crypto: talitos - Add handling for SEC 3.x treatment of link table

Later SEC revision requires the link table (used for scatter/gather)
to have an extra entry to account for the total length in descriptor [4],
which contains cipher Input and ICV.
This only applies to decrypt, not encrypt.
Without this change, on 837x, a gather return/length error results
when a decryption uses a link table to gather the fragments.
This is observed by doing a ping with size of 1447 or larger with AES,
or a ping with size 1455 or larger with 3des.

So, add check for SEC compatible "fsl,3.0" for using extra link table entry.

Signed-off-by: Lee Nipper <lee.nipper@freescale.com>
Signed-off-by: Kim Phillips <kim.phillips@freescale.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index 681c15f420834a545292b17974e523d387d2f243..ee827a7f7c6a48257e5af20219dcb256421d8994 100644 (file)
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -96,6 +96,9 @@ struct talitos_private {
        unsigned int exec_units;
        unsigned int desc_types;
 
+       /* SEC Compatibility info */
+       unsigned long features;
+
        /* next channel to be assigned next incoming descriptor */
        atomic_t last_chan;
 
@@ -133,6 +136,9 @@ struct talitos_private {
        struct hwrng rng;
 };
 
+/* .features flag */
+#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
+
 /*
  * map virtual single (contiguous) pointer to h/w descriptor pointer
  */
@@ -785,7 +791,7 @@ static void ipsec_esp_encrypt_done(struct device *dev,
        /* copy the generated ICV to dst */
        if (edesc->dma_len) {
                icvdata = &edesc->link_tbl[edesc->src_nents +
-                                          edesc->dst_nents + 1];
+                                          edesc->dst_nents + 2];
                sg = sg_last(areq->dst, edesc->dst_nents);
                memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
                       icvdata, ctx->authsize);
@@ -814,7 +820,7 @@ static void ipsec_esp_decrypt_done(struct device *dev,
                /* auth check */
                if (edesc->dma_len)
                        icvdata = &edesc->link_tbl[edesc->src_nents +
-                                                  edesc->dst_nents + 1];
+                                                  edesc->dst_nents + 2];
                else
                        icvdata = &edesc->link_tbl[0];
 
@@ -921,10 +927,30 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
                sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
                                          &edesc->link_tbl[0]);
                if (sg_count > 1) {
+                       struct talitos_ptr *link_tbl_ptr =
+                               &edesc->link_tbl[sg_count-1];
+                       struct scatterlist *sg;
+                       struct talitos_private *priv = dev_get_drvdata(dev);
+
                        desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
                        desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl);
                        dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
                                                   edesc->dma_len, DMA_BIDIRECTIONAL);
+                       /* If necessary for this SEC revision,
+                        * add a link table entry for ICV.
+                        */
+                       if ((priv->features &
+                            TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT) &&
+                           (edesc->desc.hdr & DESC_HDR_MODE0_ENCRYPT) == 0) {
+                               link_tbl_ptr->j_extent = 0;
+                               link_tbl_ptr++;
+                               link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
+                               link_tbl_ptr->len = cpu_to_be16(authsize);
+                               sg = sg_last(areq->src, edesc->src_nents ? : 1);
+                               link_tbl_ptr->ptr = cpu_to_be32(
+                                               (char *)sg_dma_address(sg)
+                                               + sg->length - authsize);
+                       }
                } else {
                        /* Only one segment now, so no link tbl needed */
                        desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
@@ -944,12 +970,11 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
                desc->ptr[5].ptr = cpu_to_be32(sg_dma_address(areq->dst));
        } else {
                struct talitos_ptr *link_tbl_ptr =
-                       &edesc->link_tbl[edesc->src_nents];
-               struct scatterlist *sg;
+                       &edesc->link_tbl[edesc->src_nents + 1];
 
                desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *)
                                               edesc->dma_link_tbl +
-                                              edesc->src_nents);
+                                              edesc->src_nents + 1);
                if (areq->src == areq->dst) {
                        memcpy(link_tbl_ptr, &edesc->link_tbl[0],
                               edesc->src_nents * sizeof(struct talitos_ptr));
@@ -957,14 +982,10 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
                        sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
                                                  link_tbl_ptr);
                }
+               /* Add an entry to the link table for ICV data */
                link_tbl_ptr += sg_count - 1;
-
-               /* handle case where sg_last contains the ICV exclusively */
-               sg = sg_last(areq->dst, edesc->dst_nents);
-               if (sg->length == ctx->authsize)
-                       link_tbl_ptr--;
-
                link_tbl_ptr->j_extent = 0;
+               sg_count++;
                link_tbl_ptr++;
                link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
                link_tbl_ptr->len = cpu_to_be16(authsize);
@@ -973,7 +994,7 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
                link_tbl_ptr->ptr = cpu_to_be32((struct talitos_ptr *)
                                                edesc->dma_link_tbl +
                                                edesc->src_nents +
-                                               edesc->dst_nents + 1);
+                                               edesc->dst_nents + 2);
 
                desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
                dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
@@ -1040,12 +1061,12 @@ static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq,
 
        /*
         * allocate space for base edesc plus the link tables,
-        * allowing for a separate entry for the generated ICV (+ 1),
+        * allowing for two separate entries for ICV and generated ICV (+ 2),
         * and the ICV data itself
         */
        alloc_len = sizeof(struct ipsec_esp_edesc);
        if (src_nents || dst_nents) {
-               dma_len = (src_nents + dst_nents + 1) *
+               dma_len = (src_nents + dst_nents + 2) *
                                 sizeof(struct talitos_ptr) + ctx->authsize;
                alloc_len += dma_len;
        } else {
@@ -1104,7 +1125,7 @@ static int aead_authenc_decrypt(struct aead_request *req)
        /* stash incoming ICV for later cmp with ICV generated by the h/w */
        if (edesc->dma_len)
                icvdata = &edesc->link_tbl[edesc->src_nents +
-                                          edesc->dst_nents + 1];
+                                          edesc->dst_nents + 2];
        else
                icvdata = &edesc->link_tbl[0];
 
@@ -1480,6 +1501,9 @@ static int talitos_probe(struct of_device *ofdev,
                goto err_out;
        }
 
+       if (of_device_is_compatible(np, "fsl,sec3.0"))
+               priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
+
        priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
                                  GFP_KERNEL);
        priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,