From 981a2e3e45c554611316b6eb28d00204586637b3 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Tue, 25 Apr 2017 11:25:40 +0300 Subject: [PATCH 001/140] crypto: tcrypt - don't disable irqs and wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tcrypt AEAD cycles speed tests disables irqs during the test, which is broken at the very least since commit '1425d2d17f7309c6 ("crypto: tcrypt - Fix AEAD speed tests")' adds a wait for completion as part of the test and probably since switching to the new AEAD API. While the result of taking a cycle count diff may not mean much on SMP systems if the task migrates, it's good enough for tcrypt being the quick & dirty dev tool it is. It's also what all the other (i.e. hash) cycle speed tests do. Signed-off-by: Gilad Ben-Yossef Reported-by: Ofir Drang Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 9a11f3c2bf98..0dd6a432d6ca 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -138,8 +138,6 @@ static int test_aead_cycles(struct aead_request *req, int enc, int blen) int ret = 0; int i; - local_irq_disable(); - /* Warm-up run. */ for (i = 0; i < 4; i++) { if (enc) @@ -169,8 +167,6 @@ static int test_aead_cycles(struct aead_request *req, int enc, int blen) } out: - local_irq_enable(); - if (ret == 0) printk("1 operation in %lu cycles (%d bytes)\n", (cycles + 4) / 8, blen); From 15a9b363de232236a31922d5a928d1830bc42060 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 25 Apr 2017 12:18:54 +0300 Subject: [PATCH 002/140] crypto: sha512-mb - add some missing unlock on error We recently added some new locking but missed the unlocks on these error paths in sha512_ctx_mgr_submit(). Fixes: c459bd7beda0 ("crypto: sha512-mb - Protect sha512 mb ctx mgr access") Signed-off-by: Dan Carpenter Acked-by: Tim Chen Signed-off-by: Herbert Xu --- arch/x86/crypto/sha512-mb/sha512_mb.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/crypto/sha512-mb/sha512_mb.c b/arch/x86/crypto/sha512-mb/sha512_mb.c index 2dd3674b5a1e..458409b7568d 100644 --- a/arch/x86/crypto/sha512-mb/sha512_mb.c +++ b/arch/x86/crypto/sha512-mb/sha512_mb.c @@ -269,19 +269,19 @@ static struct sha512_hash_ctx * LAST */ ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; - return ctx; + goto unlock; } if (ctx->status & HASH_CTX_STS_PROCESSING) { /* Cannot submit to a currently processing job. */ ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; - return ctx; + goto unlock; } if ((ctx->status & HASH_CTX_STS_COMPLETE) && !(flags & HASH_FIRST)) { /* Cannot update a finished job. */ ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; - return ctx; + goto unlock; } @@ -363,6 +363,7 @@ static struct sha512_hash_ctx } ctx = sha512_ctx_mgr_resubmit(mgr, ctx); +unlock: spin_unlock_irqrestore(&cstate->work_lock, irqflags); return ctx; } From 7fcaf62a9f6348bcdc8d2816f93f64cf7f3f87fc Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Apr 2017 16:26:36 +0300 Subject: [PATCH 003/140] crypto: caam - avoid kzalloc(0) in caam_read_raw_data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function returns NULL if buf is composed only of zeros. Signed-off-by: Tudor Ambarus Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caampkc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index 49cbdcba7883..999ba18495b0 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -374,6 +374,8 @@ static inline u8 *caam_read_raw_data(const u8 *buf, size_t *nbytes) buf++; (*nbytes)--; } + if (!*nbytes) + return NULL; val = kzalloc(*nbytes, GFP_DMA | GFP_KERNEL); if (!val) From 7ca4a9a10fe82ee50ce0da02c72791ecf7c83869 Mon Sep 17 00:00:00 2001 From: Radu Alexe Date: Tue, 25 Apr 2017 16:26:37 +0300 Subject: [PATCH 004/140] crypto: caam - incapsulate dropping leading zeros into function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This function will be used into further patches. Signed-off-by: Radu Alexe Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caampkc.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index 999ba18495b0..d2c6977ba82e 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -357,6 +357,14 @@ static void caam_rsa_free_key(struct caam_rsa_key *key) key->n_sz = 0; } +static void caam_rsa_drop_leading_zeros(const u8 **ptr, size_t *nbytes) +{ + while (!**ptr && *nbytes) { + (*ptr)++; + (*nbytes)--; + } +} + /** * caam_read_raw_data - Read a raw byte stream as a positive integer. * The function skips buffer's leading zeros, copies the remained data @@ -370,10 +378,7 @@ static inline u8 *caam_read_raw_data(const u8 *buf, size_t *nbytes) { u8 *val; - while (!*buf && *nbytes) { - buf++; - (*nbytes)--; - } + caam_rsa_drop_leading_zeros(&buf, nbytes); if (!*nbytes) return NULL; From 52e26d77b8b3de2e9ed6c7126f68e04cad5fe852 Mon Sep 17 00:00:00 2001 From: Radu Alexe Date: Tue, 25 Apr 2017 16:26:38 +0300 Subject: [PATCH 005/140] crypto: caam - add support for RSA key form 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CAAM RSA private key may have either of three representations. 1. The first representation consists of the pair (n, d), where the components have the following meanings: n the RSA modulus d the RSA private exponent 2. The second representation consists of the triplet (p, q, d), where the components have the following meanings: p the first prime factor of the RSA modulus n q the second prime factor of the RSA modulus n d the RSA private exponent 3. The third representation consists of the quintuple (p, q, dP, dQ, qInv), where the components have the following meanings: p the first prime factor of the RSA modulus n q the second prime factor of the RSA modulus n dP the first factors's CRT exponent dQ the second factors's CRT exponent qInv the (first) CRT coefficient The benefit of using the third or the second key form is lower computational cost for the decryption and signature operations. This patch adds support for the second RSA private key representation. Signed-off-by: Tudor Ambarus Signed-off-by: Radu Alexe Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caampkc.c | 231 ++++++++++++++++++++++++++++++--- drivers/crypto/caam/caampkc.h | 38 ++++++ drivers/crypto/caam/pdb.h | 29 +++++ drivers/crypto/caam/pkc_desc.c | 17 +++ 4 files changed, 298 insertions(+), 17 deletions(-) diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index d2c6977ba82e..912e41700522 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -18,6 +18,8 @@ #define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + sizeof(struct rsa_pub_pdb)) #define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \ sizeof(struct rsa_priv_f1_pdb)) +#define DESC_RSA_PRIV_F2_LEN (2 * CAAM_CMD_SZ + \ + sizeof(struct rsa_priv_f2_pdb)) static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, struct akcipher_request *req) @@ -54,6 +56,23 @@ static void rsa_priv_f1_unmap(struct device *dev, struct rsa_edesc *edesc, dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); } +static void rsa_priv_f2_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2; + size_t p_sz = key->p_sz; + size_t q_sz = key->p_sz; + + dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE); +} + /* RSA Job Completion handler */ static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context) { @@ -90,6 +109,24 @@ static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err, akcipher_request_complete(req, err); } +static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err, + void *context) +{ + struct akcipher_request *req = context; + struct rsa_edesc *edesc; + + if (err) + caam_jr_strstatus(dev, err); + + edesc = container_of(desc, struct rsa_edesc, hw_desc[0]); + + rsa_priv_f2_unmap(dev, edesc, req); + rsa_io_unmap(dev, edesc, req); + kfree(edesc); + + akcipher_request_complete(req, err); +} + static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, size_t desclen) { @@ -258,6 +295,81 @@ static int set_rsa_priv_f1_pdb(struct akcipher_request *req, return 0; } +static int set_rsa_priv_f2_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2; + int sec4_sg_index = 0; + size_t p_sz = key->p_sz; + size_t q_sz = key->p_sz; + + pdb->d_dma = dma_map_single(dev, key->d, key->d_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->d_dma)) { + dev_err(dev, "Unable to map RSA private exponent memory\n"); + return -ENOMEM; + } + + pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->p_dma)) { + dev_err(dev, "Unable to map RSA prime factor p memory\n"); + goto unmap_d; + } + + pdb->q_dma = dma_map_single(dev, key->q, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->q_dma)) { + dev_err(dev, "Unable to map RSA prime factor q memory\n"); + goto unmap_p; + } + + pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->tmp1_dma)) { + dev_err(dev, "Unable to map RSA tmp1 memory\n"); + goto unmap_q; + } + + pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->tmp2_dma)) { + dev_err(dev, "Unable to map RSA tmp2 memory\n"); + goto unmap_tmp1; + } + + if (edesc->src_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->src_nents; + } else { + pdb->g_dma = sg_dma_address(req->src); + } + + if (edesc->dst_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->f_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= (key->d_sz << RSA_PDB_D_SHIFT) | key->n_sz; + pdb->p_q_len = (q_sz << RSA_PDB_Q_SHIFT) | p_sz; + + return 0; + +unmap_tmp1: + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); +unmap_q: + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); +unmap_p: + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); +unmap_d: + dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); + + return -ENOMEM; +} + static int caam_rsa_enc(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); @@ -301,24 +413,14 @@ init_fail: return ret; } -static int caam_rsa_dec(struct akcipher_request *req) +static int caam_rsa_dec_priv_f1(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct caam_rsa_key *key = &ctx->key; struct device *jrdev = ctx->dev; struct rsa_edesc *edesc; int ret; - if (unlikely(!key->n || !key->d)) - return -EINVAL; - - if (req->dst_len < key->n_sz) { - req->dst_len = key->n_sz; - dev_err(jrdev, "Output buffer length less than parameter n\n"); - return -EOVERFLOW; - } - /* Allocate extended descriptor */ edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F1_LEN); if (IS_ERR(edesc)) @@ -344,17 +446,73 @@ init_fail: return ret; } +static int caam_rsa_dec_priv_f2(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F2_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Decrypt Protocol Data Block - Private Key Form #2 */ + ret = set_rsa_priv_f2_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_priv_f2_desc(edesc->hw_desc, &edesc->pdb.priv_f2); + + ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f2_done, req); + if (!ret) + return -EINPROGRESS; + + rsa_priv_f2_unmap(jrdev, edesc, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static int caam_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + int ret; + + if (unlikely(!key->n || !key->d)) + return -EINVAL; + + if (req->dst_len < key->n_sz) { + req->dst_len = key->n_sz; + dev_err(ctx->dev, "Output buffer length less than parameter n\n"); + return -EOVERFLOW; + } + + if (key->priv_form == FORM2) + ret = caam_rsa_dec_priv_f2(req); + else + ret = caam_rsa_dec_priv_f1(req); + + return ret; +} + static void caam_rsa_free_key(struct caam_rsa_key *key) { kzfree(key->d); + kzfree(key->p); + kzfree(key->q); + kzfree(key->tmp1); + kzfree(key->tmp2); kfree(key->e); kfree(key->n); - key->d = NULL; - key->e = NULL; - key->n = NULL; - key->d_sz = 0; - key->e_sz = 0; - key->n_sz = 0; + memset(key, 0, sizeof(*key)); } static void caam_rsa_drop_leading_zeros(const u8 **ptr, size_t *nbytes) @@ -444,6 +602,43 @@ err: return -ENOMEM; } +static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx, + struct rsa_key *raw_key) +{ + struct caam_rsa_key *rsa_key = &ctx->key; + size_t p_sz = raw_key->p_sz; + size_t q_sz = raw_key->q_sz; + + rsa_key->p = caam_read_raw_data(raw_key->p, &p_sz); + if (!rsa_key->p) + return; + rsa_key->p_sz = p_sz; + + rsa_key->q = caam_read_raw_data(raw_key->q, &q_sz); + if (!rsa_key->q) + goto free_p; + rsa_key->q_sz = q_sz; + + rsa_key->tmp1 = kzalloc(raw_key->p_sz, GFP_DMA | GFP_KERNEL); + if (!rsa_key->tmp1) + goto free_q; + + rsa_key->tmp2 = kzalloc(raw_key->q_sz, GFP_DMA | GFP_KERNEL); + if (!rsa_key->tmp2) + goto free_tmp1; + + rsa_key->priv_form = FORM2; + + return; + +free_tmp1: + kzfree(rsa_key->tmp1); +free_q: + kzfree(rsa_key->q); +free_p: + kzfree(rsa_key->p); +} + static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) { @@ -490,6 +685,8 @@ static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, memcpy(rsa_key->d, raw_key.d, raw_key.d_sz); memcpy(rsa_key->e, raw_key.e, raw_key.e_sz); + caam_rsa_set_priv_key_form(ctx, &raw_key); + return 0; err: diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h index f595d159b112..48e408ceaf75 100644 --- a/drivers/crypto/caam/caampkc.h +++ b/drivers/crypto/caam/caampkc.h @@ -12,22 +12,58 @@ #include "compat.h" #include "pdb.h" +/** + * caam_priv_key_form - CAAM RSA private key representation + * CAAM RSA private key may have either of two forms. + * + * 1. The first representation consists of the pair (n, d), where the + * components have the following meanings: + * n the RSA modulus + * d the RSA private exponent + * + * 2. The second representation consists of the triplet (p, q, d), where the + * components have the following meanings: + * p the first prime factor of the RSA modulus n + * q the second prime factor of the RSA modulus n + * d the RSA private exponent + */ +enum caam_priv_key_form { + FORM1, + FORM2, +}; + /** * caam_rsa_key - CAAM RSA key structure. Keys are allocated in DMA zone. * @n : RSA modulus raw byte stream * @e : RSA public exponent raw byte stream * @d : RSA private exponent raw byte stream + * @p : RSA prime factor p of RSA modulus n + * @q : RSA prime factor q of RSA modulus n + * @tmp1 : CAAM uses this temporary buffer as internal state buffer. + * It is assumed to be as long as p. + * @tmp2 : CAAM uses this temporary buffer as internal state buffer. + * It is assumed to be as long as q. * @n_sz : length in bytes of RSA modulus n * @e_sz : length in bytes of RSA public exponent * @d_sz : length in bytes of RSA private exponent + * @p_sz : length in bytes of RSA prime factor p of RSA modulus n + * @q_sz : length in bytes of RSA prime factor q of RSA modulus n + * @priv_form : CAAM RSA private key representation */ struct caam_rsa_key { u8 *n; u8 *e; u8 *d; + u8 *p; + u8 *q; + u8 *tmp1; + u8 *tmp2; size_t n_sz; size_t e_sz; size_t d_sz; + size_t p_sz; + size_t q_sz; + enum caam_priv_key_form priv_form; }; /** @@ -59,6 +95,7 @@ struct rsa_edesc { union { struct rsa_pub_pdb pub; struct rsa_priv_f1_pdb priv_f1; + struct rsa_priv_f2_pdb priv_f2; } pdb; u32 hw_desc[]; }; @@ -66,5 +103,6 @@ struct rsa_edesc { /* Descriptor construction primitives. */ void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb); void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb); +void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb); #endif diff --git a/drivers/crypto/caam/pdb.h b/drivers/crypto/caam/pdb.h index aaa00dd1c601..8ea5e612927e 100644 --- a/drivers/crypto/caam/pdb.h +++ b/drivers/crypto/caam/pdb.h @@ -483,6 +483,8 @@ struct dsa_verify_pdb { #define RSA_PDB_E_MASK (0xFFF << RSA_PDB_E_SHIFT) #define RSA_PDB_D_SHIFT 12 #define RSA_PDB_D_MASK (0xFFF << RSA_PDB_D_SHIFT) +#define RSA_PDB_Q_SHIFT 12 +#define RSA_PDB_Q_MASK (0xFFF << RSA_PDB_Q_SHIFT) #define RSA_PDB_SGF_F (0x8 << RSA_PDB_SGF_SHIFT) #define RSA_PDB_SGF_G (0x4 << RSA_PDB_SGF_SHIFT) @@ -490,6 +492,7 @@ struct dsa_verify_pdb { #define RSA_PRIV_PDB_SGF_G (0x8 << RSA_PDB_SGF_SHIFT) #define RSA_PRIV_KEY_FRM_1 0 +#define RSA_PRIV_KEY_FRM_2 1 /** * RSA Encrypt Protocol Data Block @@ -525,4 +528,30 @@ struct rsa_priv_f1_pdb { dma_addr_t d_dma; } __packed; +/** + * RSA Decrypt PDB - Private Key Form #2 + * @sgf : scatter-gather field + * @g_dma : dma address of encrypted input data + * @f_dma : dma address of output data + * @d_dma : dma address of RSA private exponent + * @p_dma : dma address of RSA prime factor p of RSA modulus n + * @q_dma : dma address of RSA prime factor q of RSA modulus n + * @tmp1_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as p. + * @tmp2_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as q. + * @p_q_len : length in bytes of first two prime factors of the RSA modulus n + */ +struct rsa_priv_f2_pdb { + u32 sgf; + dma_addr_t g_dma; + dma_addr_t f_dma; + dma_addr_t d_dma; + dma_addr_t p_dma; + dma_addr_t q_dma; + dma_addr_t tmp1_dma; + dma_addr_t tmp2_dma; + u32 p_q_len; +} __packed; + #endif diff --git a/drivers/crypto/caam/pkc_desc.c b/drivers/crypto/caam/pkc_desc.c index 4e4183e615ea..9d03c7055ed7 100644 --- a/drivers/crypto/caam/pkc_desc.c +++ b/drivers/crypto/caam/pkc_desc.c @@ -34,3 +34,20 @@ void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb) append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | RSA_PRIV_KEY_FRM_1); } + +/* Descriptor for RSA Private operation - Private Key Form #2 */ +void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, sizeof(*pdb)); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->d_dma); + append_ptr(desc, pdb->p_dma); + append_ptr(desc, pdb->q_dma); + append_ptr(desc, pdb->tmp1_dma); + append_ptr(desc, pdb->tmp2_dma); + append_cmd(desc, pdb->p_q_len); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | + RSA_PRIV_KEY_FRM_2); +} From 4a651b122adb8d3a9cb1c5dddfd63e708ee9bdd2 Mon Sep 17 00:00:00 2001 From: Radu Alexe Date: Tue, 25 Apr 2017 16:26:39 +0300 Subject: [PATCH 006/140] crypto: caam - add support for RSA key form 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CAAM RSA private key may have either of three representations. 1. The first representation consists of the pair (n, d), where the components have the following meanings: n the RSA modulus d the RSA private exponent 2. The second representation consists of the triplet (p, q, d), where the components have the following meanings: p the first prime factor of the RSA modulus n q the second prime factor of the RSA modulus n d the RSA private exponent 3. The third representation consists of the quintuple (p, q, dP, dQ, qInv), where the components have the following meanings: p the first prime factor of the RSA modulus n q the second prime factor of the RSA modulus n dP the first factors's CRT exponent dQ the second factors's CRT exponent qInv the (first) CRT coefficient The benefit of using the third or the second key form is lower computational cost for the decryption and signature operations. This patch adds support for the third RSA private key representations and extends caampkc to use the fastest key when all related components are present in the private key. Signed-off-by: Tudor Ambarus Signed-off-by: Radu Alexe Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caampkc.c | 219 ++++++++++++++++++++++++++++++++- drivers/crypto/caam/caampkc.h | 22 +++- drivers/crypto/caam/pdb.h | 33 +++++ drivers/crypto/caam/pkc_desc.c | 19 +++ 4 files changed, 291 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index 912e41700522..57f399caa977 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -20,6 +20,8 @@ sizeof(struct rsa_priv_f1_pdb)) #define DESC_RSA_PRIV_F2_LEN (2 * CAAM_CMD_SZ + \ sizeof(struct rsa_priv_f2_pdb)) +#define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \ + sizeof(struct rsa_priv_f3_pdb)) static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, struct akcipher_request *req) @@ -73,6 +75,25 @@ static void rsa_priv_f2_unmap(struct device *dev, struct rsa_edesc *edesc, dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE); } +static void rsa_priv_f3_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3; + size_t p_sz = key->p_sz; + size_t q_sz = key->p_sz; + + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE); +} + /* RSA Job Completion handler */ static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context) { @@ -127,6 +148,24 @@ static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err, akcipher_request_complete(req, err); } +static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err, + void *context) +{ + struct akcipher_request *req = context; + struct rsa_edesc *edesc; + + if (err) + caam_jr_strstatus(dev, err); + + edesc = container_of(desc, struct rsa_edesc, hw_desc[0]); + + rsa_priv_f3_unmap(dev, edesc, req); + rsa_io_unmap(dev, edesc, req); + kfree(edesc); + + akcipher_request_complete(req, err); +} + static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, size_t desclen) { @@ -370,6 +409,97 @@ unmap_d: return -ENOMEM; } +static int set_rsa_priv_f3_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3; + int sec4_sg_index = 0; + size_t p_sz = key->p_sz; + size_t q_sz = key->p_sz; + + pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->p_dma)) { + dev_err(dev, "Unable to map RSA prime factor p memory\n"); + return -ENOMEM; + } + + pdb->q_dma = dma_map_single(dev, key->q, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->q_dma)) { + dev_err(dev, "Unable to map RSA prime factor q memory\n"); + goto unmap_p; + } + + pdb->dp_dma = dma_map_single(dev, key->dp, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->dp_dma)) { + dev_err(dev, "Unable to map RSA exponent dp memory\n"); + goto unmap_q; + } + + pdb->dq_dma = dma_map_single(dev, key->dq, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->dq_dma)) { + dev_err(dev, "Unable to map RSA exponent dq memory\n"); + goto unmap_dp; + } + + pdb->c_dma = dma_map_single(dev, key->qinv, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->c_dma)) { + dev_err(dev, "Unable to map RSA CRT coefficient qinv memory\n"); + goto unmap_dq; + } + + pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->tmp1_dma)) { + dev_err(dev, "Unable to map RSA tmp1 memory\n"); + goto unmap_qinv; + } + + pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->tmp2_dma)) { + dev_err(dev, "Unable to map RSA tmp2 memory\n"); + goto unmap_tmp1; + } + + if (edesc->src_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->src_nents; + } else { + pdb->g_dma = sg_dma_address(req->src); + } + + if (edesc->dst_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->f_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= key->n_sz; + pdb->p_q_len = (q_sz << RSA_PDB_Q_SHIFT) | p_sz; + + return 0; + +unmap_tmp1: + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); +unmap_qinv: + dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE); +unmap_dq: + dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE); +unmap_dp: + dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE); +unmap_q: + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); +unmap_p: + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); + + return -ENOMEM; +} + static int caam_rsa_enc(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); @@ -479,6 +609,39 @@ init_fail: return ret; } +static int caam_rsa_dec_priv_f3(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F3_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Decrypt Protocol Data Block - Private Key Form #3 */ + ret = set_rsa_priv_f3_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_priv_f3_desc(edesc->hw_desc, &edesc->pdb.priv_f3); + + ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f3_done, req); + if (!ret) + return -EINPROGRESS; + + rsa_priv_f3_unmap(jrdev, edesc, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + static int caam_rsa_dec(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); @@ -495,7 +658,9 @@ static int caam_rsa_dec(struct akcipher_request *req) return -EOVERFLOW; } - if (key->priv_form == FORM2) + if (key->priv_form == FORM3) + ret = caam_rsa_dec_priv_f3(req); + else if (key->priv_form == FORM2) ret = caam_rsa_dec_priv_f2(req); else ret = caam_rsa_dec_priv_f1(req); @@ -508,6 +673,9 @@ static void caam_rsa_free_key(struct caam_rsa_key *key) kzfree(key->d); kzfree(key->p); kzfree(key->q); + kzfree(key->dp); + kzfree(key->dq); + kzfree(key->qinv); kzfree(key->tmp1); kzfree(key->tmp2); kfree(key->e); @@ -523,6 +691,34 @@ static void caam_rsa_drop_leading_zeros(const u8 **ptr, size_t *nbytes) } } +/** + * caam_read_rsa_crt - Used for reading dP, dQ, qInv CRT members. + * dP, dQ and qInv could decode to less than corresponding p, q length, as the + * BER-encoding requires that the minimum number of bytes be used to encode the + * integer. dP, dQ, qInv decoded values have to be zero-padded to appropriate + * length. + * + * @ptr : pointer to {dP, dQ, qInv} CRT member + * @nbytes: length in bytes of {dP, dQ, qInv} CRT member + * @dstlen: length in bytes of corresponding p or q prime factor + */ +static u8 *caam_read_rsa_crt(const u8 *ptr, size_t nbytes, size_t dstlen) +{ + u8 *dst; + + caam_rsa_drop_leading_zeros(&ptr, &nbytes); + if (!nbytes) + return NULL; + + dst = kzalloc(dstlen, GFP_DMA | GFP_KERNEL); + if (!dst) + return NULL; + + memcpy(dst + (dstlen - nbytes), ptr, nbytes); + + return dst; +} + /** * caam_read_raw_data - Read a raw byte stream as a positive integer. * The function skips buffer's leading zeros, copies the remained data @@ -629,8 +825,29 @@ static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx, rsa_key->priv_form = FORM2; + rsa_key->dp = caam_read_rsa_crt(raw_key->dp, raw_key->dp_sz, p_sz); + if (!rsa_key->dp) + goto free_tmp2; + + rsa_key->dq = caam_read_rsa_crt(raw_key->dq, raw_key->dq_sz, q_sz); + if (!rsa_key->dq) + goto free_dp; + + rsa_key->qinv = caam_read_rsa_crt(raw_key->qinv, raw_key->qinv_sz, + q_sz); + if (!rsa_key->qinv) + goto free_dq; + + rsa_key->priv_form = FORM3; + return; +free_dq: + kzfree(rsa_key->dq); +free_dp: + kzfree(rsa_key->dp); +free_tmp2: + kzfree(rsa_key->tmp2); free_tmp1: kzfree(rsa_key->tmp1); free_q: diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h index 48e408ceaf75..87ab75e9df43 100644 --- a/drivers/crypto/caam/caampkc.h +++ b/drivers/crypto/caam/caampkc.h @@ -14,7 +14,7 @@ /** * caam_priv_key_form - CAAM RSA private key representation - * CAAM RSA private key may have either of two forms. + * CAAM RSA private key may have either of three forms. * * 1. The first representation consists of the pair (n, d), where the * components have the following meanings: @@ -26,10 +26,22 @@ * p the first prime factor of the RSA modulus n * q the second prime factor of the RSA modulus n * d the RSA private exponent + * + * 3. The third representation consists of the quintuple (p, q, dP, dQ, qInv), + * where the components have the following meanings: + * p the first prime factor of the RSA modulus n + * q the second prime factor of the RSA modulus n + * dP the first factors's CRT exponent + * dQ the second factors's CRT exponent + * qInv the (first) CRT coefficient + * + * The benefit of using the third or the second key form is lower computational + * cost for the decryption and signature operations. */ enum caam_priv_key_form { FORM1, FORM2, + FORM3 }; /** @@ -39,6 +51,9 @@ enum caam_priv_key_form { * @d : RSA private exponent raw byte stream * @p : RSA prime factor p of RSA modulus n * @q : RSA prime factor q of RSA modulus n + * @dp : RSA CRT exponent of p + * @dp : RSA CRT exponent of q + * @qinv : RSA CRT coefficient * @tmp1 : CAAM uses this temporary buffer as internal state buffer. * It is assumed to be as long as p. * @tmp2 : CAAM uses this temporary buffer as internal state buffer. @@ -56,6 +71,9 @@ struct caam_rsa_key { u8 *d; u8 *p; u8 *q; + u8 *dp; + u8 *dq; + u8 *qinv; u8 *tmp1; u8 *tmp2; size_t n_sz; @@ -96,6 +114,7 @@ struct rsa_edesc { struct rsa_pub_pdb pub; struct rsa_priv_f1_pdb priv_f1; struct rsa_priv_f2_pdb priv_f2; + struct rsa_priv_f3_pdb priv_f3; } pdb; u32 hw_desc[]; }; @@ -104,5 +123,6 @@ struct rsa_edesc { void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb); void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb); void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb); +void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb); #endif diff --git a/drivers/crypto/caam/pdb.h b/drivers/crypto/caam/pdb.h index 8ea5e612927e..31e59963f4d2 100644 --- a/drivers/crypto/caam/pdb.h +++ b/drivers/crypto/caam/pdb.h @@ -493,6 +493,7 @@ struct dsa_verify_pdb { #define RSA_PRIV_KEY_FRM_1 0 #define RSA_PRIV_KEY_FRM_2 1 +#define RSA_PRIV_KEY_FRM_3 2 /** * RSA Encrypt Protocol Data Block @@ -554,4 +555,36 @@ struct rsa_priv_f2_pdb { u32 p_q_len; } __packed; +/** + * RSA Decrypt PDB - Private Key Form #3 + * This is the RSA Chinese Reminder Theorem (CRT) form for two prime factors of + * the RSA modulus. + * @sgf : scatter-gather field + * @g_dma : dma address of encrypted input data + * @f_dma : dma address of output data + * @c_dma : dma address of RSA CRT coefficient + * @p_dma : dma address of RSA prime factor p of RSA modulus n + * @q_dma : dma address of RSA prime factor q of RSA modulus n + * @dp_dma : dma address of RSA CRT exponent of RSA prime factor p + * @dp_dma : dma address of RSA CRT exponent of RSA prime factor q + * @tmp1_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as p. + * @tmp2_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as q. + * @p_q_len : length in bytes of first two prime factors of the RSA modulus n + */ +struct rsa_priv_f3_pdb { + u32 sgf; + dma_addr_t g_dma; + dma_addr_t f_dma; + dma_addr_t c_dma; + dma_addr_t p_dma; + dma_addr_t q_dma; + dma_addr_t dp_dma; + dma_addr_t dq_dma; + dma_addr_t tmp1_dma; + dma_addr_t tmp2_dma; + u32 p_q_len; +} __packed; + #endif diff --git a/drivers/crypto/caam/pkc_desc.c b/drivers/crypto/caam/pkc_desc.c index 9d03c7055ed7..9e2ce6fe2e43 100644 --- a/drivers/crypto/caam/pkc_desc.c +++ b/drivers/crypto/caam/pkc_desc.c @@ -51,3 +51,22 @@ void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb) append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | RSA_PRIV_KEY_FRM_2); } + +/* Descriptor for RSA Private operation - Private Key Form #3 */ +void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, sizeof(*pdb)); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->c_dma); + append_ptr(desc, pdb->p_dma); + append_ptr(desc, pdb->q_dma); + append_ptr(desc, pdb->dp_dma); + append_ptr(desc, pdb->dq_dma); + append_ptr(desc, pdb->tmp1_dma); + append_ptr(desc, pdb->tmp2_dma); + append_cmd(desc, pdb->p_q_len); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | + RSA_PRIV_KEY_FRM_3); +} From ac50b78b22cddb8a85b264b1f172c4239e826fd2 Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Tue, 25 Apr 2017 08:59:44 -0500 Subject: [PATCH 007/140] crypto: ccp - Add a module author CC: # 4.9.x+ Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index 92d1c6959f08..b7504562715c 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -31,6 +31,7 @@ #include "ccp-dev.h" MODULE_AUTHOR("Tom Lendacky "); +MODULE_AUTHOR("Gary R Hook "); MODULE_LICENSE("GPL"); MODULE_VERSION("1.0.0"); MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver"); From f4857f4c2ee9aa4e2aacac1a845352b00197fb57 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 26 Apr 2017 17:11:32 +0100 Subject: [PATCH 008/140] crypto: arm64/sha - avoid non-standard inline asm tricks Replace the inline asm which exports struct offsets as ELF symbols with proper const variables exposing the same values. This works around an issue with Clang which does not interpret the "i" (or "I") constraints in the same way as GCC. Signed-off-by: Ard Biesheuvel Tested-by: Matthias Kaehlcke Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha1-ce-core.S | 6 ++++-- arch/arm64/crypto/sha1-ce-glue.c | 11 +++-------- arch/arm64/crypto/sha2-ce-core.S | 6 ++++-- arch/arm64/crypto/sha2-ce-glue.c | 13 +++++-------- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S index c98e7e849f06..8550408735a0 100644 --- a/arch/arm64/crypto/sha1-ce-core.S +++ b/arch/arm64/crypto/sha1-ce-core.S @@ -82,7 +82,8 @@ ENTRY(sha1_ce_transform) ldr dgb, [x0, #16] /* load sha1_ce_state::finalize */ - ldr w4, [x0, #:lo12:sha1_ce_offsetof_finalize] + ldr_l w4, sha1_ce_offsetof_finalize, x4 + ldr w4, [x0, x4] /* load input */ 0: ld1 {v8.4s-v11.4s}, [x1], #64 @@ -132,7 +133,8 @@ CPU_LE( rev32 v11.16b, v11.16b ) * the padding is handled by the C code in that case. */ cbz x4, 3f - ldr x4, [x0, #:lo12:sha1_ce_offsetof_count] + ldr_l w4, sha1_ce_offsetof_count, x4 + ldr x4, [x0, x4] movi v9.2d, #0 mov x8, #0x80000000 movi v10.2d, #0 diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index aefda9868627..ea319c055f5d 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -17,9 +17,6 @@ #include #include -#define ASM_EXPORT(sym, val) \ - asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); - MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); @@ -32,6 +29,9 @@ struct sha1_ce_state { asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, int blocks); +const u32 sha1_ce_offsetof_count = offsetof(struct sha1_ce_state, sst.count); +const u32 sha1_ce_offsetof_finalize = offsetof(struct sha1_ce_state, finalize); + static int sha1_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -52,11 +52,6 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, struct sha1_ce_state *sctx = shash_desc_ctx(desc); bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE); - ASM_EXPORT(sha1_ce_offsetof_count, - offsetof(struct sha1_ce_state, sst.count)); - ASM_EXPORT(sha1_ce_offsetof_finalize, - offsetof(struct sha1_ce_state, finalize)); - /* * Allow the asm code to perform the finalization if there is no * partial data and the input is a round multiple of the block size. diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S index 01cfee066837..679c6c002f4f 100644 --- a/arch/arm64/crypto/sha2-ce-core.S +++ b/arch/arm64/crypto/sha2-ce-core.S @@ -88,7 +88,8 @@ ENTRY(sha2_ce_transform) ld1 {dgav.4s, dgbv.4s}, [x0] /* load sha256_ce_state::finalize */ - ldr w4, [x0, #:lo12:sha256_ce_offsetof_finalize] + ldr_l w4, sha256_ce_offsetof_finalize, x4 + ldr w4, [x0, x4] /* load input */ 0: ld1 {v16.4s-v19.4s}, [x1], #64 @@ -136,7 +137,8 @@ CPU_LE( rev32 v19.16b, v19.16b ) * the padding is handled by the C code in that case. */ cbz x4, 3f - ldr x4, [x0, #:lo12:sha256_ce_offsetof_count] + ldr_l w4, sha256_ce_offsetof_count, x4 + ldr x4, [x0, x4] movi v17.2d, #0 mov x8, #0x80000000 movi v18.2d, #0 diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 7cd587564a41..0ed9486f75dd 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -17,9 +17,6 @@ #include #include -#define ASM_EXPORT(sym, val) \ - asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); - MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); @@ -32,6 +29,11 @@ struct sha256_ce_state { asmlinkage void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, int blocks); +const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state, + sst.count); +const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state, + finalize); + static int sha256_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -52,11 +54,6 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, struct sha256_ce_state *sctx = shash_desc_ctx(desc); bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE); - ASM_EXPORT(sha256_ce_offsetof_count, - offsetof(struct sha256_ce_state, sst.count)); - ASM_EXPORT(sha256_ce_offsetof_finalize, - offsetof(struct sha256_ce_state, finalize)); - /* * Allow the asm code to perform the finalization if there is no * partial data and the input is a round multiple of the block size. From 0487ccac2032872ea3bcfed08d4ecaefb55cf017 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Fri, 28 Apr 2017 18:11:56 +0200 Subject: [PATCH 009/140] crypto: aesni - make non-AVX AES-GCM work with any aadlen This is the first step to make the aesni AES-GCM implementation generic. The current code was written for rfc4106, so it handles only some specific sizes of associated data. Signed-off-by: Sabrina Dubroca Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_asm.S | 169 +++++++++++++++++++++++------- 1 file changed, 132 insertions(+), 37 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 3c465184ff8a..605726aaf0a2 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -89,6 +89,29 @@ SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100 ALL_F: .octa 0xffffffffffffffffffffffffffffffff .octa 0x00000000000000000000000000000000 +.section .rodata +.align 16 +.type aad_shift_arr, @object +.size aad_shift_arr, 272 +aad_shift_arr: + .octa 0xffffffffffffffffffffffffffffffff + .octa 0xffffffffffffffffffffffffffffff0C + .octa 0xffffffffffffffffffffffffffff0D0C + .octa 0xffffffffffffffffffffffffff0E0D0C + .octa 0xffffffffffffffffffffffff0F0E0D0C + .octa 0xffffffffffffffffffffff0C0B0A0908 + .octa 0xffffffffffffffffffff0D0C0B0A0908 + .octa 0xffffffffffffffffff0E0D0C0B0A0908 + .octa 0xffffffffffffffff0F0E0D0C0B0A0908 + .octa 0xffffffffffffff0C0B0A090807060504 + .octa 0xffffffffffff0D0C0B0A090807060504 + .octa 0xffffffffff0E0D0C0B0A090807060504 + .octa 0xffffffff0F0E0D0C0B0A090807060504 + .octa 0xffffff0C0B0A09080706050403020100 + .octa 0xffff0D0C0B0A09080706050403020100 + .octa 0xff0E0D0C0B0A09080706050403020100 + .octa 0x0F0E0D0C0B0A09080706050403020100 + .text @@ -252,32 +275,66 @@ XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation mov arg8, %r12 # %r12 = aadLen mov %r12, %r11 pxor %xmm\i, %xmm\i + pxor \XMM2, \XMM2 -_get_AAD_loop\num_initial_blocks\operation: - movd (%r10), \TMP1 + cmp $16, %r11 + jl _get_AAD_rest8\num_initial_blocks\operation +_get_AAD_blocks\num_initial_blocks\operation: + movdqu (%r10), %xmm\i + PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data + pxor %xmm\i, \XMM2 + GHASH_MUL \XMM2, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + add $16, %r10 + sub $16, %r12 + sub $16, %r11 + cmp $16, %r11 + jge _get_AAD_blocks\num_initial_blocks\operation + + movdqu \XMM2, %xmm\i + cmp $0, %r11 + je _get_AAD_done\num_initial_blocks\operation + + pxor %xmm\i,%xmm\i + + /* read the last <16B of AAD. since we have at least 4B of + data right after the AAD (the ICV, and maybe some CT), we can + read 4B/8B blocks safely, and then get rid of the extra stuff */ +_get_AAD_rest8\num_initial_blocks\operation: + cmp $4, %r11 + jle _get_AAD_rest4\num_initial_blocks\operation + movq (%r10), \TMP1 + add $8, %r10 + sub $8, %r11 + pslldq $8, \TMP1 + psrldq $8, %xmm\i + pxor \TMP1, %xmm\i + jmp _get_AAD_rest8\num_initial_blocks\operation +_get_AAD_rest4\num_initial_blocks\operation: + cmp $0, %r11 + jle _get_AAD_rest0\num_initial_blocks\operation + mov (%r10), %eax + movq %rax, \TMP1 + add $4, %r10 + sub $4, %r10 pslldq $12, \TMP1 psrldq $4, %xmm\i pxor \TMP1, %xmm\i - add $4, %r10 - sub $4, %r12 - jne _get_AAD_loop\num_initial_blocks\operation - - cmp $16, %r11 - je _get_AAD_loop2_done\num_initial_blocks\operation - - mov $16, %r12 -_get_AAD_loop2\num_initial_blocks\operation: - psrldq $4, %xmm\i - sub $4, %r12 - cmp %r11, %r12 - jne _get_AAD_loop2\num_initial_blocks\operation - -_get_AAD_loop2_done\num_initial_blocks\operation: +_get_AAD_rest0\num_initial_blocks\operation: + /* finalize: shift out the extra bytes we read, and align + left. since pslldq can only shift by an immediate, we use + vpshufb and an array of shuffle masks */ + movq %r12, %r11 + salq $4, %r11 + movdqu aad_shift_arr(%r11), \TMP1 + PSHUFB_XMM \TMP1, %xmm\i +_get_AAD_rest_final\num_initial_blocks\operation: PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data + pxor \XMM2, %xmm\i + GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +_get_AAD_done\num_initial_blocks\operation: xor %r11, %r11 # initialise the data pointer offset as zero - - # start AES for num_initial_blocks blocks + # start AES for num_initial_blocks blocks mov %arg5, %rax # %rax = *Y0 movdqu (%rax), \XMM0 # XMM0 = Y0 @@ -322,7 +379,7 @@ aes_loop_initial_dec\num_initial_blocks: # prepare plaintext/ciphertext for GHASH computation .endr .endif - GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + # apply GHASH on num_initial_blocks blocks .if \i == 5 @@ -477,28 +534,66 @@ XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation mov arg8, %r12 # %r12 = aadLen mov %r12, %r11 pxor %xmm\i, %xmm\i -_get_AAD_loop\num_initial_blocks\operation: - movd (%r10), \TMP1 + pxor \XMM2, \XMM2 + + cmp $16, %r11 + jl _get_AAD_rest8\num_initial_blocks\operation +_get_AAD_blocks\num_initial_blocks\operation: + movdqu (%r10), %xmm\i + PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data + pxor %xmm\i, \XMM2 + GHASH_MUL \XMM2, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + add $16, %r10 + sub $16, %r12 + sub $16, %r11 + cmp $16, %r11 + jge _get_AAD_blocks\num_initial_blocks\operation + + movdqu \XMM2, %xmm\i + cmp $0, %r11 + je _get_AAD_done\num_initial_blocks\operation + + pxor %xmm\i,%xmm\i + + /* read the last <16B of AAD. since we have at least 4B of + data right after the AAD (the ICV, and maybe some PT), we can + read 4B/8B blocks safely, and then get rid of the extra stuff */ +_get_AAD_rest8\num_initial_blocks\operation: + cmp $4, %r11 + jle _get_AAD_rest4\num_initial_blocks\operation + movq (%r10), \TMP1 + add $8, %r10 + sub $8, %r11 + pslldq $8, \TMP1 + psrldq $8, %xmm\i + pxor \TMP1, %xmm\i + jmp _get_AAD_rest8\num_initial_blocks\operation +_get_AAD_rest4\num_initial_blocks\operation: + cmp $0, %r11 + jle _get_AAD_rest0\num_initial_blocks\operation + mov (%r10), %eax + movq %rax, \TMP1 + add $4, %r10 + sub $4, %r10 pslldq $12, \TMP1 psrldq $4, %xmm\i pxor \TMP1, %xmm\i - add $4, %r10 - sub $4, %r12 - jne _get_AAD_loop\num_initial_blocks\operation - cmp $16, %r11 - je _get_AAD_loop2_done\num_initial_blocks\operation - mov $16, %r12 -_get_AAD_loop2\num_initial_blocks\operation: - psrldq $4, %xmm\i - sub $4, %r12 - cmp %r11, %r12 - jne _get_AAD_loop2\num_initial_blocks\operation -_get_AAD_loop2_done\num_initial_blocks\operation: +_get_AAD_rest0\num_initial_blocks\operation: + /* finalize: shift out the extra bytes we read, and align + left. since pslldq can only shift by an immediate, we use + vpshufb and an array of shuffle masks */ + movq %r12, %r11 + salq $4, %r11 + movdqu aad_shift_arr(%r11), \TMP1 + PSHUFB_XMM \TMP1, %xmm\i +_get_AAD_rest_final\num_initial_blocks\operation: PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data + pxor \XMM2, %xmm\i + GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 +_get_AAD_done\num_initial_blocks\operation: xor %r11, %r11 # initialise the data pointer offset as zero - - # start AES for num_initial_blocks blocks + # start AES for num_initial_blocks blocks mov %arg5, %rax # %rax = *Y0 movdqu (%rax), \XMM0 # XMM0 = Y0 @@ -543,7 +638,7 @@ aes_loop_initial_enc\num_initial_blocks: # prepare plaintext/ciphertext for GHASH computation .endr .endif - GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 + # apply GHASH on num_initial_blocks blocks .if \i == 5 From 38d9deecab77b9eb38cb8bb5dfa43237c6710188 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Fri, 28 Apr 2017 18:11:57 +0200 Subject: [PATCH 010/140] crypto: aesni - make non-AVX AES-GCM work with all valid auth_tag_len Signed-off-by: Sabrina Dubroca Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_asm.S | 62 ++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 605726aaf0a2..16627fec80b2 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -1549,18 +1549,35 @@ _return_T_decrypt: mov arg10, %r11 # %r11 = auth_tag_len cmp $16, %r11 je _T_16_decrypt - cmp $12, %r11 - je _T_12_decrypt + cmp $8, %r11 + jl _T_4_decrypt _T_8_decrypt: MOVQ_R64_XMM %xmm0, %rax mov %rax, (%r10) - jmp _return_T_done_decrypt -_T_12_decrypt: - MOVQ_R64_XMM %xmm0, %rax - mov %rax, (%r10) + add $8, %r10 + sub $8, %r11 psrldq $8, %xmm0 + cmp $0, %r11 + je _return_T_done_decrypt +_T_4_decrypt: movd %xmm0, %eax - mov %eax, 8(%r10) + mov %eax, (%r10) + add $4, %r10 + sub $4, %r11 + psrldq $4, %xmm0 + cmp $0, %r11 + je _return_T_done_decrypt +_T_123_decrypt: + movd %xmm0, %eax + cmp $2, %r11 + jl _T_1_decrypt + mov %ax, (%r10) + cmp $2, %r11 + je _return_T_done_decrypt + add $2, %r10 + sar $16, %eax +_T_1_decrypt: + mov %al, (%r10) jmp _return_T_done_decrypt _T_16_decrypt: movdqu %xmm0, (%r10) @@ -1813,18 +1830,35 @@ _return_T_encrypt: mov arg10, %r11 # %r11 = auth_tag_len cmp $16, %r11 je _T_16_encrypt - cmp $12, %r11 - je _T_12_encrypt + cmp $8, %r11 + jl _T_4_encrypt _T_8_encrypt: MOVQ_R64_XMM %xmm0, %rax mov %rax, (%r10) - jmp _return_T_done_encrypt -_T_12_encrypt: - MOVQ_R64_XMM %xmm0, %rax - mov %rax, (%r10) + add $8, %r10 + sub $8, %r11 psrldq $8, %xmm0 + cmp $0, %r11 + je _return_T_done_encrypt +_T_4_encrypt: movd %xmm0, %eax - mov %eax, 8(%r10) + mov %eax, (%r10) + add $4, %r10 + sub $4, %r11 + psrldq $4, %xmm0 + cmp $0, %r11 + je _return_T_done_encrypt +_T_123_encrypt: + movd %xmm0, %eax + cmp $2, %r11 + jl _T_1_encrypt + mov %ax, (%r10) + cmp $2, %r11 + je _return_T_done_encrypt + add $2, %r10 + sar $16, %eax +_T_1_encrypt: + mov %al, (%r10) jmp _return_T_done_encrypt _T_16_encrypt: movdqu %xmm0, (%r10) From e10f9cb223511cfdc509f5a655f11ff6cbc5508c Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Fri, 28 Apr 2017 18:11:58 +0200 Subject: [PATCH 011/140] crypto: aesni - make AVX AES-GCM work with any aadlen This is the first step to make the aesni AES-GCM implementation generic. The current code was written for rfc4106, so it handles only some specific sizes of associated data. Signed-off-by: Sabrina Dubroca Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_avx-x86_64.S | 110 +++++++++++++++++------ 1 file changed, 82 insertions(+), 28 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S index d664382c6e56..a73117c84904 100644 --- a/arch/x86/crypto/aesni-intel_avx-x86_64.S +++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S @@ -155,6 +155,30 @@ SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100 ALL_F: .octa 0xffffffffffffffffffffffffffffffff .octa 0x00000000000000000000000000000000 +.section .rodata +.align 16 +.type aad_shift_arr, @object +.size aad_shift_arr, 272 +aad_shift_arr: + .octa 0xffffffffffffffffffffffffffffffff + .octa 0xffffffffffffffffffffffffffffff0C + .octa 0xffffffffffffffffffffffffffff0D0C + .octa 0xffffffffffffffffffffffffff0E0D0C + .octa 0xffffffffffffffffffffffff0F0E0D0C + .octa 0xffffffffffffffffffffff0C0B0A0908 + .octa 0xffffffffffffffffffff0D0C0B0A0908 + .octa 0xffffffffffffffffff0E0D0C0B0A0908 + .octa 0xffffffffffffffff0F0E0D0C0B0A0908 + .octa 0xffffffffffffff0C0B0A090807060504 + .octa 0xffffffffffff0D0C0B0A090807060504 + .octa 0xffffffffff0E0D0C0B0A090807060504 + .octa 0xffffffff0F0E0D0C0B0A090807060504 + .octa 0xffffff0C0B0A09080706050403020100 + .octa 0xffff0D0C0B0A09080706050403020100 + .octa 0xff0E0D0C0B0A09080706050403020100 + .octa 0x0F0E0D0C0B0A09080706050403020100 + + .text @@ -372,41 +396,72 @@ VARIABLE_OFFSET = 16*8 .macro INITIAL_BLOCKS_AVX num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC i = (8-\num_initial_blocks) + j = 0 setreg - mov arg6, %r10 # r10 = AAD - mov arg7, %r12 # r12 = aadLen + mov arg6, %r10 # r10 = AAD + mov arg7, %r12 # r12 = aadLen - mov %r12, %r11 + mov %r12, %r11 - vpxor reg_i, reg_i, reg_i -_get_AAD_loop\@: - vmovd (%r10), \T1 - vpslldq $12, \T1, \T1 - vpsrldq $4, reg_i, reg_i - vpxor \T1, reg_i, reg_i + vpxor reg_j, reg_j, reg_j + vpxor reg_i, reg_i, reg_i + cmp $16, %r11 + jl _get_AAD_rest8\@ +_get_AAD_blocks\@: + vmovdqu (%r10), reg_i + vpshufb SHUF_MASK(%rip), reg_i, reg_i + vpxor reg_i, reg_j, reg_j + GHASH_MUL_AVX reg_j, \T2, \T1, \T3, \T4, \T5, \T6 + add $16, %r10 + sub $16, %r12 + sub $16, %r11 + cmp $16, %r11 + jge _get_AAD_blocks\@ + vmovdqu reg_j, reg_i + cmp $0, %r11 + je _get_AAD_done\@ - add $4, %r10 - sub $4, %r12 - jg _get_AAD_loop\@ + vpxor reg_i, reg_i, reg_i + /* read the last <16B of AAD. since we have at least 4B of + data right after the AAD (the ICV, and maybe some CT), we can + read 4B/8B blocks safely, and then get rid of the extra stuff */ +_get_AAD_rest8\@: + cmp $4, %r11 + jle _get_AAD_rest4\@ + movq (%r10), \T1 + add $8, %r10 + sub $8, %r11 + vpslldq $8, \T1, \T1 + vpsrldq $8, reg_i, reg_i + vpxor \T1, reg_i, reg_i + jmp _get_AAD_rest8\@ +_get_AAD_rest4\@: + cmp $0, %r11 + jle _get_AAD_rest0\@ + mov (%r10), %eax + movq %rax, \T1 + add $4, %r10 + sub $4, %r11 + vpslldq $12, \T1, \T1 + vpsrldq $4, reg_i, reg_i + vpxor \T1, reg_i, reg_i +_get_AAD_rest0\@: + /* finalize: shift out the extra bytes we read, and align + left. since pslldq can only shift by an immediate, we use + vpshufb and an array of shuffle masks */ + movq %r12, %r11 + salq $4, %r11 + movdqu aad_shift_arr(%r11), \T1 + vpshufb \T1, reg_i, reg_i +_get_AAD_rest_final\@: + vpshufb SHUF_MASK(%rip), reg_i, reg_i + vpxor reg_j, reg_i, reg_i + GHASH_MUL_AVX reg_i, \T2, \T1, \T3, \T4, \T5, \T6 - cmp $16, %r11 - je _get_AAD_loop2_done\@ - mov $16, %r12 - -_get_AAD_loop2\@: - vpsrldq $4, reg_i, reg_i - sub $4, %r12 - cmp %r11, %r12 - jg _get_AAD_loop2\@ - -_get_AAD_loop2_done\@: - - #byte-reflect the AAD data - vpshufb SHUF_MASK(%rip), reg_i, reg_i - +_get_AAD_done\@: # initialize the data pointer offset as zero xor %r11, %r11 @@ -480,7 +535,6 @@ _get_AAD_loop2_done\@: i = (8-\num_initial_blocks) j = (9-\num_initial_blocks) setreg - GHASH_MUL_AVX reg_i, \T2, \T1, \T3, \T4, \T5, \T6 .rep \num_initial_blocks vpxor reg_i, reg_j, reg_j From 0120af7795c4e526554e7d88f135dd1a028929cf Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Fri, 28 Apr 2017 18:11:59 +0200 Subject: [PATCH 012/140] crypto: aesni - make AVX AES-GCM work with all valid auth_tag_len Signed-off-by: Sabrina Dubroca Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_avx-x86_64.S | 31 ++++++++++++++++++------ 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S index a73117c84904..ee6283120f83 100644 --- a/arch/x86/crypto/aesni-intel_avx-x86_64.S +++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S @@ -1481,19 +1481,36 @@ _return_T\@: cmp $16, %r11 je _T_16\@ - cmp $12, %r11 - je _T_12\@ + cmp $8, %r11 + jl _T_4\@ _T_8\@: vmovq %xmm9, %rax mov %rax, (%r10) - jmp _return_T_done\@ -_T_12\@: - vmovq %xmm9, %rax - mov %rax, (%r10) + add $8, %r10 + sub $8, %r11 vpsrldq $8, %xmm9, %xmm9 + cmp $0, %r11 + je _return_T_done\@ +_T_4\@: vmovd %xmm9, %eax - mov %eax, 8(%r10) + mov %eax, (%r10) + add $4, %r10 + sub $4, %r11 + vpsrldq $4, %xmm9, %xmm9 + cmp $0, %r11 + je _return_T_done\@ +_T_123\@: + vmovd %xmm9, %eax + cmp $2, %r11 + jl _T_1\@ + mov %ax, (%r10) + cmp $2, %r11 + je _return_T_done\@ + add $2, %r10 + sar $16, %eax +_T_1\@: + mov %al, (%r10) jmp _return_T_done\@ _T_16\@: From 27352c45c56181b69aa6c2e1aad48f80dd5f9ca8 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Fri, 28 Apr 2017 18:12:00 +0200 Subject: [PATCH 013/140] crypto: aesni - make AVX2 AES-GCM work with any aadlen This is the first step to make the aesni AES-GCM implementation generic. The current code was written for rfc4106, so it handles only some specific sizes of associated data. Signed-off-by: Sabrina Dubroca Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_avx-x86_64.S | 85 ++++++++++++++++-------- 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S index ee6283120f83..7230808a7cef 100644 --- a/arch/x86/crypto/aesni-intel_avx-x86_64.S +++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S @@ -1702,41 +1702,73 @@ ENDPROC(aesni_gcm_dec_avx_gen2) .macro INITIAL_BLOCKS_AVX2 num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC VER i = (8-\num_initial_blocks) + j = 0 setreg - mov arg6, %r10 # r10 = AAD - mov arg7, %r12 # r12 = aadLen + mov arg6, %r10 # r10 = AAD + mov arg7, %r12 # r12 = aadLen - mov %r12, %r11 + mov %r12, %r11 - vpxor reg_i, reg_i, reg_i -_get_AAD_loop\@: - vmovd (%r10), \T1 - vpslldq $12, \T1, \T1 - vpsrldq $4, reg_i, reg_i - vpxor \T1, reg_i, reg_i + vpxor reg_j, reg_j, reg_j + vpxor reg_i, reg_i, reg_i - add $4, %r10 - sub $4, %r12 - jg _get_AAD_loop\@ + cmp $16, %r11 + jl _get_AAD_rest8\@ +_get_AAD_blocks\@: + vmovdqu (%r10), reg_i + vpshufb SHUF_MASK(%rip), reg_i, reg_i + vpxor reg_i, reg_j, reg_j + GHASH_MUL_AVX2 reg_j, \T2, \T1, \T3, \T4, \T5, \T6 + add $16, %r10 + sub $16, %r12 + sub $16, %r11 + cmp $16, %r11 + jge _get_AAD_blocks\@ + vmovdqu reg_j, reg_i + cmp $0, %r11 + je _get_AAD_done\@ + vpxor reg_i, reg_i, reg_i - cmp $16, %r11 - je _get_AAD_loop2_done\@ - mov $16, %r12 - -_get_AAD_loop2\@: - vpsrldq $4, reg_i, reg_i - sub $4, %r12 - cmp %r11, %r12 - jg _get_AAD_loop2\@ - -_get_AAD_loop2_done\@: - - #byte-reflect the AAD data - vpshufb SHUF_MASK(%rip), reg_i, reg_i + /* read the last <16B of AAD. since we have at least 4B of + data right after the AAD (the ICV, and maybe some CT), we can + read 4B/8B blocks safely, and then get rid of the extra stuff */ +_get_AAD_rest8\@: + cmp $4, %r11 + jle _get_AAD_rest4\@ + movq (%r10), \T1 + add $8, %r10 + sub $8, %r11 + vpslldq $8, \T1, \T1 + vpsrldq $8, reg_i, reg_i + vpxor \T1, reg_i, reg_i + jmp _get_AAD_rest8\@ +_get_AAD_rest4\@: + cmp $0, %r11 + jle _get_AAD_rest0\@ + mov (%r10), %eax + movq %rax, \T1 + add $4, %r10 + sub $4, %r11 + vpslldq $12, \T1, \T1 + vpsrldq $4, reg_i, reg_i + vpxor \T1, reg_i, reg_i +_get_AAD_rest0\@: + /* finalize: shift out the extra bytes we read, and align + left. since pslldq can only shift by an immediate, we use + vpshufb and an array of shuffle masks */ + movq %r12, %r11 + salq $4, %r11 + movdqu aad_shift_arr(%r11), \T1 + vpshufb \T1, reg_i, reg_i +_get_AAD_rest_final\@: + vpshufb SHUF_MASK(%rip), reg_i, reg_i + vpxor reg_j, reg_i, reg_i + GHASH_MUL_AVX2 reg_i, \T2, \T1, \T3, \T4, \T5, \T6 +_get_AAD_done\@: # initialize the data pointer offset as zero xor %r11, %r11 @@ -1811,7 +1843,6 @@ _get_AAD_loop2_done\@: i = (8-\num_initial_blocks) j = (9-\num_initial_blocks) setreg - GHASH_MUL_AVX2 reg_i, \T2, \T1, \T3, \T4, \T5, \T6 .rep \num_initial_blocks vpxor reg_i, reg_j, reg_j From 2df7e813627622daf1168e30cd2b29467b7e14e5 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Fri, 28 Apr 2017 18:12:01 +0200 Subject: [PATCH 014/140] crypto: aesni - make AVX2 AES-GCM work with all valid auth_tag_len Signed-off-by: Sabrina Dubroca Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_avx-x86_64.S | 31 ++++++++++++++++++------ 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S index 7230808a7cef..faecb1518bf8 100644 --- a/arch/x86/crypto/aesni-intel_avx-x86_64.S +++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S @@ -2804,19 +2804,36 @@ _return_T\@: cmp $16, %r11 je _T_16\@ - cmp $12, %r11 - je _T_12\@ + cmp $8, %r11 + jl _T_4\@ _T_8\@: vmovq %xmm9, %rax mov %rax, (%r10) - jmp _return_T_done\@ -_T_12\@: - vmovq %xmm9, %rax - mov %rax, (%r10) + add $8, %r10 + sub $8, %r11 vpsrldq $8, %xmm9, %xmm9 + cmp $0, %r11 + je _return_T_done\@ +_T_4\@: vmovd %xmm9, %eax - mov %eax, 8(%r10) + mov %eax, (%r10) + add $4, %r10 + sub $4, %r11 + vpsrldq $4, %xmm9, %xmm9 + cmp $0, %r11 + je _return_T_done\@ +_T_123\@: + vmovd %xmm9, %eax + cmp $2, %r11 + jl _T_1\@ + mov %ax, (%r10) + cmp $2, %r11 + je _return_T_done\@ + add $2, %r10 + sar $16, %eax +_T_1\@: + mov %al, (%r10) jmp _return_T_done\@ _T_16\@: From cce2ea8d90fec72ed954830e73f5d4995cc79193 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Fri, 28 Apr 2017 18:12:02 +0200 Subject: [PATCH 015/140] crypto: aesni - add generic gcm(aes) Now that the asm side of things can support all the valid lengths of ICV and all lengths of associated data, provide the glue code to expose a generic gcm(aes) crypto algorithm. Signed-off-by: Sabrina Dubroca Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_glue.c | 208 ++++++++++++++++++++++------- 1 file changed, 158 insertions(+), 50 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 93de8ea51548..4a55cdcdc008 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -61,6 +61,11 @@ struct aesni_rfc4106_gcm_ctx { u8 nonce[4]; }; +struct generic_gcmaes_ctx { + u8 hash_subkey[16] AESNI_ALIGN_ATTR; + struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; +}; + struct aesni_xts_ctx { u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; @@ -102,13 +107,11 @@ asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out, * u8 *out, Ciphertext output. Encrypt in-place is allowed. * const u8 *in, Plaintext input * unsigned long plaintext_len, Length of data in bytes for encryption. - * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) - * concatenated with 8 byte Initialisation Vector (from IPSec ESP - * Payload) concatenated with 0x00000001. 16-byte aligned pointer. + * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. + * 16-byte aligned pointer. * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. * const u8 *aad, Additional Authentication Data (AAD) - * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this - * is going to be 8 or 12 bytes + * unsigned long aad_len, Length of AAD in bytes. * u8 *auth_tag, Authenticated Tag output. * unsigned long auth_tag_len), Authenticated Tag Length in bytes. * Valid values are 16 (most likely), 12 or 8. @@ -123,9 +126,8 @@ asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, * u8 *out, Plaintext output. Decrypt in-place is allowed. * const u8 *in, Ciphertext input * unsigned long ciphertext_len, Length of data in bytes for decryption. - * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) - * concatenated with 8 byte Initialisation Vector (from IPSec ESP - * Payload) concatenated with 0x00000001. 16-byte aligned pointer. + * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. + * 16-byte aligned pointer. * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. * const u8 *aad, Additional Authentication Data (AAD) * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going @@ -275,6 +277,16 @@ aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) align = 1; return PTR_ALIGN(crypto_aead_ctx(tfm), align); } + +static inline struct +generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm) +{ + unsigned long align = AESNI_ALIGN; + + if (align <= crypto_tfm_ctx_alignment()) + align = 1; + return PTR_ALIGN(crypto_aead_ctx(tfm), align); +} #endif static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) @@ -712,32 +724,34 @@ static int rfc4106_set_authsize(struct crypto_aead *parent, return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); } -static int helper_rfc4106_encrypt(struct aead_request *req) +static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 8: + case 12: + case 13: + case 14: + case 15: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen, + u8 *hash_subkey, u8 *iv, void *aes_ctx) { u8 one_entry_in_sg = 0; u8 *src, *dst, *assoc; - __be32 counter = cpu_to_be32(1); struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); - void *aes_ctx = &(ctx->aes_key_expanded); unsigned long auth_tag_len = crypto_aead_authsize(tfm); - u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); struct scatter_walk src_sg_walk; struct scatter_walk dst_sg_walk = {}; - unsigned int i; - - /* Assuming we are supporting rfc4106 64-bit extended */ - /* sequence numbers We need to have the AAD length equal */ - /* to 16 or 20 bytes */ - if (unlikely(req->assoclen != 16 && req->assoclen != 20)) - return -EINVAL; - - /* IV below built */ - for (i = 0; i < 4; i++) - *(iv+i) = ctx->nonce[i]; - for (i = 0; i < 8; i++) - *(iv+4+i) = req->iv[i]; - *((__be32 *)(iv+12)) = counter; if (sg_is_last(req->src) && (!PageHighMem(sg_page(req->src)) || @@ -768,7 +782,7 @@ static int helper_rfc4106_encrypt(struct aead_request *req) kernel_fpu_begin(); aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv, - ctx->hash_subkey, assoc, req->assoclen - 8, + hash_subkey, assoc, assoclen, dst + req->cryptlen, auth_tag_len); kernel_fpu_end(); @@ -791,37 +805,20 @@ static int helper_rfc4106_encrypt(struct aead_request *req) return 0; } -static int helper_rfc4106_decrypt(struct aead_request *req) +static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen, + u8 *hash_subkey, u8 *iv, void *aes_ctx) { u8 one_entry_in_sg = 0; u8 *src, *dst, *assoc; unsigned long tempCipherLen = 0; - __be32 counter = cpu_to_be32(1); - int retval = 0; struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); - void *aes_ctx = &(ctx->aes_key_expanded); unsigned long auth_tag_len = crypto_aead_authsize(tfm); - u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); u8 authTag[16]; struct scatter_walk src_sg_walk; struct scatter_walk dst_sg_walk = {}; - unsigned int i; - - if (unlikely(req->assoclen != 16 && req->assoclen != 20)) - return -EINVAL; - - /* Assuming we are supporting rfc4106 64-bit extended */ - /* sequence numbers We need to have the AAD length */ - /* equal to 16 or 20 bytes */ + int retval = 0; tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); - /* IV below built */ - for (i = 0; i < 4; i++) - *(iv+i) = ctx->nonce[i]; - for (i = 0; i < 8; i++) - *(iv+4+i) = req->iv[i]; - *((__be32 *)(iv+12)) = counter; if (sg_is_last(req->src) && (!PageHighMem(sg_page(req->src)) || @@ -838,7 +835,6 @@ static int helper_rfc4106_decrypt(struct aead_request *req) scatterwalk_start(&dst_sg_walk, req->dst); dst = scatterwalk_map(&dst_sg_walk) + req->assoclen; } - } else { /* Allocate memory for src, dst, assoc */ assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); @@ -850,9 +846,10 @@ static int helper_rfc4106_decrypt(struct aead_request *req) dst = src; } + kernel_fpu_begin(); aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv, - ctx->hash_subkey, assoc, req->assoclen - 8, + hash_subkey, assoc, assoclen, authTag, auth_tag_len); kernel_fpu_end(); @@ -875,6 +872,60 @@ static int helper_rfc4106_decrypt(struct aead_request *req) kfree(assoc); } return retval; + +} + +static int helper_rfc4106_encrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + void *aes_ctx = &(ctx->aes_key_expanded); + u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + unsigned int i; + __be32 counter = cpu_to_be32(1); + + /* Assuming we are supporting rfc4106 64-bit extended */ + /* sequence numbers We need to have the AAD length equal */ + /* to 16 or 20 bytes */ + if (unlikely(req->assoclen != 16 && req->assoclen != 20)) + return -EINVAL; + + /* IV below built */ + for (i = 0; i < 4; i++) + *(iv+i) = ctx->nonce[i]; + for (i = 0; i < 8; i++) + *(iv+4+i) = req->iv[i]; + *((__be32 *)(iv+12)) = counter; + + return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, + aes_ctx); +} + +static int helper_rfc4106_decrypt(struct aead_request *req) +{ + __be32 counter = cpu_to_be32(1); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + void *aes_ctx = &(ctx->aes_key_expanded); + u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + unsigned int i; + + if (unlikely(req->assoclen != 16 && req->assoclen != 20)) + return -EINVAL; + + /* Assuming we are supporting rfc4106 64-bit extended */ + /* sequence numbers We need to have the AAD length */ + /* equal to 16 or 20 bytes */ + + /* IV below built */ + for (i = 0; i < 4; i++) + *(iv+i) = ctx->nonce[i]; + for (i = 0; i < 8; i++) + *(iv+4+i) = req->iv[i]; + *((__be32 *)(iv+12)) = counter; + + return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, + aes_ctx); } static int rfc4106_encrypt(struct aead_request *req) @@ -1035,6 +1086,46 @@ struct { }; #ifdef CONFIG_X86_64 +static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key, + unsigned int key_len) +{ + struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead); + + return aes_set_key_common(crypto_aead_tfm(aead), + &ctx->aes_key_expanded, key, key_len) ?: + rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); +} + +static int generic_gcmaes_encrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); + void *aes_ctx = &(ctx->aes_key_expanded); + u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + __be32 counter = cpu_to_be32(1); + + memcpy(iv, req->iv, 12); + *((__be32 *)(iv+12)) = counter; + + return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv, + aes_ctx); +} + +static int generic_gcmaes_decrypt(struct aead_request *req) +{ + __be32 counter = cpu_to_be32(1); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + void *aes_ctx = &(ctx->aes_key_expanded); + u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + + memcpy(iv, req->iv, 12); + *((__be32 *)(iv+12)) = counter; + + return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv, + aes_ctx); +} + static struct aead_alg aesni_aead_algs[] = { { .setkey = common_rfc4106_set_key, .setauthsize = common_rfc4106_set_authsize, @@ -1069,6 +1160,23 @@ static struct aead_alg aesni_aead_algs[] = { { .cra_ctxsize = sizeof(struct cryptd_aead *), .cra_module = THIS_MODULE, }, +}, { + .setkey = generic_gcmaes_set_key, + .setauthsize = generic_gcmaes_set_authsize, + .encrypt = generic_gcmaes_encrypt, + .decrypt = generic_gcmaes_decrypt, + .ivsize = 12, + .maxauthsize = 16, + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "generic-gcm-aesni", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), + .cra_alignmask = AESNI_ALIGN - 1, + .cra_module = THIS_MODULE, + }, } }; #else static struct aead_alg aesni_aead_algs[0]; From 986130bf3701e9875a820e06ed5c6ae0a851900b Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Mon, 15 May 2017 13:52:03 +0530 Subject: [PATCH 016/140] hwrng: omap3-rom - Handle return value of clk_prepare_enable Here, Clock enable can failed. So adding an error check for clk_prepare_enable. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/char/hw_random/omap3-rom-rng.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/char/hw_random/omap3-rom-rng.c b/drivers/char/hw_random/omap3-rom-rng.c index 37a58d78aab3..38b719017186 100644 --- a/drivers/char/hw_random/omap3-rom-rng.c +++ b/drivers/char/hw_random/omap3-rom-rng.c @@ -53,7 +53,10 @@ static int omap3_rom_rng_get_random(void *buf, unsigned int count) cancel_delayed_work_sync(&idle_work); if (rng_idle) { - clk_prepare_enable(rng_clk); + r = clk_prepare_enable(rng_clk); + if (r) + return r; + r = omap3_rom_rng_call(0, 0, RNG_GEN_PRNG_HW_INIT); if (r != 0) { clk_disable_unprepare(rng_clk); @@ -88,6 +91,8 @@ static struct hwrng omap3_rom_rng_ops = { static int omap3_rom_rng_probe(struct platform_device *pdev) { + int ret = 0; + pr_info("initializing\n"); omap3_rom_rng_call = pdev->dev.platform_data; @@ -104,7 +109,9 @@ static int omap3_rom_rng_probe(struct platform_device *pdev) } /* Leave the RNG in reset state. */ - clk_prepare_enable(rng_clk); + ret = clk_prepare_enable(rng_clk); + if (ret) + return ret; omap3_rom_rng_idle(0); return hwrng_register(&omap3_rom_rng_ops); From 5f052c9c631d360d6ec6202714b9d6529a21117a Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Tue, 16 May 2017 13:57:41 +0530 Subject: [PATCH 017/140] crypto: img-hash - Handle return value of clk_prepare_enable Here, Clock enable can failed. So adding an error check for clk_prepare_enable. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/img-hash.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 9b07f3d88feb..0c6a917a9ab8 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -1088,9 +1088,17 @@ static int img_hash_suspend(struct device *dev) static int img_hash_resume(struct device *dev) { struct img_hash_dev *hdev = dev_get_drvdata(dev); + int ret; - clk_prepare_enable(hdev->hash_clk); - clk_prepare_enable(hdev->sys_clk); + ret = clk_prepare_enable(hdev->hash_clk); + if (ret) + return ret; + + ret = clk_prepare_enable(hdev->sys_clk); + if (ret) { + clk_disable_unprepare(hdev->hash_clk); + return ret; + } return 0; } From 248c65056ccca7d51db78e6821ff89a020e168bb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 16 May 2017 16:21:05 +0200 Subject: [PATCH 018/140] crypto: qat - use pcie_flr instead of duplicating it Signed-off-by: Christoph Hellwig Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_aer.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index 2839fccdd84b..d3e25c37dc33 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -109,20 +109,7 @@ EXPORT_SYMBOL_GPL(adf_reset_sbr); void adf_reset_flr(struct adf_accel_dev *accel_dev) { - struct pci_dev *pdev = accel_to_pci_dev(accel_dev); - u16 control = 0; - int pos = 0; - - dev_info(&GET_DEV(accel_dev), "Function level reset\n"); - pos = pci_pcie_cap(pdev); - if (!pos) { - dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); - return; - } - pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &control); - control |= PCI_EXP_DEVCTL_BCR_FLR; - pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, control); - msleep(100); + pcie_flr(accel_to_pci_dev(accel_dev)); } EXPORT_SYMBOL_GPL(adf_reset_flr); From 9417cd1c517a89ccfb5cc3d3f2a0e50fc64f3445 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 16 May 2017 21:03:08 -0700 Subject: [PATCH 019/140] crypto: x86/aes - Don't use %rbp as temporary register When using the "aes-asm" implementation of AES (*not* the AES-NI implementation) on an x86_64, v4.12-rc1 kernel with lockdep enabled, the following warning was reported, along with a long unwinder dump: WARNING: kernel stack regs at ffffc90000643558 in kworker/u4:2:155 has bad 'bp' value 000000000000001c The problem is that aes_enc_block() and aes_dec_block() use %rbp as a temporary register, which breaks stack traces if an interrupt occurs. Fix this by replacing %rbp with %r9, which was being used to hold the saved value of %rbp. This required rearranging the AES round macro slightly since %r9d cannot be used as the target of a move from %ah-%dh. Performance is essentially unchanged --- actually about 0.2% faster than before. Interestingly, I also measured aes-generic as being nearly 7% faster than aes-asm, so perhaps aes-asm has outlived its usefulness... Signed-off-by: Eric Biggers Reviewed-by: Josh Poimboeuf Signed-off-by: Herbert Xu --- arch/x86/crypto/aes-x86_64-asm_64.S | 47 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/arch/x86/crypto/aes-x86_64-asm_64.S b/arch/x86/crypto/aes-x86_64-asm_64.S index 910565547163..8739cf7795de 100644 --- a/arch/x86/crypto/aes-x86_64-asm_64.S +++ b/arch/x86/crypto/aes-x86_64-asm_64.S @@ -42,17 +42,15 @@ #define R5E %esi #define R6 %rdi #define R6E %edi -#define R7 %rbp -#define R7E %ebp +#define R7 %r9 /* don't use %rbp; it breaks stack traces */ +#define R7E %r9d #define R8 %r8 -#define R9 %r9 #define R10 %r10 #define R11 %r11 -#define prologue(FUNC,KEY,B128,B192,r1,r2,r3,r4,r5,r6,r7,r8,r9,r10,r11) \ +#define prologue(FUNC,KEY,B128,B192,r1,r2,r5,r6,r7,r8,r9,r10,r11) \ ENTRY(FUNC); \ movq r1,r2; \ - movq r3,r4; \ leaq KEY+48(r8),r9; \ movq r10,r11; \ movl (r7),r5 ## E; \ @@ -70,9 +68,8 @@ je B192; \ leaq 32(r9),r9; -#define epilogue(FUNC,r1,r2,r3,r4,r5,r6,r7,r8,r9) \ +#define epilogue(FUNC,r1,r2,r5,r6,r7,r8,r9) \ movq r1,r2; \ - movq r3,r4; \ movl r5 ## E,(r9); \ movl r6 ## E,4(r9); \ movl r7 ## E,8(r9); \ @@ -88,12 +85,12 @@ movl TAB(,r6,4),r6 ## E; \ roll $16,r2 ## E; \ shrl $16,r4 ## E; \ - movzbl r4 ## H,r7 ## E; \ - movzbl r4 ## L,r4 ## E; \ + movzbl r4 ## L,r7 ## E; \ + movzbl r4 ## H,r4 ## E; \ xorl OFFSET(r8),ra ## E; \ xorl OFFSET+4(r8),rb ## E; \ - xorl TAB+3072(,r7,4),r5 ## E;\ - xorl TAB+2048(,r4,4),r6 ## E;\ + xorl TAB+3072(,r4,4),r5 ## E;\ + xorl TAB+2048(,r7,4),r6 ## E;\ movzbl r1 ## L,r7 ## E; \ movzbl r1 ## H,r4 ## E; \ movl TAB+1024(,r4,4),r4 ## E;\ @@ -101,19 +98,19 @@ roll $16,r1 ## E; \ shrl $16,r3 ## E; \ xorl TAB(,r7,4),r5 ## E; \ - movzbl r3 ## H,r7 ## E; \ - movzbl r3 ## L,r3 ## E; \ - xorl TAB+3072(,r7,4),r4 ## E;\ - xorl TAB+2048(,r3,4),r5 ## E;\ - movzbl r1 ## H,r7 ## E; \ - movzbl r1 ## L,r3 ## E; \ + movzbl r3 ## L,r7 ## E; \ + movzbl r3 ## H,r3 ## E; \ + xorl TAB+3072(,r3,4),r4 ## E;\ + xorl TAB+2048(,r7,4),r5 ## E;\ + movzbl r1 ## L,r7 ## E; \ + movzbl r1 ## H,r3 ## E; \ shrl $16,r1 ## E; \ - xorl TAB+3072(,r7,4),r6 ## E;\ - movl TAB+2048(,r3,4),r3 ## E;\ - movzbl r1 ## H,r7 ## E; \ - movzbl r1 ## L,r1 ## E; \ - xorl TAB+1024(,r7,4),r6 ## E;\ - xorl TAB(,r1,4),r3 ## E; \ + xorl TAB+3072(,r3,4),r6 ## E;\ + movl TAB+2048(,r7,4),r3 ## E;\ + movzbl r1 ## L,r7 ## E; \ + movzbl r1 ## H,r1 ## E; \ + xorl TAB+1024(,r1,4),r6 ## E;\ + xorl TAB(,r7,4),r3 ## E; \ movzbl r2 ## H,r1 ## E; \ movzbl r2 ## L,r7 ## E; \ shrl $16,r2 ## E; \ @@ -131,9 +128,9 @@ movl r4 ## E,r2 ## E; #define entry(FUNC,KEY,B128,B192) \ - prologue(FUNC,KEY,B128,B192,R2,R8,R7,R9,R1,R3,R4,R6,R10,R5,R11) + prologue(FUNC,KEY,B128,B192,R2,R8,R1,R3,R4,R6,R10,R5,R11) -#define return(FUNC) epilogue(FUNC,R8,R2,R9,R7,R5,R6,R3,R4,R11) +#define return(FUNC) epilogue(FUNC,R8,R2,R5,R6,R3,R4,R11) #define encrypt_round(TAB,OFFSET) \ round(TAB,OFFSET,R1,R2,R3,R4,R5,R6,R7,R10,R5,R6,R3,R4) \ From 03d7db5654aefb78bf2ded62e2187833fda2a6cc Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Fri, 19 May 2017 08:53:23 +0200 Subject: [PATCH 020/140] crypto: hmac - add hmac IPAD/OPAD constant Many HMAC users directly use directly 0x36/0x5c values. It's better with crypto to use a name instead of directly some crypto constant. This patch simply add HMAC_IPAD_VALUE/HMAC_OPAD_VALUE defines in a new include file "crypto/hmac.h" and use them in crypto/hmac.c Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- crypto/hmac.c | 5 +++-- include/crypto/hmac.h | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 include/crypto/hmac.h diff --git a/crypto/hmac.c b/crypto/hmac.c index 72e38c098bb3..92871dc2a63e 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -16,6 +16,7 @@ * */ +#include #include #include #include @@ -74,8 +75,8 @@ static int hmac_setkey(struct crypto_shash *parent, memcpy(opad, ipad, bs); for (i = 0; i < bs; i++) { - ipad[i] ^= 0x36; - opad[i] ^= 0x5c; + ipad[i] ^= HMAC_IPAD_VALUE; + opad[i] ^= HMAC_OPAD_VALUE; } return crypto_shash_init(shash) ?: diff --git a/include/crypto/hmac.h b/include/crypto/hmac.h new file mode 100644 index 000000000000..ef09f7938204 --- /dev/null +++ b/include/crypto/hmac.h @@ -0,0 +1,7 @@ +#ifndef _CRYPTO_HMAC_H +#define _CRYPTO_HMAC_H + +#define HMAC_IPAD_VALUE 0x36 +#define HMAC_OPAD_VALUE 0x5c + +#endif /* _CRYPTO_HMAC_H */ From 1126d47db0cf40a5f31060095507ea5416a43344 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Fri, 19 May 2017 08:53:24 +0200 Subject: [PATCH 021/140] crypto: brcm - Use IPAD/OPAD constant This patch simply replace all occurrence of HMAC IPAD/OPAD value by their define. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/bcm/cipher.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index cc0d5b98006e..61393dc70b0b 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -2510,8 +2511,8 @@ static int ahash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key, memcpy(ctx->opad, ctx->ipad, blocksize); for (index = 0; index < blocksize; index++) { - ctx->ipad[index] ^= 0x36; - ctx->opad[index] ^= 0x5c; + ctx->ipad[index] ^= HMAC_IPAD_VALUE; + ctx->opad[index] ^= HMAC_OPAD_VALUE; } flow_dump(" ipad: ", ctx->ipad, blocksize); From bb9634df0526593f60294f6f3c90787f7d4df7b0 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Fri, 19 May 2017 08:53:25 +0200 Subject: [PATCH 022/140] crypto: ixp4xx - Use IPAD/OPAD constant This patch simply replace all occurrence of HMAC IPAD/OPAD value by their define. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/ixp4xx_crypto.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 771dd26c7076..427cbe012729 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -90,8 +91,6 @@ #define CTL_FLAG_PERFORM_AEAD 0x0008 #define CTL_FLAG_MASK 0x000f -#define HMAC_IPAD_VALUE 0x36 -#define HMAC_OPAD_VALUE 0x5C #define HMAC_PAD_BLOCKLEN SHA1_BLOCK_SIZE #define MD5_DIGEST_SIZE 16 From d477d813363ce6f393c310de8a2269b3eacd18a5 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Fri, 19 May 2017 08:53:26 +0200 Subject: [PATCH 023/140] crypto: marvell - Use IPAD/OPAD constant This patch simply replace all occurrence of HMAC IPAD/OPAD value by their define. Signed-off-by: Corentin Labbe Acked-by: Boris Brezillon Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 77c0fb936f47..e61b08566093 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -12,6 +12,7 @@ * by the Free Software Foundation. */ +#include #include #include @@ -1164,8 +1165,8 @@ static int mv_cesa_ahmac_pad_init(struct ahash_request *req, memcpy(opad, ipad, blocksize); for (i = 0; i < blocksize; i++) { - ipad[i] ^= 0x36; - opad[i] ^= 0x5c; + ipad[i] ^= HMAC_IPAD_VALUE; + opad[i] ^= HMAC_OPAD_VALUE; } return 0; From 8139782dce737299385d7c9d91a2f62517540f6f Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Fri, 19 May 2017 08:53:27 +0200 Subject: [PATCH 024/140] crypto: mv_cesa - Use IPAD/OPAD constant This patch simply replace all occurrence of HMAC IPAD/OPAD value by their define. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/mv_cesa.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c index 451fa18c1c7b..bf25f415eea6 100644 --- a/drivers/crypto/mv_cesa.c +++ b/drivers/crypto/mv_cesa.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -822,8 +823,8 @@ static int mv_hash_setkey(struct crypto_ahash *tfm, const u8 * key, memcpy(opad, ipad, bs); for (i = 0; i < bs; i++) { - ipad[i] ^= 0x36; - opad[i] ^= 0x5c; + ipad[i] ^= HMAC_IPAD_VALUE; + opad[i] ^= HMAC_OPAD_VALUE; } rc = crypto_shash_init(shash) ? : From ebd401e702dd88fb95a92617bf52ad41c3bf61cd Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Fri, 19 May 2017 08:53:28 +0200 Subject: [PATCH 025/140] crypto: omap-sham - Use IPAD/OPAD constant This patch simply replace all occurrence of HMAC IPAD/OPAD value by their define. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index d0b16e5e4ee5..1864a57caaa4 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #define MD5_DIGEST_SIZE 16 @@ -1326,8 +1327,8 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key, memcpy(bctx->opad, bctx->ipad, bs); for (i = 0; i < bs; i++) { - bctx->ipad[i] ^= 0x36; - bctx->opad[i] ^= 0x5c; + bctx->ipad[i] ^= HMAC_IPAD_VALUE; + bctx->opad[i] ^= HMAC_OPAD_VALUE; } } From f14011ad7cf79a590a39ad2dd7062153ef19ab6e Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Fri, 19 May 2017 08:53:29 +0200 Subject: [PATCH 026/140] crypto: qat - Use IPAD/OPAD constant This patch simply replace all occurrence of HMAC IPAD/OPAD value by their define. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 20f35df8a01f..5b5efcc52cb5 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -178,8 +179,8 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, for (i = 0; i < block_size; i++) { char *ipad_ptr = ipad + i; char *opad_ptr = opad + i; - *ipad_ptr ^= 0x36; - *opad_ptr ^= 0x5C; + *ipad_ptr ^= HMAC_IPAD_VALUE; + *opad_ptr ^= HMAC_OPAD_VALUE; } if (crypto_shash_init(shash)) From 1127eea914c74bf8c3cb3ddaa3dcaae628e87263 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Fri, 19 May 2017 08:53:30 +0200 Subject: [PATCH 027/140] crypto: mediatek - Use IPAD/OPAD constant This patch simply replace all occurrence of HMAC IPAD/OPAD value by their define. Signed-off-by: Corentin Labbe Reviewed-by: Matthias Brugger Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-sha.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index 2226f12d1c7a..5f4f845adbb8 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -12,6 +12,7 @@ * Some ideas are from atmel-sha.c and omap-sham.c drivers. */ +#include #include #include "mtk-platform.h" @@ -825,8 +826,8 @@ static int mtk_sha_setkey(struct crypto_ahash *tfm, const u8 *key, memcpy(bctx->opad, bctx->ipad, bs); for (i = 0; i < bs; i++) { - bctx->ipad[i] ^= 0x36; - bctx->opad[i] ^= 0x5c; + bctx->ipad[i] ^= HMAC_IPAD_VALUE; + bctx->opad[i] ^= HMAC_OPAD_VALUE; } return 0; From 6507c57bb013a22ae0f61bdbb2a63380598d34a2 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Fri, 19 May 2017 08:53:31 +0200 Subject: [PATCH 028/140] crypto: ccp - Use IPAD/OPAD constant This patch simply replace all occurrence of HMAC IPAD/OPAD value by their define. Signed-off-by: Corentin Labbe Acked-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-crypto-sha.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c index 6b46eea94932..ce97b3868f4a 100644 --- a/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -308,8 +309,8 @@ static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key, } for (i = 0; i < block_size; i++) { - ctx->u.sha.ipad[i] = ctx->u.sha.key[i] ^ 0x36; - ctx->u.sha.opad[i] = ctx->u.sha.key[i] ^ 0x5c; + ctx->u.sha.ipad[i] = ctx->u.sha.key[i] ^ HMAC_IPAD_VALUE; + ctx->u.sha.opad[i] = ctx->u.sha.key[i] ^ HMAC_OPAD_VALUE; } sg_init_one(&ctx->u.sha.opad_sg, ctx->u.sha.opad, block_size); From 4d8061a5918dea53402170932a861c09f3b88df5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 21 May 2017 10:23:36 +0000 Subject: [PATCH 029/140] crypto: arm/aes-ce - enable module autoloading based on CPU feature bits Make the module autoloadable by tying it to the CPU feature bit that describes whether the optional instructions it relies on are implemented by the current CPU. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-ce-glue.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c index 883b84d828c5..0f966a8ca1ce 100644 --- a/arch/arm/crypto/aes-ce-glue.c +++ b/arch/arm/crypto/aes-ce-glue.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -425,9 +426,6 @@ static int __init aes_init(void) int err; int i; - if (!(elf_hwcap2 & HWCAP2_AES)) - return -ENODEV; - err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); if (err) return err; @@ -451,5 +449,5 @@ unregister_simds: return err; } -module_init(aes_init); +module_cpu_feature_match(AES, aes_init); module_exit(aes_exit); From c9d9f608b4339435ccec6bd2ec1a5c42e066788e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 21 May 2017 10:23:37 +0000 Subject: [PATCH 030/140] crypto: arm/ghash-ce - enable module autoloading based on CPU feature bits Make the module autoloadable by tying it to the CPU feature bit that describes whether the optional instructions it relies on are implemented by the current CPU. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/ghash-ce-glue.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index 7546b3c02466..6bac8bea9f1e 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -311,9 +312,6 @@ static int __init ghash_ce_mod_init(void) { int err; - if (!(elf_hwcap2 & HWCAP2_PMULL)) - return -ENODEV; - err = crypto_register_shash(&ghash_alg); if (err) return err; @@ -334,5 +332,5 @@ static void __exit ghash_ce_mod_exit(void) crypto_unregister_shash(&ghash_alg); } -module_init(ghash_ce_mod_init); +module_cpu_feature_match(PMULL, ghash_ce_mod_init); module_exit(ghash_ce_mod_exit); From bd56f95ea933d1dd1628373a8213716225605d1c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 21 May 2017 10:23:38 +0000 Subject: [PATCH 031/140] crypto: arm/sha1-ce - enable module autoloading based on CPU feature bits Make the module autoloadable by tying it to the CPU feature bit that describes whether the optional instructions it relies on are implemented by the current CPU. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/sha1-ce-glue.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c index 80bc2fcd241a..555f72b5e659 100644 --- a/arch/arm/crypto/sha1-ce-glue.c +++ b/arch/arm/crypto/sha1-ce-glue.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -82,8 +83,6 @@ static struct shash_alg alg = { static int __init sha1_ce_mod_init(void) { - if (!(elf_hwcap2 & HWCAP2_SHA1)) - return -ENODEV; return crypto_register_shash(&alg); } @@ -92,5 +91,5 @@ static void __exit sha1_ce_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(sha1_ce_mod_init); +module_cpu_feature_match(SHA1, sha1_ce_mod_init); module_exit(sha1_ce_mod_fini); From a83ff88bedb2794f5990439276bf0abbc4b3aeb8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 21 May 2017 10:23:39 +0000 Subject: [PATCH 032/140] crypto: arm/sha2-ce - enable module autoloading based on CPU feature bits Make the module autoloadable by tying it to the CPU feature bit that describes whether the optional instructions it relies on are implemented by the current CPU. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/sha2-ce-glue.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c index 0755b2d657f3..df4dcef054ae 100644 --- a/arch/arm/crypto/sha2-ce-glue.c +++ b/arch/arm/crypto/sha2-ce-glue.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -100,8 +101,6 @@ static struct shash_alg algs[] = { { static int __init sha2_ce_mod_init(void) { - if (!(elf_hwcap2 & HWCAP2_SHA2)) - return -ENODEV; return crypto_register_shashes(algs, ARRAY_SIZE(algs)); } @@ -110,5 +109,5 @@ static void __exit sha2_ce_mod_fini(void) crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } -module_init(sha2_ce_mod_init); +module_cpu_feature_match(SHA2, sha2_ce_mod_init); module_exit(sha2_ce_mod_fini); From 2a9faf8b7e4350967ed7b59e39522972b9ead0c9 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 21 May 2017 10:23:40 +0000 Subject: [PATCH 033/140] crypto: arm/crc32 - enable module autoloading based on CPU feature bits Make the module autoloadable by tying it to the CPU feature bits that describe whether the optional instructions it relies on are implemented by the current CPU. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/crc32-ce-glue.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm/crypto/crc32-ce-glue.c b/arch/arm/crypto/crc32-ce-glue.c index e1566bec1016..1b0e0e86ee9c 100644 --- a/arch/arm/crypto/crc32-ce-glue.c +++ b/arch/arm/crypto/crc32-ce-glue.c @@ -8,6 +8,7 @@ * published by the Free Software Foundation. */ +#include #include #include #include @@ -233,6 +234,11 @@ static void __exit crc32_pmull_mod_exit(void) ARRAY_SIZE(crc32_pmull_algs)); } +static const struct cpu_feature crc32_cpu_feature[] = { + { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { } +}; +MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature); + module_init(crc32_pmull_mod_init); module_exit(crc32_pmull_mod_exit); From b14c53451719dc6ec8e6387e5be3be23294a6f1d Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Mon, 22 May 2017 14:12:23 -0700 Subject: [PATCH 034/140] dt-bindings: timeriomem_rng: Add entropy quality property Signed-off-by: Rick Altherr Signed-off-by: Herbert Xu --- Documentation/devicetree/bindings/rng/timeriomem_rng.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/devicetree/bindings/rng/timeriomem_rng.txt b/Documentation/devicetree/bindings/rng/timeriomem_rng.txt index 6616d15866a3..214940093b55 100644 --- a/Documentation/devicetree/bindings/rng/timeriomem_rng.txt +++ b/Documentation/devicetree/bindings/rng/timeriomem_rng.txt @@ -5,6 +5,13 @@ Required properties: - reg : base address to sample from - period : wait time in microseconds to use between samples +Optional properties: +- quality : estimated number of bits of true entropy per 1024 bits read from the + rng. Defaults to zero which causes the kernel's default quality to + be used instead. Note that the default quality is usually zero + which disables using this rng to automatically fill the kernel's + entropy pool. + N.B. currently 'reg' must be four bytes wide and aligned Example: From 284e76387c38260e834c99b010a68d75fc46b394 Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Mon, 22 May 2017 14:12:24 -0700 Subject: [PATCH 035/140] hw_random: timeriomem_rng: Allow setting RNG quality from platform data When a hw_random device's quality is non-zero, it will automatically be used to fill the kernel's entropy pool. Since timeriomem_rng is used by many different devices, the quality needs to be provided by platform data or device tree. Signed-off-by: Rick Altherr Signed-off-by: Herbert Xu --- drivers/char/hw_random/timeriomem-rng.c | 7 +++++++ include/linux/timeriomem-rng.h | 3 +++ 2 files changed, 10 insertions(+) diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index a0faa5f05deb..03ff5483d865 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -151,8 +151,15 @@ static int timeriomem_rng_probe(struct platform_device *pdev) dev_err(&pdev->dev, "missing period\n"); return -EINVAL; } + + if (!of_property_read_u32(pdev->dev.of_node, + "quality", &i)) + priv->rng_ops.quality = i; + else + priv->rng_ops.quality = 0; } else { period = pdata->period; + priv->rng_ops.quality = pdata->quality; } priv->period = ns_to_ktime(period * NSEC_PER_USEC); diff --git a/include/linux/timeriomem-rng.h b/include/linux/timeriomem-rng.h index 46eb27ddbfab..3e00122bcf88 100644 --- a/include/linux/timeriomem-rng.h +++ b/include/linux/timeriomem-rng.h @@ -13,4 +13,7 @@ struct timeriomem_rng_data { /* measures in usecs */ unsigned int period; + + /* bits of entropy per 1024 bits read */ + unsigned int quality; }; From 1e6adff449c12b1ee92300af9436ea70461e9887 Mon Sep 17 00:00:00 2001 From: Matthias Kaehlcke Date: Mon, 22 May 2017 15:49:43 -0700 Subject: [PATCH 036/140] crypto: rng - Remove unused function __crypto_rng_cast() This fixes the following warning when building with clang: crypto/rng.c:35:34: error: unused function '__crypto_rng_cast' [-Werror,-Wunused-function] Signed-off-by: Matthias Kaehlcke Signed-off-by: Herbert Xu --- crypto/rng.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crypto/rng.c b/crypto/rng.c index f46dac5288b9..5e8469244960 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -33,11 +33,6 @@ struct crypto_rng *crypto_default_rng; EXPORT_SYMBOL_GPL(crypto_default_rng); static int crypto_default_rng_refcnt; -static inline struct crypto_rng *__crypto_rng_cast(struct crypto_tfm *tfm) -{ - return container_of(tfm, struct crypto_rng, base); -} - int crypto_rng_reset(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) { u8 *buf = NULL; From f09b348c1ac56b61fceb12808b4003109188f1d2 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:22 +0300 Subject: [PATCH 037/140] crypto: omap-aes - drop unused flags attribute from omap_aes_ctx This is not used for anything, so drop it. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index fe32dd95ae4f..ad6e2b37dbd5 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -105,7 +105,6 @@ struct omap_aes_ctx { int keylen; u32 key[AES_KEYSIZE_256 / sizeof(u32)]; - unsigned long flags; struct crypto_skcipher *fallback; }; From 619ce700583f0f193bfb1487ca393b8ad2141a9a Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:23 +0300 Subject: [PATCH 038/140] crypto: omap-aes - fix context handling for multiple cores AES can have multiple HW accelerator cores in the system, in which case each core has its own crypto engine in use. Currently, the used hardware device is stored under the omap_aes_ctx struct, which is global for the algorithm itself, causing conflicts when used with multiple cores. Fix this by moving the used HW device under reqctx, which is stored per-request basis. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index ad6e2b37dbd5..379d70157ae7 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -101,14 +101,13 @@ #define AES_BLOCK_WORDS (AES_BLOCK_SIZE >> 2) struct omap_aes_ctx { - struct omap_aes_dev *dd; - int keylen; u32 key[AES_KEYSIZE_256 / sizeof(u32)]; struct crypto_skcipher *fallback; }; struct omap_aes_reqctx { + struct omap_aes_dev *dd; unsigned long mode; }; @@ -328,14 +327,14 @@ static void omap_aes_dma_stop(struct omap_aes_dev *dd) omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask); } -static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx) +static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx) { struct omap_aes_dev *dd; spin_lock_bh(&list_lock); dd = list_first_entry(&dev_list, struct omap_aes_dev, list); list_move_tail(&dd->list, &dev_list); - ctx->dd = dd; + rctx->dd = dd; spin_unlock_bh(&list_lock); return dd; @@ -400,12 +399,11 @@ static void sg_copy_buf(void *buf, struct scatterlist *sg, scatterwalk_done(&walk, out, 0); } -static int omap_aes_crypt_dma(struct crypto_tfm *tfm, - struct scatterlist *in_sg, struct scatterlist *out_sg, - int in_sg_len, int out_sg_len) +static int omap_aes_crypt_dma(struct omap_aes_dev *dd, + struct scatterlist *in_sg, + struct scatterlist *out_sg, + int in_sg_len, int out_sg_len) { - struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm); - struct omap_aes_dev *dd = ctx->dd; struct dma_async_tx_descriptor *tx_in, *tx_out; struct dma_slave_config cfg; int ret; @@ -483,8 +481,6 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm, static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd) { - struct crypto_tfm *tfm = crypto_ablkcipher_tfm( - crypto_ablkcipher_reqtfm(dd->req)); int err; pr_debug("total: %d\n", dd->total); @@ -505,7 +501,7 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd) } } - err = omap_aes_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len, + err = omap_aes_crypt_dma(dd, dd->in_sg, dd->out_sg, dd->in_sg_len, dd->out_sg_len); if (err && !dd->pio_only) { dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); @@ -608,8 +604,8 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, { struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx( crypto_ablkcipher_reqtfm(req)); - struct omap_aes_dev *dd = ctx->dd; - struct omap_aes_reqctx *rctx; + struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req); + struct omap_aes_dev *dd = rctx->dd; if (!dd) return -ENODEV; @@ -638,13 +634,11 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, dd->sgs_copied = 0; } - rctx = ablkcipher_request_ctx(req); - ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); rctx->mode &= FLAGS_MODE_MASK; dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; dd->ctx = ctx; - ctx->dd = dd; + rctx->dd = dd; return omap_aes_write_ctrl(dd); } @@ -652,9 +646,8 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, static int omap_aes_crypt_req(struct crypto_engine *engine, struct ablkcipher_request *req) { - struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx( - crypto_ablkcipher_reqtfm(req)); - struct omap_aes_dev *dd = ctx->dd; + struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req); + struct omap_aes_dev *dd = rctx->dd; if (!dd) return -ENODEV; @@ -725,7 +718,7 @@ static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode) skcipher_request_zero(subreq); return ret; } - dd = omap_aes_find_dev(ctx); + dd = omap_aes_find_dev(rctx); if (!dd) return -ENODEV; From a636fdce55c6ad12b07436424ceffad1045714db Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:24 +0300 Subject: [PATCH 039/140] crypto: omap-des - add check for weak keys OMAP DES crypto accelerator itself is unable to detect weak keys, so add a specific call to the generic des driver to check the key strength if requested. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-des.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index a6f65532fd16..0d68f77ea53a 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -699,16 +699,28 @@ static int omap_des_crypt(struct ablkcipher_request *req, unsigned long mode) /* ********************** ALG API ************************************ */ -static int omap_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, +static int omap_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(cipher); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); if (keylen != DES_KEY_SIZE && keylen != (3*DES_KEY_SIZE)) return -EINVAL; pr_debug("enter, keylen: %d\n", keylen); + /* Do we need to test against weak key? */ + if (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY) { + u32 tmp[DES_EXPKEY_WORDS]; + int ret = des_ekey(tmp, key); + + if (!ret) { + tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; + return -EINVAL; + } + } + memcpy(ctx->key, key, keylen); ctx->keylen = keylen; From 418f2a8cf2ebb817a2b98cb4829c13d374e06932 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:25 +0300 Subject: [PATCH 040/140] crypto: omap-des - use runtime_pm autosuspend for clock handling Convert the driver to use autosuspend for runtime_pm. This boosts the performance, and optimizes the power consumption for the driver. By default, the timeout value for autosuspend is set to one second. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-des.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 0d68f77ea53a..3783dcf6bc37 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -78,6 +78,8 @@ #define FLAGS_INIT BIT(4) #define FLAGS_BUSY BIT(6) +#define DEFAULT_AUTOSUSPEND_DELAY 1000 + struct omap_des_ctx { struct omap_des_dev *dd; @@ -506,8 +508,10 @@ static void omap_des_finish_req(struct omap_des_dev *dd, int err) pr_debug("err: %d\n", err); - pm_runtime_put(dd->dev); crypto_finalize_cipher_request(dd->engine, req, err); + + pm_runtime_mark_last_busy(dd->dev); + pm_runtime_put_autosuspend(dd->dev); } static int omap_des_crypt_dma_stop(struct omap_des_dev *dd) @@ -1044,8 +1048,10 @@ static int omap_des_probe(struct platform_device *pdev) } dd->phys_base = res->start; + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); + pm_runtime_enable(dev); - pm_runtime_irq_safe(dev); err = pm_runtime_get_sync(dev); if (err < 0) { pm_runtime_put_noidle(dev); From 74ed87e7e7f7197137164738dd0610ccd5ec5ed1 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:26 +0300 Subject: [PATCH 041/140] crypto: omap - add base support library for common routines This contains the generic APIs for aligning SG buffers. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 11 +++ drivers/crypto/Makefile | 1 + drivers/crypto/omap-crypto.c | 184 +++++++++++++++++++++++++++++++++++ drivers/crypto/omap-crypto.h | 37 +++++++ 4 files changed, 233 insertions(+) create mode 100644 drivers/crypto/omap-crypto.c create mode 100644 drivers/crypto/omap-crypto.h diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index fb1e60f5002e..64171cad735d 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -314,6 +314,15 @@ config HW_RANDOM_PPC4XX This option provides the kernel-side support for the TRNG hardware found in the security function of some PowerPC 4xx SoCs. +config CRYPTO_DEV_OMAP + tristate "Support for OMAP crypto HW accelerators" + depends on ARCH_OMAP2PLUS + help + OMAP processors have various crypto HW accelerators. Select this if + you want to use the OMAP modules for any of the crypto algorithms. + +if CRYPTO_DEV_OMAP + config CRYPTO_DEV_OMAP_SHAM tristate "Support for OMAP MD5/SHA1/SHA2 hw accelerator" depends on ARCH_OMAP2PLUS @@ -351,6 +360,8 @@ config CRYPTO_DEV_OMAP_DES the ECB and CBC modes of operation are supported by the driver. Also accesses made on unaligned boundaries are supported. +endif # CRYPTO_DEV_OMAP + config CRYPTO_DEV_PICOXCELL tristate "Support for picoXcell IPSEC and Layer2 crypto engines" depends on (ARCH_PICOXCELL || COMPILE_TEST) && HAVE_CLK diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 463f33592d93..8fd11510f6b9 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_CRYPTO_DEV_MXC_SCC) += mxc-scc.o obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o n2_crypto-y := n2_core.o n2_asm.o obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ +obj-$(CONFIG_CRYPTO_DEV_OMAP) += omap-crypto.o obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes.o obj-$(CONFIG_CRYPTO_DEV_OMAP_DES) += omap-des.o obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o diff --git a/drivers/crypto/omap-crypto.c b/drivers/crypto/omap-crypto.c new file mode 100644 index 000000000000..23e37779317e --- /dev/null +++ b/drivers/crypto/omap-crypto.c @@ -0,0 +1,184 @@ +/* + * OMAP Crypto driver common support routines. + * + * Copyright (c) 2017 Texas Instruments Incorporated + * Tero Kristo + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#include +#include +#include +#include + +#include "omap-crypto.h" + +static int omap_crypto_copy_sg_lists(int total, int bs, + struct scatterlist **sg, + struct scatterlist *new_sg, u16 flags) +{ + int n = sg_nents(*sg); + struct scatterlist *tmp; + + if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) { + new_sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL); + if (!new_sg) + return -ENOMEM; + + sg_init_table(new_sg, n); + } + + tmp = new_sg; + + while (*sg && total) { + int len = (*sg)->length; + + if (total < len) + len = total; + + if (len > 0) { + total -= len; + sg_set_page(tmp, sg_page(*sg), len, (*sg)->offset); + if (total <= 0) + sg_mark_end(tmp); + tmp = sg_next(tmp); + } + + *sg = sg_next(*sg); + } + + *sg = new_sg; + + return 0; +} + +static int omap_crypto_copy_sgs(int total, int bs, struct scatterlist **sg, + struct scatterlist *new_sg, u16 flags) +{ + void *buf; + int pages; + int new_len; + + new_len = ALIGN(total, bs); + pages = get_order(new_len); + + buf = (void *)__get_free_pages(GFP_ATOMIC, pages); + if (!buf) { + pr_err("%s: Couldn't allocate pages for unaligned cases.\n", + __func__); + return -ENOMEM; + } + + if (flags & OMAP_CRYPTO_COPY_DATA) { + scatterwalk_map_and_copy(buf, *sg, 0, total, 0); + if (flags & OMAP_CRYPTO_ZERO_BUF) + memset(buf + total, 0, new_len - total); + } + + if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) + sg_init_table(new_sg, 1); + + sg_set_buf(new_sg, buf, new_len); + + *sg = new_sg; + + return 0; +} + +static int omap_crypto_check_sg(struct scatterlist *sg, int total, int bs, + u16 flags) +{ + int len = 0; + int num_sg = 0; + + if (!IS_ALIGNED(total, bs)) + return OMAP_CRYPTO_NOT_ALIGNED; + + while (sg) { + num_sg++; + + if (!IS_ALIGNED(sg->offset, 4)) + return OMAP_CRYPTO_NOT_ALIGNED; + if (!IS_ALIGNED(sg->length, bs)) + return OMAP_CRYPTO_NOT_ALIGNED; + + len += sg->length; + sg = sg_next(sg); + + if (len >= total) + break; + } + + if ((flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY) && num_sg > 1) + return OMAP_CRYPTO_NOT_ALIGNED; + + if (len != total) + return OMAP_CRYPTO_BAD_DATA_LENGTH; + + return 0; +} + +int omap_crypto_align_sg(struct scatterlist **sg, int total, int bs, + struct scatterlist *new_sg, u16 flags, + u8 flags_shift, unsigned long *dd_flags) +{ + int ret; + + *dd_flags &= ~(OMAP_CRYPTO_COPY_MASK << flags_shift); + + if (flags & OMAP_CRYPTO_FORCE_COPY) + ret = OMAP_CRYPTO_NOT_ALIGNED; + else + ret = omap_crypto_check_sg(*sg, total, bs, flags); + + if (ret == OMAP_CRYPTO_NOT_ALIGNED) { + ret = omap_crypto_copy_sgs(total, bs, sg, new_sg, flags); + if (ret) + return ret; + *dd_flags |= OMAP_CRYPTO_DATA_COPIED << flags_shift; + } else if (ret == OMAP_CRYPTO_BAD_DATA_LENGTH) { + ret = omap_crypto_copy_sg_lists(total, bs, sg, new_sg, flags); + if (ret) + return ret; + if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) + *dd_flags |= OMAP_CRYPTO_SG_COPIED << flags_shift; + } else if (flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY) { + sg_set_buf(new_sg, sg_virt(*sg), (*sg)->length); + } + + return 0; +} +EXPORT_SYMBOL_GPL(omap_crypto_align_sg); + +void omap_crypto_cleanup(struct scatterlist *sg, struct scatterlist *orig, + int offset, int len, u8 flags_shift, + unsigned long flags) +{ + void *buf; + int pages; + + flags >>= flags_shift; + flags &= OMAP_CRYPTO_COPY_MASK; + + if (!flags) + return; + + buf = sg_virt(sg); + pages = get_order(len); + + if (orig && (flags & OMAP_CRYPTO_COPY_MASK)) + scatterwalk_map_and_copy(buf, orig, offset, len, 1); + + if (flags & OMAP_CRYPTO_DATA_COPIED) + free_pages((unsigned long)buf, pages); + else if (flags & OMAP_CRYPTO_SG_COPIED) + kfree(sg); +} +EXPORT_SYMBOL_GPL(omap_crypto_cleanup); + +MODULE_DESCRIPTION("OMAP crypto support library."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Tero Kristo "); diff --git a/drivers/crypto/omap-crypto.h b/drivers/crypto/omap-crypto.h new file mode 100644 index 000000000000..36a230eb87af --- /dev/null +++ b/drivers/crypto/omap-crypto.h @@ -0,0 +1,37 @@ +/* + * OMAP Crypto driver common support routines. + * + * Copyright (c) 2017 Texas Instruments Incorporated + * Tero Kristo + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#ifndef __CRYPTO_OMAP_CRYPTO_H +#define __CRYPTO_OMAP_CRYPTO_H + +enum { + OMAP_CRYPTO_NOT_ALIGNED = 1, + OMAP_CRYPTO_BAD_DATA_LENGTH, +}; + +#define OMAP_CRYPTO_DATA_COPIED BIT(0) +#define OMAP_CRYPTO_SG_COPIED BIT(1) + +#define OMAP_CRYPTO_COPY_MASK 0x3 + +#define OMAP_CRYPTO_COPY_DATA BIT(0) +#define OMAP_CRYPTO_FORCE_COPY BIT(1) +#define OMAP_CRYPTO_ZERO_BUF BIT(2) +#define OMAP_CRYPTO_FORCE_SINGLE_ENTRY BIT(3) + +int omap_crypto_align_sg(struct scatterlist **sg, int total, int bs, + struct scatterlist *new_sg, u16 flags, + u8 flags_shift, unsigned long *dd_flags); +void omap_crypto_cleanup(struct scatterlist *sg, struct scatterlist *orig, + int offset, int len, u8 flags_shift, + unsigned long flags); + +#endif From 9765e768613c6b11ab482df938282e097710e6aa Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:27 +0300 Subject: [PATCH 042/140] crypto: omap-des - use base omap crypto support library Use the SG alignment APIs from the OMAP crypto support library instead of using own implementations. This reduces the amount of copy-paste code. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-des.c | 112 ++++++++++---------------------------- 1 file changed, 28 insertions(+), 84 deletions(-) diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 3783dcf6bc37..0bcab00e0ff5 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -41,6 +41,8 @@ #include #include +#include "omap-crypto.h" + #define DST_MAXBURST 2 #define DES_BLOCK_WORDS (DES_BLOCK_SIZE >> 2) @@ -80,6 +82,9 @@ #define DEFAULT_AUTOSUSPEND_DELAY 1000 +#define FLAGS_IN_DATA_ST_SHIFT 8 +#define FLAGS_OUT_DATA_ST_SHIFT 10 + struct omap_des_ctx { struct omap_des_dev *dd; @@ -153,7 +158,6 @@ struct omap_des_dev { struct scatterlist in_sgl; struct scatterlist out_sgl; struct scatterlist *orig_out; - int sgs_copied; struct scatter_walk in_walk; struct scatter_walk out_walk; @@ -372,20 +376,6 @@ static void omap_des_dma_cleanup(struct omap_des_dev *dd) dma_release_channel(dd->dma_lch_in); } -static void sg_copy_buf(void *buf, struct scatterlist *sg, - unsigned int start, unsigned int nbytes, int out) -{ - struct scatter_walk walk; - - if (!nbytes) - return; - - scatterwalk_start(&walk, sg); - scatterwalk_advance(&walk, start); - scatterwalk_copychunks(buf, &walk, nbytes, out); - scatterwalk_done(&walk, out, 0); -} - static int omap_des_crypt_dma(struct crypto_tfm *tfm, struct scatterlist *in_sg, struct scatterlist *out_sg, int in_sg_len, int out_sg_len) @@ -526,55 +516,6 @@ static int omap_des_crypt_dma_stop(struct omap_des_dev *dd) return 0; } -static int omap_des_copy_needed(struct scatterlist *sg) -{ - while (sg) { - if (!IS_ALIGNED(sg->offset, 4)) - return -1; - if (!IS_ALIGNED(sg->length, DES_BLOCK_SIZE)) - return -1; - sg = sg_next(sg); - } - return 0; -} - -static int omap_des_copy_sgs(struct omap_des_dev *dd) -{ - void *buf_in, *buf_out; - int pages; - - pages = dd->total >> PAGE_SHIFT; - - if (dd->total & (PAGE_SIZE-1)) - pages++; - - BUG_ON(!pages); - - buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages); - buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages); - - if (!buf_in || !buf_out) { - pr_err("Couldn't allocated pages for unaligned cases.\n"); - return -1; - } - - dd->orig_out = dd->out_sg; - - sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0); - - sg_init_table(&dd->in_sgl, 1); - sg_set_buf(&dd->in_sgl, buf_in, dd->total); - dd->in_sg = &dd->in_sgl; - dd->in_sg_len = 1; - - sg_init_table(&dd->out_sgl, 1); - sg_set_buf(&dd->out_sgl, buf_out, dd->total); - dd->out_sg = &dd->out_sgl; - dd->out_sg_len = 1; - - return 0; -} - static int omap_des_handle_queue(struct omap_des_dev *dd, struct ablkcipher_request *req) { @@ -591,6 +532,8 @@ static int omap_des_prepare_req(struct crypto_engine *engine, crypto_ablkcipher_reqtfm(req)); struct omap_des_dev *dd = omap_des_find_dev(ctx); struct omap_des_reqctx *rctx; + int ret; + u16 flags; if (!dd) return -ENODEV; @@ -601,6 +544,23 @@ static int omap_des_prepare_req(struct crypto_engine *engine, dd->total_save = req->nbytes; dd->in_sg = req->src; dd->out_sg = req->dst; + dd->orig_out = req->dst; + + flags = OMAP_CRYPTO_COPY_DATA; + if (req->src == req->dst) + flags |= OMAP_CRYPTO_FORCE_COPY; + + ret = omap_crypto_align_sg(&dd->in_sg, dd->total, DES_BLOCK_SIZE, + &dd->in_sgl, flags, + FLAGS_IN_DATA_ST_SHIFT, &dd->flags); + if (ret) + return ret; + + ret = omap_crypto_align_sg(&dd->out_sg, dd->total, DES_BLOCK_SIZE, + &dd->out_sgl, 0, + FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); + if (ret) + return ret; dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total); if (dd->in_sg_len < 0) @@ -610,15 +570,6 @@ static int omap_des_prepare_req(struct crypto_engine *engine, if (dd->out_sg_len < 0) return dd->out_sg_len; - if (omap_des_copy_needed(dd->in_sg) || - omap_des_copy_needed(dd->out_sg)) { - if (omap_des_copy_sgs(dd)) - pr_err("Failed to copy SGs for unaligned cases\n"); - dd->sgs_copied = 1; - } else { - dd->sgs_copied = 0; - } - rctx = ablkcipher_request_ctx(req); ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); rctx->mode &= FLAGS_MODE_MASK; @@ -646,8 +597,6 @@ static int omap_des_crypt_req(struct crypto_engine *engine, static void omap_des_done_task(unsigned long data) { struct omap_des_dev *dd = (struct omap_des_dev *)data; - void *buf_in, *buf_out; - int pages; pr_debug("enter done_task\n"); @@ -660,16 +609,11 @@ static void omap_des_done_task(unsigned long data) omap_des_crypt_dma_stop(dd); } - if (dd->sgs_copied) { - buf_in = sg_virt(&dd->in_sgl); - buf_out = sg_virt(&dd->out_sgl); + omap_crypto_cleanup(&dd->in_sgl, NULL, 0, dd->total_save, + FLAGS_IN_DATA_ST_SHIFT, dd->flags); - sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1); - - pages = get_order(dd->total_save); - free_pages((unsigned long)buf_in, pages); - free_pages((unsigned long)buf_out, pages); - } + omap_crypto_cleanup(&dd->out_sgl, dd->orig_out, 0, dd->total_save, + FLAGS_OUT_DATA_ST_SHIFT, dd->flags); omap_des_finish_req(dd, 0); From afc2dc1336977c07e3aed368d128e6986d20c9a8 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:28 +0300 Subject: [PATCH 043/140] crypto: omap-aes - use base omap crypto support library Use the SG alignment APIs from the OMAP crypto support library instead of using own implementations. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 120 +++++++++----------------------------- 1 file changed, 28 insertions(+), 92 deletions(-) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 379d70157ae7..e60d2973321d 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -38,6 +38,8 @@ #include #include +#include "omap-crypto.h" + #define DST_MAXBURST 4 #define DMA_MIN (DST_MAXBURST * sizeof(u32)) @@ -98,6 +100,9 @@ #define FLAGS_FAST BIT(5) #define FLAGS_BUSY BIT(6) +#define FLAGS_IN_DATA_ST_SHIFT 8 +#define FLAGS_OUT_DATA_ST_SHIFT 10 + #define AES_BLOCK_WORDS (AES_BLOCK_SIZE >> 2) struct omap_aes_ctx { @@ -173,7 +178,6 @@ struct omap_aes_dev { struct scatterlist in_sgl; struct scatterlist out_sgl; struct scatterlist *orig_out; - int sgs_copied; struct scatter_walk in_walk; struct scatter_walk out_walk; @@ -385,20 +389,6 @@ static void omap_aes_dma_cleanup(struct omap_aes_dev *dd) dma_release_channel(dd->dma_lch_in); } -static void sg_copy_buf(void *buf, struct scatterlist *sg, - unsigned int start, unsigned int nbytes, int out) -{ - struct scatter_walk walk; - - if (!nbytes) - return; - - scatterwalk_start(&walk, sg); - scatterwalk_advance(&walk, start); - scatterwalk_copychunks(buf, &walk, nbytes, out); - scatterwalk_done(&walk, out, 0); -} - static int omap_aes_crypt_dma(struct omap_aes_dev *dd, struct scatterlist *in_sg, struct scatterlist *out_sg, @@ -534,62 +524,6 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) return 0; } -static int omap_aes_check_aligned(struct scatterlist *sg, int total) -{ - int len = 0; - - if (!IS_ALIGNED(total, AES_BLOCK_SIZE)) - return -EINVAL; - - while (sg) { - if (!IS_ALIGNED(sg->offset, 4)) - return -1; - if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE)) - return -1; - - len += sg->length; - sg = sg_next(sg); - } - - if (len != total) - return -1; - - return 0; -} - -static int omap_aes_copy_sgs(struct omap_aes_dev *dd) -{ - void *buf_in, *buf_out; - int pages, total; - - total = ALIGN(dd->total, AES_BLOCK_SIZE); - pages = get_order(total); - - buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages); - buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages); - - if (!buf_in || !buf_out) { - pr_err("Couldn't allocated pages for unaligned cases.\n"); - return -1; - } - - dd->orig_out = dd->out_sg; - - sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0); - - sg_init_table(&dd->in_sgl, 1); - sg_set_buf(&dd->in_sgl, buf_in, total); - dd->in_sg = &dd->in_sgl; - dd->in_sg_len = 1; - - sg_init_table(&dd->out_sgl, 1); - sg_set_buf(&dd->out_sgl, buf_out, total); - dd->out_sg = &dd->out_sgl; - dd->out_sg_len = 1; - - return 0; -} - static int omap_aes_handle_queue(struct omap_aes_dev *dd, struct ablkcipher_request *req) { @@ -606,6 +540,8 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, crypto_ablkcipher_reqtfm(req)); struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req); struct omap_aes_dev *dd = rctx->dd; + int ret; + u16 flags; if (!dd) return -ENODEV; @@ -616,6 +552,23 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, dd->total_save = req->nbytes; dd->in_sg = req->src; dd->out_sg = req->dst; + dd->orig_out = req->dst; + + flags = OMAP_CRYPTO_COPY_DATA; + if (req->src == req->dst) + flags |= OMAP_CRYPTO_FORCE_COPY; + + ret = omap_crypto_align_sg(&dd->in_sg, dd->total, AES_BLOCK_SIZE, + &dd->in_sgl, flags, + FLAGS_IN_DATA_ST_SHIFT, &dd->flags); + if (ret) + return ret; + + ret = omap_crypto_align_sg(&dd->out_sg, dd->total, AES_BLOCK_SIZE, + &dd->out_sgl, 0, + FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); + if (ret) + return ret; dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total); if (dd->in_sg_len < 0) @@ -625,15 +578,6 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, if (dd->out_sg_len < 0) return dd->out_sg_len; - if (omap_aes_check_aligned(dd->in_sg, dd->total) || - omap_aes_check_aligned(dd->out_sg, dd->total)) { - if (omap_aes_copy_sgs(dd)) - pr_err("Failed to copy SGs for unaligned cases\n"); - dd->sgs_copied = 1; - } else { - dd->sgs_copied = 0; - } - rctx->mode &= FLAGS_MODE_MASK; dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; @@ -658,8 +602,6 @@ static int omap_aes_crypt_req(struct crypto_engine *engine, static void omap_aes_done_task(unsigned long data) { struct omap_aes_dev *dd = (struct omap_aes_dev *)data; - void *buf_in, *buf_out; - int pages, len; pr_debug("enter done_task\n"); @@ -672,17 +614,11 @@ static void omap_aes_done_task(unsigned long data) omap_aes_crypt_dma_stop(dd); } - if (dd->sgs_copied) { - buf_in = sg_virt(&dd->in_sgl); - buf_out = sg_virt(&dd->out_sgl); + omap_crypto_cleanup(&dd->in_sgl, NULL, 0, dd->total_save, + FLAGS_IN_DATA_ST_SHIFT, dd->flags); - sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1); - - len = ALIGN(dd->total_save, AES_BLOCK_SIZE); - pages = get_order(len); - free_pages((unsigned long)buf_in, pages); - free_pages((unsigned long)buf_out, pages); - } + omap_crypto_cleanup(&dd->out_sgl, dd->orig_out, 0, dd->total_save, + FLAGS_OUT_DATA_ST_SHIFT, dd->flags); omap_aes_finish_req(dd, 0); From 5b3d4d2e613ba7274f6cc11373a5919faafdda28 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:29 +0300 Subject: [PATCH 044/140] crypto: omap-aes - move definitions over to a separate header file Move over most of the omap-aes driver internal definitions to a separate header file. This is done so that the same definitions can be used in the upcoming AES-GCM support code. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 150 +--------------------------------- drivers/crypto/omap-aes.h | 167 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 149 deletions(-) create mode 100644 drivers/crypto/omap-aes.h diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index e60d2973321d..cf7968c187ef 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -39,155 +39,7 @@ #include #include "omap-crypto.h" - -#define DST_MAXBURST 4 -#define DMA_MIN (DST_MAXBURST * sizeof(u32)) - -#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset) - -/* OMAP TRM gives bitfields as start:end, where start is the higher bit - number. For example 7:0 */ -#define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end)) -#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end)) - -#define AES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \ - ((x ^ 0x01) * 0x04)) -#define AES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04)) - -#define AES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs) -#define AES_REG_CTRL_CTR_WIDTH_MASK GENMASK(8, 7) -#define AES_REG_CTRL_CTR_WIDTH_32 0 -#define AES_REG_CTRL_CTR_WIDTH_64 BIT(7) -#define AES_REG_CTRL_CTR_WIDTH_96 BIT(8) -#define AES_REG_CTRL_CTR_WIDTH_128 GENMASK(8, 7) -#define AES_REG_CTRL_CTR BIT(6) -#define AES_REG_CTRL_CBC BIT(5) -#define AES_REG_CTRL_KEY_SIZE GENMASK(4, 3) -#define AES_REG_CTRL_DIRECTION BIT(2) -#define AES_REG_CTRL_INPUT_READY BIT(1) -#define AES_REG_CTRL_OUTPUT_READY BIT(0) -#define AES_REG_CTRL_MASK GENMASK(24, 2) - -#define AES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04)) - -#define AES_REG_REV(dd) ((dd)->pdata->rev_ofs) - -#define AES_REG_MASK(dd) ((dd)->pdata->mask_ofs) -#define AES_REG_MASK_SIDLE BIT(6) -#define AES_REG_MASK_START BIT(5) -#define AES_REG_MASK_DMA_OUT_EN BIT(3) -#define AES_REG_MASK_DMA_IN_EN BIT(2) -#define AES_REG_MASK_SOFTRESET BIT(1) -#define AES_REG_AUTOIDLE BIT(0) - -#define AES_REG_LENGTH_N(x) (0x54 + ((x) * 0x04)) - -#define AES_REG_IRQ_STATUS(dd) ((dd)->pdata->irq_status_ofs) -#define AES_REG_IRQ_ENABLE(dd) ((dd)->pdata->irq_enable_ofs) -#define AES_REG_IRQ_DATA_IN BIT(1) -#define AES_REG_IRQ_DATA_OUT BIT(2) -#define DEFAULT_TIMEOUT (5*HZ) - -#define DEFAULT_AUTOSUSPEND_DELAY 1000 - -#define FLAGS_MODE_MASK 0x000f -#define FLAGS_ENCRYPT BIT(0) -#define FLAGS_CBC BIT(1) -#define FLAGS_GIV BIT(2) -#define FLAGS_CTR BIT(3) - -#define FLAGS_INIT BIT(4) -#define FLAGS_FAST BIT(5) -#define FLAGS_BUSY BIT(6) - -#define FLAGS_IN_DATA_ST_SHIFT 8 -#define FLAGS_OUT_DATA_ST_SHIFT 10 - -#define AES_BLOCK_WORDS (AES_BLOCK_SIZE >> 2) - -struct omap_aes_ctx { - int keylen; - u32 key[AES_KEYSIZE_256 / sizeof(u32)]; - struct crypto_skcipher *fallback; -}; - -struct omap_aes_reqctx { - struct omap_aes_dev *dd; - unsigned long mode; -}; - -#define OMAP_AES_QUEUE_LENGTH 1 -#define OMAP_AES_CACHE_SIZE 0 - -struct omap_aes_algs_info { - struct crypto_alg *algs_list; - unsigned int size; - unsigned int registered; -}; - -struct omap_aes_pdata { - struct omap_aes_algs_info *algs_info; - unsigned int algs_info_size; - - void (*trigger)(struct omap_aes_dev *dd, int length); - - u32 key_ofs; - u32 iv_ofs; - u32 ctrl_ofs; - u32 data_ofs; - u32 rev_ofs; - u32 mask_ofs; - u32 irq_enable_ofs; - u32 irq_status_ofs; - - u32 dma_enable_in; - u32 dma_enable_out; - u32 dma_start; - - u32 major_mask; - u32 major_shift; - u32 minor_mask; - u32 minor_shift; -}; - -struct omap_aes_dev { - struct list_head list; - unsigned long phys_base; - void __iomem *io_base; - struct omap_aes_ctx *ctx; - struct device *dev; - unsigned long flags; - int err; - - struct tasklet_struct done_task; - - struct ablkcipher_request *req; - struct crypto_engine *engine; - - /* - * total is used by PIO mode for book keeping so introduce - * variable total_save as need it to calc page_order - */ - size_t total; - size_t total_save; - - struct scatterlist *in_sg; - struct scatterlist *out_sg; - - /* Buffers for copying for unaligned cases */ - struct scatterlist in_sgl; - struct scatterlist out_sgl; - struct scatterlist *orig_out; - - struct scatter_walk in_walk; - struct scatter_walk out_walk; - struct dma_chan *dma_lch_in; - struct dma_chan *dma_lch_out; - int in_sg_len; - int out_sg_len; - int pio_only; - const struct omap_aes_pdata *pdata; -}; +#include "omap-aes.h" /* keep registered devices data here */ static LIST_HEAD(dev_list); diff --git a/drivers/crypto/omap-aes.h b/drivers/crypto/omap-aes.h new file mode 100644 index 000000000000..ce4346e43664 --- /dev/null +++ b/drivers/crypto/omap-aes.h @@ -0,0 +1,167 @@ +/* + * Cryptographic API. + * + * Support for OMAP AES HW ACCELERATOR defines + * + * Copyright (c) 2015 Texas Instruments Incorporated + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + */ +#ifndef __OMAP_AES_H__ +#define __OMAP_AES_H__ + +#define DST_MAXBURST 4 +#define DMA_MIN (DST_MAXBURST * sizeof(u32)) + +#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset) + +/* + * OMAP TRM gives bitfields as start:end, where start is the higher bit + * number. For example 7:0 + */ +#define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end)) +#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end)) + +#define AES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \ + (((x) ^ 0x01) * 0x04)) +#define AES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04)) + +#define AES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs) +#define AES_REG_CTRL_CTR_WIDTH_MASK GENMASK(8, 7) +#define AES_REG_CTRL_CTR_WIDTH_32 0 +#define AES_REG_CTRL_CTR_WIDTH_64 BIT(7) +#define AES_REG_CTRL_CTR_WIDTH_96 BIT(8) +#define AES_REG_CTRL_CTR_WIDTH_128 GENMASK(8, 7) +#define AES_REG_CTRL_CTR BIT(6) +#define AES_REG_CTRL_CBC BIT(5) +#define AES_REG_CTRL_KEY_SIZE GENMASK(4, 3) +#define AES_REG_CTRL_DIRECTION BIT(2) +#define AES_REG_CTRL_INPUT_READY BIT(1) +#define AES_REG_CTRL_OUTPUT_READY BIT(0) +#define AES_REG_CTRL_MASK GENMASK(24, 2) + +#define AES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04)) + +#define AES_REG_REV(dd) ((dd)->pdata->rev_ofs) + +#define AES_REG_MASK(dd) ((dd)->pdata->mask_ofs) +#define AES_REG_MASK_SIDLE BIT(6) +#define AES_REG_MASK_START BIT(5) +#define AES_REG_MASK_DMA_OUT_EN BIT(3) +#define AES_REG_MASK_DMA_IN_EN BIT(2) +#define AES_REG_MASK_SOFTRESET BIT(1) +#define AES_REG_AUTOIDLE BIT(0) + +#define AES_REG_LENGTH_N(x) (0x54 + ((x) * 0x04)) + +#define AES_REG_IRQ_STATUS(dd) ((dd)->pdata->irq_status_ofs) +#define AES_REG_IRQ_ENABLE(dd) ((dd)->pdata->irq_enable_ofs) +#define AES_REG_IRQ_DATA_IN BIT(1) +#define AES_REG_IRQ_DATA_OUT BIT(2) +#define DEFAULT_TIMEOUT (5 * HZ) + +#define DEFAULT_AUTOSUSPEND_DELAY 1000 + +#define FLAGS_MODE_MASK 0x000f +#define FLAGS_ENCRYPT BIT(0) +#define FLAGS_CBC BIT(1) +#define FLAGS_GIV BIT(2) +#define FLAGS_CTR BIT(3) + +#define FLAGS_INIT BIT(4) +#define FLAGS_FAST BIT(5) +#define FLAGS_BUSY BIT(6) + +#define FLAGS_IN_DATA_ST_SHIFT 8 +#define FLAGS_OUT_DATA_ST_SHIFT 10 + +#define AES_BLOCK_WORDS (AES_BLOCK_SIZE >> 2) + +struct omap_aes_ctx { + int keylen; + u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + struct crypto_skcipher *fallback; +}; + +struct omap_aes_reqctx { + struct omap_aes_dev *dd; + unsigned long mode; +}; + +#define OMAP_AES_QUEUE_LENGTH 1 +#define OMAP_AES_CACHE_SIZE 0 + +struct omap_aes_algs_info { + struct crypto_alg *algs_list; + unsigned int size; + unsigned int registered; +}; + +struct omap_aes_pdata { + struct omap_aes_algs_info *algs_info; + unsigned int algs_info_size; + + void (*trigger)(struct omap_aes_dev *dd, int length); + + u32 key_ofs; + u32 iv_ofs; + u32 ctrl_ofs; + u32 data_ofs; + u32 rev_ofs; + u32 mask_ofs; + u32 irq_enable_ofs; + u32 irq_status_ofs; + + u32 dma_enable_in; + u32 dma_enable_out; + u32 dma_start; + + u32 major_mask; + u32 major_shift; + u32 minor_mask; + u32 minor_shift; +}; + +struct omap_aes_dev { + struct list_head list; + unsigned long phys_base; + void __iomem *io_base; + struct omap_aes_ctx *ctx; + struct device *dev; + unsigned long flags; + int err; + + struct tasklet_struct done_task; + + struct ablkcipher_request *req; + struct crypto_engine *engine; + + /* + * total is used by PIO mode for book keeping so introduce + * variable total_save as need it to calc page_order + */ + size_t total; + size_t total_save; + + struct scatterlist *in_sg; + struct scatterlist *out_sg; + + /* Buffers for copying for unaligned cases */ + struct scatterlist in_sgl; + struct scatterlist out_sgl; + struct scatterlist *orig_out; + + struct scatter_walk in_walk; + struct scatter_walk out_walk; + struct dma_chan *dma_lch_in; + struct dma_chan *dma_lch_out; + int in_sg_len; + int out_sg_len; + int pio_only; + const struct omap_aes_pdata *pdata; +}; + +#endif From d695bfd6f02adf362511a6d0139e7f7e6342d3e8 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:30 +0300 Subject: [PATCH 045/140] crypto: omap-aes - export some AES driver functionality locally These are going to be required by the addition of the GCM support. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 12 ++++++------ drivers/crypto/omap-aes.h | 7 +++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index cf7968c187ef..bf3b27d9dc33 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -55,7 +55,7 @@ static DEFINE_SPINLOCK(list_lock); _read_ret; \ }) #else -static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset) +inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset) { return __raw_readl(dd->io_base + offset); } @@ -69,7 +69,7 @@ static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset) __raw_writel(value, dd->io_base + offset); \ } while (0) #else -static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset, +inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset, u32 value) { __raw_writel(value, dd->io_base + offset); @@ -112,7 +112,7 @@ static int omap_aes_hw_init(struct omap_aes_dev *dd) return 0; } -static int omap_aes_write_ctrl(struct omap_aes_dev *dd) +int omap_aes_write_ctrl(struct omap_aes_dev *dd) { unsigned int key32; int i, err; @@ -183,7 +183,7 @@ static void omap_aes_dma_stop(struct omap_aes_dev *dd) omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask); } -static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx) +struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx) { struct omap_aes_dev *dd; @@ -321,7 +321,7 @@ static int omap_aes_crypt_dma(struct omap_aes_dev *dd, return 0; } -static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd) +int omap_aes_crypt_dma_start(struct omap_aes_dev *dd) { int err; @@ -366,7 +366,7 @@ static void omap_aes_finish_req(struct omap_aes_dev *dd, int err) pm_runtime_put_autosuspend(dd->dev); } -static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) +int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) { pr_debug("total: %d\n", dd->total); diff --git a/drivers/crypto/omap-aes.h b/drivers/crypto/omap-aes.h index ce4346e43664..11e1784cc4a5 100644 --- a/drivers/crypto/omap-aes.h +++ b/drivers/crypto/omap-aes.h @@ -164,4 +164,11 @@ struct omap_aes_dev { const struct omap_aes_pdata *pdata; }; +u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset); +void omap_aes_write(struct omap_aes_dev *dd, u32 offset, u32 value); +struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx); +int omap_aes_write_ctrl(struct omap_aes_dev *dd); +int omap_aes_crypt_dma_start(struct omap_aes_dev *dd); +int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd); + #endif From ad18cc9d0f911928704cdc37f4d126853daa9e4e Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:31 +0300 Subject: [PATCH 046/140] crypto: omap-aes - Add support for GCM mode OMAP AES hw supports AES-GCM mode. This patch adds support for GCM and RFC4106 GCM mode in omap-aes driver. The GCM implementation is mostly written into its own source file, which gets built into the same driver binary as the existing AES support. Signed-off-by: Lokesh Vutla [t-kristo@ti.com: forward port to latest upstream kernel, conversion to use omap-crypto lib and some additional fixes] Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 1 + drivers/crypto/Makefile | 3 +- drivers/crypto/omap-aes-gcm.c | 408 ++++++++++++++++++++++++++++++++++ drivers/crypto/omap-aes.c | 159 ++++++++++++- drivers/crypto/omap-aes.h | 54 ++++- 5 files changed, 612 insertions(+), 13 deletions(-) create mode 100644 drivers/crypto/omap-aes-gcm.c diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 64171cad735d..4e6591b0b128 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -344,6 +344,7 @@ config CRYPTO_DEV_OMAP_AES select CRYPTO_CBC select CRYPTO_ECB select CRYPTO_CTR + select CRYPTO_AEAD help OMAP processors have AES module accelerator. Select this if you want to use the OMAP module for AES algorithms. diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 8fd11510f6b9..8177388f5c85 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -21,7 +21,8 @@ obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o n2_crypto-y := n2_core.o n2_asm.o obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ obj-$(CONFIG_CRYPTO_DEV_OMAP) += omap-crypto.o -obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes.o +obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes-driver.o +omap-aes-driver-objs := omap-aes.o omap-aes-gcm.o obj-$(CONFIG_CRYPTO_DEV_OMAP_DES) += omap-des.o obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c new file mode 100644 index 000000000000..521a310ea699 --- /dev/null +++ b/drivers/crypto/omap-aes-gcm.c @@ -0,0 +1,408 @@ +/* + * Cryptographic API. + * + * Support for OMAP AES GCM HW acceleration. + * + * Copyright (c) 2016 Texas Instruments Incorporated + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "omap-crypto.h" +#include "omap-aes.h" + +static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, + struct aead_request *req); + +static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret) +{ + struct aead_request *req = dd->aead_req; + + dd->flags &= ~FLAGS_BUSY; + dd->in_sg = NULL; + dd->out_sg = NULL; + + req->base.complete(&req->base, ret); +} + +static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) +{ + u8 *tag; + int alen, clen, i, ret = 0, nsg; + struct omap_aes_reqctx *rctx; + + alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE); + clen = ALIGN(dd->total, AES_BLOCK_SIZE); + rctx = aead_request_ctx(dd->aead_req); + + nsg = !!(dd->assoc_len && dd->total); + + dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); + omap_aes_crypt_dma_stop(dd); + + omap_crypto_cleanup(dd->out_sg, dd->orig_out, + dd->aead_req->assoclen, dd->total, + FLAGS_OUT_DATA_ST_SHIFT, dd->flags); + + if (dd->flags & FLAGS_ENCRYPT) + scatterwalk_map_and_copy(rctx->auth_tag, + dd->aead_req->dst, + dd->total + dd->aead_req->assoclen, + dd->authsize, 1); + + omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen, + FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags); + + omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen, + FLAGS_IN_DATA_ST_SHIFT, dd->flags); + + if (!(dd->flags & FLAGS_ENCRYPT)) { + tag = (u8 *)rctx->auth_tag; + for (i = 0; i < dd->authsize; i++) { + if (tag[i]) { + dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n"); + ret = -EBADMSG; + } + } + } + + omap_aes_gcm_finish_req(dd, ret); + omap_aes_gcm_handle_queue(dd, NULL); +} + +static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd, + struct aead_request *req) +{ + int alen, clen, cryptlen, assoclen, ret; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + unsigned int authlen = crypto_aead_authsize(aead); + struct scatterlist *tmp, sg_arr[2]; + int nsg; + u16 flags; + + assoclen = req->assoclen; + cryptlen = req->cryptlen; + + if (dd->flags & FLAGS_RFC4106_GCM) + assoclen -= 8; + + if (!(dd->flags & FLAGS_ENCRYPT)) + cryptlen -= authlen; + + alen = ALIGN(assoclen, AES_BLOCK_SIZE); + clen = ALIGN(cryptlen, AES_BLOCK_SIZE); + + nsg = !!(assoclen && cryptlen); + + omap_aes_clear_copy_flags(dd); + + sg_init_table(dd->in_sgl, nsg + 1); + if (assoclen) { + tmp = req->src; + ret = omap_crypto_align_sg(&tmp, assoclen, + AES_BLOCK_SIZE, dd->in_sgl, + OMAP_CRYPTO_COPY_DATA | + OMAP_CRYPTO_ZERO_BUF | + OMAP_CRYPTO_FORCE_SINGLE_ENTRY, + FLAGS_ASSOC_DATA_ST_SHIFT, + &dd->flags); + } + + if (cryptlen) { + tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen); + + ret = omap_crypto_align_sg(&tmp, cryptlen, + AES_BLOCK_SIZE, &dd->in_sgl[nsg], + OMAP_CRYPTO_COPY_DATA | + OMAP_CRYPTO_ZERO_BUF | + OMAP_CRYPTO_FORCE_SINGLE_ENTRY, + FLAGS_IN_DATA_ST_SHIFT, + &dd->flags); + } + + dd->in_sg = dd->in_sgl; + dd->total = cryptlen; + dd->assoc_len = assoclen; + dd->authsize = authlen; + + dd->out_sg = req->dst; + dd->orig_out = req->dst; + + dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen); + + flags = 0; + if (req->src == req->dst || dd->out_sg == sg_arr) + flags |= OMAP_CRYPTO_FORCE_COPY; + + ret = omap_crypto_align_sg(&dd->out_sg, cryptlen, + AES_BLOCK_SIZE, &dd->out_sgl, + flags, + FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); + if (ret) + return ret; + + dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen); + dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen); + + return 0; +} + +static void omap_aes_gcm_complete(struct crypto_async_request *req, int err) +{ + struct omap_aes_gcm_result *res = req->data; + + if (err == -EINPROGRESS) + return; + + res->err = err; + complete(&res->completion); +} + +static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) +{ + struct scatterlist iv_sg, tag_sg; + struct skcipher_request *sk_req; + struct omap_aes_gcm_result result; + struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + int ret = 0; + + sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL); + if (!sk_req) { + pr_err("skcipher: Failed to allocate request\n"); + return -1; + } + + init_completion(&result.completion); + + sg_init_one(&iv_sg, iv, AES_BLOCK_SIZE); + sg_init_one(&tag_sg, tag, AES_BLOCK_SIZE); + skcipher_request_set_callback(sk_req, CRYPTO_TFM_REQ_MAY_BACKLOG, + omap_aes_gcm_complete, &result); + ret = crypto_skcipher_setkey(ctx->ctr, (u8 *)ctx->key, ctx->keylen); + skcipher_request_set_crypt(sk_req, &iv_sg, &tag_sg, AES_BLOCK_SIZE, + NULL); + ret = crypto_skcipher_encrypt(sk_req); + switch (ret) { + case 0: + break; + case -EINPROGRESS: + case -EBUSY: + ret = wait_for_completion_interruptible(&result.completion); + if (!ret) { + ret = result.err; + if (!ret) { + reinit_completion(&result.completion); + break; + } + } + /* fall through */ + default: + pr_err("Encryptio of IV failed for GCM mode"); + break; + } + + skcipher_request_free(sk_req); + return ret; +} + +void omap_aes_gcm_dma_out_callback(void *data) +{ + struct omap_aes_dev *dd = data; + struct omap_aes_reqctx *rctx; + int i, val; + u32 *auth_tag, tag[4]; + + if (!(dd->flags & FLAGS_ENCRYPT)) + scatterwalk_map_and_copy(tag, dd->aead_req->src, + dd->total + dd->aead_req->assoclen, + dd->authsize, 0); + + rctx = aead_request_ctx(dd->aead_req); + auth_tag = (u32 *)rctx->auth_tag; + for (i = 0; i < 4; i++) { + val = omap_aes_read(dd, AES_REG_TAG_N(dd, i)); + auth_tag[i] = val ^ auth_tag[i]; + if (!(dd->flags & FLAGS_ENCRYPT)) + auth_tag[i] = auth_tag[i] ^ tag[i]; + } + + omap_aes_gcm_done_task(dd); +} + +static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, + struct aead_request *req) +{ + struct omap_aes_ctx *ctx; + struct aead_request *backlog; + struct omap_aes_reqctx *rctx; + unsigned long flags; + int err, ret = 0; + + spin_lock_irqsave(&dd->lock, flags); + if (req) + ret = aead_enqueue_request(&dd->aead_queue, req); + if (dd->flags & FLAGS_BUSY) { + spin_unlock_irqrestore(&dd->lock, flags); + return ret; + } + + backlog = aead_get_backlog(&dd->aead_queue); + req = aead_dequeue_request(&dd->aead_queue); + if (req) + dd->flags |= FLAGS_BUSY; + spin_unlock_irqrestore(&dd->lock, flags); + + if (!req) + return ret; + + if (backlog) + backlog->base.complete(&backlog->base, -EINPROGRESS); + + ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + rctx = aead_request_ctx(req); + + dd->ctx = ctx; + rctx->dd = dd; + dd->aead_req = req; + + rctx->mode &= FLAGS_MODE_MASK; + dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; + + err = omap_aes_gcm_copy_buffers(dd, req); + if (err) + return err; + + err = omap_aes_write_ctrl(dd); + if (!err) + err = omap_aes_crypt_dma_start(dd); + + if (err) { + omap_aes_gcm_finish_req(dd, err); + omap_aes_gcm_handle_queue(dd, NULL); + } + + return ret; +} + +static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) +{ + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + unsigned int authlen = crypto_aead_authsize(aead); + struct omap_aes_dev *dd; + __be32 counter = cpu_to_be32(1); + int err, assoclen; + + memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); + memcpy(rctx->iv + 12, &counter, 4); + + err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv); + if (err) + return err; + + if (mode & FLAGS_RFC4106_GCM) + assoclen = req->assoclen - 8; + else + assoclen = req->assoclen; + if (assoclen + req->cryptlen == 0) { + scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen, + 1); + return 0; + } + + dd = omap_aes_find_dev(rctx); + if (!dd) + return -ENODEV; + rctx->mode = mode; + + return omap_aes_gcm_handle_queue(dd, req); +} + +int omap_aes_gcm_encrypt(struct aead_request *req) +{ + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + + memcpy(rctx->iv, req->iv, 12); + return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); +} + +int omap_aes_gcm_decrypt(struct aead_request *req) +{ + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + + memcpy(rctx->iv, req->iv, 12); + return omap_aes_gcm_crypt(req, FLAGS_GCM); +} + +int omap_aes_4106gcm_encrypt(struct aead_request *req) +{ + struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + + memcpy(rctx->iv, ctx->nonce, 4); + memcpy(rctx->iv + 4, req->iv, 8); + return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM | + FLAGS_RFC4106_GCM); +} + +int omap_aes_4106gcm_decrypt(struct aead_request *req) +{ + struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct omap_aes_reqctx *rctx = aead_request_ctx(req); + + memcpy(rctx->iv, ctx->nonce, 4); + memcpy(rctx->iv + 4, req->iv, 8); + return omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM); +} + +int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm); + + if (keylen < 4) + return -EINVAL; + + keylen -= 4; + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + memcpy(ctx->nonce, key + keylen, 4); + ctx->keylen = keylen; + + return 0; +} diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index bf3b27d9dc33..5120a17731d0 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -37,6 +37,7 @@ #include #include #include +#include #include "omap-crypto.h" #include "omap-aes.h" @@ -112,8 +113,16 @@ static int omap_aes_hw_init(struct omap_aes_dev *dd) return 0; } +void omap_aes_clear_copy_flags(struct omap_aes_dev *dd) +{ + dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_IN_DATA_ST_SHIFT); + dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_OUT_DATA_ST_SHIFT); + dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_ASSOC_DATA_ST_SHIFT); +} + int omap_aes_write_ctrl(struct omap_aes_dev *dd) { + struct omap_aes_reqctx *rctx; unsigned int key32; int i, err; u32 val; @@ -124,7 +133,11 @@ int omap_aes_write_ctrl(struct omap_aes_dev *dd) key32 = dd->ctx->keylen / sizeof(u32); - /* it seems a key should always be set even if it has not changed */ + /* RESET the key as previous HASH keys should not get affected*/ + if (dd->flags & FLAGS_GCM) + for (i = 0; i < 0x40; i = i + 4) + omap_aes_write(dd, i, 0x0); + for (i = 0; i < key32; i++) { omap_aes_write(dd, AES_REG_KEY(dd, i), __le32_to_cpu(dd->ctx->key[i])); @@ -133,12 +146,21 @@ int omap_aes_write_ctrl(struct omap_aes_dev *dd) if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->info) omap_aes_write_n(dd, AES_REG_IV(dd, 0), dd->req->info, 4); + if ((dd->flags & (FLAGS_GCM)) && dd->aead_req->iv) { + rctx = aead_request_ctx(dd->aead_req); + omap_aes_write_n(dd, AES_REG_IV(dd, 0), (u32 *)rctx->iv, 4); + } + val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3); if (dd->flags & FLAGS_CBC) val |= AES_REG_CTRL_CBC; - if (dd->flags & FLAGS_CTR) + + if (dd->flags & (FLAGS_CTR | FLAGS_GCM)) val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_128; + if (dd->flags & FLAGS_GCM) + val |= AES_REG_CTRL_GCM; + if (dd->flags & FLAGS_ENCRYPT) val |= AES_REG_CTRL_DIRECTION; @@ -169,6 +191,8 @@ static void omap_aes_dma_trigger_omap4(struct omap_aes_dev *dd, int length) { omap_aes_write(dd, AES_REG_LENGTH_N(0), length); omap_aes_write(dd, AES_REG_LENGTH_N(1), 0); + if (dd->flags & FLAGS_GCM) + omap_aes_write(dd, AES_REG_A_LEN, dd->assoc_len); omap_aes_dma_trigger_omap2(dd, length); } @@ -306,7 +330,10 @@ static int omap_aes_crypt_dma(struct omap_aes_dev *dd, return -EINVAL; } - tx_out->callback = omap_aes_dma_out_callback; + if (dd->flags & FLAGS_GCM) + tx_out->callback = omap_aes_gcm_dma_out_callback; + else + tx_out->callback = omap_aes_dma_out_callback; tx_out->callback_param = dd; dmaengine_submit(tx_in); @@ -411,7 +438,7 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, flags |= OMAP_CRYPTO_FORCE_COPY; ret = omap_crypto_align_sg(&dd->in_sg, dd->total, AES_BLOCK_SIZE, - &dd->in_sgl, flags, + dd->in_sgl, flags, FLAGS_IN_DATA_ST_SHIFT, &dd->flags); if (ret) return ret; @@ -466,7 +493,7 @@ static void omap_aes_done_task(unsigned long data) omap_aes_crypt_dma_stop(dd); } - omap_crypto_cleanup(&dd->in_sgl, NULL, 0, dd->total_save, + omap_crypto_cleanup(dd->in_sgl, NULL, 0, dd->total_save, FLAGS_IN_DATA_ST_SHIFT, dd->flags); omap_crypto_cleanup(&dd->out_sgl, dd->orig_out, 0, dd->total_save, @@ -591,6 +618,36 @@ static int omap_aes_cra_init(struct crypto_tfm *tfm) return 0; } +static int omap_aes_gcm_cra_init(struct crypto_aead *tfm) +{ + struct omap_aes_dev *dd = NULL; + struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm); + int err; + + /* Find AES device, currently picks the first device */ + spin_lock_bh(&list_lock); + list_for_each_entry(dd, &dev_list, list) { + break; + } + spin_unlock_bh(&list_lock); + + err = pm_runtime_get_sync(dd->dev); + if (err < 0) { + dev_err(dd->dev, "%s: failed to get_sync(%d)\n", + __func__, err); + return err; + } + + tfm->reqsize = sizeof(struct omap_aes_reqctx); + ctx->ctr = crypto_alloc_skcipher("ecb(aes)", 0, 0); + if (IS_ERR(ctx->ctr)) { + pr_warn("could not load aes driver for encrypting IV\n"); + return PTR_ERR(ctx->ctr); + } + + return 0; +} + static void omap_aes_cra_exit(struct crypto_tfm *tfm) { struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm); @@ -601,6 +658,16 @@ static void omap_aes_cra_exit(struct crypto_tfm *tfm) ctx->fallback = NULL; } +static void omap_aes_gcm_cra_exit(struct crypto_aead *tfm) +{ + struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm); + + omap_aes_cra_exit(crypto_aead_tfm(tfm)); + + if (ctx->ctr) + crypto_free_skcipher(ctx->ctr); +} + /* ********************** ALGS ************************************ */ static struct crypto_alg algs_ecb_cbc[] = { @@ -685,6 +752,54 @@ static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc[] = { }, }; +static struct aead_alg algs_aead_gcm[] = { +{ + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-omap", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct omap_aes_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .init = omap_aes_gcm_cra_init, + .exit = omap_aes_gcm_cra_exit, + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = omap_aes_gcm_setkey, + .encrypt = omap_aes_gcm_encrypt, + .decrypt = omap_aes_gcm_decrypt, +}, +{ + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-omap", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct omap_aes_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, + .init = omap_aes_gcm_cra_init, + .exit = omap_aes_gcm_cra_exit, + .maxauthsize = AES_BLOCK_SIZE, + .ivsize = 8, + .setkey = omap_aes_4106gcm_setkey, + .encrypt = omap_aes_4106gcm_encrypt, + .decrypt = omap_aes_4106gcm_decrypt, +}, +}; + +static struct omap_aes_aead_algs omap_aes_aead_info = { + .algs_list = algs_aead_gcm, + .size = ARRAY_SIZE(algs_aead_gcm), +}; + static const struct omap_aes_pdata omap_aes_pdata_omap2 = { .algs_info = omap_aes_algs_info_ecb_cbc, .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc), @@ -738,6 +853,7 @@ static const struct omap_aes_pdata omap_aes_pdata_omap3 = { static const struct omap_aes_pdata omap_aes_pdata_omap4 = { .algs_info = omap_aes_algs_info_ecb_cbc_ctr, .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr), + .aead_algs_info = &omap_aes_aead_info, .trigger = omap_aes_dma_trigger_omap4, .key_ofs = 0x3c, .iv_ofs = 0x40, @@ -920,6 +1036,7 @@ static int omap_aes_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct omap_aes_dev *dd; struct crypto_alg *algp; + struct aead_alg *aalg; struct resource res; int err = -ENOMEM, i, j, irq = -1; u32 reg; @@ -932,6 +1049,8 @@ static int omap_aes_probe(struct platform_device *pdev) dd->dev = dev; platform_set_drvdata(pdev, dd); + aead_init_queue(&dd->aead_queue, OMAP_AES_QUEUE_LENGTH); + err = (dev->of_node) ? omap_aes_get_res_of(dd, dev, &res) : omap_aes_get_res_pdev(dd, pdev, &res); if (err) @@ -987,6 +1106,7 @@ static int omap_aes_probe(struct platform_device *pdev) } } + spin_lock_init(&dd->lock); INIT_LIST_HEAD(&dd->list); spin_lock(&list_lock); @@ -1023,7 +1143,29 @@ static int omap_aes_probe(struct platform_device *pdev) } } + if (dd->pdata->aead_algs_info && + !dd->pdata->aead_algs_info->registered) { + for (i = 0; i < dd->pdata->aead_algs_info->size; i++) { + aalg = &dd->pdata->aead_algs_info->algs_list[i]; + algp = &aalg->base; + + pr_debug("reg alg: %s\n", algp->cra_name); + INIT_LIST_HEAD(&algp->cra_list); + + err = crypto_register_aead(aalg); + if (err) + goto err_aead_algs; + + dd->pdata->aead_algs_info->registered++; + } + } + return 0; +err_aead_algs: + for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) { + aalg = &dd->pdata->aead_algs_info->algs_list[i]; + crypto_unregister_aead(aalg); + } err_algs: for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) @@ -1048,6 +1190,7 @@ err_data: static int omap_aes_remove(struct platform_device *pdev) { struct omap_aes_dev *dd = platform_get_drvdata(pdev); + struct aead_alg *aalg; int i, j; if (!dd) @@ -1062,7 +1205,13 @@ static int omap_aes_remove(struct platform_device *pdev) crypto_unregister_alg( &dd->pdata->algs_info[i].algs_list[j]); + for (i = dd->pdata->aead_algs_info->size - 1; i >= 0; i--) { + aalg = &dd->pdata->aead_algs_info->algs_list[i]; + crypto_unregister_aead(aalg); + } + crypto_engine_exit(dd->engine); + tasklet_kill(&dd->done_task); omap_aes_dma_cleanup(dd); pm_runtime_disable(dd->dev); diff --git a/drivers/crypto/omap-aes.h b/drivers/crypto/omap-aes.h index 11e1784cc4a5..8906342e2b9a 100644 --- a/drivers/crypto/omap-aes.h +++ b/drivers/crypto/omap-aes.h @@ -30,11 +30,13 @@ #define AES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04)) #define AES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs) +#define AES_REG_CTRL_CONTEXT_READY BIT(31) #define AES_REG_CTRL_CTR_WIDTH_MASK GENMASK(8, 7) #define AES_REG_CTRL_CTR_WIDTH_32 0 #define AES_REG_CTRL_CTR_WIDTH_64 BIT(7) #define AES_REG_CTRL_CTR_WIDTH_96 BIT(8) #define AES_REG_CTRL_CTR_WIDTH_128 GENMASK(8, 7) +#define AES_REG_CTRL_GCM GENMASK(17, 16) #define AES_REG_CTRL_CTR BIT(6) #define AES_REG_CTRL_CBC BIT(5) #define AES_REG_CTRL_KEY_SIZE GENMASK(4, 3) @@ -43,7 +45,12 @@ #define AES_REG_CTRL_OUTPUT_READY BIT(0) #define AES_REG_CTRL_MASK GENMASK(24, 2) +#define AES_REG_C_LEN_0 0x54 +#define AES_REG_C_LEN_1 0x58 +#define AES_REG_A_LEN 0x5C + #define AES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04)) +#define AES_REG_TAG_N(dd, x) (0x70 + ((x) * 0x04)) #define AES_REG_REV(dd) ((dd)->pdata->rev_ofs) @@ -65,30 +72,41 @@ #define DEFAULT_AUTOSUSPEND_DELAY 1000 -#define FLAGS_MODE_MASK 0x000f +#define FLAGS_MODE_MASK 0x001f #define FLAGS_ENCRYPT BIT(0) #define FLAGS_CBC BIT(1) -#define FLAGS_GIV BIT(2) -#define FLAGS_CTR BIT(3) +#define FLAGS_CTR BIT(2) +#define FLAGS_GCM BIT(3) +#define FLAGS_RFC4106_GCM BIT(4) -#define FLAGS_INIT BIT(4) -#define FLAGS_FAST BIT(5) -#define FLAGS_BUSY BIT(6) +#define FLAGS_INIT BIT(5) +#define FLAGS_FAST BIT(6) +#define FLAGS_BUSY BIT(7) #define FLAGS_IN_DATA_ST_SHIFT 8 #define FLAGS_OUT_DATA_ST_SHIFT 10 +#define FLAGS_ASSOC_DATA_ST_SHIFT 12 #define AES_BLOCK_WORDS (AES_BLOCK_SIZE >> 2) +struct omap_aes_gcm_result { + struct completion completion; + int err; +}; + struct omap_aes_ctx { int keylen; u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + u8 nonce[4]; struct crypto_skcipher *fallback; + struct crypto_skcipher *ctr; }; struct omap_aes_reqctx { struct omap_aes_dev *dd; unsigned long mode; + u8 iv[AES_BLOCK_SIZE]; + u32 auth_tag[AES_BLOCK_SIZE / sizeof(u32)]; }; #define OMAP_AES_QUEUE_LENGTH 1 @@ -100,9 +118,16 @@ struct omap_aes_algs_info { unsigned int registered; }; +struct omap_aes_aead_algs { + struct aead_alg *algs_list; + unsigned int size; + unsigned int registered; +}; + struct omap_aes_pdata { struct omap_aes_algs_info *algs_info; unsigned int algs_info_size; + struct omap_aes_aead_algs *aead_algs_info; void (*trigger)(struct omap_aes_dev *dd, int length); @@ -135,8 +160,11 @@ struct omap_aes_dev { int err; struct tasklet_struct done_task; + struct aead_queue aead_queue; + spinlock_t lock; struct ablkcipher_request *req; + struct aead_request *aead_req; struct crypto_engine *engine; /* @@ -145,12 +173,14 @@ struct omap_aes_dev { */ size_t total; size_t total_save; + size_t assoc_len; + size_t authsize; struct scatterlist *in_sg; struct scatterlist *out_sg; /* Buffers for copying for unaligned cases */ - struct scatterlist in_sgl; + struct scatterlist in_sgl[2]; struct scatterlist out_sgl; struct scatterlist *orig_out; @@ -167,8 +197,18 @@ struct omap_aes_dev { u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset); void omap_aes_write(struct omap_aes_dev *dd, u32 offset, u32 value); struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx); +int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen); +int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen); +int omap_aes_gcm_encrypt(struct aead_request *req); +int omap_aes_gcm_decrypt(struct aead_request *req); +int omap_aes_4106gcm_encrypt(struct aead_request *req); +int omap_aes_4106gcm_decrypt(struct aead_request *req); int omap_aes_write_ctrl(struct omap_aes_dev *dd); int omap_aes_crypt_dma_start(struct omap_aes_dev *dd); int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd); +void omap_aes_gcm_dma_out_callback(void *data); +void omap_aes_clear_copy_flags(struct omap_aes_dev *dd); #endif From 5d78d57ede8f9e7f656c610ed25be7be337e0529 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:32 +0300 Subject: [PATCH 047/140] crypto: omap-sham - buffer handling fixes for hashing later Currently, the hash later code only handles the cases when we have either new data coming in with the request or old data in the buffer, but not the combination when we have both. Fix this by changing the ordering of the code a bit and handling both cases properly simultaneously if needed. Also, fix an issue with omap_sham_update that surfaces with this fix, so that the code checks the bufcnt instead of total data amount against buffer length to avoid any buffer overflows. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 1864a57caaa4..ca9e48a19b15 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -874,14 +874,21 @@ static int omap_sham_prepare_request(struct ahash_request *req, bool update) } if (hash_later) { - if (req->nbytes) { - scatterwalk_map_and_copy(rctx->buffer, req->src, - req->nbytes - hash_later, - hash_later, 0); - } else { + int offset = 0; + + if (hash_later > req->nbytes) { memcpy(rctx->buffer, rctx->buffer + xmit_len, - hash_later); + hash_later - req->nbytes); + offset = hash_later - req->nbytes; } + + if (req->nbytes) { + scatterwalk_map_and_copy(rctx->buffer + offset, + req->src, + offset + req->nbytes - + hash_later, hash_later, 0); + } + rctx->bufcnt = hash_later; } else { rctx->bufcnt = 0; @@ -1190,11 +1197,10 @@ static int omap_sham_update(struct ahash_request *req) if (!req->nbytes) return 0; - if (ctx->total + req->nbytes < ctx->buflen) { + if (ctx->bufcnt + req->nbytes <= ctx->buflen) { scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src, 0, req->nbytes, 0); ctx->bufcnt += req->nbytes; - ctx->total += req->nbytes; return 0; } From 898d86a565925f09de3d0b30cf3b47ec2e409680 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:33 +0300 Subject: [PATCH 048/140] crypto: omap-sham - fix closing of hash with separate finalize call Currently there is an interesting corner case failure with omap-sham driver, if the finalize call is done separately with no data, but all previous data has already been processed. In this case, it is not possible to close the hash with the hardware without providing any data, so we get incorrect results. Fix this by adjusting the size of data sent to the hardware crypto engine in case the non-final data size falls on the block size boundary, by reducing the amount of data sent by one full block. This makes it sure that we always have some data available for the finalize call and we can close the hash properly. Signed-off-by: Tero Kristo Reported-by: Aparna Balasubramanian Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index ca9e48a19b15..dc091b27affa 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -751,7 +751,10 @@ static int omap_sham_align_sgs(struct scatterlist *sg, if (final) new_len = DIV_ROUND_UP(new_len, bs) * bs; else - new_len = new_len / bs * bs; + new_len = (new_len - 1) / bs * bs; + + if (nbytes != new_len) + list_ok = false; while (nbytes > 0 && sg_tmp) { n++; @@ -847,6 +850,8 @@ static int omap_sham_prepare_request(struct ahash_request *req, bool update) xmit_len = DIV_ROUND_UP(xmit_len, bs) * bs; else xmit_len = xmit_len / bs * bs; + } else if (!final) { + xmit_len -= bs; } hash_later = rctx->total - xmit_len; @@ -1138,7 +1143,7 @@ retry: ctx = ahash_request_ctx(req); err = omap_sham_prepare_request(req, ctx->op == OP_UPDATE); - if (err) + if (err || !ctx->total) goto err1; dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", From c28e8f21642fd01a65687de9bfa5307fdcfe9966 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Wed, 24 May 2017 10:35:34 +0300 Subject: [PATCH 049/140] crypto: omap-sham - force word alignment on the xmit-buf also This was previously missed from the code, causing SDMA to hang in some cases where the buffer ended up being not aligned. Signed-off-by: Tero Kristo Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index dc091b27affa..9ad9d399daf1 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -226,7 +226,7 @@ struct omap_sham_dev { struct dma_chan *dma_lch; struct tasklet_struct done_task; u8 polling_mode; - u8 xmit_buf[BUFLEN]; + u8 xmit_buf[BUFLEN] OMAP_ALIGNED; unsigned long flags; struct crypto_queue queue; From 1b44c5a60c137e5fd0c2c8b86e58fdbc9cd181ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Wed, 24 May 2017 16:10:34 +0200 Subject: [PATCH 050/140] crypto: inside-secure - add SafeXcel EIP197 crypto engine driver Add support for Inside Secure SafeXcel EIP197 cryptographic engine, which can be found on Marvell Armada 7k and 8k boards. This driver currently implements: ecb(aes), cbc(aes), sha1, sha224, sha256 and hmac(sah1) algorithms. Two firmwares are needed for this engine to work. Their are mostly used for more advanced operations than the ones supported (as of now), but we still need them to pass the data to the internal cryptographic engine. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 17 + drivers/crypto/Makefile | 1 + drivers/crypto/inside-secure/Makefile | 2 + drivers/crypto/inside-secure/safexcel.c | 930 +++++++++++++++ drivers/crypto/inside-secure/safexcel.h | 572 +++++++++ .../crypto/inside-secure/safexcel_cipher.c | 556 +++++++++ drivers/crypto/inside-secure/safexcel_hash.c | 1045 +++++++++++++++++ drivers/crypto/inside-secure/safexcel_ring.c | 157 +++ 8 files changed, 3280 insertions(+) create mode 100644 drivers/crypto/inside-secure/Makefile create mode 100644 drivers/crypto/inside-secure/safexcel.c create mode 100644 drivers/crypto/inside-secure/safexcel.h create mode 100644 drivers/crypto/inside-secure/safexcel_cipher.c create mode 100644 drivers/crypto/inside-secure/safexcel_hash.c create mode 100644 drivers/crypto/inside-secure/safexcel_ring.c diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 4e6591b0b128..0a652baf3cb3 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -655,4 +655,21 @@ config CRYPTO_DEV_BCM_SPU source "drivers/crypto/stm32/Kconfig" +config CRYPTO_DEV_SAFEXCEL + tristate "Inside Secure's SafeXcel cryptographic engine driver" + depends on HAS_DMA && OF + depends on (ARM64 && ARCH_MVEBU) || (COMPILE_TEST && 64BIT) + select CRYPTO_AES + select CRYPTO_BLKCIPHER + select CRYPTO_HASH + select CRYPTO_HMAC + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + help + This driver interfaces with the SafeXcel EIP-197 cryptographic engine + designed by Inside Secure. Select this if you want to use CBC/ECB + chain mode, AES cipher mode and SHA1/SHA224/SHA256/SHA512 hash + algorithms. + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 8177388f5c85..28786321b118 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -41,3 +41,4 @@ obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/ obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/ +obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/ diff --git a/drivers/crypto/inside-secure/Makefile b/drivers/crypto/inside-secure/Makefile new file mode 100644 index 000000000000..302f07dde98c --- /dev/null +++ b/drivers/crypto/inside-secure/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += crypto_safexcel.o +crypto_safexcel-objs := safexcel.o safexcel_ring.o safexcel_cipher.o safexcel_hash.o diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c new file mode 100644 index 000000000000..5485e925e18d --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel.c @@ -0,0 +1,930 @@ +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + * + * This file is licensed under the terms of the GNU General Public + * License version 2. This program is licensed "as is" without any + * warranty of any kind, whether express or implied. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "safexcel.h" + +static u32 max_rings = EIP197_MAX_RINGS; +module_param(max_rings, uint, 0644); +MODULE_PARM_DESC(max_rings, "Maximum number of rings to use."); + +static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv) +{ + u32 val, htable_offset; + int i; + + /* Enable the record cache memory access */ + val = readl(priv->base + EIP197_CS_RAM_CTRL); + val &= ~EIP197_TRC_ENABLE_MASK; + val |= EIP197_TRC_ENABLE_0; + writel(val, priv->base + EIP197_CS_RAM_CTRL); + + /* Clear all ECC errors */ + writel(0, priv->base + EIP197_TRC_ECCCTRL); + + /* + * Make sure the cache memory is accessible by taking record cache into + * reset. + */ + val = readl(priv->base + EIP197_TRC_PARAMS); + val |= EIP197_TRC_PARAMS_SW_RESET; + val &= ~EIP197_TRC_PARAMS_DATA_ACCESS; + writel(val, priv->base + EIP197_TRC_PARAMS); + + /* Clear all records */ + for (i = 0; i < EIP197_CS_RC_MAX; i++) { + u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE; + + writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) | + EIP197_CS_RC_PREV(EIP197_RC_NULL), + priv->base + offset); + + val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1); + if (i == 0) + val |= EIP197_CS_RC_PREV(EIP197_RC_NULL); + else if (i == EIP197_CS_RC_MAX - 1) + val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL); + writel(val, priv->base + offset + sizeof(u32)); + } + + /* Clear the hash table entries */ + htable_offset = EIP197_CS_RC_MAX * EIP197_CS_RC_SIZE; + for (i = 0; i < 64; i++) + writel(GENMASK(29, 0), + priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32)); + + /* Disable the record cache memory access */ + val = readl(priv->base + EIP197_CS_RAM_CTRL); + val &= ~EIP197_TRC_ENABLE_MASK; + writel(val, priv->base + EIP197_CS_RAM_CTRL); + + /* Write head and tail pointers of the record free chain */ + val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) | + EIP197_TRC_FREECHAIN_TAIL_PTR(EIP197_CS_RC_MAX - 1); + writel(val, priv->base + EIP197_TRC_FREECHAIN); + + /* Configure the record cache #1 */ + val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(EIP197_CS_TRC_REC_WC) | + EIP197_TRC_PARAMS2_HTABLE_PTR(EIP197_CS_RC_MAX); + writel(val, priv->base + EIP197_TRC_PARAMS2); + + /* Configure the record cache #2 */ + val = EIP197_TRC_PARAMS_RC_SZ_LARGE(EIP197_CS_TRC_LG_REC_WC) | + EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) | + EIP197_TRC_PARAMS_HTABLE_SZ(2); + writel(val, priv->base + EIP197_TRC_PARAMS); +} + +static void eip197_write_firmware(struct safexcel_crypto_priv *priv, + const struct firmware *fw, u32 ctrl, + u32 prog_en) +{ + const u32 *data = (const u32 *)fw->data; + u32 val; + int i; + + /* Reset the engine to make its program memory accessible */ + writel(EIP197_PE_ICE_x_CTRL_SW_RESET | + EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | + EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, + priv->base + ctrl); + + /* Enable access to the program memory */ + writel(prog_en, priv->base + EIP197_PE_ICE_RAM_CTRL); + + /* Write the firmware */ + for (i = 0; i < fw->size / sizeof(u32); i++) + writel(be32_to_cpu(data[i]), + priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32)); + + /* Disable access to the program memory */ + writel(0, priv->base + EIP197_PE_ICE_RAM_CTRL); + + /* Release engine from reset */ + val = readl(priv->base + ctrl); + val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET; + writel(val, priv->base + ctrl); +} + +static int eip197_load_firmwares(struct safexcel_crypto_priv *priv) +{ + const char *fw_name[] = {"ifpp.bin", "ipue.bin"}; + const struct firmware *fw[FW_NB]; + int i, j, ret = 0; + u32 val; + + for (i = 0; i < FW_NB; i++) { + ret = request_firmware(&fw[i], fw_name[i], priv->dev); + if (ret) { + dev_err(priv->dev, + "Failed to request firmware %s (%d)\n", + fw_name[i], ret); + goto release_fw; + } + } + + /* Clear the scratchpad memory */ + val = readl(priv->base + EIP197_PE_ICE_SCRATCH_CTRL); + val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER | + EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN | + EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS | + EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS; + writel(val, priv->base + EIP197_PE_ICE_SCRATCH_CTRL); + + memset(priv->base + EIP197_PE_ICE_SCRATCH_RAM, 0, + EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32)); + + eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL, + EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN); + + eip197_write_firmware(priv, fw[FW_IPUE], EIP197_PE_ICE_PUE_CTRL, + EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN); + +release_fw: + for (j = 0; j < i; j++) + release_firmware(fw[j]); + + return ret; +} + +static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) +{ + u32 hdw, cd_size_rnd, val; + int i; + + hdw = readl(priv->base + EIP197_HIA_OPTIONS); + hdw &= GENMASK(27, 25); + hdw >>= 25; + + cd_size_rnd = (priv->config.cd_size + (BIT(hdw) - 1)) >> hdw; + + for (i = 0; i < priv->config.rings; i++) { + /* ring base address */ + writel(lower_32_bits(priv->ring[i].cdr.base_dma), + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); + writel(upper_32_bits(priv->ring[i].cdr.base_dma), + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); + + writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) | + priv->config.cd_size, + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DESC_SIZE); + writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) | + (EIP197_FETCH_COUNT * priv->config.cd_offset), + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG); + + /* Configure DMA tx control */ + val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); + val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); + writel(val, + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DMA_CFG); + + /* clear any pending interrupt */ + writel(GENMASK(5, 0), + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_STAT); + } + + return 0; +} + +static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) +{ + u32 hdw, rd_size_rnd, val; + int i; + + hdw = readl(priv->base + EIP197_HIA_OPTIONS); + hdw &= GENMASK(27, 25); + hdw >>= 25; + + rd_size_rnd = (priv->config.rd_size + (BIT(hdw) - 1)) >> hdw; + + for (i = 0; i < priv->config.rings; i++) { + /* ring base address */ + writel(lower_32_bits(priv->ring[i].rdr.base_dma), + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); + writel(upper_32_bits(priv->ring[i].rdr.base_dma), + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); + + writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) | + priv->config.rd_size, + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DESC_SIZE); + + writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) | + (EIP197_FETCH_COUNT * priv->config.rd_offset), + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG); + + /* Configure DMA tx control */ + val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); + val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); + val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUG; + writel(val, + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DMA_CFG); + + /* clear any pending interrupt */ + writel(GENMASK(7, 0), + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_STAT); + + /* enable ring interrupt */ + val = readl(priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); + val |= EIP197_RDR_IRQ(i); + writel(val, priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); + } + + return 0; +} + +static int safexcel_hw_init(struct safexcel_crypto_priv *priv) +{ + u32 version, val; + int i, ret; + + /* Determine endianess and configure byte swap */ + version = readl(priv->base + EIP197_HIA_VERSION); + val = readl(priv->base + EIP197_HIA_MST_CTRL); + + if ((version & 0xffff) == EIP197_HIA_VERSION_BE) + val |= EIP197_MST_CTRL_BYTE_SWAP; + else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE) + val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24); + + writel(val, priv->base + EIP197_HIA_MST_CTRL); + + + /* Configure wr/rd cache values */ + writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | + EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS), + priv->base + EIP197_MST_CTRL); + + /* Interrupts reset */ + + /* Disable all global interrupts */ + writel(0, priv->base + EIP197_HIA_AIC_G_ENABLE_CTRL); + + /* Clear any pending interrupt */ + writel(GENMASK(31, 0), priv->base + EIP197_HIA_AIC_G_ACK); + + /* Data Fetch Engine configuration */ + + /* Reset all DFE threads */ + writel(EIP197_DxE_THR_CTRL_RESET_PE, + priv->base + EIP197_HIA_DFE_THR_CTRL); + + /* Reset HIA input interface arbiter */ + writel(EIP197_HIA_RA_PE_CTRL_RESET, + priv->base + EIP197_HIA_RA_PE_CTRL); + + /* DMA transfer size to use */ + val = EIP197_HIA_DFE_CFG_DIS_DEBUG; + val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9); + val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7); + val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS); + val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS); + writel(val, priv->base + EIP197_HIA_DFE_CFG); + + /* Leave the DFE threads reset state */ + writel(0, priv->base + EIP197_HIA_DFE_THR_CTRL); + + /* Configure the procesing engine thresholds */ + writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(9), + priv->base + EIP197_PE_IN_DBUF_THRES); + writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(7), + priv->base + EIP197_PE_IN_TBUF_THRES); + + /* enable HIA input interface arbiter and rings */ + writel(EIP197_HIA_RA_PE_CTRL_EN | GENMASK(priv->config.rings - 1, 0), + priv->base + EIP197_HIA_RA_PE_CTRL); + + /* Data Store Engine configuration */ + + /* Reset all DSE threads */ + writel(EIP197_DxE_THR_CTRL_RESET_PE, + priv->base + EIP197_HIA_DSE_THR_CTRL); + + /* Wait for all DSE threads to complete */ + while ((readl(priv->base + EIP197_HIA_DSE_THR_STAT) & + GENMASK(15, 12)) != GENMASK(15, 12)) + ; + + /* DMA transfer size to use */ + val = EIP197_HIA_DSE_CFG_DIS_DEBUG; + val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8); + val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS); + writel(val, priv->base + EIP197_HIA_DSE_CFG); + + /* Leave the DSE threads reset state */ + writel(0, priv->base + EIP197_HIA_DSE_THR_CTRL); + + /* Configure the procesing engine thresholds */ + writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | EIP197_PE_OUT_DBUF_THRES_MAX(8), + priv->base + EIP197_PE_OUT_DBUF_THRES); + + /* Processing Engine configuration */ + + /* H/W capabilities selection */ + val = EIP197_FUNCTION_RSVD; + val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY; + val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC; + val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1; + val |= EIP197_ALG_SHA2; + writel(val, priv->base + EIP197_PE_EIP96_FUNCTION_EN); + + /* Command Descriptor Rings prepare */ + for (i = 0; i < priv->config.rings; i++) { + /* Clear interrupts for this ring */ + writel(GENMASK(31, 0), + priv->base + EIP197_HIA_AIC_R_ENABLE_CLR(i)); + + /* Disable external triggering */ + writel(0, priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG); + + /* Clear the pending prepared counter */ + writel(EIP197_xDR_PREP_CLR_COUNT, + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_COUNT); + + /* Clear the pending processed counter */ + writel(EIP197_xDR_PROC_CLR_COUNT, + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_COUNT); + + writel(0, + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_PNTR); + writel(0, + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_PNTR); + + writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2, + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_SIZE); + } + + /* Result Descriptor Ring prepare */ + for (i = 0; i < priv->config.rings; i++) { + /* Disable external triggering*/ + writel(0, priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG); + + /* Clear the pending prepared counter */ + writel(EIP197_xDR_PREP_CLR_COUNT, + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_COUNT); + + /* Clear the pending processed counter */ + writel(EIP197_xDR_PROC_CLR_COUNT, + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_COUNT); + + writel(0, + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_PNTR); + writel(0, + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_PNTR); + + /* Ring size */ + writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2, + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_SIZE); + } + + /* Enable command descriptor rings */ + writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), + priv->base + EIP197_HIA_DFE_THR_CTRL); + + /* Enable result descriptor rings */ + writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), + priv->base + EIP197_HIA_DSE_THR_CTRL); + + /* Clear any HIA interrupt */ + writel(GENMASK(30, 20), priv->base + EIP197_HIA_AIC_G_ACK); + + eip197_trc_cache_init(priv); + + ret = eip197_load_firmwares(priv); + if (ret) + return ret; + + safexcel_hw_setup_cdesc_rings(priv); + safexcel_hw_setup_rdesc_rings(priv); + + return 0; +} + +void safexcel_dequeue(struct safexcel_crypto_priv *priv) +{ + struct crypto_async_request *req, *backlog; + struct safexcel_context *ctx; + struct safexcel_request *request; + int i, ret, n = 0, nreq[EIP197_MAX_RINGS] = {0}; + int cdesc[EIP197_MAX_RINGS] = {0}, rdesc[EIP197_MAX_RINGS] = {0}; + int commands, results; + + do { + spin_lock_bh(&priv->lock); + req = crypto_dequeue_request(&priv->queue); + backlog = crypto_get_backlog(&priv->queue); + spin_unlock_bh(&priv->lock); + + if (!req) + goto finalize; + + request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req)); + if (!request) + goto requeue; + + ctx = crypto_tfm_ctx(req->tfm); + ret = ctx->send(req, ctx->ring, request, &commands, &results); + if (ret) { + kfree(request); +requeue: + spin_lock_bh(&priv->lock); + crypto_enqueue_request(&priv->queue, req); + spin_unlock_bh(&priv->lock); + + priv->need_dequeue = true; + continue; + } + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + spin_lock_bh(&priv->ring[ctx->ring].egress_lock); + list_add_tail(&request->list, &priv->ring[ctx->ring].list); + spin_unlock_bh(&priv->ring[ctx->ring].egress_lock); + + cdesc[ctx->ring] += commands; + rdesc[ctx->ring] += results; + + nreq[ctx->ring]++; + } while (n++ < EIP197_MAX_BATCH_SZ); + +finalize: + if (n == EIP197_MAX_BATCH_SZ) + priv->need_dequeue = true; + else if (!n) + return; + + for (i = 0; i < priv->config.rings; i++) { + if (!nreq[i]) + continue; + + spin_lock_bh(&priv->ring[i].lock); + + /* Configure when we want an interrupt */ + writel(EIP197_HIA_RDR_THRESH_PKT_MODE | + EIP197_HIA_RDR_THRESH_PROC_PKT(nreq[i]), + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_THRESH); + + /* let the RDR know we have pending descriptors */ + writel((rdesc[i] * priv->config.rd_offset) << 2, + priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_COUNT); + + /* let the CDR know we have pending descriptors */ + writel((cdesc[i] * priv->config.cd_offset) << 2, + priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_COUNT); + + spin_unlock_bh(&priv->ring[i].lock); + } +} + +void safexcel_free_context(struct safexcel_crypto_priv *priv, + struct crypto_async_request *req, + int result_sz) +{ + struct safexcel_context *ctx = crypto_tfm_ctx(req->tfm); + + if (ctx->result_dma) + dma_unmap_single(priv->dev, ctx->result_dma, result_sz, + DMA_FROM_DEVICE); + + if (ctx->cache) { + dma_unmap_single(priv->dev, ctx->cache_dma, ctx->cache_sz, + DMA_TO_DEVICE); + kfree(ctx->cache); + ctx->cache = NULL; + ctx->cache_sz = 0; + } +} + +void safexcel_complete(struct safexcel_crypto_priv *priv, int ring) +{ + struct safexcel_command_desc *cdesc; + + /* Acknowledge the command descriptors */ + do { + cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr); + if (IS_ERR(cdesc)) { + dev_err(priv->dev, + "Could not retrieve the command descriptor\n"); + return; + } + } while (!cdesc->last_seg); +} + +void safexcel_inv_complete(struct crypto_async_request *req, int error) +{ + struct safexcel_inv_result *result = req->data; + + if (error == -EINPROGRESS) + return; + + result->error = error; + complete(&result->completion); +} + +int safexcel_invalidate_cache(struct crypto_async_request *async, + struct safexcel_context *ctx, + struct safexcel_crypto_priv *priv, + dma_addr_t ctxr_dma, int ring, + struct safexcel_request *request) +{ + struct safexcel_command_desc *cdesc; + struct safexcel_result_desc *rdesc; + int ret = 0; + + spin_lock_bh(&priv->ring[ring].egress_lock); + + /* Prepare command descriptor */ + cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma); + if (IS_ERR(cdesc)) { + ret = PTR_ERR(cdesc); + goto unlock; + } + + cdesc->control_data.type = EIP197_TYPE_EXTENDED; + cdesc->control_data.options = 0; + cdesc->control_data.refresh = 0; + cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR; + + /* Prepare result descriptor */ + rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); + + if (IS_ERR(rdesc)) { + ret = PTR_ERR(rdesc); + goto cdesc_rollback; + } + + request->req = async; + goto unlock; + +cdesc_rollback: + safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); + +unlock: + spin_unlock_bh(&priv->ring[ring].egress_lock); + return ret; +} + +static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv, + int ring) +{ + struct safexcel_request *sreq; + struct safexcel_context *ctx; + int ret, i, nreq, ndesc = 0; + bool should_complete; + + nreq = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); + nreq >>= 24; + nreq &= GENMASK(6, 0); + if (!nreq) + return; + + for (i = 0; i < nreq; i++) { + spin_lock_bh(&priv->ring[ring].egress_lock); + sreq = list_first_entry(&priv->ring[ring].list, + struct safexcel_request, list); + list_del(&sreq->list); + spin_unlock_bh(&priv->ring[ring].egress_lock); + + ctx = crypto_tfm_ctx(sreq->req->tfm); + ndesc = ctx->handle_result(priv, ring, sreq->req, + &should_complete, &ret); + if (ndesc < 0) { + dev_err(priv->dev, "failed to handle result (%d)", ndesc); + return; + } + + writel(EIP197_xDR_PROC_xD_PKT(1) | + EIP197_xDR_PROC_xD_COUNT(ndesc * priv->config.rd_offset), + priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); + + if (should_complete) { + local_bh_disable(); + sreq->req->complete(sreq->req, ret); + local_bh_enable(); + } + + kfree(sreq); + } +} + +static void safexcel_handle_result_work(struct work_struct *work) +{ + struct safexcel_work_data *data = + container_of(work, struct safexcel_work_data, work); + struct safexcel_crypto_priv *priv = data->priv; + + safexcel_handle_result_descriptor(priv, data->ring); + + if (priv->need_dequeue) { + priv->need_dequeue = false; + safexcel_dequeue(data->priv); + } +} + +struct safexcel_ring_irq_data { + struct safexcel_crypto_priv *priv; + int ring; +}; + +static irqreturn_t safexcel_irq_ring(int irq, void *data) +{ + struct safexcel_ring_irq_data *irq_data = data; + struct safexcel_crypto_priv *priv = irq_data->priv; + int ring = irq_data->ring; + u32 status, stat; + + status = readl(priv->base + EIP197_HIA_AIC_R_ENABLED_STAT(ring)); + if (!status) + return IRQ_NONE; + + /* RDR interrupts */ + if (status & EIP197_RDR_IRQ(ring)) { + stat = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT); + + if (unlikely(stat & EIP197_xDR_ERR)) { + /* + * Fatal error, the RDR is unusable and must be + * reinitialized. This should not happen under + * normal circumstances. + */ + dev_err(priv->dev, "RDR: fatal error."); + } else if (likely(stat & EIP197_xDR_THRESH)) { + queue_work(priv->ring[ring].workqueue, &priv->ring[ring].work_data.work); + } + + /* ACK the interrupts */ + writel(stat & 0xff, + priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT); + } + + /* ACK the interrupts */ + writel(status, priv->base + EIP197_HIA_AIC_R_ACK(ring)); + + return IRQ_HANDLED; +} + +static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name, + irq_handler_t handler, + struct safexcel_ring_irq_data *ring_irq_priv) +{ + int ret, irq = platform_get_irq_byname(pdev, name); + + if (irq < 0) { + dev_err(&pdev->dev, "unable to get IRQ '%s'\n", name); + return irq; + } + + ret = devm_request_irq(&pdev->dev, irq, handler, 0, + dev_name(&pdev->dev), ring_irq_priv); + if (ret) { + dev_err(&pdev->dev, "unable to request IRQ %d\n", irq); + return ret; + } + + return irq; +} + +static struct safexcel_alg_template *safexcel_algs[] = { + &safexcel_alg_ecb_aes, + &safexcel_alg_cbc_aes, + &safexcel_alg_sha1, + &safexcel_alg_sha224, + &safexcel_alg_sha256, + &safexcel_alg_hmac_sha1, +}; + +static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) +{ + int i, j, ret = 0; + + for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { + safexcel_algs[i]->priv = priv; + + if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) + ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher); + else + ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash); + + if (ret) + goto fail; + } + + return 0; + +fail: + for (j = 0; j < i; j++) { + if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) + crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher); + else + crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash); + } + + return ret; +} + +static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { + if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) + crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher); + else + crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash); + } +} + +static void safexcel_configure(struct safexcel_crypto_priv *priv) +{ + u32 val, mask; + + val = readl(priv->base + EIP197_HIA_OPTIONS); + val = (val & GENMASK(27, 25)) >> 25; + mask = BIT(val) - 1; + + val = readl(priv->base + EIP197_HIA_OPTIONS); + priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings); + + priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32)); + priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask; + + priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32)); + priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask; +} + +static int safexcel_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *res; + struct safexcel_crypto_priv *priv; + u64 dma_mask; + int i, ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + priv->base = devm_ioremap_resource(dev, res); + if (IS_ERR(priv->base)) { + dev_err(dev, "failed to get resource\n"); + return PTR_ERR(priv->base); + } + + priv->clk = of_clk_get(dev->of_node, 0); + if (!IS_ERR(priv->clk)) { + ret = clk_prepare_enable(priv->clk); + if (ret) { + dev_err(dev, "unable to enable clk (%d)\n", ret); + return ret; + } + } else { + /* The clock isn't mandatory */ + if (PTR_ERR(priv->clk) == -EPROBE_DEFER) + return -EPROBE_DEFER; + } + + if (of_property_read_u64(dev->of_node, "dma-mask", &dma_mask)) + dma_mask = DMA_BIT_MASK(64); + ret = dma_set_mask_and_coherent(dev, dma_mask); + if (ret) + goto err_clk; + + priv->context_pool = dmam_pool_create("safexcel-context", dev, + sizeof(struct safexcel_context_record), + 1, 0); + if (!priv->context_pool) { + ret = -ENOMEM; + goto err_clk; + } + + safexcel_configure(priv); + + for (i = 0; i < priv->config.rings; i++) { + char irq_name[6] = {0}; /* "ringX\0" */ + char wq_name[9] = {0}; /* "wq_ringX\0" */ + int irq; + struct safexcel_ring_irq_data *ring_irq; + + ret = safexcel_init_ring_descriptors(priv, + &priv->ring[i].cdr, + &priv->ring[i].rdr); + if (ret) + goto err_clk; + + ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL); + if (!ring_irq) { + ret = -ENOMEM; + goto err_clk; + } + + ring_irq->priv = priv; + ring_irq->ring = i; + + snprintf(irq_name, 6, "ring%d", i); + irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring, + ring_irq); + + if (irq < 0) + goto err_clk; + + priv->ring[i].work_data.priv = priv; + priv->ring[i].work_data.ring = i; + INIT_WORK(&priv->ring[i].work_data.work, safexcel_handle_result_work); + + snprintf(wq_name, 9, "wq_ring%d", i); + priv->ring[i].workqueue = create_singlethread_workqueue(wq_name); + if (!priv->ring[i].workqueue) { + ret = -ENOMEM; + goto err_clk; + } + + INIT_LIST_HEAD(&priv->ring[i].list); + spin_lock_init(&priv->ring[i].lock); + spin_lock_init(&priv->ring[i].egress_lock); + } + + platform_set_drvdata(pdev, priv); + atomic_set(&priv->ring_used, 0); + + spin_lock_init(&priv->lock); + crypto_init_queue(&priv->queue, EIP197_DEFAULT_RING_SIZE); + + ret = safexcel_hw_init(priv); + if (ret) { + dev_err(dev, "EIP h/w init failed (%d)\n", ret); + goto err_clk; + } + + ret = safexcel_register_algorithms(priv); + if (ret) { + dev_err(dev, "Failed to register algorithms (%d)\n", ret); + goto err_clk; + } + + return 0; + +err_clk: + clk_disable_unprepare(priv->clk); + return ret; +} + + +static int safexcel_remove(struct platform_device *pdev) +{ + struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev); + int i; + + safexcel_unregister_algorithms(priv); + clk_disable_unprepare(priv->clk); + + for (i = 0; i < priv->config.rings; i++) + destroy_workqueue(priv->ring[i].workqueue); + + return 0; +} + +static const struct of_device_id safexcel_of_match_table[] = { + { .compatible = "inside-secure,safexcel-eip197" }, + {}, +}; + + +static struct platform_driver crypto_safexcel = { + .probe = safexcel_probe, + .remove = safexcel_remove, + .driver = { + .name = "crypto-safexcel", + .of_match_table = safexcel_of_match_table, + }, +}; +module_platform_driver(crypto_safexcel); + +MODULE_AUTHOR("Antoine Tenart "); +MODULE_AUTHOR("Ofer Heifetz "); +MODULE_AUTHOR("Igal Liberman "); +MODULE_DESCRIPTION("Support for SafeXcel cryptographic engine EIP197"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h new file mode 100644 index 000000000000..c17fdd40b99f --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel.h @@ -0,0 +1,572 @@ +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + * + * This file is licensed under the terms of the GNU General Public + * License version 2. This program is licensed "as is" without any + * warranty of any kind, whether express or implied. + */ + +#ifndef __SAFEXCEL_H__ +#define __SAFEXCEL_H__ + +#include +#include +#include + +#define EIP197_HIA_VERSION_LE 0xca35 +#define EIP197_HIA_VERSION_BE 0x35ca + +/* Static configuration */ +#define EIP197_DEFAULT_RING_SIZE 64 +#define EIP197_MAX_TOKENS 5 +#define EIP197_MAX_RINGS 4 +#define EIP197_FETCH_COUNT 1 +#define EIP197_MAX_BATCH_SZ 8 + +#define EIP197_GFP_FLAGS(base) ((base).flags & CRYPTO_TFM_REQ_MAY_SLEEP ? \ + GFP_KERNEL : GFP_ATOMIC) + +/* CDR/RDR register offsets */ +#define EIP197_HIA_xDR_OFF(r) (0x80000 + (r) * 0x1000) +#define EIP197_HIA_CDR(r) (EIP197_HIA_xDR_OFF(r)) +#define EIP197_HIA_RDR(r) (EIP197_HIA_xDR_OFF(r) + 0x800) +#define EIP197_HIA_xDR_RING_BASE_ADDR_LO 0x0 +#define EIP197_HIA_xDR_RING_BASE_ADDR_HI 0x4 +#define EIP197_HIA_xDR_RING_SIZE 0x18 +#define EIP197_HIA_xDR_DESC_SIZE 0x1c +#define EIP197_HIA_xDR_CFG 0x20 +#define EIP197_HIA_xDR_DMA_CFG 0x24 +#define EIP197_HIA_xDR_THRESH 0x28 +#define EIP197_HIA_xDR_PREP_COUNT 0x2c +#define EIP197_HIA_xDR_PROC_COUNT 0x30 +#define EIP197_HIA_xDR_PREP_PNTR 0x34 +#define EIP197_HIA_xDR_PROC_PNTR 0x38 +#define EIP197_HIA_xDR_STAT 0x3c + +/* register offsets */ +#define EIP197_HIA_DFE_CFG 0x8c000 +#define EIP197_HIA_DFE_THR_CTRL 0x8c040 +#define EIP197_HIA_DFE_THR_STAT 0x8c044 +#define EIP197_HIA_DSE_CFG 0x8d000 +#define EIP197_HIA_DSE_THR_CTRL 0x8d040 +#define EIP197_HIA_DSE_THR_STAT 0x8d044 +#define EIP197_HIA_RA_PE_CTRL 0x90010 +#define EIP197_HIA_RA_PE_STAT 0x90014 +#define EIP197_HIA_AIC_R_OFF(r) ((r) * 0x1000) +#define EIP197_HIA_AIC_R_ENABLE_CTRL(r) (0x9e808 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_ENABLED_STAT(r) (0x9e810 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_ACK(r) (0x9e810 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_R_ENABLE_CLR(r) (0x9e814 - EIP197_HIA_AIC_R_OFF(r)) +#define EIP197_HIA_AIC_G_ENABLE_CTRL 0x9f808 +#define EIP197_HIA_AIC_G_ENABLED_STAT 0x9f810 +#define EIP197_HIA_AIC_G_ACK 0x9f810 +#define EIP197_HIA_MST_CTRL 0x9fff4 +#define EIP197_HIA_OPTIONS 0x9fff8 +#define EIP197_HIA_VERSION 0x9fffc +#define EIP197_PE_IN_DBUF_THRES 0xa0000 +#define EIP197_PE_IN_TBUF_THRES 0xa0100 +#define EIP197_PE_ICE_SCRATCH_RAM 0xa0800 +#define EIP197_PE_ICE_PUE_CTRL 0xa0c80 +#define EIP197_PE_ICE_SCRATCH_CTRL 0xa0d04 +#define EIP197_PE_ICE_FPP_CTRL 0xa0d80 +#define EIP197_PE_ICE_RAM_CTRL 0xa0ff0 +#define EIP197_PE_EIP96_FUNCTION_EN 0xa1004 +#define EIP197_PE_EIP96_CONTEXT_CTRL 0xa1008 +#define EIP197_PE_EIP96_CONTEXT_STAT 0xa100c +#define EIP197_PE_OUT_DBUF_THRES 0xa1c00 +#define EIP197_PE_OUT_TBUF_THRES 0xa1d00 +#define EIP197_CLASSIFICATION_RAMS 0xe0000 +#define EIP197_TRC_CTRL 0xf0800 +#define EIP197_TRC_LASTRES 0xf0804 +#define EIP197_TRC_REGINDEX 0xf0808 +#define EIP197_TRC_PARAMS 0xf0820 +#define EIP197_TRC_FREECHAIN 0xf0824 +#define EIP197_TRC_PARAMS2 0xf0828 +#define EIP197_TRC_ECCCTRL 0xf0830 +#define EIP197_TRC_ECCSTAT 0xf0834 +#define EIP197_TRC_ECCADMINSTAT 0xf0838 +#define EIP197_TRC_ECCDATASTAT 0xf083c +#define EIP197_TRC_ECCDATA 0xf0840 +#define EIP197_CS_RAM_CTRL 0xf7ff0 +#define EIP197_MST_CTRL 0xffff4 + +/* EIP197_HIA_xDR_DESC_SIZE */ +#define EIP197_xDR_DESC_MODE_64BIT BIT(31) + +/* EIP197_HIA_xDR_DMA_CFG */ +#define EIP197_HIA_xDR_WR_RES_BUF BIT(22) +#define EIP197_HIA_xDR_WR_CTRL_BUG BIT(23) +#define EIP197_HIA_xDR_WR_OWN_BUF BIT(24) +#define EIP197_HIA_xDR_CFG_WR_CACHE(n) (((n) & 0x7) << 23) +#define EIP197_HIA_xDR_CFG_RD_CACHE(n) (((n) & 0x7) << 29) + +/* EIP197_HIA_CDR_THRESH */ +#define EIP197_HIA_CDR_THRESH_PROC_PKT(n) (n) +#define EIP197_HIA_CDR_THRESH_PROC_MODE BIT(22) +#define EIP197_HIA_CDR_THRESH_PKT_MODE BIT(23) +#define EIP197_HIA_CDR_THRESH_TIMEOUT(n) ((n) << 24) /* x256 clk cycles */ + +/* EIP197_HIA_RDR_THRESH */ +#define EIP197_HIA_RDR_THRESH_PROC_PKT(n) (n) +#define EIP197_HIA_RDR_THRESH_PKT_MODE BIT(23) +#define EIP197_HIA_RDR_THRESH_TIMEOUT(n) ((n) << 24) /* x256 clk cycles */ + +/* EIP197_HIA_xDR_PREP_COUNT */ +#define EIP197_xDR_PREP_CLR_COUNT BIT(31) + +/* EIP197_HIA_xDR_PROC_COUNT */ +#define EIP197_xDR_PROC_xD_COUNT(n) ((n) << 2) +#define EIP197_xDR_PROC_xD_PKT(n) ((n) << 24) +#define EIP197_xDR_PROC_CLR_COUNT BIT(31) + +/* EIP197_HIA_xDR_STAT */ +#define EIP197_xDR_DMA_ERR BIT(0) +#define EIP197_xDR_PREP_CMD_THRES BIT(1) +#define EIP197_xDR_ERR BIT(2) +#define EIP197_xDR_THRESH BIT(4) +#define EIP197_xDR_TIMEOUT BIT(5) + +#define EIP197_HIA_RA_PE_CTRL_RESET BIT(31) +#define EIP197_HIA_RA_PE_CTRL_EN BIT(30) + +/* EIP197_HIA_AIC_R_ENABLE_CTRL */ +#define EIP197_CDR_IRQ(n) BIT((n) * 2) +#define EIP197_RDR_IRQ(n) BIT((n) * 2 + 1) + +/* EIP197_HIA_DFE/DSE_CFG */ +#define EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(n) ((n) << 0) +#define EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(n) (((n) & 0x7) << 4) +#define EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(n) ((n) << 8) +#define EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(n) ((n) << 16) +#define EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(n) (((n) & 0x7) << 20) +#define EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(n) ((n) << 24) +#define EIP197_HIA_DFE_CFG_DIS_DEBUG (BIT(31) | BIT(29)) +#define EIP197_HIA_DSE_CFG_DIS_DEBUG BIT(31) + +/* EIP197_HIA_DFE/DSE_THR_CTRL */ +#define EIP197_DxE_THR_CTRL_EN BIT(30) +#define EIP197_DxE_THR_CTRL_RESET_PE BIT(31) + +/* EIP197_HIA_AIC_G_ENABLED_STAT */ +#define EIP197_G_IRQ_DFE(n) BIT((n) << 1) +#define EIP197_G_IRQ_DSE(n) BIT(((n) << 1) + 1) +#define EIP197_G_IRQ_RING BIT(16) +#define EIP197_G_IRQ_PE(n) BIT((n) + 20) + +/* EIP197_HIA_MST_CTRL */ +#define RD_CACHE_3BITS 0x5 +#define WR_CACHE_3BITS 0x3 +#define RD_CACHE_4BITS (RD_CACHE_3BITS << 1 | BIT(0)) +#define WR_CACHE_4BITS (WR_CACHE_3BITS << 1 | BIT(0)) +#define EIP197_MST_CTRL_RD_CACHE(n) (((n) & 0xf) << 0) +#define EIP197_MST_CTRL_WD_CACHE(n) (((n) & 0xf) << 4) +#define EIP197_MST_CTRL_BYTE_SWAP BIT(24) +#define EIP197_MST_CTRL_NO_BYTE_SWAP BIT(25) + +/* EIP197_PE_IN_DBUF/TBUF_THRES */ +#define EIP197_PE_IN_xBUF_THRES_MIN(n) ((n) << 8) +#define EIP197_PE_IN_xBUF_THRES_MAX(n) ((n) << 12) + +/* EIP197_PE_OUT_DBUF_THRES */ +#define EIP197_PE_OUT_DBUF_THRES_MIN(n) ((n) << 0) +#define EIP197_PE_OUT_DBUF_THRES_MAX(n) ((n) << 4) + +/* EIP197_PE_ICE_SCRATCH_CTRL */ +#define EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER BIT(2) +#define EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN BIT(3) +#define EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS BIT(24) +#define EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS BIT(25) + +/* EIP197_PE_ICE_SCRATCH_RAM */ +#define EIP197_NUM_OF_SCRATCH_BLOCKS 32 + +/* EIP197_PE_ICE_PUE/FPP_CTRL */ +#define EIP197_PE_ICE_x_CTRL_SW_RESET BIT(0) +#define EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR BIT(14) +#define EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR BIT(15) + +/* EIP197_PE_ICE_RAM_CTRL */ +#define EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN BIT(0) +#define EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN BIT(1) + +/* EIP197_PE_EIP96_FUNCTION_EN */ +#define EIP197_FUNCTION_RSVD (BIT(6) | BIT(15) | BIT(20) | BIT(23)) +#define EIP197_PROTOCOL_HASH_ONLY BIT(0) +#define EIP197_PROTOCOL_ENCRYPT_ONLY BIT(1) +#define EIP197_PROTOCOL_HASH_ENCRYPT BIT(2) +#define EIP197_PROTOCOL_HASH_DECRYPT BIT(3) +#define EIP197_PROTOCOL_ENCRYPT_HASH BIT(4) +#define EIP197_PROTOCOL_DECRYPT_HASH BIT(5) +#define EIP197_ALG_ARC4 BIT(7) +#define EIP197_ALG_AES_ECB BIT(8) +#define EIP197_ALG_AES_CBC BIT(9) +#define EIP197_ALG_AES_CTR_ICM BIT(10) +#define EIP197_ALG_AES_OFB BIT(11) +#define EIP197_ALG_AES_CFB BIT(12) +#define EIP197_ALG_DES_ECB BIT(13) +#define EIP197_ALG_DES_CBC BIT(14) +#define EIP197_ALG_DES_OFB BIT(16) +#define EIP197_ALG_DES_CFB BIT(17) +#define EIP197_ALG_3DES_ECB BIT(18) +#define EIP197_ALG_3DES_CBC BIT(19) +#define EIP197_ALG_3DES_OFB BIT(21) +#define EIP197_ALG_3DES_CFB BIT(22) +#define EIP197_ALG_MD5 BIT(24) +#define EIP197_ALG_HMAC_MD5 BIT(25) +#define EIP197_ALG_SHA1 BIT(26) +#define EIP197_ALG_HMAC_SHA1 BIT(27) +#define EIP197_ALG_SHA2 BIT(28) +#define EIP197_ALG_HMAC_SHA2 BIT(29) +#define EIP197_ALG_AES_XCBC_MAC BIT(30) +#define EIP197_ALG_GCM_HASH BIT(31) + +/* EIP197_PE_EIP96_CONTEXT_CTRL */ +#define EIP197_CONTEXT_SIZE(n) (n) +#define EIP197_ADDRESS_MODE BIT(8) +#define EIP197_CONTROL_MODE BIT(9) + +/* Context Control */ +struct safexcel_context_record { + u32 control0; + u32 control1; + + __le32 data[12]; +} __packed; + +/* control0 */ +#define CONTEXT_CONTROL_TYPE_NULL_OUT 0x0 +#define CONTEXT_CONTROL_TYPE_NULL_IN 0x1 +#define CONTEXT_CONTROL_TYPE_HASH_OUT 0x2 +#define CONTEXT_CONTROL_TYPE_HASH_IN 0x3 +#define CONTEXT_CONTROL_TYPE_CRYPTO_OUT 0x4 +#define CONTEXT_CONTROL_TYPE_CRYPTO_IN 0x5 +#define CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT 0x6 +#define CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN 0x7 +#define CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT 0x14 +#define CONTEXT_CONTROL_TYPE_HASH_DECRYPT_OUT 0x15 +#define CONTEXT_CONTROL_RESTART_HASH BIT(4) +#define CONTEXT_CONTROL_NO_FINISH_HASH BIT(5) +#define CONTEXT_CONTROL_SIZE(n) ((n) << 8) +#define CONTEXT_CONTROL_KEY_EN BIT(16) +#define CONTEXT_CONTROL_CRYPTO_ALG_AES128 (0x5 << 17) +#define CONTEXT_CONTROL_CRYPTO_ALG_AES192 (0x6 << 17) +#define CONTEXT_CONTROL_CRYPTO_ALG_AES256 (0x7 << 17) +#define CONTEXT_CONTROL_DIGEST_PRECOMPUTED (0x1 << 21) +#define CONTEXT_CONTROL_DIGEST_HMAC (0x3 << 21) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA1 (0x2 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA224 (0x4 << 23) +#define CONTEXT_CONTROL_CRYPTO_ALG_SHA256 (0x3 << 23) +#define CONTEXT_CONTROL_INV_FR (0x5 << 24) +#define CONTEXT_CONTROL_INV_TR (0x6 << 24) + +/* control1 */ +#define CONTEXT_CONTROL_CRYPTO_MODE_ECB (0 << 0) +#define CONTEXT_CONTROL_CRYPTO_MODE_CBC (1 << 0) +#define CONTEXT_CONTROL_IV0 BIT(5) +#define CONTEXT_CONTROL_IV1 BIT(6) +#define CONTEXT_CONTROL_IV2 BIT(7) +#define CONTEXT_CONTROL_IV3 BIT(8) +#define CONTEXT_CONTROL_DIGEST_CNT BIT(9) +#define CONTEXT_CONTROL_COUNTER_MODE BIT(10) +#define CONTEXT_CONTROL_HASH_STORE BIT(19) + +/* EIP197_CS_RAM_CTRL */ +#define EIP197_TRC_ENABLE_0 BIT(4) +#define EIP197_TRC_ENABLE_1 BIT(5) +#define EIP197_TRC_ENABLE_2 BIT(6) +#define EIP197_TRC_ENABLE_MASK GENMASK(6, 4) + +/* EIP197_TRC_PARAMS */ +#define EIP197_TRC_PARAMS_SW_RESET BIT(0) +#define EIP197_TRC_PARAMS_DATA_ACCESS BIT(2) +#define EIP197_TRC_PARAMS_HTABLE_SZ(x) ((x) << 4) +#define EIP197_TRC_PARAMS_BLK_TIMER_SPEED(x) ((x) << 10) +#define EIP197_TRC_PARAMS_RC_SZ_LARGE(n) ((n) << 18) + +/* EIP197_TRC_FREECHAIN */ +#define EIP197_TRC_FREECHAIN_HEAD_PTR(p) (p) +#define EIP197_TRC_FREECHAIN_TAIL_PTR(p) ((p) << 16) + +/* EIP197_TRC_PARAMS2 */ +#define EIP197_TRC_PARAMS2_HTABLE_PTR(p) (p) +#define EIP197_TRC_PARAMS2_RC_SZ_SMALL(n) ((n) << 18) + +/* Cache helpers */ +#define EIP197_CS_RC_MAX 52 +#define EIP197_CS_RC_SIZE (4 * sizeof(u32)) +#define EIP197_CS_RC_NEXT(x) (x) +#define EIP197_CS_RC_PREV(x) ((x) << 10) +#define EIP197_RC_NULL 0x3ff +#define EIP197_CS_TRC_REC_WC 59 +#define EIP197_CS_TRC_LG_REC_WC 73 + +/* Result data */ +struct result_data_desc { + u32 packet_length:17; + u32 error_code:15; + + u8 bypass_length:4; + u8 e15:1; + u16 rsvd0; + u8 hash_bytes:1; + u8 hash_length:6; + u8 generic_bytes:1; + u8 checksum:1; + u8 next_header:1; + u8 length:1; + + u16 application_id; + u16 rsvd1; + + u32 rsvd2; +} __packed; + + +/* Basic Result Descriptor format */ +struct safexcel_result_desc { + u32 particle_size:17; + u8 rsvd0:3; + u8 descriptor_overflow:1; + u8 buffer_overflow:1; + u8 last_seg:1; + u8 first_seg:1; + u16 result_size:8; + + u32 rsvd1; + + u32 data_lo; + u32 data_hi; + + struct result_data_desc result_data; +} __packed; + +struct safexcel_token { + u32 packet_length:17; + u8 stat:2; + u16 instructions:9; + u8 opcode:4; +} __packed; + +#define EIP197_TOKEN_STAT_LAST_HASH BIT(0) +#define EIP197_TOKEN_STAT_LAST_PACKET BIT(1) +#define EIP197_TOKEN_OPCODE_DIRECTION 0x0 +#define EIP197_TOKEN_OPCODE_INSERT 0x2 +#define EIP197_TOKEN_OPCODE_NOOP EIP197_TOKEN_OPCODE_INSERT +#define EIP197_TOKEN_OPCODE_BYPASS GENMASK(3, 0) + +static inline void eip197_noop_token(struct safexcel_token *token) +{ + token->opcode = EIP197_TOKEN_OPCODE_NOOP; + token->packet_length = BIT(2); +} + +/* Instructions */ +#define EIP197_TOKEN_INS_INSERT_HASH_DIGEST 0x1c +#define EIP197_TOKEN_INS_TYPE_OUTPUT BIT(5) +#define EIP197_TOKEN_INS_TYPE_HASH BIT(6) +#define EIP197_TOKEN_INS_TYPE_CRYTO BIT(7) +#define EIP197_TOKEN_INS_LAST BIT(8) + +/* Processing Engine Control Data */ +struct safexcel_control_data_desc { + u32 packet_length:17; + u16 options:13; + u8 type:2; + + u16 application_id; + u16 rsvd; + + u8 refresh:2; + u32 context_lo:30; + u32 context_hi; + + u32 control0; + u32 control1; + + u32 token[EIP197_MAX_TOKENS]; +} __packed; + +#define EIP197_OPTION_MAGIC_VALUE BIT(0) +#define EIP197_OPTION_64BIT_CTX BIT(1) +#define EIP197_OPTION_CTX_CTRL_IN_CMD BIT(8) +#define EIP197_OPTION_4_TOKEN_IV_CMD GENMASK(11, 9) + +#define EIP197_TYPE_EXTENDED 0x3 + +/* Basic Command Descriptor format */ +struct safexcel_command_desc { + u32 particle_size:17; + u8 rsvd0:5; + u8 last_seg:1; + u8 first_seg:1; + u16 additional_cdata_size:8; + + u32 rsvd1; + + u32 data_lo; + u32 data_hi; + + struct safexcel_control_data_desc control_data; +} __packed; + +/* + * Internal structures & functions + */ + +enum eip197_fw { + FW_IFPP = 0, + FW_IPUE, + FW_NB +}; + +struct safexcel_ring { + void *base; + void *base_end; + dma_addr_t base_dma; + + /* write and read pointers */ + void *write; + void *read; + + /* number of elements used in the ring */ + unsigned nr; + unsigned offset; +}; + +enum safexcel_alg_type { + SAFEXCEL_ALG_TYPE_SKCIPHER, + SAFEXCEL_ALG_TYPE_AHASH, +}; + +struct safexcel_request { + struct list_head list; + struct crypto_async_request *req; +}; + +struct safexcel_config { + u32 rings; + + u32 cd_size; + u32 cd_offset; + + u32 rd_size; + u32 rd_offset; +}; + +struct safexcel_work_data { + struct work_struct work; + struct safexcel_crypto_priv *priv; + int ring; +}; + +struct safexcel_crypto_priv { + void __iomem *base; + struct device *dev; + struct clk *clk; + struct safexcel_config config; + + spinlock_t lock; + struct crypto_queue queue; + + bool need_dequeue; + + /* context DMA pool */ + struct dma_pool *context_pool; + + atomic_t ring_used; + + struct { + spinlock_t lock; + spinlock_t egress_lock; + + struct list_head list; + struct workqueue_struct *workqueue; + struct safexcel_work_data work_data; + + /* command/result rings */ + struct safexcel_ring cdr; + struct safexcel_ring rdr; + } ring[EIP197_MAX_RINGS]; +}; + +struct safexcel_context { + int (*send)(struct crypto_async_request *req, int ring, + struct safexcel_request *request, int *commands, + int *results); + int (*handle_result)(struct safexcel_crypto_priv *priv, int ring, + struct crypto_async_request *req, bool *complete, + int *ret); + struct safexcel_context_record *ctxr; + dma_addr_t ctxr_dma; + + int ring; + bool needs_inv; + bool exit_inv; + + /* Used for ahash requests */ + dma_addr_t result_dma; + void *cache; + dma_addr_t cache_dma; + unsigned int cache_sz; +}; + +/* + * Template structure to describe the algorithms in order to register them. + * It also has the purpose to contain our private structure and is actually + * the only way I know in this framework to avoid having global pointers... + */ +struct safexcel_alg_template { + struct safexcel_crypto_priv *priv; + enum safexcel_alg_type type; + union { + struct skcipher_alg skcipher; + struct ahash_alg ahash; + } alg; +}; + +struct safexcel_inv_result { + struct completion completion; + int error; +}; + +void safexcel_dequeue(struct safexcel_crypto_priv *priv); +void safexcel_complete(struct safexcel_crypto_priv *priv, int ring); +void safexcel_free_context(struct safexcel_crypto_priv *priv, + struct crypto_async_request *req, + int result_sz); +int safexcel_invalidate_cache(struct crypto_async_request *async, + struct safexcel_context *ctx, + struct safexcel_crypto_priv *priv, + dma_addr_t ctxr_dma, int ring, + struct safexcel_request *request); +int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv, + struct safexcel_ring *cdr, + struct safexcel_ring *rdr); +int safexcel_select_ring(struct safexcel_crypto_priv *priv); +void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv, + struct safexcel_ring *ring); +void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv, + struct safexcel_ring *ring); +struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv, + int ring_id, + bool first, bool last, + dma_addr_t data, u32 len, + u32 full_data_len, + dma_addr_t context); +struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv, + int ring_id, + bool first, bool last, + dma_addr_t data, u32 len); +void safexcel_inv_complete(struct crypto_async_request *req, int error); + +/* available algorithms */ +extern struct safexcel_alg_template safexcel_alg_ecb_aes; +extern struct safexcel_alg_template safexcel_alg_cbc_aes; +extern struct safexcel_alg_template safexcel_alg_sha1; +extern struct safexcel_alg_template safexcel_alg_sha224; +extern struct safexcel_alg_template safexcel_alg_sha256; +extern struct safexcel_alg_template safexcel_alg_hmac_sha1; + +#endif diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c new file mode 100644 index 000000000000..59e6081602a2 --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -0,0 +1,556 @@ +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + * + * This file is licensed under the terms of the GNU General Public + * License version 2. This program is licensed "as is" without any + * warranty of any kind, whether express or implied. + */ + +#include +#include +#include + +#include +#include + +#include "safexcel.h" + +enum safexcel_cipher_direction { + SAFEXCEL_ENCRYPT, + SAFEXCEL_DECRYPT, +}; + +struct safexcel_cipher_ctx { + struct safexcel_context base; + struct safexcel_crypto_priv *priv; + + enum safexcel_cipher_direction direction; + u32 mode; + + __le32 key[8]; + unsigned int key_len; +}; + +static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, + struct crypto_async_request *async, + struct safexcel_command_desc *cdesc, + u32 length) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_token *token; + unsigned offset = 0; + + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) { + offset = AES_BLOCK_SIZE / sizeof(u32); + memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_SIZE); + + cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; + } + + token = (struct safexcel_token *)(cdesc->control_data.token + offset); + + token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token[0].packet_length = length; + token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET; + token[0].instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_CRYTO | + EIP197_TOKEN_INS_TYPE_OUTPUT; +} + +static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_aes_ctx aes; + int ret, i; + + ret = crypto_aes_expand_key(&aes, key, len); + if (ret) { + crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return ret; + } + + for (i = 0; i < len / sizeof(u32); i++) { + if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + ctx->base.needs_inv = true; + break; + } + } + + for (i = 0; i < len / sizeof(u32); i++) + ctx->key[i] = cpu_to_le32(aes.key_enc[i]); + + ctx->key_len = len; + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, + struct safexcel_command_desc *cdesc) +{ + struct safexcel_crypto_priv *priv = ctx->priv; + int ctrl_size; + + if (ctx->direction == SAFEXCEL_ENCRYPT) + cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT; + else + cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_IN; + + cdesc->control_data.control0 |= CONTEXT_CONTROL_KEY_EN; + cdesc->control_data.control1 |= ctx->mode; + + switch (ctx->key_len) { + case AES_KEYSIZE_128: + cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES128; + ctrl_size = 4; + break; + case AES_KEYSIZE_192: + cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES192; + ctrl_size = 6; + break; + case AES_KEYSIZE_256: + cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES256; + ctrl_size = 8; + break; + default: + dev_err(priv->dev, "aes keysize not supported: %u\n", + ctx->key_len); + return -EINVAL; + } + cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(ctrl_size); + + return 0; +} + +static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_result_desc *rdesc; + int ndesc = 0; + + *ret = 0; + + spin_lock_bh(&priv->ring[ring].egress_lock); + do { + rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); + if (IS_ERR(rdesc)) { + dev_err(priv->dev, + "cipher: result: could not retrieve the result descriptor\n"); + *ret = PTR_ERR(rdesc); + break; + } + + if (rdesc->result_data.error_code) { + dev_err(priv->dev, + "cipher: result: result descriptor error (%d)\n", + rdesc->result_data.error_code); + *ret = -EIO; + } + + ndesc++; + } while (!rdesc->last_seg); + + safexcel_complete(priv, ring); + spin_unlock_bh(&priv->ring[ring].egress_lock); + + if (req->src == req->dst) { + dma_unmap_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, req->dst, + sg_nents_for_len(req->dst, req->cryptlen), + DMA_FROM_DEVICE); + } + + *should_complete = true; + + return ndesc; +} + +static int safexcel_aes_send(struct crypto_async_request *async, + int ring, struct safexcel_request *request, + int *commands, int *results) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_command_desc *cdesc; + struct safexcel_result_desc *rdesc; + struct scatterlist *sg; + int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = req->cryptlen; + int i, ret = 0; + + request->req = &req->base; + + if (req->src == req->dst) { + nr_src = dma_map_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_BIDIRECTIONAL); + nr_dst = nr_src; + if (!nr_src) + return -EINVAL; + } else { + nr_src = dma_map_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_TO_DEVICE); + if (!nr_src) + return -EINVAL; + + nr_dst = dma_map_sg(priv->dev, req->dst, + sg_nents_for_len(req->dst, req->cryptlen), + DMA_FROM_DEVICE); + if (!nr_dst) { + dma_unmap_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_TO_DEVICE); + return -EINVAL; + } + } + + memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); + + spin_lock_bh(&priv->ring[ring].egress_lock); + + /* command descriptors */ + for_each_sg(req->src, sg, nr_src, i) { + int len = sg_dma_len(sg); + + /* Do not overflow the request */ + if (queued - len < 0) + len = queued; + + cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, !(queued - len), + sg_dma_address(sg), len, req->cryptlen, + ctx->base.ctxr_dma); + if (IS_ERR(cdesc)) { + /* No space left in the command descriptor ring */ + ret = PTR_ERR(cdesc); + goto cdesc_rollback; + } + n_cdesc++; + + if (n_cdesc == 1) { + safexcel_context_control(ctx, cdesc); + safexcel_cipher_token(ctx, async, cdesc, req->cryptlen); + } + + queued -= len; + if (!queued) + break; + } + + /* result descriptors */ + for_each_sg(req->dst, sg, nr_dst, i) { + bool first = !i, last = (i == nr_dst - 1); + u32 len = sg_dma_len(sg); + + rdesc = safexcel_add_rdesc(priv, ring, first, last, + sg_dma_address(sg), len); + if (IS_ERR(rdesc)) { + /* No space left in the result descriptor ring */ + ret = PTR_ERR(rdesc); + goto rdesc_rollback; + } + n_rdesc++; + } + + ctx->base.handle_result = safexcel_handle_result; + + spin_unlock_bh(&priv->ring[ring].egress_lock); + + *commands = n_cdesc; + *results = nr_dst; + return 0; + +rdesc_rollback: + for (i = 0; i < n_rdesc; i++) + safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr); +cdesc_rollback: + for (i = 0; i < n_cdesc; i++) + safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); + + spin_unlock_bh(&priv->ring[ring].egress_lock); + + if (req->src == req->dst) { + dma_unmap_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, req->dst, + sg_nents_for_len(req->dst, req->cryptlen), + DMA_FROM_DEVICE); + } + + return ret; +} + +static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, + int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct safexcel_result_desc *rdesc; + int ndesc = 0, enq_ret; + + *ret = 0; + + spin_lock_bh(&priv->ring[ring].egress_lock); + do { + rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); + if (IS_ERR(rdesc)) { + dev_err(priv->dev, + "cipher: invalidate: could not retrieve the result descriptor\n"); + *ret = PTR_ERR(rdesc); + break; + } + + if (rdesc->result_data.error_code) { + dev_err(priv->dev, "cipher: invalidate: result descriptor error (%d)\n", + rdesc->result_data.error_code); + *ret = -EIO; + } + + ndesc++; + } while (!rdesc->last_seg); + + safexcel_complete(priv, ring); + spin_unlock_bh(&priv->ring[ring].egress_lock); + + if (ctx->base.exit_inv) { + dma_pool_free(priv->context_pool, ctx->base.ctxr, + ctx->base.ctxr_dma); + + *should_complete = true; + + return ndesc; + } + + ctx->base.needs_inv = false; + ctx->base.ring = safexcel_select_ring(priv); + ctx->base.send = safexcel_aes_send; + + spin_lock_bh(&priv->lock); + enq_ret = crypto_enqueue_request(&priv->queue, async); + spin_unlock_bh(&priv->lock); + + if (enq_ret != -EINPROGRESS) + *ret = enq_ret; + + priv->need_dequeue = true; + *should_complete = false; + + return ndesc; +} + +static int safexcel_cipher_send_inv(struct crypto_async_request *async, + int ring, struct safexcel_request *request, + int *commands, int *results) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + int ret; + + ctx->base.handle_result = safexcel_handle_inv_result; + + ret = safexcel_invalidate_cache(async, &ctx->base, priv, + ctx->base.ctxr_dma, ring, request); + if (unlikely(ret)) + return ret; + + *commands = 1; + *results = 1; + + return 0; +} + +static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + struct skcipher_request req; + struct safexcel_inv_result result = { 0 }; + + memset(&req, 0, sizeof(struct skcipher_request)); + + /* create invalidation request */ + init_completion(&result.completion); + skcipher_request_set_callback(&req, CRYPTO_TFM_REQ_MAY_BACKLOG, + safexcel_inv_complete, &result); + + skcipher_request_set_tfm(&req, __crypto_skcipher_cast(tfm)); + ctx = crypto_tfm_ctx(req.base.tfm); + ctx->base.exit_inv = true; + ctx->base.send = safexcel_cipher_send_inv; + + spin_lock_bh(&priv->lock); + crypto_enqueue_request(&priv->queue, &req.base); + spin_unlock_bh(&priv->lock); + + if (!priv->need_dequeue) + safexcel_dequeue(priv); + + wait_for_completion_interruptible(&result.completion); + + if (result.error) { + dev_warn(priv->dev, + "cipher: sync: invalidate: completion error %d\n", + result.error); + return result.error; + } + + return 0; +} + +static int safexcel_aes(struct skcipher_request *req, + enum safexcel_cipher_direction dir, u32 mode) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + int ret; + + ctx->direction = dir; + ctx->mode = mode; + + if (ctx->base.ctxr) { + if (ctx->base.needs_inv) + ctx->base.send = safexcel_cipher_send_inv; + } else { + ctx->base.ring = safexcel_select_ring(priv); + ctx->base.send = safexcel_aes_send; + + ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, + EIP197_GFP_FLAGS(req->base), + &ctx->base.ctxr_dma); + if (!ctx->base.ctxr) + return -ENOMEM; + } + + spin_lock_bh(&priv->lock); + ret = crypto_enqueue_request(&priv->queue, &req->base); + spin_unlock_bh(&priv->lock); + + if (!priv->need_dequeue) + safexcel_dequeue(priv); + + return ret; +} + +static int safexcel_ecb_aes_encrypt(struct skcipher_request *req) +{ + return safexcel_aes(req, SAFEXCEL_ENCRYPT, + CONTEXT_CONTROL_CRYPTO_MODE_ECB); +} + +static int safexcel_ecb_aes_decrypt(struct skcipher_request *req) +{ + return safexcel_aes(req, SAFEXCEL_DECRYPT, + CONTEXT_CONTROL_CRYPTO_MODE_ECB); +} + +static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_alg_template *tmpl = + container_of(tfm->__crt_alg, struct safexcel_alg_template, + alg.skcipher.base); + + ctx->priv = tmpl->priv; + + return 0; +} + +static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + int ret; + + memzero_explicit(ctx->key, 8 * sizeof(u32)); + + /* context not allocated, skip invalidation */ + if (!ctx->base.ctxr) + return; + + memzero_explicit(ctx->base.ctxr->data, 8 * sizeof(u32)); + + ret = safexcel_cipher_exit_inv(tfm); + if (ret) + dev_warn(priv->dev, "cipher: invalidation error %d\n", ret); +} + +struct safexcel_alg_template safexcel_alg_ecb_aes = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .setkey = safexcel_aes_setkey, + .encrypt = safexcel_ecb_aes_encrypt, + .decrypt = safexcel_ecb_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "safexcel-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_cbc_aes_encrypt(struct skcipher_request *req) +{ + return safexcel_aes(req, SAFEXCEL_ENCRYPT, + CONTEXT_CONTROL_CRYPTO_MODE_CBC); +} + +static int safexcel_cbc_aes_decrypt(struct skcipher_request *req) +{ + return safexcel_aes(req, SAFEXCEL_DECRYPT, + CONTEXT_CONTROL_CRYPTO_MODE_CBC); +} + +struct safexcel_alg_template safexcel_alg_cbc_aes = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .setkey = safexcel_aes_setkey, + .encrypt = safexcel_cbc_aes_encrypt, + .decrypt = safexcel_cbc_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "safexcel-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c new file mode 100644 index 000000000000..060ea034c9da --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -0,0 +1,1045 @@ +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + * + * This file is licensed under the terms of the GNU General Public + * License version 2. This program is licensed "as is" without any + * warranty of any kind, whether express or implied. + */ + +#include +#include +#include +#include + + +#include "safexcel.h" + +struct safexcel_ahash_ctx { + struct safexcel_context base; + struct safexcel_crypto_priv *priv; + + u32 alg; + u32 digest; + + u32 ipad[SHA1_DIGEST_SIZE / sizeof(u32)]; + u32 opad[SHA1_DIGEST_SIZE / sizeof(u32)]; +}; + +struct safexcel_ahash_req { + bool last_req; + bool finish; + bool hmac; + + u8 state_sz; /* expected sate size, only set once */ + u32 state[SHA256_DIGEST_SIZE / sizeof(u32)]; + + u64 len; + u64 processed; + + u8 cache[SHA256_BLOCK_SIZE] __aligned(sizeof(u32)); + u8 cache_next[SHA256_BLOCK_SIZE] __aligned(sizeof(u32)); +}; + +struct safexcel_ahash_export_state { + u64 len; + u64 processed; + + u32 state[SHA256_DIGEST_SIZE / sizeof(u32)]; + u8 cache[SHA256_BLOCK_SIZE]; +}; + +static void safexcel_hash_token(struct safexcel_command_desc *cdesc, + u32 input_length, u32 result_length) +{ + struct safexcel_token *token = + (struct safexcel_token *)cdesc->control_data.token; + + token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token[0].packet_length = input_length; + token[0].stat = EIP197_TOKEN_STAT_LAST_HASH; + token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH; + + token[1].opcode = EIP197_TOKEN_OPCODE_INSERT; + token[1].packet_length = result_length; + token[1].stat = EIP197_TOKEN_STAT_LAST_HASH | + EIP197_TOKEN_STAT_LAST_PACKET; + token[1].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | + EIP197_TOKEN_INS_INSERT_HASH_DIGEST; +} + +static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, + struct safexcel_ahash_req *req, + struct safexcel_command_desc *cdesc, + unsigned int digestsize, + unsigned int blocksize) +{ + int i; + + cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_OUT; + cdesc->control_data.control0 |= ctx->alg; + cdesc->control_data.control0 |= ctx->digest; + + if (ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) { + if (req->processed) { + if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1) + cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(6); + else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224 || + ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256) + cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(9); + + cdesc->control_data.control1 |= CONTEXT_CONTROL_DIGEST_CNT; + } else { + cdesc->control_data.control0 |= CONTEXT_CONTROL_RESTART_HASH; + } + + if (!req->finish) + cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH; + + /* + * Copy the input digest if needed, and setup the context + * fields. Do this now as we need it to setup the first command + * descriptor. + */ + if (req->processed) { + for (i = 0; i < digestsize / sizeof(u32); i++) + ctx->base.ctxr->data[i] = cpu_to_le32(req->state[i]); + + if (req->finish) + ctx->base.ctxr->data[i] = cpu_to_le32(req->processed / blocksize); + } + } else if (ctx->digest == CONTEXT_CONTROL_DIGEST_HMAC) { + cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(10); + + memcpy(ctx->base.ctxr->data, ctx->ipad, digestsize); + memcpy(ctx->base.ctxr->data + digestsize / sizeof(u32), + ctx->opad, digestsize); + } +} + +static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct safexcel_result_desc *rdesc; + struct ahash_request *areq = ahash_request_cast(async); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_req *sreq = ahash_request_ctx(areq); + int cache_len, result_sz = sreq->state_sz; + + *ret = 0; + + spin_lock_bh(&priv->ring[ring].egress_lock); + rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); + if (IS_ERR(rdesc)) { + dev_err(priv->dev, + "hash: result: could not retrieve the result descriptor\n"); + *ret = PTR_ERR(rdesc); + } else if (rdesc->result_data.error_code) { + dev_err(priv->dev, + "hash: result: result descriptor error (%d)\n", + rdesc->result_data.error_code); + *ret = -EINVAL; + } + + safexcel_complete(priv, ring); + spin_unlock_bh(&priv->ring[ring].egress_lock); + + if (sreq->finish) + result_sz = crypto_ahash_digestsize(ahash); + memcpy(sreq->state, areq->result, result_sz); + + dma_unmap_sg(priv->dev, areq->src, + sg_nents_for_len(areq->src, areq->nbytes), DMA_TO_DEVICE); + + safexcel_free_context(priv, async, sreq->state_sz); + + cache_len = sreq->len - sreq->processed; + if (cache_len) + memcpy(sreq->cache, sreq->cache_next, cache_len); + + *should_complete = true; + + return 1; +} + +static int safexcel_ahash_send(struct crypto_async_request *async, int ring, + struct safexcel_request *request, int *commands, + int *results) +{ + struct ahash_request *areq = ahash_request_cast(async); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_command_desc *cdesc, *first_cdesc = NULL; + struct safexcel_result_desc *rdesc; + struct scatterlist *sg; + int i, nents, queued, len, cache_len, extra, n_cdesc = 0, ret = 0; + + queued = len = req->len - req->processed; + if (queued < crypto_ahash_blocksize(ahash)) + cache_len = queued; + else + cache_len = queued - areq->nbytes; + + /* + * If this is not the last request and the queued data does not fit + * into full blocks, cache it for the next send() call. + */ + extra = queued & (crypto_ahash_blocksize(ahash) - 1); + if (!req->last_req && extra) { + sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + req->cache_next, extra, areq->nbytes - extra); + + queued -= extra; + len -= extra; + } + + request->req = &areq->base; + ctx->base.handle_result = safexcel_handle_result; + + spin_lock_bh(&priv->ring[ring].egress_lock); + + /* Add a command descriptor for the cached data, if any */ + if (cache_len) { + ctx->base.cache = kzalloc(cache_len, EIP197_GFP_FLAGS(*async)); + if (!ctx->base.cache) { + ret = -ENOMEM; + goto unlock; + } + memcpy(ctx->base.cache, req->cache, cache_len); + ctx->base.cache_dma = dma_map_single(priv->dev, ctx->base.cache, + cache_len, DMA_TO_DEVICE); + if (dma_mapping_error(priv->dev, ctx->base.cache_dma)) { + ret = -EINVAL; + goto free_cache; + } + + ctx->base.cache_sz = cache_len; + first_cdesc = safexcel_add_cdesc(priv, ring, 1, + (cache_len == len), + ctx->base.cache_dma, + cache_len, len, + ctx->base.ctxr_dma); + if (IS_ERR(first_cdesc)) { + ret = PTR_ERR(first_cdesc); + goto unmap_cache; + } + n_cdesc++; + + queued -= cache_len; + if (!queued) + goto send_command; + } + + /* Now handle the current ahash request buffer(s) */ + nents = dma_map_sg(priv->dev, areq->src, + sg_nents_for_len(areq->src, areq->nbytes), + DMA_TO_DEVICE); + if (!nents) { + ret = -ENOMEM; + goto cdesc_rollback; + } + + for_each_sg(areq->src, sg, nents, i) { + int sglen = sg_dma_len(sg); + + /* Do not overflow the request */ + if (queued - sglen < 0) + sglen = queued; + + cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, + !(queued - sglen), sg_dma_address(sg), + sglen, len, ctx->base.ctxr_dma); + if (IS_ERR(cdesc)) { + ret = PTR_ERR(cdesc); + goto cdesc_rollback; + } + n_cdesc++; + + if (n_cdesc == 1) + first_cdesc = cdesc; + + queued -= sglen; + if (!queued) + break; + } + +send_command: + /* Setup the context options */ + safexcel_context_control(ctx, req, first_cdesc, req->state_sz, + crypto_ahash_blocksize(ahash)); + + /* Add the token */ + safexcel_hash_token(first_cdesc, len, req->state_sz); + + ctx->base.result_dma = dma_map_single(priv->dev, areq->result, + req->state_sz, DMA_FROM_DEVICE); + if (dma_mapping_error(priv->dev, ctx->base.result_dma)) { + ret = -EINVAL; + goto cdesc_rollback; + } + + /* Add a result descriptor */ + rdesc = safexcel_add_rdesc(priv, ring, 1, 1, ctx->base.result_dma, + req->state_sz); + if (IS_ERR(rdesc)) { + ret = PTR_ERR(rdesc); + goto cdesc_rollback; + } + + req->processed += len; + spin_unlock_bh(&priv->ring[ring].egress_lock); + + *commands = n_cdesc; + *results = 1; + return 0; + +cdesc_rollback: + for (i = 0; i < n_cdesc; i++) + safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); +unmap_cache: + if (ctx->base.cache_dma) { + dma_unmap_single(priv->dev, ctx->base.cache_dma, + ctx->base.cache_sz, DMA_TO_DEVICE); + ctx->base.cache_sz = 0; + } +free_cache: + if (ctx->base.cache) { + kfree(ctx->base.cache); + ctx->base.cache = NULL; + } + +unlock: + spin_unlock_bh(&priv->ring[ring].egress_lock); + return ret; +} + +static inline bool safexcel_ahash_needs_inv_get(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + unsigned int state_w_sz = req->state_sz / sizeof(u32); + int i; + + for (i = 0; i < state_w_sz; i++) + if (ctx->base.ctxr->data[i] != cpu_to_le32(req->state[i])) + return true; + + if (ctx->base.ctxr->data[state_w_sz] != + cpu_to_le32(req->processed / crypto_ahash_blocksize(ahash))) + return true; + + return false; +} + +static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, + int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct safexcel_result_desc *rdesc; + struct ahash_request *areq = ahash_request_cast(async); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); + int enq_ret; + + *ret = 0; + + spin_lock_bh(&priv->ring[ring].egress_lock); + rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); + if (IS_ERR(rdesc)) { + dev_err(priv->dev, + "hash: invalidate: could not retrieve the result descriptor\n"); + *ret = PTR_ERR(rdesc); + } else if (rdesc->result_data.error_code) { + dev_err(priv->dev, + "hash: invalidate: result descriptor error (%d)\n", + rdesc->result_data.error_code); + *ret = -EINVAL; + } + + safexcel_complete(priv, ring); + spin_unlock_bh(&priv->ring[ring].egress_lock); + + if (ctx->base.exit_inv) { + dma_pool_free(priv->context_pool, ctx->base.ctxr, + ctx->base.ctxr_dma); + + *should_complete = true; + return 1; + } + + ctx->base.ring = safexcel_select_ring(priv); + ctx->base.needs_inv = false; + ctx->base.send = safexcel_ahash_send; + + spin_lock_bh(&priv->lock); + enq_ret = crypto_enqueue_request(&priv->queue, async); + spin_unlock_bh(&priv->lock); + + if (enq_ret != -EINPROGRESS) + *ret = enq_ret; + + priv->need_dequeue = true; + *should_complete = false; + + return 1; +} + +static int safexcel_ahash_send_inv(struct crypto_async_request *async, + int ring, struct safexcel_request *request, + int *commands, int *results) +{ + struct ahash_request *areq = ahash_request_cast(async); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + int ret; + + ctx->base.handle_result = safexcel_handle_inv_result; + ret = safexcel_invalidate_cache(async, &ctx->base, ctx->priv, + ctx->base.ctxr_dma, ring, request); + if (unlikely(ret)) + return ret; + + *commands = 1; + *results = 1; + + return 0; +} + +static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + struct ahash_request req; + struct safexcel_inv_result result = { 0 }; + + memset(&req, 0, sizeof(struct ahash_request)); + + /* create invalidation request */ + init_completion(&result.completion); + ahash_request_set_callback(&req, CRYPTO_TFM_REQ_MAY_BACKLOG, + safexcel_inv_complete, &result); + + ahash_request_set_tfm(&req, __crypto_ahash_cast(tfm)); + ctx = crypto_tfm_ctx(req.base.tfm); + ctx->base.exit_inv = true; + ctx->base.send = safexcel_ahash_send_inv; + + spin_lock_bh(&priv->lock); + crypto_enqueue_request(&priv->queue, &req.base); + spin_unlock_bh(&priv->lock); + + if (!priv->need_dequeue) + safexcel_dequeue(priv); + + wait_for_completion_interruptible(&result.completion); + + if (result.error) { + dev_warn(priv->dev, "hash: completion error (%d)\n", + result.error); + return result.error; + } + + return 0; +} + +static int safexcel_ahash_cache(struct ahash_request *areq) +{ + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + int queued, cache_len; + + cache_len = req->len - areq->nbytes - req->processed; + queued = req->len - req->processed; + + /* + * In case there isn't enough bytes to proceed (less than a + * block size), cache the data until we have enough. + */ + if (cache_len + areq->nbytes <= crypto_ahash_blocksize(ahash)) { + sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + req->cache + cache_len, + areq->nbytes, 0); + return areq->nbytes; + } + + /* We could'nt cache all the data */ + return -E2BIG; +} + +static int safexcel_ahash_enqueue(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + struct safexcel_crypto_priv *priv = ctx->priv; + int ret; + + ctx->base.send = safexcel_ahash_send; + + if (req->processed && ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) + ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq); + + if (ctx->base.ctxr) { + if (ctx->base.needs_inv) + ctx->base.send = safexcel_ahash_send_inv; + } else { + ctx->base.ring = safexcel_select_ring(priv); + ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, + EIP197_GFP_FLAGS(areq->base), + &ctx->base.ctxr_dma); + if (!ctx->base.ctxr) + return -ENOMEM; + } + + spin_lock_bh(&priv->lock); + ret = crypto_enqueue_request(&priv->queue, &areq->base); + spin_unlock_bh(&priv->lock); + + if (!priv->need_dequeue) + safexcel_dequeue(priv); + + return ret; +} + +static int safexcel_ahash_update(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + + /* If the request is 0 length, do nothing */ + if (!areq->nbytes) + return 0; + + req->len += areq->nbytes; + + safexcel_ahash_cache(areq); + + /* + * We're not doing partial updates when performing an hmac request. + * Everything will be handled by the final() call. + */ + if (ctx->digest == CONTEXT_CONTROL_DIGEST_HMAC) + return 0; + + if (req->hmac) + return safexcel_ahash_enqueue(areq); + + if (!req->last_req && + req->len - req->processed > crypto_ahash_blocksize(ahash)) + return safexcel_ahash_enqueue(areq); + + return 0; +} + +static int safexcel_ahash_final(struct ahash_request *areq) +{ + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + + req->last_req = true; + req->finish = true; + + /* If we have an overall 0 length request */ + if (!(req->len + areq->nbytes)) { + if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1) + memcpy(areq->result, sha1_zero_message_hash, + SHA1_DIGEST_SIZE); + else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224) + memcpy(areq->result, sha224_zero_message_hash, + SHA224_DIGEST_SIZE); + else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256) + memcpy(areq->result, sha256_zero_message_hash, + SHA256_DIGEST_SIZE); + + return 0; + } + + return safexcel_ahash_enqueue(areq); +} + +static int safexcel_ahash_finup(struct ahash_request *areq) +{ + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + req->last_req = true; + req->finish = true; + + safexcel_ahash_update(areq); + return safexcel_ahash_final(areq); +} + +static int safexcel_ahash_export(struct ahash_request *areq, void *out) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + struct safexcel_ahash_export_state *export = out; + + export->len = req->len; + export->processed = req->processed; + + memcpy(export->state, req->state, req->state_sz); + memset(export->cache, 0, crypto_ahash_blocksize(ahash)); + memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash)); + + return 0; +} + +static int safexcel_ahash_import(struct ahash_request *areq, const void *in) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + const struct safexcel_ahash_export_state *export = in; + int ret; + + ret = crypto_ahash_init(areq); + if (ret) + return ret; + + req->len = export->len; + req->processed = export->processed; + + memcpy(req->cache, export->cache, crypto_ahash_blocksize(ahash)); + memcpy(req->state, export->state, req->state_sz); + + return 0; +} + +static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_alg_template *tmpl = + container_of(__crypto_ahash_alg(tfm->__crt_alg), + struct safexcel_alg_template, alg.ahash); + + ctx->priv = tmpl->priv; + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct safexcel_ahash_req)); + return 0; +} + +static int safexcel_sha1_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + req->state[0] = SHA1_H0; + req->state[1] = SHA1_H1; + req->state[2] = SHA1_H2; + req->state[3] = SHA1_H3; + req->state[4] = SHA1_H4; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; + ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA1_DIGEST_SIZE; + + return 0; +} + +static int safexcel_sha1_digest(struct ahash_request *areq) +{ + int ret = safexcel_sha1_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + int ret; + + /* context not allocated, skip invalidation */ + if (!ctx->base.ctxr) + return; + + ret = safexcel_ahash_exit_inv(tfm); + if (ret) + dev_warn(priv->dev, "hash: invalidation error %d\n", ret); +} + +struct safexcel_alg_template safexcel_alg_sha1 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .alg.ahash = { + .init = safexcel_sha1_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_sha1_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "safexcel-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_hmac_sha1_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + + safexcel_sha1_init(areq); + ctx->digest = CONTEXT_CONTROL_DIGEST_HMAC; + return 0; +} + +static int safexcel_hmac_sha1_digest(struct ahash_request *areq) +{ + int ret = safexcel_hmac_sha1_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_ahash_result { + struct completion completion; + int error; +}; + +static void safexcel_ahash_complete(struct crypto_async_request *req, int error) +{ + struct safexcel_ahash_result *result = req->data; + + if (error == -EINPROGRESS) + return; + + result->error = error; + complete(&result->completion); +} + +static int safexcel_hmac_init_pad(struct ahash_request *areq, + unsigned int blocksize, const u8 *key, + unsigned int keylen, u8 *ipad, u8 *opad) +{ + struct safexcel_ahash_result result; + struct scatterlist sg; + int ret, i; + u8 *keydup; + + if (keylen <= blocksize) { + memcpy(ipad, key, keylen); + } else { + keydup = kmemdup(key, keylen, GFP_KERNEL); + if (!keydup) + return -ENOMEM; + + ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, + safexcel_ahash_complete, &result); + sg_init_one(&sg, keydup, keylen); + ahash_request_set_crypt(areq, &sg, ipad, keylen); + init_completion(&result.completion); + + ret = crypto_ahash_digest(areq); + if (ret == -EINPROGRESS) { + wait_for_completion_interruptible(&result.completion); + ret = result.error; + } + + /* Avoid leaking */ + memzero_explicit(keydup, keylen); + kfree(keydup); + + if (ret) + return ret; + + keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); + } + + memset(ipad + keylen, 0, blocksize - keylen); + memcpy(opad, ipad, blocksize); + + for (i = 0; i < blocksize; i++) { + ipad[i] ^= 0x36; + opad[i] ^= 0x5c; + } + + return 0; +} + +static int safexcel_hmac_init_iv(struct ahash_request *areq, + unsigned int blocksize, u8 *pad, void *state) +{ + struct safexcel_ahash_result result; + struct safexcel_ahash_req *req; + struct scatterlist sg; + int ret; + + ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, + safexcel_ahash_complete, &result); + sg_init_one(&sg, pad, blocksize); + ahash_request_set_crypt(areq, &sg, pad, blocksize); + init_completion(&result.completion); + + ret = crypto_ahash_init(areq); + if (ret) + return ret; + + req = ahash_request_ctx(areq); + req->hmac = true; + req->last_req = true; + + ret = crypto_ahash_update(areq); + if (ret && ret != -EINPROGRESS) + return ret; + + wait_for_completion_interruptible(&result.completion); + if (result.error) + return result.error; + + return crypto_ahash_export(areq, state); +} + +static int safexcel_hmac_setkey(const char *alg, const u8 *key, + unsigned int keylen, void *istate, void *ostate) +{ + struct ahash_request *areq; + struct crypto_ahash *tfm; + unsigned int blocksize; + u8 *ipad, *opad; + int ret; + + tfm = crypto_alloc_ahash(alg, CRYPTO_ALG_TYPE_AHASH, + CRYPTO_ALG_TYPE_AHASH_MASK); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + areq = ahash_request_alloc(tfm, GFP_KERNEL); + if (!areq) { + ret = -ENOMEM; + goto free_ahash; + } + + crypto_ahash_clear_flags(tfm, ~0); + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + ipad = kzalloc(2 * blocksize, GFP_KERNEL); + if (!ipad) { + ret = -ENOMEM; + goto free_request; + } + + opad = ipad + blocksize; + + ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad); + if (ret) + goto free_ipad; + + ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate); + if (ret) + goto free_ipad; + + ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate); + +free_ipad: + kfree(ipad); +free_request: + ahash_request_free(areq); +free_ahash: + crypto_free_ahash(tfm); + + return ret; +} + +static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct safexcel_ahash_export_state istate, ostate; + int ret, i; + + ret = safexcel_hmac_setkey("safexcel-sha1", key, keylen, &istate, &ostate); + if (ret) + return ret; + + memcpy(ctx->ipad, &istate.state, SHA1_DIGEST_SIZE); + memcpy(ctx->opad, &ostate.state, SHA1_DIGEST_SIZE); + + for (i = 0; i < ARRAY_SIZE(istate.state); i++) { + if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) || + ctx->opad[i] != le32_to_cpu(ostate.state[i])) { + ctx->base.needs_inv = true; + break; + } + } + + return 0; +} + +struct safexcel_alg_template safexcel_alg_hmac_sha1 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .alg.ahash = { + .init = safexcel_hmac_sha1_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_hmac_sha1_digest, + .setkey = safexcel_hmac_sha1_setkey, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "safexcel-hmac-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha256_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + req->state[0] = SHA256_H0; + req->state[1] = SHA256_H1; + req->state[2] = SHA256_H2; + req->state[3] = SHA256_H3; + req->state[4] = SHA256_H4; + req->state[5] = SHA256_H5; + req->state[6] = SHA256_H6; + req->state[7] = SHA256_H7; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; + ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA256_DIGEST_SIZE; + + return 0; +} + +static int safexcel_sha256_digest(struct ahash_request *areq) +{ + int ret = safexcel_sha256_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_sha256 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .alg.ahash = { + .init = safexcel_sha256_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_sha256_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "safexcel-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; + +static int safexcel_sha224_init(struct ahash_request *areq) +{ + struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); + struct safexcel_ahash_req *req = ahash_request_ctx(areq); + + memset(req, 0, sizeof(*req)); + + req->state[0] = SHA224_H0; + req->state[1] = SHA224_H1; + req->state[2] = SHA224_H2; + req->state[3] = SHA224_H3; + req->state[4] = SHA224_H4; + req->state[5] = SHA224_H5; + req->state[6] = SHA224_H6; + req->state[7] = SHA224_H7; + + ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; + ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; + req->state_sz = SHA256_DIGEST_SIZE; + + return 0; +} + +static int safexcel_sha224_digest(struct ahash_request *areq) +{ + int ret = safexcel_sha224_init(areq); + + if (ret) + return ret; + + return safexcel_ahash_finup(areq); +} + +struct safexcel_alg_template safexcel_alg_sha224 = { + .type = SAFEXCEL_ALG_TYPE_AHASH, + .alg.ahash = { + .init = safexcel_sha224_init, + .update = safexcel_ahash_update, + .final = safexcel_ahash_final, + .finup = safexcel_ahash_finup, + .digest = safexcel_sha224_digest, + .export = safexcel_ahash_export, + .import = safexcel_ahash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct safexcel_ahash_export_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "safexcel-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), + .cra_init = safexcel_ahash_cra_init, + .cra_exit = safexcel_ahash_cra_exit, + .cra_module = THIS_MODULE, + }, + }, + }, +}; diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c new file mode 100644 index 000000000000..fdbf05ae55fc --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel_ring.c @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + * + * This file is licensed under the terms of the GNU General Public + * License version 2. This program is licensed "as is" without any + * warranty of any kind, whether express or implied. + */ + +#include +#include + +#include "safexcel.h" + +int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv, + struct safexcel_ring *cdr, + struct safexcel_ring *rdr) +{ + cdr->offset = sizeof(u32) * priv->config.cd_offset; + cdr->base = dmam_alloc_coherent(priv->dev, + cdr->offset * EIP197_DEFAULT_RING_SIZE, + &cdr->base_dma, GFP_KERNEL); + if (!cdr->base) + return -ENOMEM; + cdr->write = cdr->base; + cdr->base_end = cdr->base + cdr->offset * EIP197_DEFAULT_RING_SIZE; + cdr->read = cdr->base; + + rdr->offset = sizeof(u32) * priv->config.rd_offset; + rdr->base = dmam_alloc_coherent(priv->dev, + rdr->offset * EIP197_DEFAULT_RING_SIZE, + &rdr->base_dma, GFP_KERNEL); + if (!rdr->base) + return -ENOMEM; + rdr->write = rdr->base; + rdr->base_end = rdr->base + rdr->offset * EIP197_DEFAULT_RING_SIZE; + rdr->read = rdr->base; + + return 0; +} + +inline int safexcel_select_ring(struct safexcel_crypto_priv *priv) +{ + return (atomic_inc_return(&priv->ring_used) % priv->config.rings); +} + +static void *safexcel_ring_next_wptr(struct safexcel_crypto_priv *priv, + struct safexcel_ring *ring) +{ + void *ptr = ring->write; + + if (ring->nr == EIP197_DEFAULT_RING_SIZE - 1) + return ERR_PTR(-ENOMEM); + + ring->write += ring->offset; + if (ring->write == ring->base_end) + ring->write = ring->base; + + ring->nr++; + return ptr; +} + +void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv, + struct safexcel_ring *ring) +{ + void *ptr = ring->read; + + if (!ring->nr) + return ERR_PTR(-ENOENT); + + ring->read += ring->offset; + if (ring->read == ring->base_end) + ring->read = ring->base; + + ring->nr--; + return ptr; +} + +void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv, + struct safexcel_ring *ring) +{ + if (!ring->nr) + return; + + if (ring->write == ring->base) + ring->write += (EIP197_DEFAULT_RING_SIZE - 1) * ring->offset; + else + ring->write -= ring->offset; + + ring->nr--; +} + +struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv, + int ring_id, + bool first, bool last, + dma_addr_t data, u32 data_len, + u32 full_data_len, + dma_addr_t context) { + struct safexcel_command_desc *cdesc; + int i; + + cdesc = safexcel_ring_next_wptr(priv, &priv->ring[ring_id].cdr); + if (IS_ERR(cdesc)) + return cdesc; + + memset(cdesc, 0, sizeof(struct safexcel_command_desc)); + + cdesc->first_seg = first; + cdesc->last_seg = last; + cdesc->particle_size = data_len; + cdesc->data_lo = lower_32_bits(data); + cdesc->data_hi = upper_32_bits(data); + + if (first && context) { + struct safexcel_token *token = + (struct safexcel_token *)cdesc->control_data.token; + + cdesc->control_data.packet_length = full_data_len; + cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE | + EIP197_OPTION_64BIT_CTX | + EIP197_OPTION_CTX_CTRL_IN_CMD; + cdesc->control_data.context_lo = + (lower_32_bits(context) & GENMASK(31, 2)) >> 2; + cdesc->control_data.context_hi = upper_32_bits(context); + + /* TODO: large xform HMAC with SHA-384/512 uses refresh = 3 */ + cdesc->control_data.refresh = 2; + + for (i = 0; i < EIP197_MAX_TOKENS; i++) + eip197_noop_token(&token[i]); + } + + return cdesc; +} + +struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv, + int ring_id, + bool first, bool last, + dma_addr_t data, u32 len) +{ + struct safexcel_result_desc *rdesc; + + rdesc = safexcel_ring_next_wptr(priv, &priv->ring[ring_id].rdr); + if (IS_ERR(rdesc)) + return rdesc; + + memset(rdesc, 0, sizeof(struct safexcel_result_desc)); + + rdesc->first_seg = first; + rdesc->last_seg = last; + rdesc->particle_size = len; + rdesc->data_lo = lower_32_bits(data); + rdesc->data_hi = upper_32_bits(data); + + return rdesc; +} From ee0db78d62d81dcbfba7c3e4d523ecae9d7b7c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Wed, 24 May 2017 16:10:35 +0200 Subject: [PATCH 051/140] MAINTAINERS: add a maintainer for the Inside Secure crypto driver A new cryptographic engine driver was added in drivers/crypto/inside-secure. Add myself as a maintainer for this driver. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- MAINTAINERS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index f7d568b8f133..1f2017604f16 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6602,6 +6602,12 @@ F: Documentation/input/multi-touch-protocol.rst F: drivers/input/input-mt.c K: \b(ABS|SYN)_MT_ +INSIDE SECURE CRYPTO DRIVER +M: Antoine Tenart +F: drivers/crypto/inside-secure/ +S: Maintained +L: linux-crypto@vger.kernel.org + INTEL ASoC BDW/HSW DRIVERS M: Jie Yang L: alsa-devel@alsa-project.org (moderated for non-subscribers) From c0ca1215dc92ee6a0f975c95a92fb7f9fb31e9e1 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:03 +0300 Subject: [PATCH 052/140] crypto: kpp, (ec)dh - fix typos While here, add missing argument description (ndigits). Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/dh.c | 4 ++-- crypto/dh_helper.c | 4 ++-- crypto/ecc.h | 8 +++++--- crypto/ecdh.c | 4 ++-- crypto/ecdh_helper.c | 4 ++-- include/crypto/dh.h | 4 ++-- include/crypto/ecdh.h | 4 ++-- include/crypto/kpp.h | 4 ++-- 8 files changed, 19 insertions(+), 17 deletions(-) diff --git a/crypto/dh.c b/crypto/dh.c index 87e3542cf1b8..7cec04985b9d 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -4,9 +4,9 @@ * Authors: Salvatore Benedetto * * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public Licence + * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version - * 2 of the Licence, or (at your option) any later version. + * 2 of the License, or (at your option) any later version. */ #include diff --git a/crypto/dh_helper.c b/crypto/dh_helper.c index 02db76b20d00..8ba8a3f82620 100644 --- a/crypto/dh_helper.c +++ b/crypto/dh_helper.c @@ -3,9 +3,9 @@ * Authors: Salvatore Benedetto * * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public Licence + * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version - * 2 of the Licence, or (at your option) any later version. + * 2 of the License, or (at your option) any later version. */ #include #include diff --git a/crypto/ecc.h b/crypto/ecc.h index 663d598c7406..37f438500255 100644 --- a/crypto/ecc.h +++ b/crypto/ecc.h @@ -34,9 +34,9 @@ * ecc_is_key_valid() - Validate a given ECDH private key * * @curve_id: id representing the curve to use - * @ndigits: curve number of digits + * @ndigits: curve's number of digits * @private_key: private key to be used for the given curve - * @private_key_len: private key len + * @private_key_len: private key length * * Returns 0 if the key is acceptable, a negative value otherwise */ @@ -47,9 +47,10 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, * ecdh_make_pub_key() - Compute an ECC public key * * @curve_id: id representing the curve to use + * @ndigits: curve's number of digits * @private_key: pregenerated private key for the given curve * @private_key_len: length of private_key - * @public_key: buffer for storing the public key generated + * @public_key: buffer for storing the generated public key * @public_key_len: length of the public_key buffer * * Returns 0 if the public key was generated successfully, a negative value @@ -63,6 +64,7 @@ int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits, * crypto_ecdh_shared_secret() - Compute a shared secret * * @curve_id: id representing the curve to use + * @ndigits: curve's number of digits * @private_key: private key of part A * @private_key_len: length of private_key * @public_key: public key of counterpart B diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 63ca33771e4e..36233077a552 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -4,9 +4,9 @@ * Authors: Salvator Benedetto * * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public Licence + * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version - * 2 of the Licence, or (at your option) any later version. + * 2 of the License, or (at your option) any later version. */ #include diff --git a/crypto/ecdh_helper.c b/crypto/ecdh_helper.c index 3cd8a2414e60..f05bea5fd257 100644 --- a/crypto/ecdh_helper.c +++ b/crypto/ecdh_helper.c @@ -3,9 +3,9 @@ * Authors: Salvatore Benedetto * * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public Licence + * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version - * 2 of the Licence, or (at your option) any later version. + * 2 of the License, or (at your option) any later version. */ #include #include diff --git a/include/crypto/dh.h b/include/crypto/dh.h index 6b424ad3482e..f638998fb6d0 100644 --- a/include/crypto/dh.h +++ b/include/crypto/dh.h @@ -73,9 +73,9 @@ int crypto_dh_encode_key(char *buf, unsigned int len, const struct dh *params); /** * crypto_dh_decode_key() - decode a private key * @buf: Buffer holding a packet key that should be decoded - * @len: Lenth of the packet private key buffer + * @len: Length of the packet private key buffer * @params: Buffer allocated by the caller that is filled with the - * unpacket DH private key. + * unpacked DH private key. * * The unpacking obtains the private key by pointing @p to the correct location * in @buf. Thus, both pointers refer to the same memory. diff --git a/include/crypto/ecdh.h b/include/crypto/ecdh.h index 03a64f62ba7a..1aff2a8a3a68 100644 --- a/include/crypto/ecdh.h +++ b/include/crypto/ecdh.h @@ -74,9 +74,9 @@ int crypto_ecdh_encode_key(char *buf, unsigned int len, const struct ecdh *p); /** * crypto_ecdh_decode_key() - decode a private key * @buf: Buffer holding a packet key that should be decoded - * @len: Lenth of the packet private key buffer + * @len: Length of the packet private key buffer * @p: Buffer allocated by the caller that is filled with the - * unpacket ECDH private key. + * unpacked ECDH private key. * * The unpacking obtains the private key by pointing @p to the correct location * in @buf. Thus, both pointers refer to the same memory. diff --git a/include/crypto/kpp.h b/include/crypto/kpp.h index ce8e1f79374b..c19082587950 100644 --- a/include/crypto/kpp.h +++ b/include/crypto/kpp.h @@ -53,7 +53,7 @@ struct crypto_kpp { * * @set_secret: Function invokes the protocol specific function to * store the secret private key along with parameters. - * The implementation knows how to decode thie buffer + * The implementation knows how to decode the buffer * @generate_public_key: Function generate the public key to be sent to the * counterpart. In case of error, where output is not big * enough req->dst_len will be updated to the size @@ -102,7 +102,7 @@ struct kpp_alg { * @mask: specifies the mask for the algorithm * * Allocate a handle for kpp algorithm. The returned struct crypto_kpp - * is requeried for any following API invocation + * is required for any following API invocation * * Return: allocated handle in case of success; IS_ERR() is true in case of * an error, PTR_ERR() returns the error code. From 099054d735a5e4cfc8e90b03b7422c9b48209d8b Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:04 +0300 Subject: [PATCH 053/140] crypto: ecc - remove unused function arguments Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/ecc.c | 8 +++----- crypto/ecc.h | 13 +++---------- crypto/ecdh.c | 11 +++++------ 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/crypto/ecc.c b/crypto/ecc.c index 414c78a9c214..69b4cc4303aa 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -928,8 +928,7 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, } int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits, - const u8 *private_key, unsigned int private_key_len, - u8 *public_key, unsigned int public_key_len) + const u8 *private_key, u8 *public_key) { int ret = 0; struct ecc_point *pk; @@ -967,9 +966,8 @@ out: } int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, - const u8 *private_key, unsigned int private_key_len, - const u8 *public_key, unsigned int public_key_len, - u8 *secret, unsigned int secret_len) + const u8 *private_key, const u8 *public_key, + u8 *secret) { int ret = 0; struct ecc_point *product, *pk; diff --git a/crypto/ecc.h b/crypto/ecc.h index 37f438500255..1ca9bf7437d7 100644 --- a/crypto/ecc.h +++ b/crypto/ecc.h @@ -49,16 +49,13 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, * @curve_id: id representing the curve to use * @ndigits: curve's number of digits * @private_key: pregenerated private key for the given curve - * @private_key_len: length of private_key * @public_key: buffer for storing the generated public key - * @public_key_len: length of the public_key buffer * * Returns 0 if the public key was generated successfully, a negative value * if an error occurred. */ int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits, - const u8 *private_key, unsigned int private_key_len, - u8 *public_key, unsigned int public_key_len); + const u8 *private_key, u8 *public_key); /** * crypto_ecdh_shared_secret() - Compute a shared secret @@ -66,11 +63,8 @@ int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits, * @curve_id: id representing the curve to use * @ndigits: curve's number of digits * @private_key: private key of part A - * @private_key_len: length of private_key * @public_key: public key of counterpart B - * @public_key_len: length of public_key * @secret: buffer for storing the calculated shared secret - * @secret_len: length of the secret buffer * * Note: It is recommended that you hash the result of crypto_ecdh_shared_secret * before using it for symmetric encryption or HMAC. @@ -79,7 +73,6 @@ int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits, * if an error occurred. */ int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, - const u8 *private_key, unsigned int private_key_len, - const u8 *public_key, unsigned int public_key_len, - u8 *secret, unsigned int secret_len); + const u8 *private_key, const u8 *public_key, + u8 *secret); #endif diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 36233077a552..69c3951f0a03 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -81,16 +81,15 @@ static int ecdh_compute_value(struct kpp_request *req) return -EINVAL; ret = crypto_ecdh_shared_secret(ctx->curve_id, ctx->ndigits, - (const u8 *)ctx->private_key, nbytes, - (const u8 *)ctx->public_key, 2 * nbytes, - (u8 *)ctx->shared_secret, nbytes); + (const u8 *)ctx->private_key, + (const u8 *)ctx->public_key, + (u8 *)ctx->shared_secret); buf = ctx->shared_secret; } else { ret = ecdh_make_pub_key(ctx->curve_id, ctx->ndigits, - (const u8 *)ctx->private_key, nbytes, - (u8 *)ctx->public_key, - sizeof(ctx->public_key)); + (const u8 *)ctx->private_key, + (u8 *)ctx->public_key); buf = ctx->public_key; /* Public part is a point thus it has both coordinates */ nbytes *= 2; From ad269597107e6dde4810987eebbd075778d0262c Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:05 +0300 Subject: [PATCH 054/140] crypto: ecc - remove unnecessary casts ecc software implementation works with chunks of u64 data. There were some unnecessary casts to u8 and then back to u64 for the ecc keys. This patch removes the unnecessary casts. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/ecc.c | 28 +++++++++++++--------------- crypto/ecc.h | 8 ++++---- crypto/ecdh.c | 11 +++++------ 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/crypto/ecc.c b/crypto/ecc.c index 69b4cc4303aa..e3a2b8f78a38 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -904,7 +904,7 @@ static inline void ecc_swap_digits(const u64 *in, u64 *out, } int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, - const u8 *private_key, unsigned int private_key_len) + const u64 *private_key, unsigned int private_key_len) { int nbytes; const struct ecc_curve *curve = ecc_get_curve(curve_id); @@ -917,23 +917,22 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, if (private_key_len != nbytes) return -EINVAL; - if (vli_is_zero((const u64 *)&private_key[0], ndigits)) + if (vli_is_zero(private_key, ndigits)) return -EINVAL; /* Make sure the private key is in the range [1, n-1]. */ - if (vli_cmp(curve->n, (const u64 *)&private_key[0], ndigits) != 1) + if (vli_cmp(curve->n, private_key, ndigits) != 1) return -EINVAL; return 0; } int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits, - const u8 *private_key, u8 *public_key) + const u64 *private_key, u64 *public_key) { int ret = 0; struct ecc_point *pk; u64 priv[ndigits]; - unsigned int nbytes; const struct ecc_curve *curve = ecc_get_curve(curve_id); if (!private_key || !curve) { @@ -941,7 +940,7 @@ int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits, goto out; } - ecc_swap_digits((const u64 *)private_key, priv, ndigits); + ecc_swap_digits(private_key, priv, ndigits); pk = ecc_alloc_point(ndigits); if (!pk) { @@ -955,9 +954,8 @@ int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits, goto err_free_point; } - nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT; - ecc_swap_digits(pk->x, (u64 *)public_key, ndigits); - ecc_swap_digits(pk->y, (u64 *)&public_key[nbytes], ndigits); + ecc_swap_digits(pk->x, public_key, ndigits); + ecc_swap_digits(pk->y, &public_key[ndigits], ndigits); err_free_point: ecc_free_point(pk); @@ -966,8 +964,8 @@ out: } int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, - const u8 *private_key, const u8 *public_key, - u8 *secret) + const u64 *private_key, const u64 *public_key, + u64 *secret) { int ret = 0; struct ecc_point *product, *pk; @@ -997,13 +995,13 @@ int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, goto err_alloc_product; } - ecc_swap_digits((const u64 *)public_key, pk->x, ndigits); - ecc_swap_digits((const u64 *)&public_key[nbytes], pk->y, ndigits); - ecc_swap_digits((const u64 *)private_key, priv, ndigits); + ecc_swap_digits(public_key, pk->x, ndigits); + ecc_swap_digits(&public_key[ndigits], pk->y, ndigits); + ecc_swap_digits(private_key, priv, ndigits); ecc_point_mult(product, pk, priv, rand_z, curve->p, ndigits); - ecc_swap_digits(product->x, (u64 *)secret, ndigits); + ecc_swap_digits(product->x, secret, ndigits); if (ecc_point_is_zero(product)) ret = -EFAULT; diff --git a/crypto/ecc.h b/crypto/ecc.h index 1ca9bf7437d7..af2ffdb2dcd6 100644 --- a/crypto/ecc.h +++ b/crypto/ecc.h @@ -41,7 +41,7 @@ * Returns 0 if the key is acceptable, a negative value otherwise */ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, - const u8 *private_key, unsigned int private_key_len); + const u64 *private_key, unsigned int private_key_len); /** * ecdh_make_pub_key() - Compute an ECC public key @@ -55,7 +55,7 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, * if an error occurred. */ int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits, - const u8 *private_key, u8 *public_key); + const u64 *private_key, u64 *public_key); /** * crypto_ecdh_shared_secret() - Compute a shared secret @@ -73,6 +73,6 @@ int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits, * if an error occurred. */ int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, - const u8 *private_key, const u8 *public_key, - u8 *secret); + const u64 *private_key, const u64 *public_key, + u64 *secret); #endif diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 69c3951f0a03..c1f01630faad 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -56,7 +56,7 @@ static int ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, ctx->ndigits = ndigits; if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits, - (const u8 *)params.key, params.key_size) < 0) + (const u64 *)params.key, params.key_size) < 0) return -EINVAL; memcpy(ctx->private_key, params.key, params.key_size); @@ -81,15 +81,14 @@ static int ecdh_compute_value(struct kpp_request *req) return -EINVAL; ret = crypto_ecdh_shared_secret(ctx->curve_id, ctx->ndigits, - (const u8 *)ctx->private_key, - (const u8 *)ctx->public_key, - (u8 *)ctx->shared_secret); + ctx->private_key, + ctx->public_key, + ctx->shared_secret); buf = ctx->shared_secret; } else { ret = ecdh_make_pub_key(ctx->curve_id, ctx->ndigits, - (const u8 *)ctx->private_key, - (u8 *)ctx->public_key); + ctx->private_key, ctx->public_key); buf = ctx->public_key; /* Public part is a point thus it has both coordinates */ nbytes *= 2; From 7380c56d2fcaa882a0809aad075c3f9fdc3b3c50 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Tue, 30 May 2017 15:37:56 +0300 Subject: [PATCH 055/140] crypto: ecc - rename ecdh_make_pub_key() Rename ecdh_make_pub_key() to ecc_make_pub_key(). ecdh_make_pub_key() is not dh specific and the reference to dh is wrong. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/ecc.c | 4 ++-- crypto/ecc.h | 6 +++--- crypto/ecdh.c | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crypto/ecc.c b/crypto/ecc.c index e3a2b8f78a38..6c33c4323d6a 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -927,8 +927,8 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, return 0; } -int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits, - const u64 *private_key, u64 *public_key) +int ecc_make_pub_key(unsigned int curve_id, unsigned int ndigits, + const u64 *private_key, u64 *public_key) { int ret = 0; struct ecc_point *pk; diff --git a/crypto/ecc.h b/crypto/ecc.h index af2ffdb2dcd6..e13fe8800ee8 100644 --- a/crypto/ecc.h +++ b/crypto/ecc.h @@ -44,7 +44,7 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, const u64 *private_key, unsigned int private_key_len); /** - * ecdh_make_pub_key() - Compute an ECC public key + * ecc_make_pub_key() - Compute an ECC public key * * @curve_id: id representing the curve to use * @ndigits: curve's number of digits @@ -54,8 +54,8 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, * Returns 0 if the public key was generated successfully, a negative value * if an error occurred. */ -int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits, - const u64 *private_key, u64 *public_key); +int ecc_make_pub_key(const unsigned int curve_id, unsigned int ndigits, + const u64 *private_key, u64 *public_key); /** * crypto_ecdh_shared_secret() - Compute a shared secret diff --git a/crypto/ecdh.c b/crypto/ecdh.c index c1f01630faad..ed1464a5ce1b 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -87,8 +87,8 @@ static int ecdh_compute_value(struct kpp_request *req) buf = ctx->shared_secret; } else { - ret = ecdh_make_pub_key(ctx->curve_id, ctx->ndigits, - ctx->private_key, ctx->public_key); + ret = ecc_make_pub_key(ctx->curve_id, ctx->ndigits, + ctx->private_key, ctx->public_key); buf = ctx->public_key; /* Public part is a point thus it has both coordinates */ nbytes *= 2; From ee34e2644a78e2561742bea8c4bdcf83cabf90a7 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:07 +0300 Subject: [PATCH 056/140] crypto: dh - fix memleak in setkey setkey can be called multiple times during the existence of the transformation object. In case of multiple setkey calls, the old key was not freed and we leaked memory. Free the old MPI key if any. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/dh.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/dh.c b/crypto/dh.c index 7cec04985b9d..e151f12775ca 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -85,6 +85,9 @@ static int dh_set_secret(struct crypto_kpp *tfm, const void *buf, struct dh_ctx *ctx = dh_get_ctx(tfm); struct dh params; + /* Free the old MPI key if any */ + dh_free_ctx(ctx); + if (crypto_dh_decode_key(buf, len, ¶ms) < 0) return -EINVAL; From c444b8da87d16e6f9f11e15c8c873a641f7cd117 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:08 +0300 Subject: [PATCH 057/140] crypto: kpp - assume key is already set in maxsize As of now, crypto_kpp_maxsize() can not be reached without successfully setting the key for the transformation. kpp algorithm implementations check if the key was set and then return the output buffer size required for the given key. Change the return type to unsigned int and always assume that this function is called after a successful setkey of the transformation. kpp algorithm implementations will remove the check if key is not NULL and directly return the max size. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- include/crypto/kpp.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/include/crypto/kpp.h b/include/crypto/kpp.h index c19082587950..2133d17b7156 100644 --- a/include/crypto/kpp.h +++ b/include/crypto/kpp.h @@ -79,7 +79,7 @@ struct kpp_alg { int (*generate_public_key)(struct kpp_request *req); int (*compute_shared_secret)(struct kpp_request *req); - int (*max_size)(struct crypto_kpp *tfm); + unsigned int (*max_size)(struct crypto_kpp *tfm); int (*init)(struct crypto_kpp *tfm); void (*exit)(struct crypto_kpp *tfm); @@ -323,13 +323,14 @@ static inline int crypto_kpp_compute_shared_secret(struct kpp_request *req) /** * crypto_kpp_maxsize() - Get len for output buffer * - * Function returns the output buffer size required + * Function returns the output buffer size required for a given key. + * Function assumes that the key is already set in the transformation. If this + * function is called without a setkey or with a failed setkey, you will end up + * in a NULL dereference. * * @tfm: KPP tfm handle allocated with crypto_alloc_kpp() - * - * Return: minimum len for output buffer or error code if key hasn't been set */ -static inline int crypto_kpp_maxsize(struct crypto_kpp *tfm) +static inline unsigned int crypto_kpp_maxsize(struct crypto_kpp *tfm) { struct kpp_alg *alg = crypto_kpp_alg(tfm); From 7f69105077bfb36a8ff9c2ee20af50d921445fc0 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:09 +0300 Subject: [PATCH 058/140] crypto: dh - comply with crypto_kpp_maxsize() crypto_kpp_maxsize() asks for the output buffer size without caring for errors. It allways assume that will be called after a valid setkey. Comply with it and return what he wants. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/dh.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/dh.c b/crypto/dh.c index e151f12775ca..b1032a5c1bfa 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -147,7 +147,7 @@ err_free_val: return ret; } -static int dh_max_size(struct crypto_kpp *tfm) +static unsigned int dh_max_size(struct crypto_kpp *tfm) { struct dh_ctx *ctx = dh_get_ctx(tfm); From d0efb48b8615292e05373caa27854e36e732668b Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:10 +0300 Subject: [PATCH 059/140] crypto: ecdh - comply with crypto_kpp_maxsize() crypto_kpp_maxsize() asks for the output buffer size without caring for errors. It allways assume that will be called after a valid setkey. Comply with it and return what he wants. nbytes has no sense now, remove it and directly return the maxsize. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/ecdh.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crypto/ecdh.c b/crypto/ecdh.c index ed1464a5ce1b..4aa0b0cb4627 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -104,13 +104,12 @@ static int ecdh_compute_value(struct kpp_request *req) return ret; } -static int ecdh_max_size(struct crypto_kpp *tfm) +static unsigned int ecdh_max_size(struct crypto_kpp *tfm) { struct ecdh_ctx *ctx = ecdh_get_ctx(tfm); - int nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT; - /* Public key is made of two coordinates */ - return 2 * nbytes; + /* Public key is made of two coordinates, add one to the left shift */ + return ctx->ndigits << (ECC_DIGITS_TO_BYTES_SHIFT + 1); } static void no_exit_tfm(struct crypto_kpp *tfm) From 85ac98cbac1bb63c878486b88bfb6f5bac540e21 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:11 +0300 Subject: [PATCH 060/140] crypto: qat - comply with crypto_kpp_maxsize() crypto_kpp_maxsize() asks for the output buffer size without caring for errors. It allways assume that will be called after a valid setkey. Comply with it and return what he wants. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_asym_algs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 2aab80bc241f..1d882a7cc34f 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -521,11 +521,11 @@ static int qat_dh_set_secret(struct crypto_kpp *tfm, const void *buf, return 0; } -static int qat_dh_max_size(struct crypto_kpp *tfm) +static unsigned int qat_dh_max_size(struct crypto_kpp *tfm) { struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); - return ctx->p ? ctx->p_size : -EINVAL; + return ctx->p_size; } static int qat_dh_init_tfm(struct crypto_kpp *tfm) From 561f8e2df990a7a2859a98bd208d0f7dc932c3f7 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:12 +0300 Subject: [PATCH 061/140] crypto: akcipher - assume key is already set in maxsize As of now, crypto_akcipher_maxsize() can not be reached without successfully setting the key for the transformation. akcipher algorithm implementations check if the key was set and then return the output buffer size required for the given key. Change the return type to unsigned int and always assume that this function is called after a successful setkey of the transformation. akcipher algorithm implementations will remove the check if key is not NULL and directly return the max size. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- include/crypto/akcipher.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/include/crypto/akcipher.h b/include/crypto/akcipher.h index c37cc59e9bf2..b5e11de4d497 100644 --- a/include/crypto/akcipher.h +++ b/include/crypto/akcipher.h @@ -98,7 +98,7 @@ struct akcipher_alg { unsigned int keylen); int (*set_priv_key)(struct crypto_akcipher *tfm, const void *key, unsigned int keylen); - int (*max_size)(struct crypto_akcipher *tfm); + unsigned int (*max_size)(struct crypto_akcipher *tfm); int (*init)(struct crypto_akcipher *tfm); void (*exit)(struct crypto_akcipher *tfm); @@ -257,13 +257,14 @@ static inline void akcipher_request_set_crypt(struct akcipher_request *req, /** * crypto_akcipher_maxsize() - Get len for output buffer * - * Function returns the dest buffer size required for a given key + * Function returns the dest buffer size required for a given key. + * Function assumes that the key is already set in the transformation. If this + * function is called without a setkey or with a failed setkey, you will end up + * in a NULL dereference. * * @tfm: AKCIPHER tfm handle allocated with crypto_alloc_akcipher() - * - * Return: minimum len for output buffer or error code in key hasn't been set */ -static inline int crypto_akcipher_maxsize(struct crypto_akcipher *tfm) +static inline unsigned int crypto_akcipher_maxsize(struct crypto_akcipher *tfm) { struct akcipher_alg *alg = crypto_akcipher_alg(tfm); From 1c23b466e999911d3dd7af2ac0ec7bee8cc85635 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:13 +0300 Subject: [PATCH 062/140] crypto: rsa - comply with crypto_akcipher_maxsize() crypto_akcipher_maxsize() asks for the output buffer size without caring for errors. It allways assume that will be called after a valid setkey. Comply with it and return what he wants. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/rsa.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/rsa.c b/crypto/rsa.c index 4c280b6a3ea9..b067f3a93880 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -337,11 +337,11 @@ err: return -ENOMEM; } -static int rsa_max_size(struct crypto_akcipher *tfm) +static unsigned int rsa_max_size(struct crypto_akcipher *tfm) { struct rsa_mpi_key *pkey = akcipher_tfm_ctx(tfm); - return pkey->n ? mpi_get_size(pkey->n) : -EINVAL; + return mpi_get_size(pkey->n); } static void rsa_exit_tfm(struct crypto_akcipher *tfm) From e198429c41edaeaf5e1e2e2ef4bb5a476d8bf399 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:14 +0300 Subject: [PATCH 063/140] crypto: caampkc - comply with crypto_akcipher_maxsize() crypto_akcipher_maxsize() asks for the output buffer size without caring for errors. It allways assume that will be called after a valid setkey. Comply with it and return what he wants. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/caam/caampkc.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index 57f399caa977..9c508ba6b0f1 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -911,12 +911,11 @@ err: return -ENOMEM; } -static int caam_rsa_max_size(struct crypto_akcipher *tfm) +static unsigned int caam_rsa_max_size(struct crypto_akcipher *tfm) { struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct caam_rsa_key *key = &ctx->key; - return (key->n) ? key->n_sz : -EINVAL; + return ctx->key.n_sz; } /* Per session pkc's driver context creation function */ From 515c4d27d69acf0473157abeb21a2ae9f9328bac Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:15 +0300 Subject: [PATCH 064/140] crypto: qat - comply with crypto_akcipher_maxsize() crypto_akcipher_maxsize() asks for the output buffer size without caring for errors. It allways assume that will be called after a valid setkey. Comply with it and return what he wants. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_asym_algs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 1d882a7cc34f..6f5dd68449c6 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -1256,11 +1256,11 @@ static int qat_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, return qat_rsa_setkey(tfm, key, keylen, true); } -static int qat_rsa_max_size(struct crypto_akcipher *tfm) +static unsigned int qat_rsa_max_size(struct crypto_akcipher *tfm) { struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - return (ctx->n) ? ctx->key_sz : -EINVAL; + return ctx->key_sz; } static int qat_rsa_init_tfm(struct crypto_akcipher *tfm) From 543de102e3316983119462c2821b76648ac3e744 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 25 May 2017 10:18:16 +0300 Subject: [PATCH 065/140] crypto: pkcs1pad - comply with crypto_akcipher_maxsize() crypto_akcipher_maxsize() asks for the output buffer size without caring for errors. It allways assume that will be called after a valid setkey. Comply with it and return what he wants. crypto_akcipher_maxsize() now returns an unsigned int. Remove the unnecessary check. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/rsa-pkcs1pad.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c index 8baab4307f7b..044bcfa69cba 100644 --- a/crypto/rsa-pkcs1pad.c +++ b/crypto/rsa-pkcs1pad.c @@ -120,9 +120,6 @@ static int pkcs1pad_set_pub_key(struct crypto_akcipher *tfm, const void *key, /* Find out new modulus size from rsa implementation */ err = crypto_akcipher_maxsize(ctx->child); - if (err < 0) - return err; - if (err > PAGE_SIZE) return -ENOTSUPP; @@ -144,9 +141,6 @@ static int pkcs1pad_set_priv_key(struct crypto_akcipher *tfm, const void *key, /* Find out new modulus size from rsa implementation */ err = crypto_akcipher_maxsize(ctx->child); - if (err < 0) - return err; - if (err > PAGE_SIZE) return -ENOTSUPP; @@ -154,7 +148,7 @@ static int pkcs1pad_set_priv_key(struct crypto_akcipher *tfm, const void *key, return 0; } -static int pkcs1pad_get_max_size(struct crypto_akcipher *tfm) +static unsigned int pkcs1pad_get_max_size(struct crypto_akcipher *tfm) { struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); @@ -164,7 +158,7 @@ static int pkcs1pad_get_max_size(struct crypto_akcipher *tfm) * decrypt/verify. */ - return ctx->key_size ?: -EINVAL; + return ctx->key_size; } static void pkcs1pad_sg_set_buf(struct scatterlist *sg, void *buf, size_t len, From a18ccc365f1a29f1b6d7b4a1edbaffb7100fb1f6 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Fri, 26 May 2017 12:12:51 +0200 Subject: [PATCH 066/140] crypto: drbg - Fixes panic in wait_for_completion call Initialise ctr_completion variable before use. Signed-off-by: Harsh Jain Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/drbg.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/drbg.c b/crypto/drbg.c index fa749f470135..433c4258e484 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1691,6 +1691,7 @@ static int drbg_init_sym_kernel(struct drbg_state *drbg) return PTR_ERR(sk_tfm); } drbg->ctr_handle = sk_tfm; + init_completion(&drbg->ctr_completion); req = skcipher_request_alloc(sk_tfm, GFP_KERNEL); if (!req) { From 3bca64c1b9d2f6e7af4512bd9bfef6909d09cdbc Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 29 May 2017 16:26:22 +0300 Subject: [PATCH 067/140] X.509: Fix error code in x509_cert_parse() We forgot to set the error code on this path so it could result in returning NULL which leads to a NULL dereference. Fixes: db6c43bd2132 ("crypto: KEYS: convert public key and digsig asym to the akcipher api") Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- crypto/asymmetric_keys/x509_cert_parser.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index c80765b211cf..dd03fead1ca3 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -102,6 +102,7 @@ struct x509_certificate *x509_cert_parse(const void *data, size_t datalen) } } + ret = -ENOMEM; cert->pub->key = kmemdup(ctx->key, ctx->key_size, GFP_KERNEL); if (!cert->pub->key) goto error_decode; From 14fa93cdcd9bbd50018196c00ca16da636f965c2 Mon Sep 17 00:00:00 2001 From: Srikanth Jampala Date: Tue, 30 May 2017 17:28:01 +0530 Subject: [PATCH 068/140] crypto: cavium - Add support for CNN55XX adapters. Add Physical Function driver support for CNN55XX crypto adapters. CNN55XX adapters belongs to Cavium NITROX family series, which accelerate both Symmetric and Asymmetric crypto workloads. These adapters have crypto engines that need firmware to become operational. Signed-off-by: Srikanth Jampala Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 1 + drivers/crypto/Makefile | 1 + drivers/crypto/cavium/nitrox/Kconfig | 21 + drivers/crypto/cavium/nitrox/Makefile | 7 + drivers/crypto/cavium/nitrox/nitrox_common.h | 35 + drivers/crypto/cavium/nitrox/nitrox_csr.h | 1080 ++++++++++++++++++ drivers/crypto/cavium/nitrox/nitrox_dev.h | 175 +++ drivers/crypto/cavium/nitrox/nitrox_hal.c | 401 +++++++ drivers/crypto/cavium/nitrox/nitrox_isr.c | 467 ++++++++ drivers/crypto/cavium/nitrox/nitrox_lib.c | 172 +++ drivers/crypto/cavium/nitrox/nitrox_main.c | 465 ++++++++ drivers/crypto/cavium/nitrox/nitrox_req.h | 445 ++++++++ drivers/crypto/cavium/nitrox/nitrox_reqmgr.c | 732 ++++++++++++ 13 files changed, 4002 insertions(+) create mode 100644 drivers/crypto/cavium/nitrox/Kconfig create mode 100644 drivers/crypto/cavium/nitrox/Makefile create mode 100644 drivers/crypto/cavium/nitrox/nitrox_common.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_csr.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_dev.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_hal.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_isr.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_lib.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_main.c create mode 100644 drivers/crypto/cavium/nitrox/nitrox_req.h create mode 100644 drivers/crypto/cavium/nitrox/nitrox_reqmgr.c diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 0a652baf3cb3..0528a62a39a6 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -541,6 +541,7 @@ config CRYPTO_DEV_MXS_DCP source "drivers/crypto/qat/Kconfig" source "drivers/crypto/cavium/cpt/Kconfig" +source "drivers/crypto/cavium/nitrox/Kconfig" config CRYPTO_DEV_CAVIUM_ZIP tristate "Cavium ZIP driver" diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 28786321b118..2c555a3393b2 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -6,6 +6,7 @@ obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/ obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/ +obj-$(CONFIG_CRYPTO_DEV_NITROX) += cavium/nitrox/ obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o diff --git a/drivers/crypto/cavium/nitrox/Kconfig b/drivers/crypto/cavium/nitrox/Kconfig new file mode 100644 index 000000000000..731e6a57218a --- /dev/null +++ b/drivers/crypto/cavium/nitrox/Kconfig @@ -0,0 +1,21 @@ +# +# Cavium NITROX Crypto Device configuration +# +config CRYPTO_DEV_NITROX + tristate + select CRYPTO_BLKCIPHER + select CRYPTO_AES + select CRYPTO_DES + select FW_LOADER + +config CRYPTO_DEV_NITROX_CNN55XX + tristate "Support for Cavium CNN55XX driver" + depends on PCI_MSI && 64BIT + select CRYPTO_DEV_NITROX + default m + help + Support for Cavium NITROX family CNN55XX driver + for accelerating crypto workloads. + + To compile this as a module, choose M here: the module + will be called n5pf. diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile new file mode 100644 index 000000000000..ef457f609d88 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/Makefile @@ -0,0 +1,7 @@ +obj-$(CONFIG_CRYPTO_DEV_NITROX_CNN55XX) += n5pf.o + +n5pf-objs := nitrox_main.o \ + nitrox_isr.o \ + nitrox_lib.o \ + nitrox_hal.o \ + nitrox_reqmgr.o diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h new file mode 100644 index 000000000000..3284ffe4837a --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_common.h @@ -0,0 +1,35 @@ +#ifndef __NITROX_COMMON_H +#define __NITROX_COMMON_H + +#include "nitrox_dev.h" +#include "nitrox_req.h" + +void nitrox_pf_cleanup_isr(struct nitrox_device *ndev); +int nitrox_pf_init_isr(struct nitrox_device *ndev); + +int nitrox_common_sw_init(struct nitrox_device *ndev); +void nitrox_common_sw_cleanup(struct nitrox_device *ndev); + +void pkt_slc_resp_handler(unsigned long data); +int nitrox_process_se_request(struct nitrox_device *ndev, + struct se_crypto_request *req, + completion_t cb, + struct skcipher_request *skreq); +void backlog_qflush_work(struct work_struct *work); + +void nitrox_config_emu_unit(struct nitrox_device *ndev); +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev); +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev); +void nitrox_config_vfmode(struct nitrox_device *ndev, int mode); +void nitrox_config_nps_unit(struct nitrox_device *ndev); +void nitrox_config_pom_unit(struct nitrox_device *ndev); +void nitrox_config_rand_unit(struct nitrox_device *ndev); +void nitrox_config_efl_unit(struct nitrox_device *ndev); +void nitrox_config_bmi_unit(struct nitrox_device *ndev); +void nitrox_config_bmo_unit(struct nitrox_device *ndev); +void nitrox_config_lbc_unit(struct nitrox_device *ndev); +void invalidate_lbc(struct nitrox_device *ndev); +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring); +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port); + +#endif /* __NITROX_COMMON_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h new file mode 100644 index 000000000000..583465190882 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h @@ -0,0 +1,1080 @@ +#ifndef __NITROX_CSR_H +#define __NITROX_CSR_H + +#include +#include + +/* EMU clusters */ +#define NR_CLUSTERS 4 +#define AE_CORES_PER_CLUSTER 20 +#define SE_CORES_PER_CLUSTER 16 + +/* BIST registers */ +#define EMU_BIST_STATUSX(_i) (0x1402700 + ((_i) * 0x40000)) +#define UCD_BIST_STATUS 0x12C0070 +#define NPS_CORE_BIST_REG 0x10000E8 +#define NPS_CORE_NPC_BIST_REG 0x1000128 +#define NPS_PKT_SLC_BIST_REG 0x1040088 +#define NPS_PKT_IN_BIST_REG 0x1040100 +#define POM_BIST_REG 0x11C0100 +#define BMI_BIST_REG 0x1140080 +#define EFL_CORE_BIST_REGX(_i) (0x1240100 + ((_i) * 0x400)) +#define EFL_TOP_BIST_STAT 0x1241090 +#define BMO_BIST_REG 0x1180080 +#define LBC_BIST_STATUS 0x1200020 +#define PEM_BIST_STATUSX(_i) (0x1080468 | ((_i) << 18)) + +/* EMU registers */ +#define EMU_SE_ENABLEX(_i) (0x1400000 + ((_i) * 0x40000)) +#define EMU_AE_ENABLEX(_i) (0x1400008 + ((_i) * 0x40000)) +#define EMU_WD_INT_ENA_W1SX(_i) (0x1402318 + ((_i) * 0x40000)) +#define EMU_GE_INT_ENA_W1SX(_i) (0x1402518 + ((_i) * 0x40000)) +#define EMU_FUSE_MAPX(_i) (0x1402708 + ((_i) * 0x40000)) + +/* UCD registers */ +#define UCD_UCODE_LOAD_BLOCK_NUM 0x12C0010 +#define UCD_UCODE_LOAD_IDX_DATAX(_i) (0x12C0018 + ((_i) * 0x20)) +#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000)) + +/* NPS core registers */ +#define NPS_CORE_GBL_VFCFG 0x1000000 +#define NPS_CORE_CONTROL 0x1000008 +#define NPS_CORE_INT_ACTIVE 0x1000080 +#define NPS_CORE_INT 0x10000A0 +#define NPS_CORE_INT_ENA_W1S 0x10000B8 + +/* NPS packet registers */ +#define NPS_PKT_INT 0x1040018 +#define NPS_PKT_IN_RERR_HI 0x1040108 +#define NPS_PKT_IN_RERR_HI_ENA_W1S 0x1040120 +#define NPS_PKT_IN_RERR_LO 0x1040128 +#define NPS_PKT_IN_RERR_LO_ENA_W1S 0x1040140 +#define NPS_PKT_IN_ERR_TYPE 0x1040148 +#define NPS_PKT_IN_ERR_TYPE_ENA_W1S 0x1040160 +#define NPS_PKT_IN_INSTR_CTLX(_i) (0x10060 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_BADDRX(_i) (0x10068 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_RSIZEX(_i) (0x10070 + ((_i) * 0x40000)) +#define NPS_PKT_IN_DONE_CNTSX(_i) (0x10080 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_BAOFF_DBELLX(_i) (0x10078 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INT_LEVELSX(_i) (0x10088 + ((_i) * 0x40000)) + +#define NPS_PKT_SLC_RERR_HI 0x1040208 +#define NPS_PKT_SLC_RERR_HI_ENA_W1S 0x1040220 +#define NPS_PKT_SLC_RERR_LO 0x1040228 +#define NPS_PKT_SLC_RERR_LO_ENA_W1S 0x1040240 +#define NPS_PKT_SLC_ERR_TYPE 0x1040248 +#define NPS_PKT_SLC_ERR_TYPE_ENA_W1S 0x1040260 +#define NPS_PKT_SLC_CTLX(_i) (0x10000 + ((_i) * 0x40000)) +#define NPS_PKT_SLC_CNTSX(_i) (0x10008 + ((_i) * 0x40000)) +#define NPS_PKT_SLC_INT_LEVELSX(_i) (0x10010 + ((_i) * 0x40000)) + +/* POM registers */ +#define POM_INT_ENA_W1S 0x11C0018 +#define POM_GRP_EXECMASKX(_i) (0x11C1100 | ((_i) * 8)) +#define POM_INT 0x11C0000 +#define POM_PERF_CTL 0x11CC400 + +/* BMI registers */ +#define BMI_INT 0x1140000 +#define BMI_CTL 0x1140020 +#define BMI_INT_ENA_W1S 0x1140018 + +/* EFL registers */ +#define EFL_CORE_INT_ENA_W1SX(_i) (0x1240018 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT0X(_i) (0x1240050 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT0_ENA_W1SX(_i) (0x1240068 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT1X(_i) (0x1240070 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT1_ENA_W1SX(_i) (0x1240088 + ((_i) * 0x400)) +#define EFL_CORE_SE_ERR_INTX(_i) (0x12400A0 + ((_i) * 0x400)) +#define EFL_RNM_CTL_STATUS 0x1241800 +#define EFL_CORE_INTX(_i) (0x1240000 + ((_i) * 0x400)) + +/* BMO registers */ +#define BMO_CTL2 0x1180028 + +/* LBC registers */ +#define LBC_INT 0x1200000 +#define LBC_INVAL_CTL 0x1201010 +#define LBC_PLM_VF1_64_INT 0x1202008 +#define LBC_INVAL_STATUS 0x1202010 +#define LBC_INT_ENA_W1S 0x1203000 +#define LBC_PLM_VF1_64_INT_ENA_W1S 0x1205008 +#define LBC_PLM_VF65_128_INT 0x1206008 +#define LBC_ELM_VF1_64_INT 0x1208000 +#define LBC_PLM_VF65_128_INT_ENA_W1S 0x1209008 +#define LBC_ELM_VF1_64_INT_ENA_W1S 0x120B000 +#define LBC_ELM_VF65_128_INT 0x120C000 +#define LBC_ELM_VF65_128_INT_ENA_W1S 0x120F000 + +/* PEM registers */ +#define PEM0_INT 0x1080428 + +/** + * struct emu_fuse_map - EMU Fuse Map Registers + * @ae_fuse: Fuse settings for AE 19..0 + * @se_fuse: Fuse settings for SE 15..0 + * + * A set bit indicates the unit is fuse disabled. + */ +union emu_fuse_map { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 valid : 1; + u64 raz_52_62 : 11; + u64 ae_fuse : 20; + u64 raz_16_31 : 16; + u64 se_fuse : 16; +#else + u64 se_fuse : 16; + u64 raz_16_31 : 16; + u64 ae_fuse : 20; + u64 raz_52_62 : 11; + u64 valid : 1; +#endif + } s; +}; + +/** + * struct emu_se_enable - Symmetric Engine Enable Registers + * @enable: Individual enables for each of the clusters + * 16 symmetric engines. + */ +union emu_se_enable { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 48; + u64 enable : 16; +#else + u64 enable : 16; + u64 raz : 48; +#endif + } s; +}; + +/** + * struct emu_ae_enable - EMU Asymmetric engines. + * @enable: Individual enables for each of the cluster's + * 20 Asymmetric Engines. + */ +union emu_ae_enable { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 44; + u64 enable : 20; +#else + u64 enable : 20; + u64 raz : 44; +#endif + } s; +}; + +/** + * struct emu_wd_int_ena_w1s - EMU Interrupt Enable Registers + * @ae_wd: Reads or sets enable for EMU(0..3)_WD_INT[AE_WD] + * @se_wd: Reads or sets enable for EMU(0..3)_WD_INT[SE_WD] + */ +union emu_wd_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 12; + u64 ae_wd : 20; + u64 raz1 : 16; + u64 se_wd : 16; +#else + u64 se_wd : 16; + u64 raz1 : 16; + u64 ae_wd : 20; + u64 raz2 : 12; +#endif + } s; +}; + +/** + * struct emu_ge_int_ena_w1s - EMU Interrupt Enable set registers + * @ae_ge: Reads or sets enable for EMU(0..3)_GE_INT[AE_GE] + * @se_ge: Reads or sets enable for EMU(0..3)_GE_INT[SE_GE] + */ +union emu_ge_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_52_63 : 12; + u64 ae_ge : 20; + u64 raz_16_31: 16; + u64 se_ge : 16; +#else + u64 se_ge : 16; + u64 raz_16_31: 16; + u64 ae_ge : 20; + u64 raz_52_63 : 12; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_ctl - Solicited Packet Out Control Registers + * @rh: Indicates whether to remove or include the response header + * 1 = Include, 0 = Remove + * @z: If set, 8 trailing 0x00 bytes will be added to the end of the + * outgoing packet. + * @enb: Enable for this port. + */ +union nps_pkt_slc_ctl { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz : 61; + u64 rh : 1; + u64 z : 1; + u64 enb : 1; +#else + u64 enb : 1; + u64 z : 1; + u64 rh : 1; + u64 raz : 61; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_cnts - Solicited Packet Out Count Registers + * @slc_int: Returns a 1 when: + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET]. + * To clear the bit, the CNTS register must be written to clear. + * @in_int: Returns a 1 when: + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT]. + * To clear the bit, the DONE_CNTS register must be written to clear. + * @mbox_int: Returns a 1 when: + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. To clear the bit, + * write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] with 1. + * @timer: Timer, incremented every 2048 coprocessor clock cycles + * when [CNT] is not zero. The hardware clears both [TIMER] and + * [INT] when [CNT] goes to 0. + * @cnt: Packet counter. Hardware adds to [CNT] as it sends packets out. + * On a write to this CSR, hardware subtracts the amount written to the + * [CNT] field from [CNT]. + */ +union nps_pkt_slc_cnts { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 slc_int : 1; + u64 uns_int : 1; + u64 in_int : 1; + u64 mbox_int : 1; + u64 resend : 1; + u64 raz : 5; + u64 timer : 22; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 timer : 22; + u64 raz : 5; + u64 resend : 1; + u64 mbox_int : 1; + u64 in_int : 1; + u64 uns_int : 1; + u64 slc_int : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_int_levels - Solicited Packet Out Interrupt Levels + * Registers. + * @bmode: Determines whether NPS_PKT_SLC_CNTS[CNT] is a byte or + * packet counter. + * @timet: Output port counter time interrupt threshold. + * @cnt: Output port counter interrupt threshold. + */ +union nps_pkt_slc_int_levels { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 bmode : 1; + u64 raz : 9; + u64 timet : 22; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 timet : 22; + u64 raz : 9; + u64 bmode : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_inst - NPS Packet Interrupt Register + * @in_err: Set when any NPS_PKT_IN_RERR_HI/LO bit and + * corresponding NPS_PKT_IN_RERR_*_ENA_* bit are bot set. + * @uns_err: Set when any NSP_PKT_UNS_RERR_HI/LO bit and + * corresponding NPS_PKT_UNS_RERR_*_ENA_* bit are both set. + * @slc_er: Set when any NSP_PKT_SLC_RERR_HI/LO bit and + * corresponding NPS_PKT_SLC_RERR_*_ENA_* bit are both set. + */ +union nps_pkt_int { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz : 54; + u64 uns_wto : 1; + u64 in_err : 1; + u64 uns_err : 1; + u64 slc_err : 1; + u64 in_dbe : 1; + u64 in_sbe : 1; + u64 uns_dbe : 1; + u64 uns_sbe : 1; + u64 slc_dbe : 1; + u64 slc_sbe : 1; +#else + u64 slc_sbe : 1; + u64 slc_dbe : 1; + u64 uns_sbe : 1; + u64 uns_dbe : 1; + u64 in_sbe : 1; + u64 in_dbe : 1; + u64 slc_err : 1; + u64 uns_err : 1; + u64 in_err : 1; + u64 uns_wto : 1; + u64 raz : 54; +#endif + } s; +}; + +/** + * struct nps_pkt_in_done_cnts - Input instruction ring counts registers + * @slc_cnt: Returns a 1 when: + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SCL(i)_INT_LEVELS[TIMET] + * To clear the bit, the CNTS register must be + * written to clear the underlying condition + * @uns_int: Return a 1 when: + * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT], or + * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET] + * To clear the bit, the CNTS register must be + * written to clear the underlying condition + * @in_int: Returns a 1 when: + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT] + * To clear the bit, the DONE_CNTS register + * must be written to clear the underlying condition + * @mbox_int: Returns a 1 when: + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. + * To clear the bit, write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] + * with 1. + * @resend: A write of 1 will resend an MSI-X interrupt message if any + * of the following conditions are true for this ring "i". + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT] + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET] + * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT] + * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET] + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT] + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set + * @cnt: Packet counter. Hardware adds to [CNT] as it reads + * packets. On a write to this CSR, hardware substracts the + * amount written to the [CNT] field from [CNT], which will + * clear PKT_IN(i)_INT_STATUS[INTR] if [CNT] becomes <= + * NPS_PKT_IN(i)_INT_LEVELS[CNT]. This register should be + * cleared before enabling a ring by reading the current + * value and writing it back. + */ +union nps_pkt_in_done_cnts { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 slc_int : 1; + u64 uns_int : 1; + u64 in_int : 1; + u64 mbox_int : 1; + u64 resend : 1; + u64 raz : 27; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 raz : 27; + u64 resend : 1; + u64 mbox_int : 1; + u64 in_int : 1; + u64 uns_int : 1; + u64 slc_int : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_ctl - Input Instruction Ring Control Registers. + * @is64b: If 1, the ring uses 64-byte instructions. If 0, the + * ring uses 32-byte instructions. + * @enb: Enable for the input ring. + */ +union nps_pkt_in_instr_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 62; + u64 is64b : 1; + u64 enb : 1; +#else + u64 enb : 1; + u64 is64b : 1; + u64 raz : 62; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_rsize - Input instruction ring size registers + * @rsize: Ring size (number of instructions) + */ +union nps_pkt_in_instr_rsize { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 32; + u64 rsize : 32; +#else + u64 rsize : 32; + u64 raz : 32; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_baoff_dbell - Input instruction ring + * base address offset and doorbell registers + * @aoff: Address offset. The offset from the NPS_PKT_IN_INSTR_BADDR + * where the next pointer is read. + * @dbell: Pointer list doorbell count. Write operations to this field + * increments the present value here. Read operations return the + * present value. + */ +union nps_pkt_in_instr_baoff_dbell { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 aoff : 32; + u64 dbell : 32; +#else + u64 dbell : 32; + u64 aoff : 32; +#endif + } s; +}; + +/** + * struct nps_core_int_ena_w1s - NPS core interrupt enable set register + * @host_nps_wr_err: Reads or sets enable for + * NPS_CORE_INT[HOST_NPS_WR_ERR]. + * @npco_dma_malform: Reads or sets enable for + * NPS_CORE_INT[NPCO_DMA_MALFORM]. + * @exec_wr_timeout: Reads or sets enable for + * NPS_CORE_INT[EXEC_WR_TIMEOUT]. + * @host_wr_timeout: Reads or sets enable for + * NPS_CORE_INT[HOST_WR_TIMEOUT]. + * @host_wr_err: Reads or sets enable for + * NPS_CORE_INT[HOST_WR_ERR] + */ +union nps_core_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz4 : 55; + u64 host_nps_wr_err : 1; + u64 npco_dma_malform : 1; + u64 exec_wr_timeout : 1; + u64 host_wr_timeout : 1; + u64 host_wr_err : 1; + u64 raz3 : 1; + u64 raz2 : 1; + u64 raz1 : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 raz1 : 1; + u64 raz2 : 1; + u64 raz3 : 1; + u64 host_wr_err : 1; + u64 host_wr_timeout : 1; + u64 exec_wr_timeout : 1; + u64 npco_dma_malform : 1; + u64 host_nps_wr_err : 1; + u64 raz4 : 55; +#endif + } s; +}; + +/** + * struct nps_core_gbl_vfcfg - Global VF Configuration Register. + * @ilk_disable: When set, this bit indicates that the ILK interface has + * been disabled. + * @obaf: BMO allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @ibaf: BMI allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @zaf: ZIP allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @aeaf: AE allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @seaf: SE allocation control + * 0 = allocation per queue + * 1 = allocate per VF + * @cfg: VF/PF mode. + */ +union nps_core_gbl_vfcfg { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz :55; + u64 ilk_disable :1; + u64 obaf :1; + u64 ibaf :1; + u64 zaf :1; + u64 aeaf :1; + u64 seaf :1; + u64 cfg :3; +#else + u64 cfg :3; + u64 seaf :1; + u64 aeaf :1; + u64 zaf :1; + u64 ibaf :1; + u64 obaf :1; + u64 ilk_disable :1; + u64 raz :55; +#endif + } s; +}; + +/** + * struct nps_core_int_active - NPS Core Interrupt Active Register + * @resend: Resend MSI-X interrupt if needs to handle interrupts + * Sofware can set this bit and then exit the ISR. + * @ocla: Set when any OCLA(0)_INT and corresponding OCLA(0_INT_ENA_W1C + * bit are set + * @mbox: Set when any NPS_PKT_MBOX_INT_LO/HI and corresponding + * NPS_PKT_MBOX_INT_LO_ENA_W1C/HI_ENA_W1C bits are set + * @emu: bit i is set in [EMU] when any EMU(i)_INT bit is set + * @bmo: Set when any BMO_INT bit is set + * @bmi: Set when any BMI_INT bit is set or when any non-RO + * BMI_INT and corresponding BMI_INT_ENA_W1C bits are both set + * @aqm: Set when any AQM_INT bit is set + * @zqm: Set when any ZQM_INT bit is set + * @efl: Set when any EFL_INT RO bit is set or when any non-RO EFL_INT + * and corresponding EFL_INT_ENA_W1C bits are both set + * @ilk: Set when any ILK_INT bit is set + * @lbc: Set when any LBC_INT RO bit is set or when any non-RO LBC_INT + * and corresponding LBC_INT_ENA_W1C bits are bot set + * @pem: Set when any PEM(0)_INT RO bit is set or when any non-RO + * PEM(0)_INT and corresponding PEM(0)_INT_ENA_W1C bit are both set + * @ucd: Set when any UCD_INT bit is set + * @zctl: Set when any ZIP_INT RO bit is set or when any non-RO ZIP_INT + * and corresponding ZIP_INT_ENA_W1C bits are both set + * @lbm: Set when any LBM_INT bit is set + * @nps_pkt: Set when any NPS_PKT_INT bit is set + * @nps_core: Set when any NPS_CORE_INT RO bit is set or when non-RO + * NPS_CORE_INT and corresponding NSP_CORE_INT_ENA_W1C bits are both set + */ +union nps_core_int_active { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 resend : 1; + u64 raz : 43; + u64 ocla : 1; + u64 mbox : 1; + u64 emu : 4; + u64 bmo : 1; + u64 bmi : 1; + u64 aqm : 1; + u64 zqm : 1; + u64 efl : 1; + u64 ilk : 1; + u64 lbc : 1; + u64 pem : 1; + u64 pom : 1; + u64 ucd : 1; + u64 zctl : 1; + u64 lbm : 1; + u64 nps_pkt : 1; + u64 nps_core : 1; +#else + u64 nps_core : 1; + u64 nps_pkt : 1; + u64 lbm : 1; + u64 zctl: 1; + u64 ucd : 1; + u64 pom : 1; + u64 pem : 1; + u64 lbc : 1; + u64 ilk : 1; + u64 efl : 1; + u64 zqm : 1; + u64 aqm : 1; + u64 bmi : 1; + u64 bmo : 1; + u64 emu : 4; + u64 mbox : 1; + u64 ocla : 1; + u64 raz : 43; + u64 resend : 1; +#endif + } s; +}; + +/** + * struct efl_core_int - EFL Interrupt Registers + * @epci_decode_err: EPCI decoded a transacation that was unknown + * This error should only occurred when there is a micrcode/SE error + * and should be considered fatal + * @ae_err: An AE uncorrectable error occurred. + * See EFL_CORE(0..3)_AE_ERR_INT + * @se_err: An SE uncorrectable error occurred. + * See EFL_CORE(0..3)_SE_ERR_INT + * @dbe: Double-bit error occurred in EFL + * @sbe: Single-bit error occurred in EFL + * @d_left: Asserted when new POM-Header-BMI-data is + * being sent to an Exec, and that Exec has Not read all BMI + * data associated with the previous POM header + * @len_ovr: Asserted when an Exec-Read is issued that is more than + * 14 greater in length that the BMI data left to be read + */ +union efl_core_int { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 57; + u64 epci_decode_err : 1; + u64 ae_err : 1; + u64 se_err : 1; + u64 dbe : 1; + u64 sbe : 1; + u64 d_left : 1; + u64 len_ovr : 1; +#else + u64 len_ovr : 1; + u64 d_left : 1; + u64 sbe : 1; + u64 dbe : 1; + u64 se_err : 1; + u64 ae_err : 1; + u64 epci_decode_err : 1; + u64 raz : 57; +#endif + } s; +}; + +/** + * struct efl_core_int_ena_w1s - EFL core interrupt enable set register + * @epci_decode_err: Reads or sets enable for + * EFL_CORE(0..3)_INT[EPCI_DECODE_ERR]. + * @d_left: Reads or sets enable for + * EFL_CORE(0..3)_INT[D_LEFT]. + * @len_ovr: Reads or sets enable for + * EFL_CORE(0..3)_INT[LEN_OVR]. + */ +union efl_core_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_7_63 : 57; + u64 epci_decode_err : 1; + u64 raz_2_5 : 4; + u64 d_left : 1; + u64 len_ovr : 1; +#else + u64 len_ovr : 1; + u64 d_left : 1; + u64 raz_2_5 : 4; + u64 epci_decode_err : 1; + u64 raz_7_63 : 57; +#endif + } s; +}; + +/** + * struct efl_rnm_ctl_status - RNM Control and Status Register + * @ent_sel: Select input to RNM FIFO + * @exp_ent: Exported entropy enable for random number generator + * @rng_rst: Reset to RNG. Setting this bit to 1 cancels the generation + * of the current random number. + * @rnm_rst: Reset the RNM. Setting this bit to 1 clears all sorted numbers + * in the random number memory. + * @rng_en: Enabled the output of the RNG. + * @ent_en: Entropy enable for random number generator. + */ +union efl_rnm_ctl_status { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_9_63 : 55; + u64 ent_sel : 4; + u64 exp_ent : 1; + u64 rng_rst : 1; + u64 rnm_rst : 1; + u64 rng_en : 1; + u64 ent_en : 1; +#else + u64 ent_en : 1; + u64 rng_en : 1; + u64 rnm_rst : 1; + u64 rng_rst : 1; + u64 exp_ent : 1; + u64 ent_sel : 4; + u64 raz_9_63 : 55; +#endif + } s; +}; + +/** + * struct bmi_ctl - BMI control register + * @ilk_hdrq_thrsh: Maximum number of header queue locations + * that ILK packets may consume. When the threshold is + * exceeded ILK_XOFF is sent to the BMI_X2P_ARB. + * @nps_hdrq_thrsh: Maximum number of header queue locations + * that NPS packets may consume. When the threshold is + * exceeded NPS_XOFF is sent to the BMI_X2P_ARB. + * @totl_hdrq_thrsh: Maximum number of header queue locations + * that the sum of ILK and NPS packets may consume. + * @ilk_free_thrsh: Maximum number of buffers that ILK packet + * flows may consume before ILK_XOFF is sent to the BMI_X2P_ARB. + * @nps_free_thrsh: Maximum number of buffers that NPS packet + * flows may consume before NPS XOFF is sent to the BMI_X2p_ARB. + * @totl_free_thrsh: Maximum number of buffers that bot ILK and NPS + * packet flows may consume before both NPS_XOFF and ILK_XOFF + * are asserted to the BMI_X2P_ARB. + * @max_pkt_len: Maximum packet length, integral number of 256B + * buffers. + */ +union bmi_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_56_63 : 8; + u64 ilk_hdrq_thrsh : 8; + u64 nps_hdrq_thrsh : 8; + u64 totl_hdrq_thrsh : 8; + u64 ilk_free_thrsh : 8; + u64 nps_free_thrsh : 8; + u64 totl_free_thrsh : 8; + u64 max_pkt_len : 8; +#else + u64 max_pkt_len : 8; + u64 totl_free_thrsh : 8; + u64 nps_free_thrsh : 8; + u64 ilk_free_thrsh : 8; + u64 totl_hdrq_thrsh : 8; + u64 nps_hdrq_thrsh : 8; + u64 ilk_hdrq_thrsh : 8; + u64 raz_56_63 : 8; +#endif + } s; +}; + +/** + * struct bmi_int_ena_w1s - BMI interrupt enable set register + * @ilk_req_oflw: Reads or sets enable for + * BMI_INT[ILK_REQ_OFLW]. + * @nps_req_oflw: Reads or sets enable for + * BMI_INT[NPS_REQ_OFLW]. + * @fpf_undrrn: Reads or sets enable for + * BMI_INT[FPF_UNDRRN]. + * @eop_err_ilk: Reads or sets enable for + * BMI_INT[EOP_ERR_ILK]. + * @eop_err_nps: Reads or sets enable for + * BMI_INT[EOP_ERR_NPS]. + * @sop_err_ilk: Reads or sets enable for + * BMI_INT[SOP_ERR_ILK]. + * @sop_err_nps: Reads or sets enable for + * BMI_INT[SOP_ERR_NPS]. + * @pkt_rcv_err_ilk: Reads or sets enable for + * BMI_INT[PKT_RCV_ERR_ILK]. + * @pkt_rcv_err_nps: Reads or sets enable for + * BMI_INT[PKT_RCV_ERR_NPS]. + * @max_len_err_ilk: Reads or sets enable for + * BMI_INT[MAX_LEN_ERR_ILK]. + * @max_len_err_nps: Reads or sets enable for + * BMI_INT[MAX_LEN_ERR_NPS]. + */ +union bmi_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_13_63 : 51; + u64 ilk_req_oflw : 1; + u64 nps_req_oflw : 1; + u64 raz_10 : 1; + u64 raz_9 : 1; + u64 fpf_undrrn : 1; + u64 eop_err_ilk : 1; + u64 eop_err_nps : 1; + u64 sop_err_ilk : 1; + u64 sop_err_nps : 1; + u64 pkt_rcv_err_ilk : 1; + u64 pkt_rcv_err_nps : 1; + u64 max_len_err_ilk : 1; + u64 max_len_err_nps : 1; +#else + u64 max_len_err_nps : 1; + u64 max_len_err_ilk : 1; + u64 pkt_rcv_err_nps : 1; + u64 pkt_rcv_err_ilk : 1; + u64 sop_err_nps : 1; + u64 sop_err_ilk : 1; + u64 eop_err_nps : 1; + u64 eop_err_ilk : 1; + u64 fpf_undrrn : 1; + u64 raz_9 : 1; + u64 raz_10 : 1; + u64 nps_req_oflw : 1; + u64 ilk_req_oflw : 1; + u64 raz_13_63 : 51; +#endif + } s; +}; + +/** + * struct bmo_ctl2 - BMO Control2 Register + * @arb_sel: Determines P2X Arbitration + * @ilk_buf_thrsh: Maximum number of buffers that the + * ILK packet flows may consume before ILK XOFF is + * asserted to the POM. + * @nps_slc_buf_thrsh: Maximum number of buffers that the + * NPS_SLC packet flow may consume before NPS_SLC XOFF is + * asserted to the POM. + * @nps_uns_buf_thrsh: Maximum number of buffers that the + * NPS_UNS packet flow may consume before NPS_UNS XOFF is + * asserted to the POM. + * @totl_buf_thrsh: Maximum number of buffers that ILK, NPS_UNS and + * NPS_SLC packet flows may consume before NPS_UNS XOFF, NSP_SLC and + * ILK_XOFF are all asserted POM. + */ +union bmo_ctl2 { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 arb_sel : 1; + u64 raz_32_62 : 31; + u64 ilk_buf_thrsh : 8; + u64 nps_slc_buf_thrsh : 8; + u64 nps_uns_buf_thrsh : 8; + u64 totl_buf_thrsh : 8; +#else + u64 totl_buf_thrsh : 8; + u64 nps_uns_buf_thrsh : 8; + u64 nps_slc_buf_thrsh : 8; + u64 ilk_buf_thrsh : 8; + u64 raz_32_62 : 31; + u64 arb_sel : 1; +#endif + } s; +}; + +/** + * struct pom_int_ena_w1s - POM interrupt enable set register + * @illegal_intf: Reads or sets enable for POM_INT[ILLEGAL_INTF]. + * @illegal_dport: Reads or sets enable for POM_INT[ILLEGAL_DPORT]. + */ +union pom_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 60; + u64 illegal_intf : 1; + u64 illegal_dport : 1; + u64 raz1 : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 raz1 : 1; + u64 illegal_dport : 1; + u64 illegal_intf : 1; + u64 raz2 : 60; +#endif + } s; +}; + +/** + * struct lbc_inval_ctl - LBC invalidation control register + * @wait_timer: Wait timer for wait state. [WAIT_TIMER] must + * always be written with its reset value. + * @cam_inval_start: Software should write [CAM_INVAL_START]=1 + * to initiate an LBC cache invalidation. After this, software + * should read LBC_INVAL_STATUS until LBC_INVAL_STATUS[DONE] is set. + * LBC hardware clears [CAVM_INVAL_START] before software can + * observed LBC_INVAL_STATUS[DONE] to be set + */ +union lbc_inval_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 48; + u64 wait_timer : 8; + u64 raz1 : 6; + u64 cam_inval_start : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 cam_inval_start : 1; + u64 raz1 : 6; + u64 wait_timer : 8; + u64 raz2 : 48; +#endif + } s; +}; + +/** + * struct lbc_int_ena_w1s - LBC interrupt enable set register + * @cam_hard_err: Reads or sets enable for LBC_INT[CAM_HARD_ERR]. + * @cam_inval_abort: Reads or sets enable for LBC_INT[CAM_INVAL_ABORT]. + * @over_fetch_err: Reads or sets enable for LBC_INT[OVER_FETCH_ERR]. + * @cache_line_to_err: Reads or sets enable for + * LBC_INT[CACHE_LINE_TO_ERR]. + * @cam_soft_err: Reads or sets enable for + * LBC_INT[CAM_SOFT_ERR]. + * @dma_rd_err: Reads or sets enable for + * LBC_INT[DMA_RD_ERR]. + */ +union lbc_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_10_63 : 54; + u64 cam_hard_err : 1; + u64 cam_inval_abort : 1; + u64 over_fetch_err : 1; + u64 cache_line_to_err : 1; + u64 raz_2_5 : 4; + u64 cam_soft_err : 1; + u64 dma_rd_err : 1; +#else + u64 dma_rd_err : 1; + u64 cam_soft_err : 1; + u64 raz_2_5 : 4; + u64 cache_line_to_err : 1; + u64 over_fetch_err : 1; + u64 cam_inval_abort : 1; + u64 cam_hard_err : 1; + u64 raz_10_63 : 54; +#endif + } s; +}; + +/** + * struct lbc_int - LBC interrupt summary register + * @cam_hard_err: indicates a fatal hardware error. + * It requires system reset. + * When [CAM_HARD_ERR] is set, LBC stops logging any new information in + * LBC_POM_MISS_INFO_LOG, + * LBC_POM_MISS_ADDR_LOG, + * LBC_EFL_MISS_INFO_LOG, and + * LBC_EFL_MISS_ADDR_LOG. + * Software should sample them. + * @cam_inval_abort: indicates a fatal hardware error. + * System reset is required. + * @over_fetch_err: indicates a fatal hardware error + * System reset is required + * @cache_line_to_err: is a debug feature. + * This timeout interrupt bit tells the software that + * a cacheline in LBC has non-zero usage and the context + * has not been used for greater than the + * LBC_TO_CNT[TO_CNT] time interval. + * @sbe: Memory SBE error. This is recoverable via ECC. + * See LBC_ECC_INT for more details. + * @dbe: Memory DBE error. This is a fatal and requires a + * system reset. + * @pref_dat_len_mismatch_err: Summary bit for context length + * mismatch errors. + * @rd_dat_len_mismatch_err: Summary bit for SE read data length + * greater than data prefect length errors. + * @cam_soft_err: is recoverable. Software must complete a + * LBC_INVAL_CTL[CAM_INVAL_START] invalidation sequence and + * then clear [CAM_SOFT_ERR]. + * @dma_rd_err: A context prefect read of host memory returned with + * a read error. + */ +union lbc_int { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_10_63 : 54; + u64 cam_hard_err : 1; + u64 cam_inval_abort : 1; + u64 over_fetch_err : 1; + u64 cache_line_to_err : 1; + u64 sbe : 1; + u64 dbe : 1; + u64 pref_dat_len_mismatch_err : 1; + u64 rd_dat_len_mismatch_err : 1; + u64 cam_soft_err : 1; + u64 dma_rd_err : 1; +#else + u64 dma_rd_err : 1; + u64 cam_soft_err : 1; + u64 rd_dat_len_mismatch_err : 1; + u64 pref_dat_len_mismatch_err : 1; + u64 dbe : 1; + u64 sbe : 1; + u64 cache_line_to_err : 1; + u64 over_fetch_err : 1; + u64 cam_inval_abort : 1; + u64 cam_hard_err : 1; + u64 raz_10_63 : 54; +#endif + } s; +}; + +/** + * struct lbc_inval_status: LBC Invalidation status register + * @cam_clean_entry_complete_cnt: The number of entries that are + * cleaned up successfully. + * @cam_clean_entry_cnt: The number of entries that have the CAM + * inval command issued. + * @cam_inval_state: cam invalidation FSM state + * @cam_inval_abort: cam invalidation abort + * @cam_rst_rdy: lbc_cam reset ready + * @done: LBC clears [DONE] when + * LBC_INVAL_CTL[CAM_INVAL_START] is written with a one, + * and sets [DONE] when it completes the invalidation + * sequence. + */ +union lbc_inval_status { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz3 : 23; + u64 cam_clean_entry_complete_cnt : 9; + u64 raz2 : 7; + u64 cam_clean_entry_cnt : 9; + u64 raz1 : 5; + u64 cam_inval_state : 3; + u64 raz0 : 5; + u64 cam_inval_abort : 1; + u64 cam_rst_rdy : 1; + u64 done : 1; +#else + u64 done : 1; + u64 cam_rst_rdy : 1; + u64 cam_inval_abort : 1; + u64 raz0 : 5; + u64 cam_inval_state : 3; + u64 raz1 : 5; + u64 cam_clean_entry_cnt : 9; + u64 raz2 : 7; + u64 cam_clean_entry_complete_cnt : 9; + u64 raz3 : 23; +#endif + } s; +}; + +#endif /* __NITROX_CSR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h new file mode 100644 index 000000000000..2793a106b552 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h @@ -0,0 +1,175 @@ +#ifndef __NITROX_DEV_H +#define __NITROX_DEV_H + +#include +#include +#include + +#define VERSION_LEN 32 + +struct nitrox_cmdq { + /* command queue lock */ + spinlock_t cmdq_lock; + /* response list lock */ + spinlock_t response_lock; + /* backlog list lock */ + spinlock_t backlog_lock; + + /* request submitted to chip, in progress */ + struct list_head response_head; + /* hw queue full, hold in backlog list */ + struct list_head backlog_head; + + /* doorbell address */ + u8 __iomem *dbell_csr_addr; + /* base address of the queue */ + u8 *head; + + struct nitrox_device *ndev; + /* flush pending backlog commands */ + struct work_struct backlog_qflush; + + /* requests posted waiting for completion */ + atomic_t pending_count; + /* requests in backlog queues */ + atomic_t backlog_count; + + /* command size 32B/64B */ + u8 instr_size; + u8 qno; + u32 qsize; + + /* unaligned addresses */ + u8 *head_unaligned; + dma_addr_t dma_unaligned; + /* dma address of the base */ + dma_addr_t dma; +}; + +struct nitrox_hw { + /* firmware version */ + char fw_name[VERSION_LEN]; + + u16 vendor_id; + u16 device_id; + u8 revision_id; + + /* CNN55XX cores */ + u8 se_cores; + u8 ae_cores; + u8 zip_cores; +}; + +#define MAX_MSIX_VECTOR_NAME 20 +/** + * vectors for queues (64 AE, 64 SE and 64 ZIP) and + * error condition/mailbox. + */ +#define MAX_MSIX_VECTORS 192 + +struct nitrox_msix { + struct msix_entry *entries; + char **names; + DECLARE_BITMAP(irqs, MAX_MSIX_VECTORS); + u32 nr_entries; +}; + +struct bh_data { + /* slc port completion count address */ + u8 __iomem *completion_cnt_csr_addr; + + struct nitrox_cmdq *cmdq; + struct tasklet_struct resp_handler; +}; + +struct nitrox_bh { + struct bh_data *slc; +}; + +/* NITROX-5 driver state */ +#define NITROX_UCODE_LOADED 0 +#define NITROX_READY 1 + +/* command queue size */ +#define DEFAULT_CMD_QLEN 2048 +/* command timeout in milliseconds */ +#define CMD_TIMEOUT 2000 + +#define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev)) +#define PF_MODE 0 + +#define NITROX_CSR_ADDR(ndev, offset) \ + ((ndev)->bar_addr + (offset)) + +/** + * struct nitrox_device - NITROX Device Information. + * @list: pointer to linked list of devices + * @bar_addr: iomap address + * @pdev: PCI device information + * @status: NITROX status + * @timeout: Request timeout in jiffies + * @refcnt: Device usage count + * @idx: device index (0..N) + * @node: NUMA node id attached + * @qlen: Command queue length + * @nr_queues: Number of command queues + * @ctx_pool: DMA pool for crypto context + * @pkt_cmdqs: SE Command queues + * @msix: MSI-X information + * @bh: post processing work + * @hw: hardware information + */ +struct nitrox_device { + struct list_head list; + + u8 __iomem *bar_addr; + struct pci_dev *pdev; + + unsigned long status; + unsigned long timeout; + refcount_t refcnt; + + u8 idx; + int node; + u16 qlen; + u16 nr_queues; + + struct dma_pool *ctx_pool; + struct nitrox_cmdq *pkt_cmdqs; + + struct nitrox_msix msix; + struct nitrox_bh bh; + + struct nitrox_hw hw; +}; + +/** + * nitrox_read_csr - Read from device register + * @ndev: NITROX device + * @offset: offset of the register to read + * + * Returns: value read + */ +static inline u64 nitrox_read_csr(struct nitrox_device *ndev, u64 offset) +{ + return readq(ndev->bar_addr + offset); +} + +/** + * nitrox_write_csr - Write to device register + * @ndev: NITROX device + * @offset: offset of the register to write + * @value: value to write + */ +static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset, + u64 value) +{ + writeq(value, (ndev->bar_addr + offset)); +} + +static inline int nitrox_ready(struct nitrox_device *ndev) +{ + return test_bit(NITROX_READY, &ndev->status); +} + +#endif /* __NITROX_DEV_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c new file mode 100644 index 000000000000..f0655f82fa7d --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -0,0 +1,401 @@ +#include + +#include "nitrox_dev.h" +#include "nitrox_csr.h" + +/** + * emu_enable_cores - Enable EMU cluster cores. + * @ndev: N5 device + */ +static void emu_enable_cores(struct nitrox_device *ndev) +{ + union emu_se_enable emu_se; + union emu_ae_enable emu_ae; + int i; + + /* AE cores 20 per cluster */ + emu_ae.value = 0; + emu_ae.s.enable = 0xfffff; + + /* SE cores 16 per cluster */ + emu_se.value = 0; + emu_se.s.enable = 0xffff; + + /* enable per cluster cores */ + for (i = 0; i < NR_CLUSTERS; i++) { + nitrox_write_csr(ndev, EMU_AE_ENABLEX(i), emu_ae.value); + nitrox_write_csr(ndev, EMU_SE_ENABLEX(i), emu_se.value); + } +} + +/** + * nitrox_config_emu_unit - configure EMU unit. + * @ndev: N5 device + */ +void nitrox_config_emu_unit(struct nitrox_device *ndev) +{ + union emu_wd_int_ena_w1s emu_wd_int; + union emu_ge_int_ena_w1s emu_ge_int; + u64 offset; + int i; + + /* enable cores */ + emu_enable_cores(ndev); + + /* enable general error and watch dog interrupts */ + emu_ge_int.value = 0; + emu_ge_int.s.se_ge = 0xffff; + emu_ge_int.s.ae_ge = 0xfffff; + emu_wd_int.value = 0; + emu_wd_int.s.se_wd = 1; + + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EMU_WD_INT_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, emu_wd_int.value); + offset = EMU_GE_INT_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, emu_ge_int.value); + } +} + +static void reset_pkt_input_ring(struct nitrox_device *ndev, int ring) +{ + union nps_pkt_in_instr_ctl pkt_in_ctl; + union nps_pkt_in_instr_baoff_dbell pkt_in_dbell; + union nps_pkt_in_done_cnts pkt_in_cnts; + u64 offset; + + offset = NPS_PKT_IN_INSTR_CTLX(ring); + /* disable the ring */ + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + pkt_in_ctl.s.enb = 0; + nitrox_write_csr(ndev, offset, pkt_in_ctl.value); + usleep_range(100, 150); + + /* wait to clear [ENB] */ + do { + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + } while (pkt_in_ctl.s.enb); + + /* clear off door bell counts */ + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(ring); + pkt_in_dbell.value = 0; + pkt_in_dbell.s.dbell = 0xffffffff; + nitrox_write_csr(ndev, offset, pkt_in_dbell.value); + + /* clear done counts */ + offset = NPS_PKT_IN_DONE_CNTSX(ring); + pkt_in_cnts.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, pkt_in_cnts.value); + usleep_range(50, 100); +} + +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring) +{ + union nps_pkt_in_instr_ctl pkt_in_ctl; + u64 offset; + + /* 64-byte instruction size */ + offset = NPS_PKT_IN_INSTR_CTLX(ring); + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + pkt_in_ctl.s.is64b = 1; + pkt_in_ctl.s.enb = 1; + nitrox_write_csr(ndev, offset, pkt_in_ctl.value); + + /* wait for set [ENB] */ + do { + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + } while (!pkt_in_ctl.s.enb); +} + +/** + * nitrox_config_pkt_input_rings - configure Packet Input Rings + * @ndev: N5 device + */ +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i]; + union nps_pkt_in_instr_rsize pkt_in_rsize; + u64 offset; + + reset_pkt_input_ring(ndev, i); + + /* configure ring base address 16-byte aligned, + * size and interrupt threshold. + */ + offset = NPS_PKT_IN_INSTR_BADDRX(i); + nitrox_write_csr(ndev, NPS_PKT_IN_INSTR_BADDRX(i), cmdq->dma); + + /* configure ring size */ + offset = NPS_PKT_IN_INSTR_RSIZEX(i); + pkt_in_rsize.value = 0; + pkt_in_rsize.s.rsize = ndev->qlen; + nitrox_write_csr(ndev, offset, pkt_in_rsize.value); + + /* set high threshold for pkt input ring interrupts */ + offset = NPS_PKT_IN_INT_LEVELSX(i); + nitrox_write_csr(ndev, offset, 0xffffffff); + + enable_pkt_input_ring(ndev, i); + } +} + +static void reset_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_ctl pkt_slc_ctl; + union nps_pkt_slc_cnts pkt_slc_cnts; + u64 offset; + + /* disable slc port */ + offset = NPS_PKT_SLC_CTLX(port); + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + pkt_slc_ctl.s.enb = 0; + nitrox_write_csr(ndev, offset, pkt_slc_ctl.value); + usleep_range(100, 150); + + /* wait to clear [ENB] */ + do { + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + } while (pkt_slc_ctl.s.enb); + + /* clear slc counters */ + offset = NPS_PKT_SLC_CNTSX(port); + pkt_slc_cnts.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, pkt_slc_cnts.value); + usleep_range(50, 100); +} + +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_ctl pkt_slc_ctl; + u64 offset; + + offset = NPS_PKT_SLC_CTLX(port); + pkt_slc_ctl.value = 0; + pkt_slc_ctl.s.enb = 1; + + /* + * 8 trailing 0x00 bytes will be added + * to the end of the outgoing packet. + */ + pkt_slc_ctl.s.z = 1; + /* enable response header */ + pkt_slc_ctl.s.rh = 1; + nitrox_write_csr(ndev, offset, pkt_slc_ctl.value); + + /* wait to set [ENB] */ + do { + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + } while (!pkt_slc_ctl.s.enb); +} + +static void config_single_pkt_solicit_port(struct nitrox_device *ndev, + int port) +{ + union nps_pkt_slc_int_levels pkt_slc_int; + u64 offset; + + reset_pkt_solicit_port(ndev, port); + + offset = NPS_PKT_SLC_INT_LEVELSX(port); + pkt_slc_int.value = 0; + /* time interrupt threshold */ + pkt_slc_int.s.timet = 0x3fffff; + nitrox_write_csr(ndev, offset, pkt_slc_int.value); + + enable_pkt_solicit_port(ndev, port); +} + +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) + config_single_pkt_solicit_port(ndev, i); +} + +/** + * enable_nps_interrupts - enable NPS interrutps + * @ndev: N5 device. + * + * This includes NPS core, packet in and slc interrupts. + */ +static void enable_nps_interrupts(struct nitrox_device *ndev) +{ + union nps_core_int_ena_w1s core_int; + + /* NPS core interrutps */ + core_int.value = 0; + core_int.s.host_wr_err = 1; + core_int.s.host_wr_timeout = 1; + core_int.s.exec_wr_timeout = 1; + core_int.s.npco_dma_malform = 1; + core_int.s.host_nps_wr_err = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value); + + /* NPS packet in ring interrupts */ + nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL)); + /* NPS packet slc port interrupts */ + nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL)); +} + +void nitrox_config_nps_unit(struct nitrox_device *ndev) +{ + union nps_core_gbl_vfcfg core_gbl_vfcfg; + + /* endian control information */ + nitrox_write_csr(ndev, NPS_CORE_CONTROL, 1ULL); + + /* disable ILK interface */ + core_gbl_vfcfg.value = 0; + core_gbl_vfcfg.s.ilk_disable = 1; + core_gbl_vfcfg.s.cfg = PF_MODE; + nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value); + /* config input and solicit ports */ + nitrox_config_pkt_input_rings(ndev); + nitrox_config_pkt_solicit_ports(ndev); + + /* enable interrupts */ + enable_nps_interrupts(ndev); +} + +void nitrox_config_pom_unit(struct nitrox_device *ndev) +{ + union pom_int_ena_w1s pom_int; + int i; + + /* enable pom interrupts */ + pom_int.value = 0; + pom_int.s.illegal_dport = 1; + nitrox_write_csr(ndev, POM_INT_ENA_W1S, pom_int.value); + + /* enable perf counters */ + for (i = 0; i < ndev->hw.se_cores; i++) + nitrox_write_csr(ndev, POM_PERF_CTL, BIT_ULL(i)); +} + +/** + * nitrox_config_rand_unit - enable N5 random number unit + * @ndev: N5 device + */ +void nitrox_config_rand_unit(struct nitrox_device *ndev) +{ + union efl_rnm_ctl_status efl_rnm_ctl; + u64 offset; + + offset = EFL_RNM_CTL_STATUS; + efl_rnm_ctl.value = nitrox_read_csr(ndev, offset); + efl_rnm_ctl.s.ent_en = 1; + efl_rnm_ctl.s.rng_en = 1; + nitrox_write_csr(ndev, offset, efl_rnm_ctl.value); +} + +void nitrox_config_efl_unit(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + union efl_core_int_ena_w1s efl_core_int; + u64 offset; + + /* EFL core interrupts */ + offset = EFL_CORE_INT_ENA_W1SX(i); + efl_core_int.value = 0; + efl_core_int.s.len_ovr = 1; + efl_core_int.s.d_left = 1; + efl_core_int.s.epci_decode_err = 1; + nitrox_write_csr(ndev, offset, efl_core_int.value); + + offset = EFL_CORE_VF_ERR_INT0_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = EFL_CORE_VF_ERR_INT1_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, (~0ULL)); + } +} + +void nitrox_config_bmi_unit(struct nitrox_device *ndev) +{ + union bmi_ctl bmi_ctl; + union bmi_int_ena_w1s bmi_int_ena; + u64 offset; + + /* no threshold limits for PCIe */ + offset = BMI_CTL; + bmi_ctl.value = nitrox_read_csr(ndev, offset); + bmi_ctl.s.max_pkt_len = 0xff; + bmi_ctl.s.nps_free_thrsh = 0xff; + bmi_ctl.s.nps_hdrq_thrsh = 0x7a; + nitrox_write_csr(ndev, offset, bmi_ctl.value); + + /* enable interrupts */ + offset = BMI_INT_ENA_W1S; + bmi_int_ena.value = 0; + bmi_int_ena.s.max_len_err_nps = 1; + bmi_int_ena.s.pkt_rcv_err_nps = 1; + bmi_int_ena.s.fpf_undrrn = 1; + nitrox_write_csr(ndev, offset, bmi_int_ena.value); +} + +void nitrox_config_bmo_unit(struct nitrox_device *ndev) +{ + union bmo_ctl2 bmo_ctl2; + u64 offset; + + /* no threshold limits for PCIe */ + offset = BMO_CTL2; + bmo_ctl2.value = nitrox_read_csr(ndev, offset); + bmo_ctl2.s.nps_slc_buf_thrsh = 0xff; + nitrox_write_csr(ndev, offset, bmo_ctl2.value); +} + +void invalidate_lbc(struct nitrox_device *ndev) +{ + union lbc_inval_ctl lbc_ctl; + union lbc_inval_status lbc_stat; + u64 offset; + + /* invalidate LBC */ + offset = LBC_INVAL_CTL; + lbc_ctl.value = nitrox_read_csr(ndev, offset); + lbc_ctl.s.cam_inval_start = 1; + nitrox_write_csr(ndev, offset, lbc_ctl.value); + + offset = LBC_INVAL_STATUS; + + do { + lbc_stat.value = nitrox_read_csr(ndev, offset); + } while (!lbc_stat.s.done); +} + +void nitrox_config_lbc_unit(struct nitrox_device *ndev) +{ + union lbc_int_ena_w1s lbc_int_ena; + u64 offset; + + invalidate_lbc(ndev); + + /* enable interrupts */ + offset = LBC_INT_ENA_W1S; + lbc_int_ena.value = 0; + lbc_int_ena.s.dma_rd_err = 1; + lbc_int_ena.s.over_fetch_err = 1; + lbc_int_ena.s.cam_inval_abort = 1; + lbc_int_ena.s.cam_hard_err = 1; + nitrox_write_csr(ndev, offset, lbc_int_ena.value); + + offset = LBC_PLM_VF1_64_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = LBC_PLM_VF65_128_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + + offset = LBC_ELM_VF1_64_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = LBC_ELM_VF65_128_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c new file mode 100644 index 000000000000..71f934871a89 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -0,0 +1,467 @@ +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_csr.h" +#include "nitrox_common.h" + +#define NR_RING_VECTORS 3 +#define NPS_CORE_INT_ACTIVE_ENTRY 192 + +/** + * nps_pkt_slc_isr - IRQ handler for NPS solicit port + * @irq: irq number + * @data: argument + */ +static irqreturn_t nps_pkt_slc_isr(int irq, void *data) +{ + struct bh_data *slc = data; + union nps_pkt_slc_cnts pkt_slc_cnts; + + pkt_slc_cnts.value = readq(slc->completion_cnt_csr_addr); + /* New packet on SLC output port */ + if (pkt_slc_cnts.s.slc_int) + tasklet_hi_schedule(&slc->resp_handler); + + return IRQ_HANDLED; +} + +static void clear_nps_core_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + /* Write 1 to clear */ + value = nitrox_read_csr(ndev, NPS_CORE_INT); + nitrox_write_csr(ndev, NPS_CORE_INT, value); + + dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT 0x%016llx\n", value); +} + +static void clear_nps_pkt_err_intr(struct nitrox_device *ndev) +{ + union nps_pkt_int pkt_int; + unsigned long value, offset; + int i; + + pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT); + dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT 0x%016llx\n", + pkt_int.value); + + if (pkt_int.s.slc_err) { + offset = NPS_PKT_SLC_ERR_TYPE; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_ERR_TYPE 0x%016lx\n", value); + + offset = NPS_PKT_SLC_RERR_LO; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + /* enable the solicit ports */ + for_each_set_bit(i, &value, BITS_PER_LONG) + enable_pkt_solicit_port(ndev, i); + + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_RERR_LO 0x%016lx\n", value); + + offset = NPS_PKT_SLC_RERR_HI; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_RERR_HI 0x%016lx\n", value); + } + + if (pkt_int.s.in_err) { + offset = NPS_PKT_IN_ERR_TYPE; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_ERR_TYPE 0x%016lx\n", value); + offset = NPS_PKT_IN_RERR_LO; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + /* enable the input ring */ + for_each_set_bit(i, &value, BITS_PER_LONG) + enable_pkt_input_ring(ndev, i); + + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_RERR_LO 0x%016lx\n", value); + + offset = NPS_PKT_IN_RERR_HI; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_RERR_HI 0x%016lx\n", value); + } +} + +static void clear_pom_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, POM_INT); + nitrox_write_csr(ndev, POM_INT, value); + dev_err_ratelimited(DEV(ndev), "POM_INT 0x%016llx\n", value); +} + +static void clear_pem_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, PEM0_INT); + nitrox_write_csr(ndev, PEM0_INT, value); + dev_err_ratelimited(DEV(ndev), "PEM(0)_INT 0x%016llx\n", value); +} + +static void clear_lbc_err_intr(struct nitrox_device *ndev) +{ + union lbc_int lbc_int; + u64 value, offset; + int i; + + lbc_int.value = nitrox_read_csr(ndev, LBC_INT); + dev_err_ratelimited(DEV(ndev), "LBC_INT 0x%016llx\n", lbc_int.value); + + if (lbc_int.s.dma_rd_err) { + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EFL_CORE_VF_ERR_INT0X(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = EFL_CORE_VF_ERR_INT1X(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + } + + if (lbc_int.s.cam_soft_err) { + dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n"); + invalidate_lbc(ndev); + } + + if (lbc_int.s.pref_dat_len_mismatch_err) { + offset = LBC_PLM_VF1_64_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = LBC_PLM_VF65_128_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + + if (lbc_int.s.rd_dat_len_mismatch_err) { + offset = LBC_ELM_VF1_64_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = LBC_ELM_VF65_128_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + nitrox_write_csr(ndev, LBC_INT, lbc_int.value); +} + +static void clear_efl_err_intr(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + union efl_core_int core_int; + u64 value, offset; + + offset = EFL_CORE_INTX(i); + core_int.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, core_int.value); + dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT 0x%016llx\n", + i, core_int.value); + if (core_int.s.se_err) { + offset = EFL_CORE_SE_ERR_INTX(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + } +} + +static void clear_bmi_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, BMI_INT); + nitrox_write_csr(ndev, BMI_INT, value); + dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value); +} + +/** + * clear_nps_core_int_active - clear NPS_CORE_INT_ACTIVE interrupts + * @ndev: NITROX device + */ +static void clear_nps_core_int_active(struct nitrox_device *ndev) +{ + union nps_core_int_active core_int_active; + + core_int_active.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE); + + if (core_int_active.s.nps_core) + clear_nps_core_err_intr(ndev); + + if (core_int_active.s.nps_pkt) + clear_nps_pkt_err_intr(ndev); + + if (core_int_active.s.pom) + clear_pom_err_intr(ndev); + + if (core_int_active.s.pem) + clear_pem_err_intr(ndev); + + if (core_int_active.s.lbc) + clear_lbc_err_intr(ndev); + + if (core_int_active.s.efl) + clear_efl_err_intr(ndev); + + if (core_int_active.s.bmi) + clear_bmi_err_intr(ndev); + + /* If more work callback the ISR, set resend */ + core_int_active.s.resend = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int_active.value); +} + +static irqreturn_t nps_core_int_isr(int irq, void *data) +{ + struct nitrox_device *ndev = data; + + clear_nps_core_int_active(ndev); + + return IRQ_HANDLED; +} + +static int nitrox_enable_msix(struct nitrox_device *ndev) +{ + struct msix_entry *entries; + char **names; + int i, nr_entries, ret; + + /* + * PF MSI-X vectors + * + * Entry 0: NPS PKT ring 0 + * Entry 1: AQMQ ring 0 + * Entry 2: ZQM ring 0 + * Entry 3: NPS PKT ring 1 + * Entry 4: AQMQ ring 1 + * Entry 5: ZQM ring 1 + * .... + * Entry 192: NPS_CORE_INT_ACTIVE + */ + nr_entries = (ndev->nr_queues * NR_RING_VECTORS) + 1; + entries = kzalloc_node(nr_entries * sizeof(struct msix_entry), + GFP_KERNEL, ndev->node); + if (!entries) + return -ENOMEM; + + names = kcalloc(nr_entries, sizeof(char *), GFP_KERNEL); + if (!names) { + kfree(entries); + return -ENOMEM; + } + + /* fill entires */ + for (i = 0; i < (nr_entries - 1); i++) + entries[i].entry = i; + + entries[i].entry = NPS_CORE_INT_ACTIVE_ENTRY; + + for (i = 0; i < nr_entries; i++) { + *(names + i) = kzalloc(MAX_MSIX_VECTOR_NAME, GFP_KERNEL); + if (!(*(names + i))) { + ret = -ENOMEM; + goto msix_fail; + } + } + ndev->msix.entries = entries; + ndev->msix.names = names; + ndev->msix.nr_entries = nr_entries; + + ret = pci_enable_msix_exact(ndev->pdev, ndev->msix.entries, + ndev->msix.nr_entries); + if (ret) { + dev_err(&ndev->pdev->dev, "Failed to enable MSI-X IRQ(s) %d\n", + ret); + goto msix_fail; + } + return 0; + +msix_fail: + for (i = 0; i < nr_entries; i++) + kfree(*(names + i)); + + kfree(entries); + kfree(names); + return ret; +} + +static void nitrox_cleanup_pkt_slc_bh(struct nitrox_device *ndev) +{ + int i; + + if (!ndev->bh.slc) + return; + + for (i = 0; i < ndev->nr_queues; i++) { + struct bh_data *bh = &ndev->bh.slc[i]; + + tasklet_disable(&bh->resp_handler); + tasklet_kill(&bh->resp_handler); + } + kfree(ndev->bh.slc); + ndev->bh.slc = NULL; +} + +static int nitrox_setup_pkt_slc_bh(struct nitrox_device *ndev) +{ + u32 size; + int i; + + size = ndev->nr_queues * sizeof(struct bh_data); + ndev->bh.slc = kzalloc(size, GFP_KERNEL); + if (!ndev->bh.slc) + return -ENOMEM; + + for (i = 0; i < ndev->nr_queues; i++) { + struct bh_data *bh = &ndev->bh.slc[i]; + u64 offset; + + offset = NPS_PKT_SLC_CNTSX(i); + /* pre calculate completion count address */ + bh->completion_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset); + bh->cmdq = &ndev->pkt_cmdqs[i]; + + tasklet_init(&bh->resp_handler, pkt_slc_resp_handler, + (unsigned long)bh); + } + + return 0; +} + +static int nitrox_request_irqs(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + struct msix_entry *msix_ent = ndev->msix.entries; + int nr_ring_vectors, i = 0, ring, cpu, ret; + char *name; + + /* + * PF MSI-X vectors + * + * Entry 0: NPS PKT ring 0 + * Entry 1: AQMQ ring 0 + * Entry 2: ZQM ring 0 + * Entry 3: NPS PKT ring 1 + * .... + * Entry 192: NPS_CORE_INT_ACTIVE + */ + nr_ring_vectors = ndev->nr_queues * NR_RING_VECTORS; + + /* request irq for pkt ring/ports only */ + while (i < nr_ring_vectors) { + name = *(ndev->msix.names + i); + ring = (i / NR_RING_VECTORS); + snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-slc-ring%d", + ndev->idx, ring); + + ret = request_irq(msix_ent[i].vector, nps_pkt_slc_isr, 0, + name, &ndev->bh.slc[ring]); + if (ret) { + dev_err(&pdev->dev, "failed to get irq %d for %s\n", + msix_ent[i].vector, name); + return ret; + } + cpu = ring % num_online_cpus(); + irq_set_affinity_hint(msix_ent[i].vector, get_cpu_mask(cpu)); + + set_bit(i, ndev->msix.irqs); + i += NR_RING_VECTORS; + } + + /* Request IRQ for NPS_CORE_INT_ACTIVE */ + name = *(ndev->msix.names + i); + snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-nps-core-int", ndev->idx); + ret = request_irq(msix_ent[i].vector, nps_core_int_isr, 0, name, ndev); + if (ret) { + dev_err(&pdev->dev, "failed to get irq %d for %s\n", + msix_ent[i].vector, name); + return ret; + } + set_bit(i, ndev->msix.irqs); + + return 0; +} + +static void nitrox_disable_msix(struct nitrox_device *ndev) +{ + struct msix_entry *msix_ent = ndev->msix.entries; + char **names = ndev->msix.names; + int i = 0, ring, nr_ring_vectors; + + nr_ring_vectors = ndev->msix.nr_entries - 1; + + /* clear pkt ring irqs */ + while (i < nr_ring_vectors) { + if (test_and_clear_bit(i, ndev->msix.irqs)) { + ring = (i / NR_RING_VECTORS); + irq_set_affinity_hint(msix_ent[i].vector, NULL); + free_irq(msix_ent[i].vector, &ndev->bh.slc[ring]); + } + i += NR_RING_VECTORS; + } + irq_set_affinity_hint(msix_ent[i].vector, NULL); + free_irq(msix_ent[i].vector, ndev); + clear_bit(i, ndev->msix.irqs); + + kfree(ndev->msix.entries); + for (i = 0; i < ndev->msix.nr_entries; i++) + kfree(*(names + i)); + + kfree(names); + pci_disable_msix(ndev->pdev); +} + +/** + * nitrox_pf_cleanup_isr: Cleanup PF MSI-X and IRQ + * @ndev: NITROX device + */ +void nitrox_pf_cleanup_isr(struct nitrox_device *ndev) +{ + nitrox_disable_msix(ndev); + nitrox_cleanup_pkt_slc_bh(ndev); +} + +/** + * nitrox_init_isr - Initialize PF MSI-X vectors and IRQ + * @ndev: NITROX device + * + * Return: 0 on success, a negative value on failure. + */ +int nitrox_pf_init_isr(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_setup_pkt_slc_bh(ndev); + if (err) + return err; + + err = nitrox_enable_msix(ndev); + if (err) + goto msix_fail; + + err = nitrox_request_irqs(ndev); + if (err) + goto irq_fail; + + return 0; + +irq_fail: + nitrox_disable_msix(ndev); +msix_fail: + nitrox_cleanup_pkt_slc_bh(ndev); + return err; +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c new file mode 100644 index 000000000000..846e990d4f35 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c @@ -0,0 +1,172 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" +#include "nitrox_csr.h" + +#define CRYPTO_CTX_SIZE 256 + +/* command queue alignments */ +#define PKT_IN_ALIGN 16 + +static int cmdq_common_init(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + u32 qsize; + + qsize = (ndev->qlen) * cmdq->instr_size; + cmdq->head_unaligned = dma_zalloc_coherent(DEV(ndev), + (qsize + PKT_IN_ALIGN), + &cmdq->dma_unaligned, + GFP_KERNEL); + if (!cmdq->head_unaligned) + return -ENOMEM; + + cmdq->head = PTR_ALIGN(cmdq->head_unaligned, PKT_IN_ALIGN); + cmdq->dma = PTR_ALIGN(cmdq->dma_unaligned, PKT_IN_ALIGN); + cmdq->qsize = (qsize + PKT_IN_ALIGN); + + spin_lock_init(&cmdq->response_lock); + spin_lock_init(&cmdq->cmdq_lock); + spin_lock_init(&cmdq->backlog_lock); + + INIT_LIST_HEAD(&cmdq->response_head); + INIT_LIST_HEAD(&cmdq->backlog_head); + INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work); + + atomic_set(&cmdq->pending_count, 0); + atomic_set(&cmdq->backlog_count, 0); + return 0; +} + +static void cmdq_common_cleanup(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + + cancel_work_sync(&cmdq->backlog_qflush); + + dma_free_coherent(DEV(ndev), cmdq->qsize, + cmdq->head_unaligned, cmdq->dma_unaligned); + + atomic_set(&cmdq->pending_count, 0); + atomic_set(&cmdq->backlog_count, 0); + + cmdq->dbell_csr_addr = NULL; + cmdq->head = NULL; + cmdq->dma = 0; + cmdq->qsize = 0; + cmdq->instr_size = 0; +} + +static void nitrox_cleanup_pkt_cmdqs(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i]; + + cmdq_common_cleanup(cmdq); + } + kfree(ndev->pkt_cmdqs); + ndev->pkt_cmdqs = NULL; +} + +static int nitrox_init_pkt_cmdqs(struct nitrox_device *ndev) +{ + int i, err, size; + + size = ndev->nr_queues * sizeof(struct nitrox_cmdq); + ndev->pkt_cmdqs = kzalloc(size, GFP_KERNEL); + if (!ndev->pkt_cmdqs) + return -ENOMEM; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq; + u64 offset; + + cmdq = &ndev->pkt_cmdqs[i]; + cmdq->ndev = ndev; + cmdq->qno = i; + cmdq->instr_size = sizeof(struct nps_pkt_instr); + + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i); + /* SE ring doorbell address for this queue */ + cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset); + + err = cmdq_common_init(cmdq); + if (err) + goto pkt_cmdq_fail; + } + return 0; + +pkt_cmdq_fail: + nitrox_cleanup_pkt_cmdqs(ndev); + return err; +} + +static int create_crypto_dma_pool(struct nitrox_device *ndev) +{ + size_t size; + + /* Crypto context pool, 16 byte aligned */ + size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr); + ndev->ctx_pool = dma_pool_create("crypto-context", + DEV(ndev), size, 16, 0); + if (!ndev->ctx_pool) + return -ENOMEM; + + return 0; +} + +static void destroy_crypto_dma_pool(struct nitrox_device *ndev) +{ + if (!ndev->ctx_pool) + return; + + dma_pool_destroy(ndev->ctx_pool); + ndev->ctx_pool = NULL; +} + +/** + * nitrox_common_sw_init - allocate software resources. + * @ndev: NITROX device + * + * Allocates crypto context pools and command queues etc. + * + * Return: 0 on success, or a negative error code on error. + */ +int nitrox_common_sw_init(struct nitrox_device *ndev) +{ + int err = 0; + + /* per device crypto context pool */ + err = create_crypto_dma_pool(ndev); + if (err) + return err; + + err = nitrox_init_pkt_cmdqs(ndev); + if (err) + destroy_crypto_dma_pool(ndev); + + return err; +} + +/** + * nitrox_common_sw_cleanup - free software resources. + * @ndev: NITROX device + */ +void nitrox_common_sw_cleanup(struct nitrox_device *ndev) +{ + nitrox_cleanup_pkt_cmdqs(ndev); + destroy_crypto_dma_pool(ndev); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c new file mode 100644 index 000000000000..aa331202dcfa --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -0,0 +1,465 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_csr.h" + +#define CNN55XX_DEV_ID 0x12 +#define MAX_PF_QUEUES 64 +#define UCODE_HLEN 48 +#define SE_GROUP 0 + +#define DRIVER_VERSION "1.0" +/* SE microcode */ +#define SE_FW "cnn55xx_se.fw" + +static const char nitrox_driver_name[] = "CNN55XX"; + +static LIST_HEAD(ndevlist); +static DEFINE_MUTEX(devlist_lock); +static unsigned int num_devices; + +/** + * nitrox_pci_tbl - PCI Device ID Table + */ +static const struct pci_device_id nitrox_pci_tbl[] = { + {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0}, + /* required last entry */ + {0, } +}; +MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl); + +static unsigned int qlen = DEFAULT_CMD_QLEN; +module_param(qlen, uint, 0644); +MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); + +/** + * struct ucode - Firmware Header + * @id: microcode ID + * @version: firmware version + * @code_size: code section size + * @raz: alignment + * @code: code section + */ +struct ucode { + u8 id; + char version[VERSION_LEN - 1]; + __be32 code_size; + u8 raz[12]; + u64 code[0]; +}; + +/** + * write_to_ucd_unit - Write Firmware to NITROX UCD unit + */ +static void write_to_ucd_unit(struct nitrox_device *ndev, + struct ucode *ucode) +{ + u32 code_size = be32_to_cpu(ucode->code_size) * 2; + u64 offset, data; + int i = 0; + + /* + * UCD structure + * + * ------------- + * | BLK 7 | + * ------------- + * | BLK 6 | + * ------------- + * | ... | + * ------------- + * | BLK 0 | + * ------------- + * Total of 8 blocks, each size 32KB + */ + + /* set the block number */ + offset = UCD_UCODE_LOAD_BLOCK_NUM; + nitrox_write_csr(ndev, offset, 0); + + code_size = roundup(code_size, 8); + while (code_size) { + data = ucode->code[i]; + /* write 8 bytes at a time */ + offset = UCD_UCODE_LOAD_IDX_DATAX(i); + nitrox_write_csr(ndev, offset, data); + code_size -= 8; + i++; + } + + /* put all SE cores in group 0 */ + offset = POM_GRP_EXECMASKX(SE_GROUP); + nitrox_write_csr(ndev, offset, (~0ULL)); + + for (i = 0; i < ndev->hw.se_cores; i++) { + /* + * write block number and firware length + * bit:<2:0> block number + * bit:3 is set SE uses 32KB microcode + * bit:3 is clear SE uses 64KB microcode + */ + offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i); + nitrox_write_csr(ndev, offset, 0x8); + } + usleep_range(300, 400); +} + +static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name) +{ + const struct firmware *fw; + struct ucode *ucode; + int ret; + + dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name); + + ret = request_firmware(&fw, fw_name, DEV(ndev)); + if (ret < 0) { + dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name); + return ret; + } + + ucode = (struct ucode *)fw->data; + /* copy the firmware version */ + memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2)); + ndev->hw.fw_name[VERSION_LEN - 1] = '\0'; + + write_to_ucd_unit(ndev, ucode); + release_firmware(fw); + + set_bit(NITROX_UCODE_LOADED, &ndev->status); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + return 0; +} + +/** + * nitrox_add_to_devlist - add NITROX device to global device list + * @ndev: NITROX device + */ +static int nitrox_add_to_devlist(struct nitrox_device *ndev) +{ + struct nitrox_device *dev; + int ret = 0; + + INIT_LIST_HEAD(&ndev->list); + refcount_set(&ndev->refcnt, 1); + + mutex_lock(&devlist_lock); + list_for_each_entry(dev, &ndevlist, list) { + if (dev == ndev) { + ret = -EEXIST; + goto unlock; + } + } + ndev->idx = num_devices++; + list_add_tail(&ndev->list, &ndevlist); +unlock: + mutex_unlock(&devlist_lock); + return ret; +} + +/** + * nitrox_remove_from_devlist - remove NITROX device from + * global device list + * @ndev: NITROX device + */ +static void nitrox_remove_from_devlist(struct nitrox_device *ndev) +{ + mutex_lock(&devlist_lock); + list_del(&ndev->list); + num_devices--; + mutex_unlock(&devlist_lock); +} + +static int nitrox_reset_device(struct pci_dev *pdev) +{ + int pos = 0; + + pos = pci_save_state(pdev); + if (pos) { + dev_err(&pdev->dev, "Failed to save pci state\n"); + return -ENOMEM; + } + + pos = pci_pcie_cap(pdev); + if (!pos) + return -ENOTTY; + + if (!pci_wait_for_pending_transaction(pdev)) + dev_err(&pdev->dev, "waiting for pending transaction\n"); + + pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR); + msleep(100); + pci_restore_state(pdev); + + return 0; +} + +static int nitrox_pf_sw_init(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_common_sw_init(ndev); + if (err) + return err; + + err = nitrox_pf_init_isr(ndev); + if (err) + nitrox_common_sw_cleanup(ndev); + + return err; +} + +static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev) +{ + nitrox_pf_cleanup_isr(ndev); + nitrox_common_sw_cleanup(ndev); +} + +/** + * nitrox_bist_check - Check NITORX BIST registers status + * @ndev: NITROX device + */ +static int nitrox_bist_check(struct nitrox_device *ndev) +{ + u64 value = 0; + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i)); + value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i)); + } + value += nitrox_read_csr(ndev, UCD_BIST_STATUS); + value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG); + value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG); + value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG); + value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG); + value += nitrox_read_csr(ndev, POM_BIST_REG); + value += nitrox_read_csr(ndev, BMI_BIST_REG); + value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT); + value += nitrox_read_csr(ndev, BMO_BIST_REG); + value += nitrox_read_csr(ndev, LBC_BIST_STATUS); + value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0)); + if (value) + return -EIO; + return 0; +} + +static void nitrox_get_hwinfo(struct nitrox_device *ndev) +{ + union emu_fuse_map emu_fuse; + u64 offset; + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + u8 dead_cores; + + offset = EMU_FUSE_MAPX(i); + emu_fuse.value = nitrox_read_csr(ndev, offset); + if (emu_fuse.s.valid) { + dead_cores = hweight32(emu_fuse.s.ae_fuse); + ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores; + dead_cores = hweight16(emu_fuse.s.se_fuse); + ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores; + } + } +} + +static int nitrox_pf_hw_init(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_bist_check(ndev); + if (err) { + dev_err(&ndev->pdev->dev, "BIST check failed\n"); + return err; + } + /* get cores information */ + nitrox_get_hwinfo(ndev); + + nitrox_config_nps_unit(ndev); + nitrox_config_pom_unit(ndev); + nitrox_config_efl_unit(ndev); + /* configure IO units */ + nitrox_config_bmi_unit(ndev); + nitrox_config_bmo_unit(ndev); + /* configure Local Buffer Cache */ + nitrox_config_lbc_unit(ndev); + nitrox_config_rand_unit(ndev); + + /* load firmware on SE cores */ + err = nitrox_load_fw(ndev, SE_FW); + if (err) + return err; + + nitrox_config_emu_unit(ndev); + + return 0; +} + +/** + * nitrox_probe - NITROX Initialization function. + * @pdev: PCI device information struct + * @id: entry in nitrox_pci_tbl + * + * Return: 0, if the driver is bound to the device, or + * a negative error if there is failure. + */ +static int nitrox_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct nitrox_device *ndev; + int err; + + dev_info_once(&pdev->dev, "%s driver version %s\n", + nitrox_driver_name, DRIVER_VERSION); + + err = pci_enable_device_mem(pdev); + if (err) + return err; + + /* do FLR */ + err = nitrox_reset_device(pdev); + if (err) { + dev_err(&pdev->dev, "FLR failed\n"); + pci_disable_device(pdev); + return err; + } + + if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { + dev_dbg(&pdev->dev, "DMA to 64-BIT address\n"); + } else { + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (err) { + dev_err(&pdev->dev, "DMA configuration failed\n"); + pci_disable_device(pdev); + return err; + } + } + + err = pci_request_mem_regions(pdev, nitrox_driver_name); + if (err) { + pci_disable_device(pdev); + return err; + } + pci_set_master(pdev); + + ndev = kzalloc(sizeof(*ndev), GFP_KERNEL); + if (!ndev) + goto ndev_fail; + + pci_set_drvdata(pdev, ndev); + ndev->pdev = pdev; + + /* add to device list */ + nitrox_add_to_devlist(ndev); + + ndev->hw.vendor_id = pdev->vendor; + ndev->hw.device_id = pdev->device; + ndev->hw.revision_id = pdev->revision; + /* command timeout in jiffies */ + ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT); + ndev->node = dev_to_node(&pdev->dev); + if (ndev->node == NUMA_NO_NODE) + ndev->node = 0; + + ndev->bar_addr = ioremap(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); + if (!ndev->bar_addr) { + err = -EIO; + goto ioremap_err; + } + /* allocate command queus based on cpus, max queues are 64 */ + ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus()); + ndev->qlen = qlen; + + err = nitrox_pf_sw_init(ndev); + if (err) + goto ioremap_err; + + err = nitrox_pf_hw_init(ndev); + if (err) + goto pf_hw_fail; + + set_bit(NITROX_READY, &ndev->status); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + return 0; + +pf_hw_fail: + nitrox_pf_sw_cleanup(ndev); +ioremap_err: + nitrox_remove_from_devlist(ndev); + kfree(ndev); + pci_set_drvdata(pdev, NULL); +ndev_fail: + pci_release_mem_regions(pdev); + pci_disable_device(pdev); + return err; +} + +/** + * nitrox_remove - Unbind the driver from the device. + * @pdev: PCI device information struct + */ +static void nitrox_remove(struct pci_dev *pdev) +{ + struct nitrox_device *ndev = pci_get_drvdata(pdev); + + if (!ndev) + return; + + if (!refcount_dec_and_test(&ndev->refcnt)) { + dev_err(DEV(ndev), "Device refcnt not zero (%d)\n", + refcount_read(&ndev->refcnt)); + return; + } + + dev_info(DEV(ndev), "Removing Device %x:%x\n", + ndev->hw.vendor_id, ndev->hw.device_id); + + clear_bit(NITROX_READY, &ndev->status); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + + nitrox_remove_from_devlist(ndev); + nitrox_pf_sw_cleanup(ndev); + + iounmap(ndev->bar_addr); + kfree(ndev); + + pci_set_drvdata(pdev, NULL); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +static void nitrox_shutdown(struct pci_dev *pdev) +{ + pci_set_drvdata(pdev, NULL); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +static struct pci_driver nitrox_driver = { + .name = nitrox_driver_name, + .id_table = nitrox_pci_tbl, + .probe = nitrox_probe, + .remove = nitrox_remove, + .shutdown = nitrox_shutdown, +}; + +module_pci_driver(nitrox_driver); + +MODULE_AUTHOR("Srikanth Jampala "); +MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " "); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRIVER_VERSION); +MODULE_FIRMWARE(SE_FW); diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h new file mode 100644 index 000000000000..74f4c20dc87d --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_req.h @@ -0,0 +1,445 @@ +#ifndef __NITROX_REQ_H +#define __NITROX_REQ_H + +#include +#include + +#include "nitrox_dev.h" + +/** + * struct gphdr - General purpose Header + * @param0: first parameter. + * @param1: second parameter. + * @param2: third parameter. + * @param3: fourth parameter. + * + * Params tell the iv and enc/dec data offsets. + */ +struct gphdr { + __be16 param0; + __be16 param1; + __be16 param2; + __be16 param3; +}; + +/** + * struct se_req_ctrl - SE request information. + * @arg: Minor number of the opcode + * @ctxc: Context control. + * @unca: Uncertainity enabled. + * @info: Additional information for SE cores. + * @ctxl: Context length in bytes. + * @uddl: User defined data length + */ +union se_req_ctrl { + u64 value; + struct { + u64 raz : 22; + u64 arg : 8; + u64 ctxc : 2; + u64 unca : 1; + u64 info : 3; + u64 unc : 8; + u64 ctxl : 12; + u64 uddl : 8; + } s; +}; + +struct nitrox_sglist { + u16 len; + u16 raz0; + u32 raz1; + dma_addr_t dma; +}; + +#define MAX_IV_LEN 16 + +/** + * struct se_crypto_request - SE crypto request structure. + * @opcode: Request opcode (enc/dec) + * @flags: flags from crypto subsystem + * @ctx_handle: Crypto context handle. + * @gph: GP Header + * @ctrl: Request Information. + * @in: Input sglist + * @out: Output sglist + */ +struct se_crypto_request { + u8 opcode; + gfp_t gfp; + u32 flags; + u64 ctx_handle; + + struct gphdr gph; + union se_req_ctrl ctrl; + + u8 iv[MAX_IV_LEN]; + u16 ivsize; + + struct scatterlist *src; + struct scatterlist *dst; +}; + +/* Crypto opcodes */ +#define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33 +#define ENCRYPT 0 +#define DECRYPT 1 + +/* IV from context */ +#define IV_FROM_CTX 0 +/* IV from Input data */ +#define IV_FROM_DPTR 1 + +/** + * cipher opcodes for firmware + */ +enum flexi_cipher { + CIPHER_NULL = 0, + CIPHER_3DES_CBC, + CIPHER_3DES_ECB, + CIPHER_AES_CBC, + CIPHER_AES_ECB, + CIPHER_AES_CFB, + CIPHER_AES_CTR, + CIPHER_AES_GCM, + CIPHER_AES_XTS, + CIPHER_AES_CCM, + CIPHER_AES_CBC_CTS, + CIPHER_AES_ECB_CTS, + CIPHER_INVALID +}; + +/** + * struct crypto_keys - Crypto keys + * @key: Encryption key or KEY1 for AES-XTS + * @iv: Encryption IV or Tweak for AES-XTS + */ +struct crypto_keys { + union { + u8 key[AES_MAX_KEY_SIZE]; + u8 key1[AES_MAX_KEY_SIZE]; + } u; + u8 iv[AES_BLOCK_SIZE]; +}; + +/** + * struct auth_keys - Authentication keys + * @ipad: IPAD or KEY2 for AES-XTS + * @opad: OPAD or AUTH KEY if auth_input_type = 1 + */ +struct auth_keys { + union { + u8 ipad[64]; + u8 key2[64]; + } u; + u8 opad[64]; +}; + +/** + * struct flexi_crypto_context - Crypto context + * @cipher_type: Encryption cipher type + * @aes_keylen: AES key length + * @iv_source: Encryption IV source + * @hash_type: Authentication type + * @auth_input_type: Authentication input type + * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD + * 0 - Authentication OPAD/IPAD + * @mac_len: mac length + * @crypto: Crypto keys + * @auth: Authentication keys + */ +struct flexi_crypto_context { + union { + __be64 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 cipher_type : 4; + u64 reserved_59 : 1; + u64 aes_keylen : 2; + u64 iv_source : 1; + u64 hash_type : 4; + u64 reserved_49_51 : 3; + u64 auth_input_type: 1; + u64 mac_len : 8; + u64 reserved_0_39 : 40; +#else + u64 reserved_0_39 : 40; + u64 mac_len : 8; + u64 auth_input_type: 1; + u64 reserved_49_51 : 3; + u64 hash_type : 4; + u64 iv_source : 1; + u64 aes_keylen : 2; + u64 reserved_59 : 1; + u64 cipher_type : 4; +#endif + } w0; + }; + + struct crypto_keys crypto; + struct auth_keys auth; +}; + +struct nitrox_crypto_ctx { + struct nitrox_device *ndev; + union { + u64 ctx_handle; + struct flexi_crypto_context *fctx; + } u; +}; + +struct nitrox_kcrypt_request { + struct se_crypto_request creq; + struct nitrox_crypto_ctx *nctx; + struct skcipher_request *skreq; +}; + +/** + * struct pkt_instr_hdr - Packet Instruction Header + * @g: Gather used + * When [G] is set and [GSZ] != 0, the instruction is + * indirect gather instruction. + * When [G] is set and [GSZ] = 0, the instruction is + * direct gather instruction. + * @gsz: Number of pointers in the indirect gather list + * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr + * and adds them to the packet after the pkt_instr_hdr but before any UDD + * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ] + * when [IHI] is set. + * @fsz: The number of front data bytes directly included in the + * PCIe instruction. + * @tlen: The length of the input packet in bytes, include: + * - 16B pkt_hdr + * - Inline context bytes if any, + * - UDD if any, + * - packet payload bytes + */ +union pkt_instr_hdr { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz_48_63 : 16; + u64 g : 1; + u64 gsz : 7; + u64 ihi : 1; + u64 ssz : 7; + u64 raz_30_31 : 2; + u64 fsz : 6; + u64 raz_16_23 : 8; + u64 tlen : 16; +#else + u64 tlen : 16; + u64 raz_16_23 : 8; + u64 fsz : 6; + u64 raz_30_31 : 2; + u64 ssz : 7; + u64 ihi : 1; + u64 gsz : 7; + u64 g : 1; + u64 raz_48_63 : 16; +#endif + } s; +}; + +/** + * struct pkt_hdr - Packet Input Header + * @opcode: Request opcode (Major) + * @arg: Request opcode (Minor) + * @ctxc: Context control. + * @unca: When set [UNC] is the uncertainty count for an input packet. + * The hardware uses uncertainty counts to predict + * output buffer use and avoid deadlock. + * @info: Not used by input hardware. Available for use + * during SE processing. + * @destport: The expected destination port/ring/channel for the packet. + * @unc: Uncertainty count for an input packet. + * @grp: SE group that will process the input packet. + * @ctxl: Context Length in 64-bit words. + * @uddl: User-defined data (UDD) length in bytes. + * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases. + */ +union pkt_hdr { + u64 value[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 opcode : 8; + u64 arg : 8; + u64 ctxc : 2; + u64 unca : 1; + u64 raz_44 : 1; + u64 info : 3; + u64 destport : 9; + u64 unc : 8; + u64 raz_19_23 : 5; + u64 grp : 3; + u64 raz_15 : 1; + u64 ctxl : 7; + u64 uddl : 8; +#else + u64 uddl : 8; + u64 ctxl : 7; + u64 raz_15 : 1; + u64 grp : 3; + u64 raz_19_23 : 5; + u64 unc : 8; + u64 destport : 9; + u64 info : 3; + u64 raz_44 : 1; + u64 unca : 1; + u64 ctxc : 2; + u64 arg : 8; + u64 opcode : 8; +#endif + __be64 ctxp; + } s; +}; + +/** + * struct slc_store_info - Solicited Paceket Output Store Information. + * @ssz: The number of scatterlist pointers for the solicited output port + * packet. + * @rptr: The result pointer for the solicited output port packet. + * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote + * host that is large enough to hold the entire output packet. + * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4 + * sglist components at [RPTR] on the remote host. + */ +union slc_store_info { + u64 value[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz_39_63 : 25; + u64 ssz : 7; + u64 raz_0_31 : 32; +#else + u64 raz_0_31 : 32; + u64 ssz : 7; + u64 raz_39_63 : 25; +#endif + __be64 rptr; + } s; +}; + +/** + * struct nps_pkt_instr - NPS Packet Instruction of SE cores. + * @dptr0 : Input pointer points to buffer in remote host. + * @ih: Packet Instruction Header (8 bytes) + * @irh: Packet Input Header (16 bytes) + * @slc: Solicited Packet Output Store Information (16 bytes) + * @fdata: Front data + * + * 64-Byte Instruction Format + */ +struct nps_pkt_instr { + __be64 dptr0; + union pkt_instr_hdr ih; + union pkt_hdr irh; + union slc_store_info slc; + u64 fdata[2]; +}; + +/** + * struct ctx_hdr - Book keeping data about the crypto context + * @pool: Pool used to allocate crypto context + * @dma: Base DMA address of the cypto context + * @ctx_dma: Actual usable crypto context for NITROX + */ +struct ctx_hdr { + struct dma_pool *pool; + dma_addr_t dma; + dma_addr_t ctx_dma; +}; + +/* + * struct sglist_component - SG list component format + * @len0: The number of bytes at [PTR0] on the remote host. + * @len1: The number of bytes at [PTR1] on the remote host. + * @len2: The number of bytes at [PTR2] on the remote host. + * @len3: The number of bytes at [PTR3] on the remote host. + * @dma0: First pointer point to buffer in remote host. + * @dma1: Second pointer point to buffer in remote host. + * @dma2: Third pointer point to buffer in remote host. + * @dma3: Fourth pointer point to buffer in remote host. + */ +struct nitrox_sgcomp { + __be16 len[4]; + __be64 dma[4]; +}; + +/* + * strutct nitrox_sgtable - SG list information + * @map_cnt: Number of buffers mapped + * @nr_comp: Number of sglist components + * @total_bytes: Total bytes in sglist. + * @len: Total sglist components length. + * @dma: DMA address of sglist component. + * @dir: DMA direction. + * @buf: crypto request buffer. + * @sglist: SG list of input/output buffers. + * @sgcomp: sglist component for NITROX. + */ +struct nitrox_sgtable { + u8 map_bufs_cnt; + u8 nr_sgcomp; + u16 total_bytes; + u32 len; + dma_addr_t dma; + enum dma_data_direction dir; + + struct scatterlist *buf; + struct nitrox_sglist *sglist; + struct nitrox_sgcomp *sgcomp; +}; + +/* Response Header Length */ +#define ORH_HLEN 8 +/* Completion bytes Length */ +#define COMP_HLEN 8 + +struct resp_hdr { + u64 orh; + dma_addr_t orh_dma; + u64 completion; + dma_addr_t completion_dma; +}; + +typedef void (*completion_t)(struct skcipher_request *skreq, int err); + +/** + * struct nitrox_softreq - Represents the NIROX Request. + * @response: response list entry + * @backlog: Backlog list entry + * @ndev: Device used to submit the request + * @cmdq: Command queue for submission + * @resp: Response headers + * @instr: 64B instruction + * @in: SG table for input + * @out SG table for output + * @tstamp: Request submitted time in jiffies + * @callback: callback after request completion/timeout + * @cb_arg: callback argument + */ +struct nitrox_softreq { + struct list_head response; + struct list_head backlog; + + u32 flags; + gfp_t gfp; + atomic_t status; + bool inplace; + + struct nitrox_device *ndev; + struct nitrox_cmdq *cmdq; + + struct nps_pkt_instr instr; + struct resp_hdr resp; + struct nitrox_sgtable in; + struct nitrox_sgtable out; + + unsigned long tstamp; + + completion_t callback; + struct skcipher_request *skreq; +}; + +#endif /* __NITROX_REQ_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c new file mode 100644 index 000000000000..b6bd2a870028 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -0,0 +1,732 @@ +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_req.h" +#include "nitrox_csr.h" +#include "nitrox_req.h" + +/* SLC_STORE_INFO */ +#define MIN_UDD_LEN 16 +/* PKT_IN_HDR + SLC_STORE_INFO */ +#define FDATA_SIZE 32 +/* Base destination port for the solicited requests */ +#define SOLICIT_BASE_DPORT 256 +#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL + +#define REQ_NOT_POSTED 1 +#define REQ_BACKLOG 2 +#define REQ_POSTED 3 + +/** + * Response codes from SE microcode + * 0x00 - Success + * Completion with no error + * 0x43 - ERR_GC_DATA_LEN_INVALID + * Invalid Data length if Encryption Data length is + * less than 16 bytes for AES-XTS and AES-CTS. + * 0x45 - ERR_GC_CTX_LEN_INVALID + * Invalid context length: CTXL != 23 words. + * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID + * DOCSIS support is enabled with other than + * AES/DES-CBC mode encryption. + * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID + * Authentication offset is other than 0 with + * Encryption IV source = 0. + * Authentication offset is other than 8 (DES)/16 (AES) + * with Encryption IV source = 1 + * 0x51 - ERR_GC_CRC32_INVALID_SELECTION + * CRC32 is enabled for other than DOCSIS encryption. + * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID + * Invalid flag options in AES-CCM IV. + */ + +/** + * dma_free_sglist - unmap and free the sg lists. + * @ndev: N5 device + * @sgtbl: SG table + */ +static void softreq_unmap_sgbufs(struct nitrox_softreq *sr) +{ + struct nitrox_device *ndev = sr->ndev; + struct device *dev = DEV(ndev); + struct nitrox_sglist *sglist; + + /* unmap in sgbuf */ + sglist = sr->in.sglist; + if (!sglist) + goto out_unmap; + + /* unmap iv */ + dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL); + /* unmpa src sglist */ + dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir); + /* unamp gather component */ + dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE); + kfree(sr->in.sglist); + kfree(sr->in.sgcomp); + sr->in.sglist = NULL; + sr->in.buf = NULL; + sr->in.map_bufs_cnt = 0; + +out_unmap: + /* unmap out sgbuf */ + sglist = sr->out.sglist; + if (!sglist) + return; + + /* unmap orh */ + dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir); + + /* unmap dst sglist */ + if (!sr->inplace) { + dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3), + sr->out.dir); + } + /* unmap completion */ + dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir); + + /* unmap scatter component */ + dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE); + kfree(sr->out.sglist); + kfree(sr->out.sgcomp); + sr->out.sglist = NULL; + sr->out.buf = NULL; + sr->out.map_bufs_cnt = 0; +} + +static void softreq_destroy(struct nitrox_softreq *sr) +{ + softreq_unmap_sgbufs(sr); + kfree(sr); +} + +/** + * create_sg_component - create SG componets for N5 device. + * @sr: Request structure + * @sgtbl: SG table + * @nr_comp: total number of components required + * + * Component structure + * + * 63 48 47 32 31 16 15 0 + * -------------------------------------- + * | LEN0 | LEN1 | LEN2 | LEN3 | + * |------------------------------------- + * | PTR0 | + * -------------------------------------- + * | PTR1 | + * -------------------------------------- + * | PTR2 | + * -------------------------------------- + * | PTR3 | + * -------------------------------------- + * + * Returns 0 if success or a negative errno code on error. + */ +static int create_sg_component(struct nitrox_softreq *sr, + struct nitrox_sgtable *sgtbl, int map_nents) +{ + struct nitrox_device *ndev = sr->ndev; + struct nitrox_sgcomp *sgcomp; + struct nitrox_sglist *sglist; + dma_addr_t dma; + size_t sz_comp; + int i, j, nr_sgcomp; + + nr_sgcomp = roundup(map_nents, 4) / 4; + + /* each component holds 4 dma pointers */ + sz_comp = nr_sgcomp * sizeof(*sgcomp); + sgcomp = kzalloc(sz_comp, sr->gfp); + if (!sgcomp) + return -ENOMEM; + + sgtbl->sgcomp = sgcomp; + sgtbl->nr_sgcomp = nr_sgcomp; + + sglist = sgtbl->sglist; + /* populate device sg component */ + for (i = 0; i < nr_sgcomp; i++) { + for (j = 0; j < 4; j++) { + sgcomp->len[j] = cpu_to_be16(sglist->len); + sgcomp->dma[j] = cpu_to_be64(sglist->dma); + sglist++; + } + sgcomp++; + } + /* map the device sg component */ + dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); + if (dma_mapping_error(DEV(ndev), dma)) { + kfree(sgtbl->sgcomp); + sgtbl->sgcomp = NULL; + return -ENOMEM; + } + + sgtbl->dma = dma; + sgtbl->len = sz_comp; + + return 0; +} + +/** + * dma_map_inbufs - DMA map input sglist and creates sglist component + * for N5 device. + * @sr: Request structure + * @req: Crypto request structre + * + * Returns 0 if successful or a negative errno code on error. + */ +static int dma_map_inbufs(struct nitrox_softreq *sr, + struct se_crypto_request *req) +{ + struct device *dev = DEV(sr->ndev); + struct scatterlist *sg = req->src; + struct nitrox_sglist *glist; + int i, nents, ret = 0; + dma_addr_t dma; + size_t sz; + + nents = sg_nents(req->src); + + /* creater gather list IV and src entries */ + sz = roundup((1 + nents), 4) * sizeof(*glist); + glist = kzalloc(sz, sr->gfp); + if (!glist) + return -ENOMEM; + + sr->in.sglist = glist; + /* map IV */ + dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL); + ret = dma_mapping_error(dev, dma); + if (ret) + goto iv_map_err; + + sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; + /* map src entries */ + nents = dma_map_sg(dev, req->src, nents, sr->in.dir); + if (!nents) { + ret = -EINVAL; + goto src_map_err; + } + sr->in.buf = req->src; + + /* store the mappings */ + glist->len = req->ivsize; + glist->dma = dma; + glist++; + sr->in.total_bytes += req->ivsize; + + for_each_sg(req->src, sg, nents, i) { + glist->len = sg_dma_len(sg); + glist->dma = sg_dma_address(sg); + sr->in.total_bytes += glist->len; + glist++; + } + /* roundup map count to align with entires in sg component */ + sr->in.map_bufs_cnt = (1 + nents); + + /* create NITROX gather component */ + ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt); + if (ret) + goto incomp_err; + + return 0; + +incomp_err: + dma_unmap_sg(dev, req->src, nents, sr->in.dir); + sr->in.map_bufs_cnt = 0; +src_map_err: + dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL); +iv_map_err: + kfree(sr->in.sglist); + sr->in.sglist = NULL; + return ret; +} + +static int dma_map_outbufs(struct nitrox_softreq *sr, + struct se_crypto_request *req) +{ + struct device *dev = DEV(sr->ndev); + struct nitrox_sglist *glist = sr->in.sglist; + struct nitrox_sglist *slist; + struct scatterlist *sg; + int i, nents, map_bufs_cnt, ret = 0; + size_t sz; + + nents = sg_nents(req->dst); + + /* create scatter list ORH, IV, dst entries and Completion header */ + sz = roundup((3 + nents), 4) * sizeof(*slist); + slist = kzalloc(sz, sr->gfp); + if (!slist) + return -ENOMEM; + + sr->out.sglist = slist; + sr->out.dir = DMA_BIDIRECTIONAL; + /* map ORH */ + sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN, + sr->out.dir); + ret = dma_mapping_error(dev, sr->resp.orh_dma); + if (ret) + goto orh_map_err; + + /* map completion */ + sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion, + COMP_HLEN, sr->out.dir); + ret = dma_mapping_error(dev, sr->resp.completion_dma); + if (ret) + goto compl_map_err; + + sr->inplace = (req->src == req->dst) ? true : false; + /* out place */ + if (!sr->inplace) { + nents = dma_map_sg(dev, req->dst, nents, sr->out.dir); + if (!nents) { + ret = -EINVAL; + goto dst_map_err; + } + } + sr->out.buf = req->dst; + + /* store the mappings */ + /* orh */ + slist->len = ORH_HLEN; + slist->dma = sr->resp.orh_dma; + slist++; + + /* copy the glist mappings */ + if (sr->inplace) { + nents = sr->in.map_bufs_cnt - 1; + map_bufs_cnt = sr->in.map_bufs_cnt; + while (map_bufs_cnt--) { + slist->len = glist->len; + slist->dma = glist->dma; + slist++; + glist++; + } + } else { + /* copy iv mapping */ + slist->len = glist->len; + slist->dma = glist->dma; + slist++; + /* copy remaining maps */ + for_each_sg(req->dst, sg, nents, i) { + slist->len = sg_dma_len(sg); + slist->dma = sg_dma_address(sg); + slist++; + } + } + + /* completion */ + slist->len = COMP_HLEN; + slist->dma = sr->resp.completion_dma; + + sr->out.map_bufs_cnt = (3 + nents); + + ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt); + if (ret) + goto outcomp_map_err; + + return 0; + +outcomp_map_err: + if (!sr->inplace) + dma_unmap_sg(dev, req->dst, nents, sr->out.dir); + sr->out.map_bufs_cnt = 0; + sr->out.buf = NULL; +dst_map_err: + dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir); + sr->resp.completion_dma = 0; +compl_map_err: + dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir); + sr->resp.orh_dma = 0; +orh_map_err: + kfree(sr->out.sglist); + sr->out.sglist = NULL; + return ret; +} + +static inline int softreq_map_iobuf(struct nitrox_softreq *sr, + struct se_crypto_request *creq) +{ + int ret; + + ret = dma_map_inbufs(sr, creq); + if (ret) + return ret; + + ret = dma_map_outbufs(sr, creq); + if (ret) + softreq_unmap_sgbufs(sr); + + return ret; +} + +static inline void backlog_list_add(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + INIT_LIST_HEAD(&sr->backlog); + + spin_lock_bh(&cmdq->backlog_lock); + list_add_tail(&sr->backlog, &cmdq->backlog_head); + atomic_inc(&cmdq->backlog_count); + atomic_set(&sr->status, REQ_BACKLOG); + spin_unlock_bh(&cmdq->backlog_lock); +} + +static inline void response_list_add(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + INIT_LIST_HEAD(&sr->response); + + spin_lock_bh(&cmdq->response_lock); + list_add_tail(&sr->response, &cmdq->response_head); + spin_unlock_bh(&cmdq->response_lock); +} + +static inline void response_list_del(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + spin_lock_bh(&cmdq->response_lock); + list_del(&sr->response); + spin_unlock_bh(&cmdq->response_lock); +} + +static struct nitrox_softreq * +get_first_response_entry(struct nitrox_cmdq *cmdq) +{ + return list_first_entry_or_null(&cmdq->response_head, + struct nitrox_softreq, response); +} + +static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) +{ + if (atomic_inc_return(&cmdq->pending_count) > qlen) { + atomic_dec(&cmdq->pending_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + return true; + } + return false; +} + +/** + * post_se_instr - Post SE instruction to Packet Input ring + * @sr: Request structure + * + * Returns 0 if successful or a negative error code, + * if no space in ring. + */ +static void post_se_instr(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = sr->ndev; + union nps_pkt_in_instr_baoff_dbell pkt_in_baoff_dbell; + u64 offset; + u8 *ent; + + spin_lock_bh(&cmdq->cmdq_lock); + + /* get the next write offset */ + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(cmdq->qno); + pkt_in_baoff_dbell.value = nitrox_read_csr(ndev, offset); + /* copy the instruction */ + ent = cmdq->head + pkt_in_baoff_dbell.s.aoff; + memcpy(ent, &sr->instr, cmdq->instr_size); + /* flush the command queue updates */ + dma_wmb(); + + sr->tstamp = jiffies; + atomic_set(&sr->status, REQ_POSTED); + response_list_add(sr, cmdq); + + /* Ring doorbell with count 1 */ + writeq(1, cmdq->dbell_csr_addr); + /* orders the doorbell rings */ + mmiowb(); + + spin_unlock_bh(&cmdq->cmdq_lock); +} + +static int post_backlog_cmds(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + struct nitrox_softreq *sr, *tmp; + int ret = 0; + + spin_lock_bh(&cmdq->backlog_lock); + + list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { + struct skcipher_request *skreq; + + /* submit until space available */ + if (unlikely(cmdq_full(cmdq, ndev->qlen))) { + ret = -EBUSY; + break; + } + /* delete from backlog list */ + list_del(&sr->backlog); + atomic_dec(&cmdq->backlog_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + + skreq = sr->skreq; + /* post the command */ + post_se_instr(sr, cmdq); + + /* backlog requests are posted, wakeup with -EINPROGRESS */ + skcipher_request_complete(skreq, -EINPROGRESS); + } + spin_unlock_bh(&cmdq->backlog_lock); + + return ret; +} + +static int nitrox_enqueue_request(struct nitrox_softreq *sr) +{ + struct nitrox_cmdq *cmdq = sr->cmdq; + struct nitrox_device *ndev = sr->ndev; + int ret = -EBUSY; + + if (unlikely(cmdq_full(cmdq, ndev->qlen))) { + if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return -EAGAIN; + + backlog_list_add(sr, cmdq); + } else { + ret = post_backlog_cmds(cmdq); + if (ret) { + backlog_list_add(sr, cmdq); + return ret; + } + post_se_instr(sr, cmdq); + ret = -EINPROGRESS; + } + return ret; +} + +/** + * nitrox_se_request - Send request to SE core + * @ndev: NITROX device + * @req: Crypto request + * + * Returns 0 on success, or a negative error code. + */ +int nitrox_process_se_request(struct nitrox_device *ndev, + struct se_crypto_request *req, + completion_t callback, + struct skcipher_request *skreq) +{ + struct nitrox_softreq *sr; + dma_addr_t ctx_handle = 0; + int qno, ret = 0; + + if (!nitrox_ready(ndev)) + return -ENODEV; + + sr = kzalloc(sizeof(*sr), req->gfp); + if (!sr) + return -ENOMEM; + + sr->ndev = ndev; + sr->flags = req->flags; + sr->gfp = req->gfp; + sr->callback = callback; + sr->skreq = skreq; + + atomic_set(&sr->status, REQ_NOT_POSTED); + + WRITE_ONCE(sr->resp.orh, PENDING_SIG); + WRITE_ONCE(sr->resp.completion, PENDING_SIG); + + ret = softreq_map_iobuf(sr, req); + if (ret) { + kfree(sr); + return ret; + } + + /* get the context handle */ + if (req->ctx_handle) { + struct ctx_hdr *hdr; + u8 *ctx_ptr; + + ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; + hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); + ctx_handle = hdr->ctx_dma; + } + + /* select the queue */ + qno = smp_processor_id() % ndev->nr_queues; + + sr->cmdq = &ndev->pkt_cmdqs[qno]; + + /* + * 64-Byte Instruction Format + * + * ---------------------- + * | DPTR0 | 8 bytes + * ---------------------- + * | PKT_IN_INSTR_HDR | 8 bytes + * ---------------------- + * | PKT_IN_HDR | 16 bytes + * ---------------------- + * | SLC_INFO | 16 bytes + * ---------------------- + * | Front data | 16 bytes + * ---------------------- + */ + + /* fill the packet instruction */ + /* word 0 */ + sr->instr.dptr0 = cpu_to_be64(sr->in.dma); + + /* word 1 */ + sr->instr.ih.value = 0; + sr->instr.ih.s.g = 1; + sr->instr.ih.s.gsz = sr->in.map_bufs_cnt; + sr->instr.ih.s.ssz = sr->out.map_bufs_cnt; + sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); + sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; + sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value); + + /* word 2 */ + sr->instr.irh.value[0] = 0; + sr->instr.irh.s.uddl = MIN_UDD_LEN; + /* context length in 64-bit words */ + sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); + /* offset from solicit base port 256 */ + sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; + sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; + sr->instr.irh.s.arg = req->ctrl.s.arg; + sr->instr.irh.s.opcode = req->opcode; + sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]); + + /* word 3 */ + sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); + + /* word 4 */ + sr->instr.slc.value[0] = 0; + sr->instr.slc.s.ssz = sr->out.map_bufs_cnt; + sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]); + + /* word 5 */ + sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma); + + /* + * No conversion for front data, + * It goes into payload + * put GP Header in front data + */ + sr->instr.fdata[0] = *((u64 *)&req->gph); + sr->instr.fdata[1] = 0; + /* flush the soft_req changes before posting the cmd */ + wmb(); + + ret = nitrox_enqueue_request(sr); + if (ret == -EAGAIN) + goto send_fail; + + return ret; + +send_fail: + softreq_destroy(sr); + return ret; +} + +static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout) +{ + return time_after_eq(jiffies, (tstamp + timeout)); +} + +void backlog_qflush_work(struct work_struct *work) +{ + struct nitrox_cmdq *cmdq; + + cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); + post_backlog_cmds(cmdq); +} + +/** + * process_request_list - process completed requests + * @ndev: N5 device + * @qno: queue to operate + * + * Returns the number of responses processed. + */ +static void process_response_list(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + struct nitrox_softreq *sr; + struct skcipher_request *skreq; + completion_t callback; + int req_completed = 0, err = 0, budget; + + /* check all pending requests */ + budget = atomic_read(&cmdq->pending_count); + + while (req_completed < budget) { + sr = get_first_response_entry(cmdq); + if (!sr) + break; + + if (atomic_read(&sr->status) != REQ_POSTED) + break; + + /* check orh and completion bytes updates */ + if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) { + /* request not completed, check for timeout */ + if (!cmd_timeout(sr->tstamp, ndev->timeout)) + break; + dev_err_ratelimited(DEV(ndev), + "Request timeout, orh 0x%016llx\n", + READ_ONCE(sr->resp.orh)); + } + atomic_dec(&cmdq->pending_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + /* remove from response list */ + response_list_del(sr, cmdq); + + callback = sr->callback; + skreq = sr->skreq; + + /* ORH error code */ + err = READ_ONCE(sr->resp.orh) & 0xff; + softreq_destroy(sr); + + if (callback) + callback(skreq, err); + + req_completed++; + } +} + +/** + * pkt_slc_resp_handler - post processing of SE responses + */ +void pkt_slc_resp_handler(unsigned long data) +{ + struct bh_data *bh = (void *)(uintptr_t)(data); + struct nitrox_cmdq *cmdq = bh->cmdq; + union nps_pkt_slc_cnts pkt_slc_cnts; + + /* read completion count */ + pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr); + /* resend the interrupt if more work to do */ + pkt_slc_cnts.s.resend = 1; + + process_response_list(cmdq); + + /* + * clear the interrupt with resend bit enabled, + * MSI-X interrupt generates if Completion count > Threshold + */ + writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr); + /* order the writes */ + mmiowb(); + + if (atomic_read(&cmdq->backlog_count)) + schedule_work(&cmdq->backlog_qflush); +} From 086eac9eb4a05d053f8abf661c36e77c40264e41 Mon Sep 17 00:00:00 2001 From: Srikanth Jampala Date: Tue, 30 May 2017 17:28:02 +0530 Subject: [PATCH 069/140] crypto: cavium - Add debugfs support in CNN55XX driver. Add debugfs support in CNN55XX Physical Function driver. Provides hardware counters and firmware information. Signed-off-by: Srikanth Jampala Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/nitrox_csr.h | 4 + drivers/crypto/cavium/nitrox/nitrox_dev.h | 4 + drivers/crypto/cavium/nitrox/nitrox_main.c | 135 +++++++++++++++++++++ 3 files changed, 143 insertions(+) diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h index 583465190882..30b04c4c6076 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_csr.h +++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h @@ -42,6 +42,8 @@ #define NPS_CORE_INT_ACTIVE 0x1000080 #define NPS_CORE_INT 0x10000A0 #define NPS_CORE_INT_ENA_W1S 0x10000B8 +#define NPS_STATS_PKT_DMA_RD_CNT 0x1000180 +#define NPS_STATS_PKT_DMA_WR_CNT 0x1000190 /* NPS packet registers */ #define NPS_PKT_INT 0x1040018 @@ -78,6 +80,7 @@ #define BMI_INT 0x1140000 #define BMI_CTL 0x1140020 #define BMI_INT_ENA_W1S 0x1140018 +#define BMI_NPS_PKT_CNT 0x1140070 /* EFL registers */ #define EFL_CORE_INT_ENA_W1SX(_i) (0x1240018 + ((_i) * 0x400)) @@ -91,6 +94,7 @@ /* BMO registers */ #define BMO_CTL2 0x1180028 +#define BMO_NPS_SLC_PKT_CNT 0x1180078 /* LBC registers */ #define LBC_INT 0x1200000 diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h index 2793a106b552..57858b04f165 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_dev.h +++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h @@ -118,6 +118,7 @@ struct nitrox_bh { * @msix: MSI-X information * @bh: post processing work * @hw: hardware information + * @debugfs_dir: debugfs directory */ struct nitrox_device { struct list_head list; @@ -141,6 +142,9 @@ struct nitrox_device { struct nitrox_bh bh; struct nitrox_hw hw; +#if IS_ENABLED(CONFIG_DEBUG_FS) + struct dentry *debugfs_dir; +#endif }; /** diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index aa331202dcfa..9b07003e39f7 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -305,6 +306,135 @@ static int nitrox_pf_hw_init(struct nitrox_device *ndev) return 0; } +#if IS_ENABLED(CONFIG_DEBUG_FS) +static int registers_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + u64 offset; + + /* NPS DMA stats */ + offset = NPS_STATS_PKT_DMA_RD_CNT; + seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT 0x%016llx\n", + nitrox_read_csr(ndev, offset)); + offset = NPS_STATS_PKT_DMA_WR_CNT; + seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT 0x%016llx\n", + nitrox_read_csr(ndev, offset)); + + /* BMI/BMO stats */ + offset = BMI_NPS_PKT_CNT; + seq_printf(s, "BMI_NPS_PKT_CNT 0x%016llx\n", + nitrox_read_csr(ndev, offset)); + offset = BMO_NPS_SLC_PKT_CNT; + seq_printf(s, "BMO_NPS_PKT_CNT 0x%016llx\n", + nitrox_read_csr(ndev, offset)); + + return 0; +} + +static int registers_open(struct inode *inode, struct file *file) +{ + return single_open(file, registers_show, inode->i_private); +} + +static const struct file_operations register_fops = { + .owner = THIS_MODULE, + .open = registers_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int firmware_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "Version: %s\n", ndev->hw.fw_name); + return 0; +} + +static int firmware_open(struct inode *inode, struct file *file) +{ + return single_open(file, firmware_show, inode->i_private); +} + +static const struct file_operations firmware_fops = { + .owner = THIS_MODULE, + .open = firmware_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int nitrox_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx); + seq_printf(s, " Revsion ID: 0x%0x\n", ndev->hw.revision_id); + seq_printf(s, " Cores [AE: %u SE: %u]\n", + ndev->hw.ae_cores, ndev->hw.se_cores); + seq_printf(s, " Number of Queues: %u\n", ndev->nr_queues); + seq_printf(s, " Queue length: %u\n", ndev->qlen); + seq_printf(s, " Node: %u\n", ndev->node); + + return 0; +} + +static int nitrox_open(struct inode *inode, struct file *file) +{ + return single_open(file, nitrox_show, inode->i_private); +} + +static const struct file_operations nitrox_fops = { + .owner = THIS_MODULE, + .open = nitrox_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static void nitrox_debugfs_exit(struct nitrox_device *ndev) +{ + debugfs_remove_recursive(ndev->debugfs_dir); + ndev->debugfs_dir = NULL; +} + +static int nitrox_debugfs_init(struct nitrox_device *ndev) +{ + struct dentry *dir, *f; + + dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + if (!dir) + return -ENOMEM; + + ndev->debugfs_dir = dir; + f = debugfs_create_file("counters", 0400, dir, ndev, ®ister_fops); + if (!f) + goto err; + f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops); + if (!f) + goto err; + f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops); + if (!f) + goto err; + + return 0; + +err: + nitrox_debugfs_exit(ndev); + return -ENODEV; +} +#else +static int nitrox_debugfs_init(struct nitrox_device *ndev) +{ + return 0; +} + +static void nitrox_debugfs_exit(struct nitrox_device *ndev) +{ +} +#endif + /** * nitrox_probe - NITROX Initialization function. * @pdev: PCI device information struct @@ -389,6 +519,10 @@ static int nitrox_probe(struct pci_dev *pdev, if (err) goto pf_hw_fail; + err = nitrox_debugfs_init(ndev); + if (err) + goto pf_hw_fail; + set_bit(NITROX_READY, &ndev->status); /* barrier to sync with other cpus */ smp_mb__after_atomic(); @@ -431,6 +565,7 @@ static void nitrox_remove(struct pci_dev *pdev) smp_mb__after_atomic(); nitrox_remove_from_devlist(ndev); + nitrox_debugfs_exit(ndev); nitrox_pf_sw_cleanup(ndev); iounmap(ndev->bar_addr); From f2663872f073c874495b793721a47cc7f30eaec7 Mon Sep 17 00:00:00 2001 From: Srikanth Jampala Date: Tue, 30 May 2017 17:28:03 +0530 Subject: [PATCH 070/140] crypto: cavium - Register the CNN55XX supported crypto algorithms. Register the Symmetric crypto algorithms supported by CNN55XX driver with crypto subsystem. The following Symmetric crypto algorithms are supported, - aes with cbc, ecb, cfb, xts, ctr and cts modes - des3_ede with cbc and ecb modes Signed-off-by: Srikanth Jampala Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/Makefile | 3 +- drivers/crypto/cavium/nitrox/nitrox_algs.c | 457 +++++++++++++++++++ drivers/crypto/cavium/nitrox/nitrox_common.h | 7 + drivers/crypto/cavium/nitrox/nitrox_lib.c | 38 ++ drivers/crypto/cavium/nitrox/nitrox_main.c | 40 ++ 5 files changed, 544 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cavium/nitrox/nitrox_algs.c diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile index ef457f609d88..5af2e4368267 100644 --- a/drivers/crypto/cavium/nitrox/Makefile +++ b/drivers/crypto/cavium/nitrox/Makefile @@ -4,4 +4,5 @@ n5pf-objs := nitrox_main.o \ nitrox_isr.o \ nitrox_lib.o \ nitrox_hal.o \ - nitrox_reqmgr.o + nitrox_reqmgr.o \ + nitrox_algs.o diff --git a/drivers/crypto/cavium/nitrox/nitrox_algs.c b/drivers/crypto/cavium/nitrox/nitrox_algs.c new file mode 100644 index 000000000000..ce330278ef8a --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_algs.c @@ -0,0 +1,457 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" + +#define PRIO 4001 + +struct nitrox_cipher { + const char *name; + enum flexi_cipher value; +}; + +/** + * supported cipher list + */ +static const struct nitrox_cipher flexi_cipher_table[] = { + { "null", CIPHER_NULL }, + { "cbc(des3_ede)", CIPHER_3DES_CBC }, + { "ecb(des3_ede)", CIPHER_3DES_ECB }, + { "cbc(aes)", CIPHER_AES_CBC }, + { "ecb(aes)", CIPHER_AES_ECB }, + { "cfb(aes)", CIPHER_AES_CFB }, + { "rfc3686(ctr(aes))", CIPHER_AES_CTR }, + { "xts(aes)", CIPHER_AES_XTS }, + { "cts(cbc(aes))", CIPHER_AES_CBC_CTS }, + { NULL, CIPHER_INVALID } +}; + +static enum flexi_cipher flexi_cipher_type(const char *name) +{ + const struct nitrox_cipher *cipher = flexi_cipher_table; + + while (cipher->name) { + if (!strcmp(cipher->name, name)) + break; + cipher++; + } + return cipher->value; +} + +static int flexi_aes_keylen(int keylen) +{ + int aes_keylen; + + switch (keylen) { + case AES_KEYSIZE_128: + aes_keylen = 1; + break; + case AES_KEYSIZE_192: + aes_keylen = 2; + break; + case AES_KEYSIZE_256: + aes_keylen = 3; + break; + default: + aes_keylen = -EINVAL; + break; + } + return aes_keylen; +} + +static int nitrox_skcipher_init(struct crypto_skcipher *tfm) +{ + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); + void *fctx; + + /* get the first device */ + nctx->ndev = nitrox_get_first_device(); + if (!nctx->ndev) + return -ENODEV; + + /* allocate nitrox crypto context */ + fctx = crypto_alloc_context(nctx->ndev); + if (!fctx) { + nitrox_put_device(nctx->ndev); + return -ENOMEM; + } + nctx->u.ctx_handle = (uintptr_t)fctx; + crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) + + sizeof(struct nitrox_kcrypt_request)); + return 0; +} + +static void nitrox_skcipher_exit(struct crypto_skcipher *tfm) +{ + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); + + /* free the nitrox crypto context */ + if (nctx->u.ctx_handle) { + struct flexi_crypto_context *fctx = nctx->u.fctx; + + memset(&fctx->crypto, 0, sizeof(struct crypto_keys)); + memset(&fctx->auth, 0, sizeof(struct auth_keys)); + crypto_free_context((void *)fctx); + } + nitrox_put_device(nctx->ndev); + + nctx->u.ctx_handle = 0; + nctx->ndev = NULL; +} + +static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher, + int aes_keylen, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); + struct flexi_crypto_context *fctx; + enum flexi_cipher cipher_type; + const char *name; + + name = crypto_tfm_alg_name(tfm); + cipher_type = flexi_cipher_type(name); + if (unlikely(cipher_type == CIPHER_INVALID)) { + pr_err("unsupported cipher: %s\n", name); + return -EINVAL; + } + + /* fill crypto context */ + fctx = nctx->u.fctx; + fctx->flags = 0; + fctx->w0.cipher_type = cipher_type; + fctx->w0.aes_keylen = aes_keylen; + fctx->w0.iv_source = IV_FROM_DPTR; + fctx->flags = cpu_to_be64(*(u64 *)&fctx->w0); + /* copy the key to context */ + memcpy(fctx->crypto.u.key, key, keylen); + + return 0; +} + +static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + int aes_keylen; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) { + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static void nitrox_skcipher_callback(struct skcipher_request *skreq, + int err) +{ + if (err) { + pr_err_ratelimited("request failed status 0x%0x\n", err); + err = -EINVAL; + } + skcipher_request_complete(skreq, err); +} + +static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + int ivsize = crypto_skcipher_ivsize(cipher); + struct se_crypto_request *creq; + + creq = &nkreq->creq; + creq->flags = skreq->base.flags; + creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + + /* fill the request */ + creq->ctrl.value = 0; + creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; + creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT); + /* param0: length of the data to be encrypted */ + creq->gph.param0 = cpu_to_be16(skreq->cryptlen); + creq->gph.param1 = 0; + /* param2: encryption data offset */ + creq->gph.param2 = cpu_to_be16(ivsize); + creq->gph.param3 = 0; + + creq->ctx_handle = nctx->u.ctx_handle; + creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); + + /* copy the iv */ + memcpy(creq->iv, skreq->iv, ivsize); + creq->ivsize = ivsize; + creq->src = skreq->src; + creq->dst = skreq->dst; + + nkreq->nctx = nctx; + nkreq->skreq = skreq; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, + nitrox_skcipher_callback, skreq); +} + +static int nitrox_aes_encrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, true); +} + +static int nitrox_aes_decrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, false); +} + +static int nitrox_3des_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + if (keylen != DES3_EDE_KEY_SIZE) { + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + return nitrox_skcipher_setkey(cipher, 0, key, keylen); +} + +static int nitrox_3des_encrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, true); +} + +static int nitrox_3des_decrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, false); +} + +static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); + struct flexi_crypto_context *fctx; + int aes_keylen, ret; + + ret = xts_check_key(tfm, key, keylen); + if (ret) + return ret; + + keylen /= 2; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) { + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + fctx = nctx->u.fctx; + /* copy KEY2 */ + memcpy(fctx->auth.u.key2, (key + keylen), keylen); + + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); + struct flexi_crypto_context *fctx; + int aes_keylen; + + if (keylen < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + fctx = nctx->u.fctx; + + memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE), + CTR_RFC3686_NONCE_SIZE); + + keylen -= CTR_RFC3686_NONCE_SIZE; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) { + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static struct skcipher_alg nitrox_skciphers[] = { { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "n5_cbc(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "n5_ecb(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "cfb(aes)", + .cra_driver_name = "n5_cfb(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "n5_xts(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_xts_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "n5_rfc3686(ctr(aes))", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, + .setkey = nitrox_aes_ctr_rfc3686_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, +}, { + .base = { + .cra_name = "cts(cbc(aes))", + .cra_driver_name = "n5_cts(cbc(aes))", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "n5_cbc(des3_ede)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = nitrox_3des_setkey, + .encrypt = nitrox_3des_encrypt, + .decrypt = nitrox_3des_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "n5_ecb(des3_ede)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = nitrox_3des_setkey, + .encrypt = nitrox_3des_encrypt, + .decrypt = nitrox_3des_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +} + +}; + +int nitrox_crypto_register(void) +{ + return crypto_register_skciphers(nitrox_skciphers, + ARRAY_SIZE(nitrox_skciphers)); +} + +void nitrox_crypto_unregister(void) +{ + crypto_unregister_skciphers(nitrox_skciphers, + ARRAY_SIZE(nitrox_skciphers)); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h index 3284ffe4837a..4888c7823a5f 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_common.h +++ b/drivers/crypto/cavium/nitrox/nitrox_common.h @@ -4,6 +4,13 @@ #include "nitrox_dev.h" #include "nitrox_req.h" +int nitrox_crypto_register(void); +void nitrox_crypto_unregister(void); +void *crypto_alloc_context(struct nitrox_device *ndev); +void crypto_free_context(void *ctx); +struct nitrox_device *nitrox_get_first_device(void); +void nitrox_put_device(struct nitrox_device *ndev); + void nitrox_pf_cleanup_isr(struct nitrox_device *ndev); int nitrox_pf_init_isr(struct nitrox_device *ndev); diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c index 846e990d4f35..b4a391adb9b6 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_lib.c +++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c @@ -137,6 +137,44 @@ static void destroy_crypto_dma_pool(struct nitrox_device *ndev) ndev->ctx_pool = NULL; } +/* + * crypto_alloc_context - Allocate crypto context from pool + * @ndev: NITROX Device + */ +void *crypto_alloc_context(struct nitrox_device *ndev) +{ + struct ctx_hdr *ctx; + void *vaddr; + dma_addr_t dma; + + vaddr = dma_pool_alloc(ndev->ctx_pool, (GFP_ATOMIC | __GFP_ZERO), &dma); + if (!vaddr) + return NULL; + + /* fill meta data */ + ctx = vaddr; + ctx->pool = ndev->ctx_pool; + ctx->dma = dma; + ctx->ctx_dma = dma + sizeof(struct ctx_hdr); + + return ((u8 *)vaddr + sizeof(struct ctx_hdr)); +} + +/** + * crypto_free_context - Free crypto context to pool + * @ctx: context to free + */ +void crypto_free_context(void *ctx) +{ + struct ctx_hdr *ctxp; + + if (!ctx) + return; + + ctxp = (struct ctx_hdr *)((u8 *)ctx - sizeof(struct ctx_hdr)); + dma_pool_free(ctxp->pool, ctxp, ctxp->dma); +} + /** * nitrox_common_sw_init - allocate software resources. * @ndev: NITROX device diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index 9b07003e39f7..eee6fb501580 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -180,6 +180,35 @@ static void nitrox_remove_from_devlist(struct nitrox_device *ndev) mutex_unlock(&devlist_lock); } +struct nitrox_device *nitrox_get_first_device(void) +{ + struct nitrox_device *ndev = NULL; + + mutex_lock(&devlist_lock); + list_for_each_entry(ndev, &ndevlist, list) { + if (nitrox_ready(ndev)) + break; + } + mutex_unlock(&devlist_lock); + if (!ndev) + return NULL; + + refcount_inc(&ndev->refcnt); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + return ndev; +} + +void nitrox_put_device(struct nitrox_device *ndev) +{ + if (!ndev) + return; + + refcount_dec(&ndev->refcnt); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); +} + static int nitrox_reset_device(struct pci_dev *pdev) { int pos = 0; @@ -526,8 +555,18 @@ static int nitrox_probe(struct pci_dev *pdev, set_bit(NITROX_READY, &ndev->status); /* barrier to sync with other cpus */ smp_mb__after_atomic(); + + err = nitrox_crypto_register(); + if (err) + goto crypto_fail; + return 0; +crypto_fail: + nitrox_debugfs_exit(ndev); + clear_bit(NITROX_READY, &ndev->status); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); pf_hw_fail: nitrox_pf_sw_cleanup(ndev); ioremap_err: @@ -565,6 +604,7 @@ static void nitrox_remove(struct pci_dev *pdev) smp_mb__after_atomic(); nitrox_remove_from_devlist(ndev); + nitrox_crypto_unregister(); nitrox_debugfs_exit(ndev); nitrox_pf_sw_cleanup(ndev); From 6755fd269d5c100b0eca420db501ae58435efd6e Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Tue, 30 May 2017 17:52:48 +0300 Subject: [PATCH 071/140] crypto: ecdh - add privkey generation support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for generating ecc private keys. Generation of ecc private keys is helpful in a user-space to kernel ecdh offload because the keys are not revealed to user-space. Private key generation is also helpful to implement forward secrecy. If the user provides a NULL ecc private key, the kernel will generate it and further use it for ecdh. Move ecdh's object files below drbg's. drbg must be present in the kernel at the time of calling. Signed-off-by: Tudor Ambarus Reviewed-by: Stephan Müller Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 + crypto/Makefile | 9 ++++---- crypto/ecc.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ crypto/ecc.h | 14 +++++++++++++ crypto/ecdh.c | 4 ++++ 5 files changed, 80 insertions(+), 4 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index aac4bc90a138..caa770e535a2 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -130,6 +130,7 @@ config CRYPTO_DH config CRYPTO_ECDH tristate "ECDH algorithm" select CRYTPO_KPP + select CRYPTO_RNG_DEFAULT help Generic implementation of the ECDH algorithm diff --git a/crypto/Makefile b/crypto/Makefile index 8a44057240d5..d41f0331b085 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -33,10 +33,6 @@ obj-$(CONFIG_CRYPTO_KPP2) += kpp.o dh_generic-y := dh.o dh_generic-y += dh_helper.o obj-$(CONFIG_CRYPTO_DH) += dh_generic.o -ecdh_generic-y := ecc.o -ecdh_generic-y += ecdh.o -ecdh_generic-y += ecdh_helper.o -obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o $(obj)/rsapubkey-asn1.o: $(obj)/rsapubkey-asn1.c $(obj)/rsapubkey-asn1.h $(obj)/rsaprivkey-asn1.o: $(obj)/rsaprivkey-asn1.c $(obj)/rsaprivkey-asn1.h @@ -138,6 +134,11 @@ obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o +ecdh_generic-y := ecc.o +ecdh_generic-y += ecdh.o +ecdh_generic-y += ecdh_helper.o +obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o + # # generic algorithms and the async_tx api # diff --git a/crypto/ecc.c b/crypto/ecc.c index 6c33c4323d6a..633a9bcdc574 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -29,6 +29,7 @@ #include #include #include +#include #include "ecc.h" #include "ecc_curve_defs.h" @@ -927,6 +928,61 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, return 0; } +/* + * ECC private keys are generated using the method of extra random bits, + * equivalent to that described in FIPS 186-4, Appendix B.4.1. + * + * d = (c mod(n–1)) + 1 where c is a string of random bits, 64 bits longer + * than requested + * 0 <= c mod(n-1) <= n-2 and implies that + * 1 <= d <= n-1 + * + * This method generates a private key uniformly distributed in the range + * [1, n-1]. + */ +int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, u64 *privkey) +{ + const struct ecc_curve *curve = ecc_get_curve(curve_id); + u64 priv[ndigits]; + unsigned int nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT; + unsigned int nbits = vli_num_bits(curve->n, ndigits); + int err; + + /* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */ + if (nbits < 160) + return -EINVAL; + + /* + * FIPS 186-4 recommends that the private key should be obtained from a + * RBG with a security strength equal to or greater than the security + * strength associated with N. + * + * The maximum security strength identified by NIST SP800-57pt1r4 for + * ECC is 256 (N >= 512). + * + * This condition is met by the default RNG because it selects a favored + * DRBG with a security strength of 256. + */ + if (crypto_get_default_rng()) + err = -EFAULT; + + err = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes); + crypto_put_default_rng(); + if (err) + return err; + + if (vli_is_zero(priv, ndigits)) + return -EINVAL; + + /* Make sure the private key is in the range [1, n-1]. */ + if (vli_cmp(curve->n, priv, ndigits) != 1) + return -EINVAL; + + ecc_swap_digits(priv, privkey, ndigits); + + return 0; +} + int ecc_make_pub_key(unsigned int curve_id, unsigned int ndigits, const u64 *private_key, u64 *public_key) { diff --git a/crypto/ecc.h b/crypto/ecc.h index e13fe8800ee8..e4fd4492c765 100644 --- a/crypto/ecc.h +++ b/crypto/ecc.h @@ -43,6 +43,20 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, const u64 *private_key, unsigned int private_key_len); +/** + * ecc_gen_privkey() - Generates an ECC private key. + * The private key is a random integer in the range 0 < random < n, where n is a + * prime that is the order of the cyclic subgroup generated by the distinguished + * point G. + * @curve_id: id representing the curve to use + * @ndigits: curve number of digits + * @private_key: buffer for storing the generated private key + * + * Returns 0 if the private key was generated successfully, a negative value + * if an error occurred. + */ +int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, u64 *privkey); + /** * ecc_make_pub_key() - Compute an ECC public key * diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 4aa0b0cb4627..61c7708905d0 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -55,6 +55,10 @@ static int ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, ctx->curve_id = params.curve_id; ctx->ndigits = ndigits; + if (!params.key || !params.key_size) + return ecc_gen_privkey(ctx->curve_id, ctx->ndigits, + ctx->private_key); + if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits, (const u64 *)params.key, params.key_size) < 0) return -EINVAL; From 47d3fd390e899e0403da35733b75832eacfbbc3f Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Tue, 30 May 2017 17:52:49 +0300 Subject: [PATCH 072/140] crypto: testmgr - add genkey kpp test The test considers a party that already has a private-public key pair and a party that provides a NULL key. The kernel will generate the private-public key pair for the latter, computes the shared secret on both ends and verifies if it's the same. The explicit private-public key pair was copied from the previous test vector. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/testmgr.c | 76 +++++++++++++++++++++++++++++++++++++++++------- crypto/testmgr.h | 47 ++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 11 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 6f5f3ed8376c..5f8e6838e5a8 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1997,6 +1997,9 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, struct kpp_request *req; void *input_buf = NULL; void *output_buf = NULL; + void *a_public = NULL; + void *a_ss = NULL; + void *shared_secret = NULL; struct tcrypt_result result; unsigned int out_len_max; int err = -ENOMEM; @@ -2026,20 +2029,31 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, tcrypt_complete, &result); - /* Compute public key */ + /* Compute party A's public key */ err = wait_async_op(&result, crypto_kpp_generate_public_key(req)); if (err) { - pr_err("alg: %s: generate public key test failed. err %d\n", + pr_err("alg: %s: Party A: generate public key test failed. err %d\n", alg, err); goto free_output; } - /* Verify calculated public key */ - if (memcmp(vec->expected_a_public, sg_virt(req->dst), - vec->expected_a_public_size)) { - pr_err("alg: %s: generate public key test failed. Invalid output\n", - alg); - err = -EINVAL; - goto free_output; + + if (vec->genkey) { + /* Save party A's public key */ + a_public = kzalloc(out_len_max, GFP_KERNEL); + if (!a_public) { + err = -ENOMEM; + goto free_output; + } + memcpy(a_public, sg_virt(req->dst), out_len_max); + } else { + /* Verify calculated public key */ + if (memcmp(vec->expected_a_public, sg_virt(req->dst), + vec->expected_a_public_size)) { + pr_err("alg: %s: Party A: generate public key test failed. Invalid output\n", + alg); + err = -EINVAL; + goto free_output; + } } /* Calculate shared secret key by using counter part (b) public key. */ @@ -2058,15 +2072,53 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, tcrypt_complete, &result); err = wait_async_op(&result, crypto_kpp_compute_shared_secret(req)); if (err) { - pr_err("alg: %s: compute shard secret test failed. err %d\n", + pr_err("alg: %s: Party A: compute shared secret test failed. err %d\n", alg, err); goto free_all; } + + if (vec->genkey) { + /* Save the shared secret obtained by party A */ + a_ss = kzalloc(vec->expected_ss_size, GFP_KERNEL); + if (!a_ss) { + err = -ENOMEM; + goto free_all; + } + memcpy(a_ss, sg_virt(req->dst), vec->expected_ss_size); + + /* + * Calculate party B's shared secret by using party A's + * public key. + */ + err = crypto_kpp_set_secret(tfm, vec->b_secret, + vec->b_secret_size); + if (err < 0) + goto free_all; + + sg_init_one(&src, a_public, vec->expected_a_public_size); + sg_init_one(&dst, output_buf, out_len_max); + kpp_request_set_input(req, &src, vec->expected_a_public_size); + kpp_request_set_output(req, &dst, out_len_max); + kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + tcrypt_complete, &result); + err = wait_async_op(&result, + crypto_kpp_compute_shared_secret(req)); + if (err) { + pr_err("alg: %s: Party B: compute shared secret failed. err %d\n", + alg, err); + goto free_all; + } + + shared_secret = a_ss; + } else { + shared_secret = (void *)vec->expected_ss; + } + /* * verify shared secret from which the user will derive * secret key by executing whatever hash it has chosen */ - if (memcmp(vec->expected_ss, sg_virt(req->dst), + if (memcmp(shared_secret, sg_virt(req->dst), vec->expected_ss_size)) { pr_err("alg: %s: compute shared secret test failed. Invalid output\n", alg); @@ -2074,8 +2126,10 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, } free_all: + kfree(a_ss); kfree(input_buf); free_output: + kfree(a_public); kfree(output_buf); free_req: kpp_request_free(req); diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 429357339dcc..db2e26c0c065 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -137,13 +137,16 @@ struct akcipher_testvec { struct kpp_testvec { const unsigned char *secret; + const unsigned char *b_secret; const unsigned char *b_public; const unsigned char *expected_a_public; const unsigned char *expected_ss; unsigned short secret_size; + unsigned short b_secret_size; unsigned short b_public_size; unsigned short expected_a_public_size; unsigned short expected_ss_size; + bool genkey; }; static const char zeroed_string[48]; @@ -840,6 +843,50 @@ static const struct kpp_testvec ecdh_tv_template[] = { .b_public_size = 64, .expected_a_public_size = 64, .expected_ss_size = 32 + }, { + .secret = +#ifdef __LITTLE_ENDIAN + "\x02\x00" /* type */ + "\x08\x00" /* len */ + "\x02\x00" /* curve_id */ + "\x00\x00", /* key_size */ +#else + "\x00\x02" /* type */ + "\x00\x08" /* len */ + "\x00\x02" /* curve_id */ + "\x00\x00", /* key_size */ +#endif + .b_secret = +#ifdef __LITTLE_ENDIAN + "\x02\x00" /* type */ + "\x28\x00" /* len */ + "\x02\x00" /* curve_id */ + "\x20\x00" /* key_size */ +#else + "\x00\x02" /* type */ + "\x00\x28" /* len */ + "\x00\x02" /* curve_id */ + "\x00\x20" /* key_size */ +#endif + "\x24\xd1\x21\xeb\xe5\xcf\x2d\x83" + "\xf6\x62\x1b\x6e\x43\x84\x3a\xa3" + "\x8b\xe0\x86\xc3\x20\x19\xda\x92" + "\x50\x53\x03\xe1\xc0\xea\xb8\x82", + .b_public = + "\x1a\x7f\xeb\x52\x00\xbd\x3c\x31" + "\x7d\xb6\x70\xc1\x86\xa6\xc7\xc4" + "\x3b\xc5\x5f\x6c\x6f\x58\x3c\xf5" + "\xb6\x63\x82\x77\x33\x24\xa1\x5f" + "\x6a\xca\x43\x6f\xf7\x7e\xff\x02" + "\x37\x08\xcc\x40\x5e\x7a\xfd\x6a" + "\x6a\x02\x6e\x41\x87\x68\x38\x77" + "\xfa\xa9\x44\x43\x2d\xef\x09\xdf", + .secret_size = 8, + .b_secret_size = 40, + .b_public_size = 64, + .expected_a_public_size = 64, + .expected_ss_size = 32, + .genkey = true, } }; From 8fa23a29eb77698cdcc6ea173577ec3af463fe53 Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Thu, 1 Jun 2017 10:30:21 +0800 Subject: [PATCH 073/140] crypto: mediatek - remove redundant clock setting This patch removes redundant clock setting for 'clk_ethif', which is the parent of 'clk_cryp'. Hence, we just need to handle its child. Signed-off-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-platform.c | 10 +--------- drivers/crypto/mediatek/mtk-platform.h | 2 -- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c index b6ecc288b540..9b4437c857b9 100644 --- a/drivers/crypto/mediatek/mtk-platform.c +++ b/drivers/crypto/mediatek/mtk-platform.c @@ -504,19 +504,14 @@ static int mtk_crypto_probe(struct platform_device *pdev) } } - cryp->clk_ethif = devm_clk_get(&pdev->dev, "ethif"); cryp->clk_cryp = devm_clk_get(&pdev->dev, "cryp"); - if (IS_ERR(cryp->clk_ethif) || IS_ERR(cryp->clk_cryp)) + if (IS_ERR(cryp->clk_cryp)) return -EPROBE_DEFER; cryp->dev = &pdev->dev; pm_runtime_enable(cryp->dev); pm_runtime_get_sync(cryp->dev); - err = clk_prepare_enable(cryp->clk_ethif); - if (err) - goto err_clk_ethif; - err = clk_prepare_enable(cryp->clk_cryp); if (err) goto err_clk_cryp; @@ -559,8 +554,6 @@ err_engine: err_resource: clk_disable_unprepare(cryp->clk_cryp); err_clk_cryp: - clk_disable_unprepare(cryp->clk_ethif); -err_clk_ethif: pm_runtime_put_sync(cryp->dev); pm_runtime_disable(cryp->dev); @@ -576,7 +569,6 @@ static int mtk_crypto_remove(struct platform_device *pdev) mtk_desc_dma_free(cryp); clk_disable_unprepare(cryp->clk_cryp); - clk_disable_unprepare(cryp->clk_ethif); pm_runtime_put_sync(cryp->dev); pm_runtime_disable(cryp->dev); diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h index 303c152dc931..f0831f1742ab 100644 --- a/drivers/crypto/mediatek/mtk-platform.h +++ b/drivers/crypto/mediatek/mtk-platform.h @@ -200,7 +200,6 @@ struct mtk_sha_rec { * struct mtk_cryp - Cryptographic device * @base: pointer to mapped register I/O base * @dev: pointer to device - * @clk_ethif: pointer to ethif clock * @clk_cryp: pointer to crypto clock * @irq: global system and rings IRQ * @ring: pointer to descriptor rings @@ -215,7 +214,6 @@ struct mtk_sha_rec { struct mtk_cryp { void __iomem *base; struct device *dev; - struct clk *clk_ethif; struct clk *clk_cryp; int irq[MTK_IRQ_NUM]; From 48df28b85991ac67a2270f0bb818221527834b78 Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Thu, 1 Jun 2017 10:30:22 +0800 Subject: [PATCH 074/140] dt-bindings: crypto: remove mediatek ethif clock This patch removes the parent clock 'ethif' in bindings, since we don't need to control the parent of a clock in current clock framework. Moreover, the clocks are get by name in the driver, thus this change does not break backwards compatibility. Signed-off-by: Ryder Lee Reviewed-by: Matthias Brugger Acked-by: Rob Herring Signed-off-by: Herbert Xu --- .../devicetree/bindings/crypto/mediatek-crypto.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt b/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt index c204725e5873..450da3661cad 100644 --- a/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt +++ b/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt @@ -6,8 +6,7 @@ Required properties: - interrupts: Should contain the five crypto engines interrupts in numeric order. These are global system and four descriptor rings. - clocks: the clock used by the core -- clock-names: the names of the clock listed in the clocks property. These are - "ethif", "cryp" +- clock-names: Must contain "cryp". - power-domains: Must contain a reference to the PM domain. @@ -20,8 +19,7 @@ Example: , , ; - clocks = <&topckgen CLK_TOP_ETHIF_SEL>, - <ðsys CLK_ETHSYS_CRYPTO>; - clock-names = "ethif","cryp"; + clocks = <ðsys CLK_ETHSYS_CRYPTO>; + clock-names = "cryp"; power-domains = <&scpsys MT2701_POWER_DOMAIN_ETH>; }; From 03d2c5114c95797c0aa7d9f463348b171a274fd4 Mon Sep 17 00:00:00 2001 From: Martin Hicks Date: Tue, 2 May 2017 09:38:35 -0400 Subject: [PATCH 075/140] crypto: talitos - Extend max key length for SHA384/512-HMAC and AEAD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An updated patch that also handles the additional key length requirements for the AEAD algorithms. The max keysize is not 96. For SHA384/512 it's 128, and for the AEAD algorithms it's longer still. Extend the max keysize for the AEAD size for AES256 + HMAC(SHA512). Cc: # 3.6+ Fixes: 357fb60502ede ("crypto: talitos - add sha224, sha384 and sha512 to existing AEAD algorithms") Signed-off-by: Martin Hicks Acked-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 0bba6a19d36a..79791c690858 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -816,7 +816,7 @@ static void talitos_unregister_rng(struct device *dev) * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP */ #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) -#define TALITOS_MAX_KEY_SIZE 96 +#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ struct talitos_ctx { @@ -1495,6 +1495,11 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, { struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); + if (keylen > TALITOS_MAX_KEY_SIZE) { + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + memcpy(&ctx->key, key, keylen); ctx->keylen = keylen; From 3cdbe346ed3f380eae1cb3e9febfe703e7d8a7b0 Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Tue, 2 May 2017 17:33:40 -0500 Subject: [PATCH 076/140] crypto: ccp - Add debugfs entries for CCP information Expose some data about the configuration and operation of the CCP through debugfs entries: device name, capabilities, configuration, statistics. Allow the user to reset the counters to zero by writing (any value) to the 'stats' file. This can be done per queue or per device. Changes from V1: - Correct polarity of test when destroying devices at module unload Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/Makefile | 3 +- drivers/crypto/ccp/ccp-debugfs.c | 345 +++++++++++++++++++++++++++++++ drivers/crypto/ccp/ccp-dev-v5.c | 28 ++- drivers/crypto/ccp/ccp-dev.c | 2 +- drivers/crypto/ccp/ccp-dev.h | 20 ++ 5 files changed, 395 insertions(+), 3 deletions(-) create mode 100644 drivers/crypto/ccp/ccp-debugfs.c diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile index 60919a3ec53b..59493fd3a751 100644 --- a/drivers/crypto/ccp/Makefile +++ b/drivers/crypto/ccp/Makefile @@ -4,7 +4,8 @@ ccp-objs := ccp-dev.o \ ccp-dev-v3.o \ ccp-dev-v5.o \ ccp-platform.o \ - ccp-dmaengine.o + ccp-dmaengine.o \ + ccp-debugfs.o ccp-$(CONFIG_PCI) += ccp-pci.o obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o diff --git a/drivers/crypto/ccp/ccp-debugfs.c b/drivers/crypto/ccp/ccp-debugfs.c new file mode 100644 index 000000000000..6d86693b117f --- /dev/null +++ b/drivers/crypto/ccp/ccp-debugfs.c @@ -0,0 +1,345 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2017 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + +#include "ccp-dev.h" + +/* DebugFS helpers */ +#define OBUFP (obuf + oboff) +#define OBUFLEN 512 +#define OBUFSPC (OBUFLEN - oboff) +#define OSCNPRINTF(fmt, ...) \ + scnprintf(OBUFP, OBUFSPC, fmt, ## __VA_ARGS__) + +#define BUFLEN 63 + +#define RI_VERSION_NUM 0x0000003F +#define RI_AES_PRESENT 0x00000040 +#define RI_3DES_PRESENT 0x00000080 +#define RI_SHA_PRESENT 0x00000100 +#define RI_RSA_PRESENT 0x00000200 +#define RI_ECC_PRESENT 0x00000400 +#define RI_ZDE_PRESENT 0x00000800 +#define RI_ZCE_PRESENT 0x00001000 +#define RI_TRNG_PRESENT 0x00002000 +#define RI_ELFC_PRESENT 0x00004000 +#define RI_ELFC_SHIFT 14 +#define RI_NUM_VQM 0x00078000 +#define RI_NVQM_SHIFT 15 +#define RI_NVQM(r) (((r) * RI_NUM_VQM) >> RI_NVQM_SHIFT) +#define RI_LSB_ENTRIES 0x0FF80000 +#define RI_NLSB_SHIFT 19 +#define RI_NLSB(r) (((r) * RI_LSB_ENTRIES) >> RI_NLSB_SHIFT) + +static ssize_t ccp5_debugfs_info_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *offp) +{ + struct ccp_device *ccp = filp->private_data; + unsigned int oboff = 0; + unsigned int regval; + ssize_t ret; + char *obuf; + + if (!ccp) + return 0; + + obuf = kmalloc(OBUFLEN, GFP_KERNEL); + if (!obuf) + return -ENOMEM; + + oboff += OSCNPRINTF("Device name: %s\n", ccp->name); + oboff += OSCNPRINTF(" RNG name: %s\n", ccp->rngname); + oboff += OSCNPRINTF(" # Queues: %d\n", ccp->cmd_q_count); + oboff += OSCNPRINTF(" # Cmds: %d\n", ccp->cmd_count); + + regval = ioread32(ccp->io_regs + CMD5_PSP_CCP_VERSION); + oboff += OSCNPRINTF(" Version: %d\n", regval & RI_VERSION_NUM); + oboff += OSCNPRINTF(" Engines:"); + if (regval & RI_AES_PRESENT) + oboff += OSCNPRINTF(" AES"); + if (regval & RI_3DES_PRESENT) + oboff += OSCNPRINTF(" 3DES"); + if (regval & RI_SHA_PRESENT) + oboff += OSCNPRINTF(" SHA"); + if (regval & RI_RSA_PRESENT) + oboff += OSCNPRINTF(" RSA"); + if (regval & RI_ECC_PRESENT) + oboff += OSCNPRINTF(" ECC"); + if (regval & RI_ZDE_PRESENT) + oboff += OSCNPRINTF(" ZDE"); + if (regval & RI_ZCE_PRESENT) + oboff += OSCNPRINTF(" ZCE"); + if (regval & RI_TRNG_PRESENT) + oboff += OSCNPRINTF(" TRNG"); + oboff += OSCNPRINTF("\n"); + oboff += OSCNPRINTF(" Queues: %d\n", + (regval & RI_NUM_VQM) >> RI_NVQM_SHIFT); + oboff += OSCNPRINTF("LSB Entries: %d\n", + (regval & RI_LSB_ENTRIES) >> RI_NLSB_SHIFT); + + ret = simple_read_from_buffer(ubuf, count, offp, obuf, oboff); + kfree(obuf); + + return ret; +} + +/* Return a formatted buffer containing the current + * statistics across all queues for a CCP. + */ +static ssize_t ccp5_debugfs_stats_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *offp) +{ + struct ccp_device *ccp = filp->private_data; + unsigned long total_xts_aes_ops = 0; + unsigned long total_3des_ops = 0; + unsigned long total_aes_ops = 0; + unsigned long total_sha_ops = 0; + unsigned long total_rsa_ops = 0; + unsigned long total_ecc_ops = 0; + unsigned long total_pt_ops = 0; + unsigned long total_ops = 0; + unsigned int oboff = 0; + ssize_t ret = 0; + unsigned int i; + char *obuf; + + for (i = 0; i < ccp->cmd_q_count; i++) { + struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; + + total_ops += cmd_q->total_ops; + total_aes_ops += cmd_q->total_aes_ops; + total_xts_aes_ops += cmd_q->total_xts_aes_ops; + total_3des_ops += cmd_q->total_3des_ops; + total_sha_ops += cmd_q->total_sha_ops; + total_rsa_ops += cmd_q->total_rsa_ops; + total_pt_ops += cmd_q->total_pt_ops; + total_ecc_ops += cmd_q->total_ecc_ops; + } + + obuf = kmalloc(OBUFLEN, GFP_KERNEL); + if (!obuf) + return -ENOMEM; + + oboff += OSCNPRINTF("Total Interrupts Handled: %ld\n", + ccp->total_interrupts); + oboff += OSCNPRINTF(" Total Operations: %ld\n", + total_ops); + oboff += OSCNPRINTF(" AES: %ld\n", + total_aes_ops); + oboff += OSCNPRINTF(" XTS AES: %ld\n", + total_xts_aes_ops); + oboff += OSCNPRINTF(" SHA: %ld\n", + total_3des_ops); + oboff += OSCNPRINTF(" SHA: %ld\n", + total_sha_ops); + oboff += OSCNPRINTF(" RSA: %ld\n", + total_rsa_ops); + oboff += OSCNPRINTF(" Pass-Thru: %ld\n", + total_pt_ops); + oboff += OSCNPRINTF(" ECC: %ld\n", + total_ecc_ops); + + ret = simple_read_from_buffer(ubuf, count, offp, obuf, oboff); + kfree(obuf); + + return ret; +} + +/* Reset the counters in a queue + */ +static void ccp5_debugfs_reset_queue_stats(struct ccp_cmd_queue *cmd_q) +{ + cmd_q->total_ops = 0L; + cmd_q->total_aes_ops = 0L; + cmd_q->total_xts_aes_ops = 0L; + cmd_q->total_3des_ops = 0L; + cmd_q->total_sha_ops = 0L; + cmd_q->total_rsa_ops = 0L; + cmd_q->total_pt_ops = 0L; + cmd_q->total_ecc_ops = 0L; +} + +/* A value was written to the stats variable, which + * should be used to reset the queue counters across + * that device. + */ +static ssize_t ccp5_debugfs_stats_write(struct file *filp, + const char __user *ubuf, + size_t count, loff_t *offp) +{ + struct ccp_device *ccp = filp->private_data; + int i; + + for (i = 0; i < ccp->cmd_q_count; i++) + ccp5_debugfs_reset_queue_stats(&ccp->cmd_q[i]); + ccp->total_interrupts = 0L; + + return count; +} + +/* Return a formatted buffer containing the current information + * for that queue + */ +static ssize_t ccp5_debugfs_queue_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *offp) +{ + struct ccp_cmd_queue *cmd_q = filp->private_data; + unsigned int oboff = 0; + unsigned int regval; + ssize_t ret; + char *obuf; + + if (!cmd_q) + return 0; + + obuf = kmalloc(OBUFLEN, GFP_KERNEL); + if (!obuf) + return -ENOMEM; + + oboff += OSCNPRINTF(" Total Queue Operations: %ld\n", + cmd_q->total_ops); + oboff += OSCNPRINTF(" AES: %ld\n", + cmd_q->total_aes_ops); + oboff += OSCNPRINTF(" XTS AES: %ld\n", + cmd_q->total_xts_aes_ops); + oboff += OSCNPRINTF(" SHA: %ld\n", + cmd_q->total_3des_ops); + oboff += OSCNPRINTF(" SHA: %ld\n", + cmd_q->total_sha_ops); + oboff += OSCNPRINTF(" RSA: %ld\n", + cmd_q->total_rsa_ops); + oboff += OSCNPRINTF(" Pass-Thru: %ld\n", + cmd_q->total_pt_ops); + oboff += OSCNPRINTF(" ECC: %ld\n", + cmd_q->total_ecc_ops); + + regval = ioread32(cmd_q->reg_int_enable); + oboff += OSCNPRINTF(" Enabled Interrupts:"); + if (regval & INT_EMPTY_QUEUE) + oboff += OSCNPRINTF(" EMPTY"); + if (regval & INT_QUEUE_STOPPED) + oboff += OSCNPRINTF(" STOPPED"); + if (regval & INT_ERROR) + oboff += OSCNPRINTF(" ERROR"); + if (regval & INT_COMPLETION) + oboff += OSCNPRINTF(" COMPLETION"); + oboff += OSCNPRINTF("\n"); + + ret = simple_read_from_buffer(ubuf, count, offp, obuf, oboff); + kfree(obuf); + + return ret; +} + +/* A value was written to the stats variable for a + * queue. Reset the queue counters to this value. + */ +static ssize_t ccp5_debugfs_queue_write(struct file *filp, + const char __user *ubuf, + size_t count, loff_t *offp) +{ + struct ccp_cmd_queue *cmd_q = filp->private_data; + + ccp5_debugfs_reset_queue_stats(cmd_q); + + return count; +} + +static const struct file_operations ccp_debugfs_info_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ccp5_debugfs_info_read, + .write = NULL, +}; + +static const struct file_operations ccp_debugfs_queue_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ccp5_debugfs_queue_read, + .write = ccp5_debugfs_queue_write, +}; + +static const struct file_operations ccp_debugfs_stats_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ccp5_debugfs_stats_read, + .write = ccp5_debugfs_stats_write, +}; + +static struct dentry *ccp_debugfs_dir; +static DEFINE_RWLOCK(ccp_debugfs_lock); + +#define MAX_NAME_LEN 20 + +void ccp5_debugfs_setup(struct ccp_device *ccp) +{ + struct ccp_cmd_queue *cmd_q; + char name[MAX_NAME_LEN + 1]; + struct dentry *debugfs_info; + struct dentry *debugfs_stats; + struct dentry *debugfs_q_instance; + struct dentry *debugfs_q_stats; + unsigned long flags; + int i; + + if (!debugfs_initialized()) + return; + + write_lock_irqsave(&ccp_debugfs_lock, flags); + if (!ccp_debugfs_dir) { + ccp_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + if (!ccp_debugfs_dir) + return; + } + write_unlock_irqrestore(&ccp_debugfs_lock, flags); + + ccp->debugfs_instance = debugfs_create_dir(ccp->name, ccp_debugfs_dir); + if (!ccp->debugfs_instance) + return; + + debugfs_info = debugfs_create_file("info", 0400, + ccp->debugfs_instance, ccp, + &ccp_debugfs_info_ops); + if (!debugfs_info) + return; + + debugfs_stats = debugfs_create_file("stats", 0600, + ccp->debugfs_instance, ccp, + &ccp_debugfs_stats_ops); + if (!debugfs_stats) + return; + + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + snprintf(name, MAX_NAME_LEN - 1, "q%d", cmd_q->id); + + debugfs_q_instance = + debugfs_create_dir(name, ccp->debugfs_instance); + if (!debugfs_q_instance) + return; + + debugfs_q_stats = + debugfs_create_file("stats", 0600, + debugfs_q_instance, cmd_q, + &ccp_debugfs_queue_ops); + if (!debugfs_q_stats) + return; + } +} + +void ccp5_debugfs_destroy(void) +{ + debugfs_remove_recursive(ccp_debugfs_dir); +} diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index ccbe32d5dd1c..b10d2d2075cb 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -231,6 +232,8 @@ static int ccp5_do_cmd(struct ccp5_desc *desc, int i; int ret = 0; + cmd_q->total_ops++; + if (CCP5_CMD_SOC(desc)) { CCP5_CMD_IOC(desc) = 1; CCP5_CMD_SOC(desc) = 0; @@ -282,6 +285,8 @@ static int ccp5_perform_aes(struct ccp_op *op) union ccp_function function; u32 key_addr = op->sb_key * LSB_ITEM_SIZE; + op->cmd_q->total_aes_ops++; + /* Zero out all the fields of the command desc */ memset(&desc, 0, Q_DESC_SIZE); @@ -325,6 +330,8 @@ static int ccp5_perform_xts_aes(struct ccp_op *op) union ccp_function function; u32 key_addr = op->sb_key * LSB_ITEM_SIZE; + op->cmd_q->total_xts_aes_ops++; + /* Zero out all the fields of the command desc */ memset(&desc, 0, Q_DESC_SIZE); @@ -364,6 +371,8 @@ static int ccp5_perform_sha(struct ccp_op *op) struct ccp5_desc desc; union ccp_function function; + op->cmd_q->total_sha_ops++; + /* Zero out all the fields of the command desc */ memset(&desc, 0, Q_DESC_SIZE); @@ -404,6 +413,8 @@ static int ccp5_perform_des3(struct ccp_op *op) union ccp_function function; u32 key_addr = op->sb_key * LSB_ITEM_SIZE; + op->cmd_q->total_3des_ops++; + /* Zero out all the fields of the command desc */ memset(&desc, 0, sizeof(struct ccp5_desc)); @@ -444,6 +455,8 @@ static int ccp5_perform_rsa(struct ccp_op *op) struct ccp5_desc desc; union ccp_function function; + op->cmd_q->total_rsa_ops++; + /* Zero out all the fields of the command desc */ memset(&desc, 0, Q_DESC_SIZE); @@ -487,6 +500,8 @@ static int ccp5_perform_passthru(struct ccp_op *op) struct ccp_dma_info *daddr = &op->dst.u.dma; + op->cmd_q->total_pt_ops++; + memset(&desc, 0, Q_DESC_SIZE); CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU; @@ -543,6 +558,8 @@ static int ccp5_perform_ecc(struct ccp_op *op) struct ccp5_desc desc; union ccp_function function; + op->cmd_q->total_ecc_ops++; + /* Zero out all the fields of the command desc */ memset(&desc, 0, Q_DESC_SIZE); @@ -592,7 +609,6 @@ static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status) return queues ? 0 : -EINVAL; } - static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp, int lsb_cnt, int n_lsbs, unsigned long *lsb_pub) @@ -757,6 +773,7 @@ static irqreturn_t ccp5_irq_handler(int irq, void *data) struct ccp_device *ccp = dev_get_drvdata(dev); ccp5_disable_queue_interrupts(ccp); + ccp->total_interrupts++; if (ccp->use_tasklet) tasklet_schedule(&ccp->irq_tasklet); else @@ -956,6 +973,9 @@ static int ccp5_init(struct ccp_device *ccp) if (ret) goto e_hwrng; + /* Set up debugfs entries */ + ccp5_debugfs_setup(ccp); + return 0; e_hwrng: @@ -992,6 +1012,12 @@ static void ccp5_destroy(struct ccp_device *ccp) /* Remove this device from the list of available units first */ ccp_del_device(ccp); + /* We're in the process of tearing down the entire driver; + * when all the devices are gone clean up debugfs + */ + if (ccp_present()) + ccp5_debugfs_destroy(); + /* Disable and clear interrupts */ ccp5_disable_queue_interrupts(ccp); for (i = 0; i < ccp->cmd_q_count; i++) { diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index b7504562715c..2506b5025700 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -33,7 +33,7 @@ MODULE_AUTHOR("Tom Lendacky "); MODULE_AUTHOR("Gary R Hook "); MODULE_LICENSE("GPL"); -MODULE_VERSION("1.0.0"); +MODULE_VERSION("1.1.0"); MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver"); struct ccp_tasklet_data { diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 0cb09d0feeaf..a70154ac7405 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -70,6 +70,7 @@ #define LSB_PUBLIC_MASK_HI_OFFSET 0x1C #define LSB_PRIVATE_MASK_LO_OFFSET 0x20 #define LSB_PRIVATE_MASK_HI_OFFSET 0x24 +#define CMD5_PSP_CCP_VERSION 0x100 #define CMD5_Q_CONTROL_BASE 0x0000 #define CMD5_Q_TAIL_LO_BASE 0x0004 @@ -322,6 +323,16 @@ struct ccp_cmd_queue { /* Interrupt wait queue */ wait_queue_head_t int_queue; unsigned int int_rcvd; + + /* Per-queue Statistics */ + unsigned long total_ops; + unsigned long total_aes_ops; + unsigned long total_xts_aes_ops; + unsigned long total_3des_ops; + unsigned long total_sha_ops; + unsigned long total_rsa_ops; + unsigned long total_pt_ops; + unsigned long total_ecc_ops; } ____cacheline_aligned; struct ccp_device { @@ -419,6 +430,12 @@ struct ccp_device { /* DMA caching attribute support */ unsigned int axcache; + + /* Device Statistics */ + unsigned long total_interrupts; + + /* DebugFS info */ + struct dentry *debugfs_instance; }; enum ccp_memtype { @@ -632,6 +649,9 @@ void ccp_unregister_rng(struct ccp_device *ccp); int ccp_dmaengine_register(struct ccp_device *ccp); void ccp_dmaengine_unregister(struct ccp_device *ccp); +void ccp5_debugfs_setup(struct ccp_device *ccp); +void ccp5_debugfs_destroy(void); + /* Structure for computation functions that are device-specific */ struct ccp_actions { int (*aes)(struct ccp_op *); From 9d1fb19668207b66bc54f8e10fa91a2fcff061e4 Mon Sep 17 00:00:00 2001 From: pjambhlekar Date: Wed, 3 May 2017 09:32:09 +0530 Subject: [PATCH 077/140] crypto: ccp - return NULL instead of 0 This change is to handle sparse warning. Return type of function is a pointer to the structure and it returns 0. Instead it should return NULL. Signed-off-by: Pushkar Jambhlekar Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-platform.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/ccp/ccp-platform.c b/drivers/crypto/ccp/ccp-platform.c index 351f28d8c336..e26969e601ad 100644 --- a/drivers/crypto/ccp/ccp-platform.c +++ b/drivers/crypto/ccp/ccp-platform.c @@ -44,7 +44,7 @@ static struct ccp_vdata *ccp_get_of_version(struct platform_device *pdev) if (match && match->data) return (struct ccp_vdata *)match->data; #endif - return 0; + return NULL; } static struct ccp_vdata *ccp_get_acpi_version(struct platform_device *pdev) @@ -56,7 +56,7 @@ static struct ccp_vdata *ccp_get_acpi_version(struct platform_device *pdev) if (match && match->driver_data) return (struct ccp_vdata *)match->driver_data; #endif - return 0; + return NULL; } static int ccp_get_irq(struct ccp_device *ccp) From cc53e92ab36fdadfc9ecf97ff6f711691f49d3a1 Mon Sep 17 00:00:00 2001 From: George Cherian Date: Thu, 4 May 2017 11:34:44 +0000 Subject: [PATCH 078/140] crypto: cavium - Downgrade the annoying misc interrupt print from dev_err to dev_dbg Mailbox interrupt is common and it is not an error interrupt. So downgrade the print from dev_err to dev_dbg. Signed-off-by: George Cherian Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cptvf_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index 6ffc740c7431..5c796ed55eba 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -525,7 +525,7 @@ static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq) intr = cptvf_read_vf_misc_intr_status(cptvf); /*Check for MISC interrupt types*/ if (likely(intr & CPT_VF_INTR_MBOX_MASK)) { - dev_err(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n", + dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n", intr, cptvf->vfid); cptvf_handle_mbox_intr(cptvf); cptvf_clear_mbox_intr(cptvf); From e2eb769ed0bdc06cb523f475db411ce3a5f1c465 Mon Sep 17 00:00:00 2001 From: George Cherian Date: Thu, 4 May 2017 11:34:45 +0000 Subject: [PATCH 079/140] crypto: cavium - Remove the individual encrypt/decrypt function for each algorithm Remove the individual encrypt/decrypt function for easch algorithm. This is in prepration of adding more crypto algorithms supported by hardware. While at that simplify create_ctx_hdr/create_input_list function interfaces. Signed-off-by: George Cherian Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cptvf_algs.c | 153 ++++++++++++------------- drivers/crypto/cavium/cpt/cptvf_algs.h | 7 ++ 2 files changed, 83 insertions(+), 77 deletions(-) diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c index cc853f913d4b..443c3623d3be 100644 --- a/drivers/crypto/cavium/cpt/cptvf_algs.c +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -98,7 +98,6 @@ static inline void update_output_data(struct cpt_request_info *req_info, } static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc, - u32 cipher_type, u32 aes_key_type, u32 *argcnt) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); @@ -124,11 +123,11 @@ static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc, req_info->req.param1 = req->nbytes; /* Encryption Data length */ req_info->req.param2 = 0; /*Auth data length */ - fctx->enc.enc_ctrl.e.enc_cipher = cipher_type; - fctx->enc.enc_ctrl.e.aes_key = aes_key_type; + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR; - if (cipher_type == AES_XTS) + if (ctx->cipher_type == AES_XTS) memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); else memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); @@ -154,14 +153,13 @@ static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc, } static inline u32 create_input_list(struct ablkcipher_request *req, u32 enc, - u32 cipher_type, u32 aes_key_type, u32 enc_iv_len) { struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req); struct cpt_request_info *req_info = &rctx->cpt_req; u32 argcnt = 0; - create_ctx_hdr(req, enc, cipher_type, aes_key_type, &argcnt); + create_ctx_hdr(req, enc, &argcnt); update_input_iv(req_info, req->info, enc_iv_len, &argcnt); update_input_data(req_info, req->src, req->nbytes, &argcnt); req_info->incnt = argcnt; @@ -177,7 +175,6 @@ static inline void store_cb_info(struct ablkcipher_request *req, } static inline void create_output_list(struct ablkcipher_request *req, - u32 cipher_type, u32 enc_iv_len) { struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req); @@ -197,12 +194,9 @@ static inline void create_output_list(struct ablkcipher_request *req, req_info->outcnt = argcnt; } -static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc, - u32 cipher_type) +static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm); - u32 key_type = AES_128_BIT; struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req); u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm); struct fc_context *fctx = &rctx->fctx; @@ -210,36 +204,10 @@ static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc, void *cdev = NULL; int status; - switch (ctx->key_len) { - case 16: - key_type = AES_128_BIT; - break; - case 24: - key_type = AES_192_BIT; - break; - case 32: - if (cipher_type == AES_XTS) - key_type = AES_128_BIT; - else - key_type = AES_256_BIT; - break; - case 64: - if (cipher_type == AES_XTS) - key_type = AES_256_BIT; - else - return -EINVAL; - break; - default: - return -EINVAL; - } - - if (cipher_type == DES3_CBC) - key_type = 0; - memset(req_info, 0, sizeof(struct cpt_request_info)); memset(fctx, 0, sizeof(struct fc_context)); - create_input_list(req, enc, cipher_type, key_type, enc_iv_len); - create_output_list(req, cipher_type, enc_iv_len); + create_input_list(req, enc, enc_iv_len); + create_output_list(req, enc_iv_len); store_cb_info(req, req_info); cdev = dev_handle.cdev[smp_processor_id()]; status = cptvf_do_request(cdev, req_info); @@ -254,34 +222,14 @@ static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc, return -EINPROGRESS; } -int cvm_des3_encrypt_cbc(struct ablkcipher_request *req) +int cvm_encrypt(struct ablkcipher_request *req) { - return cvm_enc_dec(req, true, DES3_CBC); + return cvm_enc_dec(req, true); } -int cvm_des3_decrypt_cbc(struct ablkcipher_request *req) +int cvm_decrypt(struct ablkcipher_request *req) { - return cvm_enc_dec(req, false, DES3_CBC); -} - -int cvm_aes_encrypt_xts(struct ablkcipher_request *req) -{ - return cvm_enc_dec(req, true, AES_XTS); -} - -int cvm_aes_decrypt_xts(struct ablkcipher_request *req) -{ - return cvm_enc_dec(req, false, AES_XTS); -} - -int cvm_aes_encrypt_cbc(struct ablkcipher_request *req) -{ - return cvm_enc_dec(req, true, AES_CBC); -} - -int cvm_aes_decrypt_cbc(struct ablkcipher_request *req) -{ - return cvm_enc_dec(req, false, AES_CBC); + return cvm_enc_dec(req, false); } int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, @@ -299,24 +247,75 @@ int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, ctx->key_len = keylen; memcpy(ctx->enc_key, key1, keylen / 2); memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); + ctx->cipher_type = AES_XTS; + switch (ctx->key_len) { + case 32: + ctx->key_type = AES_128_BIT; + break; + case 64: + ctx->key_type = AES_256_BIT; + break; + default: + return -EINVAL; + } return 0; } -int cvm_enc_dec_setkey(struct crypto_ablkcipher *cipher, const u8 *key, - u32 keylen) +static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen) +{ + if ((keylen == 16) || (keylen == 24) || (keylen == 32)) { + ctx->key_len = keylen; + switch (ctx->key_len) { + case 16: + ctx->key_type = AES_128_BIT; + break; + case 24: + ctx->key_type = AES_192_BIT; + break; + case 32: + ctx->key_type = AES_256_BIT; + break; + default: + return -EINVAL; + } + + if (ctx->cipher_type == DES3_CBC) + ctx->key_type = 0; + + return 0; + } + + return -EINVAL; +} + +static int cvm_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + u32 keylen, u8 cipher_type) { struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm); - if ((keylen == 16) || (keylen == 24) || (keylen == 32)) { - ctx->key_len = keylen; + ctx->cipher_type = cipher_type; + if (!cvm_validate_keylen(ctx, keylen)) { memcpy(ctx->enc_key, key, keylen); return 0; + } else { + crypto_ablkcipher_set_flags(cipher, + CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; } - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); +} - return -EINVAL; +static int cvm_cbc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + u32 keylen) +{ + return cvm_setkey(cipher, key, keylen, AES_CBC); +} + +static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + u32 keylen) +{ + return cvm_setkey(cipher, key, keylen, DES3_CBC); } int cvm_enc_dec_init(struct crypto_tfm *tfm) @@ -349,8 +348,8 @@ struct crypto_alg algs[] = { { .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, .setkey = cvm_xts_setkey, - .encrypt = cvm_aes_encrypt_xts, - .decrypt = cvm_aes_decrypt_xts, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, }, }, .cra_init = cvm_enc_dec_init, @@ -369,9 +368,9 @@ struct crypto_alg algs[] = { { .ivsize = AES_BLOCK_SIZE, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, - .setkey = cvm_enc_dec_setkey, - .encrypt = cvm_aes_encrypt_cbc, - .decrypt = cvm_aes_decrypt_cbc, + .setkey = cvm_cbc_aes_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, }, }, .cra_init = cvm_enc_dec_init, @@ -390,9 +389,9 @@ struct crypto_alg algs[] = { { .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, .ivsize = DES_BLOCK_SIZE, - .setkey = cvm_enc_dec_setkey, - .encrypt = cvm_des3_encrypt_cbc, - .decrypt = cvm_des3_decrypt_cbc, + .setkey = cvm_cbc_des3_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, }, }, .cra_init = cvm_enc_dec_init, diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.h b/drivers/crypto/cavium/cpt/cptvf_algs.h index a12050d11b0c..902f25751123 100644 --- a/drivers/crypto/cavium/cpt/cptvf_algs.h +++ b/drivers/crypto/cavium/cpt/cptvf_algs.h @@ -77,6 +77,11 @@ union encr_ctrl { } e; }; +struct cvm_cipher { + const char *name; + u8 value; +}; + struct enc_context { union encr_ctrl enc_ctrl; u8 encr_key[32]; @@ -96,6 +101,8 @@ struct fc_context { struct cvm_enc_ctx { u32 key_len; u8 enc_key[MAX_KEY_SIZE]; + u8 cipher_type:4; + u8 key_type:2; }; struct cvm_des3_ctx { From 10d82222d9411dcc0ebcb11ca110597246696af4 Mon Sep 17 00:00:00 2001 From: George Cherian Date: Thu, 4 May 2017 11:34:46 +0000 Subject: [PATCH 080/140] crypto: cavium - Add more algorithms Add more algorithm support for the driver. Add support for ecb(aes), cfb(aes) and ecb(des3_ede). Signed-off-by: George Cherian Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cptvf_algs.c | 81 ++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c index 443c3623d3be..1b220f3ed017 100644 --- a/drivers/crypto/cavium/cpt/cptvf_algs.c +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -312,12 +312,30 @@ static int cvm_cbc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, return cvm_setkey(cipher, key, keylen, AES_CBC); } +static int cvm_ecb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + u32 keylen) +{ + return cvm_setkey(cipher, key, keylen, AES_ECB); +} + +static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + u32 keylen) +{ + return cvm_setkey(cipher, key, keylen, AES_CFB); +} + static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key, u32 keylen) { return cvm_setkey(cipher, key, keylen, DES3_CBC); } +static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + u32 keylen) +{ + return cvm_setkey(cipher, key, keylen, DES3_ECB); +} + int cvm_enc_dec_init(struct crypto_tfm *tfm) { struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm); @@ -375,6 +393,48 @@ struct crypto_alg algs[] = { { }, .cra_init = cvm_enc_dec_init, .cra_module = THIS_MODULE, +}, { + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct cvm_enc_ctx), + .cra_alignmask = 7, + .cra_priority = 4001, + .cra_name = "ecb(aes)", + .cra_driver_name = "cavium-ecb-aes", + .cra_type = &crypto_ablkcipher_type, + .cra_u = { + .ablkcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = cvm_ecb_aes_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + }, + }, + .cra_init = cvm_enc_dec_init, + .cra_module = THIS_MODULE, +}, { + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct cvm_enc_ctx), + .cra_alignmask = 7, + .cra_priority = 4001, + .cra_name = "cfb(aes)", + .cra_driver_name = "cavium-cfb-aes", + .cra_type = &crypto_ablkcipher_type, + .cra_u = { + .ablkcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = cvm_cfb_aes_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + }, + }, + .cra_init = cvm_enc_dec_init, + .cra_module = THIS_MODULE, }, { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = DES3_EDE_BLOCK_SIZE, @@ -396,6 +456,27 @@ struct crypto_alg algs[] = { { }, .cra_init = cvm_enc_dec_init, .cra_module = THIS_MODULE, +}, { + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct cvm_des3_ctx), + .cra_alignmask = 7, + .cra_priority = 4001, + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "cavium-ecb-des3_ede", + .cra_type = &crypto_ablkcipher_type, + .cra_u = { + .ablkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = cvm_ecb_des3_setkey, + .encrypt = cvm_encrypt, + .decrypt = cvm_decrypt, + }, + }, + .cra_init = cvm_enc_dec_init, + .cra_module = THIS_MODULE, } }; static inline int cav_register_algs(void) From b9162917fafd36690491af35a262e628c2ffd8a7 Mon Sep 17 00:00:00 2001 From: Geliang Tang Date: Sat, 6 May 2017 23:35:29 +0800 Subject: [PATCH 081/140] crypto: mediatek - drop .owner field in mtk_crypto_driver Drop .owner field in mtk_crypto_driver, since platform_driver_register() will set it automatically. Signed-off-by: Geliang Tang Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-platform.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c index 9b4437c857b9..000b6500a22d 100644 --- a/drivers/crypto/mediatek/mtk-platform.c +++ b/drivers/crypto/mediatek/mtk-platform.c @@ -588,7 +588,6 @@ static struct platform_driver mtk_crypto_driver = { .remove = mtk_crypto_remove, .driver = { .name = "mtk-crypto", - .owner = THIS_MODULE, .of_match_table = of_crypto_id, }, }; From fa598d0a7026219461a1ad76e27117ce325fa0e4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 9 May 2017 17:20:27 -0700 Subject: [PATCH 082/140] crypto: aes_ti - fix comment for MixColumns step mix_columns() contains a comment which shows the matrix used by the MixColumns step of AES, but the last entry in this matrix was incorrect Signed-off-by: Herbert Xu --- crypto/aes_ti.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/aes_ti.c b/crypto/aes_ti.c index 92644fd1ac19..03023b2290e8 100644 --- a/crypto/aes_ti.c +++ b/crypto/aes_ti.c @@ -114,7 +114,7 @@ static u32 mix_columns(u32 x) * | 0x2 0x3 0x1 0x1 | | x[0] | * | 0x1 0x2 0x3 0x1 | | x[1] | * | 0x1 0x1 0x2 0x3 | x | x[2] | - * | 0x3 0x1 0x1 0x3 | | x[3] | + * | 0x3 0x1 0x1 0x2 | | x[3] | */ u32 y = mul_by_x(x) ^ ror32(x, 16); From 8270f5d799c946d7636ef317eff990830d2fe89f Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 10 May 2017 08:52:26 +0300 Subject: [PATCH 083/140] crypto: glue_helper - Delete some dead code We checked (nbytes < bsize) inside the loops so it's not possible to hit the "goto done;" here. This code is cut and paste from other slightly different loops where we don't have the check inside the loop. Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- arch/x86/crypto/glue_helper.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c index 24ac9fad832d..d61e57960fe0 100644 --- a/arch/x86/crypto/glue_helper.c +++ b/arch/x86/crypto/glue_helper.c @@ -176,9 +176,6 @@ __glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx, src -= 1; dst -= 1; } while (nbytes >= func_bytes); - - if (nbytes < bsize) - goto done; } } From ea644b8ca44b9fcbb41c58e1d51a0c8cdad0a61b Mon Sep 17 00:00:00 2001 From: Kamil Konieczny Date: Fri, 12 May 2017 17:38:02 +0200 Subject: [PATCH 084/140] crypto: doc - Fixed bugs, added example usage of calc_hash(). - Fixed bugs in example for shash and rng (added missing "*" and " *"). - Corrected pr_info() in calc_hash(). - Added example usage of calc_hash(). - No need for negate PTR_ERR to get error code, as crypto_alloc_rng already returns negative values like ERR_PTR(-ENOMEM). Fixed. Signed-off-by: Kamil Konieczny Signed-off-by: Herbert Xu --- Documentation/crypto/api-samples.rst | 38 ++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/Documentation/crypto/api-samples.rst b/Documentation/crypto/api-samples.rst index d021fd96a76d..2531948db89f 100644 --- a/Documentation/crypto/api-samples.rst +++ b/Documentation/crypto/api-samples.rst @@ -155,9 +155,9 @@ Code Example For Use of Operational State Memory With SHASH char ctx[]; }; - static struct sdesc init_sdesc(struct crypto_shash *alg) + static struct sdesc *init_sdesc(struct crypto_shash *alg) { - struct sdesc sdesc; + struct sdesc *sdesc; int size; size = sizeof(struct shash_desc) + crypto_shash_descsize(alg); @@ -169,15 +169,16 @@ Code Example For Use of Operational State Memory With SHASH return sdesc; } - static int calc_hash(struct crypto_shashalg, - const unsigned chardata, unsigned int datalen, - unsigned chardigest) { - struct sdesc sdesc; + static int calc_hash(struct crypto_shash *alg, + const unsigned char *data, unsigned int datalen, + unsigned char *digest) + { + struct sdesc *sdesc; int ret; sdesc = init_sdesc(alg); if (IS_ERR(sdesc)) { - pr_info("trusted_key: can't alloc %s\n", hash_alg); + pr_info("can't alloc sdesc\n"); return PTR_ERR(sdesc); } @@ -186,6 +187,23 @@ Code Example For Use of Operational State Memory With SHASH return ret; } + static int test_hash(const unsigned char *data, unsigned int datalen, + unsigned char *digest) + { + struct crypto_shash *alg; + char *hash_alg_name = "sha1-padlock-nano"; + int ret; + + alg = crypto_alloc_shash(hash_alg_name, CRYPTO_ALG_TYPE_SHASH, 0); + if (IS_ERR(alg)) { + pr_info("can't alloc alg %s\n", hash_alg_name); + return PTR_ERR(alg); + } + ret = calc_hash(alg, data, datalen, digest); + crypto_free_shash(alg); + return ret; + } + Code Example For Random Number Generator Usage ---------------------------------------------- @@ -195,8 +213,8 @@ Code Example For Random Number Generator Usage static int get_random_numbers(u8 *buf, unsigned int len) { - struct crypto_rngrng = NULL; - chardrbg = "drbg_nopr_sha256"; /* Hash DRBG with SHA-256, no PR */ + struct crypto_rng *rng = NULL; + char *drbg = "drbg_nopr_sha256"; /* Hash DRBG with SHA-256, no PR */ int ret; if (!buf || !len) { @@ -207,7 +225,7 @@ Code Example For Random Number Generator Usage rng = crypto_alloc_rng(drbg, 0, 0); if (IS_ERR(rng)) { pr_debug("could not allocate RNG handle for %s\n", drbg); - return -PTR_ERR(rng); + return PTR_ERR(rng); } ret = crypto_rng_get_bytes(rng, buf, len); From c72358571aaadf46755d3e38f671233a7dd48f00 Mon Sep 17 00:00:00 2001 From: Karim Eshapa Date: Sat, 13 May 2017 21:05:19 +0200 Subject: [PATCH 085/140] crypto: algapi - Use pr_err common logging style. Use more common error logging style. Signed-off-by: Karim Eshapa Signed-off-by: Herbert Xu --- crypto/algapi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 9eed4ef9c971..e4cc7615a139 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -260,7 +260,7 @@ void crypto_alg_tested(const char *name, int err) goto found; } - printk(KERN_ERR "alg: Unexpected test result for %s: %d\n", name, err); + pr_err("alg: Unexpected test result for %s: %d\n", name, err); goto unlock; found: From e3f9490e89c2240f87c6d040575cdc11dd405afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:38:53 +0200 Subject: [PATCH 086/140] crypto: sun4i-ss - group variable definitions in sun4i_hash() Cosmetic change to avoid having a full screen a variable definitions. It also helps to see which variables share the same type. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-hash.c | 33 +++++++++---------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index 0de2f62d51ff..855325338844 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -167,37 +167,28 @@ int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in) */ static int sun4i_hash(struct ahash_request *areq) { - u32 v, ivmode = 0; - unsigned int i = 0; /* * i is the total bytes read from SGs, to be compared to areq->nbytes * i is important because we cannot rely on SG length since the sum of * SG->length could be greater than areq->nbytes + * + * end is the position when we need to stop writing to the device, + * to be compared to i + * + * in_i: advancement in the current SG */ - + unsigned int i = 0, end, index, padlen, nwait, nbw = 0, j = 0, todo; + unsigned int in_i = 0; + u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32], wb = 0, v, ivmode = 0; struct sun4i_req_ctx *op = ahash_request_ctx(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun4i_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); struct sun4i_ss_ctx *ss = tfmctx->ss; - unsigned int in_i = 0; /* advancement in the current SG */ - unsigned int end; - /* - * end is the position when we need to stop writing to the device, - * to be compared to i - */ - int in_r, err = 0; - unsigned int todo; - u32 spaces, rx_cnt = SS_RX_DEFAULT; - size_t copied = 0; - struct sg_mapping_iter mi; - unsigned int j = 0; - int zeros; - unsigned int index, padlen; - __be64 bits; - u32 bf[32]; - u32 wb = 0; - unsigned int nwait, nbw = 0; struct scatterlist *in_sg = areq->src; + struct sg_mapping_iter mi; + int in_r, err = 0, zeros; + size_t copied = 0; + __be64 bits; dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x", __func__, crypto_tfm_alg_name(areq->base.tfm), From a595e60a70c0d035b5a088f90ae3fde475c7e755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:38:54 +0200 Subject: [PATCH 087/140] crypto: sun4i-ss - remove conditional checks against 0 Cosmetic clean up if conditional checks on 0s values. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-cipher.c | 28 ++++++++-------- drivers/crypto/sunxi-ss/sun4i-ss-core.c | 10 +++--- drivers/crypto/sunxi-ss/sun4i-ss-hash.c | 40 +++++++++++------------ 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c index 90efd10d57a1..23e549204365 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -38,7 +38,7 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) unsigned int oi, oo; /* offset for in and out */ unsigned long flags; - if (areq->nbytes == 0) + if (!areq->nbytes) return 0; if (!areq->info) { @@ -82,7 +82,7 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) oo = 0; do { todo = min3(rx_cnt, ileft, (mi.length - oi) / 4); - if (todo > 0) { + if (todo) { ileft -= todo; writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); oi += todo * 4; @@ -97,7 +97,7 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) tx_cnt = SS_TXFIFO_SPACES(spaces); todo = min3(tx_cnt, oleft, (mo.length - oo) / 4); - if (todo > 0) { + if (todo) { oleft -= todo; readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); oo += todo * 4; @@ -106,7 +106,7 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) sg_miter_next(&mo); oo = 0; } - } while (oleft > 0); + } while (oleft); if (areq->info) { for (i = 0; i < 4 && i < ivsize / 4; i++) { @@ -154,7 +154,7 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) unsigned int obl = 0; /* length of data in bufo */ unsigned long flags; - if (areq->nbytes == 0) + if (!areq->nbytes) return 0; if (!areq->info) { @@ -172,12 +172,12 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) * we can use the SS optimized function */ while (in_sg && no_chunk == 1) { - if ((in_sg->length % 4) != 0) + if (in_sg->length % 4) no_chunk = 0; in_sg = sg_next(in_sg); } while (out_sg && no_chunk == 1) { - if ((out_sg->length % 4) != 0) + if (out_sg->length % 4) no_chunk = 0; out_sg = sg_next(out_sg); } @@ -214,14 +214,14 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) oi = 0; oo = 0; - while (oleft > 0) { - if (ileft > 0) { + while (oleft) { + if (ileft) { /* * todo is the number of consecutive 4byte word that we * can read from current SG */ todo = min3(rx_cnt, ileft / 4, (mi.length - oi) / 4); - if (todo > 0 && ob == 0) { + if (todo && !ob) { writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); ileft -= todo * 4; @@ -240,7 +240,7 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) ileft -= todo; oi += todo; ob += todo; - if (ob % 4 == 0) { + if (!(ob % 4)) { writesl(ss->base + SS_RXFIFO, buf, ob / 4); ob = 0; @@ -260,11 +260,11 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) oi, mi.length, ileft, areq->nbytes, rx_cnt, oo, mo.length, oleft, areq->nbytes, tx_cnt, ob); - if (tx_cnt == 0) + if (!tx_cnt) continue; /* todo in 4bytes word */ todo = min3(tx_cnt, oleft / 4, (mo.length - oo) / 4); - if (todo > 0) { + if (todo) { readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); oleft -= todo * 4; oo += todo * 4; @@ -516,7 +516,7 @@ int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, flags = crypto_ablkcipher_get_flags(tfm); ret = des_ekey(tmp, key); - if (unlikely(ret == 0) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + if (unlikely(!ret) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) { crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); dev_dbg(ss->dev, "Weak key %u\n", keylen); return -EINVAL; diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c index 3ac6c6c4ad18..50af81c3eb45 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c @@ -266,12 +266,12 @@ static int sun4i_ss_probe(struct platform_device *pdev) /* Enable both clocks */ err = clk_prepare_enable(ss->busclk); - if (err != 0) { + if (err) { dev_err(&pdev->dev, "Cannot prepare_enable busclk\n"); return err; } err = clk_prepare_enable(ss->ssclk); - if (err != 0) { + if (err) { dev_err(&pdev->dev, "Cannot prepare_enable ssclk\n"); goto error_ssclk; } @@ -281,7 +281,7 @@ static int sun4i_ss_probe(struct platform_device *pdev) * Try to set the clock to the maximum allowed */ err = clk_set_rate(ss->ssclk, cr_mod); - if (err != 0) { + if (err) { dev_err(&pdev->dev, "Cannot set clock rate to ssclk\n"); goto error_clk; } @@ -342,7 +342,7 @@ static int sun4i_ss_probe(struct platform_device *pdev) switch (ss_algs[i].type) { case CRYPTO_ALG_TYPE_ABLKCIPHER: err = crypto_register_alg(&ss_algs[i].alg.crypto); - if (err != 0) { + if (err) { dev_err(ss->dev, "Fail to register %s\n", ss_algs[i].alg.crypto.cra_name); goto error_alg; @@ -350,7 +350,7 @@ static int sun4i_ss_probe(struct platform_device *pdev) break; case CRYPTO_ALG_TYPE_AHASH: err = crypto_register_ahash(&ss_algs[i].alg.hash); - if (err != 0) { + if (err) { dev_err(ss->dev, "Fail to register %s\n", ss_algs[i].alg.hash.halg.base.cra_name); goto error_alg; diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index 855325338844..39f0abe5ee25 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -60,7 +60,7 @@ int sun4i_hash_export_md5(struct ahash_request *areq, void *out) memcpy(octx->block, op->buf, op->len); - if (op->byte_count > 0) { + if (op->byte_count) { for (i = 0; i < 4; i++) octx->hash[i] = op->hash[i]; } else { @@ -102,7 +102,7 @@ int sun4i_hash_export_sha1(struct ahash_request *areq, void *out) memcpy(octx->buffer, op->buf, op->len); - if (op->byte_count > 0) { + if (op->byte_count) { for (i = 0; i < 5; i++) octx->state[i] = op->hash[i]; } else { @@ -195,7 +195,7 @@ static int sun4i_hash(struct ahash_request *areq) op->byte_count, areq->nbytes, op->mode, op->len, op->hash[0]); - if (unlikely(areq->nbytes == 0) && (op->flags & SS_HASH_FINAL) == 0) + if (unlikely(!areq->nbytes) && !(op->flags & SS_HASH_FINAL)) return 0; /* protect against overflow */ @@ -204,7 +204,7 @@ static int sun4i_hash(struct ahash_request *areq) return -EINVAL; } - if (op->len + areq->nbytes < 64 && (op->flags & SS_HASH_FINAL) == 0) { + if (op->len + areq->nbytes < 64 && !(op->flags & SS_HASH_FINAL)) { /* linearize data to op->buf */ copied = sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), op->buf + op->len, areq->nbytes, 0); @@ -218,7 +218,7 @@ static int sun4i_hash(struct ahash_request *areq) * if some data have been processed before, * we need to restore the partial hash state */ - if (op->byte_count > 0) { + if (op->byte_count) { ivmode = SS_IV_ARBITRARY; for (i = 0; i < 5; i++) writel(op->hash[i], ss->base + SS_IV0 + i * 4); @@ -226,11 +226,11 @@ static int sun4i_hash(struct ahash_request *areq) /* Enable the device */ writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL); - if ((op->flags & SS_HASH_UPDATE) == 0) + if (!(op->flags & SS_HASH_UPDATE)) goto hash_final; /* start of handling data */ - if ((op->flags & SS_HASH_FINAL) == 0) { + if (!(op->flags & SS_HASH_FINAL)) { end = ((areq->nbytes + op->len) / 64) * 64 - op->len; if (end > areq->nbytes || areq->nbytes - end > 63) { @@ -244,14 +244,14 @@ static int sun4i_hash(struct ahash_request *areq) end = ((areq->nbytes + op->len) / 4) * 4 - op->len; } - /* TODO if SGlen % 4 and op->len == 0 then DMA */ + /* TODO if SGlen % 4 and !op->len then DMA */ i = 1; while (in_sg && i == 1) { - if ((in_sg->length % 4) != 0) + if (in_sg->length % 4) i = 0; in_sg = sg_next(in_sg); } - if (i == 1 && op->len == 0) + if (i == 1 && !op->len) dev_dbg(ss->dev, "We can DMA\n"); i = 0; @@ -266,7 +266,7 @@ static int sun4i_hash(struct ahash_request *areq) * - the buffer is already used * - the SG does not have enough byte remaining ( < 4) */ - if (op->len > 0 || (mi.length - in_i) < 4) { + if (op->len || (mi.length - in_i) < 4) { /* * if we have entered here we have two reason to stop * - the buffer is full @@ -285,7 +285,7 @@ static int sun4i_hash(struct ahash_request *areq) in_i = 0; } } - if (op->len > 3 && (op->len % 4) == 0) { + if (op->len > 3 && !(op->len % 4)) { /* write buf to the device */ writesl(ss->base + SS_RXFIFO, op->buf, op->len / 4); @@ -304,7 +304,7 @@ static int sun4i_hash(struct ahash_request *areq) i += todo * 4; in_i += todo * 4; rx_cnt -= todo; - if (rx_cnt == 0) { + if (!rx_cnt) { spaces = readl(ss->base + SS_FCSR); rx_cnt = SS_RXFIFO_SPACES(spaces); } @@ -342,7 +342,7 @@ static int sun4i_hash(struct ahash_request *areq) * Now if we have the flag final go to finalize part * If not, store the partial hash */ - if ((op->flags & SS_HASH_FINAL) > 0) + if (op->flags & SS_HASH_FINAL) goto hash_final; writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL); @@ -350,7 +350,7 @@ static int sun4i_hash(struct ahash_request *areq) do { v = readl(ss->base + SS_CTL); i++; - } while (i < SS_TIMEOUT && (v & SS_DATA_END) > 0); + } while (i < SS_TIMEOUT && (v & SS_DATA_END)); if (unlikely(i >= SS_TIMEOUT)) { dev_err_ratelimited(ss->dev, "ERROR: hash end timeout %d>%d ctl=%x len=%u\n", @@ -379,9 +379,9 @@ static int sun4i_hash(struct ahash_request *areq) hash_final: /* write the remaining words of the wait buffer */ - if (op->len > 0) { + if (op->len) { nwait = op->len / 4; - if (nwait > 0) { + if (nwait) { writesl(ss->base + SS_RXFIFO, op->buf, nwait); op->byte_count += 4 * nwait; } @@ -391,7 +391,7 @@ hash_final: } /* write the remaining bytes of the nbw buffer */ - if (nbw > 0) { + if (nbw) { wb |= ((1 << 7) << (nbw * 8)); bf[j++] = wb; } else { @@ -444,7 +444,7 @@ hash_final: do { v = readl(ss->base + SS_CTL); i++; - } while (i < SS_TIMEOUT && (v & SS_DATA_END) > 0); + } while (i < SS_TIMEOUT && (v & SS_DATA_END)); if (unlikely(i >= SS_TIMEOUT)) { dev_err_ratelimited(ss->dev, "ERROR: hash end timeout %d>%d ctl=%x len=%u\n", @@ -504,7 +504,7 @@ int sun4i_hash_digest(struct ahash_request *areq) struct sun4i_req_ctx *op = ahash_request_ctx(areq); err = sun4i_hash_init(areq); - if (err != 0) + if (err) return err; op->flags = SS_HASH_UPDATE | SS_HASH_FINAL; From 11be0107ab1a893121ff31325e1bf82d843ea4dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:38:55 +0200 Subject: [PATCH 088/140] crypto: sun4i-ss - use lower/upper_32_bits helpers Replace custom bit shifts and masks with lower/upper_32_bits helpers. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-hash.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index 39f0abe5ee25..e53c3127b743 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -188,7 +188,6 @@ static int sun4i_hash(struct ahash_request *areq) struct sg_mapping_iter mi; int in_r, err = 0, zeros; size_t copied = 0; - __be64 bits; dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x", __func__, crypto_tfm_alg_name(areq->base.tfm), @@ -423,12 +422,13 @@ hash_final: /* write the length of data */ if (op->mode == SS_OP_SHA1) { - bits = cpu_to_be64(op->byte_count << 3); - bf[j++] = bits & 0xffffffff; - bf[j++] = (bits >> 32) & 0xffffffff; + __be64 bits = cpu_to_be64(op->byte_count << 3); + bf[j++] = lower_32_bits(bits); + bf[j++] = upper_32_bits(bits); } else { - bf[j++] = (op->byte_count << 3) & 0xffffffff; - bf[j++] = (op->byte_count >> 29) & 0xffffffff; + __le64 bits = op->byte_count << 3; + bf[j++] = lower_32_bits(bits); + bf[j++] = upper_32_bits(bits); } writesl(ss->base + SS_RXFIFO, bf, j); From 0f52ddaed6500d0b74fcaf5568726068d385129f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:38:56 +0200 Subject: [PATCH 089/140] crypto: sun4i-ss - cannot use DMA is the request is 0 length Do not use DMA is the request is 0 length. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index e53c3127b743..bc3cbde07219 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -250,7 +250,7 @@ static int sun4i_hash(struct ahash_request *areq) i = 0; in_sg = sg_next(in_sg); } - if (i == 1 && !op->len) + if (i == 1 && !op->len && areq->nbytes) dev_dbg(ss->dev, "We can DMA\n"); i = 0; From 7e6df1f7a3a5f379b04eee177be0571beb32b811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:38:57 +0200 Subject: [PATCH 090/140] crypto: sun4i-ss - do not dynamically set parts of the last buffer to 0 Parts of the bf buffer were dynamically set to 0. Change this to set the whole buffer to 0 by default to avoid any mistake. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-hash.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index bc3cbde07219..518bc7a7df6a 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -179,7 +179,7 @@ static int sun4i_hash(struct ahash_request *areq) */ unsigned int i = 0, end, index, padlen, nwait, nbw = 0, j = 0, todo; unsigned int in_i = 0; - u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32], wb = 0, v, ivmode = 0; + u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32] = {0}, wb = 0, v, ivmode = 0; struct sun4i_req_ctx *op = ahash_request_ctx(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun4i_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); @@ -417,7 +417,6 @@ hash_final: zeros = (padlen - 1) / 4; } - memset(bf + j, 0, 4 * zeros); j += zeros; /* write the length of data */ From 214a9bd0f8dec468de6a32f445c540f786773ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:38:58 +0200 Subject: [PATCH 091/140] crypto: sun4i-ss - simplify the pad length calculation When sending the last block of data to the engine, it should be padded so that the total length of the request can be given to the engine as the last 2 words of the last 64 bytes block. Simplify the calculation of this pad offset. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-hash.c | 29 ++++++++++--------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index 518bc7a7df6a..59d9b4412aca 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -177,7 +177,7 @@ static int sun4i_hash(struct ahash_request *areq) * * in_i: advancement in the current SG */ - unsigned int i = 0, end, index, padlen, nwait, nbw = 0, j = 0, todo; + unsigned int i = 0, end, fill, min_fill, nwait, nbw = 0, j = 0, todo; unsigned int in_i = 0; u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32] = {0}, wb = 0, v, ivmode = 0; struct sun4i_req_ctx *op = ahash_request_ctx(areq); @@ -186,7 +186,7 @@ static int sun4i_hash(struct ahash_request *areq) struct sun4i_ss_ctx *ss = tfmctx->ss; struct scatterlist *in_sg = areq->src; struct sg_mapping_iter mi; - int in_r, err = 0, zeros; + int in_r, err = 0; size_t copied = 0; dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x", @@ -387,6 +387,8 @@ hash_final: nbw = op->len - 4 * nwait; wb = *(u32 *)(op->buf + nwait * 4); wb &= (0xFFFFFFFF >> (4 - nbw) * 8); + + op->byte_count += nbw; } /* write the remaining bytes of the nbw buffer */ @@ -402,22 +404,15 @@ hash_final: * I take the operations from other MD5/SHA1 implementations */ - /* we have already send 4 more byte of which nbw data */ - if (op->mode == SS_OP_MD5) { - index = (op->byte_count + 4) & 0x3f; - op->byte_count += nbw; - if (index > 56) - zeros = (120 - index) / 4; - else - zeros = (56 - index) / 4; - } else { - op->byte_count += nbw; - index = op->byte_count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); - zeros = (padlen - 1) / 4; - } + /* last block size */ + fill = 64 - (op->byte_count % 64); + min_fill = 2 * sizeof(u32) + (nbw ? 0 : sizeof(u32)); - j += zeros; + /* if we can't fill all data, jump to the next 64 block */ + if (fill < min_fill) + fill += 64; + + j += (fill - min_fill) / sizeof(u32); /* write the length of data */ if (op->mode == SS_OP_SHA1) { From 303391d69b6ae7dff30f7886b717e39f47067d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:38:59 +0200 Subject: [PATCH 092/140] crypto: sun4i-ss - simplify the appended bit assignment A bit is appended at the end of the input buffer for sha1. Simplify the code assigning it. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-hash.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index 59d9b4412aca..0c2efc88bc0a 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -392,12 +392,8 @@ hash_final: } /* write the remaining bytes of the nbw buffer */ - if (nbw) { - wb |= ((1 << 7) << (nbw * 8)); - bf[j++] = wb; - } else { - bf[j++] = 1 << 7; - } + wb |= ((1 << 7) << (nbw * 8)); + bf[j++] = wb; /* * number of space to pad to obtain 64o minus 8(size) minus 4 (final 1) From d78867a94ea74b8d2dafdbf370e0707f37a58f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:39:00 +0200 Subject: [PATCH 093/140] crypto: sun4i-ss - use GENMASK to generate masks Use the GENMASK helper instead of custom calculations to generate masks, It also helps the readability. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-hash.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index 0c2efc88bc0a..685de5b6ab17 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -384,11 +384,14 @@ hash_final: writesl(ss->base + SS_RXFIFO, op->buf, nwait); op->byte_count += 4 * nwait; } - nbw = op->len - 4 * nwait; - wb = *(u32 *)(op->buf + nwait * 4); - wb &= (0xFFFFFFFF >> (4 - nbw) * 8); - op->byte_count += nbw; + nbw = op->len - 4 * nwait; + if (nbw) { + wb = *(u32 *)(op->buf + nwait * 4); + wb &= GENMASK((nbw * 8) - 1, 0); + + op->byte_count += nbw; + } } /* write the remaining bytes of the nbw buffer */ From 317cbacf720c35c3c47e9d167c1946fa144a1e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:39:01 +0200 Subject: [PATCH 094/140] crypto: sun4i-ss - move from ablkcipher to skcipher API Update the sun4i-ss driver to use the skcipher API instead of the old ablkcipher one. It's a bit more tricky than s/ablkcipher/skcipher/, but still nothing special and the driver's logic stays the same. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-cipher.c | 189 +++++++++--------- drivers/crypto/sunxi-ss/sun4i-ss-core.c | 222 +++++++++++----------- drivers/crypto/sunxi-ss/sun4i-ss.h | 34 ++-- 3 files changed, 221 insertions(+), 224 deletions(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c index 23e549204365..5cf64746731a 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -16,13 +16,13 @@ */ #include "sun4i-ss.h" -static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) +static int sun4i_ss_opti_poll(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun4i_ss_ctx *ss = op->ss; - unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); - struct sun4i_cipher_req_ctx *ctx = ablkcipher_request_ctx(areq); + unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); u32 mode = ctx->mode; /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ u32 rx_cnt = SS_RX_DEFAULT; @@ -31,17 +31,17 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) u32 v; int err = 0; unsigned int i; - unsigned int ileft = areq->nbytes; - unsigned int oleft = areq->nbytes; + unsigned int ileft = areq->cryptlen; + unsigned int oleft = areq->cryptlen; unsigned int todo; struct sg_mapping_iter mi, mo; unsigned int oi, oo; /* offset for in and out */ unsigned long flags; - if (!areq->nbytes) + if (!areq->cryptlen) return 0; - if (!areq->info) { + if (!areq->iv) { dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n"); return -EINVAL; } @@ -56,9 +56,9 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) for (i = 0; i < op->keylen; i += 4) writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); - if (areq->info) { + if (areq->iv) { for (i = 0; i < 4 && i < ivsize / 4; i++) { - v = *(u32 *)(areq->info + i * 4); + v = *(u32 *)(areq->iv + i * 4); writel(v, ss->base + SS_IV0 + i * 4); } } @@ -76,8 +76,8 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) goto release_ss; } - ileft = areq->nbytes / 4; - oleft = areq->nbytes / 4; + ileft = areq->cryptlen / 4; + oleft = areq->cryptlen / 4; oi = 0; oo = 0; do { @@ -108,10 +108,10 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) } } while (oleft); - if (areq->info) { + if (areq->iv) { for (i = 0; i < 4 && i < ivsize / 4; i++) { v = readl(ss->base + SS_IV0 + i * 4); - *(u32 *)(areq->info + i * 4) = v; + *(u32 *)(areq->iv + i * 4) = v; } } @@ -124,16 +124,16 @@ release_ss: } /* Generic function that support SG with size not multiple of 4 */ -static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) +static int sun4i_ss_cipher_poll(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun4i_ss_ctx *ss = op->ss; int no_chunk = 1; struct scatterlist *in_sg = areq->src; struct scatterlist *out_sg = areq->dst; - unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); - struct sun4i_cipher_req_ctx *ctx = ablkcipher_request_ctx(areq); + unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); u32 mode = ctx->mode; /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ u32 rx_cnt = SS_RX_DEFAULT; @@ -142,8 +142,8 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) u32 spaces; int err = 0; unsigned int i; - unsigned int ileft = areq->nbytes; - unsigned int oleft = areq->nbytes; + unsigned int ileft = areq->cryptlen; + unsigned int oleft = areq->cryptlen; unsigned int todo; struct sg_mapping_iter mi, mo; unsigned int oi, oo; /* offset for in and out */ @@ -154,10 +154,10 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) unsigned int obl = 0; /* length of data in bufo */ unsigned long flags; - if (!areq->nbytes) + if (!areq->cryptlen) return 0; - if (!areq->info) { + if (!areq->iv) { dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n"); return -EINVAL; } @@ -190,9 +190,9 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) for (i = 0; i < op->keylen; i += 4) writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); - if (areq->info) { + if (areq->iv) { for (i = 0; i < 4 && i < ivsize / 4; i++) { - v = *(u32 *)(areq->info + i * 4); + v = *(u32 *)(areq->iv + i * 4); writel(v, ss->base + SS_IV0 + i * 4); } } @@ -209,8 +209,8 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) err = -EINVAL; goto release_ss; } - ileft = areq->nbytes; - oleft = areq->nbytes; + ileft = areq->cryptlen; + oleft = areq->cryptlen; oi = 0; oo = 0; @@ -257,8 +257,8 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) tx_cnt = SS_TXFIFO_SPACES(spaces); dev_dbg(ss->dev, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u\n", mode, - oi, mi.length, ileft, areq->nbytes, rx_cnt, - oo, mo.length, oleft, areq->nbytes, tx_cnt, ob); + oi, mi.length, ileft, areq->cryptlen, rx_cnt, + oo, mo.length, oleft, areq->cryptlen, tx_cnt, ob); if (!tx_cnt) continue; @@ -300,10 +300,10 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) /* bufo must be fully used here */ } } - if (areq->info) { + if (areq->iv) { for (i = 0; i < 4 && i < ivsize / 4; i++) { v = readl(ss->base + SS_IV0 + i * 4); - *(u32 *)(areq->info + i * 4) = v; + *(u32 *)(areq->iv + i * 4) = v; } } @@ -317,22 +317,22 @@ release_ss: } /* CBC AES */ -int sun4i_ss_cbc_aes_encrypt(struct ablkcipher_request *areq) +int sun4i_ss_cbc_aes_encrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | op->keymode; return sun4i_ss_cipher_poll(areq); } -int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq) +int sun4i_ss_cbc_aes_decrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_DECRYPTION | op->keymode; @@ -340,22 +340,22 @@ int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq) } /* ECB AES */ -int sun4i_ss_ecb_aes_encrypt(struct ablkcipher_request *areq) +int sun4i_ss_ecb_aes_encrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | op->keymode; return sun4i_ss_cipher_poll(areq); } -int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq) +int sun4i_ss_ecb_aes_decrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_DECRYPTION | op->keymode; @@ -363,22 +363,22 @@ int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq) } /* CBC DES */ -int sun4i_ss_cbc_des_encrypt(struct ablkcipher_request *areq) +int sun4i_ss_cbc_des_encrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | op->keymode; return sun4i_ss_cipher_poll(areq); } -int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq) +int sun4i_ss_cbc_des_decrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_DECRYPTION | op->keymode; @@ -386,22 +386,22 @@ int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq) } /* ECB DES */ -int sun4i_ss_ecb_des_encrypt(struct ablkcipher_request *areq) +int sun4i_ss_ecb_des_encrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | op->keymode; return sun4i_ss_cipher_poll(areq); } -int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq) +int sun4i_ss_ecb_des_decrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_DECRYPTION | op->keymode; @@ -409,22 +409,22 @@ int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq) } /* CBC 3DES */ -int sun4i_ss_cbc_des3_encrypt(struct ablkcipher_request *areq) +int sun4i_ss_cbc_des3_encrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | op->keymode; return sun4i_ss_cipher_poll(areq); } -int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq) +int sun4i_ss_cbc_des3_decrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_DECRYPTION | op->keymode; @@ -432,22 +432,22 @@ int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq) } /* ECB 3DES */ -int sun4i_ss_ecb_des3_encrypt(struct ablkcipher_request *areq) +int sun4i_ss_ecb_des3_encrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | op->keymode; return sun4i_ss_cipher_poll(areq); } -int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq) +int sun4i_ss_ecb_des3_decrypt(struct skcipher_request *areq) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); - struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_DECRYPTION | op->keymode; @@ -457,24 +457,25 @@ int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq) int sun4i_ss_cipher_init(struct crypto_tfm *tfm) { struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm); - struct crypto_alg *alg = tfm->__crt_alg; struct sun4i_ss_alg_template *algt; memset(op, 0, sizeof(struct sun4i_tfm_ctx)); - algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + algt = container_of(tfm->__crt_alg, struct sun4i_ss_alg_template, + alg.crypto.base); op->ss = algt->ss; - tfm->crt_ablkcipher.reqsize = sizeof(struct sun4i_cipher_req_ctx); + crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), + sizeof(struct sun4i_cipher_req_ctx)); return 0; } /* check and set the AES key, prepare the mode to be used */ -int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, +int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun4i_ss_ctx *ss = op->ss; switch (keylen) { @@ -489,7 +490,7 @@ int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, break; default: dev_err(ss->dev, "ERROR: Invalid keylen %u\n", keylen); - crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } op->keylen = keylen; @@ -498,10 +499,10 @@ int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, } /* check and set the DES key, prepare the mode to be used */ -int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, +int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun4i_ss_ctx *ss = op->ss; u32 flags; u32 tmp[DES_EXPKEY_WORDS]; @@ -509,15 +510,15 @@ int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, if (unlikely(keylen != DES_KEY_SIZE)) { dev_err(ss->dev, "Invalid keylen %u\n", keylen); - crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } - flags = crypto_ablkcipher_get_flags(tfm); + flags = crypto_skcipher_get_flags(tfm); ret = des_ekey(tmp, key); if (unlikely(!ret) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) { - crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); dev_dbg(ss->dev, "Weak key %u\n", keylen); return -EINVAL; } @@ -528,15 +529,15 @@ int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, } /* check and set the 3DES key, prepare the mode to be used */ -int sun4i_ss_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, +int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { - struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun4i_ss_ctx *ss = op->ss; if (unlikely(keylen != 3 * DES_KEY_SIZE)) { dev_err(ss->dev, "Invalid keylen %u\n", keylen); - crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } op->keylen = keylen; diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c index 50af81c3eb45..f7b99974d1bb 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c @@ -83,134 +83,128 @@ static struct sun4i_ss_alg_template ss_algs[] = { } } }, -{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.crypto = { - .cra_name = "cbc(aes)", - .cra_driver_name = "cbc-aes-sun4i-ss", - .cra_priority = 300, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, - .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), - .cra_module = THIS_MODULE, - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_init = sun4i_ss_cipher_init, - .cra_ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = sun4i_ss_aes_setkey, - .encrypt = sun4i_ss_cbc_aes_encrypt, - .decrypt = sun4i_ss_cbc_aes_decrypt, + .setkey = sun4i_ss_aes_setkey, + .encrypt = sun4i_ss_cbc_aes_encrypt, + .decrypt = sun4i_ss_cbc_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, } } }, -{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.crypto = { - .cra_name = "ecb(aes)", - .cra_driver_name = "ecb-aes-sun4i-ss", - .cra_priority = 300, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, - .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), - .cra_module = THIS_MODULE, - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_init = sun4i_ss_cipher_init, - .cra_ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = sun4i_ss_aes_setkey, - .encrypt = sun4i_ss_ecb_aes_encrypt, - .decrypt = sun4i_ss_ecb_aes_decrypt, + .setkey = sun4i_ss_aes_setkey, + .encrypt = sun4i_ss_ecb_aes_encrypt, + .decrypt = sun4i_ss_ecb_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, } } }, -{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.crypto = { - .cra_name = "cbc(des)", - .cra_driver_name = "cbc-des-sun4i-ss", - .cra_priority = 300, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, - .cra_ctxsize = sizeof(struct sun4i_req_ctx), - .cra_module = THIS_MODULE, - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_init = sun4i_ss_cipher_init, - .cra_u.ablkcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = sun4i_ss_des_setkey, - .encrypt = sun4i_ss_cbc_des_encrypt, - .decrypt = sun4i_ss_cbc_des_decrypt, + .setkey = sun4i_ss_des_setkey, + .encrypt = sun4i_ss_cbc_des_encrypt, + .decrypt = sun4i_ss_cbc_des_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, } } }, -{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.crypto = { - .cra_name = "ecb(des)", - .cra_driver_name = "ecb-des-sun4i-ss", - .cra_priority = 300, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, - .cra_ctxsize = sizeof(struct sun4i_req_ctx), - .cra_module = THIS_MODULE, - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_init = sun4i_ss_cipher_init, - .cra_u.ablkcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = sun4i_ss_des_setkey, - .encrypt = sun4i_ss_ecb_des_encrypt, - .decrypt = sun4i_ss_ecb_des_decrypt, + .setkey = sun4i_ss_des_setkey, + .encrypt = sun4i_ss_ecb_des_encrypt, + .decrypt = sun4i_ss_ecb_des_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "ecb-des-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, } } }, -{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.crypto = { - .cra_name = "cbc(des3_ede)", - .cra_driver_name = "cbc-des3-sun4i-ss", - .cra_priority = 300, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, - .cra_ctxsize = sizeof(struct sun4i_req_ctx), - .cra_module = THIS_MODULE, - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_init = sun4i_ss_cipher_init, - .cra_u.ablkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .setkey = sun4i_ss_des3_setkey, - .encrypt = sun4i_ss_cbc_des3_encrypt, - .decrypt = sun4i_ss_cbc_des3_decrypt, + .setkey = sun4i_ss_des3_setkey, + .encrypt = sun4i_ss_cbc_des3_encrypt, + .decrypt = sun4i_ss_cbc_des3_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-des3-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, } } }, -{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, +{ .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.crypto = { - .cra_name = "ecb(des3_ede)", - .cra_driver_name = "ecb-des3-sun4i-ss", - .cra_priority = 300, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, - .cra_ctxsize = sizeof(struct sun4i_req_ctx), - .cra_module = THIS_MODULE, - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_init = sun4i_ss_cipher_init, - .cra_u.ablkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .setkey = sun4i_ss_des3_setkey, - .encrypt = sun4i_ss_ecb_des3_encrypt, - .decrypt = sun4i_ss_ecb_des3_decrypt, + .setkey = sun4i_ss_des3_setkey, + .encrypt = sun4i_ss_ecb_des3_encrypt, + .decrypt = sun4i_ss_ecb_des3_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-des3-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_init = sun4i_ss_cipher_init, } } }, @@ -340,11 +334,11 @@ static int sun4i_ss_probe(struct platform_device *pdev) for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { ss_algs[i].ss = ss; switch (ss_algs[i].type) { - case CRYPTO_ALG_TYPE_ABLKCIPHER: - err = crypto_register_alg(&ss_algs[i].alg.crypto); + case CRYPTO_ALG_TYPE_SKCIPHER: + err = crypto_register_skcipher(&ss_algs[i].alg.crypto); if (err) { dev_err(ss->dev, "Fail to register %s\n", - ss_algs[i].alg.crypto.cra_name); + ss_algs[i].alg.crypto.base.cra_name); goto error_alg; } break; @@ -364,8 +358,8 @@ error_alg: i--; for (; i >= 0; i--) { switch (ss_algs[i].type) { - case CRYPTO_ALG_TYPE_ABLKCIPHER: - crypto_unregister_alg(&ss_algs[i].alg.crypto); + case CRYPTO_ALG_TYPE_SKCIPHER: + crypto_unregister_skcipher(&ss_algs[i].alg.crypto); break; case CRYPTO_ALG_TYPE_AHASH: crypto_unregister_ahash(&ss_algs[i].alg.hash); @@ -388,8 +382,8 @@ static int sun4i_ss_remove(struct platform_device *pdev) for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { switch (ss_algs[i].type) { - case CRYPTO_ALG_TYPE_ABLKCIPHER: - crypto_unregister_alg(&ss_algs[i].alg.crypto); + case CRYPTO_ALG_TYPE_SKCIPHER: + crypto_unregister_skcipher(&ss_algs[i].alg.crypto); break; case CRYPTO_ALG_TYPE_AHASH: crypto_unregister_ahash(&ss_algs[i].alg.hash); diff --git a/drivers/crypto/sunxi-ss/sun4i-ss.h b/drivers/crypto/sunxi-ss/sun4i-ss.h index f04c0f8cf026..a0e1efc1cb2a 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss.h +++ b/drivers/crypto/sunxi-ss/sun4i-ss.h @@ -24,9 +24,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -140,7 +142,7 @@ struct sun4i_ss_alg_template { u32 type; u32 mode; union { - struct crypto_alg crypto; + struct skcipher_alg crypto; struct ahash_alg hash; } alg; struct sun4i_ss_ctx *ss; @@ -177,25 +179,25 @@ int sun4i_hash_import_md5(struct ahash_request *areq, const void *in); int sun4i_hash_export_sha1(struct ahash_request *areq, void *out); int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in); -int sun4i_ss_cbc_aes_encrypt(struct ablkcipher_request *areq); -int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq); -int sun4i_ss_ecb_aes_encrypt(struct ablkcipher_request *areq); -int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq); +int sun4i_ss_cbc_aes_encrypt(struct skcipher_request *areq); +int sun4i_ss_cbc_aes_decrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_aes_encrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_aes_decrypt(struct skcipher_request *areq); -int sun4i_ss_cbc_des_encrypt(struct ablkcipher_request *areq); -int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq); -int sun4i_ss_ecb_des_encrypt(struct ablkcipher_request *areq); -int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq); +int sun4i_ss_cbc_des_encrypt(struct skcipher_request *areq); +int sun4i_ss_cbc_des_decrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_des_encrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_des_decrypt(struct skcipher_request *areq); -int sun4i_ss_cbc_des3_encrypt(struct ablkcipher_request *areq); -int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq); -int sun4i_ss_ecb_des3_encrypt(struct ablkcipher_request *areq); -int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq); +int sun4i_ss_cbc_des3_encrypt(struct skcipher_request *areq); +int sun4i_ss_cbc_des3_decrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_des3_encrypt(struct skcipher_request *areq); +int sun4i_ss_ecb_des3_decrypt(struct skcipher_request *areq); int sun4i_ss_cipher_init(struct crypto_tfm *tfm); -int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, +int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen); -int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, +int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen); -int sun4i_ss_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, +int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen); From 0d9c68a9bfc7259f847704f28c64549fc4078aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:39:02 +0200 Subject: [PATCH 095/140] crypto: sun4i-ss - add the CRYPTO_ALG_KERN_DRIVER_ONLY flag The CRYPTO_ALG_KERN_DRIVER_ONLY flag is set for hardware accelerated ciphers accessible through a kernel driver only. This is the case for ciphers exposed by the sun4i-ss driver. This patch sets this flag. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-core.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c index f7b99974d1bb..02ad8256e900 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c @@ -96,7 +96,8 @@ static struct sun4i_ss_alg_template ss_algs[] = { .cra_driver_name = "cbc-aes-sun4i-ss", .cra_priority = 300, .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, @@ -117,7 +118,8 @@ static struct sun4i_ss_alg_template ss_algs[] = { .cra_driver_name = "ecb-aes-sun4i-ss", .cra_priority = 300, .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, @@ -138,7 +140,8 @@ static struct sun4i_ss_alg_template ss_algs[] = { .cra_driver_name = "cbc-des-sun4i-ss", .cra_priority = 300, .cra_blocksize = DES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_ctxsize = sizeof(struct sun4i_req_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, @@ -158,7 +161,8 @@ static struct sun4i_ss_alg_template ss_algs[] = { .cra_driver_name = "ecb-des-sun4i-ss", .cra_priority = 300, .cra_blocksize = DES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_ctxsize = sizeof(struct sun4i_req_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, @@ -179,7 +183,8 @@ static struct sun4i_ss_alg_template ss_algs[] = { .cra_driver_name = "cbc-des3-sun4i-ss", .cra_priority = 300, .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_ctxsize = sizeof(struct sun4i_req_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, From 049655499e2ecc4d7d3920da44f117e091a2240b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 1 Jun 2017 21:39:03 +0200 Subject: [PATCH 096/140] crypto: sun4i-ss - fix large block size support The run-time self-tests fail quite early, as soon as the input block size is larger than 64 bytes: alg: hash: Test 4 failed for sha1-sun4i-ss 00000000: b9 c9 1e 52 c0 26 d8 39 81 ff f2 3c 99 b1 27 b2 00000010: 30 d6 c9 85 One thing to notice is the value of the last word, which is the one expected (it can sometime be the last two words). The datasheet isn't very clear about when the digest is ready to retrieve and is seems the bit SS_DATA_END is cleared when the digest was computed *but* that doesn't mean the digest is ready to retrieve in the registers. A ndelay(1) is added before reading the computed digest to ensure it is available in the SS_MD[] registers. Signed-off-by: Antoine Tenart Tested-by: Corentin Labbe Acked-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-hash.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index 685de5b6ab17..a4b5ff2b72f8 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -358,6 +358,15 @@ static int sun4i_hash(struct ahash_request *areq) goto release_ss; } + /* + * The datasheet isn't very clear about when to retrieve the digest. The + * bit SS_DATA_END is cleared when the engine has processed the data and + * when the digest is computed *but* it doesn't mean the digest is + * available in the digest registers. Hence the delay to be sure we can + * read it. + */ + ndelay(1); + for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) op->hash[i] = readl(ss->base + SS_MD0 + i * 4); @@ -446,6 +455,15 @@ hash_final: goto release_ss; } + /* + * The datasheet isn't very clear about when to retrieve the digest. The + * bit SS_DATA_END is cleared when the engine has processed the data and + * when the digest is computed *but* it doesn't mean the digest is + * available in the digest registers. Hence the delay to be sure we can + * read it. + */ + ndelay(1); + /* Get the hash from the device */ if (op->mode == SS_OP_SHA1) { for (i = 0; i < 5; i++) { From b1a4b182c1a529a19c600fa4e9b1c44fa44132bc Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Sun, 4 Jun 2017 19:29:20 +0100 Subject: [PATCH 097/140] crypto: brcm - fix spelling mistake: "fallbck" -> "fallback" Trivial fix to spelling mistake in flow_log message Signed-off-by: Colin Ian King Reviewed-by: Steve Lin Signed-off-by: Herbert Xu --- drivers/crypto/bcm/cipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 61393dc70b0b..9cfd36c1bcb6 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -2639,7 +2639,7 @@ static int aead_need_fallback(struct aead_request *req) (spu->spu_type == SPU_TYPE_SPUM) && (ctx->digestsize != 8) && (ctx->digestsize != 12) && (ctx->digestsize != 16)) { - flow_log("%s() AES CCM needs fallbck for digest size %d\n", + flow_log("%s() AES CCM needs fallback for digest size %d\n", __func__, ctx->digestsize); return 1; } From cf3f9609c9b2f36760f8181d11b8085c357c27ee Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Mon, 5 Jun 2017 08:33:43 +0300 Subject: [PATCH 098/140] crypto: testmgr - use consistent format for errors Fix inconsistent format and spelling in hash tests error messages. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- crypto/testmgr.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 5f8e6838e5a8..127a0f545ec2 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -218,14 +218,14 @@ static int ahash_partial_update(struct ahash_request **preq, crypto_ahash_reqtfm(req)); state = kmalloc(statesize + sizeof(guard), GFP_KERNEL); if (!state) { - pr_err("alt: hash: Failed to alloc state for %s\n", algo); + pr_err("alg: hash: Failed to alloc state for %s\n", algo); goto out_nostate; } memcpy(state + statesize, guard, sizeof(guard)); ret = crypto_ahash_export(req, state); WARN_ON(memcmp(state + statesize, guard, sizeof(guard))); if (ret) { - pr_err("alt: hash: Failed to export() for %s\n", algo); + pr_err("alg: hash: Failed to export() for %s\n", algo); goto out; } ahash_request_free(req); @@ -344,19 +344,19 @@ static int __test_hash(struct crypto_ahash *tfm, } else { ret = wait_async_op(&tresult, crypto_ahash_init(req)); if (ret) { - pr_err("alt: hash: init failed on test %d " + pr_err("alg: hash: init failed on test %d " "for %s: ret=%d\n", j, algo, -ret); goto out; } ret = wait_async_op(&tresult, crypto_ahash_update(req)); if (ret) { - pr_err("alt: hash: update failed on test %d " + pr_err("alg: hash: update failed on test %d " "for %s: ret=%d\n", j, algo, -ret); goto out; } ret = wait_async_op(&tresult, crypto_ahash_final(req)); if (ret) { - pr_err("alt: hash: final failed on test %d " + pr_err("alg: hash: final failed on test %d " "for %s: ret=%d\n", j, algo, -ret); goto out; } @@ -488,13 +488,13 @@ static int __test_hash(struct crypto_ahash *tfm, ahash_request_set_crypt(req, sg, result, template[i].tap[0]); ret = wait_async_op(&tresult, crypto_ahash_init(req)); if (ret) { - pr_err("alt: hash: init failed on test %d for %s: ret=%d\n", + pr_err("alg: hash: init failed on test %d for %s: ret=%d\n", j, algo, -ret); goto out; } ret = wait_async_op(&tresult, crypto_ahash_update(req)); if (ret) { - pr_err("alt: hash: update failed on test %d for %s: ret=%d\n", + pr_err("alg: hash: update failed on test %d for %s: ret=%d\n", j, algo, -ret); goto out; } @@ -505,7 +505,7 @@ static int __test_hash(struct crypto_ahash *tfm, hash_buff, k, temp, &sg[0], algo, result, &tresult); if (ret) { - pr_err("hash: partial update failed on test %d for %s: ret=%d\n", + pr_err("alg: hash: partial update failed on test %d for %s: ret=%d\n", j, algo, -ret); goto out_noreq; } @@ -513,7 +513,7 @@ static int __test_hash(struct crypto_ahash *tfm, } ret = wait_async_op(&tresult, crypto_ahash_final(req)); if (ret) { - pr_err("alt: hash: final failed on test %d for %s: ret=%d\n", + pr_err("alg: hash: final failed on test %d for %s: ret=%d\n", j, algo, -ret); goto out; } From 88d58ef891d868303acd7951cb1282c911f736ac Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 6 Jun 2017 15:44:16 +0200 Subject: [PATCH 099/140] crypto: engine - replace pr_xxx by dev_xxx By adding a struct device *dev to struct engine, we could store the device used at register time and so use all dev_xxx functions instead of pr_xxx. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- crypto/crypto_engine.c | 23 +++++++++++++---------- include/crypto/engine.h | 1 + 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c index 727bd5c3569e..61e7c4e02fd2 100644 --- a/crypto/crypto_engine.c +++ b/crypto/crypto_engine.c @@ -70,7 +70,7 @@ static void crypto_pump_requests(struct crypto_engine *engine, if (engine->unprepare_crypt_hardware && engine->unprepare_crypt_hardware(engine)) - pr_err("failed to unprepare crypt hardware\n"); + dev_err(engine->dev, "failed to unprepare crypt hardware\n"); spin_lock_irqsave(&engine->queue_lock, flags); engine->idling = false; @@ -99,7 +99,7 @@ static void crypto_pump_requests(struct crypto_engine *engine, if (!was_busy && engine->prepare_crypt_hardware) { ret = engine->prepare_crypt_hardware(engine); if (ret) { - pr_err("failed to prepare crypt hardware\n"); + dev_err(engine->dev, "failed to prepare crypt hardware\n"); goto req_err; } } @@ -110,14 +110,15 @@ static void crypto_pump_requests(struct crypto_engine *engine, if (engine->prepare_hash_request) { ret = engine->prepare_hash_request(engine, hreq); if (ret) { - pr_err("failed to prepare request: %d\n", ret); + dev_err(engine->dev, "failed to prepare request: %d\n", + ret); goto req_err; } engine->cur_req_prepared = true; } ret = engine->hash_one_request(engine, hreq); if (ret) { - pr_err("failed to hash one request from queue\n"); + dev_err(engine->dev, "failed to hash one request from queue\n"); goto req_err; } return; @@ -126,19 +127,20 @@ static void crypto_pump_requests(struct crypto_engine *engine, if (engine->prepare_cipher_request) { ret = engine->prepare_cipher_request(engine, breq); if (ret) { - pr_err("failed to prepare request: %d\n", ret); + dev_err(engine->dev, "failed to prepare request: %d\n", + ret); goto req_err; } engine->cur_req_prepared = true; } ret = engine->cipher_one_request(engine, breq); if (ret) { - pr_err("failed to cipher one request from queue\n"); + dev_err(engine->dev, "failed to cipher one request from queue\n"); goto req_err; } return; default: - pr_err("failed to prepare request of unknown type\n"); + dev_err(engine->dev, "failed to prepare request of unknown type\n"); return; } @@ -275,7 +277,7 @@ void crypto_finalize_cipher_request(struct crypto_engine *engine, engine->unprepare_cipher_request) { ret = engine->unprepare_cipher_request(engine, req); if (ret) - pr_err("failed to unprepare request\n"); + dev_err(engine->dev, "failed to unprepare request\n"); } spin_lock_irqsave(&engine->queue_lock, flags); engine->cur_req = NULL; @@ -312,7 +314,7 @@ void crypto_finalize_hash_request(struct crypto_engine *engine, engine->unprepare_hash_request) { ret = engine->unprepare_hash_request(engine, req); if (ret) - pr_err("failed to unprepare request\n"); + dev_err(engine->dev, "failed to unprepare request\n"); } spin_lock_irqsave(&engine->queue_lock, flags); engine->cur_req = NULL; @@ -384,7 +386,7 @@ int crypto_engine_stop(struct crypto_engine *engine) spin_unlock_irqrestore(&engine->queue_lock, flags); if (ret) - pr_warn("could not stop engine\n"); + dev_warn(engine->dev, "could not stop engine\n"); return ret; } @@ -411,6 +413,7 @@ struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) if (!engine) return NULL; + engine->dev = dev; engine->rt = rt; engine->running = false; engine->busy = false; diff --git a/include/crypto/engine.h b/include/crypto/engine.h index 1bf600fc99f7..dd04c1699b51 100644 --- a/include/crypto/engine.h +++ b/include/crypto/engine.h @@ -58,6 +58,7 @@ struct crypto_engine { struct list_head list; spinlock_t queue_lock; struct crypto_queue queue; + struct device *dev; bool rt; From ffe55266e36bef40b940b38bfef4fcff4957624d Mon Sep 17 00:00:00 2001 From: Christophe Jaillet Date: Sat, 10 Jun 2017 14:54:33 +0200 Subject: [PATCH 100/140] crypto: crypto4xx - fix an error code If 'kzalloc' fails, we return 0 which means success. return -ENOMEM instead as already done a few lines above. Signed-off-by: Christophe JAILLET Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index fdc83a2281ca..65dc78b91dea 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1179,6 +1179,7 @@ static int crypto4xx_probe(struct platform_device *ofdev) dev_set_drvdata(dev, core_dev); core_dev->ofdev = ofdev; core_dev->dev = kzalloc(sizeof(struct crypto4xx_device), GFP_KERNEL); + rc = -ENOMEM; if (!core_dev->dev) goto err_alloc_dev; From fec17cb2231733174e039ad9054fa16bb358e2ec Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sun, 11 Jun 2017 23:20:23 +0200 Subject: [PATCH 101/140] crypto: rsa-pkcs1pad - use constant time memory comparison for MACs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise, we enable all sorts of forgeries via timing attack. Signed-off-by: Jason A. Donenfeld Suggested-by: Stephan Müller Cc: stable@vger.kernel.org Cc: Herbert Xu Cc: linux-crypto@vger.kernel.org Signed-off-by: Herbert Xu --- crypto/rsa-pkcs1pad.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c index 044bcfa69cba..407c64bdcdd9 100644 --- a/crypto/rsa-pkcs1pad.c +++ b/crypto/rsa-pkcs1pad.c @@ -490,7 +490,7 @@ static int pkcs1pad_verify_complete(struct akcipher_request *req, int err) goto done; pos++; - if (memcmp(out_buf + pos, digest_info->data, digest_info->size)) + if (crypto_memneq(out_buf + pos, digest_info->data, digest_info->size)) goto done; pos += digest_info->size; From 34194913ea6eae8b7df439d01e85ad20c6d0ab7b Mon Sep 17 00:00:00 2001 From: Sean Wang Date: Mon, 12 Jun 2017 23:56:54 +0800 Subject: [PATCH 102/140] dt-bindings: rng: add MediaTek MT7622 Hardware Random Generator bindings Document the bindings used by MediaTek MT7622 SoC hardware random number generator. Signed-off-by: Sean Wang Reviewed-by: Matthias Brugger Acked-by: Rob Herring Signed-off-by: Herbert Xu --- Documentation/devicetree/bindings/rng/mtk-rng.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/rng/mtk-rng.txt b/Documentation/devicetree/bindings/rng/mtk-rng.txt index a6d62a2abd39..366b99bff8cd 100644 --- a/Documentation/devicetree/bindings/rng/mtk-rng.txt +++ b/Documentation/devicetree/bindings/rng/mtk-rng.txt @@ -2,7 +2,9 @@ Device-Tree bindings for Mediatek random number generator found in Mediatek SoC family Required properties: -- compatible : Should be "mediatek,mt7623-rng" +- compatible : Should be + "mediatek,mt7622-rng", "mediatek,mt7623-rng" : for MT7622 + "mediatek,mt7623-rng" : for MT7623 - clocks : list of clock specifiers, corresponding to entries in clock-names property; - clock-names : Should contain "rng" entries; From 81d2b34508c6a35dd1525ee08e3439dce14e0615 Mon Sep 17 00:00:00 2001 From: Sean Wang Date: Mon, 12 Jun 2017 23:56:55 +0800 Subject: [PATCH 103/140] hwrng: mtk - add runtime PM support Add runtime PM support. There will be the benefit on SoCs where the clock to the RNG used can be shutdown. Signed-off-by: Sean Wang Signed-off-by: Herbert Xu --- drivers/char/hw_random/mtk-rng.c | 42 ++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c index df8eb54fd5a3..8da7bcf54105 100644 --- a/drivers/char/hw_random/mtk-rng.c +++ b/drivers/char/hw_random/mtk-rng.c @@ -25,6 +25,10 @@ #include #include #include +#include + +/* Runtime PM autosuspend timeout: */ +#define RNG_AUTOSUSPEND_TIMEOUT 100 #define USEC_POLL 2 #define TIMEOUT_POLL 20 @@ -90,6 +94,8 @@ static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) struct mtk_rng *priv = to_mtk_rng(rng); int retval = 0; + pm_runtime_get_sync((struct device *)priv->rng.priv); + while (max >= sizeof(u32)) { if (!mtk_rng_wait_ready(rng, wait)) break; @@ -100,6 +106,9 @@ static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) max -= sizeof(u32); } + pm_runtime_mark_last_busy((struct device *)priv->rng.priv); + pm_runtime_put_sync_autosuspend((struct device *)priv->rng.priv); + return retval || !wait ? retval : -EIO; } @@ -120,9 +129,12 @@ static int mtk_rng_probe(struct platform_device *pdev) return -ENOMEM; priv->rng.name = pdev->name; +#ifndef CONFIG_PM priv->rng.init = mtk_rng_init; priv->rng.cleanup = mtk_rng_cleanup; +#endif priv->rng.read = mtk_rng_read; + priv->rng.priv = (unsigned long)&pdev->dev; priv->clk = devm_clk_get(&pdev->dev, "rng"); if (IS_ERR(priv->clk)) { @@ -142,11 +154,40 @@ static int mtk_rng_probe(struct platform_device *pdev) return ret; } + dev_set_drvdata(&pdev->dev, priv); + pm_runtime_set_autosuspend_delay(&pdev->dev, RNG_AUTOSUSPEND_TIMEOUT); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_enable(&pdev->dev); + dev_info(&pdev->dev, "registered RNG driver\n"); return 0; } +#ifdef CONFIG_PM +static int mtk_rng_runtime_suspend(struct device *dev) +{ + struct mtk_rng *priv = dev_get_drvdata(dev); + + mtk_rng_cleanup(&priv->rng); + + return 0; +} + +static int mtk_rng_runtime_resume(struct device *dev) +{ + struct mtk_rng *priv = dev_get_drvdata(dev); + + return mtk_rng_init(&priv->rng); +} + +static UNIVERSAL_DEV_PM_OPS(mtk_rng_pm_ops, mtk_rng_runtime_suspend, + mtk_rng_runtime_resume, NULL); +#define MTK_RNG_PM_OPS (&mtk_rng_pm_ops) +#else /* CONFIG_PM */ +#define MTK_RNG_PM_OPS NULL +#endif /* CONFIG_PM */ + static const struct of_device_id mtk_rng_match[] = { { .compatible = "mediatek,mt7623-rng" }, {}, @@ -157,6 +198,7 @@ static struct platform_driver mtk_rng_driver = { .probe = mtk_rng_probe, .driver = { .name = MTK_RNG_DEV, + .pm = MTK_RNG_PM_OPS, .of_match_table = mtk_rng_match, }, }; From c4a29891161b7fd8666263b78918f051f8703424 Mon Sep 17 00:00:00 2001 From: Sean Wang Date: Mon, 12 Jun 2017 23:56:56 +0800 Subject: [PATCH 104/140] MAINTAINERS: add entry for MediaTek Random Number Generator I work for MediaTek on maintaining the MediaTek RNG driver for the existing SoCs and adding support for the following SoCs. Signed-off-by: Sean Wang Signed-off-by: Herbert Xu --- MAINTAINERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 1f2017604f16..8d64b890669d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8267,6 +8267,11 @@ L: linux-wireless@vger.kernel.org S: Maintained F: drivers/net/wireless/mediatek/mt7601u/ +MEDIATEK RANDOM NUMBER GENERATOR SUPPORT +M: Sean Wang +S: Maintained +F: drivers/char/hw_random/mtk-rng.c + MEGACHIPS STDPXXXX-GE-B850V3-FW LVDS/DP++ BRIDGES M: Peter Senna Tschudin M: Martin Donnelly From 1207107c7168f779643c91e30f8d6c2f8a4afd4d Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Mon, 12 Jun 2017 23:27:51 +0200 Subject: [PATCH 105/140] crypto: testmgr - add testvector for pkcs1pad(rsa) The PKCS#1 RSA implementation is provided with a self test with RSA 2048 and SHA-256. This self test implicitly covers other RSA keys and other hashes. Also, this self test implies that the pkcs1pad(rsa) is FIPS 140-2 compliant. Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/testmgr.c | 33 +++++++++++++++-- crypto/testmgr.h | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 4 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 127a0f545ec2..32087a7acbfb 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -2222,8 +2222,11 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, tcrypt_complete, &result); - /* Run RSA encrypt - c = m^e mod n;*/ - err = wait_async_op(&result, crypto_akcipher_encrypt(req)); + err = wait_async_op(&result, vecs->siggen_sigver_test ? + /* Run asymmetric signature generation */ + crypto_akcipher_sign(req) : + /* Run asymmetric encrypt */ + crypto_akcipher_encrypt(req)); if (err) { pr_err("alg: akcipher: encrypt test failed. err %d\n", err); goto free_all; @@ -2261,8 +2264,11 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, init_completion(&result.completion); akcipher_request_set_crypt(req, &src, &dst, vecs->c_size, out_len_max); - /* Run RSA decrypt - m = c^d mod n;*/ - err = wait_async_op(&result, crypto_akcipher_decrypt(req)); + err = wait_async_op(&result, vecs->siggen_sigver_test ? + /* Run asymmetric signature verification */ + crypto_akcipher_verify(req) : + /* Run asymmetric decrypt */ + crypto_akcipher_decrypt(req)); if (err) { pr_err("alg: akcipher: decrypt test failed. err %d\n", err); goto free_all; @@ -3308,6 +3314,25 @@ static const struct alg_test_desc alg_test_descs[] = { .dec = __VECS(fcrypt_pcbc_dec_tv_template) } } + }, { + .alg = "pkcs1pad(rsa,sha224)", + .test = alg_test_null, + .fips_allowed = 1, + }, { + .alg = "pkcs1pad(rsa,sha256)", + .test = alg_test_akcipher, + .fips_allowed = 1, + .suite = { + .akcipher = __VECS(pkcs1pad_rsa_tv_template) + } + }, { + .alg = "pkcs1pad(rsa,sha384)", + .test = alg_test_null, + .fips_allowed = 1, + }, { + .alg = "pkcs1pad(rsa,sha512)", + .test = alg_test_null, + .fips_allowed = 1, }, { .alg = "poly1305", .test = alg_test_hash, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index db2e26c0c065..6ceb0e2758bb 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -133,6 +133,7 @@ struct akcipher_testvec { unsigned int m_size; unsigned int c_size; bool public_key_vec; + bool siggen_sigver_test; }; struct kpp_testvec { @@ -541,6 +542,101 @@ static const struct akcipher_testvec rsa_tv_template[] = { } }; +/* + * PKCS#1 RSA test vectors. Obtained from CAVS testing. + */ +static const struct akcipher_testvec pkcs1pad_rsa_tv_template[] = { + { + .key = + "\x30\x82\x03\x1f\x02\x01\x10\x02\x82\x01\x01\x00\xd7\x1e\x77\x82" + "\x8c\x92\x31\xe7\x69\x02\xa2\xd5\x5c\x78\xde\xa2\x0c\x8f\xfe\x28" + "\x59\x31\xdf\x40\x9c\x60\x61\x06\xb9\x2f\x62\x40\x80\x76\xcb\x67" + "\x4a\xb5\x59\x56\x69\x17\x07\xfa\xf9\x4c\xbd\x6c\x37\x7a\x46\x7d" + "\x70\xa7\x67\x22\xb3\x4d\x7a\x94\xc3\xba\x4b\x7c\x4b\xa9\x32\x7c" + "\xb7\x38\x95\x45\x64\xa4\x05\xa8\x9f\x12\x7c\x4e\xc6\xc8\x2d\x40" + "\x06\x30\xf4\x60\xa6\x91\xbb\x9b\xca\x04\x79\x11\x13\x75\xf0\xae" + "\xd3\x51\x89\xc5\x74\xb9\xaa\x3f\xb6\x83\xe4\x78\x6b\xcd\xf9\x5c" + "\x4c\x85\xea\x52\x3b\x51\x93\xfc\x14\x6b\x33\x5d\x30\x70\xfa\x50" + "\x1b\x1b\x38\x81\x13\x8d\xf7\xa5\x0c\xc0\x8e\xf9\x63\x52\x18\x4e" + "\xa9\xf9\xf8\x5c\x5d\xcd\x7a\x0d\xd4\x8e\x7b\xee\x91\x7b\xad\x7d" + "\xb4\x92\xd5\xab\x16\x3b\x0a\x8a\xce\x8e\xde\x47\x1a\x17\x01\x86" + "\x7b\xab\x99\xf1\x4b\x0c\x3a\x0d\x82\x47\xc1\x91\x8c\xbb\x2e\x22" + "\x9e\x49\x63\x6e\x02\xc1\xc9\x3a\x9b\xa5\x22\x1b\x07\x95\xd6\x10" + "\x02\x50\xfd\xfd\xd1\x9b\xbe\xab\xc2\xc0\x74\xd7\xec\x00\xfb\x11" + "\x71\xcb\x7a\xdc\x81\x79\x9f\x86\x68\x46\x63\x82\x4d\xb7\xf1\xe6" + "\x16\x6f\x42\x63\xf4\x94\xa0\xca\x33\xcc\x75\x13\x02\x82\x01\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x01" + "\x02\x82\x01\x00\x62\xb5\x60\x31\x4f\x3f\x66\x16\xc1\x60\xac\x47" + "\x2a\xff\x6b\x69\x00\x4a\xb2\x5c\xe1\x50\xb9\x18\x74\xa8\xe4\xdc" + "\xa8\xec\xcd\x30\xbb\xc1\xc6\xe3\xc6\xac\x20\x2a\x3e\x5e\x8b\x12" + "\xe6\x82\x08\x09\x38\x0b\xab\x7c\xb3\xcc\x9c\xce\x97\x67\xdd\xef" + "\x95\x40\x4e\x92\xe2\x44\xe9\x1d\xc1\x14\xfd\xa9\xb1\xdc\x71\x9c" + "\x46\x21\xbd\x58\x88\x6e\x22\x15\x56\xc1\xef\xe0\xc9\x8d\xe5\x80" + "\x3e\xda\x7e\x93\x0f\x52\xf6\xf5\xc1\x91\x90\x9e\x42\x49\x4f\x8d" + "\x9c\xba\x38\x83\xe9\x33\xc2\x50\x4f\xec\xc2\xf0\xa8\xb7\x6e\x28" + "\x25\x56\x6b\x62\x67\xfe\x08\xf1\x56\xe5\x6f\x0e\x99\xf1\xe5\x95" + "\x7b\xef\xeb\x0a\x2c\x92\x97\x57\x23\x33\x36\x07\xdd\xfb\xae\xf1" + "\xb1\xd8\x33\xb7\x96\x71\x42\x36\xc5\xa4\xa9\x19\x4b\x1b\x52\x4c" + "\x50\x69\x91\xf0\x0e\xfa\x80\x37\x4b\xb5\xd0\x2f\xb7\x44\x0d\xd4" + "\xf8\x39\x8d\xab\x71\x67\x59\x05\x88\x3d\xeb\x48\x48\x33\x88\x4e" + "\xfe\xf8\x27\x1b\xd6\x55\x60\x5e\x48\xb7\x6d\x9a\xa8\x37\xf9\x7a" + "\xde\x1b\xcd\x5d\x1a\x30\xd4\xe9\x9e\x5b\x3c\x15\xf8\x9c\x1f\xda" + "\xd1\x86\x48\x55\xce\x83\xee\x8e\x51\xc7\xde\x32\x12\x47\x7d\x46" + "\xb8\x35\xdf\x41\x02\x01\x30\x02\x01\x30\x02\x01\x30\x02\x01\x30" + "\x02\x01\x30", + .key_len = 804, + /* + * m is SHA256 hash of following message: + * "\x49\x41\xbe\x0a\x0c\xc9\xf6\x35\x51\xe4\x27\x56\x13\x71\x4b\xd0" + * "\x36\x92\x84\x89\x1b\xf8\x56\x4a\x72\x61\x14\x69\x4f\x5e\x98\xa5" + * "\x80\x5a\x37\x51\x1f\xd8\xf5\xb5\x63\xfc\xf4\xb1\xbb\x4d\x33\xa3" + * "\x1e\xb9\x75\x8b\x9c\xda\x7e\x6d\x3a\x77\x85\xf7\xfc\x4e\xe7\x64" + * "\x43\x10\x19\xa0\x59\xae\xe0\xad\x4b\xd3\xc4\x45\xf7\xb1\xc2\xc1" + * "\x65\x01\x41\x39\x5b\x45\x47\xed\x2b\x51\xed\xe3\xd0\x09\x10\xd2" + * "\x39\x6c\x4a\x3f\xe5\xd2\x20\xe6\xb0\x71\x7d\x5b\xed\x26\x60\xf1" + * "\xb4\x73\xd1\xdb\x7d\xc4\x19\x91\xee\xf6\x32\x76\xf2\x19\x7d\xb7" + */ + .m = + "\x3e\xc8\xa1\x26\x20\x54\x44\x52\x48\x0d\xe5\x66\xf3\xb3\xf5\x04" + "\xbe\x10\xa8\x48\x94\x22\x2d\xdd\xba\x7a\xb4\x76\x8d\x79\x98\x89", + .m_size = 32, + .c = + "\xc7\xa3\x98\xeb\x43\xd1\x08\xc2\x3d\x78\x45\x04\x70\xc9\x01\xee" + "\xf8\x85\x37\x7c\x0b\xf9\x19\x70\x5c\x45\x7b\x2f\x3a\x0b\xb7\x8b" + "\xc4\x0d\x7b\x3a\x64\x0b\x0f\xdb\x78\xa9\x0b\xfd\x8d\x82\xa4\x86" + "\x39\xbf\x21\xb8\x84\xc4\xce\x9f\xc2\xe8\xb6\x61\x46\x17\xb9\x4e" + "\x0b\x57\x05\xb4\x4f\xf9\x9c\x93\x2d\x9b\xd5\x48\x1d\x80\x12\xef" + "\x3a\x77\x7f\xbc\xb5\x8e\x2b\x6b\x7c\xfc\x9f\x8c\x9d\xa2\xc4\x85" + "\xb0\x87\xe9\x17\x9b\xb6\x23\x62\xd2\xa9\x9f\x57\xe8\xf7\x04\x45" + "\x24\x3a\x45\xeb\xeb\x6a\x08\x8e\xaf\xc8\xa0\x84\xbc\x5d\x13\x38" + "\xf5\x17\x8c\xa3\x96\x9b\xa9\x38\x8d\xf0\x35\xad\x32\x8a\x72\x5b" + "\xdf\x21\xab\x4b\x0e\xa8\x29\xbb\x61\x54\xbf\x05\xdb\x84\x84\xde" + "\xdd\x16\x36\x31\xda\xf3\x42\x6d\x7a\x90\x22\x9b\x11\x29\xa6\xf8" + "\x30\x61\xda\xd3\x8b\x54\x1e\x42\xd1\x47\x1d\x6f\xd1\xcd\x42\x0b" + "\xd1\xe4\x15\x85\x7e\x08\xd6\x59\x64\x4c\x01\x34\x91\x92\x26\xe8" + "\xb0\x25\x8c\xf8\xf4\xfa\x8b\xc9\x31\x33\x76\x72\xfb\x64\x92\x9f" + "\xda\x62\x8d\xe1\x2a\x71\x91\x43\x40\x61\x3c\x5a\xbe\x86\xfc\x5b" + "\xe6\xf9\xa9\x16\x31\x1f\xaf\x25\x6d\xc2\x4a\x23\x6e\x63\x02\xa2", + .c_size = 256, + .siggen_sigver_test = true, + } +}; + static const struct kpp_testvec dh_tv_template[] = { { .secret = From 783a5665ca06a71defc0685d9f86be244b5efb1b Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 13 Jun 2017 09:46:47 +0100 Subject: [PATCH 106/140] crypto: omap-aes - fix spelling mistake "Encryptio" -> "Encryption" Trivial fix to spelling mistake in pr_err message Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes-gcm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c index 521a310ea699..7d4f8a4be6d8 100644 --- a/drivers/crypto/omap-aes-gcm.c +++ b/drivers/crypto/omap-aes-gcm.c @@ -214,7 +214,7 @@ static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) } /* fall through */ default: - pr_err("Encryptio of IV failed for GCM mode"); + pr_err("Encryption of IV failed for GCM mode"); break; } From 7c742df55035ce566b2ca7d4125d1202d41a99f3 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 13 Jun 2017 09:52:54 +0100 Subject: [PATCH 107/140] crypto: cavium - fix spelling mistake "Revsion" -> "Revision" Trivial fix to spelling mistake in seq_printf message Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/nitrox_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index eee6fb501580..ae44a464cd2d 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -399,7 +399,7 @@ static int nitrox_show(struct seq_file *s, void *v) struct nitrox_device *ndev = s->private; seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx); - seq_printf(s, " Revsion ID: 0x%0x\n", ndev->hw.revision_id); + seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id); seq_printf(s, " Cores [AE: %u SE: %u]\n", ndev->hw.ae_cores, ndev->hw.se_cores); seq_printf(s, " Number of Queues: %u\n", ndev->nr_queues); From 2512a62414235be0cf4c2529f468546c57b96f60 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 15 Jun 2017 12:43:39 +0530 Subject: [PATCH 108/140] crypto: chcr - Pass lcb bit setting to firmware GCM and CBC mode of operation requires Last Cipher Block. This patch set lcb bit in WR header when required. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 18 +++++++++++------- drivers/crypto/chelsio/chcr_algo.h | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index f00e0d8bd039..e8ff505c3beb 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -518,7 +518,8 @@ static inline void create_wreq(struct chcr_context *ctx, void *req, struct sk_buff *skb, int kctx_len, int hash_sz, int is_iv, - unsigned int sc_len) + unsigned int sc_len, + unsigned int lcb) { struct uld_ctx *u_ctx = ULD_CTX(ctx); int iv_loc = IV_DSGL; @@ -543,7 +544,8 @@ static inline void create_wreq(struct chcr_context *ctx, chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, - is_iv ? iv_loc : IV_NOP, ctx->tx_qidx); + is_iv ? iv_loc : IV_NOP, !!lcb, + ctx->tx_qidx); chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, qid); @@ -652,7 +654,8 @@ static struct sk_buff write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, req->src, req->nbytes); create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1, - sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl); + sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl, + ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); reqctx->skb = skb; skb_get(skb); return skb; @@ -923,7 +926,7 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, write_sg_to_skb(skb, &frags, req->src, param->sg_len); create_wreq(ctx, chcr_req, req, skb, kctx_len, hash_size_in_response, 0, - DUMMY_BYTES); + DUMMY_BYTES, 0); req_ctx->skb = skb; skb_get(skb); return skb; @@ -1508,7 +1511,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, write_buffer_to_skb(skb, &frags, req->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size); + sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); reqctx->skb = skb; skb_get(skb); @@ -1804,7 +1807,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, skb_set_transport_header(skb, transhdr_len); frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size); + sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); reqctx->skb = skb; skb_get(skb); return skb; @@ -1950,7 +1953,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size); + sizeof(struct cpl_rx_phys_dsgl) + dst_size, + reqctx->verify); reqctx->skb = skb; skb_get(skb); return skb; diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 751d06a58101..9894c7b4d764 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -185,11 +185,11 @@ FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \ FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len))) -#define FILL_WR_RX_Q_ID(cid, qid, wr_iv, fid) \ +#define FILL_WR_RX_Q_ID(cid, qid, wr_iv, lcb, fid) \ htonl( \ FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \ FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \ - FW_CRYPTO_LOOKASIDE_WR_LCB_V(0) | \ + FW_CRYPTO_LOOKASIDE_WR_LCB_V((lcb)) | \ FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)) | \ FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(fid)) From 4dbeae4237c15d3104147ab5f79ecbde9511f49f Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 15 Jun 2017 12:43:40 +0530 Subject: [PATCH 109/140] crypto: chcr - Fix fallback key setting Set key of fallback tfm for rfc4309. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index e8ff505c3beb..14641c66c3eb 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -2210,7 +2210,8 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + int error; if (keylen < 3) { crypto_tfm_set_flags((struct crypto_tfm *)aead, @@ -2218,6 +2219,15 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, aeadctx->enckey_len = 0; return -EINVAL; } + crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & + CRYPTO_TFM_REQ_MASK); + error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); + crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); + crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & + CRYPTO_TFM_RES_MASK); + if (error) + return error; keylen -= 3; memcpy(aeadctx->salt, key + keylen, 3); return chcr_ccm_common_setkey(aead, key, keylen); From 5fe8c7117d78dea231bc7a546a68c1af9fd37cc0 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 15 Jun 2017 12:43:41 +0530 Subject: [PATCH 110/140] crypto: chcr - Return correct error code Return correct error instead of EINVAL. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 76 +++++++++++++++++------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 14641c66c3eb..156065dc4c11 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -1399,7 +1399,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, unsigned short stop_offset = 0; unsigned int assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int err = -EINVAL, src_nent; + int error = -EINVAL, src_nent; int null = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1416,9 +1416,9 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, reqctx->dst = src; if (req->src != req->dst) { - err = chcr_copy_assoc(req, aeadctx); - if (err) - return ERR_PTR(err); + error = chcr_copy_assoc(req, aeadctx); + if (error) + return ERR_PTR(error); reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, req->assoclen); } @@ -1430,6 +1430,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, (op_type ? -authsize : authsize)); if (reqctx->dst_nents < 0) { pr_err("AUTHENC:Invalid Destination sg entries\n"); + error = -EINVAL; goto err; } dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); @@ -1443,8 +1444,10 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); - if (!skb) + if (!skb) { + error = -ENOMEM; goto err; + } /* LLD is going to write the sge hdr. */ skb_reserve(skb, sizeof(struct sge_opaque_hdr)); @@ -1496,9 +1499,9 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, sg_param.nents = reqctx->dst_nents; sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); sg_param.qid = qid; - sg_param.align = 0; - if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst, - &sg_param)) + error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, + reqctx->dst, &sg_param); + if (error) goto dstmap_fail; skb_set_transport_header(skb, transhdr_len); @@ -1520,7 +1523,7 @@ dstmap_fail: /* ivmap_fail: */ kfree_skb(skb); err: - return ERR_PTR(-EINVAL); + return ERR_PTR(error); } static int set_msg_len(u8 *block, unsigned int msglen, int csize) @@ -1730,7 +1733,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, unsigned int dst_size = 0, kctx_len; unsigned int sub_type; unsigned int authsize = crypto_aead_authsize(tfm); - int err = -EINVAL, src_nent; + int error = -EINVAL, src_nent; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1746,10 +1749,10 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, reqctx->dst = src; if (req->src != req->dst) { - err = chcr_copy_assoc(req, aeadctx); - if (err) { + error = chcr_copy_assoc(req, aeadctx); + if (error) { pr_err("AAD copy to destination buffer fails\n"); - return ERR_PTR(err); + return ERR_PTR(error); } reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, req->assoclen); @@ -1758,11 +1761,11 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, (op_type ? -authsize : authsize)); if (reqctx->dst_nents < 0) { pr_err("CCM:Invalid Destination sg entries\n"); + error = -EINVAL; goto err; } - - - if (aead_ccm_validate_input(op_type, req, aeadctx, sub_type)) + error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); + if (error) goto err; dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); @@ -1777,8 +1780,10 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); - if (!skb) + if (!skb) { + error = -ENOMEM; goto err; + } skb_reserve(skb, sizeof(struct sge_opaque_hdr)); @@ -1793,15 +1798,16 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 16), aeadctx->key, aeadctx->enckey_len); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - if (ccm_format_packet(req, aeadctx, sub_type, op_type)) + error = ccm_format_packet(req, aeadctx, sub_type, op_type); + if (error) goto dstmap_fail; sg_param.nents = reqctx->dst_nents; sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); sg_param.qid = qid; - sg_param.align = 0; - if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst, - &sg_param)) + error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, + reqctx->dst, &sg_param); + if (error) goto dstmap_fail; skb_set_transport_header(skb, transhdr_len); @@ -1813,9 +1819,8 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, return skb; dstmap_fail: kfree_skb(skb); - skb = NULL; err: - return ERR_PTR(-EINVAL); + return ERR_PTR(error); } static struct sk_buff *create_gcm_wr(struct aead_request *req, @@ -1839,7 +1844,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, unsigned char tag_offset = 0; unsigned int crypt_len = 0; unsigned int authsize = crypto_aead_authsize(tfm); - int err = -EINVAL, src_nent; + int error = -EINVAL, src_nent; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1856,9 +1861,9 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); reqctx->dst = src; if (req->src != req->dst) { - err = chcr_copy_assoc(req, aeadctx); - if (err) - return ERR_PTR(err); + error = chcr_copy_assoc(req, aeadctx); + if (error) + return ERR_PTR(error); reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, req->assoclen); } @@ -1874,6 +1879,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, (op_type ? -authsize : authsize)); if (reqctx->dst_nents < 0) { pr_err("GCM:Invalid Destination sg entries\n"); + error = -EINVAL; goto err; } @@ -1889,8 +1895,10 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); - if (!skb) + if (!skb) { + error = -ENOMEM; goto err; + } /* NIC driver is going to write the sge hdr. */ skb_reserve(skb, sizeof(struct sge_opaque_hdr)); @@ -1941,9 +1949,9 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, sg_param.nents = reqctx->dst_nents; sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); sg_param.qid = qid; - sg_param.align = 0; - if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst, - &sg_param)) + error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, + reqctx->dst, &sg_param); + if (error) goto dstmap_fail; skb_set_transport_header(skb, transhdr_len); @@ -1962,9 +1970,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, dstmap_fail: /* ivmap_fail: */ kfree_skb(skb); - skb = NULL; err: - return skb; + return ERR_PTR(error); } @@ -1976,7 +1983,8 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm) struct aead_alg *alg = crypto_aead_alg(tfm); aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, - CRYPTO_ALG_NEED_FALLBACK); + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); if (IS_ERR(aeadctx->sw_cipher)) return PTR_ERR(aeadctx->sw_cipher); crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), From d600fc8aae746de71631b52ff40d02cf90dd3b74 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 15 Jun 2017 12:43:42 +0530 Subject: [PATCH 111/140] crypto: chcr - Avoid changing request structure Do not update assoclen received in aead_request. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 37 +++++++++++------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 156065dc4c11..9c839c6de76c 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -126,13 +126,13 @@ static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) fw6_pld = (struct cpl_fw6_pld *)input; if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) || (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) { - cmp = memcmp(&fw6_pld->data[2], (fw6_pld + 1), authsize); + cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize); } else { sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp, authsize, req->assoclen + req->cryptlen - authsize); - cmp = memcmp(temp, (fw6_pld + 1), authsize); + cmp = crypto_memneq(temp, (fw6_pld + 1), authsize); } if (cmp) *err = -EBADMSG; @@ -1840,9 +1840,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, struct scatterlist *src; unsigned int frags = 0, transhdr_len; unsigned int ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len; + unsigned int dst_size = 0, kctx_len, assoclen = req->assoclen; unsigned char tag_offset = 0; - unsigned int crypt_len = 0; unsigned int authsize = crypto_aead_authsize(tfm); int error = -EINVAL, src_nent; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : @@ -1854,27 +1853,21 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) goto err; - src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); + src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen); if (src_nent < 0) goto err; - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); + src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen); reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) return ERR_PTR(error); reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); + assoclen); } - if (!req->cryptlen) - /* null-payload is not supported in the hardware. - * software is sending block size - */ - crypt_len = AES_BLOCK_SIZE; - else - crypt_len = req->cryptlen; + reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); if (reqctx->dst_nents < 0) { @@ -1907,19 +1900,19 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, memset(chcr_req, 0, transhdr_len); if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) - req->assoclen -= 8; + assoclen = req->assoclen - 8; tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( ctx->dev->rx_channel_id, 2, (ivsize ? - (req->assoclen + 1) : 0)); + (assoclen + 1) : 0)); chcr_req->sec_cpl.pldlen = - htonl(req->assoclen + ivsize + req->cryptlen); + htonl(assoclen + ivsize + req->cryptlen); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( - req->assoclen ? 1 : 0, req->assoclen, - req->assoclen + ivsize + 1, 0); + assoclen ? 1 : 0, assoclen, + assoclen + ivsize + 1, 0); chcr_req->sec_cpl.cipherstop_lo_authinsert = - FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + ivsize + 1, + FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1, tag_offset, tag_offset); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == @@ -1955,9 +1948,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, goto dstmap_fail; skb_set_transport_header(skb, transhdr_len); - - write_sg_to_skb(skb, &frags, req->src, req->assoclen); - + write_sg_to_skb(skb, &frags, req->src, assoclen); write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1, From b8fd1f4170e7e8bda45d7bcc750e909c859ec714 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 15 Jun 2017 12:43:43 +0530 Subject: [PATCH 112/140] crypto: chcr - Add ctr mode and process large sg entries for cipher It send multiple WRs to H/W to handle large sg lists. Adds ctr(aes) and rfc(ctr(aes)) modes. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 786 ++++++++++++++++++++++----- drivers/crypto/chelsio/chcr_algo.h | 26 +- drivers/crypto/chelsio/chcr_core.c | 1 - drivers/crypto/chelsio/chcr_core.h | 3 + drivers/crypto/chelsio/chcr_crypto.h | 19 +- 5 files changed, 690 insertions(+), 145 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 9c839c6de76c..03b817f185dd 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -55,6 +55,8 @@ #include #include #include +#include +#include #include #include #include @@ -151,12 +153,11 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, struct chcr_context *ctx = crypto_tfm_ctx(tfm); struct uld_ctx *u_ctx = ULD_CTX(ctx); struct chcr_req_ctx ctx_req; - struct cpl_fw6_pld *fw6_pld; unsigned int digestsize, updated_digestsize; switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_AEAD: - ctx_req.req.aead_req = (struct aead_request *)req; + ctx_req.req.aead_req = aead_request_cast(req); ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req); dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst, ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE); @@ -169,27 +170,16 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, &err); ctx_req.ctx.reqctx->verify = VERIFY_HW; } + ctx_req.req.aead_req->base.complete(req, err); break; case CRYPTO_ALG_TYPE_ABLKCIPHER: - ctx_req.req.ablk_req = (struct ablkcipher_request *)req; - ctx_req.ctx.ablk_ctx = - ablkcipher_request_ctx(ctx_req.req.ablk_req); - if (!err) { - fw6_pld = (struct cpl_fw6_pld *)input; - memcpy(ctx_req.req.ablk_req->info, &fw6_pld->data[2], - AES_BLOCK_SIZE); - } - dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.ablk_req->dst, - ctx_req.ctx.ablk_ctx->dst_nents, DMA_FROM_DEVICE); - if (ctx_req.ctx.ablk_ctx->skb) { - kfree_skb(ctx_req.ctx.ablk_ctx->skb); - ctx_req.ctx.ablk_ctx->skb = NULL; - } + err = chcr_handle_cipher_resp(ablkcipher_request_cast(req), + input, err); break; case CRYPTO_ALG_TYPE_AHASH: - ctx_req.req.ahash_req = (struct ahash_request *)req; + ctx_req.req.ahash_req = ahash_request_cast(req); ctx_req.ctx.ahash_ctx = ahash_request_ctx(ctx_req.req.ahash_req); digestsize = @@ -214,6 +204,7 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, sizeof(struct cpl_fw6_pld), updated_digestsize); } + ctx_req.req.ahash_req->base.complete(req, err); break; } return err; @@ -392,7 +383,7 @@ static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, struct phys_sge_parm *sg_param) { struct phys_sge_pairs *to; - int out_buf_size = sg_param->obsize; + unsigned int len = 0, left_size = sg_param->obsize; unsigned int nents = sg_param->nents, i, j = 0; phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) @@ -409,20 +400,15 @@ static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, phys_cpl->rss_hdr_int.hash_val = 0; to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + sizeof(struct cpl_rx_phys_dsgl)); - - for (i = 0; nents; to++) { - for (j = 0; j < 8 && nents; j++, nents--) { - out_buf_size -= sg_dma_len(sg); - to->len[j] = htons(sg_dma_len(sg)); + for (i = 0; nents && left_size; to++) { + for (j = 0; j < 8 && nents && left_size; j++, nents--) { + len = min(left_size, sg_dma_len(sg)); + to->len[j] = htons(len); to->addr[j] = cpu_to_be64(sg_dma_address(sg)); + left_size -= len; sg = sg_next(sg); } } - if (out_buf_size) { - j--; - to--; - to->len[j] = htons(ntohs(to->len[j]) + (out_buf_size)); - } } static inline int map_writesg_phys_cpl(struct device *dev, @@ -431,7 +417,7 @@ static inline int map_writesg_phys_cpl(struct device *dev, struct phys_sge_parm *sg_param) { if (!sg || !sg_param->nents) - return 0; + return -EINVAL; sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE); if (sg_param->nents == 0) { @@ -498,6 +484,24 @@ write_sg_to_skb(struct sk_buff *skb, unsigned int *frags, } } +static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) +{ + struct adapter *adap = netdev2adap(dev); + struct sge_uld_txq_info *txq_info = + adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; + struct sge_uld_txq *txq; + int ret = 0; + + local_bh_disable(); + txq = &txq_info->uldtxq[idx]; + spin_lock(&txq->sendq.lock); + if (txq->full) + ret = -1; + spin_unlock(&txq->sendq.lock); + local_bh_enable(); + return ret; +} + static int generate_copy_rrkey(struct ablk_ctx *ablkctx, struct _key_ctx *key_ctx) { @@ -512,7 +516,60 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx, } return 0; } +static int chcr_sg_ent_in_wr(struct scatterlist *src, + struct scatterlist *dst, + unsigned int minsg, + unsigned int space, + short int *sent, + short int *dent) +{ + int srclen = 0, dstlen = 0; + int srcsg = minsg, dstsg = 0; + *sent = 0; + *dent = 0; + while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) && + space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { + srclen += src->length; + srcsg++; + while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && + space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { + if (srclen <= dstlen) + break; + dstlen += dst->length; + dst = sg_next(dst); + dstsg++; + } + src = sg_next(src); + } + *sent = srcsg - minsg; + *dent = dstsg; + return min(srclen, dstlen); +} + +static int chcr_cipher_fallback(struct crypto_skcipher *cipher, + u32 flags, + struct scatterlist *src, + struct scatterlist *dst, + unsigned int nbytes, + u8 *iv, + unsigned short op_type) +{ + int err; + + SKCIPHER_REQUEST_ON_STACK(subreq, cipher); + skcipher_request_set_tfm(subreq, cipher); + skcipher_request_set_callback(subreq, flags, NULL, NULL); + skcipher_request_set_crypt(subreq, src, dst, + nbytes, iv); + + err = op_type ? crypto_skcipher_decrypt(subreq) : + crypto_skcipher_encrypt(subreq); + skcipher_request_zero(subreq); + + return err; + +} static inline void create_wreq(struct chcr_context *ctx, struct chcr_wr *chcr_req, void *req, struct sk_buff *skb, @@ -565,69 +622,61 @@ static inline void create_wreq(struct chcr_context *ctx, * @qid: ingress qid where response of this WR should be received. * @op_type: encryption or decryption */ -static struct sk_buff -*create_cipher_wr(struct ablkcipher_request *req, - unsigned short qid, - unsigned short op_type) +static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req); struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); struct uld_ctx *u_ctx = ULD_CTX(ctx); struct ablk_ctx *ablkctx = ABLK_CTX(ctx); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + struct chcr_blkcipher_req_ctx *reqctx = + ablkcipher_request_ctx(wrparam->req); struct phys_sge_parm sg_param; unsigned int frags = 0, transhdr_len, phys_dsgl; - unsigned int ivsize = crypto_ablkcipher_ivsize(tfm), kctx_len; - gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : - GFP_ATOMIC; - - if (!req->info) - return ERR_PTR(-EINVAL); - reqctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes); - if (reqctx->dst_nents <= 0) { - pr_err("AES:Invalid Destination sg lists\n"); - return ERR_PTR(-EINVAL); - } - if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || - (req->nbytes <= 0) || (req->nbytes % AES_BLOCK_SIZE)) { - pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", - ablkctx->enckey_len, req->nbytes, ivsize); - return ERR_PTR(-EINVAL); - } + int error; + unsigned int ivsize = AES_BLOCK_SIZE, kctx_len; + gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents); kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); - if (!skb) - return ERR_PTR(-ENOMEM); + if (!skb) { + error = -ENOMEM; + goto err; + } skb_reserve(skb, sizeof(struct sge_opaque_hdr)); chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len); memset(chcr_req, 0, transhdr_len); chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1); - chcr_req->sec_cpl.pldlen = htonl(ivsize + req->nbytes); + chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0); chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); - chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 0, + chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, ablkctx->ciph_mode, 0, 0, ivsize >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, 0, 1, phys_dsgl); chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; - if (op_type == CHCR_DECRYPT_OP) { + if ((reqctx->op == CHCR_DECRYPT_OP) && + (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == + CRYPTO_ALG_SUB_TYPE_CTR)) && + (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == + CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) { generate_copy_rrkey(ablkctx, &chcr_req->key_ctx); } else { - if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { + if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) || + (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) { memcpy(chcr_req->key_ctx.key, ablkctx->key, ablkctx->enckey_len); } else { @@ -642,18 +691,17 @@ static struct sk_buff } phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->nbytes; - sg_param.qid = qid; - sg_param.align = 1; - if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, req->dst, - &sg_param)) + sg_param.obsize = wrparam->bytes; + sg_param.qid = wrparam->qid; + error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, + reqctx->dst, &sg_param); + if (error) goto map_fail1; skb_set_transport_header(skb, transhdr_len); - memcpy(reqctx->iv, req->info, ivsize); write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - write_sg_to_skb(skb, &frags, req->src, req->nbytes); - create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1, + write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes); + create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1, sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl, ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); reqctx->skb = skb; @@ -661,27 +709,61 @@ static struct sk_buff return skb; map_fail1: kfree_skb(skb); - return ERR_PTR(-ENOMEM); +err: + return ERR_PTR(error); } -static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key, +static inline int chcr_keyctx_ck_size(unsigned int keylen) +{ + int ck_size = 0; + + if (keylen == AES_KEYSIZE_128) + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + else if (keylen == AES_KEYSIZE_192) + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; + else if (keylen == AES_KEYSIZE_256) + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; + else + ck_size = 0; + + return ck_size; +} +static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + int err = 0; + + crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen); + tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm->crt_flags |= + crypto_skcipher_get_flags(ablkctx->sw_cipher) & + CRYPTO_TFM_RES_MASK; + return err; +} + +static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); + struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); struct ablk_ctx *ablkctx = ABLK_CTX(ctx); unsigned int ck_size, context_size; u16 alignment = 0; + int err; - if (keylen == AES_KEYSIZE_128) { - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; - } else if (keylen == AES_KEYSIZE_192) { - alignment = 8; - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; - } else if (keylen == AES_KEYSIZE_256) { - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; - } else { + err = chcr_cipher_fallback_setkey(cipher, key, keylen); + if (err) goto badkey_err; - } + + ck_size = chcr_keyctx_ck_size(keylen); + alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0; memcpy(ablkctx->key, key, keylen); ablkctx->enckey_len = keylen; get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3); @@ -693,35 +775,387 @@ static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key, ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; return 0; badkey_err: - crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); ablkctx->enckey_len = 0; - return -EINVAL; + + return err; } -static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) +static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, + unsigned int keylen) { - struct adapter *adap = netdev2adap(dev); - struct sge_uld_txq_info *txq_info = - adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; - struct sge_uld_txq *txq; + struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + unsigned int ck_size, context_size; + u16 alignment = 0; + int err; + + err = chcr_cipher_fallback_setkey(cipher, key, keylen); + if (err) + goto badkey_err; + ck_size = chcr_keyctx_ck_size(keylen); + alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; + memcpy(ablkctx->key, key, keylen); + ablkctx->enckey_len = keylen; + context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + + keylen + alignment) >> 4; + + ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, + 0, 0, context_size); + ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; + + return 0; +badkey_err: + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + ablkctx->enckey_len = 0; + + return err; +} + +static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, + unsigned int keylen) +{ + struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + unsigned int ck_size, context_size; + u16 alignment = 0; + int err; + + if (keylen < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE), + CTR_RFC3686_NONCE_SIZE); + + keylen -= CTR_RFC3686_NONCE_SIZE; + err = chcr_cipher_fallback_setkey(cipher, key, keylen); + if (err) + goto badkey_err; + + ck_size = chcr_keyctx_ck_size(keylen); + alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; + memcpy(ablkctx->key, key, keylen); + ablkctx->enckey_len = keylen; + context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + + keylen + alignment) >> 4; + + ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, + 0, 0, context_size); + ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; + + return 0; +badkey_err: + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + ablkctx->enckey_len = 0; + + return err; +} +static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add) +{ + unsigned int size = AES_BLOCK_SIZE; + __be32 *b = (__be32 *)(dstiv + size); + u32 c, prev; + + memcpy(dstiv, srciv, AES_BLOCK_SIZE); + for (; size >= 4; size -= 4) { + prev = be32_to_cpu(*--b); + c = prev + add; + *b = cpu_to_be32(c); + if (prev < c) + break; + add = 1; + } + +} + +static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) +{ + __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE); + u64 c; + u32 temp = be32_to_cpu(*--b); + + temp = ~temp; + c = (u64)temp + 1; // No of block can processed withou overflow + if ((bytes / AES_BLOCK_SIZE) > c) + bytes = c * AES_BLOCK_SIZE; + return bytes; +} + +static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + struct crypto_cipher *cipher; + int ret, i; + u8 *key; + unsigned int keylen; + + cipher = crypto_alloc_cipher("aes-generic", 0, 0); + memcpy(iv, req->info, AES_BLOCK_SIZE); + + if (IS_ERR(cipher)) { + ret = -ENOMEM; + goto out; + } + keylen = ablkctx->enckey_len / 2; + key = ablkctx->key + keylen; + ret = crypto_cipher_setkey(cipher, key, keylen); + if (ret) + goto out1; + + crypto_cipher_encrypt_one(cipher, iv, iv); + for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++) + gf128mul_x_ble((le128 *)iv, (le128 *)iv); + + crypto_cipher_decrypt_one(cipher, iv, iv); +out1: + crypto_free_cipher(cipher); +out: + return ret; +} + +static int chcr_update_cipher_iv(struct ablkcipher_request *req, + struct cpl_fw6_pld *fw6_pld, u8 *iv) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)); int ret = 0; - local_bh_disable(); - txq = &txq_info->uldtxq[idx]; - spin_lock(&txq->sendq.lock); - if (txq->full) - ret = -1; - spin_unlock(&txq->sendq.lock); - local_bh_enable(); + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) + ctr_add_iv(iv, req->info, (reqctx->processed / + AES_BLOCK_SIZE)); + else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) + *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + + CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed / + AES_BLOCK_SIZE) + 1); + else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) + ret = chcr_update_tweak(req, iv); + else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { + if (reqctx->op) + sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv, + 16, + reqctx->processed - AES_BLOCK_SIZE); + else + memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); + } + return ret; + +} + +/* We need separate function for final iv because in rfc3686 Initial counter + * starts from 1 and buffer size of iv is 8 byte only which remains constant + * for subsequent update requests + */ + +static int chcr_final_cipher_iv(struct ablkcipher_request *req, + struct cpl_fw6_pld *fw6_pld, u8 *iv) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)); + int ret = 0; + + if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) + ctr_add_iv(iv, req->info, (reqctx->processed / + AES_BLOCK_SIZE)); + else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) + ret = chcr_update_tweak(req, iv); + else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { + if (reqctx->op) + sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv, + 16, + reqctx->processed - AES_BLOCK_SIZE); + else + memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); + + } + return ret; + +} + + +static int chcr_handle_cipher_resp(struct ablkcipher_request *req, + unsigned char *input, int err) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); + struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct sk_buff *skb; + struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + struct cipher_wr_param wrparam; + int bytes; + + dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents, + DMA_FROM_DEVICE); + + if (reqctx->skb) { + kfree_skb(reqctx->skb); + reqctx->skb = NULL; + } + if (err) + goto complete; + + if (req->nbytes == reqctx->processed) { + err = chcr_final_cipher_iv(req, fw6_pld, req->info); + goto complete; + } + + if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], + ctx->tx_qidx))) { + if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + err = -EBUSY; + goto complete; + } + + } + wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src, + reqctx->processed); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg, + reqctx->processed); + if (!wrparam.srcsg || !reqctx->dst) { + pr_err("Input sg list length less that nbytes\n"); + err = -EINVAL; + goto complete; + } + bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1, + SPACE_LEFT(ablkctx->enckey_len), + &wrparam.snent, &reqctx->dst_nents); + if ((bytes + reqctx->processed) >= req->nbytes) + bytes = req->nbytes - reqctx->processed; + else + bytes = ROUND_16(bytes); + err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); + if (err) + goto complete; + + if (unlikely(bytes == 0)) { + err = chcr_cipher_fallback(ablkctx->sw_cipher, + req->base.flags, + wrparam.srcsg, + reqctx->dst, + req->nbytes - reqctx->processed, + reqctx->iv, + reqctx->op); + goto complete; + } + + if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == + CRYPTO_ALG_SUB_TYPE_CTR) + bytes = adjust_ctr_overflow(reqctx->iv, bytes); + reqctx->processed += bytes; + wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; + wrparam.req = req; + wrparam.bytes = bytes; + skb = create_cipher_wr(&wrparam); + if (IS_ERR(skb)) { + pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); + err = PTR_ERR(skb); + goto complete; + } + skb->dev = u_ctx->lldi.ports[0]; + set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + chcr_send_wr(skb); + return 0; +complete: + req->base.complete(&req->base, err); + return err; +} + +static int process_cipher(struct ablkcipher_request *req, + unsigned short qid, + struct sk_buff **skb, + unsigned short op_type) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct cipher_wr_param wrparam; + int bytes, err = -EINVAL; + + reqctx->newdstsg = NULL; + reqctx->processed = 0; + if (!req->info) + goto error; + if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || + (req->nbytes == 0) || + (req->nbytes % crypto_ablkcipher_blocksize(tfm))) { + pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", + ablkctx->enckey_len, req->nbytes, ivsize); + goto error; + } + wrparam.srcsg = req->src; + reqctx->dstsg = req->dst; + bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG, + SPACE_LEFT(ablkctx->enckey_len), + &wrparam.snent, + &reqctx->dst_nents); + if ((bytes + reqctx->processed) >= req->nbytes) + bytes = req->nbytes - reqctx->processed; + else + bytes = ROUND_16(bytes); + if (unlikely(bytes > req->nbytes)) + bytes = req->nbytes; + if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == + CRYPTO_ALG_SUB_TYPE_CTR) { + bytes = adjust_ctr_overflow(req->info, bytes); + } + if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == + CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) { + memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE); + memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info, + CTR_RFC3686_IV_SIZE); + + /* initialize counter portion of counter block */ + *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + + CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); + + } else { + + memcpy(reqctx->iv, req->info, ivsize); + } + if (unlikely(bytes == 0)) { + err = chcr_cipher_fallback(ablkctx->sw_cipher, + req->base.flags, + req->src, + req->dst, + req->nbytes, + req->info, + op_type); + goto error; + } + reqctx->processed = bytes; + reqctx->dst = reqctx->dstsg; + reqctx->op = op_type; + wrparam.qid = qid; + wrparam.req = req; + wrparam.bytes = bytes; + *skb = create_cipher_wr(&wrparam); + if (IS_ERR(*skb)) { + err = PTR_ERR(*skb); + goto error; + } + + return 0; +error: + return err; } static int chcr_aes_encrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); + struct sk_buff *skb = NULL; + int err; struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct sk_buff *skb; if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], ctx->tx_qidx))) { @@ -729,12 +1163,10 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req) return -EBUSY; } - skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], + err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, CHCR_ENCRYPT_OP); - if (IS_ERR(skb)) { - pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); - return PTR_ERR(skb); - } + if (err || !skb) + return err; skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); chcr_send_wr(skb); @@ -746,7 +1178,8 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req) struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct sk_buff *skb; + struct sk_buff *skb = NULL; + int err; if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], ctx->tx_qidx))) { @@ -754,12 +1187,10 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req) return -EBUSY; } - skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], + err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, CHCR_DECRYPT_OP); - if (IS_ERR(skb)) { - pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); - return PTR_ERR(skb); - } + if (err || !skb) + return err; skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); chcr_send_wr(skb); @@ -804,10 +1235,48 @@ out: static int chcr_cra_init(struct crypto_tfm *tfm) { + struct crypto_alg *alg = tfm->__crt_alg; + struct chcr_context *ctx = crypto_tfm_ctx(tfm); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + + ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0, + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ablkctx->sw_cipher)) { + pr_err("failed to allocate fallback for %s\n", alg->cra_name); + return PTR_ERR(ablkctx->sw_cipher); + } tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); return chcr_device_init(crypto_tfm_ctx(tfm)); } +static int chcr_rfc3686_init(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct chcr_context *ctx = crypto_tfm_ctx(tfm); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + + /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes)) + * cannot be used as fallback in chcr_handle_cipher_response + */ + ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0, + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ablkctx->sw_cipher)) { + pr_err("failed to allocate fallback for %s\n", alg->cra_name); + return PTR_ERR(ablkctx->sw_cipher); + } + tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); + return chcr_device_init(crypto_tfm_ctx(tfm)); +} + + +static void chcr_cra_exit(struct crypto_tfm *tfm) +{ + struct chcr_context *ctx = crypto_tfm_ctx(tfm); + struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + + crypto_free_skcipher(ablkctx->sw_cipher); +} + static int get_alg_config(struct algo_param *params, unsigned int auth_size) { @@ -925,8 +1394,8 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, if (param->sg_len != 0) write_sg_to_skb(skb, &frags, req->src, param->sg_len); - create_wreq(ctx, chcr_req, req, skb, kctx_len, hash_size_in_response, 0, - DUMMY_BYTES, 0); + create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, + hash_size_in_response, 0, DUMMY_BYTES, 0); req_ctx->skb = skb; skb_get(skb); return skb; @@ -1229,21 +1698,17 @@ out: return err; } -static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, +static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int key_len) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); + struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); struct ablk_ctx *ablkctx = ABLK_CTX(ctx); unsigned short context_size = 0; + int err; - if ((key_len != (AES_KEYSIZE_128 << 1)) && - (key_len != (AES_KEYSIZE_256 << 1))) { - crypto_tfm_set_flags((struct crypto_tfm *)tfm, - CRYPTO_TFM_RES_BAD_KEY_LEN); - ablkctx->enckey_len = 0; - return -EINVAL; - - } + err = chcr_cipher_fallback_setkey(cipher, key, key_len); + if (err) + goto badkey_err; memcpy(ablkctx->key, key, key_len); ablkctx->enckey_len = key_len; @@ -1257,6 +1722,11 @@ static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 0, context_size); ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; return 0; +badkey_err: + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + ablkctx->enckey_len = 0; + + return err; } static int chcr_sha_init(struct ahash_request *areq) @@ -1513,7 +1983,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, } write_buffer_to_skb(skb, &frags, req->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); - create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1, + create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); reqctx->skb = skb; skb_get(skb); @@ -1812,7 +2282,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, skb_set_transport_header(skb, transhdr_len); frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); - create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1, + create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1, sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); reqctx->skb = skb; skb_get(skb); @@ -1951,7 +2421,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, write_sg_to_skb(skb, &frags, req->src, assoclen); write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); - create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1, + create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, sizeof(struct cpl_rx_phys_dsgl) + dst_size, reqctx->verify); reqctx->skb = skb; @@ -2565,22 +3035,14 @@ static int chcr_aead_op(struct aead_request *req, static struct chcr_alg_template driver_algs[] = { /* AES-CBC */ { - .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC, .is_registered = 0, .alg.crypto = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-chcr", - .cra_priority = CHCR_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct chcr_context) - + sizeof(struct ablk_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, .cra_init = chcr_cra_init, - .cra_exit = NULL, + .cra_exit = chcr_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -2592,24 +3054,15 @@ static struct chcr_alg_template driver_algs[] = { } }, { - .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS, .is_registered = 0, .alg.crypto = { .cra_name = "xts(aes)", .cra_driver_name = "xts-aes-chcr", - .cra_priority = CHCR_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct chcr_context) + - sizeof(struct ablk_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, .cra_init = chcr_cra_init, .cra_exit = NULL, - .cra_u = { - .ablkcipher = { + .cra_u .ablkcipher = { .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, @@ -2618,6 +3071,47 @@ static struct chcr_alg_template driver_algs[] = { .decrypt = chcr_aes_decrypt, } } + }, + { + .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR, + .is_registered = 0, + .alg.crypto = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-chcr", + .cra_blocksize = 1, + .cra_init = chcr_cra_init, + .cra_exit = chcr_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = chcr_aes_ctr_setkey, + .encrypt = chcr_aes_encrypt, + .decrypt = chcr_aes_decrypt, + } + } + }, + { + .type = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_SUB_TYPE_CTR_RFC3686, + .is_registered = 0, + .alg.crypto = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "rfc3686-ctr-aes-chcr", + .cra_blocksize = 1, + .cra_init = chcr_rfc3686_init, + .cra_exit = chcr_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .setkey = chcr_aes_rfc3686_setkey, + .encrypt = chcr_aes_encrypt, + .decrypt = chcr_aes_decrypt, + .geniv = "seqiv", + } } }, /* SHA */ @@ -2999,6 +3493,18 @@ static int chcr_register_alg(void) continue; switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_ABLKCIPHER: + driver_algs[i].alg.crypto.cra_priority = + CHCR_CRA_PRIORITY; + driver_algs[i].alg.crypto.cra_module = THIS_MODULE; + driver_algs[i].alg.crypto.cra_flags = + CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK; + driver_algs[i].alg.crypto.cra_ctxsize = + sizeof(struct chcr_context) + + sizeof(struct ablk_ctx); + driver_algs[i].alg.crypto.cra_alignmask = 0; + driver_algs[i].alg.crypto.cra_type = + &crypto_ablkcipher_type; err = crypto_register_alg(&driver_algs[i].alg.crypto); name = driver_algs[i].alg.crypto.cra_driver_name; break; diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 9894c7b4d764..583008de51a3 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -219,9 +219,26 @@ #define MAX_NK 8 #define CRYPTO_MAX_IMM_TX_PKT_LEN 256 #define MAX_WR_SIZE 512 +#define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0) +#define MAX_DSGL_ENT 32 +#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 2) +#define MIN_CIPHER_SG 1 /* IV */ #define MIN_AUTH_SG 2 /*IV + AAD*/ #define MIN_GCM_SG 2 /* IV + AAD*/ +#define MIN_DIGEST_SG 1 /*Partial Buffer*/ #define MIN_CCM_SG 3 /*IV+AAD+B0*/ +#define SPACE_LEFT(len) \ + ((MAX_WR_SIZE - WR_MIN_LEN - (len))) + +unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, + 48, 64, 72, 88, + 96, 112, 120, 136, + 144, 160, 168, 184, + 192}; +unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, + 112, 112, 128, 128, 144, 144, 160, 160, + 192, 192, 208, 208, 224, 224, 240, 240, + 272, 272, 288, 288, 304, 304, 320, 320}; struct algo_param { unsigned int auth_mode; @@ -239,6 +256,14 @@ struct hash_wr_param { u64 scmd1; }; +struct cipher_wr_param { + struct ablkcipher_request *req; + struct scatterlist *srcsg; + char *iv; + int bytes; + short int snent; + unsigned short qid; +}; enum { AES_KEYLENGTH_128BIT = 128, AES_KEYLENGTH_192BIT = 192, @@ -293,7 +318,6 @@ struct phys_sge_parm { unsigned int nents; unsigned int obsize; unsigned short qid; - unsigned char align; }; struct crypto_result { diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index c28e018e0773..a6c938a913c7 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -115,7 +115,6 @@ static int cpl_fw6_pld_handler(struct chcr_dev *dev, /* call completion callback with failure status */ if (req) { error_status = chcr_handle_resp(req, input, error_status); - req->complete(req, error_status); } else { pr_err("Incorrect request address from the firmware\n"); return -EFAULT; diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index cd0c35a18d92..ddfb2c934551 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -53,6 +53,9 @@ #define MAC_ERROR_BIT 0 #define CHK_MAC_ERR_BIT(x) (((x) >> MAC_ERROR_BIT) & 1) #define MAX_SALT 4 +#define WR_MIN_LEN (sizeof(struct chcr_wr) + \ + sizeof(struct cpl_rx_phys_dsgl) + \ + sizeof(struct ulptx_sgl)) #define padap(dev) pci_get_drvdata(dev->u_ctx->lldi.pdev) diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 5b2fabb14229..b1b794435503 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -139,6 +139,9 @@ #define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309 0x06000000 #define CRYPTO_ALG_SUB_TYPE_AEAD_NULL 0x07000000 #define CRYPTO_ALG_SUB_TYPE_CTR 0x08000000 +#define CRYPTO_ALG_SUB_TYPE_CTR_RFC3686 0x09000000 +#define CRYPTO_ALG_SUB_TYPE_XTS 0x0a000000 +#define CRYPTO_ALG_SUB_TYPE_CBC 0x0b000000 #define CRYPTO_ALG_TYPE_HMAC (CRYPTO_ALG_TYPE_AHASH |\ CRYPTO_ALG_SUB_TYPE_HASH_HMAC) @@ -150,10 +153,12 @@ /* Aligned to 128 bit boundary */ struct ablk_ctx { + struct crypto_skcipher *sw_cipher; __be32 key_ctx_hdr; unsigned int enckey_len; - u8 key[CHCR_AES_MAX_KEY_LEN]; unsigned char ciph_mode; + u8 key[CHCR_AES_MAX_KEY_LEN]; + u8 nonce[4]; u8 rrkey[AES_MAX_KEY_SIZE]; }; struct chcr_aead_reqctx { @@ -233,7 +238,14 @@ struct chcr_ahash_req_ctx { struct chcr_blkcipher_req_ctx { struct sk_buff *skb; - unsigned int dst_nents; + struct scatterlist srcffwd[2]; + struct scatterlist dstffwd[2]; + struct scatterlist *dstsg; + struct scatterlist *dst; + struct scatterlist *newdstsg; + unsigned int processed; + unsigned int op; + short int dst_nents; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; }; @@ -275,5 +287,6 @@ static int chcr_aead_op(struct aead_request *req_base, int size, create_wr_t create_wr_fn); static inline int get_aead_subtype(struct crypto_aead *aead); - +static int chcr_handle_cipher_resp(struct ablkcipher_request *req, + unsigned char *input, int err); #endif /* __CHCR_CRYPTO_H__ */ From ee0863ba118d37609a795a15c08e9acf6809c038 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 15 Jun 2017 12:43:44 +0530 Subject: [PATCH 113/140] chcr - Add debug counters Count types of operation done by HW. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 16 ++++++++- drivers/crypto/chelsio/chcr_core.c | 2 ++ drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 1 + .../ethernet/chelsio/cxgb4/cxgb4_debugfs.c | 35 +++++++++++++++++++ .../net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 9 +++++ 5 files changed, 62 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 03b817f185dd..2f388af232ee 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -154,6 +154,7 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, struct uld_ctx *u_ctx = ULD_CTX(ctx); struct chcr_req_ctx ctx_req; unsigned int digestsize, updated_digestsize; + struct adapter *adap = padap(ctx->dev); switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_AEAD: @@ -207,6 +208,7 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, ctx_req.req.ahash_req->base.complete(req, err); break; } + atomic_inc(&adap->chcr_stats.complete); return err; } @@ -639,6 +641,7 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) unsigned int ivsize = AES_BLOCK_SIZE, kctx_len; gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + struct adapter *adap = padap(ctx->dev); phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents); @@ -701,6 +704,7 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) skb_set_transport_header(skb, transhdr_len); write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes); + atomic_inc(&adap->chcr_stats.cipher_rqst); create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1, sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl, ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); @@ -1337,6 +1341,7 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, u8 hash_size_in_response = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + struct adapter *adap = padap(ctx->dev); iopad_alignment = KEYCTX_ALIGN_PAD(digestsize); kctx_len = param->alg_prm.result_size + iopad_alignment; @@ -1393,7 +1398,7 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, param->bfr_len); if (param->sg_len != 0) write_sg_to_skb(skb, &frags, req->src, param->sg_len); - + atomic_inc(&adap->chcr_stats.digest_rqst); create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, hash_size_in_response, 0, DUMMY_BYTES, 0); req_ctx->skb = skb; @@ -1873,6 +1878,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, int null = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + struct adapter *adap = padap(ctx->dev); if (aeadctx->enckey_len == 0 || (req->cryptlen == 0)) goto err; @@ -1911,6 +1917,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, T6_MAX_AAD_SIZE, transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), op_type)) { + atomic_inc(&adap->chcr_stats.fallback); return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -1983,6 +1990,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, } write_buffer_to_skb(skb, &frags, req->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); + atomic_inc(&adap->chcr_stats.cipher_rqst); create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); reqctx->skb = skb; @@ -2206,6 +2214,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, int error = -EINVAL, src_nent; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + struct adapter *adap = padap(ctx->dev); if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) @@ -2245,6 +2254,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, T6_MAX_AAD_SIZE - 18, transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), op_type)) { + atomic_inc(&adap->chcr_stats.fallback); return ERR_PTR(chcr_aead_fallback(req, op_type)); } @@ -2282,6 +2292,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, skb_set_transport_header(skb, transhdr_len); frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); + atomic_inc(&adap->chcr_stats.aead_rqst); create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1, sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); reqctx->skb = skb; @@ -2316,6 +2327,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, int error = -EINVAL, src_nent; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + struct adapter *adap = padap(ctx->dev); /* validate key size */ if (aeadctx->enckey_len == 0) @@ -2355,6 +2367,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, T6_MAX_AAD_SIZE, transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), op_type)) { + atomic_inc(&adap->chcr_stats.fallback); return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -2421,6 +2434,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, write_sg_to_skb(skb, &frags, req->src, assoclen); write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); + atomic_inc(&adap->chcr_stats.aead_rqst); create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, sizeof(struct cpl_rx_phys_dsgl) + dst_size, reqctx->verify); diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index a6c938a913c7..5ae659af6a90 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -100,6 +100,7 @@ static int cpl_fw6_pld_handler(struct chcr_dev *dev, struct cpl_fw6_pld *fw6_pld; u32 ack_err_status = 0; int error_status = 0; + struct adapter *adap = padap(dev); fw6_pld = (struct cpl_fw6_pld *)input; req = (struct crypto_async_request *)(uintptr_t)be64_to_cpu( @@ -111,6 +112,7 @@ static int cpl_fw6_pld_handler(struct chcr_dev *dev, if (CHK_MAC_ERR_BIT(ack_err_status) || CHK_PAD_ERR_BIT(ack_err_status)) error_status = -EBADMSG; + atomic_inc(&adap->chcr_stats.error); } /* call completion callback with failure status */ if (req) { diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h index e88c1808e46f..b32eb8c7c1e3 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h @@ -868,6 +868,7 @@ struct adapter { /* TC u32 offload */ struct cxgb4_tc_u32_table *tc_u32; + struct chcr_stats_debug chcr_stats; }; /* Support for "sched-class" command to allow a TX Scheduling Class to be diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c index 1fa34b009891..77a59d7db7f5 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c @@ -3069,6 +3069,40 @@ static const struct file_operations meminfo_fops = { .llseek = seq_lseek, .release = single_release, }; + +static int chcr_show(struct seq_file *seq, void *v) +{ + struct adapter *adap = seq->private; + + seq_puts(seq, "Chelsio Crypto Accelerator Stats \n"); + seq_printf(seq, "Cipher Ops: %10u \n", + atomic_read(&adap->chcr_stats.cipher_rqst)); + seq_printf(seq, "Digest Ops: %10u \n", + atomic_read(&adap->chcr_stats.digest_rqst)); + seq_printf(seq, "Aead Ops: %10u \n", + atomic_read(&adap->chcr_stats.aead_rqst)); + seq_printf(seq, "Completion: %10u \n", + atomic_read(&adap->chcr_stats.complete)); + seq_printf(seq, "Error: %10u \n", + atomic_read(&adap->chcr_stats.error)); + seq_printf(seq, "Fallback: %10u \n", + atomic_read(&adap->chcr_stats.fallback)); + return 0; +} + + +static int chcr_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, chcr_show, inode->i_private); +} + +static const struct file_operations chcr_stats_debugfs_fops = { + .owner = THIS_MODULE, + .open = chcr_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; /* Add an array of Debug FS files. */ void add_debugfs_files(struct adapter *adap, @@ -3143,6 +3177,7 @@ int t4_setup_debugfs(struct adapter *adap) { "tids", &tid_info_debugfs_fops, S_IRUSR, 0}, { "blocked_fl", &blocked_fl_fops, S_IRUSR | S_IWUSR, 0 }, { "meminfo", &meminfo_fops, S_IRUSR, 0 }, + { "crypto", &chcr_stats_debugfs_fops, S_IRUSR, 0 }, }; /* Debug FS nodes common to all T5 and later adapters. diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h index 6e74040af49a..35f4d9f6382b 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h @@ -275,6 +275,15 @@ struct cxgb4_virt_res { /* virtualized HW resources */ unsigned int ncrypto_fc; }; +struct chcr_stats_debug { + atomic_t cipher_rqst; + atomic_t digest_rqst; + atomic_t aead_rqst; + atomic_t complete; + atomic_t error; + atomic_t fallback; +}; + #define OCQ_WIN_OFFSET(pdev, vres) \ (pci_resource_len((pdev), 2) - roundup_pow_of_two((vres)->ocq.size)) From 72cb471c13c5df00cc85dcfe5a2849db79570e4e Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 15 Jun 2017 12:43:45 +0530 Subject: [PATCH 114/140] MAINTAINERS: Add maintainer for chelsio crypto driver Add myself as maintainer for chcr. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- MAINTAINERS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 8d64b890669d..a24eb8c3a227 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3706,6 +3706,13 @@ S: Supported F: drivers/infiniband/hw/cxgb4/ F: include/uapi/rdma/cxgb4-abi.h +CXGB4 CRYPTO DRIVER (chcr) +M: Harsh Jain +L: linux-crypto@vger.kernel.org +W: http://www.chelsio.com +S: Supported +F: drivers/crypto/chelsio + CXGB4VF ETHERNET DRIVER (CXGB4VF) M: Casey Leedom L: netdev@vger.kernel.org From 738bff48871902bd37c56e6c6b492bb8e37315d5 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 15 Jun 2017 12:43:46 +0530 Subject: [PATCH 115/140] crypto: chcr - Ensure Destination sg entry size less than 2k Allocate new sg list in case received destination sg list has entry greater that 2k. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 153 ++++++++++++++++++++++++--- drivers/crypto/chelsio/chcr_crypto.h | 6 ++ 2 files changed, 142 insertions(+), 17 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 2f388af232ee..9a84ffa2adf3 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -166,6 +166,8 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, kfree_skb(ctx_req.ctx.reqctx->skb); ctx_req.ctx.reqctx->skb = NULL; } + free_new_sg(ctx_req.ctx.reqctx->newdstsg); + ctx_req.ctx.reqctx->newdstsg = NULL; if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { chcr_verify_tag(ctx_req.req.aead_req, input, &err); @@ -1068,6 +1070,8 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, chcr_send_wr(skb); return 0; complete: + free_new_sg(reqctx->newdstsg); + reqctx->newdstsg = NULL; req->base.complete(&req->base, err); return err; } @@ -1083,7 +1087,7 @@ static int process_cipher(struct ablkcipher_request *req, struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); struct ablk_ctx *ablkctx = ABLK_CTX(ctx); struct cipher_wr_param wrparam; - int bytes, err = -EINVAL; + int bytes, nents, err = -EINVAL; reqctx->newdstsg = NULL; reqctx->processed = 0; @@ -1097,7 +1101,14 @@ static int process_cipher(struct ablkcipher_request *req, goto error; } wrparam.srcsg = req->src; - reqctx->dstsg = req->dst; + if (is_newsg(req->dst, &nents)) { + reqctx->newdstsg = alloc_new_sg(req->dst, nents); + if (IS_ERR(reqctx->newdstsg)) + return PTR_ERR(reqctx->newdstsg); + reqctx->dstsg = reqctx->newdstsg; + } else { + reqctx->dstsg = req->dst; + } bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG, SPACE_LEFT(ablkctx->enckey_len), &wrparam.snent, @@ -1150,6 +1161,8 @@ static int process_cipher(struct ablkcipher_request *req, return 0; error: + free_new_sg(reqctx->newdstsg); + reqctx->newdstsg = NULL; return err; } @@ -1808,6 +1821,63 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) } } +static int is_newsg(struct scatterlist *sgl, unsigned int *newents) +{ + int nents = 0; + int ret = 0; + + while (sgl) { + if (sgl->length > CHCR_SG_SIZE) + ret = 1; + nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE); + sgl = sg_next(sgl); + } + *newents = nents; + return ret; +} + +static inline void free_new_sg(struct scatterlist *sgl) +{ + kfree(sgl); +} + +static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, + unsigned int nents) +{ + struct scatterlist *newsg, *sg; + int i, len, processed = 0; + struct page *spage; + int offset; + + newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL); + if (!newsg) + return ERR_PTR(-ENOMEM); + sg = newsg; + sg_init_table(sg, nents); + offset = sgl->offset; + spage = sg_page(sgl); + for (i = 0; i < nents; i++) { + len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE); + sg_set_page(sg, spage, len, offset); + processed += len; + offset += len; + if (offset >= PAGE_SIZE) { + offset = offset % PAGE_SIZE; + spage++; + } + if (processed == sgl->length) { + processed = 0; + sgl = sg_next(sgl); + if (!sgl) + break; + spage = sg_page(sgl); + offset = sgl->offset; + } + sg = sg_next(sg); + } + return newsg; +} + static int chcr_copy_assoc(struct aead_request *req, struct chcr_aead_ctx *ctx) { @@ -1870,7 +1940,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, struct scatterlist *src; unsigned int frags = 0, transhdr_len; unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0; - unsigned int kctx_len = 0; + unsigned int kctx_len = 0, nents; unsigned short stop_offset = 0; unsigned int assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); @@ -1880,7 +1950,10 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - if (aeadctx->enckey_len == 0 || (req->cryptlen == 0)) + reqctx->newdstsg = NULL; + dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : + authsize); + if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0)) goto err; if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) @@ -1889,14 +1962,24 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, if (src_nent < 0) goto err; src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) return ERR_PTR(error); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); + } + if (dst_size && is_newsg(req->dst, &nents)) { + reqctx->newdstsg = alloc_new_sg(req->dst, nents); + if (IS_ERR(reqctx->newdstsg)) + return ERR_CAST(reqctx->newdstsg); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, + reqctx->newdstsg, req->assoclen); + } else { + if (req->src == req->dst) + reqctx->dst = src; + else + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, + req->dst, req->assoclen); } if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { null = 1; @@ -1918,6 +2001,8 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); + free_new_sg(reqctx->newdstsg); + reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -2001,6 +2086,8 @@ dstmap_fail: /* ivmap_fail: */ kfree_skb(skb); err: + free_new_sg(reqctx->newdstsg); + reqctx->newdstsg = NULL; return ERR_PTR(error); } @@ -2208,7 +2295,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, struct phys_sge_parm sg_param; struct scatterlist *src; unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len; + unsigned int dst_size = 0, kctx_len, nents; unsigned int sub_type; unsigned int authsize = crypto_aead_authsize(tfm); int error = -EINVAL, src_nent; @@ -2216,7 +2303,9 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - + dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : + authsize); + reqctx->newdstsg = NULL; if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) goto err; src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); @@ -2225,16 +2314,25 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, sub_type = get_aead_subtype(tfm); src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - reqctx->dst = src; - if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) { pr_err("AAD copy to destination buffer fails\n"); return ERR_PTR(error); } - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); + } + if (dst_size && is_newsg(req->dst, &nents)) { + reqctx->newdstsg = alloc_new_sg(req->dst, nents); + if (IS_ERR(reqctx->newdstsg)) + return ERR_CAST(reqctx->newdstsg); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, + reqctx->newdstsg, req->assoclen); + } else { + if (req->src == req->dst) + reqctx->dst = src; + else + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, + req->dst, req->assoclen); } reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); @@ -2255,6 +2353,8 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); + free_new_sg(reqctx->newdstsg); + reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } @@ -2301,6 +2401,8 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, dstmap_fail: kfree_skb(skb); err: + free_new_sg(reqctx->newdstsg); + reqctx->newdstsg = NULL; return ERR_PTR(error); } @@ -2321,7 +2423,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, struct scatterlist *src; unsigned int frags = 0, transhdr_len; unsigned int ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len, assoclen = req->assoclen; + unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen; unsigned char tag_offset = 0; unsigned int authsize = crypto_aead_authsize(tfm); int error = -EINVAL, src_nent; @@ -2329,6 +2431,9 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); + reqctx->newdstsg = NULL; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : + authsize); /* validate key size */ if (aeadctx->enckey_len == 0) goto err; @@ -2340,15 +2445,25 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, goto err; src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen); - reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) return ERR_PTR(error); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - assoclen); } + if (dst_size && is_newsg(req->dst, &nents)) { + reqctx->newdstsg = alloc_new_sg(req->dst, nents); + if (IS_ERR(reqctx->newdstsg)) + return ERR_CAST(reqctx->newdstsg); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, + reqctx->newdstsg, assoclen); + } else { + if (req->src == req->dst) + reqctx->dst = src; + else + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, + req->dst, assoclen); + } reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); @@ -2368,6 +2483,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); + free_new_sg(reqctx->newdstsg); + reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -2446,6 +2563,8 @@ dstmap_fail: /* ivmap_fail: */ kfree_skb(skb); err: + free_new_sg(reqctx->newdstsg); + reqctx->newdstsg = NULL; return ERR_PTR(error); } diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index b1b794435503..a4f95b014b46 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -149,6 +149,7 @@ #define CHCR_HASH_MAX_BLOCK_SIZE_64 64 #define CHCR_HASH_MAX_BLOCK_SIZE_128 128 +#define CHCR_SG_SIZE 2048 /* Aligned to 128 bit boundary */ @@ -164,6 +165,7 @@ struct ablk_ctx { struct chcr_aead_reqctx { struct sk_buff *skb; struct scatterlist *dst; + struct scatterlist *newdstsg; struct scatterlist srcffwd[2]; struct scatterlist dstffwd[2]; short int dst_nents; @@ -287,6 +289,10 @@ static int chcr_aead_op(struct aead_request *req_base, int size, create_wr_t create_wr_fn); static inline int get_aead_subtype(struct crypto_aead *aead); +static int is_newsg(struct scatterlist *sgl, unsigned int *newents); +static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, + unsigned int nents); +static inline void free_new_sg(struct scatterlist *sgl); static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err); #endif /* __CHCR_CRYPTO_H__ */ From 14c19b178a012dd984721c1312114d4b8f3b4e81 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Thu, 15 Jun 2017 12:43:47 +0530 Subject: [PATCH 116/140] crypto: chcr - Select device in Round Robin fashion When multiple devices are present in system select device in round-robin fashion for crypto operations Signed-off-by: Atul Gupta Reviewed-by: Ganesh Goudar Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 8 +-- drivers/crypto/chelsio/chcr_core.c | 53 +++++++++++++------ drivers/crypto/chelsio/chcr_core.h | 2 +- .../net/ethernet/chelsio/cxgb4/cxgb4_uld.c | 1 + .../net/ethernet/chelsio/cxgb4/cxgb4_uld.h | 1 + 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 9a84ffa2adf3..aa4e5b88483d 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -1216,7 +1216,7 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req) static int chcr_device_init(struct chcr_context *ctx) { - struct uld_ctx *u_ctx; + struct uld_ctx *u_ctx = NULL; struct adapter *adap; unsigned int id; int txq_perchan, txq_idx, ntxq; @@ -1224,12 +1224,12 @@ static int chcr_device_init(struct chcr_context *ctx) id = smp_processor_id(); if (!ctx->dev) { - err = assign_chcr_device(&ctx->dev); - if (err) { + u_ctx = assign_chcr_device(); + if (!u_ctx) { pr_err("chcr device assignment fails\n"); goto out; } - u_ctx = ULD_CTX(ctx); + ctx->dev = u_ctx->dev; adap = padap(ctx->dev); ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq, adap->vres.ncrypto_fc); diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index 5ae659af6a90..b6dd9cbe815f 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -29,6 +29,7 @@ static LIST_HEAD(uld_ctx_list); static DEFINE_MUTEX(dev_mutex); static atomic_t dev_count; +static struct uld_ctx *ctx_rr; typedef int (*chcr_handler_func)(struct chcr_dev *dev, unsigned char *input); static int cpl_fw6_pld_handler(struct chcr_dev *dev, unsigned char *input); @@ -49,25 +50,28 @@ static struct cxgb4_uld_info chcr_uld_info = { .rx_handler = chcr_uld_rx_handler, }; -int assign_chcr_device(struct chcr_dev **dev) +struct uld_ctx *assign_chcr_device(void) { - struct uld_ctx *u_ctx; - int ret = -ENXIO; + struct uld_ctx *u_ctx = NULL; /* - * Which device to use if multiple devices are available TODO - * May be select the device based on round robin. One session - * must go to the same device to maintain the ordering. + * When multiple devices are present in system select + * device in round-robin fashion for crypto operations + * Although One session must use the same device to + * maintain request-response ordering. */ - mutex_lock(&dev_mutex); /* TODO ? */ - list_for_each_entry(u_ctx, &uld_ctx_list, entry) - if (u_ctx->dev) { - *dev = u_ctx->dev; - ret = 0; - break; + mutex_lock(&dev_mutex); + if (!list_empty(&uld_ctx_list)) { + u_ctx = ctx_rr; + if (list_is_last(&ctx_rr->entry, &uld_ctx_list)) + ctx_rr = list_first_entry(&uld_ctx_list, + struct uld_ctx, + entry); + else + ctx_rr = list_next_entry(ctx_rr, entry); } mutex_unlock(&dev_mutex); - return ret; + return u_ctx; } static int chcr_dev_add(struct uld_ctx *u_ctx) @@ -82,11 +86,27 @@ static int chcr_dev_add(struct uld_ctx *u_ctx) u_ctx->dev = dev; dev->u_ctx = u_ctx; atomic_inc(&dev_count); + mutex_lock(&dev_mutex); + list_add_tail(&u_ctx->entry, &uld_ctx_list); + if (!ctx_rr) + ctx_rr = u_ctx; + mutex_unlock(&dev_mutex); return 0; } static int chcr_dev_remove(struct uld_ctx *u_ctx) { + if (ctx_rr == u_ctx) { + if (list_is_last(&ctx_rr->entry, &uld_ctx_list)) + ctx_rr = list_first_entry(&uld_ctx_list, + struct uld_ctx, + entry); + else + ctx_rr = list_next_entry(ctx_rr, entry); + } + list_del(&u_ctx->entry); + if (list_empty(&uld_ctx_list)) + ctx_rr = NULL; kfree(u_ctx->dev); u_ctx->dev = NULL; atomic_dec(&dev_count); @@ -139,10 +159,11 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld) u_ctx = ERR_PTR(-ENOMEM); goto out; } + if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) { + u_ctx = ERR_PTR(-ENOMEM); + goto out; + } u_ctx->lldi = *lld; - mutex_lock(&dev_mutex); - list_add_tail(&u_ctx->entry, &uld_ctx_list); - mutex_unlock(&dev_mutex); out: return u_ctx; } diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index ddfb2c934551..c9a19b2a1e9f 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -89,7 +89,7 @@ struct uld_ctx { struct chcr_dev *dev; }; -int assign_chcr_device(struct chcr_dev **dev); +struct uld_ctx * assign_chcr_device(void); int chcr_send_wr(struct sk_buff *skb); int start_crypto(void); int stop_crypto(void); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c index d0868c2320da..ec53fe9dec68 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c @@ -642,6 +642,7 @@ static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld) lld->sge_ingpadboundary = adap->sge.fl_align; lld->sge_egrstatuspagesize = adap->sge.stat_len; lld->sge_pktshift = adap->sge.pktshift; + lld->ulp_crypto = adap->params.crypto; lld->enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN; lld->max_ordird_qp = adap->params.max_ordird_qp; lld->max_ird_adapter = adap->params.max_ird_adapter; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h index 35f4d9f6382b..8f1c874cfe21 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h @@ -331,6 +331,7 @@ struct cxgb4_lld_info { unsigned int iscsi_tagmask; /* iscsi ddp tag mask */ unsigned int iscsi_pgsz_order; /* iscsi ddp page size orders */ unsigned int iscsi_llimit; /* chip's iscsi region llimit */ + unsigned int ulp_crypto; /* crypto lookaside support */ void **iscsi_ppm; /* iscsi page pod manager */ int nodeid; /* device numa node id */ bool fr_nsmr_tpte_wr_support; /* FW supports FR_NSMR_TPTE_WR */ From aed3731e9039285c1d98f7636f5be92ea5db9e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 15 Jun 2017 09:56:17 +0200 Subject: [PATCH 117/140] crypto: inside-secure - use hmac ipad/opad constants Replace the hmac ipad/opad values by their defined constants. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 060ea034c9da..6eee1a502225 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -8,6 +8,7 @@ * warranty of any kind, whether express or implied. */ +#include #include #include #include @@ -774,8 +775,8 @@ static int safexcel_hmac_init_pad(struct ahash_request *areq, memcpy(opad, ipad, blocksize); for (i = 0; i < blocksize; i++) { - ipad[i] ^= 0x36; - opad[i] ^= 0x5c; + ipad[i] ^= HMAC_IPAD_VALUE; + opad[i] ^= HMAC_OPAD_VALUE; } return 0; From aefa794efed40cc6f8225646d75deda879b98256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 15 Jun 2017 09:56:18 +0200 Subject: [PATCH 118/140] crypto: inside-secure - fix the ring wr_cache offset The EIP197_HIA_xDR_CFG_WR_CACHE macro was defined to use an offset of 23, which is wrong as it's actually 25. Fix this. Reported-by: Igal Liberman Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index c17fdd40b99f..0328a9314b90 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -99,7 +99,7 @@ #define EIP197_HIA_xDR_WR_RES_BUF BIT(22) #define EIP197_HIA_xDR_WR_CTRL_BUG BIT(23) #define EIP197_HIA_xDR_WR_OWN_BUF BIT(24) -#define EIP197_HIA_xDR_CFG_WR_CACHE(n) (((n) & 0x7) << 23) +#define EIP197_HIA_xDR_CFG_WR_CACHE(n) (((n) & 0x7) << 25) #define EIP197_HIA_xDR_CFG_RD_CACHE(n) (((n) & 0x7) << 29) /* EIP197_HIA_CDR_THRESH */ From 39ba1bb4a4624bc0215531e0d5fdbbf0325c0c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 15 Jun 2017 09:56:19 +0200 Subject: [PATCH 119/140] crypto: inside-secure - fix incorrect DSE data cache setting Set the correct value to the DSE data cache, using WR_CACHE_3BITS instead of RD_CACHE_3BITS. This fixes an incorrect setting and helps improving performances. Reported-by: Igal Liberman Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 5485e925e18d..99755fc1a161 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -328,7 +328,7 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) /* DMA transfer size to use */ val = EIP197_HIA_DSE_CFG_DIS_DEBUG; val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8); - val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS); + val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); writel(val, priv->base + EIP197_HIA_DSE_CFG); /* Leave the DSE threads reset state */ From c87925bfd234f09b1b423f7a466a998d398f5a56 Mon Sep 17 00:00:00 2001 From: Igal Liberman Date: Thu, 15 Jun 2017 09:56:20 +0200 Subject: [PATCH 120/140] crypto: inside-secure - enable single WR in DSE configuration When enable_single_wr is not enabled, the DSE will only write those parts of a result descriptor that need updating, which means a final result descriptor will be written in 2 or 3 smaller transfers. When enable_single_wr is enabled the DSE will combine these 2-3 updates into one large write transfer, generally improving performance. Signed-off-by: Igal Liberman Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 99755fc1a161..658b307c6a11 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -329,6 +329,7 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) val = EIP197_HIA_DSE_CFG_DIS_DEBUG; val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8); val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); + val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR; writel(val, priv->base + EIP197_HIA_DSE_CFG); /* Leave the DSE threads reset state */ diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 0328a9314b90..b8a81c568c99 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -143,6 +143,7 @@ #define EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(n) (((n) & 0x7) << 20) #define EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(n) ((n) << 24) #define EIP197_HIA_DFE_CFG_DIS_DEBUG (BIT(31) | BIT(29)) +#define EIP197_HIA_DSE_CFG_EN_SINGLE_WR BIT(29) #define EIP197_HIA_DSE_CFG_DIS_DEBUG BIT(31) /* EIP197_HIA_DFE/DSE_THR_CTRL */ From ee1fd870ee09ea5636b3d7a73769cd699241c77c Mon Sep 17 00:00:00 2001 From: Igal Liberman Date: Thu, 15 Jun 2017 09:56:21 +0200 Subject: [PATCH 121/140] crypto: inside-secure - optimize DSE bufferability control Configure the data write bufferability to always buffer packets in the DSE. This change slightly improves performance. Signed-off-by: Igal Liberman Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 1 + drivers/crypto/inside-secure/safexcel.h | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 658b307c6a11..73f4ef8d71f3 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -329,6 +329,7 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) val = EIP197_HIA_DSE_CFG_DIS_DEBUG; val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8); val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); + val |= EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE; val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR; writel(val, priv->base + EIP197_HIA_DSE_CFG); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index b8a81c568c99..7e3cbb9ac98e 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -139,6 +139,7 @@ #define EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(n) ((n) << 0) #define EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(n) (((n) & 0x7) << 4) #define EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(n) ((n) << 8) +#define EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE GENMASK(15, 14) #define EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(n) ((n) << 16) #define EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(n) (((n) & 0x7) << 20) #define EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(n) ((n) << 24) From 152bdf4cdb9c20e1858bc1b754c7bfb984a6ea14 Mon Sep 17 00:00:00 2001 From: Ofer Heifetz Date: Thu, 15 Jun 2017 09:56:22 +0200 Subject: [PATCH 122/140] crypto: inside-secure - align the cipher and hash send functions The cipher and hash send functions both set the results variable, to be used later by the caller. Align the way they're setting this variable. Signed-off-by: Ofer Heifetz Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_cipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 59e6081602a2..8eea4d30db31 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -269,7 +269,7 @@ static int safexcel_aes_send(struct crypto_async_request *async, spin_unlock_bh(&priv->ring[ring].egress_lock); *commands = n_cdesc; - *results = nr_dst; + *results = n_rdesc; return 0; rdesc_rollback: From 9785843424c803cbe9515c477a6b89181914ea09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 15 Jun 2017 09:56:23 +0200 Subject: [PATCH 123/140] crypto: inside-secure - update the context and request later This move the context and request updates at the end of the cipher and hash send() functions. This way the context and request fields are set only when everything else was successful in the send() functions. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_cipher.c | 7 +++---- drivers/crypto/inside-secure/safexcel_hash.c | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 8eea4d30db31..6037cdfc1f16 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -190,8 +190,6 @@ static int safexcel_aes_send(struct crypto_async_request *async, int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = req->cryptlen; int i, ret = 0; - request->req = &req->base; - if (req->src == req->dst) { nr_src = dma_map_sg(priv->dev, req->src, sg_nents_for_len(req->src, req->cryptlen), @@ -264,10 +262,11 @@ static int safexcel_aes_send(struct crypto_async_request *async, n_rdesc++; } - ctx->base.handle_result = safexcel_handle_result; - spin_unlock_bh(&priv->ring[ring].egress_lock); + request->req = &req->base; + ctx->base.handle_result = safexcel_handle_result; + *commands = n_cdesc; *results = n_rdesc; return 0; diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 6eee1a502225..4e526372464f 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -198,9 +198,6 @@ static int safexcel_ahash_send(struct crypto_async_request *async, int ring, len -= extra; } - request->req = &areq->base; - ctx->base.handle_result = safexcel_handle_result; - spin_lock_bh(&priv->ring[ring].egress_lock); /* Add a command descriptor for the cached data, if any */ @@ -291,9 +288,12 @@ send_command: goto cdesc_rollback; } - req->processed += len; spin_unlock_bh(&priv->ring[ring].egress_lock); + req->processed += len; + request->req = &areq->base; + ctx->base.handle_result = safexcel_handle_result; + *commands = n_cdesc; *results = 1; return 0; From 86671abbbbfc959c4e4cbd2c618d5868a6f8dcf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 15 Jun 2017 09:56:24 +0200 Subject: [PATCH 124/140] crypto: inside-secure - use one queue per hw ring Update the inside-secure safexcel driver from using one global queue to one queue per hw ring. This ease the request management and keep the hw in sync with what's done in sw. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 86 +++++++++---------- drivers/crypto/inside-secure/safexcel.h | 12 +-- .../crypto/inside-secure/safexcel_cipher.c | 38 ++++---- drivers/crypto/inside-secure/safexcel_hash.c | 38 ++++---- 4 files changed, 89 insertions(+), 85 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 73f4ef8d71f3..8956b23803a8 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -422,20 +422,18 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) return 0; } -void safexcel_dequeue(struct safexcel_crypto_priv *priv) +void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) { struct crypto_async_request *req, *backlog; struct safexcel_context *ctx; struct safexcel_request *request; - int i, ret, n = 0, nreq[EIP197_MAX_RINGS] = {0}; - int cdesc[EIP197_MAX_RINGS] = {0}, rdesc[EIP197_MAX_RINGS] = {0}; - int commands, results; + int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; do { - spin_lock_bh(&priv->lock); - req = crypto_dequeue_request(&priv->queue); - backlog = crypto_get_backlog(&priv->queue); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + req = crypto_dequeue_request(&priv->ring[ring].queue); + backlog = crypto_get_backlog(&priv->ring[ring].queue); + spin_unlock_bh(&priv->ring[ring].queue_lock); if (!req) goto finalize; @@ -445,58 +443,51 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv) goto requeue; ctx = crypto_tfm_ctx(req->tfm); - ret = ctx->send(req, ctx->ring, request, &commands, &results); + ret = ctx->send(req, ring, request, &commands, &results); if (ret) { kfree(request); requeue: - spin_lock_bh(&priv->lock); - crypto_enqueue_request(&priv->queue, req); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + crypto_enqueue_request(&priv->ring[ring].queue, req); + spin_unlock_bh(&priv->ring[ring].queue_lock); - priv->need_dequeue = true; + priv->ring[ring].need_dequeue = true; continue; } if (backlog) backlog->complete(backlog, -EINPROGRESS); - spin_lock_bh(&priv->ring[ctx->ring].egress_lock); - list_add_tail(&request->list, &priv->ring[ctx->ring].list); - spin_unlock_bh(&priv->ring[ctx->ring].egress_lock); + spin_lock_bh(&priv->ring[ring].egress_lock); + list_add_tail(&request->list, &priv->ring[ring].list); + spin_unlock_bh(&priv->ring[ring].egress_lock); - cdesc[ctx->ring] += commands; - rdesc[ctx->ring] += results; - - nreq[ctx->ring]++; - } while (n++ < EIP197_MAX_BATCH_SZ); + cdesc += commands; + rdesc += results; + } while (nreq++ < EIP197_MAX_BATCH_SZ); finalize: - if (n == EIP197_MAX_BATCH_SZ) - priv->need_dequeue = true; - else if (!n) + if (nreq == EIP197_MAX_BATCH_SZ) + priv->ring[ring].need_dequeue = true; + else if (!nreq) return; - for (i = 0; i < priv->config.rings; i++) { - if (!nreq[i]) - continue; + spin_lock_bh(&priv->ring[ring].lock); - spin_lock_bh(&priv->ring[i].lock); + /* Configure when we want an interrupt */ + writel(EIP197_HIA_RDR_THRESH_PKT_MODE | + EIP197_HIA_RDR_THRESH_PROC_PKT(nreq), + priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_THRESH); - /* Configure when we want an interrupt */ - writel(EIP197_HIA_RDR_THRESH_PKT_MODE | - EIP197_HIA_RDR_THRESH_PROC_PKT(nreq[i]), - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_THRESH); + /* let the RDR know we have pending descriptors */ + writel((rdesc * priv->config.rd_offset) << 2, + priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PREP_COUNT); - /* let the RDR know we have pending descriptors */ - writel((rdesc[i] * priv->config.rd_offset) << 2, - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_COUNT); + /* let the CDR know we have pending descriptors */ + writel((cdesc * priv->config.cd_offset) << 2, + priv->base + EIP197_HIA_CDR(ring) + EIP197_HIA_xDR_PREP_COUNT); - /* let the CDR know we have pending descriptors */ - writel((cdesc[i] * priv->config.cd_offset) << 2, - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_COUNT); - - spin_unlock_bh(&priv->ring[i].lock); - } + spin_unlock_bh(&priv->ring[ring].lock); } void safexcel_free_context(struct safexcel_crypto_priv *priv, @@ -638,9 +629,9 @@ static void safexcel_handle_result_work(struct work_struct *work) safexcel_handle_result_descriptor(priv, data->ring); - if (priv->need_dequeue) { - priv->need_dequeue = false; - safexcel_dequeue(data->priv); + if (priv->ring[data->ring].need_dequeue) { + priv->ring[data->ring].need_dequeue = false; + safexcel_dequeue(data->priv, data->ring); } } @@ -864,17 +855,18 @@ static int safexcel_probe(struct platform_device *pdev) goto err_clk; } + crypto_init_queue(&priv->ring[i].queue, + EIP197_DEFAULT_RING_SIZE); + INIT_LIST_HEAD(&priv->ring[i].list); spin_lock_init(&priv->ring[i].lock); spin_lock_init(&priv->ring[i].egress_lock); + spin_lock_init(&priv->ring[i].queue_lock); } platform_set_drvdata(pdev, priv); atomic_set(&priv->ring_used, 0); - spin_lock_init(&priv->lock); - crypto_init_queue(&priv->queue, EIP197_DEFAULT_RING_SIZE); - ret = safexcel_hw_init(priv); if (ret) { dev_err(dev, "EIP h/w init failed (%d)\n", ret); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 7e3cbb9ac98e..abe0f59d1473 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -469,11 +469,6 @@ struct safexcel_crypto_priv { struct clk *clk; struct safexcel_config config; - spinlock_t lock; - struct crypto_queue queue; - - bool need_dequeue; - /* context DMA pool */ struct dma_pool *context_pool; @@ -490,6 +485,11 @@ struct safexcel_crypto_priv { /* command/result rings */ struct safexcel_ring cdr; struct safexcel_ring rdr; + + /* queue */ + struct crypto_queue queue; + spinlock_t queue_lock; + bool need_dequeue; } ring[EIP197_MAX_RINGS]; }; @@ -533,7 +533,7 @@ struct safexcel_inv_result { int error; }; -void safexcel_dequeue(struct safexcel_crypto_priv *priv); +void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring); void safexcel_complete(struct safexcel_crypto_priv *priv, int ring); void safexcel_free_context(struct safexcel_crypto_priv *priv, struct crypto_async_request *req, diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 6037cdfc1f16..d2207ac5ba19 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -339,18 +339,21 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, return ndesc; } + ring = safexcel_select_ring(priv); + ctx->base.ring = ring; ctx->base.needs_inv = false; - ctx->base.ring = safexcel_select_ring(priv); ctx->base.send = safexcel_aes_send; - spin_lock_bh(&priv->lock); - enq_ret = crypto_enqueue_request(&priv->queue, async); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async); + spin_unlock_bh(&priv->ring[ring].queue_lock); if (enq_ret != -EINPROGRESS) *ret = enq_ret; - priv->need_dequeue = true; + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); + *should_complete = false; return ndesc; @@ -384,6 +387,7 @@ static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm) struct safexcel_crypto_priv *priv = ctx->priv; struct skcipher_request req; struct safexcel_inv_result result = { 0 }; + int ring = ctx->base.ring; memset(&req, 0, sizeof(struct skcipher_request)); @@ -397,12 +401,12 @@ static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm) ctx->base.exit_inv = true; ctx->base.send = safexcel_cipher_send_inv; - spin_lock_bh(&priv->lock); - crypto_enqueue_request(&priv->queue, &req.base); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + crypto_enqueue_request(&priv->ring[ring].queue, &req.base); + spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!priv->need_dequeue) - safexcel_dequeue(priv); + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); wait_for_completion_interruptible(&result.completion); @@ -421,7 +425,7 @@ static int safexcel_aes(struct skcipher_request *req, { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct safexcel_crypto_priv *priv = ctx->priv; - int ret; + int ret, ring; ctx->direction = dir; ctx->mode = mode; @@ -440,12 +444,14 @@ static int safexcel_aes(struct skcipher_request *req, return -ENOMEM; } - spin_lock_bh(&priv->lock); - ret = crypto_enqueue_request(&priv->queue, &req->base); - spin_unlock_bh(&priv->lock); + ring = ctx->base.ring; - if (!priv->need_dequeue) - safexcel_dequeue(priv); + spin_lock_bh(&priv->ring[ring].queue_lock); + ret = crypto_enqueue_request(&priv->ring[ring].queue, &req->base); + spin_unlock_bh(&priv->ring[ring].queue_lock); + + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); return ret; } diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 4e526372464f..8527a5899a2f 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -374,18 +374,21 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, return 1; } - ctx->base.ring = safexcel_select_ring(priv); + ring = safexcel_select_ring(priv); + ctx->base.ring = ring; ctx->base.needs_inv = false; ctx->base.send = safexcel_ahash_send; - spin_lock_bh(&priv->lock); - enq_ret = crypto_enqueue_request(&priv->queue, async); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async); + spin_unlock_bh(&priv->ring[ring].queue_lock); if (enq_ret != -EINPROGRESS) *ret = enq_ret; - priv->need_dequeue = true; + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); + *should_complete = false; return 1; @@ -417,6 +420,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) struct safexcel_crypto_priv *priv = ctx->priv; struct ahash_request req; struct safexcel_inv_result result = { 0 }; + int ring = ctx->base.ring; memset(&req, 0, sizeof(struct ahash_request)); @@ -430,12 +434,12 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) ctx->base.exit_inv = true; ctx->base.send = safexcel_ahash_send_inv; - spin_lock_bh(&priv->lock); - crypto_enqueue_request(&priv->queue, &req.base); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + crypto_enqueue_request(&priv->ring[ring].queue, &req.base); + spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!priv->need_dequeue) - safexcel_dequeue(priv); + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); wait_for_completion_interruptible(&result.completion); @@ -477,7 +481,7 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); struct safexcel_ahash_req *req = ahash_request_ctx(areq); struct safexcel_crypto_priv *priv = ctx->priv; - int ret; + int ret, ring; ctx->base.send = safexcel_ahash_send; @@ -496,12 +500,14 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) return -ENOMEM; } - spin_lock_bh(&priv->lock); - ret = crypto_enqueue_request(&priv->queue, &areq->base); - spin_unlock_bh(&priv->lock); + ring = ctx->base.ring; - if (!priv->need_dequeue) - safexcel_dequeue(priv); + spin_lock_bh(&priv->ring[ring].queue_lock); + ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base); + spin_unlock_bh(&priv->ring[ring].queue_lock); + + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); return ret; } From c5acabd33c85e501f8838af12491f7143b2a9051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 15 Jun 2017 09:56:25 +0200 Subject: [PATCH 125/140] crypto: inside-secure - stop requeueing failed requests This update the dequeue function of the inside-secure safexcel driver so that failed requests aren't requeued when they fail (for whatever reason, which can be because the hw ring is full). Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 8956b23803a8..8ae133a9e3f2 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -439,20 +439,22 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) goto finalize; request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req)); - if (!request) - goto requeue; - - ctx = crypto_tfm_ctx(req->tfm); - ret = ctx->send(req, ring, request, &commands, &results); - if (ret) { - kfree(request); -requeue: + if (!request) { spin_lock_bh(&priv->ring[ring].queue_lock); crypto_enqueue_request(&priv->ring[ring].queue, req); spin_unlock_bh(&priv->ring[ring].queue_lock); priv->ring[ring].need_dequeue = true; - continue; + goto finalize; + } + + ctx = crypto_tfm_ctx(req->tfm); + ret = ctx->send(req, ring, request, &commands, &results); + if (ret) { + kfree(request); + req->complete(req, ret); + priv->ring[ring].need_dequeue = true; + goto finalize; } if (backlog) From b1deb47ad10b71a7d62dfdb049735a1be2214eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 15 Jun 2017 09:56:26 +0200 Subject: [PATCH 126/140] crypto: inside-secure - get the backlog before dequeueing the request Get the backlog before dequeuing the request otherwise we'll miss the first request in line. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 8ae133a9e3f2..8f195e031938 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -431,8 +431,8 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) do { spin_lock_bh(&priv->ring[ring].queue_lock); - req = crypto_dequeue_request(&priv->ring[ring].queue); backlog = crypto_get_backlog(&priv->ring[ring].queue); + req = crypto_dequeue_request(&priv->ring[ring].queue); spin_unlock_bh(&priv->ring[ring].queue_lock); if (!req) From 5eb0cc66f3a3cad46c2e432342f2a3c20c992b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 15 Jun 2017 09:56:27 +0200 Subject: [PATCH 127/140] crypto: inside-secure - only dequeue when needed This force the need_dequeue flag to be unset whenever the dequeue function is called, to avoid calling it when it is not necessary. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 8f195e031938..e7f87ac12685 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -429,6 +429,8 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) struct safexcel_request *request; int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; + priv->ring[ring].need_dequeue = false; + do { spin_lock_bh(&priv->ring[ring].queue_lock); backlog = crypto_get_backlog(&priv->ring[ring].queue); @@ -631,10 +633,8 @@ static void safexcel_handle_result_work(struct work_struct *work) safexcel_handle_result_descriptor(priv, data->ring); - if (priv->ring[data->ring].need_dequeue) { - priv->ring[data->ring].need_dequeue = false; + if (priv->ring[data->ring].need_dequeue) safexcel_dequeue(data->priv, data->ring); - } } struct safexcel_ring_irq_data { From e826934e359efa75bade6e3603432e31d5794281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 15 Jun 2017 09:56:28 +0200 Subject: [PATCH 128/140] crypto: inside-secure - increase the batch size Increase the batch size to the maximum number of requests a ring can handle at a time (its size). This is possible now that the request queues are per hw ring. This improves performances. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index abe0f59d1473..304c5838c11a 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -23,7 +23,7 @@ #define EIP197_MAX_TOKENS 5 #define EIP197_MAX_RINGS 4 #define EIP197_FETCH_COUNT 1 -#define EIP197_MAX_BATCH_SZ 8 +#define EIP197_MAX_BATCH_SZ EIP197_DEFAULT_RING_SIZE #define EIP197_GFP_FLAGS(base) ((base).flags & CRYPTO_TFM_REQ_MAY_SLEEP ? \ GFP_KERNEL : GFP_ATOMIC) From cfb73f89534fb246155a05a52d63559cb5d58749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Thu, 15 Jun 2017 09:56:29 +0200 Subject: [PATCH 129/140] crypto: inside-secure - use the base_end pointer in ring rollback A base_end pointer is set and provided. Use it in the ring rollback function to avoid using build-in defines. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_ring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c index fdbf05ae55fc..c9d2a8716b5b 100644 --- a/drivers/crypto/inside-secure/safexcel_ring.c +++ b/drivers/crypto/inside-secure/safexcel_ring.c @@ -84,7 +84,7 @@ void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv, return; if (ring->write == ring->base) - ring->write += (EIP197_DEFAULT_RING_SIZE - 1) * ring->offset; + ring->write = ring->base_end - ring->offset; else ring->write -= ring->offset; From 4914b90b7baa8f21a4bb4ddc3d92b92e3324767e Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Thu, 15 Jun 2017 17:28:10 +0530 Subject: [PATCH 130/140] crypto: n2 - make of_device_ids const of_device_ids are not supposed to change at runtime. All functions working with of_device_ids provided by work with const of_device_ids. So mark the non-const structs as const. File size before: text data bss dec hex filename 2001 2168 48 4217 1079 drivers/crypto/n2_core.o File size after constify dummy_tlb_ops.: text data bss dec hex filename 3601 536 48 4185 1059 drivers/crypto/n2_core.o Signed-off-by: Arvind Yadav Acked-by: David S. Miller Signed-off-by: Herbert Xu --- drivers/crypto/n2_core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 4ecb77aa60e1..269451375b63 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -2169,7 +2169,7 @@ static int n2_mau_remove(struct platform_device *dev) return 0; } -static struct of_device_id n2_crypto_match[] = { +static const struct of_device_id n2_crypto_match[] = { { .name = "n2cp", .compatible = "SUNW,n2-cwq", @@ -2196,7 +2196,7 @@ static struct platform_driver n2_crypto_driver = { .remove = n2_crypto_remove, }; -static struct of_device_id n2_mau_match[] = { +static const struct of_device_id n2_mau_match[] = { { .name = "ncp", .compatible = "SUNW,n2-mau", From 78557e77b25a98c153f87182a0f48b63e474ac9f Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Fri, 16 Jun 2017 11:39:48 +0300 Subject: [PATCH 131/140] crypto: vmx - remove unnecessary check You can't reach init() if parent alg_name is invalid. Moreover, cypto_alloc_base() will return ENOENT if alg_name is NULL. Found while grasping the fallback mechanism. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/vmx/aes.c | 7 +------ drivers/crypto/vmx/aes_cbc.c | 7 +------ drivers/crypto/vmx/aes_ctr.c | 7 +------ drivers/crypto/vmx/aes_xts.c | 7 +------ 4 files changed, 4 insertions(+), 24 deletions(-) diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c index 022c7ab7351a..96072b9b55c4 100644 --- a/drivers/crypto/vmx/aes.c +++ b/drivers/crypto/vmx/aes.c @@ -37,15 +37,10 @@ struct p8_aes_ctx { static int p8_aes_init(struct crypto_tfm *tfm) { - const char *alg; + const char *alg = crypto_tfm_alg_name(tfm); struct crypto_cipher *fallback; struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); - if (!(alg = crypto_tfm_alg_name(tfm))) { - printk(KERN_ERR "Failed to get algorithm name.\n"); - return -ENOENT; - } - fallback = crypto_alloc_cipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { printk(KERN_ERR diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c index 72a26eb4e954..7394d35d5936 100644 --- a/drivers/crypto/vmx/aes_cbc.c +++ b/drivers/crypto/vmx/aes_cbc.c @@ -39,15 +39,10 @@ struct p8_aes_cbc_ctx { static int p8_aes_cbc_init(struct crypto_tfm *tfm) { - const char *alg; + const char *alg = crypto_tfm_alg_name(tfm); struct crypto_skcipher *fallback; struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); - if (!(alg = crypto_tfm_alg_name(tfm))) { - printk(KERN_ERR "Failed to get algorithm name.\n"); - return -ENOENT; - } - fallback = crypto_alloc_skcipher(alg, 0, CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c index 7cf6d31c1123..9c26d9e8dbea 100644 --- a/drivers/crypto/vmx/aes_ctr.c +++ b/drivers/crypto/vmx/aes_ctr.c @@ -36,15 +36,10 @@ struct p8_aes_ctr_ctx { static int p8_aes_ctr_init(struct crypto_tfm *tfm) { - const char *alg; + const char *alg = crypto_tfm_alg_name(tfm); struct crypto_blkcipher *fallback; struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); - if (!(alg = crypto_tfm_alg_name(tfm))) { - printk(KERN_ERR "Failed to get algorithm name.\n"); - return -ENOENT; - } - fallback = crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c index 6adc9290557a..8cd6e62e4c90 100644 --- a/drivers/crypto/vmx/aes_xts.c +++ b/drivers/crypto/vmx/aes_xts.c @@ -41,15 +41,10 @@ struct p8_aes_xts_ctx { static int p8_aes_xts_init(struct crypto_tfm *tfm) { - const char *alg; + const char *alg = crypto_tfm_alg_name(tfm); struct crypto_skcipher *fallback; struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); - if (!(alg = crypto_tfm_alg_name(tfm))) { - printk(KERN_ERR "Failed to get algorithm name.\n"); - return -ENOENT; - } - fallback = crypto_alloc_skcipher(alg, 0, CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { From 52a33d99882291808681af8582358ddca5b0d0bc Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Fri, 16 Jun 2017 14:46:44 +0530 Subject: [PATCH 132/140] crypto: caam - make of_device_ids const. of_device_ids are not supposed to change at runtime. All functions working with of_device_ids provided by work with const of_device_ids. So mark the non-const structs as const. File size before: text data bss dec hex filename 2376 808 128 3312 cf0 drivers/crypto/caam/jr.o File size after constify caam_jr_match: text data bss dec hex filename 2976 192 128 3296 ce0 drivers/crypto/caam/jr.o Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/caam/jr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 27631000b9f8..1ccfb317d468 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -536,7 +536,7 @@ static int caam_jr_probe(struct platform_device *pdev) return 0; } -static struct of_device_id caam_jr_match[] = { +static const struct of_device_id caam_jr_match[] = { { .compatible = "fsl,sec-v4.0-job-ring", }, From b61929c654f2e725644935737c4c1ea9c741e2f8 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Fri, 26 May 2017 12:11:31 +0200 Subject: [PATCH 133/140] crypto: drbg - Fixes panic in wait_for_completion call Initialise ctr_completion variable before use. Cc: Signed-off-by: Harsh Jain Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/drbg.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/drbg.c b/crypto/drbg.c index cdb27ac4b226..633a88e93ab0 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1691,6 +1691,7 @@ static int drbg_init_sym_kernel(struct drbg_state *drbg) return PTR_ERR(sk_tfm); } drbg->ctr_handle = sk_tfm; + init_completion(&drbg->ctr_completion); req = skcipher_request_alloc(sk_tfm, GFP_KERNEL); if (!req) { From 42cfcafb91dabb0f9d9e08396c39824535948c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Mon, 19 Jun 2017 11:44:45 +0300 Subject: [PATCH 134/140] crypto: caam - fix gfp allocation flags (part I) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes in the SW cts (ciphertext stealing) code in commit 0605c41cc53ca ("crypto: cts - Convert to skcipher") revealed a problem in the CAAM driver: when cts(cbc(aes)) is executed and cts runs in SW, cbc(aes) is offloaded in CAAM; cts encrypts the last block in atomic context and CAAM incorrectly decides to use GFP_KERNEL for memory allocation. Fix this by allowing GFP_KERNEL (sleeping) only when MAY_SLEEP flag is set, i.e. remove MAY_BACKLOG flag. We split the fix in two parts - first is sent to -stable, while the second is not (since there is no known failure case). Link: http://lkml.kernel.org/g/20170602122446.2427-1-david@sigma-star.at Cc: # 4.8+ Reported-by: David Gstir Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 398807d1b77e..4ecf92e3b404 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1475,8 +1475,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; struct ablkcipher_edesc *edesc; From 019d62db54017f4639fd7d4f6592f5a116a16695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Mon, 19 Jun 2017 11:44:46 +0300 Subject: [PATCH 135/140] crypto: caam - fix gfp allocation flags (part II) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the 2nd part of fixing the usage of GFP_KERNEL for memory allocations, taking care off all the places that haven't caused a real problem / failure. Again, the issue being fixed is that GFP_KERNEL should be used only when MAY_SLEEP flag is set, i.e. MAY_BACKLOG flag usage is orthogonal. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 7 +++---- drivers/crypto/caam/caamalg_qi.c | 10 ++++------ drivers/crypto/caam/caamhash.c | 32 ++++++++++++++++---------------- drivers/crypto/caam/caampkc.c | 4 ++-- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 4ecf92e3b404..fde399c88779 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1187,8 +1187,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; struct aead_edesc *edesc; int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; @@ -1680,8 +1680,7 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; struct ablkcipher_edesc *edesc; diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index ea0e5b8b9171..78c4c0485c58 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -555,8 +555,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), typeof(*alg), aead); struct device *qidev = ctx->qidev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; struct aead_edesc *edesc; dma_addr_t qm_sg_dma, iv_dma = 0; @@ -808,8 +808,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); struct device *qidev = ctx->qidev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; struct ablkcipher_edesc *edesc; @@ -953,8 +952,7 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); struct device *qidev = ctx->qidev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; struct ablkcipher_edesc *edesc; diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index da4f94eab3da..7c44c90ad593 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -719,8 +719,8 @@ static int ahash_update_ctx(struct ahash_request *req) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; u8 *buf = current_buf(state); int *buflen = current_buflen(state); u8 *next_buf = alt_buf(state); @@ -849,8 +849,8 @@ static int ahash_final_ctx(struct ahash_request *req) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; int buflen = *current_buflen(state); u32 *desc; int sec4_sg_bytes, sec4_sg_src_index; @@ -926,8 +926,8 @@ static int ahash_finup_ctx(struct ahash_request *req) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; int buflen = *current_buflen(state); u32 *desc; int sec4_sg_src_index; @@ -1013,8 +1013,8 @@ static int ahash_digest(struct ahash_request *req) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; u32 *desc; int digestsize = crypto_ahash_digestsize(ahash); int src_nents, mapped_nents; @@ -1093,8 +1093,8 @@ static int ahash_final_no_ctx(struct ahash_request *req) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; u8 *buf = current_buf(state); int buflen = *current_buflen(state); u32 *desc; @@ -1154,8 +1154,8 @@ static int ahash_update_no_ctx(struct ahash_request *req) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; u8 *buf = current_buf(state); int *buflen = current_buflen(state); u8 *next_buf = alt_buf(state); @@ -1280,8 +1280,8 @@ static int ahash_finup_no_ctx(struct ahash_request *req) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; int buflen = *current_buflen(state); u32 *desc; int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents; @@ -1370,8 +1370,8 @@ static int ahash_update_first(struct ahash_request *req) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; u8 *next_buf = alt_buf(state); int *next_buflen = alt_buflen(state); int to_hash; diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index 49cbdcba7883..67df644bb96c 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -97,8 +97,8 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct device *dev = ctx->dev; struct rsa_edesc *edesc; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; int sgc; int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; int src_nents, dst_nents; From 0f89b39bc60a8ea42e9e4b0f2e80659e7b22b003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20T=C3=A9nart?= Date: Wed, 24 May 2017 16:10:31 +0200 Subject: [PATCH 136/140] Documentation/bindings: Document the SafeXel cryptographic engine driver The Inside Secure Safexcel cryptographic engine is found on some Marvell SoCs (7k/8k). Document the bindings used by its driver. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- .../crypto/inside-secure-safexcel.txt | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt diff --git a/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt b/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt new file mode 100644 index 000000000000..f69773f4252b --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt @@ -0,0 +1,29 @@ +Inside Secure SafeXcel cryptographic engine + +Required properties: +- compatible: Should be "inside-secure,safexcel-eip197". +- reg: Base physical address of the engine and length of memory mapped region. +- interrupts: Interrupt numbers for the rings and engine. +- interrupt-names: Should be "ring0", "ring1", "ring2", "ring3", "eip", "mem". + +Optional properties: +- clocks: Reference to the crypto engine clock. +- dma-mask: The address mask limitation. Defaults to 64. + +Example: + + crypto: crypto@800000 { + compatible = "inside-secure,safexcel-eip197"; + reg = <0x800000 0x200000>; + interrupts = , + , + , + , + , + ; + interrupt-names = "mem", "ring0", "ring1", "ring2", "ring3", + "eip"; + clocks = <&cpm_syscon0 1 26>; + dma-mask = <0xff 0xffffffff>; + status = "disabled"; + }; From 8bd1d400f6a4d1fdcf79c64012602518781573e5 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 18 Jun 2017 23:53:41 -0700 Subject: [PATCH 137/140] crypto: doc - fix typo in docs Signed-off-by: Benjamin Peterson Signed-off-by: Herbert Xu --- Documentation/crypto/userspace-if.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/crypto/userspace-if.rst b/Documentation/crypto/userspace-if.rst index de5a72e32bc9..ff86befa61e0 100644 --- a/Documentation/crypto/userspace-if.rst +++ b/Documentation/crypto/userspace-if.rst @@ -327,7 +327,7 @@ boundary. Non-aligned data can be used as well, but may require more operations of the kernel which would defeat the speed gains obtained from the zero-copy interface. -The system-interent limit for the size of one zero-copy operation is 16 +The system-inherent limit for the size of one zero-copy operation is 16 pages. If more data is to be sent to AF_ALG, user space must slice the input into segments with a maximum size of 16 pages. From f2339eb9b9b01c6728ed36c274ca381bc6c452c0 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 19 Jun 2017 13:51:25 +0300 Subject: [PATCH 138/140] crypto: cavium/nitrox - dma_mapping_error() returns bool We want to return negative error codes here, but we're accidentally propogating the "true" return from dma_mapping_error(). Fixes: 14fa93cdcd9b ("crypto: cavium - Add support for CNN55XX adapters.") Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/nitrox_reqmgr.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c index b6bd2a870028..4bb4377c5ac0 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -199,9 +199,10 @@ static int dma_map_inbufs(struct nitrox_softreq *sr, sr->in.sglist = glist; /* map IV */ dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL); - ret = dma_mapping_error(dev, dma); - if (ret) + if (dma_mapping_error(dev, dma)) { + ret = -EINVAL; goto iv_map_err; + } sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; /* map src entries */ @@ -268,16 +269,18 @@ static int dma_map_outbufs(struct nitrox_softreq *sr, /* map ORH */ sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN, sr->out.dir); - ret = dma_mapping_error(dev, sr->resp.orh_dma); - if (ret) + if (dma_mapping_error(dev, sr->resp.orh_dma)) { + ret = -EINVAL; goto orh_map_err; + } /* map completion */ sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion, COMP_HLEN, sr->out.dir); - ret = dma_mapping_error(dev, sr->resp.completion_dma); - if (ret) + if (dma_mapping_error(dev, sr->resp.completion_dma)) { + ret = -EINVAL; goto compl_map_err; + } sr->inplace = (req->src == req->dst) ? true : false; /* out place */ From 30b4c54ccdebfc5a0210d9a4d77cc3671c9d1576 Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Mon, 19 Jun 2017 12:31:17 -0500 Subject: [PATCH 139/140] crypto: ccp - Release locks before returning krobot warning: make sure that all error return paths release locks. Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-debugfs.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/ccp/ccp-debugfs.c b/drivers/crypto/ccp/ccp-debugfs.c index 6d86693b117f..3cd6c83754e0 100644 --- a/drivers/crypto/ccp/ccp-debugfs.c +++ b/drivers/crypto/ccp/ccp-debugfs.c @@ -297,12 +297,11 @@ void ccp5_debugfs_setup(struct ccp_device *ccp) return; write_lock_irqsave(&ccp_debugfs_lock, flags); - if (!ccp_debugfs_dir) { + if (!ccp_debugfs_dir) ccp_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); - if (!ccp_debugfs_dir) - return; - } write_unlock_irqrestore(&ccp_debugfs_lock, flags); + if (!ccp_debugfs_dir) + return; ccp->debugfs_instance = debugfs_create_dir(ccp->name, ccp_debugfs_dir); if (!ccp->debugfs_instance) From bcf741cb779283081db47853264cc94854e7ad83 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 28 Jun 2017 19:09:07 +0800 Subject: [PATCH 140/140] crypto: testmgr - Reenable sha1/aes in FIPS mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The combination of sha1 and aes was disabled in FIPS Mode accidentally. This patch reenables it. Fixes: 284a0f6e87b0 ("crypto: testmgr - Disable fips-allowed for...") Signed-off-by: Herbert Xu Acked-by: Stephan Müller --- crypto/testmgr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 32087a7acbfb..7125ba3880af 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -2366,6 +2366,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "authenc(hmac(sha1),cbc(aes))", .test = alg_test_aead, + .fips_allowed = 1, .suite = { .aead = { .enc = __VECS(hmac_sha1_aes_cbc_enc_tv_temp)