From 933224985a0abdca8d60f7c4626068fbdead949f Mon Sep 17 00:00:00 2001 From: Xin Zeng Date: Mon, 29 Apr 2019 16:43:15 +0100 Subject: [PATCH 001/143] crypto: qat - remove spin_lock in qat_ablkcipher_setkey Remove unnecessary spin lock in qat_ablkcipher_setkey. Reviewed-by: Conor Mcloughlin Tested-by: Sergey Portnoy Signed-off-by: Xin Zeng Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index c8d401646902..413e05e8891e 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -131,7 +131,6 @@ struct qat_alg_ablkcipher_ctx { struct icp_qat_fw_la_bulk_req dec_fw_req; struct qat_crypto_instance *inst; struct crypto_tfm *tfm; - spinlock_t lock; /* protects qat_alg_ablkcipher_ctx struct */ }; static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) @@ -912,7 +911,6 @@ static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct device *dev; - spin_lock(&ctx->lock); if (ctx->enc_cd) { /* rekeying */ dev = &GET_DEV(ctx->inst->accel_dev); @@ -925,29 +923,22 @@ static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, int node = get_current_node(); struct qat_crypto_instance *inst = qat_crypto_get_instance_node(node); - if (!inst) { - spin_unlock(&ctx->lock); + if (!inst) return -EINVAL; - } dev = &GET_DEV(inst->accel_dev); ctx->inst = inst; ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), &ctx->enc_cd_paddr, GFP_ATOMIC); - if (!ctx->enc_cd) { - spin_unlock(&ctx->lock); + if (!ctx->enc_cd) return -ENOMEM; - } ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), &ctx->dec_cd_paddr, GFP_ATOMIC); - if (!ctx->dec_cd) { - spin_unlock(&ctx->lock); + if (!ctx->dec_cd) goto out_free_enc; - } } - spin_unlock(&ctx->lock); if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode)) goto out_free_all; @@ -1119,7 +1110,6 @@ static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm) { struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); - spin_lock_init(&ctx->lock); tfm->crt_ablkcipher.reqsize = sizeof(struct qat_crypto_request); ctx->tfm = tfm; return 0; From c044b62c3679fb2814fd4684fd8bb51ba38d6f3c Mon Sep 17 00:00:00 2001 From: Shant KumarX Sonnad Date: Mon, 29 Apr 2019 16:43:16 +0100 Subject: [PATCH 002/143] crypto: qat - add check for negative offset in alg precompute function The offset is calculated based on type of hash algorithum. If the algorithum is invalid the offset can have negative value. Hence added negative offset check and return -EFAULT. Reviewed-by: Conor Mcloughlin Tested-by: Sergey Portnoy Signed-off-by: Shant KumarX Sonnad Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 413e05e8891e..b60156d987eb 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -222,6 +222,9 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, return -EFAULT; offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8); + if (offset < 0) + return -EFAULT; + hash_state_out = (__be32 *)(hash->sha.state1 + offset); hash512_state_out = (__be64 *)hash_state_out; From 15b5e9112c156b66176751302584b73f799c2383 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 29 Apr 2019 16:43:17 +0100 Subject: [PATCH 003/143] crypto: qat - update iv after encryption or decryption operations Allocate a contiguous buffer and instruct the qat hardware to return the iv at the end of an encryption or decryption operation. The iv is copied to the array provided by the user in the callback function. This problem was found with by the crypto self test. Reviewed-by: Conor Mcloughlin Tested-by: Sergey Portnoy Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 71 ++++++++++++++++++---- drivers/crypto/qat/qat_common/qat_crypto.h | 2 + 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index b60156d987eb..6be3e7413beb 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -255,7 +255,24 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, return 0; } -static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) +static void qat_alg_init_hdr_iv_updt(struct icp_qat_fw_comn_req_hdr *header) +{ + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, + ICP_QAT_FW_CIPH_IV_64BIT_PTR); + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_UPDATE_STATE); +} + +static void qat_alg_init_hdr_no_iv_updt(struct icp_qat_fw_comn_req_hdr *header) +{ + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, + ICP_QAT_FW_CIPH_IV_16BYTE_DATA); + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_UPDATE_STATE); +} + +static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, + int aead) { header->hdr_flags = ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); @@ -265,12 +282,12 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) QAT_COMN_PTR_TYPE_SGL); ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, ICP_QAT_FW_LA_PARTIAL_NONE); - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, - ICP_QAT_FW_CIPH_IV_16BYTE_DATA); + if (aead) + qat_alg_init_hdr_no_iv_updt(header); + else + qat_alg_init_hdr_iv_updt(header); ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_PROTO); - ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_UPDATE_STATE); } static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, @@ -305,7 +322,7 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, return -EFAULT; /* Request setup */ - qat_alg_init_common_hdr(header); + qat_alg_init_common_hdr(header, 1); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); @@ -392,7 +409,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, return -EFAULT; /* Request setup */ - qat_alg_init_common_hdr(header); + qat_alg_init_common_hdr(header, 1); header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); @@ -456,7 +473,7 @@ static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx, struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; memcpy(cd->aes.key, key, keylen); - qat_alg_init_common_hdr(header); + qat_alg_init_common_hdr(header, 0); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; cd_pars->u.s.content_desc_params_sz = sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; @@ -803,11 +820,17 @@ static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, struct qat_crypto_instance *inst = ctx->inst; struct ablkcipher_request *areq = qat_req->ablkcipher_req; uint8_t stat_filed = qat_resp->comn_resp.comn_status; + struct device *dev = &GET_DEV(ctx->inst->accel_dev); int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); qat_alg_free_bufl(inst, qat_req); if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) res = -EINVAL; + + memcpy(areq->info, qat_req->iv, AES_BLOCK_SIZE); + dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, + qat_req->iv_paddr); + areq->base.complete(&areq->base, res); } @@ -989,11 +1012,20 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_bulk_req *msg; + struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; + qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, + &qat_req->iv_paddr, GFP_ATOMIC); + if (!qat_req->iv) + return -ENOMEM; + ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); - if (unlikely(ret)) + if (unlikely(ret)) { + dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, + qat_req->iv_paddr); return ret; + } msg = &qat_req->req; *msg = ctx->enc_fw_req; @@ -1006,13 +1038,16 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->nbytes; cipher_param->cipher_offset = 0; - memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE); + cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr; + memcpy(qat_req->iv, req->info, AES_BLOCK_SIZE); do { ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); } while (ret == -EAGAIN && ctr++ < 10); if (ret == -EAGAIN) { qat_alg_free_bufl(ctx->inst, qat_req); + dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, + qat_req->iv_paddr); return -EBUSY; } return -EINPROGRESS; @@ -1026,11 +1061,20 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_bulk_req *msg; + struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; + qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, + &qat_req->iv_paddr, GFP_ATOMIC); + if (!qat_req->iv) + return -ENOMEM; + ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); - if (unlikely(ret)) + if (unlikely(ret)) { + dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, + qat_req->iv_paddr); return ret; + } msg = &qat_req->req; *msg = ctx->dec_fw_req; @@ -1043,13 +1087,16 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->nbytes; cipher_param->cipher_offset = 0; - memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE); + cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr; + memcpy(qat_req->iv, req->info, AES_BLOCK_SIZE); do { ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); } while (ret == -EAGAIN && ctr++ < 10); if (ret == -EAGAIN) { qat_alg_free_bufl(ctx->inst, qat_req); + dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, + qat_req->iv_paddr); return -EBUSY; } return -EINPROGRESS; diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index dc0273fe3620..c77a80020cde 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -88,6 +88,8 @@ struct qat_crypto_request { struct qat_crypto_request_buffs buf; void (*cb)(struct icp_qat_fw_la_resp *resp, struct qat_crypto_request *req); + void *iv; + dma_addr_t iv_paddr; }; #endif From 51d33c2f05edeace67e6f7f0603dc7aa4667e9a6 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 29 Apr 2019 16:43:18 +0100 Subject: [PATCH 004/143] crypto: qat - fix block size for aes ctr mode The block size for aes counter mode was improperly set to AES_BLOCK_SIZE. This sets it to 1 as it is a stream cipher. This problem was found with by the new extra run-time crypto self test. Reviewed-by: Conor Mcloughlin Tested-by: Sergey Portnoy Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 6be3e7413beb..5ca5cf9f6be5 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -1273,7 +1273,7 @@ static struct crypto_alg qat_algs[] = { { .cra_driver_name = "qat_aes_ctr", .cra_priority = 4001, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, From 92fec16d1f4d852711511844b02f5353825d1578 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 29 Apr 2019 16:43:19 +0100 Subject: [PATCH 005/143] crypto: qat - return proper error code in setkey If an invalid key is provided as input to the setkey function, the function always failed returning -ENOMEM rather than -EINVAL. Furthermore, if setkey was called multiple times with an invalid key, the device instance was getting leaked. This patch fixes the error paths in the setkey functions by returning the correct error code in case of error and freeing all the resources allocated in this function in case of failure. This problem was found with by the new extra run-time crypto self test. Reviewed-by: Conor Mcloughlin Tested-by: Sergey Portnoy Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 173 ++++++++++++++--------- 1 file changed, 108 insertions(+), 65 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 5ca5cf9f6be5..f9a46918c9d1 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -595,45 +595,52 @@ bad_key: return -EINVAL; } -static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, +static int qat_alg_aead_rekey(struct crypto_aead *tfm, const uint8_t *key, + unsigned int keylen) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); + memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); + + return qat_alg_aead_init_sessions(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CBC_MODE); +} + +static int qat_alg_aead_newkey(struct crypto_aead *tfm, const uint8_t *key, unsigned int keylen) { struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct qat_crypto_instance *inst = NULL; + int node = get_current_node(); struct device *dev; + int ret; - if (ctx->enc_cd) { - /* rekeying */ - dev = &GET_DEV(ctx->inst->accel_dev); - memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); - memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); - memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); - memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); - } else { - /* new key */ - int node = get_current_node(); - struct qat_crypto_instance *inst = - qat_crypto_get_instance_node(node); - if (!inst) { - return -EINVAL; - } - - dev = &GET_DEV(inst->accel_dev); - ctx->inst = inst; - ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), - &ctx->enc_cd_paddr, - GFP_ATOMIC); - if (!ctx->enc_cd) { - return -ENOMEM; - } - ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), - &ctx->dec_cd_paddr, - GFP_ATOMIC); - if (!ctx->dec_cd) { - goto out_free_enc; - } + inst = qat_crypto_get_instance_node(node); + if (!inst) + return -EINVAL; + dev = &GET_DEV(inst->accel_dev); + ctx->inst = inst; + ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), + &ctx->enc_cd_paddr, + GFP_ATOMIC); + if (!ctx->enc_cd) { + ret = -ENOMEM; + goto out_free_inst; } - if (qat_alg_aead_init_sessions(tfm, key, keylen, - ICP_QAT_HW_CIPHER_CBC_MODE)) + ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), + &ctx->dec_cd_paddr, + GFP_ATOMIC); + if (!ctx->dec_cd) { + ret = -ENOMEM; + goto out_free_enc; + } + + ret = qat_alg_aead_init_sessions(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CBC_MODE); + if (ret) goto out_free_all; return 0; @@ -648,7 +655,21 @@ out_free_enc: dma_free_coherent(dev, sizeof(struct qat_alg_cd), ctx->enc_cd, ctx->enc_cd_paddr); ctx->enc_cd = NULL; - return -ENOMEM; +out_free_inst: + ctx->inst = NULL; + qat_crypto_put_instance(inst); + return ret; +} + +static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, + unsigned int keylen) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + + if (ctx->enc_cd) + return qat_alg_aead_rekey(tfm, key, keylen); + else + return qat_alg_aead_newkey(tfm, key, keylen); } static void qat_alg_free_bufl(struct qat_crypto_instance *inst, @@ -930,42 +951,49 @@ static int qat_alg_aead_enc(struct aead_request *areq) return -EINPROGRESS; } -static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, +static int qat_alg_ablkcipher_rekey(struct qat_alg_ablkcipher_ctx *ctx, + const u8 *key, unsigned int keylen, + int mode) +{ + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); + memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); + + return qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode); +} + +static int qat_alg_ablkcipher_newkey(struct qat_alg_ablkcipher_ctx *ctx, const u8 *key, unsigned int keylen, int mode) { - struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct qat_crypto_instance *inst = NULL; struct device *dev; + int node = get_current_node(); + int ret; - if (ctx->enc_cd) { - /* rekeying */ - dev = &GET_DEV(ctx->inst->accel_dev); - memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); - memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); - memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); - memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); - } else { - /* new key */ - int node = get_current_node(); - struct qat_crypto_instance *inst = - qat_crypto_get_instance_node(node); - if (!inst) - return -EINVAL; - - dev = &GET_DEV(inst->accel_dev); - ctx->inst = inst; - ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), - &ctx->enc_cd_paddr, - GFP_ATOMIC); - if (!ctx->enc_cd) - return -ENOMEM; - ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), - &ctx->dec_cd_paddr, - GFP_ATOMIC); - if (!ctx->dec_cd) - goto out_free_enc; + inst = qat_crypto_get_instance_node(node); + if (!inst) + return -EINVAL; + dev = &GET_DEV(inst->accel_dev); + ctx->inst = inst; + ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), + &ctx->enc_cd_paddr, + GFP_ATOMIC); + if (!ctx->enc_cd) { + ret = -ENOMEM; + goto out_free_instance; } - if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode)) + ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), + &ctx->dec_cd_paddr, + GFP_ATOMIC); + if (!ctx->dec_cd) { + ret = -ENOMEM; + goto out_free_enc; + } + + ret = qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode); + if (ret) goto out_free_all; return 0; @@ -980,7 +1008,22 @@ out_free_enc: dma_free_coherent(dev, sizeof(*ctx->enc_cd), ctx->enc_cd, ctx->enc_cd_paddr); ctx->enc_cd = NULL; - return -ENOMEM; +out_free_instance: + ctx->inst = NULL; + qat_crypto_put_instance(inst); + return ret; +} + +static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, + const u8 *key, unsigned int keylen, + int mode) +{ + struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + if (ctx->enc_cd) + return qat_alg_ablkcipher_rekey(ctx, key, keylen, mode); + else + return qat_alg_ablkcipher_newkey(ctx, key, keylen, mode); } static int qat_alg_ablkcipher_cbc_setkey(struct crypto_ablkcipher *tfm, From 96ee111a659e0f0bdfc48402b9d4162e14016ff1 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 29 Apr 2019 16:43:20 +0100 Subject: [PATCH 006/143] crypto: qat - return error for block ciphers for invalid requests Return -EINVAL if a request for a block cipher is not multiple of the size of the block. This problem was found with by the new extra run-time crypto self test. Reviewed-by: Conor Mcloughlin Tested-by: Sergey Portnoy Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index f9a46918c9d1..868fefa9bb65 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -1096,6 +1096,14 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) return -EINPROGRESS; } +static int qat_alg_ablkcipher_blk_encrypt(struct ablkcipher_request *req) +{ + if (req->nbytes % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return qat_alg_ablkcipher_encrypt(req); +} + static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); @@ -1145,6 +1153,13 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) return -EINPROGRESS; } +static int qat_alg_ablkcipher_blk_decrypt(struct ablkcipher_request *req) +{ + if (req->nbytes % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return qat_alg_ablkcipher_decrypt(req); +} static int qat_alg_aead_init(struct crypto_aead *tfm, enum icp_qat_hw_auth_algo hash, const char *hash_name) @@ -1304,8 +1319,8 @@ static struct crypto_alg qat_algs[] = { { .cra_u = { .ablkcipher = { .setkey = qat_alg_ablkcipher_cbc_setkey, - .decrypt = qat_alg_ablkcipher_decrypt, - .encrypt = qat_alg_ablkcipher_encrypt, + .decrypt = qat_alg_ablkcipher_blk_decrypt, + .encrypt = qat_alg_ablkcipher_blk_encrypt, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, @@ -1348,8 +1363,8 @@ static struct crypto_alg qat_algs[] = { { .cra_u = { .ablkcipher = { .setkey = qat_alg_ablkcipher_xts_setkey, - .decrypt = qat_alg_ablkcipher_decrypt, - .encrypt = qat_alg_ablkcipher_encrypt, + .decrypt = qat_alg_ablkcipher_blk_decrypt, + .encrypt = qat_alg_ablkcipher_blk_encrypt, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, From a3af11399aa28de0779a187f1a30504129ad754a Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 29 Apr 2019 16:43:21 +0100 Subject: [PATCH 007/143] crypto: qat - do not offload zero length requests If a zero length request is submitted through the skcipher api, do not offload it and return success. Reviewed-by: Conor Mcloughlin Tested-by: Sergey Portnoy Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 868fefa9bb65..2842b2cdaa90 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -1058,6 +1058,9 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; + if (req->nbytes == 0) + return 0; + qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, &qat_req->iv_paddr, GFP_ATOMIC); if (!qat_req->iv) @@ -1115,6 +1118,9 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; + if (req->nbytes == 0) + return 0; + qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, &qat_req->iv_paddr, GFP_ATOMIC); if (!qat_req->iv) From dcd9c76e5a183af4f793beb5141efcd260b8d09f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 3 May 2019 17:17:37 +0300 Subject: [PATCH 008/143] crypto: caam - avoid S/G table fetching for AEAD zero-length output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When enabling IOMMU support, the following issue becomes visible in the AEAD zero-length case. Even though the output sequence length is set to zero, the crypto engine tries to prefetch 4 S/G table entries (since SGF bit is set in SEQ OUT PTR command - which is either generated in SW in case of caam/jr or in HW in case of caam/qi, caam/qi2). The DMA read operation will trigger an IOMMU fault since the address in the SEQ OUT PTR is "dummy" (set to zero / not obtained via DMA API mapping). 1. In case of caam/jr, avoid the IOMMU fault by clearing the SGF bit in SEQ OUT PTR command. 2. In case of caam/qi - setting address, bpid, length to zero for output entry in the compound frame has a special meaning (cf. CAAM RM): "Output frame = Unspecified, Input address = Y. A unspecified frame is indicated by an unused SGT entry (an entry in which the Address, Length, and BPID fields are all zero). SEC obtains output buffers from BMan as prescribed by the preheader." Since no output buffers are needed, modify the preheader by setting (ABS = 1, ADDBUF = 0): -"ABS = 1 means obtain the number of buffers in ADDBUF (0 or 1) from the pool POOL ID" -ADDBUF: "If ABS is set, ADD BUF specifies whether to allocate a buffer or not" 3. In case of caam/qi2, since engine: -does not support FLE[FMT]=2'b11 ("unused" entry) mentioned in DPAA2 RM -requires output entry to be present, even if not used the solution chosen is to leave output frame list entry zeroized. Fixes: 763069ba49d3 ("crypto: caam - handle zero-length AEAD output") Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 1 + drivers/crypto/caam/caamalg_qi.c | 2 +- drivers/crypto/caam/caamalg_qi2.c | 9 +++++++++ drivers/crypto/caam/qi.c | 3 +++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index c0ece44f303b..df416e6c1468 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1106,6 +1106,7 @@ static void init_aead_job(struct aead_request *req, if (unlikely(req->src != req->dst)) { if (!edesc->mapped_dst_nents) { dst_dma = 0; + out_options = 0; } else if (edesc->mapped_dst_nents == 1) { dst_dma = sg_dma_address(req->dst); out_options = 0; diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index d290d6b41825..116cbc81fa8d 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -1109,7 +1109,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + (1 + !!ivsize) * sizeof(*sg_table), out_len, 0); - } else if (mapped_dst_nents == 1) { + } else if (mapped_dst_nents <= 1) { dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, 0); } else { diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 2b2980a8a9b9..b949944c8e55 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -559,6 +559,14 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dpaa2_fl_set_addr(out_fle, qm_sg_dma + (1 + !!ivsize) * sizeof(*sg_table)); } + } else if (!mapped_dst_nents) { + /* + * crypto engine requires the output entry to be present when + * "frame list" FD is used. + * Since engine does not support FMT=2'b11 (unused entry type), + * leaving out_fle zeroized is the best option. + */ + goto skip_out_fle; } else if (mapped_dst_nents == 1) { dpaa2_fl_set_format(out_fle, dpaa2_fl_single); dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); @@ -570,6 +578,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dpaa2_fl_set_len(out_fle, out_len); +skip_out_fle: return edesc; } diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 9f08f84cca59..2d9b0485141f 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -18,6 +18,7 @@ #include "desc_constr.h" #define PREHDR_RSLS_SHIFT 31 +#define PREHDR_ABS BIT(25) /* * Use a reasonable backlog of frames (per CPU) as congestion threshold, @@ -346,6 +347,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) */ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | num_words); + drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS); memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); dma_sync_single_for_device(qidev, drv_ctx->context_a, sizeof(drv_ctx->sh_desc) + @@ -401,6 +403,7 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, */ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | num_words); + drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS); memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc); hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size, From a5e5c13398f353bb7ebbe913a7bb0c2a77b2ae10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 3 May 2019 17:17:38 +0300 Subject: [PATCH 009/143] crypto: caam - fix S/G table passing page boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit According to CAAM RM: -crypto engine reads 4 S/G entries (64 bytes) at a time, even if the S/G table has fewer entries -it's the responsibility of the user / programmer to make sure this HW behaviour has no side effect The drivers do not take care of this currently, leading to IOMMU faults when the S/G table ends close to a page boundary - since only one page is DMA mapped, while CAAM's DMA engine accesses two pages. Fix this by rounding up the number of allocated S/G table entries to a multiple of 4. Note that in case of two *contiguous* S/G tables, only the last table might needs extra entries. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 30 ++++++++++++++- drivers/crypto/caam/caamalg_qi.c | 40 ++++++++++++++++++-- drivers/crypto/caam/caamalg_qi2.c | 63 ++++++++++++++++++++++++------- drivers/crypto/caam/caamhash.c | 33 ++++++++-------- drivers/crypto/caam/caampkc.c | 7 +++- drivers/crypto/caam/desc_constr.h | 11 ++++++ 6 files changed, 147 insertions(+), 37 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index df416e6c1468..9f3028c72953 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1381,8 +1381,16 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, } } + /* + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. + */ sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; - sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + if (mapped_dst_nents > 1) + sec4_sg_len += pad_sg_nents(mapped_dst_nents); + else + sec4_sg_len = pad_sg_nents(sec4_sg_len); + sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ @@ -1720,7 +1728,25 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, else sec4_sg_ents = mapped_src_nents + !!ivsize; dst_sg_idx = sec4_sg_ents; - sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + + /* + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed + */ + if (mapped_dst_nents > 1) + sec4_sg_ents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + sec4_sg_ents = max(pad_sg_nents(sec4_sg_ents), + !!ivsize + pad_sg_nents(mapped_src_nents)); + else + sec4_sg_ents = pad_sg_nents(sec4_sg_ents); + sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); /* diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 116cbc81fa8d..4ccaa4b7936c 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -4,7 +4,7 @@ * Based on caamalg.c * * Copyright 2013-2016 Freescale Semiconductor, Inc. - * Copyright 2016-2018 NXP + * Copyright 2016-2019 NXP */ #include "compat.h" @@ -1019,9 +1019,24 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, /* * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. * Input is not contiguous. + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed */ - qm_sg_ents = 1 + !!ivsize + mapped_src_nents + - (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + qm_sg_ents = 1 + !!ivsize + mapped_src_nents; + if (mapped_dst_nents > 1) + qm_sg_ents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + qm_sg_ents = max(pad_sg_nents(qm_sg_ents), + 1 + !!ivsize + pad_sg_nents(mapped_src_nents)); + else + qm_sg_ents = pad_sg_nents(qm_sg_ents); + sg_table = &edesc->sgt[0]; qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > @@ -1276,7 +1291,24 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, qm_sg_ents = 1 + mapped_src_nents; dst_sg_idx = qm_sg_ents; - qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + /* + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed + */ + if (mapped_dst_nents > 1) + qm_sg_ents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + qm_sg_ents = max(pad_sg_nents(qm_sg_ents), + 1 + pad_sg_nents(mapped_src_nents)); + else + qm_sg_ents = pad_sg_nents(qm_sg_ents); + qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + ivsize > CAAM_QI_MEMCACHE_SIZE)) { diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index b949944c8e55..c5d869ea95c5 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) /* * Copyright 2015-2016 Freescale Semiconductor Inc. - * Copyright 2017-2018 NXP + * Copyright 2017-2019 NXP */ #include "compat.h" @@ -460,9 +460,25 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, /* * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. * Input is not contiguous. + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed */ - qm_sg_nents = 1 + !!ivsize + mapped_src_nents + - (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + qm_sg_nents = 1 + !!ivsize + mapped_src_nents; + if (mapped_dst_nents > 1) + qm_sg_nents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + qm_sg_nents = max(pad_sg_nents(qm_sg_nents), + 1 + !!ivsize + + pad_sg_nents(mapped_src_nents)); + else + qm_sg_nents = pad_sg_nents(qm_sg_nents); + sg_table = &edesc->sgt[0]; qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > @@ -1086,7 +1102,24 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) qm_sg_ents = 1 + mapped_src_nents; dst_sg_idx = qm_sg_ents; - qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + /* + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed + */ + if (mapped_dst_nents > 1) + qm_sg_ents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + qm_sg_ents = max(pad_sg_nents(qm_sg_ents), + 1 + pad_sg_nents(mapped_src_nents)); + else + qm_sg_ents = pad_sg_nents(qm_sg_ents); + qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + ivsize > CAAM_QI_MEMCACHE_SIZE)) { @@ -3418,7 +3451,7 @@ static int ahash_update_ctx(struct ahash_request *req) edesc->src_nents = src_nents; qm_sg_src_index = 1 + (*buflen ? 1 : 0); - qm_sg_bytes = (qm_sg_src_index + mapped_nents) * + qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * sizeof(*sg_table); sg_table = &edesc->sgt[0]; @@ -3503,7 +3536,7 @@ static int ahash_final_ctx(struct ahash_request *req) gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int buflen = *current_buflen(state); - int qm_sg_bytes, qm_sg_src_index; + int qm_sg_bytes; int digestsize = crypto_ahash_digestsize(ahash); struct ahash_edesc *edesc; struct dpaa2_sg_entry *sg_table; @@ -3514,8 +3547,7 @@ static int ahash_final_ctx(struct ahash_request *req) if (!edesc) return -ENOMEM; - qm_sg_src_index = 1 + (buflen ? 1 : 0); - qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, @@ -3527,7 +3559,7 @@ static int ahash_final_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true); + dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); @@ -3608,7 +3640,8 @@ static int ahash_finup_ctx(struct ahash_request *req) edesc->src_nents = src_nents; qm_sg_src_index = 1 + (buflen ? 1 : 0); - qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * + sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, @@ -3705,7 +3738,7 @@ static int ahash_digest(struct ahash_request *req) int qm_sg_bytes; struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; - qm_sg_bytes = mapped_nents * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); @@ -3877,7 +3910,8 @@ static int ahash_update_no_ctx(struct ahash_request *req) } edesc->src_nents = src_nents; - qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * + sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); @@ -3996,7 +4030,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req) } edesc->src_nents = src_nents; - qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); @@ -4111,7 +4145,8 @@ static int ahash_update_first(struct ahash_request *req) int qm_sg_bytes; sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); - qm_sg_bytes = mapped_nents * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(mapped_nents) * + sizeof(*sg_table); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 7205d9f4029e..3710f393d1b5 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -759,7 +759,8 @@ static int ahash_edesc_add_src(struct caam_hash_ctx *ctx, if (nents > 1 || first_sg) { struct sec4_sg_entry *sg = edesc->sec4_sg; - unsigned int sgsize = sizeof(*sg) * (first_sg + nents); + unsigned int sgsize = sizeof(*sg) * + pad_sg_nents(first_sg + nents); sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0); @@ -819,6 +820,8 @@ static int ahash_update_ctx(struct ahash_request *req) } if (to_hash) { + int pad_nents; + src_nents = sg_nents_for_len(req->src, req->nbytes - (*next_buflen)); if (src_nents < 0) { @@ -838,15 +841,14 @@ static int ahash_update_ctx(struct ahash_request *req) } sec4_sg_src_index = 1 + (*buflen ? 1 : 0); - sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * - sizeof(struct sec4_sg_entry); + pad_nents = pad_sg_nents(sec4_sg_src_index + mapped_nents); + sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry); /* * allocate space for base edesc and hw desc commands, * link tables */ - edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, - ctx->sh_desc_update, + edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update, ctx->sh_desc_update_dma, flags); if (!edesc) { dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); @@ -935,18 +937,17 @@ static int ahash_final_ctx(struct ahash_request *req) GFP_KERNEL : GFP_ATOMIC; int buflen = *current_buflen(state); u32 *desc; - int sec4_sg_bytes, sec4_sg_src_index; + int sec4_sg_bytes; int digestsize = crypto_ahash_digestsize(ahash); struct ahash_edesc *edesc; int ret; - sec4_sg_src_index = 1 + (buflen ? 1 : 0); - sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry); + sec4_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * + sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index, - ctx->sh_desc_fin, ctx->sh_desc_fin_dma, - flags); + edesc = ahash_edesc_alloc(ctx, 4, ctx->sh_desc_fin, + ctx->sh_desc_fin_dma, flags); if (!edesc) return -ENOMEM; @@ -963,7 +964,7 @@ static int ahash_final_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1); + sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0)); edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); @@ -1246,6 +1247,8 @@ static int ahash_update_no_ctx(struct ahash_request *req) } if (to_hash) { + int pad_nents; + src_nents = sg_nents_for_len(req->src, req->nbytes - *next_buflen); if (src_nents < 0) { @@ -1264,14 +1267,14 @@ static int ahash_update_no_ctx(struct ahash_request *req) mapped_nents = 0; } - sec4_sg_bytes = (1 + mapped_nents) * - sizeof(struct sec4_sg_entry); + pad_nents = pad_sg_nents(1 + mapped_nents); + sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry); /* * allocate space for base edesc and hw desc commands, * link tables */ - edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents, + edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update_first, ctx->sh_desc_update_first_dma, flags); diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index fe24485274e1..d97ffb03afc0 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -3,7 +3,7 @@ * caam - Freescale FSL CAAM support for Public Key Cryptography * * Copyright 2016 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2019 NXP * * There is no Shared Descriptor for PKC so that the Job Descriptor must carry * all the desired key parameters, input and output pointers. @@ -239,8 +239,11 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, if (src_nents > 1) sec4_sg_len = src_nents; + if (dst_nents > 1) - sec4_sg_len += dst_nents; + sec4_sg_len += pad_sg_nents(dst_nents); + else + sec4_sg_len = pad_sg_nents(sec4_sg_len); sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 2980b8ef1fb1..5988a26a2441 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -3,6 +3,7 @@ * caam descriptor construction helper functions * * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019 NXP */ #ifndef DESC_CONSTR_H @@ -37,6 +38,16 @@ extern bool caam_little_end; +/* + * HW fetches 4 S/G table entries at a time, irrespective of how many entries + * are in the table. It's SW's responsibility to make sure these accesses + * do not have side effects. + */ +static inline int pad_sg_nents(int sg_nents) +{ + return ALIGN(sg_nents, 4); +} + static inline int desc_len(u32 * const desc) { return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK; From 1b46c90c8e002028dc622bf7bd8cf89efcaab274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 3 May 2019 17:17:39 +0300 Subject: [PATCH 010/143] crypto: caam - convert top level drivers to libraries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently we allow top level code, i.e. that which sits between the low level (HW-specific) drivers and crypto API, to be built as several drivers: caamalg, caamhash, caam_pkc, caamrng, caamalg_qi. There is no advantage in this, more it interferes with adding support for deferred probing (there are no corresponding devices and thus no bus). Convert these drivers and call init() / exit() manually at the right time. Move algorithms initialization at JR probe / remove time: -the first probed JR registers the crypto algs -the last removed JR unregisters the crypto algs Note: caam_qi_init() is called before JR platform devices creation (of_populate_bus()), such that QI interface is initialized when the caam/qi algorithms are registered in the JR driver (by calling caam_qi_algapi_init(). While here, fix the Kconfig entries under CRYPTO_DEV_FSL_CAAM_JR to be aligned. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/Kconfig | 46 ++++++---------- drivers/crypto/caam/Makefile | 18 +++---- drivers/crypto/caam/caamalg.c | 43 ++------------- drivers/crypto/caam/caamalg_qi.c | 45 ++-------------- drivers/crypto/caam/caamhash.c | 48 ++--------------- drivers/crypto/caam/caampkc.c | 50 ++--------------- drivers/crypto/caam/caamrng.c | 54 +++---------------- drivers/crypto/caam/ctrl.c | 45 +++++++++------- drivers/crypto/caam/intern.h | 93 ++++++++++++++++++++++++++++++-- drivers/crypto/caam/jr.c | 43 +++++++++++++++ 10 files changed, 208 insertions(+), 277 deletions(-) diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index 577c9844b322..3720ddabb507 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -2,6 +2,12 @@ config CRYPTO_DEV_FSL_CAAM_COMMON tristate +config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + tristate + +config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC + tristate + config CRYPTO_DEV_FSL_CAAM tristate "Freescale CAAM-Multicore platform driver backend" depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE @@ -25,7 +31,7 @@ config CRYPTO_DEV_FSL_CAAM_DEBUG Selecting this will enable printing of various debug information in the CAAM driver. -config CRYPTO_DEV_FSL_CAAM_JR +menuconfig CRYPTO_DEV_FSL_CAAM_JR tristate "Freescale CAAM Job Ring driver backend" default y help @@ -86,8 +92,9 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD threshold. Range is 1-65535. config CRYPTO_DEV_FSL_CAAM_CRYPTO_API - tristate "Register algorithm implementations with the Crypto API" + bool "Register algorithm implementations with the Crypto API" default y + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC select CRYPTO_AEAD select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER @@ -97,13 +104,11 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API scatterlist crypto API (such as the linux native IPSec stack) to the SEC4 via job ring. - To compile this as a module, choose M here: the module - will be called caamalg. - config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI - tristate "Queue Interface as Crypto API backend" + bool "Queue Interface as Crypto API backend" depends on FSL_DPAA && NET default y + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER help @@ -114,33 +119,26 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI assigned to the kernel should also be more than the number of job rings. - To compile this as a module, choose M here: the module - will be called caamalg_qi. - config CRYPTO_DEV_FSL_CAAM_AHASH_API - tristate "Register hash algorithm implementations with Crypto API" + bool "Register hash algorithm implementations with Crypto API" default y + select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC select CRYPTO_HASH help Selecting this will offload ahash for users of the scatterlist crypto API to the SEC4 via job ring. - To compile this as a module, choose M here: the module - will be called caamhash. - config CRYPTO_DEV_FSL_CAAM_PKC_API - tristate "Register public key cryptography implementations with Crypto API" + bool "Register public key cryptography implementations with Crypto API" default y select CRYPTO_RSA help Selecting this will allow SEC Public key support for RSA. Supported cryptographic primitives: encryption, decryption, signature and verification. - To compile this as a module, choose M here: the module - will be called caam_pkc. config CRYPTO_DEV_FSL_CAAM_RNG_API - tristate "Register caam device for hwrng API" + bool "Register caam device for hwrng API" default y select CRYPTO_RNG select HW_RANDOM @@ -148,9 +146,6 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API Selecting this will register the SEC4 hardware rng to the hw_random API for suppying the kernel entropy pool. - To compile this as a module, choose M here: the module - will be called caamrng. - endif # CRYPTO_DEV_FSL_CAAM_JR endif # CRYPTO_DEV_FSL_CAAM @@ -160,6 +155,8 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM depends on FSL_MC_DPIO depends on NETDEVICES select CRYPTO_DEV_FSL_CAAM_COMMON + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC select CRYPTO_BLKCIPHER select CRYPTO_AUTHENC select CRYPTO_AEAD @@ -171,12 +168,3 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM To compile this as a module, choose M here: the module will be called dpaa2_caam. - -config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC - def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \ - CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI || \ - CRYPTO_DEV_FSL_DPAA2_CAAM) - -config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC - def_tristate (CRYPTO_DEV_FSL_CAAM_AHASH_API || \ - CRYPTO_DEV_FSL_DPAA2_CAAM) diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index 7bbfd06a11ff..9ab4e81ea21e 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -11,20 +11,20 @@ ccflags-y += -DVERSION=\"\" obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o -caam-objs := ctrl.o -caam_jr-objs := jr.o key_gen.o -caam_pkc-y := caampkc.o pkc_desc.o +caam-y := ctrl.o +caam_jr-y := jr.o key_gen.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caampkc.o pkc_desc.o + +caam-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += qi.o ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),) ccflags-y += -DCONFIG_CAAM_QI - caam-objs += qi.o endif obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 9f3028c72953..a5a4b5218efa 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -3471,7 +3471,7 @@ static void caam_aead_exit(struct crypto_aead *tfm) caam_exit_common(crypto_aead_ctx(tfm)); } -static void __exit caam_algapi_exit(void) +void caam_algapi_exit(void) { int i; @@ -3516,43 +3516,15 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg) alg->exit = caam_aead_exit; } -static int __init caam_algapi_init(void) +int caam_algapi_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); int i = 0, err = 0; u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst; u32 arc4_inst; unsigned int md_limit = SHA512_DIGEST_SIZE; bool registered = false, gcm_support; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - priv = dev_get_drvdata(&pdev->dev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - - /* * Register crypto algorithms the device supports. * First, detect presence and attributes of DES, AES, and MD blocks. @@ -3695,14 +3667,5 @@ static int __init caam_algapi_init(void) if (registered) pr_info("caam algorithms registered in /proc/crypto\n"); -out_put_dev: - put_device(&pdev->dev); return err; } - -module_init(caam_algapi_init); -module_exit(caam_algapi_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("FSL CAAM support for crypto API"); -MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 4ccaa4b7936c..2737aa2875c0 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -2492,7 +2492,7 @@ static void caam_aead_exit(struct crypto_aead *tfm) caam_exit_common(crypto_aead_ctx(tfm)); } -static void __exit caam_qi_algapi_exit(void) +void caam_qi_algapi_exit(void) { int i; @@ -2537,45 +2537,17 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg) alg->exit = caam_aead_exit; } -static int __init caam_qi_algapi_init(void) +int caam_qi_algapi_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); int i = 0, err = 0; u32 aes_vid, aes_inst, des_inst, md_vid, md_inst; unsigned int md_limit = SHA512_DIGEST_SIZE; bool registered = false; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - of_node_put(dev_node); - if (!pdev) - return -ENODEV; - - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv || !priv->qi_present) { - err = -ENODEV; - goto out_put_dev; - } - if (caam_dpaa2) { dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); - err = -ENODEV; - goto out_put_dev; + return -ENODEV; } /* @@ -2688,14 +2660,5 @@ static int __init caam_qi_algapi_init(void) if (registered) dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); -out_put_dev: - put_device(ctrldev); return err; } - -module_init(caam_qi_algapi_init); -module_exit(caam_qi_algapi_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend"); -MODULE_AUTHOR("Freescale Semiconductor"); diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 3710f393d1b5..a67c62e444ca 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -1933,7 +1933,7 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm) caam_jr_free(ctx->jrdev); } -static void __exit caam_algapi_hash_exit(void) +void caam_algapi_hash_exit(void) { struct caam_hash_alg *t_alg, *n; @@ -1991,40 +1991,13 @@ caam_hash_alloc(struct caam_hash_template *template, return t_alg; } -static int __init caam_algapi_hash_init(void) +int caam_algapi_hash_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; int i = 0, err = 0; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); unsigned int md_limit = SHA512_DIGEST_SIZE; u32 md_inst, md_vid; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - priv = dev_get_drvdata(&pdev->dev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - /* * Register crypto algorithms the device supports. First, identify * presence and attributes of MD block. @@ -2045,10 +2018,8 @@ static int __init caam_algapi_hash_init(void) * Skip registration of any hashing algorithms if MD block * is not present. */ - if (!md_inst) { - err = -ENODEV; - goto out_put_dev; - } + if (!md_inst) + return -ENODEV; /* Limit digest size based on LP256 */ if (md_vid == CHA_VER_VID_MD_LP256) @@ -2105,14 +2076,5 @@ static int __init caam_algapi_hash_init(void) list_add_tail(&t_alg->entry, &hash_list); } -out_put_dev: - put_device(&pdev->dev); return err; } - -module_init(caam_algapi_hash_init); -module_exit(caam_algapi_hash_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API"); -MODULE_AUTHOR("Freescale Semiconductor - NMG"); diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index d97ffb03afc0..34e37f9bd828 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -1013,41 +1013,12 @@ static struct akcipher_alg caam_rsa = { }; /* Public Key Cryptography module initialization handler */ -static int __init caam_pkc_init(void) +int caam_pkc_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); u32 pk_inst; int err; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - /* Determine public key hardware accelerator presence. */ if (priv->era < 10) pk_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & @@ -1056,10 +1027,8 @@ static int __init caam_pkc_init(void) pk_inst = rd_reg32(&priv->ctrl->vreg.pkha) & CHA_VER_NUM_MASK; /* Do not register algorithms if PKHA is not present. */ - if (!pk_inst) { - err = -ENODEV; - goto out_put_dev; - } + if (!pk_inst) + return 0; err = crypto_register_akcipher(&caam_rsa); if (err) @@ -1068,19 +1037,10 @@ static int __init caam_pkc_init(void) else dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n"); -out_put_dev: - put_device(ctrldev); return err; } -static void __exit caam_pkc_exit(void) +void caam_pkc_exit(void) { crypto_unregister_akcipher(&caam_rsa); } - -module_init(caam_pkc_init); -module_exit(caam_pkc_exit); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_DESCRIPTION("FSL CAAM support for PKC functions of crypto API"); -MODULE_AUTHOR("Freescale Semiconductor"); diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index 95eb5402c59f..1ece4ed571a2 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -3,7 +3,7 @@ * caam - Freescale FSL CAAM support for hw_random * * Copyright 2011 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2019 NXP * * Based on caamalg.c crypto API driver. * @@ -296,47 +296,20 @@ static struct hwrng caam_rng = { .read = caam_read, }; -static void __exit caam_rng_exit(void) +void caam_rng_exit(void) { caam_jr_free(rng_ctx->jrdev); hwrng_unregister(&caam_rng); kfree(rng_ctx); } -static int __init caam_rng_init(void) +int caam_rng_init(struct device *ctrldev) { struct device *dev; - struct device_node *dev_node; - struct platform_device *pdev; - struct caam_drv_private *priv; u32 rng_inst; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); int err; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - priv = dev_get_drvdata(&pdev->dev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - /* Check for an instantiated RNG before registration */ if (priv->era < 10) rng_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & @@ -344,16 +317,13 @@ static int __init caam_rng_init(void) else rng_inst = rd_reg32(&priv->ctrl->vreg.rng) & CHA_VER_NUM_MASK; - if (!rng_inst) { - err = -ENODEV; - goto out_put_dev; - } + if (!rng_inst) + return 0; dev = caam_jr_alloc(); if (IS_ERR(dev)) { pr_err("Job Ring Device allocation for transform failed\n"); - err = PTR_ERR(dev); - goto out_put_dev; + return PTR_ERR(dev); } rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA | GFP_KERNEL); if (!rng_ctx) { @@ -364,7 +334,6 @@ static int __init caam_rng_init(void) if (err) goto free_rng_ctx; - put_device(&pdev->dev); dev_info(dev, "registering rng-caam\n"); return hwrng_register(&caam_rng); @@ -372,14 +341,5 @@ free_rng_ctx: kfree(rng_ctx); free_caam_alloc: caam_jr_free(dev); -out_put_dev: - put_device(&pdev->dev); return err; } - -module_init(caam_rng_init); -module_exit(caam_rng_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("FSL CAAM support for hw_random API"); -MODULE_AUTHOR("Freescale Semiconductor - NMG"); diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index e2ba3d202da5..36c2f15100a4 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -3,7 +3,7 @@ * Controller-level driver, kernel property detection, initialization * * Copyright 2008-2012 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2019 NXP */ #include @@ -703,12 +703,6 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->era = caam_get_era(ctrl); - ret = of_platform_populate(nprop, caam_match, NULL, dev); - if (ret) { - dev_err(dev, "JR platform devices creation error\n"); - goto iounmap_ctrl; - } - #ifdef CONFIG_DEBUG_FS /* * FIXME: needs better naming distinction, as some amalgamation of @@ -721,19 +715,6 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); #endif - ring = 0; - for_each_available_child_of_node(nprop, np) - if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || - of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { - ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *) - ((__force uint8_t *)ctrl + - (ring + JR_BLOCK_NUMBER) * - BLOCK_OFFSET - ); - ctrlpriv->total_jobrs++; - ring++; - } - /* Check to see if (DPAA 1.x) QI present. If so, enable */ ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK); if (ctrlpriv->qi_present && !caam_dpaa2) { @@ -752,6 +733,25 @@ static int caam_probe(struct platform_device *pdev) #endif } + ret = of_platform_populate(nprop, caam_match, NULL, dev); + if (ret) { + dev_err(dev, "JR platform devices creation error\n"); + goto shutdown_qi; + } + + ring = 0; + for_each_available_child_of_node(nprop, np) + if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || + of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { + ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *) + ((__force uint8_t *)ctrl + + (ring + JR_BLOCK_NUMBER) * + BLOCK_OFFSET + ); + ctrlpriv->total_jobrs++; + ring++; + } + /* If no QI and no rings specified, quit and go home */ if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) { dev_err(dev, "no queues configured, terminating\n"); @@ -898,6 +898,11 @@ caam_remove: caam_remove(pdev); return ret; +shutdown_qi: +#ifdef CONFIG_CAAM_QI + if (ctrlpriv->qidev) + caam_qi_shutdown(ctrlpriv->qidev); +#endif iounmap_ctrl: iounmap(ctrl); disable_caam_emi_slow: diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 3392615dc91b..50e24ebc533b 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -4,7 +4,7 @@ * Private/internal definitions between modules * * Copyright 2008-2011 Freescale Semiconductor, Inc. - * + * Copyright 2019 NXP */ #ifndef INTERN_H @@ -107,8 +107,95 @@ struct caam_drv_private { #endif }; -void caam_jr_algapi_init(struct device *dev); -void caam_jr_algapi_remove(struct device *dev); +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API + +int caam_algapi_init(struct device *dev); +void caam_algapi_exit(void); + +#else + +static inline int caam_algapi_init(struct device *dev) +{ + return 0; +} + +static inline void caam_algapi_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API + +int caam_algapi_hash_init(struct device *dev); +void caam_algapi_hash_exit(void); + +#else + +static inline int caam_algapi_hash_init(struct device *dev) +{ + return 0; +} + +static inline void caam_algapi_hash_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API + +int caam_pkc_init(struct device *dev); +void caam_pkc_exit(void); + +#else + +static inline int caam_pkc_init(struct device *dev) +{ + return 0; +} + +static inline void caam_pkc_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API + +int caam_rng_init(struct device *dev); +void caam_rng_exit(void); + +#else + +static inline int caam_rng_init(struct device *dev) +{ + return 0; +} + +static inline void caam_rng_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API */ + +#ifdef CONFIG_CAAM_QI + +int caam_qi_algapi_init(struct device *dev); +void caam_qi_algapi_exit(void); + +#else + +static inline int caam_qi_algapi_init(struct device *dev) +{ + return 0; +} + +static inline void caam_qi_algapi_exit(void) +{ +} + +#endif /* CONFIG_CAAM_QI */ #ifdef CONFIG_DEBUG_FS static int caam_debugfs_u64_get(void *data, u64 *val) diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 1de2562d0982..cea811fed320 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -4,6 +4,7 @@ * JobR backend functionality * * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019 NXP */ #include @@ -23,6 +24,43 @@ struct jr_driver_data { } ____cacheline_aligned; static struct jr_driver_data driver_data; +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs; + +static void register_algs(struct device *dev) +{ + mutex_lock(&algs_lock); + + if (++active_devs != 1) + goto algs_unlock; + + caam_algapi_init(dev); + caam_algapi_hash_init(dev); + caam_pkc_init(dev); + caam_rng_init(dev); + caam_qi_algapi_init(dev); + +algs_unlock: + mutex_unlock(&algs_lock); +} + +static void unregister_algs(void) +{ + mutex_lock(&algs_lock); + + if (--active_devs != 0) + goto algs_unlock; + + caam_qi_algapi_exit(); + + caam_rng_exit(); + caam_pkc_exit(); + caam_algapi_hash_exit(); + caam_algapi_exit(); + +algs_unlock: + mutex_unlock(&algs_lock); +} static int caam_reset_hw_jr(struct device *dev) { @@ -109,6 +147,9 @@ static int caam_jr_remove(struct platform_device *pdev) return -EBUSY; } + /* Unregister JR-based RNG & crypto algorithms */ + unregister_algs(); + /* Remove the node from Physical JobR list maintained by driver */ spin_lock(&driver_data.jr_alloc_lock); list_del(&jrpriv->list_node); @@ -541,6 +582,8 @@ static int caam_jr_probe(struct platform_device *pdev) atomic_set(&jrpriv->tfm_count, 0); + register_algs(jrdev->parent); + return 0; } From 6b175685b4a12ab0aa048078bd7ca1fdb20ca766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 3 May 2019 17:17:40 +0300 Subject: [PATCH 011/143] crypto: caam/qi - don't allocate an extra platform device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the controller device for caam/qi instead of allocating a new platform device. This is needed as a preparation to add support for working behind an SMMU. A platform device allocated using platform_device_register_full() is not completely set up - most importantly .dma_configure() is not called. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg_qi.c | 6 +++--- drivers/crypto/caam/ctrl.c | 8 ++++---- drivers/crypto/caam/intern.h | 7 +++---- drivers/crypto/caam/qi.c | 33 +++++--------------------------- 4 files changed, 15 insertions(+), 39 deletions(-) diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 2737aa2875c0..f30959335567 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -2443,7 +2443,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; - ctx->qidev = priv->qidev; + ctx->qidev = ctx->jrdev->parent; spin_lock_init(&ctx->lock); ctx->drv_ctx[ENCRYPT] = NULL; @@ -2602,7 +2602,7 @@ int caam_qi_algapi_init(struct device *ctrldev) err = crypto_register_skcipher(&t_alg->skcipher); if (err) { - dev_warn(priv->qidev, "%s alg registration failed\n", + dev_warn(ctrldev, "%s alg registration failed\n", t_alg->skcipher.base.cra_driver_name); continue; } @@ -2658,7 +2658,7 @@ int caam_qi_algapi_init(struct device *ctrldev) } if (registered) - dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); + dev_info(ctrldev, "algorithms registered in /proc/crypto\n"); return err; } diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 36c2f15100a4..38bcbbccdfda 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -323,8 +323,8 @@ static int caam_remove(struct platform_device *pdev) of_platform_depopulate(ctrldev); #ifdef CONFIG_CAAM_QI - if (ctrlpriv->qidev) - caam_qi_shutdown(ctrlpriv->qidev); + if (ctrlpriv->qi_init) + caam_qi_shutdown(ctrldev); #endif /* @@ -900,8 +900,8 @@ caam_remove: shutdown_qi: #ifdef CONFIG_CAAM_QI - if (ctrlpriv->qidev) - caam_qi_shutdown(ctrlpriv->qidev); + if (ctrlpriv->qi_init) + caam_qi_shutdown(dev); #endif iounmap_ctrl: iounmap(ctrl); diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 50e24ebc533b..c9089da5dbaf 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -63,10 +63,6 @@ struct caam_drv_private_jr { * Driver-private storage for a single CAAM block instance */ struct caam_drv_private { -#ifdef CONFIG_CAAM_QI - struct device *qidev; -#endif - /* Physical-presence section */ struct caam_ctrl __iomem *ctrl; /* controller region */ struct caam_deco __iomem *deco; /* DECO/CCB views */ @@ -80,6 +76,9 @@ struct caam_drv_private { */ u8 total_jobrs; /* Total Job Rings in device */ u8 qi_present; /* Nonzero if QI present in device */ +#ifdef CONFIG_CAAM_QI + u8 qi_init; /* Nonzero if QI has been initialized */ +#endif u8 mc_en; /* Nonzero if MC f/w is active */ int secvio_irq; /* Security violation interrupt number */ int virt_en; /* Virtualization enabled in CAAM */ diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 2d9b0485141f..46fca2c9fb24 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -4,7 +4,7 @@ * Queue Interface backend functionality * * Copyright 2013-2016 Freescale Semiconductor, Inc. - * Copyright 2016-2017 NXP + * Copyright 2016-2017, 2019 NXP */ #include @@ -59,11 +59,9 @@ static DEFINE_PER_CPU(int, last_cpu); /* * caam_qi_priv - CAAM QI backend private params * @cgr: QMan congestion group - * @qi_pdev: platform device for QI backend */ struct caam_qi_priv { struct qman_cgr cgr; - struct platform_device *qi_pdev; }; static struct caam_qi_priv qipriv ____cacheline_aligned; @@ -491,7 +489,7 @@ EXPORT_SYMBOL(caam_drv_ctx_rel); void caam_qi_shutdown(struct device *qidev) { int i; - struct caam_qi_priv *priv = dev_get_drvdata(qidev); + struct caam_qi_priv *priv = &qipriv; const cpumask_t *cpus = qman_affine_cpus(); for_each_cpu(i, cpus) { @@ -509,8 +507,6 @@ void caam_qi_shutdown(struct device *qidev) qman_release_cgrid(priv->cgr.cgrid); kmem_cache_destroy(qi_cache); - - platform_device_unregister(priv->qi_pdev); } static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) @@ -695,33 +691,17 @@ static void free_rsp_fqs(void) int caam_qi_init(struct platform_device *caam_pdev) { int err, i; - struct platform_device *qi_pdev; struct device *ctrldev = &caam_pdev->dev, *qidev; struct caam_drv_private *ctrlpriv; const cpumask_t *cpus = qman_affine_cpus(); - static struct platform_device_info qi_pdev_info = { - .name = "caam_qi", - .id = PLATFORM_DEVID_NONE - }; - - qi_pdev_info.parent = ctrldev; - qi_pdev_info.dma_mask = dma_get_mask(ctrldev); - qi_pdev = platform_device_register_full(&qi_pdev_info); - if (IS_ERR(qi_pdev)) - return PTR_ERR(qi_pdev); - set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev)); ctrlpriv = dev_get_drvdata(ctrldev); - qidev = &qi_pdev->dev; - - qipriv.qi_pdev = qi_pdev; - dev_set_drvdata(qidev, &qipriv); + qidev = ctrldev; /* Initialize the congestion detection */ err = init_cgr(qidev); if (err) { dev_err(qidev, "CGR initialization failed: %d\n", err); - platform_device_unregister(qi_pdev); return err; } @@ -730,7 +710,6 @@ int caam_qi_init(struct platform_device *caam_pdev) if (err) { dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err); free_rsp_fqs(); - platform_device_unregister(qi_pdev); return err; } @@ -753,15 +732,11 @@ int caam_qi_init(struct platform_device *caam_pdev) napi_enable(irqtask); } - /* Hook up QI device to parent controlling caam device */ - ctrlpriv->qidev = qidev; - qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0, SLAB_CACHE_DMA, NULL); if (!qi_cache) { dev_err(qidev, "Can't allocate CAAM cache\n"); free_rsp_fqs(); - platform_device_unregister(qi_pdev); return -ENOMEM; } @@ -769,6 +744,8 @@ int caam_qi_init(struct platform_device *caam_pdev) debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, ×_congested, &caam_fops_u64_ro); #endif + + ctrlpriv->qi_init = 1; dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n"); return 0; } From b2b2ee350e701df9086275b8f857f6db10546f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 3 May 2019 17:17:41 +0300 Subject: [PATCH 012/143] crypto: caam/qi - fix address translations with IOMMU enabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When IOMMU is enabled, iova -> phys address translation should be performed using iommu_ops, not dma_to_phys(). Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/ctrl.c | 1 + drivers/crypto/caam/intern.h | 2 ++ drivers/crypto/caam/qi.c | 16 ++++++++++++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 38bcbbccdfda..bbde6efce8af 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -702,6 +702,7 @@ static int caam_probe(struct platform_device *pdev) } ctrlpriv->era = caam_get_era(ctrl); + ctrlpriv->domain = iommu_get_domain_for_dev(dev); #ifdef CONFIG_DEBUG_FS /* diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index c9089da5dbaf..6af84bbc612c 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -70,6 +70,8 @@ struct caam_drv_private { struct caam_queue_if __iomem *qi; /* QI control region */ struct caam_job_ring __iomem *jr[4]; /* JobR's register space */ + struct iommu_domain *domain; + /* * Detected geometry block. Filled in from device tree if powerpc, * or from register-based version detection code diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 46fca2c9fb24..0fe618e3804a 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -94,6 +94,16 @@ static u64 times_congested; */ static struct kmem_cache *qi_cache; +static void *caam_iova_to_virt(struct iommu_domain *domain, + dma_addr_t iova_addr) +{ + phys_addr_t phys_addr; + + phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; + + return phys_to_virt(phys_addr); +} + int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req) { struct qm_fd fd; @@ -134,6 +144,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, const struct qm_fd *fd; struct caam_drv_req *drv_req; struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); + struct caam_drv_private *priv = dev_get_drvdata(qidev); fd = &msg->ern.fd; @@ -142,7 +153,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, return; } - drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd)); + drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd)); if (!drv_req) { dev_err(qidev, "Can't find original request for CAAM response\n"); @@ -549,6 +560,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, struct caam_drv_req *drv_req; const struct qm_fd *fd; struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); + struct caam_drv_private *priv = dev_get_drvdata(qidev); u32 status; if (caam_qi_napi_schedule(p, caam_napi)) @@ -571,7 +583,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, return qman_cb_dqrr_consume; } - drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd)); + drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd)); if (unlikely(!drv_req)) { dev_err(qidev, "Can't find original request for caam response\n"); From a7cd942bb6e3559d86257fe3b57844e55341a3a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 3 May 2019 17:17:42 +0300 Subject: [PATCH 013/143] crypto: caam/qi - DMA map keys using proper device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently there is a mismatch b/w the ICID (Isolation Context ID) used for DMA mapping keys and ICID used for accessing them. -keys are DMA mapped using a job ring device, thus a job ring ICID -keys are accessed from descriptors enqueued via Queue Interface, thus using QI ICID [Note: ICIDs of JRs, QI are configured by U-boot / other entity by: -fixing up the corresponding job ring and controller DT nodes -setting up corresponding caam ICID registers] In order to avoid IOMMU faults, DMA map the key using the controller device instead of a job ring device. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg_qi.c | 33 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index f30959335567..c6bb804d82a8 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -237,7 +237,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, memcpy(ctx->key, keys.authkey, keys.authkeylen); memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); goto skip_split_key; @@ -251,8 +251,9 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, /* postpend encryption key to auth split key */ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + - keys.enckeylen, ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->adata.keylen_pad + keys.enckeylen, + ctx->dir); #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, @@ -392,7 +393,8 @@ static int gcm_setkey(struct crypto_aead *aead, #endif memcpy(ctx->key, key, keylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen, + ctx->dir); ctx->cdata.keylen = keylen; ret = gcm_set_sh_desc(aead); @@ -496,8 +498,8 @@ static int rfc4106_setkey(struct crypto_aead *aead, * in the nonce. Update the AES key length. */ ctx->cdata.keylen = keylen - 4; - dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, - ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->cdata.keylen, ctx->dir); ret = rfc4106_set_sh_desc(aead); if (ret) @@ -600,8 +602,8 @@ static int rfc4543_setkey(struct crypto_aead *aead, * in the nonce. Update the AES key length. */ ctx->cdata.keylen = keylen - 4; - dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, - ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->cdata.keylen, ctx->dir); ret = rfc4543_set_sh_desc(aead); if (ret) @@ -2414,6 +2416,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, bool uses_dkp) { struct caam_drv_private *priv; + struct device *dev; /* * distribute tfms across job rings to ensure in-order @@ -2425,16 +2428,17 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, return PTR_ERR(ctx->jrdev); } - priv = dev_get_drvdata(ctx->jrdev->parent); + dev = ctx->jrdev->parent; + priv = dev_get_drvdata(dev); if (priv->era >= 6 && uses_dkp) ctx->dir = DMA_BIDIRECTIONAL; else ctx->dir = DMA_TO_DEVICE; - ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), + ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key), ctx->dir); - if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { - dev_err(ctx->jrdev, "unable to map key\n"); + if (dma_mapping_error(dev, ctx->key_dma)) { + dev_err(dev, "unable to map key\n"); caam_jr_free(ctx->jrdev); return -ENOMEM; } @@ -2443,7 +2447,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; - ctx->qidev = ctx->jrdev->parent; + ctx->qidev = dev; spin_lock_init(&ctx->lock); ctx->drv_ctx[ENCRYPT] = NULL; @@ -2477,7 +2481,8 @@ static void caam_exit_common(struct caam_ctx *ctx) caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); - dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir); + dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key), + ctx->dir); caam_jr_free(ctx->jrdev); } From db07cd26ac6a418dc2823187958edcfdb415fa83 Mon Sep 17 00:00:00 2001 From: Stephan Mueller Date: Wed, 8 May 2019 16:19:24 +0200 Subject: [PATCH 014/143] crypto: drbg - add FIPS 140-2 CTRNG for noise source FIPS 140-2 section 4.9.2 requires a continuous self test of the noise source. Up to kernel 4.8 drivers/char/random.c provided this continuous self test. Afterwards it was moved to a location that is inconsistent with the FIPS 140-2 requirements. The relevant patch was e192be9d9a30555aae2ca1dc3aad37cba484cd4a . Thus, the FIPS 140-2 CTRNG is added to the DRBG when it obtains the seed. This patch resurrects the function drbg_fips_continous_test that existed some time ago and applies it to the noise sources. The patch that removed the drbg_fips_continous_test was b3614763059b82c26bdd02ffcb1c016c1132aad0 . The Jitter RNG implements its own FIPS 140-2 self test and thus does not need to be subjected to the test in the DRBG. The patch contains a tiny fix to ensure proper zeroization in case of an error during the Jitter RNG data gathering. Signed-off-by: Stephan Mueller Reviewed-by: Yann Droneaud Signed-off-by: Herbert Xu --- crypto/drbg.c | 94 +++++++++++++++++++++++++++++++++++++++++-- include/crypto/drbg.h | 2 + 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index 2a5b16bb000c..b6929eb5f565 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -219,6 +219,57 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) } } +/* + * FIPS 140-2 continuous self test for the noise source + * The test is performed on the noise source input data. Thus, the function + * implicitly knows the size of the buffer to be equal to the security + * strength. + * + * Note, this function disregards the nonce trailing the entropy data during + * initial seeding. + * + * drbg->drbg_mutex must have been taken. + * + * @drbg DRBG handle + * @entropy buffer of seed data to be checked + * + * return: + * 0 on success + * -EAGAIN on when the CTRNG is not yet primed + * < 0 on error + */ +static int drbg_fips_continuous_test(struct drbg_state *drbg, + const unsigned char *entropy) +{ + unsigned short entropylen = drbg_sec_strength(drbg->core->flags); + int ret = 0; + + if (!IS_ENABLED(CONFIG_CRYPTO_FIPS)) + return 0; + + /* skip test if we test the overall system */ + if (list_empty(&drbg->test_data.list)) + return 0; + /* only perform test in FIPS mode */ + if (!fips_enabled) + return 0; + + if (!drbg->fips_primed) { + /* Priming of FIPS test */ + memcpy(drbg->prev, entropy, entropylen); + drbg->fips_primed = true; + /* priming: another round is needed */ + return -EAGAIN; + } + ret = memcmp(drbg->prev, entropy, entropylen); + if (!ret) + panic("DRBG continuous self test failed\n"); + memcpy(drbg->prev, entropy, entropylen); + + /* the test shall pass when the two values are not equal */ + return 0; +} + /* * Convert an integer into a byte representation of this integer. * The byte representation is big-endian @@ -998,6 +1049,22 @@ static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed, return ret; } +static inline int drbg_get_random_bytes(struct drbg_state *drbg, + unsigned char *entropy, + unsigned int entropylen) +{ + int ret; + + do { + get_random_bytes(entropy, entropylen); + ret = drbg_fips_continuous_test(drbg, entropy); + if (ret && ret != -EAGAIN) + return ret; + } while (ret); + + return 0; +} + static void drbg_async_seed(struct work_struct *work) { struct drbg_string data; @@ -1006,16 +1073,20 @@ static void drbg_async_seed(struct work_struct *work) seed_work); unsigned int entropylen = drbg_sec_strength(drbg->core->flags); unsigned char entropy[32]; + int ret; BUG_ON(!entropylen); BUG_ON(entropylen > sizeof(entropy)); - get_random_bytes(entropy, entropylen); drbg_string_fill(&data, entropy, entropylen); list_add_tail(&data.list, &seedlist); mutex_lock(&drbg->drbg_mutex); + ret = drbg_get_random_bytes(drbg, entropy, entropylen); + if (ret) + goto unlock; + /* If nonblocking pool is initialized, deactivate Jitter RNG */ crypto_free_rng(drbg->jent); drbg->jent = NULL; @@ -1030,6 +1101,7 @@ static void drbg_async_seed(struct work_struct *work) if (drbg->seeded) drbg->reseed_threshold = drbg_max_requests(drbg); +unlock: mutex_unlock(&drbg->drbg_mutex); memzero_explicit(entropy, entropylen); @@ -1081,7 +1153,9 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, BUG_ON((entropylen * 2) > sizeof(entropy)); /* Get seed from in-kernel /dev/urandom */ - get_random_bytes(entropy, entropylen); + ret = drbg_get_random_bytes(drbg, entropy, entropylen); + if (ret) + goto out; if (!drbg->jent) { drbg_string_fill(&data1, entropy, entropylen); @@ -1094,7 +1168,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, entropylen); if (ret) { pr_devel("DRBG: jent failed with %d\n", ret); - return ret; + goto out; } drbg_string_fill(&data1, entropy, entropylen * 2); @@ -1121,6 +1195,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, ret = __drbg_seed(drbg, &seedlist, reseed); +out: memzero_explicit(entropy, entropylen * 2); return ret; @@ -1142,6 +1217,11 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) drbg->reseed_ctr = 0; drbg->d_ops = NULL; drbg->core = NULL; + if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) { + kzfree(drbg->prev); + drbg->prev = NULL; + drbg->fips_primed = false; + } } /* @@ -1211,6 +1291,14 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) drbg->scratchpad = PTR_ALIGN(drbg->scratchpadbuf, ret + 1); } + if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) { + drbg->prev = kzalloc(drbg_sec_strength(drbg->core->flags), + GFP_KERNEL); + if (!drbg->prev) + goto fini; + drbg->fips_primed = false; + } + return 0; fini: diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h index 3fb581bf3b87..8c9af21efce1 100644 --- a/include/crypto/drbg.h +++ b/include/crypto/drbg.h @@ -129,6 +129,8 @@ struct drbg_state { bool seeded; /* DRBG fully seeded? */ bool pr; /* Prediction resistance enabled? */ + bool fips_primed; /* Continuous test primed? */ + unsigned char *prev; /* FIPS 140-2 continuous test value */ struct work_struct seed_work; /* asynchronous seeding support */ struct crypto_rng *jent; const struct drbg_state_ops *d_ops; From efc77e8107c5c0b301686d52bd11e0ec2b10cb61 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Sun, 12 May 2019 17:05:40 +0800 Subject: [PATCH 015/143] crypto: arm/sha512 - Make sha512_arm_final static Fix sparse warning: arch/arm/crypto/sha512-glue.c:40:5: warning: symbol 'sha512_arm_final' was not declared. Should it be static? Reported-by: Hulk Robot Signed-off-by: YueHaibing Signed-off-by: Herbert Xu --- arch/arm/crypto/sha512-glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c index 86540cd4a6fa..23fc3813a91b 100644 --- a/arch/arm/crypto/sha512-glue.c +++ b/arch/arm/crypto/sha512-glue.c @@ -37,7 +37,7 @@ int sha512_arm_update(struct shash_desc *desc, const u8 *data, (sha512_block_fn *)sha512_block_data_order); } -int sha512_arm_final(struct shash_desc *desc, u8 *out) +static int sha512_arm_final(struct shash_desc *desc, u8 *out) { sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_block_data_order); From 499df9674c54a733f4deb3329dfcfc5487eb7738 Mon Sep 17 00:00:00 2001 From: "Hook, Gary" Date: Tue, 14 May 2019 21:53:16 +0000 Subject: [PATCH 016/143] crypto: ccp - AES CFB mode is a stream cipher CFB mode should be treated as a stream cipher, not block. Fixes: 63b945091a07 ('crypto: ccp - CCP device driver and interface support') Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-ops.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index 267a367bd076..3ecadeab919c 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -1,7 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2013,2018 Advanced Micro Devices, Inc. + * Copyright (C) 2013-2019 Advanced Micro Devices, Inc. * * Author: Tom Lendacky * Author: Gary R Hook @@ -893,8 +894,7 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) return -EINVAL; if (((aes->mode == CCP_AES_MODE_ECB) || - (aes->mode == CCP_AES_MODE_CBC) || - (aes->mode == CCP_AES_MODE_CFB)) && + (aes->mode == CCP_AES_MODE_CBC)) && (aes->src_len & (AES_BLOCK_SIZE - 1))) return -EINVAL; From c3b359d6567c0b8f413e924feb37cf025067d55a Mon Sep 17 00:00:00 2001 From: "Hook, Gary" Date: Tue, 14 May 2019 21:53:23 +0000 Subject: [PATCH 017/143] crypto: ccp - fix AES CFB error exposed by new test vectors Updated testmgr will exhibit this error message when loading the ccp-crypto module: alg: skcipher: cfb-aes-ccp encryption failed with err -22 on test vector 3, cfg="in-place" Update the CCP crypto driver to correctly treat CFB as a streaming mode cipher (instead of block mode). Update the configuration for CFB to specify the block size as a single byte; Fixes: 2b789435d7f3 ('crypto: ccp - CCP AES crypto API support') Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-crypto-aes.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c index 89291c15015c..3f768699332b 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes.c +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -1,7 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) AES crypto API support * - * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. + * Copyright (C) 2013-2019 Advanced Micro Devices, Inc. * * Author: Tom Lendacky * @@ -79,8 +80,7 @@ static int ccp_aes_crypt(struct ablkcipher_request *req, bool encrypt) return -EINVAL; if (((ctx->u.aes.mode == CCP_AES_MODE_ECB) || - (ctx->u.aes.mode == CCP_AES_MODE_CBC) || - (ctx->u.aes.mode == CCP_AES_MODE_CFB)) && + (ctx->u.aes.mode == CCP_AES_MODE_CBC)) && (req->nbytes & (AES_BLOCK_SIZE - 1))) return -EINVAL; @@ -291,7 +291,7 @@ static struct ccp_aes_def aes_algs[] = { .version = CCP_VERSION(3, 0), .name = "cfb(aes)", .driver_name = "cfb-aes-ccp", - .blocksize = AES_BLOCK_SIZE, + .blocksize = 1, .ivsize = AES_BLOCK_SIZE, .alg_defaults = &ccp_aes_defaults, }, From 89646fdda4cae203185444ac7988835f36a21ee1 Mon Sep 17 00:00:00 2001 From: "Hook, Gary" Date: Tue, 14 May 2019 21:53:30 +0000 Subject: [PATCH 018/143] crypto: ccp - Fix 3DES complaint from ccp-crypto module Crypto self-tests reveal an error: alg: skcipher: cbc-des3-ccp encryption test failed (wrong output IV) on test vector 0, cfg="in-place" The offset value should not be recomputed when retrieving the context. Also, a code path exists which makes decisions based on older (version 3) hardware; a v3 device deosn't support 3DES so remove this check. Fixes: 990672d48515 ('crypto: ccp - Enable 3DES function on v5 CCPs') Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-ops.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index 3ecadeab919c..b116d62991c6 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -1267,6 +1267,9 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) int ret; /* Error checks */ + if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) + return -EINVAL; + if (!cmd_q->ccp->vdata->perform->des3) return -EINVAL; @@ -1349,8 +1352,6 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) * passthru option to convert from big endian to little endian. */ if (des3->mode != CCP_DES3_MODE_ECB) { - u32 load_mode; - op.sb_ctx = cmd_q->sb_ctx; ret = ccp_init_dm_workarea(&ctx, cmd_q, @@ -1366,12 +1367,8 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) if (ret) goto e_ctx; - if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) - load_mode = CCP_PASSTHRU_BYTESWAP_NOOP; - else - load_mode = CCP_PASSTHRU_BYTESWAP_256BIT; ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, - load_mode); + CCP_PASSTHRU_BYTESWAP_256BIT); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_ctx; @@ -1433,10 +1430,6 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) } /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ - if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) - dm_offset = CCP_SB_BYTES - des3->iv_len; - else - dm_offset = 0; ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, DES3_EDE_BLOCK_SIZE); } From 3e03e792865ae48b8cfc69a0b4d65f02f467389f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 15 May 2019 12:29:03 +0000 Subject: [PATCH 019/143] crypto: talitos - fix skcipher failure due to wrong output IV MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Selftests report the following: [ 2.984845] alg: skcipher: cbc-aes-talitos encryption test failed (wrong output IV) on test vector 0, cfg="in-place" [ 2.995377] 00000000: 3d af ba 42 9d 9e b4 30 b4 22 da 80 2c 9f ac 41 [ 3.032673] alg: skcipher: cbc-des-talitos encryption test failed (wrong output IV) on test vector 0, cfg="in-place" [ 3.043185] 00000000: fe dc ba 98 76 54 32 10 [ 3.063238] alg: skcipher: cbc-3des-talitos encryption test failed (wrong output IV) on test vector 0, cfg="in-place" [ 3.073818] 00000000: 7d 33 88 93 0f 93 b2 42 This above dumps show that the actual output IV is indeed the input IV. This is due to the IV not being copied back into the request. This patch fixes that. Signed-off-by: Christophe Leroy Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 1d429fc073d1..f443cbe7da80 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1637,11 +1637,15 @@ static void ablkcipher_done(struct device *dev, int err) { struct ablkcipher_request *areq = context; + struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); + unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); struct talitos_edesc *edesc; edesc = container_of(desc, struct talitos_edesc, desc); common_nonsnoop_unmap(dev, edesc, areq); + memcpy(areq->info, ctx->iv, ivsize); kfree(edesc); From bb992bc418e12d4658d39e037f610e0d03940d86 Mon Sep 17 00:00:00 2001 From: Sascha Hauer Date: Thu, 16 May 2019 16:24:42 +0200 Subject: [PATCH 020/143] crypto: caam - print debugging hex dumps after unmapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For encryption the destination pointer was still mapped, so the hex dump may be wrong. The IV still contained the input IV while printing instead of the output IV as intended. For decryption the destination pointer was still mapped, so the hex dump may be wrong. The IV dump was correct. Do the hex dumps consistenly after the buffers have been unmapped and in case of IV copied to their final destination. Signed-off-by: Sascha Hauer Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index a5a4b5218efa..08c36ae5d12d 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1010,15 +1010,6 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, if (err) caam_jr_strstatus(jrdev, err); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents > 1 ? 100 : ivsize, 1); -#endif - caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->dst, - edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); - skcipher_unmap(jrdev, edesc, req); /* @@ -1029,6 +1020,15 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize, ivsize, 0); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); +#endif + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); + kfree(edesc); skcipher_request_complete(req, err); @@ -1050,6 +1050,8 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, if (err) caam_jr_strstatus(jrdev, err); + skcipher_unmap(jrdev, edesc, req); + #ifdef DEBUG print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); @@ -1058,7 +1060,6 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); - skcipher_unmap(jrdev, edesc, req); kfree(edesc); skcipher_request_complete(req, err); From bfa2ba7d9e6b20aca82b99e6842fe18842ae3a0f Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 17 May 2019 23:15:57 +0200 Subject: [PATCH 021/143] crypto: crypto4xx - fix AES CTR blocksize value This patch fixes a issue with crypto4xx's ctr(aes) that was discovered by libcapi's kcapi-enc-test.sh test. The some of the ctr(aes) encryptions test were failing on the non-power-of-two test: kcapi-enc - Error: encryption failed with error 0 kcapi-enc - Error: decryption failed with error 0 [FAILED: 32-bit - 5.1.0-rc1+] 15 bytes: STDIN / STDOUT enc test (128 bits): original file (1d100e..cc96184c) and generated file (e3b0c442..1b7852b855) [FAILED: 32-bit - 5.1.0-rc1+] 15 bytes: STDIN / STDOUT enc test (128 bits) (openssl generated CT): original file (e3b0..5) and generated file (3..8e) [PASSED: 32-bit - 5.1.0-rc1+] 15 bytes: STDIN / STDOUT enc test (128 bits) (openssl generated PT) [FAILED: 32-bit - 5.1.0-rc1+] 15 bytes: STDIN / STDOUT enc test (password): original file (1d1..84c) and generated file (e3b..852b855) But the 16, 32, 512, 65536 tests always worked. Thankfully, this isn't a hidden hardware problem like previously, instead this turned out to be a copy and paste issue. With this patch, all the tests are passing with and kcapi-enc-test.sh gives crypto4xx's a clean bill of health: "Number of failures: 0" :). Cc: stable@vger.kernel.org Fixes: 98e87e3d933b ("crypto: crypto4xx - add aes-ctr support") Fixes: f2a13e7cba9e ("crypto: crypto4xx - enable AES RFC3686, ECB, CFB and OFB offloads") Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 3934c2523762..0322ae8ac466 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1252,7 +1252,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_module = THIS_MODULE, }, @@ -1272,7 +1272,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_module = THIS_MODULE, }, From 70c4997f34b6c6888b3ac157adec49e01d0df2d5 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Sat, 18 May 2019 23:28:11 +0200 Subject: [PATCH 022/143] crypto: crypto4xx - fix blocksize for cfb and ofb While the hardware consider them to be blockciphers, the reference implementation defines them as streamciphers. Do the right thing and set the blocksize to 1. This was found by CONFIG_CRYPTO_MANAGER_EXTRA_TESTS. This fixes the following issues: skcipher: blocksize for ofb-aes-ppc4xx (16) doesn't match generic impl (1) skcipher: blocksize for cfb-aes-ppc4xx (16) doesn't match generic impl (1) Cc: Eric Biggers Cc: stable@vger.kernel.org Fixes: f2a13e7cba9e ("crypto: crypto4xx - enable AES RFC3686, ECB, CFB and OFB offloads") Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 0322ae8ac466..5f2709cffc5b 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1231,7 +1231,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_module = THIS_MODULE, }, @@ -1311,7 +1311,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_module = THIS_MODULE, }, From 0f7a81374060828280fcfdfbaa162cb559017f9f Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Sat, 18 May 2019 23:28:12 +0200 Subject: [PATCH 023/143] crypto: crypto4xx - block ciphers should only accept complete blocks The hardware automatically zero pads incomplete block ciphers blocks without raising any errors. This is a screw-up. This was noticed by CONFIG_CRYPTO_MANAGER_EXTRA_TESTS tests that sent a incomplete blocks and expect them to fail. This fixes: cbc-aes-ppc4xx encryption unexpectedly succeeded on test vector "random: len=2409 klen=32"; expected_error=-22, cfg="random: may_sleep use_digest src_divs=[96.90%@+2295, 2.34%@+4066, 0.32%@alignmask+12, 0.34%@+4087, 0.9%@alignmask+1787, 0.1%@+3767] iv_offset=6" ecb-aes-ppc4xx encryption unexpectedly succeeded on test vector "random: len=1011 klen=32"; expected_error=-22, cfg="random: may_sleep use_digest src_divs=[100.0%@alignmask+20] dst_divs=[3.12%@+3001, 96.88%@+4070]" Cc: Eric Biggers Cc: stable@vger.kernel.org [4.19, 5.0 and 5.1] Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 36 +++++++++++++++++++--------- drivers/crypto/amcc/crypto4xx_core.c | 16 ++++++------- drivers/crypto/amcc/crypto4xx_core.h | 10 ++++---- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 307f5cfa9ba4..26f86fd7532b 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -76,12 +76,16 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, } static inline int crypto4xx_crypt(struct skcipher_request *req, - const unsigned int ivlen, bool decrypt) + const unsigned int ivlen, bool decrypt, + bool check_blocksize) { struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); __le32 iv[AES_IV_SIZE]; + if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) + return -EINVAL; + if (ivlen) crypto4xx_memcpy_to_le32(iv, req->iv, ivlen); @@ -90,24 +94,34 @@ static inline int crypto4xx_crypt(struct skcipher_request *req, ctx->sa_len, 0, NULL); } -int crypto4xx_encrypt_noiv(struct skcipher_request *req) +int crypto4xx_encrypt_noiv_block(struct skcipher_request *req) { - return crypto4xx_crypt(req, 0, false); + return crypto4xx_crypt(req, 0, false, true); } -int crypto4xx_encrypt_iv(struct skcipher_request *req) +int crypto4xx_encrypt_iv_stream(struct skcipher_request *req) { - return crypto4xx_crypt(req, AES_IV_SIZE, false); + return crypto4xx_crypt(req, AES_IV_SIZE, false, false); } -int crypto4xx_decrypt_noiv(struct skcipher_request *req) +int crypto4xx_decrypt_noiv_block(struct skcipher_request *req) { - return crypto4xx_crypt(req, 0, true); + return crypto4xx_crypt(req, 0, true, true); } -int crypto4xx_decrypt_iv(struct skcipher_request *req) +int crypto4xx_decrypt_iv_stream(struct skcipher_request *req) { - return crypto4xx_crypt(req, AES_IV_SIZE, true); + return crypto4xx_crypt(req, AES_IV_SIZE, true, false); +} + +int crypto4xx_encrypt_iv_block(struct skcipher_request *req) +{ + return crypto4xx_crypt(req, AES_IV_SIZE, false, true); +} + +int crypto4xx_decrypt_iv_block(struct skcipher_request *req) +{ + return crypto4xx_crypt(req, AES_IV_SIZE, true, true); } /** @@ -278,8 +292,8 @@ crypto4xx_ctr_crypt(struct skcipher_request *req, bool encrypt) return ret; } - return encrypt ? crypto4xx_encrypt_iv(req) - : crypto4xx_decrypt_iv(req); + return encrypt ? crypto4xx_encrypt_iv_stream(req) + : crypto4xx_decrypt_iv_stream(req); } static int crypto4xx_sk_setup_fallback(struct crypto4xx_ctx *ctx, diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 5f2709cffc5b..45f65d638caf 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1219,8 +1219,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_IV_SIZE, .setkey = crypto4xx_setkey_aes_cbc, - .encrypt = crypto4xx_encrypt_iv, - .decrypt = crypto4xx_decrypt_iv, + .encrypt = crypto4xx_encrypt_iv_block, + .decrypt = crypto4xx_decrypt_iv_block, .init = crypto4xx_sk_init, .exit = crypto4xx_sk_exit, } }, @@ -1239,8 +1239,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_IV_SIZE, .setkey = crypto4xx_setkey_aes_cfb, - .encrypt = crypto4xx_encrypt_iv, - .decrypt = crypto4xx_decrypt_iv, + .encrypt = crypto4xx_encrypt_iv_stream, + .decrypt = crypto4xx_decrypt_iv_stream, .init = crypto4xx_sk_init, .exit = crypto4xx_sk_exit, } }, @@ -1299,8 +1299,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = crypto4xx_setkey_aes_ecb, - .encrypt = crypto4xx_encrypt_noiv, - .decrypt = crypto4xx_decrypt_noiv, + .encrypt = crypto4xx_encrypt_noiv_block, + .decrypt = crypto4xx_decrypt_noiv_block, .init = crypto4xx_sk_init, .exit = crypto4xx_sk_exit, } }, @@ -1319,8 +1319,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_IV_SIZE, .setkey = crypto4xx_setkey_aes_ofb, - .encrypt = crypto4xx_encrypt_iv, - .decrypt = crypto4xx_decrypt_iv, + .encrypt = crypto4xx_encrypt_iv_stream, + .decrypt = crypto4xx_decrypt_iv_stream, .init = crypto4xx_sk_init, .exit = crypto4xx_sk_exit, } }, diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index c624f8cd3d2e..8ca082666736 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -182,10 +182,12 @@ int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); int crypto4xx_encrypt_ctr(struct skcipher_request *req); int crypto4xx_decrypt_ctr(struct skcipher_request *req); -int crypto4xx_encrypt_iv(struct skcipher_request *req); -int crypto4xx_decrypt_iv(struct skcipher_request *req); -int crypto4xx_encrypt_noiv(struct skcipher_request *req); -int crypto4xx_decrypt_noiv(struct skcipher_request *req); +int crypto4xx_encrypt_iv_stream(struct skcipher_request *req); +int crypto4xx_decrypt_iv_stream(struct skcipher_request *req); +int crypto4xx_encrypt_iv_block(struct skcipher_request *req); +int crypto4xx_decrypt_iv_block(struct skcipher_request *req); +int crypto4xx_encrypt_noiv_block(struct skcipher_request *req); +int crypto4xx_decrypt_noiv_block(struct skcipher_request *req); int crypto4xx_rfc3686_encrypt(struct skcipher_request *req); int crypto4xx_rfc3686_decrypt(struct skcipher_request *req); int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); From 1a9e93722967cb3a6b3834973cbca91f38ba791e Mon Sep 17 00:00:00 2001 From: Neil Armstrong Date: Mon, 20 May 2019 15:59:19 +0200 Subject: [PATCH 024/143] hwrng: meson - update with SPDX Licence identifier Signed-off-by: Neil Armstrong Signed-off-by: Herbert Xu --- drivers/char/hw_random/meson-rng.c | 52 +----------------------------- 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c index 2e23be802a62..76e693da5dde 100644 --- a/drivers/char/hw_random/meson-rng.c +++ b/drivers/char/hw_random/meson-rng.c @@ -1,58 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * * Copyright (c) 2016 BayLibre, SAS. * Author: Neil Armstrong * Copyright (C) 2014 Amlogic, Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, see . - * The full GNU General Public License is included in this distribution - * in the file called COPYING. - * - * BSD LICENSE - * - * Copyright (c) 2016 BayLibre, SAS. - * Author: Neil Armstrong - * Copyright (C) 2014 Amlogic, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include From 1fa0a7dcf7599f318e10e42ae66a0fe670fdc7bd Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 20 May 2019 09:42:32 -0700 Subject: [PATCH 025/143] crypto: vmx - convert to SPDX license identifiers Remove the boilerplate license text and replace it with the equivalent SPDX license identifier. Signed-off-by: Eric Biggers Reviewed-by: Michael Ellerman Signed-off-by: Herbert Xu --- drivers/crypto/vmx/aes.c | 14 +------------- drivers/crypto/vmx/aes_cbc.c | 14 +------------- drivers/crypto/vmx/aes_ctr.c | 14 +------------- drivers/crypto/vmx/aes_xts.c | 14 +------------- drivers/crypto/vmx/vmx.c | 14 +------------- 5 files changed, 5 insertions(+), 65 deletions(-) diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c index 603a62081994..2e9476158df4 100644 --- a/drivers/crypto/vmx/aes.c +++ b/drivers/crypto/vmx/aes.c @@ -1,21 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /** * AES routines supporting VMX instructions on the Power 8 * * Copyright (C) 2015 International Business Machines Inc. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 only. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - * * Author: Marcelo Henrique Cerri */ diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c index a1a9a6f0d42c..dae8af3c46dc 100644 --- a/drivers/crypto/vmx/aes_cbc.c +++ b/drivers/crypto/vmx/aes_cbc.c @@ -1,21 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /** * AES CBC routines supporting VMX instructions on the Power 8 * * Copyright (C) 2015 International Business Machines Inc. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 only. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - * * Author: Marcelo Henrique Cerri */ diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c index 192a53512f5e..dc3110117844 100644 --- a/drivers/crypto/vmx/aes_ctr.c +++ b/drivers/crypto/vmx/aes_ctr.c @@ -1,21 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /** * AES CTR routines supporting VMX instructions on the Power 8 * * Copyright (C) 2015 International Business Machines Inc. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 only. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - * * Author: Marcelo Henrique Cerri */ diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c index 00d412d811ae..aee1339f134e 100644 --- a/drivers/crypto/vmx/aes_xts.c +++ b/drivers/crypto/vmx/aes_xts.c @@ -1,21 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /** * AES XTS routines supporting VMX In-core instructions on Power 8 * * Copyright (C) 2015 International Business Machines Inc. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundations; version 2 only. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY of FITNESS FOR A PARTICUPAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - * * Author: Leonidas S. Barbosa */ diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c index a9f519830615..abd89c2bcec4 100644 --- a/drivers/crypto/vmx/vmx.c +++ b/drivers/crypto/vmx/vmx.c @@ -1,21 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /** * Routines supporting VMX instructions on the Power 8 * * Copyright (C) 2015 International Business Machines Inc. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 only. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - * * Author: Marcelo Henrique Cerri */ From 2621a8699e81c0a4f17d7b98ef22f1f89975a7b5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 20 May 2019 09:44:48 -0700 Subject: [PATCH 026/143] crypto: vmx - convert to skcipher API Convert the VMX implementations of AES-CBC, AES-CTR, and AES-XTS from the deprecated "blkcipher" API to the "skcipher" API. As part of this, I moved the skcipher_request for the fallback algorithm off the stack and into the request context of the parent algorithm. I tested this in a PowerPC VM with CONFIG_CRYPTO_MANAGER_EXTRA_TESTS=y. Signed-off-by: Eric Biggers Tested-by: Michael Ellerman Signed-off-by: Herbert Xu --- drivers/crypto/vmx/aes_cbc.c | 181 ++++++++++++--------------------- drivers/crypto/vmx/aes_ctr.c | 163 +++++++++++++---------------- drivers/crypto/vmx/aes_xts.c | 175 ++++++++++++++----------------- drivers/crypto/vmx/aesp8-ppc.h | 2 - drivers/crypto/vmx/vmx.c | 74 +++++++------- 5 files changed, 251 insertions(+), 344 deletions(-) diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c index dae8af3c46dc..92e75a05d6a9 100644 --- a/drivers/crypto/vmx/aes_cbc.c +++ b/drivers/crypto/vmx/aes_cbc.c @@ -7,64 +7,52 @@ * Author: Marcelo Henrique Cerri */ -#include -#include -#include -#include #include #include #include #include -#include -#include +#include #include "aesp8-ppc.h" struct p8_aes_cbc_ctx { - struct crypto_sync_skcipher *fallback; + struct crypto_skcipher *fallback; struct aes_key enc_key; struct aes_key dec_key; }; -static int p8_aes_cbc_init(struct crypto_tfm *tfm) +static int p8_aes_cbc_init(struct crypto_skcipher *tfm) { - const char *alg = crypto_tfm_alg_name(tfm); - struct crypto_sync_skcipher *fallback; - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); - - fallback = crypto_alloc_sync_skcipher(alg, 0, - CRYPTO_ALG_NEED_FALLBACK); + struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *fallback; + fallback = crypto_alloc_skcipher("cbc(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); if (IS_ERR(fallback)) { - printk(KERN_ERR - "Failed to allocate transformation for '%s': %ld\n", - alg, PTR_ERR(fallback)); + pr_err("Failed to allocate cbc(aes) fallback: %ld\n", + PTR_ERR(fallback)); return PTR_ERR(fallback); } - crypto_sync_skcipher_set_flags( - fallback, - crypto_skcipher_get_flags((struct crypto_skcipher *)tfm)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(fallback)); ctx->fallback = fallback; - return 0; } -static void p8_aes_cbc_exit(struct crypto_tfm *tfm) +static void p8_aes_cbc_exit(struct crypto_skcipher *tfm) { - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); - if (ctx->fallback) { - crypto_free_sync_skcipher(ctx->fallback); - ctx->fallback = NULL; - } + crypto_free_skcipher(ctx->fallback); } -static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key, +static int p8_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { + struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); int ret; - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); preempt_disable(); pagefault_disable(); @@ -75,108 +63,71 @@ static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_enable(); preempt_enable(); - ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); + ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen); return ret ? -EINVAL : 0; } -static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int p8_aes_cbc_crypt(struct skcipher_request *req, int enc) { + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; int ret; - struct blkcipher_walk walk; - struct p8_aes_cbc_ctx *ctx = - crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); if (!crypto_simd_usable()) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); - skcipher_request_set_sync_tfm(req, ctx->fallback); - skcipher_request_set_callback(req, desc->flags, NULL, NULL); - skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); - } else { - blkcipher_walk_init(&walk, dst, src, nbytes); - ret = blkcipher_walk_virt(desc, &walk); - while ((nbytes = walk.nbytes)) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - aes_p8_cbc_encrypt(walk.src.virt.addr, - walk.dst.virt.addr, - nbytes & AES_BLOCK_MASK, - &ctx->enc_key, walk.iv, 1); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); + struct skcipher_request *subreq = skcipher_request_ctx(req); - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, &walk, nbytes); - } + *subreq = *req; + skcipher_request_set_tfm(subreq, ctx->fallback); + return enc ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); } + ret = skcipher_walk_virt(&walk, req, false); + while ((nbytes = walk.nbytes) != 0) { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + aes_p8_cbc_encrypt(walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + enc ? &ctx->enc_key : &ctx->dec_key, + walk.iv, enc); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); + } return ret; } -static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int p8_aes_cbc_encrypt(struct skcipher_request *req) { - int ret; - struct blkcipher_walk walk; - struct p8_aes_cbc_ctx *ctx = - crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); - - if (!crypto_simd_usable()) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); - skcipher_request_set_sync_tfm(req, ctx->fallback); - skcipher_request_set_callback(req, desc->flags, NULL, NULL); - skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = crypto_skcipher_decrypt(req); - skcipher_request_zero(req); - } else { - blkcipher_walk_init(&walk, dst, src, nbytes); - ret = blkcipher_walk_virt(desc, &walk); - while ((nbytes = walk.nbytes)) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - aes_p8_cbc_encrypt(walk.src.virt.addr, - walk.dst.virt.addr, - nbytes & AES_BLOCK_MASK, - &ctx->dec_key, walk.iv, 0); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, &walk, nbytes); - } - } - - return ret; + return p8_aes_cbc_crypt(req, 1); } +static int p8_aes_cbc_decrypt(struct skcipher_request *req) +{ + return p8_aes_cbc_crypt(req, 0); +} -struct crypto_alg p8_aes_cbc_alg = { - .cra_name = "cbc(aes)", - .cra_driver_name = "p8_aes_cbc", - .cra_module = THIS_MODULE, - .cra_priority = 2000, - .cra_type = &crypto_blkcipher_type, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, - .cra_alignmask = 0, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct p8_aes_cbc_ctx), - .cra_init = p8_aes_cbc_init, - .cra_exit = p8_aes_cbc_exit, - .cra_blkcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = p8_aes_cbc_setkey, - .encrypt = p8_aes_cbc_encrypt, - .decrypt = p8_aes_cbc_decrypt, - }, +struct skcipher_alg p8_aes_cbc_alg = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "p8_aes_cbc", + .base.cra_module = THIS_MODULE, + .base.cra_priority = 2000, + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct p8_aes_cbc_ctx), + .setkey = p8_aes_cbc_setkey, + .encrypt = p8_aes_cbc_encrypt, + .decrypt = p8_aes_cbc_decrypt, + .init = p8_aes_cbc_init, + .exit = p8_aes_cbc_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, }; diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c index dc3110117844..c4d2809a5d9e 100644 --- a/drivers/crypto/vmx/aes_ctr.c +++ b/drivers/crypto/vmx/aes_ctr.c @@ -7,62 +7,51 @@ * Author: Marcelo Henrique Cerri */ -#include -#include -#include -#include #include #include #include #include -#include -#include +#include #include "aesp8-ppc.h" struct p8_aes_ctr_ctx { - struct crypto_sync_skcipher *fallback; + struct crypto_skcipher *fallback; struct aes_key enc_key; }; -static int p8_aes_ctr_init(struct crypto_tfm *tfm) +static int p8_aes_ctr_init(struct crypto_skcipher *tfm) { - const char *alg = crypto_tfm_alg_name(tfm); - struct crypto_sync_skcipher *fallback; - struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *fallback; - fallback = crypto_alloc_sync_skcipher(alg, 0, - CRYPTO_ALG_NEED_FALLBACK); + fallback = crypto_alloc_skcipher("ctr(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); if (IS_ERR(fallback)) { - printk(KERN_ERR - "Failed to allocate transformation for '%s': %ld\n", - alg, PTR_ERR(fallback)); + pr_err("Failed to allocate ctr(aes) fallback: %ld\n", + PTR_ERR(fallback)); return PTR_ERR(fallback); } - crypto_sync_skcipher_set_flags( - fallback, - crypto_skcipher_get_flags((struct crypto_skcipher *)tfm)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(fallback)); ctx->fallback = fallback; - return 0; } -static void p8_aes_ctr_exit(struct crypto_tfm *tfm) +static void p8_aes_ctr_exit(struct crypto_skcipher *tfm) { - struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - if (ctx->fallback) { - crypto_free_sync_skcipher(ctx->fallback); - ctx->fallback = NULL; - } + crypto_free_skcipher(ctx->fallback); } -static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key, +static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { + struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); int ret; - struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); preempt_disable(); pagefault_disable(); @@ -72,13 +61,13 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_enable(); preempt_enable(); - ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); + ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen); return ret ? -EINVAL : 0; } -static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx, - struct blkcipher_walk *walk) +static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx, + struct skcipher_walk *walk) { u8 *ctrblk = walk->iv; u8 keystream[AES_BLOCK_SIZE]; @@ -98,77 +87,63 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx, crypto_inc(ctrblk, AES_BLOCK_SIZE); } -static int p8_aes_ctr_crypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int p8_aes_ctr_crypt(struct skcipher_request *req) { + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; int ret; - u64 inc; - struct blkcipher_walk walk; - struct p8_aes_ctr_ctx *ctx = - crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); if (!crypto_simd_usable()) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); - skcipher_request_set_sync_tfm(req, ctx->fallback); - skcipher_request_set_callback(req, desc->flags, NULL, NULL); - skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); - } else { - blkcipher_walk_init(&walk, dst, src, nbytes); - ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); - while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, - walk.dst.virt.addr, - (nbytes & - AES_BLOCK_MASK) / - AES_BLOCK_SIZE, - &ctx->enc_key, - walk.iv); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); + struct skcipher_request *subreq = skcipher_request_ctx(req); - /* We need to update IV mostly for last bytes/round */ - inc = (nbytes & AES_BLOCK_MASK) / AES_BLOCK_SIZE; - if (inc > 0) - while (inc--) - crypto_inc(walk.iv, AES_BLOCK_SIZE); - - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, &walk, nbytes); - } - if (walk.nbytes) { - p8_aes_ctr_final(ctx, &walk); - ret = blkcipher_walk_done(desc, &walk, 0); - } + *subreq = *req; + skcipher_request_set_tfm(subreq, ctx->fallback); + return crypto_skcipher_encrypt(subreq); } + ret = skcipher_walk_virt(&walk, req, false); + while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, + walk.dst.virt.addr, + nbytes / AES_BLOCK_SIZE, + &ctx->enc_key, walk.iv); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + do { + crypto_inc(walk.iv, AES_BLOCK_SIZE); + } while ((nbytes -= AES_BLOCK_SIZE) >= AES_BLOCK_SIZE); + + ret = skcipher_walk_done(&walk, nbytes); + } + if (nbytes) { + p8_aes_ctr_final(ctx, &walk); + ret = skcipher_walk_done(&walk, 0); + } return ret; } -struct crypto_alg p8_aes_ctr_alg = { - .cra_name = "ctr(aes)", - .cra_driver_name = "p8_aes_ctr", - .cra_module = THIS_MODULE, - .cra_priority = 2000, - .cra_type = &crypto_blkcipher_type, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, - .cra_alignmask = 0, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct p8_aes_ctr_ctx), - .cra_init = p8_aes_ctr_init, - .cra_exit = p8_aes_ctr_exit, - .cra_blkcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = p8_aes_ctr_setkey, - .encrypt = p8_aes_ctr_crypt, - .decrypt = p8_aes_ctr_crypt, - }, +struct skcipher_alg p8_aes_ctr_alg = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "p8_aes_ctr", + .base.cra_module = THIS_MODULE, + .base.cra_priority = 2000, + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct p8_aes_ctr_ctx), + .setkey = p8_aes_ctr_setkey, + .encrypt = p8_aes_ctr_crypt, + .decrypt = p8_aes_ctr_crypt, + .init = p8_aes_ctr_init, + .exit = p8_aes_ctr_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, }; diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c index aee1339f134e..965d8e03321c 100644 --- a/drivers/crypto/vmx/aes_xts.c +++ b/drivers/crypto/vmx/aes_xts.c @@ -7,67 +7,56 @@ * Author: Leonidas S. Barbosa */ -#include -#include -#include -#include #include #include #include #include -#include +#include #include -#include #include "aesp8-ppc.h" struct p8_aes_xts_ctx { - struct crypto_sync_skcipher *fallback; + struct crypto_skcipher *fallback; struct aes_key enc_key; struct aes_key dec_key; struct aes_key tweak_key; }; -static int p8_aes_xts_init(struct crypto_tfm *tfm) +static int p8_aes_xts_init(struct crypto_skcipher *tfm) { - const char *alg = crypto_tfm_alg_name(tfm); - struct crypto_sync_skcipher *fallback; - struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *fallback; - fallback = crypto_alloc_sync_skcipher(alg, 0, - CRYPTO_ALG_NEED_FALLBACK); + fallback = crypto_alloc_skcipher("xts(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); if (IS_ERR(fallback)) { - printk(KERN_ERR - "Failed to allocate transformation for '%s': %ld\n", - alg, PTR_ERR(fallback)); + pr_err("Failed to allocate xts(aes) fallback: %ld\n", + PTR_ERR(fallback)); return PTR_ERR(fallback); } - crypto_sync_skcipher_set_flags( - fallback, - crypto_skcipher_get_flags((struct crypto_skcipher *)tfm)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(fallback)); ctx->fallback = fallback; - return 0; } -static void p8_aes_xts_exit(struct crypto_tfm *tfm) +static void p8_aes_xts_exit(struct crypto_skcipher *tfm) { - struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - if (ctx->fallback) { - crypto_free_sync_skcipher(ctx->fallback); - ctx->fallback = NULL; - } + crypto_free_skcipher(ctx->fallback); } -static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key, +static int p8_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { + struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); int ret; - struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); - ret = xts_check_key(tfm, key, keylen); + ret = xts_verify_key(tfm, key, keylen); if (ret) return ret; @@ -81,100 +70,90 @@ static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_enable(); preempt_enable(); - ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); + ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen); return ret ? -EINVAL : 0; } -static int p8_aes_xts_crypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, - unsigned int nbytes, int enc) +static int p8_aes_xts_crypt(struct skcipher_request *req, int enc) { - int ret; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; u8 tweak[AES_BLOCK_SIZE]; - u8 *iv; - struct blkcipher_walk walk; - struct p8_aes_xts_ctx *ctx = - crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); + int ret; if (!crypto_simd_usable()) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); - skcipher_request_set_sync_tfm(req, ctx->fallback); - skcipher_request_set_callback(req, desc->flags, NULL, NULL); - skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = enc? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req); - skcipher_request_zero(req); - } else { - blkcipher_walk_init(&walk, dst, src, nbytes); + struct skcipher_request *subreq = skcipher_request_ctx(req); - ret = blkcipher_walk_virt(desc, &walk); + *subreq = *req; + skcipher_request_set_tfm(subreq, ctx->fallback); + return enc ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + } + ret = skcipher_walk_virt(&walk, req, false); + if (ret) + return ret; + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + + aes_p8_encrypt(walk.iv, tweak, &ctx->tweak_key); + + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + while ((nbytes = walk.nbytes) != 0) { preempt_disable(); pagefault_disable(); enable_kernel_vsx(); - - iv = walk.iv; - memset(tweak, 0, AES_BLOCK_SIZE); - aes_p8_encrypt(iv, tweak, &ctx->tweak_key); - + if (enc) + aes_p8_xts_encrypt(walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + &ctx->enc_key, NULL, tweak); + else + aes_p8_xts_decrypt(walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + &ctx->dec_key, NULL, tweak); disable_kernel_vsx(); pagefault_enable(); preempt_enable(); - while ((nbytes = walk.nbytes)) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - if (enc) - aes_p8_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr, - nbytes & AES_BLOCK_MASK, &ctx->enc_key, NULL, tweak); - else - aes_p8_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr, - nbytes & AES_BLOCK_MASK, &ctx->dec_key, NULL, tweak); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, &walk, nbytes); - } + ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); } return ret; } -static int p8_aes_xts_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int p8_aes_xts_encrypt(struct skcipher_request *req) { - return p8_aes_xts_crypt(desc, dst, src, nbytes, 1); + return p8_aes_xts_crypt(req, 1); } -static int p8_aes_xts_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int p8_aes_xts_decrypt(struct skcipher_request *req) { - return p8_aes_xts_crypt(desc, dst, src, nbytes, 0); + return p8_aes_xts_crypt(req, 0); } -struct crypto_alg p8_aes_xts_alg = { - .cra_name = "xts(aes)", - .cra_driver_name = "p8_aes_xts", - .cra_module = THIS_MODULE, - .cra_priority = 2000, - .cra_type = &crypto_blkcipher_type, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, - .cra_alignmask = 0, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct p8_aes_xts_ctx), - .cra_init = p8_aes_xts_init, - .cra_exit = p8_aes_xts_exit, - .cra_blkcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = 2 * AES_MIN_KEY_SIZE, - .max_keysize = 2 * AES_MAX_KEY_SIZE, - .setkey = p8_aes_xts_setkey, - .encrypt = p8_aes_xts_encrypt, - .decrypt = p8_aes_xts_decrypt, - } +struct skcipher_alg p8_aes_xts_alg = { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "p8_aes_xts", + .base.cra_module = THIS_MODULE, + .base.cra_priority = 2000, + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct p8_aes_xts_ctx), + .setkey = p8_aes_xts_setkey, + .encrypt = p8_aes_xts_encrypt, + .decrypt = p8_aes_xts_decrypt, + .init = p8_aes_xts_init, + .exit = p8_aes_xts_exit, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, }; diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h index 349646b73754..01774a4d26a2 100644 --- a/drivers/crypto/vmx/aesp8-ppc.h +++ b/drivers/crypto/vmx/aesp8-ppc.h @@ -2,8 +2,6 @@ #include #include -#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) - struct aes_key { u8 key[AES_MAX_KEYLENGTH]; int rounds; diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c index abd89c2bcec4..eff03fdf964f 100644 --- a/drivers/crypto/vmx/vmx.c +++ b/drivers/crypto/vmx/vmx.c @@ -15,54 +15,58 @@ #include #include #include +#include extern struct shash_alg p8_ghash_alg; extern struct crypto_alg p8_aes_alg; -extern struct crypto_alg p8_aes_cbc_alg; -extern struct crypto_alg p8_aes_ctr_alg; -extern struct crypto_alg p8_aes_xts_alg; -static struct crypto_alg *algs[] = { - &p8_aes_alg, - &p8_aes_cbc_alg, - &p8_aes_ctr_alg, - &p8_aes_xts_alg, - NULL, -}; +extern struct skcipher_alg p8_aes_cbc_alg; +extern struct skcipher_alg p8_aes_ctr_alg; +extern struct skcipher_alg p8_aes_xts_alg; static int __init p8_init(void) { - int ret = 0; - struct crypto_alg **alg_it; - - for (alg_it = algs; *alg_it; alg_it++) { - ret = crypto_register_alg(*alg_it); - printk(KERN_INFO "crypto_register_alg '%s' = %d\n", - (*alg_it)->cra_name, ret); - if (ret) { - for (alg_it--; alg_it >= algs; alg_it--) - crypto_unregister_alg(*alg_it); - break; - } - } - if (ret) - return ret; + int ret; ret = crypto_register_shash(&p8_ghash_alg); - if (ret) { - for (alg_it = algs; *alg_it; alg_it++) - crypto_unregister_alg(*alg_it); - } + if (ret) + goto err; + + ret = crypto_register_alg(&p8_aes_alg); + if (ret) + goto err_unregister_ghash; + + ret = crypto_register_skcipher(&p8_aes_cbc_alg); + if (ret) + goto err_unregister_aes; + + ret = crypto_register_skcipher(&p8_aes_ctr_alg); + if (ret) + goto err_unregister_aes_cbc; + + ret = crypto_register_skcipher(&p8_aes_xts_alg); + if (ret) + goto err_unregister_aes_ctr; + + return 0; + +err_unregister_aes_ctr: + crypto_unregister_skcipher(&p8_aes_ctr_alg); +err_unregister_aes_cbc: + crypto_unregister_skcipher(&p8_aes_cbc_alg); +err_unregister_aes: + crypto_unregister_alg(&p8_aes_alg); +err_unregister_ghash: + crypto_unregister_shash(&p8_ghash_alg); +err: return ret; } static void __exit p8_exit(void) { - struct crypto_alg **alg_it; - - for (alg_it = algs; *alg_it; alg_it++) { - printk(KERN_INFO "Removing '%s'\n", (*alg_it)->cra_name); - crypto_unregister_alg(*alg_it); - } + crypto_unregister_skcipher(&p8_aes_xts_alg); + crypto_unregister_skcipher(&p8_aes_ctr_alg); + crypto_unregister_skcipher(&p8_aes_cbc_alg); + crypto_unregister_alg(&p8_aes_alg); crypto_unregister_shash(&p8_ghash_alg); } From e944eab37a72bf171647d691dda4614f7151191f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 20 May 2019 09:47:19 -0700 Subject: [PATCH 027/143] crypto: testmgr - fix length truncation with large page size On PowerPC with CONFIG_CRYPTO_MANAGER_EXTRA_TESTS=y, there is sometimes a crash in generate_random_aead_testvec(). The problem is that the generated test vectors use data lengths of up to about 2 * PAGE_SIZE, which is 128 KiB on PowerPC; however, the data length fields in the test vectors are 'unsigned short', so the lengths get truncated. Fix this by changing the relevant fields to 'unsigned int'. Fixes: 40153b10d91c ("crypto: testmgr - fuzz AEADs against their generic implementation") Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crypto/testmgr.h b/crypto/testmgr.h index b6daae1f6a1d..2655f41d4d23 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -43,7 +43,7 @@ struct hash_testvec { const char *key; const char *plaintext; const char *digest; - unsigned short psize; + unsigned int psize; unsigned short ksize; int setkey_error; int digest_error; @@ -74,7 +74,7 @@ struct cipher_testvec { const char *ctext; unsigned char wk; /* weak key flag */ unsigned short klen; - unsigned short len; + unsigned int len; bool fips_skip; bool generates_iv; int setkey_error; @@ -110,9 +110,9 @@ struct aead_testvec { unsigned char novrfy; unsigned char wk; unsigned char klen; - unsigned short plen; - unsigned short clen; - unsigned short alen; + unsigned int plen; + unsigned int clen; + unsigned int alen; int setkey_error; int setauthsize_error; int crypt_error; From 929d34cac156a55bf9e5a88a1fa5191be1fcdc35 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 20 May 2019 09:48:29 -0700 Subject: [PATCH 028/143] crypto: testmgr - make extra tests depend on cryptomgr The crypto self-tests are part of the "cryptomgr" module, which can technically be disabled (though it rarely is). If you do so, currently you can still enable CRYPTO_MANAGER_EXTRA_TESTS, which doesn't make sense since in that case testmgr.c isn't compiled at all. Fix it by making it CRYPTO_MANAGER_EXTRA_TESTS depend on CRYPTO_MANAGER2, like CRYPTO_MANAGER_DISABLE_TESTS already does. Fixes: 5b2706a4d459 ("crypto: testmgr - introduce CONFIG_CRYPTO_MANAGER_EXTRA_TESTS") Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 3d056e7da65f..7009aff745cb 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -137,10 +137,11 @@ config CRYPTO_USER Userspace configuration for cryptographic instantiations such as cbc(aes). +if CRYPTO_MANAGER2 + config CRYPTO_MANAGER_DISABLE_TESTS bool "Disable run-time self tests" default y - depends on CRYPTO_MANAGER2 help Disable run-time self tests that normally take place at algorithm registration. @@ -155,6 +156,8 @@ config CRYPTO_MANAGER_EXTRA_TESTS This is intended for developer use only, as these tests take much longer to run than the normal self tests. +endif # if CRYPTO_MANAGER2 + config CRYPTO_GF128MUL tristate "GF(2^128) multiplication functions" help From c8a3315a5f691cc4be16421aeb0608495c9a33ba Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 20 May 2019 09:49:46 -0700 Subject: [PATCH 029/143] crypto: make all templates select CRYPTO_MANAGER The "cryptomgr" module is required for templates to be used. Many templates select it, but others don't. Make all templates select it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crypto/Kconfig b/crypto/Kconfig index 7009aff745cb..af8c6b4e6a83 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -282,6 +282,7 @@ config CRYPTO_CCM select CRYPTO_CTR select CRYPTO_HASH select CRYPTO_AEAD + select CRYPTO_MANAGER help Support for Counter with CBC MAC. Required for IPsec. @@ -291,6 +292,7 @@ config CRYPTO_GCM select CRYPTO_AEAD select CRYPTO_GHASH select CRYPTO_NULL + select CRYPTO_MANAGER help Support for Galois/Counter Mode (GCM) and Galois Message Authentication Code (GMAC). Required for IPSec. @@ -300,6 +302,7 @@ config CRYPTO_CHACHA20POLY1305 select CRYPTO_CHACHA20 select CRYPTO_POLY1305 select CRYPTO_AEAD + select CRYPTO_MANAGER help ChaCha20-Poly1305 AEAD support, RFC7539. @@ -414,6 +417,7 @@ config CRYPTO_SEQIV select CRYPTO_BLKCIPHER select CRYPTO_NULL select CRYPTO_RNG_DEFAULT + select CRYPTO_MANAGER help This IV generator generates an IV based on a sequence number by xoring it with a salt. This algorithm is mainly useful for CTR @@ -423,6 +427,7 @@ config CRYPTO_ECHAINIV select CRYPTO_AEAD select CRYPTO_NULL select CRYPTO_RNG_DEFAULT + select CRYPTO_MANAGER default m help This IV generator generates an IV based on the encryption of @@ -459,6 +464,7 @@ config CRYPTO_CTR config CRYPTO_CTS tristate "CTS support" select CRYPTO_BLKCIPHER + select CRYPTO_MANAGER help CTS: Cipher Text Stealing This is the Cipher Text Stealing mode as described by @@ -524,6 +530,7 @@ config CRYPTO_XTS config CRYPTO_KEYWRAP tristate "Key wrapping support" select CRYPTO_BLKCIPHER + select CRYPTO_MANAGER help Support for key wrapping (NIST SP800-38F / RFC3394) without padding. @@ -554,6 +561,7 @@ config CRYPTO_ADIANTUM select CRYPTO_CHACHA20 select CRYPTO_POLY1305 select CRYPTO_NHPOLY1305 + select CRYPTO_MANAGER help Adiantum is a tweakable, length-preserving encryption mode designed for fast and secure disk encryption, especially on From 87804144cb7ad09126f1d16e205a7d6b4bc35404 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 20 May 2019 09:52:07 -0700 Subject: [PATCH 030/143] crypto: echainiv - change to 'default n' echainiv is the only algorithm or template in the crypto API that is enabled by default. But there doesn't seem to be a good reason for it. And it pulls in a lot of stuff as dependencies, like AEAD support and a "NIST SP800-90A DRBG" including HMAC-SHA256. The commit which made it default 'm', commit 3491244c6298 ("crypto: echainiv - Set Kconfig default to m"), mentioned that it's needed for IPsec. However, later commit 32b6170ca59c ("ipv4+ipv6: Make INET*_ESP select CRYPTO_ECHAINIV") made the IPsec kconfig options select it. So, remove the 'default m'. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index af8c6b4e6a83..1062e1031f73 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -428,7 +428,6 @@ config CRYPTO_ECHAINIV select CRYPTO_NULL select CRYPTO_RNG_DEFAULT select CRYPTO_MANAGER - default m help This IV generator generates an IV based on the encryption of a sequence number xored with a salt. This is the default From e590e1321c13ae736bd0e0da48a61a58ef084875 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 20 May 2019 09:53:43 -0700 Subject: [PATCH 031/143] crypto: gf128mul - make unselectable by user There's no reason for users to select CONFIG_CRYPTO_GF128MUL, since it's just some helper functions, and algorithms that need it select it. Remove the prompt string so that it's not shown to users. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 1062e1031f73..5350aa9368ec 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -159,13 +159,7 @@ config CRYPTO_MANAGER_EXTRA_TESTS endif # if CRYPTO_MANAGER2 config CRYPTO_GF128MUL - tristate "GF(2^128) multiplication functions" - help - Efficient table driven implementation of multiplications in the - field GF(2^128). This is needed by some cypher modes. This - option will be selected automatically if you select such a - cipher mode. Only select this option by hand if you expect to load - an external module that requires these functions. + tristate config CRYPTO_NULL tristate "Null algorithms" From 3e56e168638b3e7147902c3b7257a57ea573a30e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 20 May 2019 09:53:58 -0700 Subject: [PATCH 032/143] crypto: cryptd - move kcrypto_wq into cryptd kcrypto_wq is only used by cryptd, so move it into cryptd.c and change the workqueue name from "crypto" to "cryptd". Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 5 ---- crypto/Makefile | 2 -- crypto/cryptd.c | 24 ++++++++++++---- crypto/crypto_wq.c | 40 -------------------------- drivers/crypto/cavium/cpt/cptvf_algs.c | 1 - include/crypto/crypto_wq.h | 8 ------ 6 files changed, 19 insertions(+), 61 deletions(-) delete mode 100644 crypto/crypto_wq.c delete mode 100644 include/crypto/crypto_wq.h diff --git a/crypto/Kconfig b/crypto/Kconfig index 5350aa9368ec..9cdd92520f00 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -61,7 +61,6 @@ config CRYPTO_BLKCIPHER2 tristate select CRYPTO_ALGAPI2 select CRYPTO_RNG2 - select CRYPTO_WORKQUEUE config CRYPTO_HASH tristate @@ -183,15 +182,11 @@ config CRYPTO_PCRYPT This converts an arbitrary crypto algorithm into a parallel algorithm that executes in kernel threads. -config CRYPTO_WORKQUEUE - tristate - config CRYPTO_CRYPTD tristate "Software async crypto daemon" select CRYPTO_BLKCIPHER select CRYPTO_HASH select CRYPTO_MANAGER - select CRYPTO_WORKQUEUE help This is a generic software asynchronous crypto daemon that converts an arbitrary synchronous software crypto algorithm diff --git a/crypto/Makefile b/crypto/Makefile index 266a4cdbb9e2..2adf06b178e9 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -6,8 +6,6 @@ obj-$(CONFIG_CRYPTO) += crypto.o crypto-y := api.o cipher.o compress.o memneq.o -obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o - obj-$(CONFIG_CRYPTO_ENGINE) += crypto_engine.o obj-$(CONFIG_CRYPTO_FIPS) += fips.o diff --git a/crypto/cryptd.c b/crypto/cryptd.c index b3bb99390ae7..1bf777b76512 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -31,11 +30,14 @@ #include #include #include +#include static unsigned int cryptd_max_cpu_qlen = 1000; module_param(cryptd_max_cpu_qlen, uint, 0); MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); +static struct workqueue_struct *cryptd_wq; + struct cryptd_cpu_queue { struct crypto_queue queue; struct work_struct work; @@ -141,7 +143,7 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue, if (err == -ENOSPC) goto out_put_cpu; - queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); + queue_work_on(cpu, cryptd_wq, &cpu_queue->work); if (!atomic_read(refcnt)) goto out_put_cpu; @@ -184,7 +186,7 @@ static void cryptd_queue_worker(struct work_struct *work) req->complete(req, 0); if (cpu_queue->queue.qlen) - queue_work(kcrypto_wq, &cpu_queue->work); + queue_work(cryptd_wq, &cpu_queue->work); } static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) @@ -1123,19 +1125,31 @@ static int __init cryptd_init(void) { int err; + cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, + 1); + if (!cryptd_wq) + return -ENOMEM; + err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen); if (err) - return err; + goto err_destroy_wq; err = crypto_register_template(&cryptd_tmpl); if (err) - cryptd_fini_queue(&queue); + goto err_fini_queue; + return 0; + +err_fini_queue: + cryptd_fini_queue(&queue); +err_destroy_wq: + destroy_workqueue(cryptd_wq); return err; } static void __exit cryptd_exit(void) { + destroy_workqueue(cryptd_wq); cryptd_fini_queue(&queue); crypto_unregister_template(&cryptd_tmpl); } diff --git a/crypto/crypto_wq.c b/crypto/crypto_wq.c deleted file mode 100644 index 2f1b8d12952a..000000000000 --- a/crypto/crypto_wq.c +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Workqueue for crypto subsystem - * - * Copyright (c) 2009 Intel Corp. - * Author: Huang Ying - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - */ - -#include -#include -#include -#include - -struct workqueue_struct *kcrypto_wq; -EXPORT_SYMBOL_GPL(kcrypto_wq); - -static int __init crypto_wq_init(void) -{ - kcrypto_wq = alloc_workqueue("crypto", - WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1); - if (unlikely(!kcrypto_wq)) - return -ENOMEM; - return 0; -} - -static void __exit crypto_wq_exit(void) -{ - destroy_workqueue(kcrypto_wq); -} - -subsys_initcall(crypto_wq_init); -module_exit(crypto_wq_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Workqueue for crypto subsystem"); diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c index 9810ad8ac519..f6b0c9df12ed 100644 --- a/drivers/crypto/cavium/cpt/cptvf_algs.c +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include diff --git a/include/crypto/crypto_wq.h b/include/crypto/crypto_wq.h deleted file mode 100644 index 23114746ac08..000000000000 --- a/include/crypto/crypto_wq.h +++ /dev/null @@ -1,8 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef CRYPTO_WQ_H -#define CRYPTO_WQ_H - -#include - -extern struct workqueue_struct *kcrypto_wq; -#endif From 84ede58dfcd1db6f04f71dd3ccd5328271b346da Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 20 May 2019 09:54:46 -0700 Subject: [PATCH 033/143] crypto: hash - remove CRYPTO_ALG_TYPE_DIGEST Remove the unnecessary constant CRYPTO_ALG_TYPE_DIGEST, which has the same value as CRYPTO_ALG_TYPE_HASH. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- Documentation/crypto/architecture.rst | 4 +--- crypto/cryptd.c | 2 +- include/linux/crypto.h | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Documentation/crypto/architecture.rst b/Documentation/crypto/architecture.rst index ee8ff0762d7f..3eae1ae7f798 100644 --- a/Documentation/crypto/architecture.rst +++ b/Documentation/crypto/architecture.rst @@ -208,9 +208,7 @@ the aforementioned cipher types: - CRYPTO_ALG_TYPE_KPP Key-agreement Protocol Primitive (KPP) such as an ECDH or DH implementation -- CRYPTO_ALG_TYPE_DIGEST Raw message digest - -- CRYPTO_ALG_TYPE_HASH Alias for CRYPTO_ALG_TYPE_DIGEST +- CRYPTO_ALG_TYPE_HASH Raw message digest - CRYPTO_ALG_TYPE_SHASH Synchronous multi-block hash diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 1bf777b76512..c34d10309b1b 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -925,7 +925,7 @@ static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_BLKCIPHER: return cryptd_create_skcipher(tmpl, tb, &queue); - case CRYPTO_ALG_TYPE_DIGEST: + case CRYPTO_ALG_TYPE_HASH: return cryptd_create_hash(tmpl, tb, &queue); case CRYPTO_ALG_TYPE_AEAD: return cryptd_create_aead(tmpl, tb, &queue); diff --git a/include/linux/crypto.h b/include/linux/crypto.h index f2565a103158..311237b1dab0 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -54,7 +54,6 @@ #define CRYPTO_ALG_TYPE_SCOMPRESS 0x0000000b #define CRYPTO_ALG_TYPE_RNG 0x0000000c #define CRYPTO_ALG_TYPE_AKCIPHER 0x0000000d -#define CRYPTO_ALG_TYPE_DIGEST 0x0000000e #define CRYPTO_ALG_TYPE_HASH 0x0000000e #define CRYPTO_ALG_TYPE_SHASH 0x0000000e #define CRYPTO_ALG_TYPE_AHASH 0x0000000f From 5e99a0a7a9468a18efec66ee97f1e34886e55550 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 20 May 2019 09:55:15 -0700 Subject: [PATCH 034/143] crypto: algapi - remove crypto_tfm_in_queue() Remove the crypto_tfm_in_queue() function, which is unused. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/algapi.c | 13 ------------- include/crypto/algapi.h | 7 ------- include/crypto/internal/hash.h | 6 ------ 3 files changed, 26 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 4c9c86b55738..7c51f45d1cf1 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -952,19 +952,6 @@ struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) } EXPORT_SYMBOL_GPL(crypto_dequeue_request); -int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm) -{ - struct crypto_async_request *req; - - list_for_each_entry(req, &queue->list, list) { - if (req->tfm == tfm) - return 1; - } - - return 0; -} -EXPORT_SYMBOL_GPL(crypto_tfm_in_queue); - static inline void crypto_inc_byte(u8 *a, unsigned int size) { u8 *b = (a + size); diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 4be38cd0b8d5..964a26fa4ff4 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -194,7 +194,6 @@ void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen); int crypto_enqueue_request(struct crypto_queue *queue, struct crypto_async_request *request); struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue); -int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm); static inline unsigned int crypto_queue_len(struct crypto_queue *queue) { return queue->qlen; @@ -376,12 +375,6 @@ static inline void *ablkcipher_request_ctx(struct ablkcipher_request *req) return req->__ctx; } -static inline int ablkcipher_tfm_in_queue(struct crypto_queue *queue, - struct crypto_ablkcipher *tfm) -{ - return crypto_tfm_in_queue(queue, crypto_ablkcipher_tfm(tfm)); -} - static inline struct crypto_alg *crypto_get_attr_alg(struct rtattr **tb, u32 type, u32 mask) { diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index e355fdb642a9..669b4ee5c16d 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -201,12 +201,6 @@ static inline struct ahash_request *ahash_dequeue_request( return ahash_request_cast(crypto_dequeue_request(queue)); } -static inline int ahash_tfm_in_queue(struct crypto_queue *queue, - struct crypto_ahash *tfm) -{ - return crypto_tfm_in_queue(queue, crypto_ahash_tfm(tfm)); -} - static inline void *crypto_shash_ctx(struct crypto_shash *tfm) { return crypto_tfm_ctx(&tfm->base); From a1a42f84011fae6ff08441a91aefeb7febc984fc Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:08 +0000 Subject: [PATCH 035/143] crypto: talitos - rename alternative AEAD algos. The talitos driver has two ways to perform AEAD depending on the HW capability. Some HW support both. It is needed to give them different names to distingish which one it is for instance when a test fails. Signed-off-by: Christophe Leroy Fixes: 7405c8d7ff97 ("crypto: talitos - templates for AEAD using HMAC_SNOOP_NO_AFEU") Cc: stable@vger.kernel.org Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index f443cbe7da80..6f8bc6467706 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -2356,7 +2356,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(sha1),cbc(aes))", .cra_driver_name = "authenc-hmac-sha1-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2401,7 +2401,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_name = "authenc(hmac(sha1)," "cbc(des3_ede))", .cra_driver_name = "authenc-hmac-sha1-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2444,7 +2444,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(sha224),cbc(aes))", .cra_driver_name = "authenc-hmac-sha224-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2489,7 +2489,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_name = "authenc(hmac(sha224)," "cbc(des3_ede))", .cra_driver_name = "authenc-hmac-sha224-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2532,7 +2532,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(sha256),cbc(aes))", .cra_driver_name = "authenc-hmac-sha256-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2577,7 +2577,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_name = "authenc(hmac(sha256)," "cbc(des3_ede))", .cra_driver_name = "authenc-hmac-sha256-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2706,7 +2706,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(md5),cbc(aes))", .cra_driver_name = "authenc-hmac-md5-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2749,7 +2749,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(md5),cbc(des3_ede))", .cra_driver_name = "authenc-hmac-md5-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, From b8fbdc2bc4e71b62646031d5df5f08aafe15d5ad Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:09 +0000 Subject: [PATCH 036/143] crypto: talitos - reduce max key size for SEC1 SEC1 doesn't support SHA384/512, so it doesn't require longer keys. This patch reduces the max key size when the driver is built for SEC1 only. Signed-off-by: Christophe Leroy Fixes: 03d2c5114c95 ("crypto: talitos - Extend max key length for SHA384/512-HMAC and AEAD") Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 6f8bc6467706..6312f8d501b1 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -837,7 +837,11 @@ static void talitos_unregister_rng(struct device *dev) * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP */ #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) +#ifdef CONFIG_CRYPTO_DEV_TALITOS_SEC2 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) +#else +#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) +#endif #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ struct talitos_ctx { From 1ba34e71e9e56ac29a52e0d42b6290f3dc5bfd90 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:10 +0000 Subject: [PATCH 037/143] crypto: talitos - check AES key size Although the HW accepts any size and silently truncates it to the correct length, the extra tests expects EINVAL to be returned when the key size is not valid. Signed-off-by: Christophe Leroy Fixes: 4de9d0b547b9 ("crypto: talitos - Add ablkcipher algorithms") Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 6312f8d501b1..95f71e18bf55 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1622,6 +1622,18 @@ static int ablkcipher_des3_setkey(struct crypto_ablkcipher *cipher, return ablkcipher_setkey(cipher, key, keylen); } +static int ablkcipher_aes_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 || + keylen == AES_KEYSIZE_256) + return ablkcipher_setkey(cipher, key, keylen); + + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + + return -EINVAL; +} + static void common_nonsnoop_unmap(struct device *dev, struct talitos_edesc *edesc, struct ablkcipher_request *areq) @@ -2782,6 +2794,7 @@ static struct talitos_alg_template driver_algs[] = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, + .setkey = ablkcipher_aes_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2798,6 +2811,7 @@ static struct talitos_alg_template driver_algs[] = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, + .setkey = ablkcipher_aes_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2815,6 +2829,7 @@ static struct talitos_alg_template driver_algs[] = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, + .setkey = ablkcipher_aes_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | From b9a05b6041cb9810a291315569b2af0d63c3680a Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:11 +0000 Subject: [PATCH 038/143] crypto: talitos - fix CTR alg blocksize CTR has a blocksize of 1. Signed-off-by: Christophe Leroy Fixes: 5e75ae1b3cef ("crypto: talitos - add new crypto modes") Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 95f71e18bf55..8b9a529f1b66 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -2822,7 +2822,7 @@ static struct talitos_alg_template driver_algs[] = { .alg.crypto = { .cra_name = "ctr(aes)", .cra_driver_name = "ctr-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_ablkcipher = { From ee483d32ee1a1a7f7d7e918fbc350c790a5af64a Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:12 +0000 Subject: [PATCH 039/143] crypto: talitos - check data blocksize in ablkcipher. When data size is not a multiple of the alg's block size, the SEC generates an error interrupt and dumps the registers. And for NULL size, the SEC does just nothing and the interrupt is awaited forever. This patch ensures the data size is correct before submitting the request to the SEC engine. Signed-off-by: Christophe Leroy Fixes: 4de9d0b547b9 ("crypto: talitos - Add ablkcipher algorithms") Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 8b9a529f1b66..1e5410f92166 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1756,6 +1756,14 @@ static int ablkcipher_encrypt(struct ablkcipher_request *areq) struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); struct talitos_edesc *edesc; + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher)); + + if (!areq->nbytes) + return 0; + + if (areq->nbytes % blocksize) + return -EINVAL; /* allocate extended descriptor */ edesc = ablkcipher_edesc_alloc(areq, true); @@ -1773,6 +1781,14 @@ static int ablkcipher_decrypt(struct ablkcipher_request *areq) struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); struct talitos_edesc *edesc; + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher)); + + if (!areq->nbytes) + return 0; + + if (areq->nbytes % blocksize) + return -EINVAL; /* allocate extended descriptor */ edesc = ablkcipher_edesc_alloc(areq, false); From d84cc9c9524ec5973a337533e6d8ccd3e5f05f2b Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:13 +0000 Subject: [PATCH 040/143] crypto: talitos - fix ECB algs ivsize ECB's ivsize must be 0. Signed-off-by: Christophe Leroy Fixes: 5e75ae1b3cef ("crypto: talitos - add new crypto modes") Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 1e5410f92166..6f6f34754ad8 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -2809,7 +2809,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, .setkey = ablkcipher_aes_setkey, } }, @@ -2862,7 +2861,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, .setkey = ablkcipher_des_setkey, } }, @@ -2897,7 +2895,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_ablkcipher = { .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, .setkey = ablkcipher_des3_setkey, } }, From 7ede4c36cf7c6516986ee9d75b197c8bf73ea96f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:14 +0000 Subject: [PATCH 041/143] crypto: talitos - Do not modify req->cryptlen on decryption. For decrypt, req->cryptlen includes the size of the authentication part while all functions of the driver expect cryptlen to be the size of the encrypted data. As it is not expected to change req->cryptlen, this patch implements local calculation of cryptlen. Signed-off-by: Christophe Leroy Fixes: 9c4a79653b35 ("crypto: talitos - Freescale integrated security engine (SEC) driver") Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 6f6f34754ad8..a15aa6d6ec33 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1025,11 +1025,13 @@ static void talitos_sg_unmap(struct device *dev, static void ipsec_esp_unmap(struct device *dev, struct talitos_edesc *edesc, - struct aead_request *areq) + struct aead_request *areq, bool encrypt) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(aead); unsigned int ivsize = crypto_aead_ivsize(aead); + unsigned int authsize = crypto_aead_authsize(aead); + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; @@ -1038,7 +1040,7 @@ static void ipsec_esp_unmap(struct device *dev, DMA_FROM_DEVICE); unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); - talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, + talitos_sg_unmap(dev, edesc, areq->src, areq->dst, cryptlen, areq->assoclen); if (edesc->dma_len) @@ -1049,7 +1051,7 @@ static void ipsec_esp_unmap(struct device *dev, unsigned int dst_nents = edesc->dst_nents ? : 1; sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, - areq->assoclen + areq->cryptlen - ivsize); + areq->assoclen + cryptlen - ivsize); } } @@ -1072,7 +1074,7 @@ static void ipsec_esp_encrypt_done(struct device *dev, edesc = container_of(desc, struct talitos_edesc, desc); - ipsec_esp_unmap(dev, edesc, areq); + ipsec_esp_unmap(dev, edesc, areq, true); /* copy the generated ICV to dst */ if (edesc->icv_ool) { @@ -1108,7 +1110,7 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, edesc = container_of(desc, struct talitos_edesc, desc); - ipsec_esp_unmap(dev, edesc, req); + ipsec_esp_unmap(dev, edesc, req, false); if (!err) { /* auth check */ @@ -1145,7 +1147,7 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev, edesc = container_of(desc, struct talitos_edesc, desc); - ipsec_esp_unmap(dev, edesc, req); + ipsec_esp_unmap(dev, edesc, req, false); /* check ICV auth status */ if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != @@ -1248,6 +1250,7 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src, * fill in and submit ipsec_esp descriptor */ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, + bool encrypt, void (*callback)(struct device *dev, struct talitos_desc *desc, void *context, int error)) @@ -1257,7 +1260,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, struct talitos_ctx *ctx = crypto_aead_ctx(aead); struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; - unsigned int cryptlen = areq->cryptlen; + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); unsigned int ivsize = crypto_aead_ivsize(aead); int tbl_off = 0; int sg_count, ret; @@ -1384,7 +1387,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { - ipsec_esp_unmap(dev, edesc, areq); + ipsec_esp_unmap(dev, edesc, areq, encrypt); kfree(edesc); } return ret; @@ -1502,9 +1505,10 @@ static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); unsigned int ivsize = crypto_aead_ivsize(authenc); + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, - iv, areq->assoclen, areq->cryptlen, + iv, areq->assoclen, cryptlen, authsize, ivsize, icv_stashing, areq->base.flags, encrypt); } @@ -1523,7 +1527,7 @@ static int aead_encrypt(struct aead_request *req) /* set encrypt */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; - return ipsec_esp(edesc, req, ipsec_esp_encrypt_done); + return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done); } static int aead_decrypt(struct aead_request *req) @@ -1536,8 +1540,6 @@ static int aead_decrypt(struct aead_request *req) struct scatterlist *sg; void *icvdata; - req->cryptlen -= authsize; - /* allocate extended descriptor */ edesc = aead_edesc_alloc(req, req->iv, 1, false); if (IS_ERR(edesc)) @@ -1554,7 +1556,8 @@ static int aead_decrypt(struct aead_request *req) /* reset integrity check result bits */ - return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done); + return ipsec_esp(edesc, req, false, + ipsec_esp_decrypt_hwauth_done); } /* Have to check the ICV with software */ @@ -1571,7 +1574,7 @@ static int aead_decrypt(struct aead_request *req) memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize); - return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done); + return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done); } static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, From 4bbfb839259a9c96a0be872e16f7471b7136aee5 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:15 +0000 Subject: [PATCH 042/143] crypto: talitos - HMAC SNOOP NO AFEU mode requires SW icv checking. In that mode, hardware ICV verification is not supported. Signed-off-by: Christophe Leroy Fixes: 7405c8d7ff97 ("crypto: talitos - templates for AEAD using HMAC_SNOOP_NO_AFEU") Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index a15aa6d6ec33..e35581d67315 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1545,7 +1545,8 @@ static int aead_decrypt(struct aead_request *req) if (IS_ERR(edesc)) return PTR_ERR(edesc); - if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) && + if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && + (priv->features & TALITOS_FTR_HW_AUTH_CHECK) && ((!edesc->src_nents && !edesc->dst_nents) || priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { From eae55a586c3c8b50982bad3c3426e9c9dd7a0075 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:17 +0000 Subject: [PATCH 043/143] crypto: talitos - properly handle split ICV. The driver assumes that the ICV is as a single piece in the last element of the scatterlist. This assumption is wrong. This patch ensures that the ICV is properly handled regardless of the scatterlist layout. Fixes: 9c4a79653b35 ("crypto: talitos - Freescale integrated security engine (SEC) driver") Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index e35581d67315..7c8a3a717b91 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1069,7 +1069,6 @@ static void ipsec_esp_encrypt_done(struct device *dev, unsigned int authsize = crypto_aead_authsize(authenc); unsigned int ivsize = crypto_aead_ivsize(authenc); struct talitos_edesc *edesc; - struct scatterlist *sg; void *icvdata; edesc = container_of(desc, struct talitos_edesc, desc); @@ -1083,9 +1082,8 @@ static void ipsec_esp_encrypt_done(struct device *dev, else icvdata = &edesc->link_tbl[edesc->src_nents + edesc->dst_nents + 2]; - sg = sg_last(areq->dst, edesc->dst_nents); - memcpy((char *)sg_virt(sg) + sg->length - authsize, - icvdata, authsize); + sg_pcopy_from_buffer(areq->dst, edesc->dst_nents ? : 1, icvdata, + authsize, areq->assoclen + areq->cryptlen); } dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); @@ -1103,7 +1101,6 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, struct crypto_aead *authenc = crypto_aead_reqtfm(req); unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_edesc *edesc; - struct scatterlist *sg; char *oicv, *icv; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); @@ -1113,9 +1110,18 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, ipsec_esp_unmap(dev, edesc, req, false); if (!err) { + char icvdata[SHA512_DIGEST_SIZE]; + int nents = edesc->dst_nents ? : 1; + unsigned int len = req->assoclen + req->cryptlen; + /* auth check */ - sg = sg_last(req->dst, edesc->dst_nents ? : 1); - icv = (char *)sg_virt(sg) + sg->length - authsize; + if (nents > 1) { + sg_pcopy_to_buffer(req->dst, nents, icvdata, authsize, + len - authsize); + icv = icvdata; + } else { + icv = (char *)sg_virt(req->dst) + len - authsize; + } if (edesc->dma_len) { if (is_sec1) @@ -1537,7 +1543,6 @@ static int aead_decrypt(struct aead_request *req) struct talitos_ctx *ctx = crypto_aead_ctx(authenc); struct talitos_private *priv = dev_get_drvdata(ctx->dev); struct talitos_edesc *edesc; - struct scatterlist *sg; void *icvdata; /* allocate extended descriptor */ @@ -1571,9 +1576,8 @@ static int aead_decrypt(struct aead_request *req) else icvdata = &edesc->link_tbl[0]; - sg = sg_last(req->src, edesc->src_nents ? : 1); - - memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize); + sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize, + req->assoclen + req->cryptlen - authsize); return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done); } From c9cca7034b34a2d82e9a03b757de2485c294851c Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:18 +0000 Subject: [PATCH 044/143] crypto: talitos - Align SEC1 accesses to 32 bits boundaries. The MPC885 reference manual states: SEC Lite-initiated 8xx writes can occur only on 32-bit-word boundaries, but reads can occur on any byte boundary. Writing back a header read from a non-32-bit-word boundary will yield unpredictable results. In order to ensure that, cra_alignmask is set to 3 for SEC1. Signed-off-by: Christophe Leroy Fixes: 9c4a79653b35 ("crypto: talitos - Freescale integrated security engine (SEC) driver") Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 7c8a3a717b91..750b0159e654 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -3327,7 +3327,10 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, alg->cra_priority = t_alg->algt.priority; else alg->cra_priority = TALITOS_CRA_PRIORITY; - alg->cra_alignmask = 0; + if (has_ftr_sec1(priv)) + alg->cra_alignmask = 3; + else + alg->cra_alignmask = 0; alg->cra_ctxsize = sizeof(struct talitos_ctx); alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; From e345177ded17611e36c067751d63d64bf106cb54 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:19 +0000 Subject: [PATCH 045/143] crypto: talitos - fix AEAD processing. This driver is working well in 'simple cases', but as soon as more exotic SG lists are provided (dst different from src, auth part not in a single SG fragment, ...) there are wrong results, overruns, etc ... This patch cleans up the AEAD processing by: - Simplifying the location of 'out of line' ICV - Never using 'out of line' ICV on encryp - Always using 'out of line' ICV on decrypt - Forcing the generation of a SG table on decrypt Signed-off-by: Christophe Leroy Fixes: aeb4c132f33d ("crypto: talitos - Convert to new AEAD interface") Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 162 ++++++++++++++------------------------- drivers/crypto/talitos.h | 2 +- 2 files changed, 57 insertions(+), 107 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 750b0159e654..12917fd54bcb 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1040,8 +1040,8 @@ static void ipsec_esp_unmap(struct device *dev, DMA_FROM_DEVICE); unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); - talitos_sg_unmap(dev, edesc, areq->src, areq->dst, cryptlen, - areq->assoclen); + talitos_sg_unmap(dev, edesc, areq->src, areq->dst, + cryptlen + authsize, areq->assoclen); if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, @@ -1062,30 +1062,15 @@ static void ipsec_esp_encrypt_done(struct device *dev, struct talitos_desc *desc, void *context, int err) { - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); struct aead_request *areq = context; struct crypto_aead *authenc = crypto_aead_reqtfm(areq); - unsigned int authsize = crypto_aead_authsize(authenc); unsigned int ivsize = crypto_aead_ivsize(authenc); struct talitos_edesc *edesc; - void *icvdata; edesc = container_of(desc, struct talitos_edesc, desc); ipsec_esp_unmap(dev, edesc, areq, true); - /* copy the generated ICV to dst */ - if (edesc->icv_ool) { - if (is_sec1) - icvdata = edesc->buf + areq->assoclen + areq->cryptlen; - else - icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; - sg_pcopy_from_buffer(areq->dst, edesc->dst_nents ? : 1, icvdata, - authsize, areq->assoclen + areq->cryptlen); - } - dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); kfree(edesc); @@ -1102,39 +1087,15 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_edesc *edesc; char *oicv, *icv; - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); edesc = container_of(desc, struct talitos_edesc, desc); ipsec_esp_unmap(dev, edesc, req, false); if (!err) { - char icvdata[SHA512_DIGEST_SIZE]; - int nents = edesc->dst_nents ? : 1; - unsigned int len = req->assoclen + req->cryptlen; - /* auth check */ - if (nents > 1) { - sg_pcopy_to_buffer(req->dst, nents, icvdata, authsize, - len - authsize); - icv = icvdata; - } else { - icv = (char *)sg_virt(req->dst) + len - authsize; - } - - if (edesc->dma_len) { - if (is_sec1) - oicv = (char *)&edesc->dma_link_tbl + - req->assoclen + req->cryptlen; - else - oicv = (char *) - &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; - if (edesc->icv_ool) - icv = oicv + authsize; - } else - oicv = (char *)&edesc->link_tbl[0]; + oicv = edesc->buf + edesc->dma_len; + icv = oicv - authsize; err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; } @@ -1170,11 +1131,12 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev, * stop at cryptlen bytes */ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, - unsigned int offset, int cryptlen, + unsigned int offset, int datalen, int elen, struct talitos_ptr *link_tbl_ptr) { - int n_sg = sg_count; + int n_sg = elen ? sg_count + 1 : sg_count; int count = 0; + int cryptlen = datalen + elen; while (cryptlen && sg && n_sg--) { unsigned int len = sg_dma_len(sg); @@ -1189,11 +1151,20 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, if (len > cryptlen) len = cryptlen; + if (datalen > 0 && len > datalen) { + to_talitos_ptr(link_tbl_ptr + count, + sg_dma_address(sg) + offset, datalen, 0); + to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); + count++; + len -= datalen; + offset += datalen; + } to_talitos_ptr(link_tbl_ptr + count, sg_dma_address(sg) + offset, len, 0); to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); count++; cryptlen -= len; + datalen -= len; offset = 0; next: @@ -1203,7 +1174,7 @@ next: /* tag end of link table */ if (count > 0) to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, - DESC_PTR_LNKTBL_RETURN, 0); + DESC_PTR_LNKTBL_RET, 0); return count; } @@ -1211,7 +1182,8 @@ next: static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, unsigned int len, struct talitos_edesc *edesc, struct talitos_ptr *ptr, int sg_count, - unsigned int offset, int tbl_off, int elen) + unsigned int offset, int tbl_off, int elen, + bool force) { struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); @@ -1221,7 +1193,7 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, return 1; } to_talitos_ptr_ext_set(ptr, elen, is_sec1); - if (sg_count == 1) { + if (sg_count == 1 && !force) { to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1); return sg_count; } @@ -1229,9 +1201,9 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1); return sg_count; } - sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len + elen, + sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, &edesc->link_tbl[tbl_off]); - if (sg_count == 1) { + if (sg_count == 1 && !force) { /* Only one segment now, so no link tbl needed*/ copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); return sg_count; @@ -1249,7 +1221,7 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src, unsigned int offset, int tbl_off) { return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, - tbl_off, 0); + tbl_off, 0, false); } /* @@ -1277,6 +1249,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; + dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize; /* hmac key */ to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); @@ -1316,7 +1289,8 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, elen = authsize; ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], - sg_count, areq->assoclen, tbl_off, elen); + sg_count, areq->assoclen, tbl_off, elen, + false); if (ret > 1) { tbl_off += ret; @@ -1330,55 +1304,35 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); } - ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], - sg_count, areq->assoclen, tbl_off); - - if (is_ipsec_esp) - to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); + if (is_ipsec_esp && encrypt) + elen = authsize; + else + elen = 0; + ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], + sg_count, areq->assoclen, tbl_off, elen, + is_ipsec_esp && !encrypt); + tbl_off += ret; /* ICV data */ - if (ret > 1) { - tbl_off += ret; - edesc->icv_ool = true; + edesc->icv_ool = !encrypt; + + if (!encrypt && is_ipsec_esp) { + struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; + + /* Add an entry to the link table for ICV data */ + to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); + to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1); + + /* icv data follows link tables */ + to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1); + to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); + sync_needed = true; + } else if (!encrypt) { + to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1); sync_needed = true; - - if (is_ipsec_esp) { - struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; - int offset = (edesc->src_nents + edesc->dst_nents + 2) * - sizeof(struct talitos_ptr) + authsize; - - /* Add an entry to the link table for ICV data */ - to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); - to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN, - is_sec1); - - /* icv data follows link tables */ - to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset, - authsize, is_sec1); - } else { - dma_addr_t addr = edesc->dma_link_tbl; - - if (is_sec1) - addr += areq->assoclen + cryptlen; - else - addr += sizeof(struct talitos_ptr) * tbl_off; - - to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1); - } } else if (!is_ipsec_esp) { - ret = talitos_sg_map(dev, areq->dst, authsize, edesc, - &desc->ptr[6], sg_count, areq->assoclen + - cryptlen, - tbl_off); - if (ret > 1) { - tbl_off += ret; - edesc->icv_ool = true; - sync_needed = true; - } else { - edesc->icv_ool = false; - } - } else { - edesc->icv_ool = false; + talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6], + sg_count, areq->assoclen + cryptlen, tbl_off); } /* iv out */ @@ -1461,18 +1415,18 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, * and space for two sets of ICVs (stashed and generated) */ alloc_len = sizeof(struct talitos_edesc); - if (src_nents || dst_nents) { + if (src_nents || dst_nents || !encrypt) { if (is_sec1) dma_len = (src_nents ? src_len : 0) + - (dst_nents ? dst_len : 0); + (dst_nents ? dst_len : 0) + authsize; else dma_len = (src_nents + dst_nents + 2) * - sizeof(struct talitos_ptr) + authsize * 2; + sizeof(struct talitos_ptr) + authsize; alloc_len += dma_len; } else { dma_len = 0; - alloc_len += icv_stashing ? authsize : 0; } + alloc_len += icv_stashing ? authsize : 0; /* if its a ahash, add space for a second desc next to the first one */ if (is_sec1 && !dst) @@ -1570,11 +1524,7 @@ static int aead_decrypt(struct aead_request *req) edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; /* stash incoming ICV for later cmp with ICV generated by the h/w */ - if (edesc->dma_len) - icvdata = (char *)&edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; - else - icvdata = &edesc->link_tbl[0]; + icvdata = edesc->buf + edesc->dma_len; sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize, req->assoclen + req->cryptlen - authsize); diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index a65a63e0d6c1..dbedd0956c8a 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -412,5 +412,5 @@ static inline bool has_ftr_sec1(struct talitos_private *priv) /* link table extent field bits */ #define DESC_PTR_LNKTBL_JUMP 0x80 -#define DESC_PTR_LNKTBL_RETURN 0x02 +#define DESC_PTR_LNKTBL_RET 0x02 #define DESC_PTR_LNKTBL_NEXT 0x01 From fbb8d46e161644a13b81337f1235cd419e046318 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:20 +0000 Subject: [PATCH 046/143] Revert "crypto: talitos - export the talitos_submit function" There is no other file using talitos_submit in the kernel tree, so it doesn't need to be exported nor made global. This reverts commit 865d506155b117edc7e668ced373030ce7108ce9. Signed-off-by: Christophe Leroy Fixes: 865d506155b1 ("crypto: talitos - export the talitos_submit function") Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 11 +++++------ drivers/crypto/talitos.h | 6 ------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 12917fd54bcb..6fe900d8e5d8 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -278,11 +278,11 @@ static int init_device(struct device *dev) * callback must check err and feedback in descriptor header * for device processing status. */ -int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, - void (*callback)(struct device *dev, - struct talitos_desc *desc, - void *context, int error), - void *context) +static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, + void (*callback)(struct device *dev, + struct talitos_desc *desc, + void *context, int error), + void *context) { struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request; @@ -332,7 +332,6 @@ int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, return -EINPROGRESS; } -EXPORT_SYMBOL(talitos_submit); /* * process what was done, notify callback of error if not diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index dbedd0956c8a..95e97951b924 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -150,12 +150,6 @@ struct talitos_private { bool rng_registered; }; -extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, - void (*callback)(struct device *dev, - struct talitos_desc *desc, - void *context, int error), - void *context); - /* .features flag */ #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001 #define TALITOS_FTR_HW_AUTH_CHECK 0x00000002 From 89b32dfe008caa9c14ed42cdba656b6d1b753550 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:21 +0000 Subject: [PATCH 047/143] crypto: talitos - use IS_ENABLED() in has_ftr_sec1() This patch rewrites has_ftr_sec1() using IS_ENABLED() instead of #ifdefs Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.h | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 95e97951b924..5699d46401e6 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -164,13 +164,11 @@ struct talitos_private { */ static inline bool has_ftr_sec1(struct talitos_private *priv) { -#if defined(CONFIG_CRYPTO_DEV_TALITOS1) && defined(CONFIG_CRYPTO_DEV_TALITOS2) - return priv->features & TALITOS_FTR_SEC1 ? true : false; -#elif defined(CONFIG_CRYPTO_DEV_TALITOS1) - return true; -#else - return false; -#endif + if (IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS1) && + IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS2)) + return priv->features & TALITOS_FTR_SEC1; + + return IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS1); } /* From ff9f902cce5dc59392f5d0f9dc7f9979a565feb1 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 21 May 2019 13:34:22 +0000 Subject: [PATCH 048/143] crypto: talitos - use SPDX-License-Identifier This patch drops the license text and replaces it with an SPDX-License-Identifier tag. Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 15 +-------------- drivers/crypto/talitos.h | 25 +------------------------ 2 files changed, 2 insertions(+), 38 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 6fe900d8e5d8..32a7e747dc5f 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * talitos - Freescale Integrated Security Engine (SEC) device driver * @@ -9,20 +10,6 @@ * Crypto algorithm registration code copied from hifn driver: * 2007+ Copyright (c) Evgeniy Polyakov * All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 5699d46401e6..32ad4fc679ed 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -1,31 +1,8 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ /* * Freescale SEC (talitos) device register and descriptor header defines * * Copyright (c) 2006-2011 Freescale Semiconductor, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * */ #define TALITOS_TIMEOUT 100000 From ff455ad9a7f850daf874114e8d38c04aab9f39a2 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 23 May 2019 14:35:30 +0800 Subject: [PATCH 049/143] crypto: ixp4xx - Fix cross-compile errors due to type mismatch This patch changes multiple uses of u32s to dma_addr_t where the physical address is used. This fixes COMPILE_TEST errors on 64-bit platforms. Signed-off-by: Herbert Xu --- drivers/crypto/ixp4xx_crypto.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index f5414b6dfb55..3f40be34ac95 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -104,7 +104,7 @@ struct buffer_desc { u16 pkt_len; u16 buf_len; #endif - u32 phys_addr; + dma_addr_t phys_addr; u32 __reserved[4]; struct buffer_desc *next; enum dma_data_direction dir; @@ -121,9 +121,9 @@ struct crypt_ctl { u8 mode; /* NPE_OP_* operation mode */ #endif u8 iv[MAX_IVLEN]; /* IV for CBC mode or CTR IV for CTR mode */ - u32 icv_rev_aes; /* icv or rev aes */ - u32 src_buf; - u32 dst_buf; + dma_addr_t icv_rev_aes; /* icv or rev aes */ + dma_addr_t src_buf; + dma_addr_t dst_buf; #ifdef __ARMEB__ u16 auth_offs; /* Authentication start offset */ u16 auth_len; /* Authentication data length */ @@ -324,7 +324,8 @@ static struct crypt_ctl *get_crypt_desc_emerg(void) } } -static void free_buf_chain(struct device *dev, struct buffer_desc *buf,u32 phys) +static void free_buf_chain(struct device *dev, struct buffer_desc *buf, + dma_addr_t phys) { while (buf) { struct buffer_desc *buf1; @@ -606,7 +607,7 @@ static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target, struct buffer_desc *buf; int i; u8 *pad; - u32 pad_phys, buf_phys; + dma_addr_t pad_phys, buf_phys; BUILD_BUG_ON(NPE_CTX_LEN < HMAC_PAD_BLOCKLEN); pad = dma_pool_alloc(ctx_pool, GFP_KERNEL, &pad_phys); @@ -791,7 +792,7 @@ static struct buffer_desc *chainup_buffers(struct device *dev, for (; nbytes > 0; sg = sg_next(sg)) { unsigned len = min(nbytes, sg->length); struct buffer_desc *next_buf; - u32 next_buf_phys; + dma_addr_t next_buf_phys; void *ptr; nbytes -= len; From f6b99b6e18657aa1222b0c51ad20ac674e764281 Mon Sep 17 00:00:00 2001 From: Sascha Hauer Date: Thu, 23 May 2019 10:50:27 +0200 Subject: [PATCH 050/143] crypto: caam - print IV only when non NULL Since eaed71a44ad9 ("crypto: caam - add ecb(*) support") the IV can be NULL, so only dump it when it's non NULL as designated by the ivsize variable. Fixes: eaed71a44ad9 ("crypto: caam - add ecb(*) support") Signed-off-by: Sascha Hauer Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 08c36ae5d12d..3a64f74dbdce 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1021,9 +1021,10 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, ivsize, ivsize, 0); #ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents > 1 ? 100 : ivsize, 1); + if (ivsize) + print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); #endif caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, From 32f75e67f3457479d79bc5c26b0eb4e2bc150cf0 Mon Sep 17 00:00:00 2001 From: Sascha Hauer Date: Thu, 23 May 2019 10:50:28 +0200 Subject: [PATCH 051/143] crypto: caam - remove unused defines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CAAM driver defines its own debug() macro, but it is unused. Remove it. Signed-off-by: Sascha Hauer Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 7 ------- drivers/crypto/caam/caamhash.c | 8 -------- 2 files changed, 15 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 3a64f74dbdce..757675cc3f99 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -77,13 +77,6 @@ #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) -#ifdef DEBUG -/* for print_hex_dumps with line references */ -#define debug(format, arg...) printk(format, arg) -#else -#define debug(format, arg...) -#endif - struct caam_alg_entry { int class1_alg_type; int class2_alg_type; diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index a67c62e444ca..508de427a55f 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -82,14 +82,6 @@ #define HASH_MSG_LEN 8 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) -#ifdef DEBUG -/* for print_hex_dumps with line references */ -#define debug(format, arg...) printk(format, arg) -#else -#define debug(format, arg...) -#endif - - static struct list_head hash_list; /* ahash per-session context */ From 6e005503199b9bf1b385949c05897fd6567b5af4 Mon Sep 17 00:00:00 2001 From: Sascha Hauer Date: Thu, 23 May 2019 10:50:29 +0200 Subject: [PATCH 052/143] crypto: caam - print debug messages at debug level The CAAM driver used to put its debug messages inside #ifdef DEBUG and then prints the messages at KERN_ERR level. Replace this with proper functions printing at KERN_DEBUG level. The #ifdef DEBUG gets unnecessary when the right functions are used. This replaces: - print_hex_dump(KERN_ERR ...) inside #ifdef DEBUG with print_hex_dump_debug(...) - dev_err() inside #ifdef DEBUG with dev_dbg() - printk(KERN_ERR ...) inside #ifdef DEBUG with dev_dbg() Some parts of the driver use these functions already, so it is only consequent to use the debug function consistently. Signed-off-by: Sascha Hauer Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 145 ++++++++----------- drivers/crypto/caam/caamalg_desc.c | 116 ++++++--------- drivers/crypto/caam/caamalg_qi.c | 45 ++---- drivers/crypto/caam/caamhash.c | 225 ++++++++++++----------------- drivers/crypto/caam/caamrng.c | 22 ++- drivers/crypto/caam/key_gen.c | 28 ++-- drivers/crypto/caam/sg_sw_sec4.h | 8 +- 7 files changed, 236 insertions(+), 353 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 757675cc3f99..1598ce7b6cf1 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -576,13 +576,11 @@ static int aead_setkey(struct crypto_aead *aead, if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; -#ifdef DEBUG - printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", + dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* * If DKP is supported, use it in the shared descriptor to generate @@ -616,11 +614,10 @@ static int aead_setkey(struct crypto_aead *aead, memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, - ctx->adata.keylen_pad + keys.enckeylen, 1); -#endif + + print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); skip_split_key: ctx->cdata.keylen = keys.enckeylen; @@ -671,10 +668,8 @@ static int gcm_setkey(struct crypto_aead *aead, struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); @@ -692,10 +687,8 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -718,10 +711,8 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -750,10 +741,8 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* * AES-CTR needs to load IV in CONTEXT1 reg * at an offset of 128bits (16bytes) @@ -942,9 +931,7 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, struct aead_request *req = context; struct aead_edesc *edesc; -#ifdef DEBUG - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct aead_edesc, hw_desc[0]); @@ -964,9 +951,7 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, struct aead_request *req = context; struct aead_edesc *edesc; -#ifdef DEBUG - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct aead_edesc, hw_desc[0]); @@ -994,9 +979,7 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); int ivsize = crypto_skcipher_ivsize(skcipher); -#ifdef DEBUG - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]); @@ -1013,12 +996,11 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize, ivsize, 0); -#ifdef DEBUG if (ivsize) - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents > 1 ? 100 : ivsize, 1); -#endif + print_hex_dump_debug("dstiv @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); @@ -1033,12 +1015,10 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, { struct skcipher_request *req = context; struct skcipher_edesc *edesc; -#ifdef DEBUG struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); int ivsize = crypto_skcipher_ivsize(skcipher); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]); if (err) @@ -1046,10 +1026,8 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, skcipher_unmap(jrdev, edesc, req); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); -#endif + print_hex_dump_debug("dstiv @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); @@ -1245,6 +1223,7 @@ static void init_skcipher_job(struct skcipher_request *req, { struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); + struct device *jrdev = ctx->jrdev; int ivsize = crypto_skcipher_ivsize(skcipher); u32 *desc = edesc->hw_desc; u32 *sh_desc; @@ -1252,12 +1231,11 @@ static void init_skcipher_job(struct skcipher_request *req, dma_addr_t src_dma, dst_dma, ptr; int len, sec4_sg_index = 0; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); - pr_err("asked=%d, cryptlen%d\n", + print_hex_dump_debug("presciv@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); + dev_dbg(jrdev, "asked=%d, cryptlen%d\n", (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen); -#endif + caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->src, edesc->src_nents > 1 ? 100 : req->cryptlen, 1); @@ -1450,11 +1428,10 @@ static int gcm_encrypt(struct aead_request *req) /* Create and submit job descriptor */ init_gcm_job(req, edesc, all_contig, true); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); @@ -1560,11 +1537,10 @@ static int aead_encrypt(struct aead_request *req) /* Create and submit job descriptor */ init_authenc_job(req, edesc, all_contig, true); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); @@ -1595,11 +1571,10 @@ static int gcm_decrypt(struct aead_request *req) /* Create and submit job descriptor*/ init_gcm_job(req, edesc, all_contig, false); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); @@ -1643,11 +1618,10 @@ static int aead_decrypt(struct aead_request *req) /* Create and submit job descriptor*/ init_authenc_job(req, edesc, all_contig, false); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); @@ -1804,11 +1778,9 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, edesc->iv_dma = iv_dma; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "skcipher sec4_sg@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, - sec4_sg_bytes, 1); -#endif + print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, + sec4_sg_bytes, 1); return edesc; } @@ -1829,11 +1801,11 @@ static int skcipher_encrypt(struct skcipher_request *req) /* Create and submit job descriptor*/ init_skcipher_job(req, edesc, true); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); + desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, skcipher_encrypt_done, req); @@ -1873,11 +1845,10 @@ static int skcipher_decrypt(struct skcipher_request *req) /* Create and submit job descriptor*/ init_skcipher_job(req, edesc, false); desc = edesc->hw_desc; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); ret = caam_jr_enqueue(jrdev, desc, skcipher_decrypt_done, req); if (!ret) { diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index 1e1a376edc2f..a73b79c5d46c 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -115,11 +115,9 @@ void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead null enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); @@ -204,11 +202,9 @@ void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead null dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); @@ -358,10 +354,9 @@ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_encap); @@ -475,10 +470,9 @@ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_decap); @@ -613,11 +607,9 @@ copy_iv: append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead givenc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); @@ -742,10 +734,9 @@ void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); @@ -838,10 +829,9 @@ void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); @@ -933,11 +923,9 @@ void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4106 enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); @@ -1030,11 +1018,9 @@ void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4106 dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); @@ -1115,11 +1101,9 @@ void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4543 enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); @@ -1205,11 +1189,9 @@ void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4543 dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); @@ -1416,11 +1398,9 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "skcipher enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap); @@ -1487,11 +1467,9 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "skcipher dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap); @@ -1538,11 +1516,9 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "xts skcipher enc shdesc@" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, + desc, desc_bytes(desc), 1); } EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap); @@ -1588,11 +1564,9 @@ void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata) /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "xts skcipher dec shdesc@" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); } EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap); diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index c6bb804d82a8..742b111e3c6e 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -214,13 +214,11 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; -#ifdef DEBUG - dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", + dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* * If DKP is supported, use it in the shared descriptor to generate @@ -387,10 +385,8 @@ static int gcm_setkey(struct crypto_aead *aead, struct device *jrdev = ctx->jrdev; int ret; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen, @@ -487,10 +483,8 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); /* @@ -591,10 +585,8 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); /* @@ -646,10 +638,9 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, const bool is_rfc3686 = alg->caam.rfc3686; int ret = 0; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + /* * AES-CTR needs to load IV in CONTEXT1 reg * at an offset of 128bits (16bytes) @@ -1199,23 +1190,19 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status) struct device *qidev = caam_ctx->qidev; int ivsize = crypto_skcipher_ivsize(skcipher); -#ifdef DEBUG - dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); -#endif + dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); edesc = container_of(drv_req, typeof(*edesc), drv_req); if (status) caam_jr_strstatus(qidev, status); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents > 1 ? 100 : ivsize, 1); + print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); -#endif skcipher_unmap(qidev, edesc, req); diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 508de427a55f..a87526f62737 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -235,11 +235,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, true, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash update shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("ahash update shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); /* ahash_update_first shared descriptor */ desc = ctx->sh_desc_update_first; @@ -247,11 +246,9 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash update first shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("ahash update first shdesc@"__stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); /* ahash_final shared descriptor */ desc = ctx->sh_desc_fin; @@ -259,11 +256,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, true, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("ahash final shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); /* ahash_digest shared descriptor */ desc = ctx->sh_desc_digest; @@ -271,12 +267,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash digest shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("ahash digest shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); return 0; } @@ -320,9 +314,9 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, ctx->key_dma); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, desc_bytes(desc), ctx->dir); - print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)" : ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), - 1); + print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__) + " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); /* shared descriptor for ahash_digest */ desc = ctx->sh_desc_digest; @@ -369,8 +363,8 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, 0); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, desc_bytes(desc), ctx->dir); - print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)" : ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, + print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__) + " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); /* shared descriptor for ahash_digest */ @@ -421,12 +415,11 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("key_in@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); result.err = 0; init_completion(&result.completion); @@ -436,11 +429,10 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, /* in progress */ wait_for_completion(&result.completion); ret = result.err; -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "digested key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1); -#endif + + print_hex_dump_debug("digested key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, + digestsize, 1); } dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL); @@ -455,15 +447,14 @@ static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, unsigned int keylen) { struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct device *jrdev = ctx->jrdev; int blocksize = crypto_tfm_alg_blocksize(&ahash->base); int digestsize = crypto_ahash_digestsize(ahash); struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); int ret; u8 *hashed_key = NULL; -#ifdef DEBUG - printk(KERN_ERR "keylen %d\n", keylen); -#endif + dev_dbg(jrdev, "keylen %d\n", keylen); if (keylen > blocksize) { hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); @@ -592,11 +583,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); int digestsize = crypto_ahash_digestsize(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -606,11 +595,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err, memcpy(req->result, state->caam_ctx, digestsize); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); -#endif + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); req->base.complete(&req->base, err); } @@ -623,11 +610,9 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG int digestsize = crypto_ahash_digestsize(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -637,15 +622,13 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, switch_buf(state); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); if (req->result) - print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); -#endif + print_hex_dump_debug("result@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); req->base.complete(&req->base, err); } @@ -658,11 +641,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); int digestsize = crypto_ahash_digestsize(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -672,11 +653,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, memcpy(req->result, state->caam_ctx, digestsize); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); -#endif + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); req->base.complete(&req->base, err); } @@ -689,11 +668,9 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG int digestsize = crypto_ahash_digestsize(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -703,15 +680,13 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, switch_buf(state); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); if (req->result) - print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); -#endif + print_hex_dump_debug("result@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); req->base.complete(&req->base, err); } @@ -887,11 +862,9 @@ static int ahash_update_ctx(struct ahash_request *req) append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req); if (ret) @@ -904,13 +877,12 @@ static int ahash_update_ctx(struct ahash_request *req) *buflen = *next_buflen; *next_buflen = last_buflen; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); - print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); -#endif + + print_hex_dump_debug("buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); + print_hex_dump_debug("next buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, + *next_buflen, 1); return ret; unmap_ctx: @@ -970,10 +942,9 @@ static int ahash_final_ctx(struct ahash_request *req) LDST_SGF); append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); if (ret) @@ -1051,10 +1022,9 @@ static int ahash_finup_ctx(struct ahash_request *req) append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); if (ret) @@ -1128,10 +1098,9 @@ static int ahash_digest(struct ahash_request *req) return -ENOMEM; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); if (!ret) { @@ -1183,10 +1152,9 @@ static int ahash_final_no_ctx(struct ahash_request *req) if (ret) goto unmap; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); if (!ret) { @@ -1308,11 +1276,9 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); if (ret) @@ -1328,13 +1294,12 @@ static int ahash_update_no_ctx(struct ahash_request *req) *buflen = *next_buflen; *next_buflen = 0; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); - print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); -#endif + + print_hex_dump_debug("buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); + print_hex_dump_debug("next buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, + 1); return ret; unmap_ctx: @@ -1409,10 +1374,9 @@ static int ahash_finup_no_ctx(struct ahash_request *req) if (ret) goto unmap; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); if (!ret) { @@ -1512,11 +1476,9 @@ static int ahash_update_first(struct ahash_request *req) if (ret) goto unmap_ctx; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); if (ret) @@ -1534,11 +1496,10 @@ static int ahash_update_first(struct ahash_request *req) req->nbytes, 0); switch_buf(state); } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); -#endif + + print_hex_dump_debug("next buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, + 1); return ret; unmap_ctx: diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index 1ece4ed571a2..561bcb535184 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -113,10 +113,8 @@ static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context) /* Buffer refilled, invalidate cache */ dma_sync_single_for_cpu(jrdev, bd->addr, RN_BUF_SIZE, DMA_FROM_DEVICE); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rng refreshed buf@: ", - DUMP_PREFIX_ADDRESS, 16, 4, bd->buf, RN_BUF_SIZE, 1); -#endif + print_hex_dump_debug("rng refreshed buf@: ", DUMP_PREFIX_ADDRESS, 16, 4, + bd->buf, RN_BUF_SIZE, 1); } static inline int submit_job(struct caam_rng_ctx *ctx, int to_current) @@ -209,10 +207,10 @@ static inline int rng_create_sh_desc(struct caam_rng_ctx *ctx) dev_err(jrdev, "unable to map shared descriptor\n"); return -ENOMEM; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4, - desc, desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4, + desc, desc_bytes(desc), 1); + return 0; } @@ -233,10 +231,10 @@ static inline int rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id) } append_seq_out_ptr_intlen(desc, bd->addr, RN_BUF_SIZE, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4, - desc, desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4, + desc, desc_bytes(desc), 1); + return 0; } diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c index 8d0713fae6ac..48dd3536060d 100644 --- a/drivers/crypto/caam/key_gen.c +++ b/drivers/crypto/caam/key_gen.c @@ -16,9 +16,7 @@ void split_key_done(struct device *dev, u32 *desc, u32 err, { struct split_key_result *res = context; -#ifdef DEBUG - dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); if (err) caam_jr_strstatus(dev, err); @@ -55,12 +53,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out, adata->keylen_pad = split_key_pad_len(adata->algtype & OP_ALG_ALGSEL_MASK); -#ifdef DEBUG - dev_err(jrdev, "split keylen %d split keylen padded %d\n", + dev_dbg(jrdev, "split keylen %d split keylen padded %d\n", adata->keylen, adata->keylen_pad); - print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); -#endif + print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); if (adata->keylen_pad > max_keylen) return -EINVAL; @@ -102,10 +98,9 @@ int gen_split_key(struct device *jrdev, u8 *key_out, append_fifo_store(desc, dma_addr, adata->keylen, LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); result.err = 0; init_completion(&result.completion); @@ -115,11 +110,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out, /* in progress */ wait_for_completion(&result.completion); ret = result.err; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_out, - adata->keylen_pad, 1); -#endif + + print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_out, + adata->keylen_pad, 1); } dma_unmap_single(jrdev, dma_addr, adata->keylen_pad, DMA_BIDIRECTIONAL); diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index dbfa9fce33e0..8f9555d01011 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -35,11 +35,9 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK); } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ", - DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr, - sizeof(struct sec4_sg_entry), 1); -#endif + + print_hex_dump_debug("sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4, + sec4_sg_ptr, sizeof(struct sec4_sg_entry), 1); } /* From 8a82451bd04f57bfde8915f289fc2d1f9457abae Mon Sep 17 00:00:00 2001 From: Sascha Hauer Date: Thu, 23 May 2019 10:50:30 +0200 Subject: [PATCH 053/143] crypto: caam - print messages in caam_dump_sg at debug level caam_dump_sg() is only compiled in when DEBUG is defined, hence the messages are debug messages. Remove the @level argument from caam_dump_sg() and print all messages at debug level. Signed-off-by: Sascha Hauer Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 8 ++++---- drivers/crypto/caam/caamalg_qi.c | 2 +- drivers/crypto/caam/caamalg_qi2.c | 4 ++-- drivers/crypto/caam/error.c | 8 ++++---- drivers/crypto/caam/error.h | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 1598ce7b6cf1..1efa6f5b62cf 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1001,7 +1001,7 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, DUMP_PREFIX_ADDRESS, 16, 4, req->iv, edesc->src_nents > 1 ? 100 : ivsize, 1); - caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); @@ -1028,7 +1028,7 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, print_hex_dump_debug("dstiv @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); - caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); @@ -1236,7 +1236,7 @@ static void init_skcipher_job(struct skcipher_request *req, dev_dbg(jrdev, "asked=%d, cryptlen%d\n", (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen); - caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ", + caam_dump_sg("src @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->src, edesc->src_nents > 1 ? 100 : req->cryptlen, 1); @@ -1606,7 +1606,7 @@ static int aead_decrypt(struct aead_request *req) u32 *desc; int ret = 0; - caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ", + caam_dump_sg("dec src@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->src, req->assoclen + req->cryptlen, 1); diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 742b111e3c6e..490a2bbdbb93 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -1200,7 +1200,7 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status) print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv, edesc->src_nents > 1 ? 100 : ivsize, 1); - caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index c5d869ea95c5..2621d657048a 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -1366,7 +1366,7 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status) print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv, edesc->src_nents > 1 ? 100 : ivsize, 1); - caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ", + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); @@ -1404,7 +1404,7 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status) print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv, edesc->src_nents > 1 ? 100 : ivsize, 1); - caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ", + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c index 4da844e4b61d..4f0d45865aa2 100644 --- a/drivers/crypto/caam/error.c +++ b/drivers/crypto/caam/error.c @@ -13,7 +13,7 @@ #ifdef DEBUG #include -void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, +void caam_dump_sg(const char *prefix_str, int prefix_type, int rowsize, int groupsize, struct scatterlist *sg, size_t tlen, bool ascii) { @@ -35,15 +35,15 @@ void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, buf = it_page + it->offset; len = min_t(size_t, tlen, it->length); - print_hex_dump(level, prefix_str, prefix_type, rowsize, - groupsize, buf, len, ascii); + print_hex_dump_debug(prefix_str, prefix_type, rowsize, + groupsize, buf, len, ascii); tlen -= len; kunmap_atomic(it_page); } } #else -void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, +void caam_dump_sg(const char *prefix_str, int prefix_type, int rowsize, int groupsize, struct scatterlist *sg, size_t tlen, bool ascii) {} diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h index 8c6b83e02a70..d9726e66edbf 100644 --- a/drivers/crypto/caam/error.h +++ b/drivers/crypto/caam/error.h @@ -17,7 +17,7 @@ void caam_strstatus(struct device *dev, u32 status, bool qi_v2); #define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false) #define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true) -void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, +void caam_dump_sg(const char *prefix_str, int prefix_type, int rowsize, int groupsize, struct scatterlist *sg, size_t tlen, bool ascii); From 4befedc045367c359de8ba021120fcf3bf797992 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 24 May 2019 18:26:46 +0200 Subject: [PATCH 054/143] i2c: acpi: permit bus speed to be discovered after enumeration Currently, the I2C ACPI enumeration code only permits the max bus rate to be discovered before enumerating the slaves on the bus. In some cases, drivers for slave devices may require this information, e.g., some ATmel crypto drivers need to generate a so-called wake token of a fixed duration, regardless of the bus rate. So tweak the code so i2c_acpi_lookup_speed() is able to obtain this information after enumeration as well. Acked-by: Mika Westerberg Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- drivers/i2c/i2c-core-acpi.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c index 272800692088..7240cc07abb4 100644 --- a/drivers/i2c/i2c-core-acpi.c +++ b/drivers/i2c/i2c-core-acpi.c @@ -115,8 +115,7 @@ static int i2c_acpi_do_lookup(struct acpi_device *adev, struct list_head resource_list; int ret; - if (acpi_bus_get_status(adev) || !adev->status.present || - acpi_device_enumerated(adev)) + if (acpi_bus_get_status(adev) || !adev->status.present) return -EINVAL; if (acpi_match_device_ids(adev, i2c_acpi_ignored_device_ids) == 0) @@ -151,6 +150,9 @@ static int i2c_acpi_get_info(struct acpi_device *adev, lookup.info = info; lookup.index = -1; + if (acpi_device_enumerated(adev)) + return -EINVAL; + ret = i2c_acpi_do_lookup(adev, &lookup); if (ret) return ret; From 3c756aa346dfac5ae6b7fe81cb1c6380a20b1e41 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 24 May 2019 18:26:47 +0200 Subject: [PATCH 055/143] crypto: atmel-ecc - add support for ACPI probing on non-AT91 platforms The Atmel/Microchip EC508A is a I2C device that could be wired into any platform, and is being used on the Linaro/96boards Secure96 mezzanine adapter. This means it could be found on any platform, even on ones that use ACPI enumeration (via PRP0001 devices). So update the code to enable this use case. This involves tweaking the bus rate discovery code to take ACPI probing into account, which records the maximum bus rate as a property of the slave device. For the atmel-ecc code, this means that the effective bus rate should never exceed the maximum rate, unless we are dealing with buggy firmware. Nonetheless, let's just use the existing plumbing to discover the bus rate and keep the existing logic intact. Signed-off-by: Ard Biesheuvel Reviewed-by: Linus Walleij Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 1 - drivers/crypto/atmel-ecc.c | 13 ++++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 177b7713bd2d..119ba073f424 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -521,7 +521,6 @@ config CRYPTO_DEV_ATMEL_SHA config CRYPTO_DEV_ATMEL_ECC tristate "Support for Microchip / Atmel ECC hw accelerator" - depends on ARCH_AT91 || COMPILE_TEST depends on I2C select CRYPTO_ECDH select CRC16 diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index ba00e4563ca0..5705348f540f 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -657,11 +657,14 @@ static int atmel_ecc_probe(struct i2c_client *client, return -ENODEV; } - ret = of_property_read_u32(client->adapter->dev.of_node, - "clock-frequency", &bus_clk_rate); - if (ret) { - dev_err(dev, "of: failed to read clock-frequency property\n"); - return ret; + clk_rate = i2c_acpi_find_bus_speed(&client->adapter->dev); + if (!clk_rate) { + ret = device_property_read_u32(&client->adapter->dev, + "clock-frequency", &bus_clk_rate); + if (ret) { + dev_err(dev, "failed to read clock-frequency property\n"); + return ret; + } } if (bus_clk_rate > 1000000L) { From c34a320176a59445d76783e5ee043d6ecd22d011 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 24 May 2019 18:26:48 +0200 Subject: [PATCH 056/143] crypto: atmel-ecc - factor out code that can be shared In preparation of adding support for the random number generator in Atmel atsha204a devices, refactor the existing atmel-ecc driver (which drives hardware that is closely related) so we can share the basic I2C and command queuing routines. Reviewed-by: Linus Walleij Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 4 + drivers/crypto/Makefile | 1 + drivers/crypto/atmel-ecc.c | 406 ++------------------ drivers/crypto/atmel-i2c.c | 349 +++++++++++++++++ drivers/crypto/{atmel-ecc.h => atmel-i2c.h} | 80 +++- 5 files changed, 451 insertions(+), 389 deletions(-) create mode 100644 drivers/crypto/atmel-i2c.c rename drivers/crypto/{atmel-ecc.h => atmel-i2c.h} (51%) diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 119ba073f424..20674842261e 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -519,9 +519,13 @@ config CRYPTO_DEV_ATMEL_SHA To compile this driver as a module, choose M here: the module will be called atmel-sha. +config CRYPTO_DEV_ATMEL_I2C + tristate + config CRYPTO_DEV_ATMEL_ECC tristate "Support for Microchip / Atmel ECC hw accelerator" depends on I2C + select CRYPTO_DEV_ATMEL_I2C select CRYPTO_ECDH select CRC16 help diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index a23a7197fcd7..394e84089924 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_I2C) += atmel-i2c.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index 5705348f540f..ff02cc05affb 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -6,8 +6,6 @@ * Author: Tudor Ambarus */ -#include -#include #include #include #include @@ -23,41 +21,10 @@ #include #include #include -#include "atmel-ecc.h" - -/* Used for binding tfm objects to i2c clients. */ -struct atmel_ecc_driver_data { - struct list_head i2c_client_list; - spinlock_t i2c_list_lock; -} ____cacheline_aligned; +#include "atmel-i2c.h" static struct atmel_ecc_driver_data driver_data; -/** - * atmel_ecc_i2c_client_priv - i2c_client private data - * @client : pointer to i2c client device - * @i2c_client_list_node: part of i2c_client_list - * @lock : lock for sending i2c commands - * @wake_token : wake token array of zeros - * @wake_token_sz : size in bytes of the wake_token - * @tfm_count : number of active crypto transformations on i2c client - * - * Reads and writes from/to the i2c client are sequential. The first byte - * transmitted to the device is treated as the byte size. Any attempt to send - * more than this number of bytes will cause the device to not ACK those bytes. - * After the host writes a single command byte to the input buffer, reads are - * prohibited until after the device completes command execution. Use a mutex - * when sending i2c commands. - */ -struct atmel_ecc_i2c_client_priv { - struct i2c_client *client; - struct list_head i2c_client_list_node; - struct mutex lock; - u8 wake_token[WAKE_TOKEN_MAX_SIZE]; - size_t wake_token_sz; - atomic_t tfm_count ____cacheline_aligned; -}; - /** * atmel_ecdh_ctx - transformation context * @client : pointer to i2c client device @@ -80,188 +47,12 @@ struct atmel_ecdh_ctx { bool do_fallback; }; -/** - * atmel_ecc_work_data - data structure representing the work - * @ctx : transformation context. - * @cbk : pointer to a callback function to be invoked upon completion of this - * request. This has the form: - * callback(struct atmel_ecc_work_data *work_data, void *areq, u8 status) - * where: - * @work_data: data structure representing the work - * @areq : optional pointer to an argument passed with the original - * request. - * @status : status returned from the i2c client device or i2c error. - * @areq: optional pointer to a user argument for use at callback time. - * @work: describes the task to be executed. - * @cmd : structure used for communicating with the device. - */ -struct atmel_ecc_work_data { - struct atmel_ecdh_ctx *ctx; - void (*cbk)(struct atmel_ecc_work_data *work_data, void *areq, - int status); - void *areq; - struct work_struct work; - struct atmel_ecc_cmd cmd; -}; - -static u16 atmel_ecc_crc16(u16 crc, const u8 *buffer, size_t len) -{ - return cpu_to_le16(bitrev16(crc16(crc, buffer, len))); -} - -/** - * atmel_ecc_checksum() - Generate 16-bit CRC as required by ATMEL ECC. - * CRC16 verification of the count, opcode, param1, param2 and data bytes. - * The checksum is saved in little-endian format in the least significant - * two bytes of the command. CRC polynomial is 0x8005 and the initial register - * value should be zero. - * - * @cmd : structure used for communicating with the device. - */ -static void atmel_ecc_checksum(struct atmel_ecc_cmd *cmd) -{ - u8 *data = &cmd->count; - size_t len = cmd->count - CRC_SIZE; - u16 *crc16 = (u16 *)(data + len); - - *crc16 = atmel_ecc_crc16(0, data, len); -} - -static void atmel_ecc_init_read_cmd(struct atmel_ecc_cmd *cmd) -{ - cmd->word_addr = COMMAND; - cmd->opcode = OPCODE_READ; - /* - * Read the word from Configuration zone that contains the lock bytes - * (UserExtra, Selector, LockValue, LockConfig). - */ - cmd->param1 = CONFIG_ZONE; - cmd->param2 = DEVICE_LOCK_ADDR; - cmd->count = READ_COUNT; - - atmel_ecc_checksum(cmd); - - cmd->msecs = MAX_EXEC_TIME_READ; - cmd->rxsize = READ_RSP_SIZE; -} - -static void atmel_ecc_init_genkey_cmd(struct atmel_ecc_cmd *cmd, u16 keyid) -{ - cmd->word_addr = COMMAND; - cmd->count = GENKEY_COUNT; - cmd->opcode = OPCODE_GENKEY; - cmd->param1 = GENKEY_MODE_PRIVATE; - /* a random private key will be generated and stored in slot keyID */ - cmd->param2 = cpu_to_le16(keyid); - - atmel_ecc_checksum(cmd); - - cmd->msecs = MAX_EXEC_TIME_GENKEY; - cmd->rxsize = GENKEY_RSP_SIZE; -} - -static int atmel_ecc_init_ecdh_cmd(struct atmel_ecc_cmd *cmd, - struct scatterlist *pubkey) -{ - size_t copied; - - cmd->word_addr = COMMAND; - cmd->count = ECDH_COUNT; - cmd->opcode = OPCODE_ECDH; - cmd->param1 = ECDH_PREFIX_MODE; - /* private key slot */ - cmd->param2 = cpu_to_le16(DATA_SLOT_2); - - /* - * The device only supports NIST P256 ECC keys. The public key size will - * always be the same. Use a macro for the key size to avoid unnecessary - * computations. - */ - copied = sg_copy_to_buffer(pubkey, - sg_nents_for_len(pubkey, - ATMEL_ECC_PUBKEY_SIZE), - cmd->data, ATMEL_ECC_PUBKEY_SIZE); - if (copied != ATMEL_ECC_PUBKEY_SIZE) - return -EINVAL; - - atmel_ecc_checksum(cmd); - - cmd->msecs = MAX_EXEC_TIME_ECDH; - cmd->rxsize = ECDH_RSP_SIZE; - - return 0; -} - -/* - * After wake and after execution of a command, there will be error, status, or - * result bytes in the device's output register that can be retrieved by the - * system. When the length of that group is four bytes, the codes returned are - * detailed in error_list. - */ -static int atmel_ecc_status(struct device *dev, u8 *status) -{ - size_t err_list_len = ARRAY_SIZE(error_list); - int i; - u8 err_id = status[1]; - - if (*status != STATUS_SIZE) - return 0; - - if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR) - return 0; - - for (i = 0; i < err_list_len; i++) - if (error_list[i].value == err_id) - break; - - /* if err_id is not in the error_list then ignore it */ - if (i != err_list_len) { - dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text); - return err_id; - } - - return 0; -} - -static int atmel_ecc_wakeup(struct i2c_client *client) -{ - struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); - u8 status[STATUS_RSP_SIZE]; - int ret; - - /* - * The device ignores any levels or transitions on the SCL pin when the - * device is idle, asleep or during waking up. Don't check for error - * when waking up the device. - */ - i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz); - - /* - * Wait to wake the device. Typical execution times for ecdh and genkey - * are around tens of milliseconds. Delta is chosen to 50 microseconds. - */ - usleep_range(TWHI_MIN, TWHI_MAX); - - ret = i2c_master_recv(client, status, STATUS_SIZE); - if (ret < 0) - return ret; - - return atmel_ecc_status(&client->dev, status); -} - -static int atmel_ecc_sleep(struct i2c_client *client) -{ - u8 sleep = SLEEP_TOKEN; - - return i2c_master_send(client, &sleep, 1); -} - -static void atmel_ecdh_done(struct atmel_ecc_work_data *work_data, void *areq, +static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq, int status) { struct kpp_request *req = areq; struct atmel_ecdh_ctx *ctx = work_data->ctx; - struct atmel_ecc_cmd *cmd = &work_data->cmd; + struct atmel_i2c_cmd *cmd = &work_data->cmd; size_t copied, n_sz; if (status) @@ -282,82 +73,6 @@ free_work_data: kpp_request_complete(req, status); } -/* - * atmel_ecc_send_receive() - send a command to the device and receive its - * response. - * @client: i2c client device - * @cmd : structure used to communicate with the device - * - * After the device receives a Wake token, a watchdog counter starts within the - * device. After the watchdog timer expires, the device enters sleep mode - * regardless of whether some I/O transmission or command execution is in - * progress. If a command is attempted when insufficient time remains prior to - * watchdog timer execution, the device will return the watchdog timeout error - * code without attempting to execute the command. There is no way to reset the - * counter other than to put the device into sleep or idle mode and then - * wake it up again. - */ -static int atmel_ecc_send_receive(struct i2c_client *client, - struct atmel_ecc_cmd *cmd) -{ - struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); - int ret; - - mutex_lock(&i2c_priv->lock); - - ret = atmel_ecc_wakeup(client); - if (ret) - goto err; - - /* send the command */ - ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE); - if (ret < 0) - goto err; - - /* delay the appropriate amount of time for command to execute */ - msleep(cmd->msecs); - - /* receive the response */ - ret = i2c_master_recv(client, cmd->data, cmd->rxsize); - if (ret < 0) - goto err; - - /* put the device into low-power mode */ - ret = atmel_ecc_sleep(client); - if (ret < 0) - goto err; - - mutex_unlock(&i2c_priv->lock); - return atmel_ecc_status(&client->dev, cmd->data); -err: - mutex_unlock(&i2c_priv->lock); - return ret; -} - -static void atmel_ecc_work_handler(struct work_struct *work) -{ - struct atmel_ecc_work_data *work_data = - container_of(work, struct atmel_ecc_work_data, work); - struct atmel_ecc_cmd *cmd = &work_data->cmd; - struct i2c_client *client = work_data->ctx->client; - int status; - - status = atmel_ecc_send_receive(client, cmd); - work_data->cbk(work_data, work_data->areq, status); -} - -static void atmel_ecc_enqueue(struct atmel_ecc_work_data *work_data, - void (*cbk)(struct atmel_ecc_work_data *work_data, - void *areq, int status), - void *areq) -{ - work_data->cbk = (void *)cbk; - work_data->areq = areq; - - INIT_WORK(&work_data->work, atmel_ecc_work_handler); - schedule_work(&work_data->work); -} - static unsigned int atmel_ecdh_supported_curve(unsigned int curve_id) { if (curve_id == ECC_CURVE_NIST_P256) @@ -374,7 +89,7 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, unsigned int len) { struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); - struct atmel_ecc_cmd *cmd; + struct atmel_i2c_cmd *cmd; void *public_key; struct ecdh params; int ret = -ENOMEM; @@ -412,9 +127,9 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, ctx->do_fallback = false; ctx->curve_id = params.curve_id; - atmel_ecc_init_genkey_cmd(cmd, DATA_SLOT_2); + atmel_i2c_init_genkey_cmd(cmd, DATA_SLOT_2); - ret = atmel_ecc_send_receive(ctx->client, cmd); + ret = atmel_i2c_send_receive(ctx->client, cmd); if (ret) goto free_public_key; @@ -444,6 +159,9 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req) return crypto_kpp_generate_public_key(req); } + if (!ctx->public_key) + return -EINVAL; + /* might want less than we've got */ nbytes = min_t(size_t, ATMEL_ECC_PUBKEY_SIZE, req->dst_len); @@ -461,7 +179,7 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req) { struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); - struct atmel_ecc_work_data *work_data; + struct atmel_i2c_work_data *work_data; gfp_t gfp; int ret; @@ -482,12 +200,13 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req) return -ENOMEM; work_data->ctx = ctx; + work_data->client = ctx->client; - ret = atmel_ecc_init_ecdh_cmd(&work_data->cmd, req->src); + ret = atmel_i2c_init_ecdh_cmd(&work_data->cmd, req->src); if (ret) goto free_work_data; - atmel_ecc_enqueue(work_data, atmel_ecdh_done, req); + atmel_i2c_enqueue(work_data, atmel_ecdh_done, req); return -EINPROGRESS; @@ -498,7 +217,7 @@ free_work_data: static struct i2c_client *atmel_ecc_i2c_client_alloc(void) { - struct atmel_ecc_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL; + struct atmel_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL; struct i2c_client *client = ERR_PTR(-ENODEV); int min_tfm_cnt = INT_MAX; int tfm_cnt; @@ -533,7 +252,7 @@ static struct i2c_client *atmel_ecc_i2c_client_alloc(void) static void atmel_ecc_i2c_client_free(struct i2c_client *client) { - struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); atomic_dec(&i2c_priv->tfm_count); } @@ -604,99 +323,18 @@ static struct kpp_alg atmel_ecdh = { }, }; -static inline size_t atmel_ecc_wake_token_sz(u32 bus_clk_rate) -{ - u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC); - - /* return the size of the wake_token in bytes */ - return DIV_ROUND_UP(no_of_bits, 8); -} - -static int device_sanity_check(struct i2c_client *client) -{ - struct atmel_ecc_cmd *cmd; - int ret; - - cmd = kmalloc(sizeof(*cmd), GFP_KERNEL); - if (!cmd) - return -ENOMEM; - - atmel_ecc_init_read_cmd(cmd); - - ret = atmel_ecc_send_receive(client, cmd); - if (ret) - goto free_cmd; - - /* - * It is vital that the Configuration, Data and OTP zones be locked - * prior to release into the field of the system containing the device. - * Failure to lock these zones may permit modification of any secret - * keys and may lead to other security problems. - */ - if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) { - dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n"); - ret = -ENOTSUPP; - } - - /* fall through */ -free_cmd: - kfree(cmd); - return ret; -} - static int atmel_ecc_probe(struct i2c_client *client, const struct i2c_device_id *id) { - struct atmel_ecc_i2c_client_priv *i2c_priv; - struct device *dev = &client->dev; + struct atmel_i2c_client_priv *i2c_priv; int ret; - u32 bus_clk_rate; - if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { - dev_err(dev, "I2C_FUNC_I2C not supported\n"); - return -ENODEV; - } - - clk_rate = i2c_acpi_find_bus_speed(&client->adapter->dev); - if (!clk_rate) { - ret = device_property_read_u32(&client->adapter->dev, - "clock-frequency", &bus_clk_rate); - if (ret) { - dev_err(dev, "failed to read clock-frequency property\n"); - return ret; - } - } - - if (bus_clk_rate > 1000000L) { - dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n", - bus_clk_rate); - return -EINVAL; - } - - i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL); - if (!i2c_priv) - return -ENOMEM; - - i2c_priv->client = client; - mutex_init(&i2c_priv->lock); - - /* - * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate - - * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz - * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE. - */ - i2c_priv->wake_token_sz = atmel_ecc_wake_token_sz(bus_clk_rate); - - memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token)); - - atomic_set(&i2c_priv->tfm_count, 0); - - i2c_set_clientdata(client, i2c_priv); - - ret = device_sanity_check(client); + ret = atmel_i2c_probe(client, id); if (ret) return ret; + i2c_priv = i2c_get_clientdata(client); + spin_lock(&driver_data.i2c_list_lock); list_add_tail(&i2c_priv->i2c_client_list_node, &driver_data.i2c_client_list); @@ -708,10 +346,10 @@ static int atmel_ecc_probe(struct i2c_client *client, list_del(&i2c_priv->i2c_client_list_node); spin_unlock(&driver_data.i2c_list_lock); - dev_err(dev, "%s alg registration failed\n", + dev_err(&client->dev, "%s alg registration failed\n", atmel_ecdh.base.cra_driver_name); } else { - dev_info(dev, "atmel ecc algorithms registered in /proc/crypto\n"); + dev_info(&client->dev, "atmel ecc algorithms registered in /proc/crypto\n"); } return ret; @@ -719,7 +357,7 @@ static int atmel_ecc_probe(struct i2c_client *client, static int atmel_ecc_remove(struct i2c_client *client) { - struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); /* Return EBUSY if i2c client already allocated. */ if (atomic_read(&i2c_priv->tfm_count)) { diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c new file mode 100644 index 000000000000..5e099368d120 --- /dev/null +++ b/drivers/crypto/atmel-i2c.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip / Atmel ECC (I2C) driver. + * + * Copyright (c) 2017, Microchip Technology Inc. + * Author: Tudor Ambarus + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "atmel-i2c.h" + +/** + * atmel_i2c_checksum() - Generate 16-bit CRC as required by ATMEL ECC. + * CRC16 verification of the count, opcode, param1, param2 and data bytes. + * The checksum is saved in little-endian format in the least significant + * two bytes of the command. CRC polynomial is 0x8005 and the initial register + * value should be zero. + * + * @cmd : structure used for communicating with the device. + */ +static void atmel_i2c_checksum(struct atmel_i2c_cmd *cmd) +{ + u8 *data = &cmd->count; + size_t len = cmd->count - CRC_SIZE; + u16 *__crc16 = (u16 *)(data + len); + + *__crc16 = cpu_to_le16(bitrev16(crc16(0, data, len))); +} + +void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd) +{ + cmd->word_addr = COMMAND; + cmd->opcode = OPCODE_READ; + /* + * Read the word from Configuration zone that contains the lock bytes + * (UserExtra, Selector, LockValue, LockConfig). + */ + cmd->param1 = CONFIG_ZONE; + cmd->param2 = DEVICE_LOCK_ADDR; + cmd->count = READ_COUNT; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_READ; + cmd->rxsize = READ_RSP_SIZE; +} +EXPORT_SYMBOL(atmel_i2c_init_read_cmd); + +void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid) +{ + cmd->word_addr = COMMAND; + cmd->count = GENKEY_COUNT; + cmd->opcode = OPCODE_GENKEY; + cmd->param1 = GENKEY_MODE_PRIVATE; + /* a random private key will be generated and stored in slot keyID */ + cmd->param2 = cpu_to_le16(keyid); + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_GENKEY; + cmd->rxsize = GENKEY_RSP_SIZE; +} +EXPORT_SYMBOL(atmel_i2c_init_genkey_cmd); + +int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd, + struct scatterlist *pubkey) +{ + size_t copied; + + cmd->word_addr = COMMAND; + cmd->count = ECDH_COUNT; + cmd->opcode = OPCODE_ECDH; + cmd->param1 = ECDH_PREFIX_MODE; + /* private key slot */ + cmd->param2 = cpu_to_le16(DATA_SLOT_2); + + /* + * The device only supports NIST P256 ECC keys. The public key size will + * always be the same. Use a macro for the key size to avoid unnecessary + * computations. + */ + copied = sg_copy_to_buffer(pubkey, + sg_nents_for_len(pubkey, + ATMEL_ECC_PUBKEY_SIZE), + cmd->data, ATMEL_ECC_PUBKEY_SIZE); + if (copied != ATMEL_ECC_PUBKEY_SIZE) + return -EINVAL; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_ECDH; + cmd->rxsize = ECDH_RSP_SIZE; + + return 0; +} +EXPORT_SYMBOL(atmel_i2c_init_ecdh_cmd); + +/* + * After wake and after execution of a command, there will be error, status, or + * result bytes in the device's output register that can be retrieved by the + * system. When the length of that group is four bytes, the codes returned are + * detailed in error_list. + */ +static int atmel_i2c_status(struct device *dev, u8 *status) +{ + size_t err_list_len = ARRAY_SIZE(error_list); + int i; + u8 err_id = status[1]; + + if (*status != STATUS_SIZE) + return 0; + + if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR) + return 0; + + for (i = 0; i < err_list_len; i++) + if (error_list[i].value == err_id) + break; + + /* if err_id is not in the error_list then ignore it */ + if (i != err_list_len) { + dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text); + return err_id; + } + + return 0; +} + +static int atmel_i2c_wakeup(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + u8 status[STATUS_RSP_SIZE]; + int ret; + + /* + * The device ignores any levels or transitions on the SCL pin when the + * device is idle, asleep or during waking up. Don't check for error + * when waking up the device. + */ + i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz); + + /* + * Wait to wake the device. Typical execution times for ecdh and genkey + * are around tens of milliseconds. Delta is chosen to 50 microseconds. + */ + usleep_range(TWHI_MIN, TWHI_MAX); + + ret = i2c_master_recv(client, status, STATUS_SIZE); + if (ret < 0) + return ret; + + return atmel_i2c_status(&client->dev, status); +} + +static int atmel_i2c_sleep(struct i2c_client *client) +{ + u8 sleep = SLEEP_TOKEN; + + return i2c_master_send(client, &sleep, 1); +} + +/* + * atmel_i2c_send_receive() - send a command to the device and receive its + * response. + * @client: i2c client device + * @cmd : structure used to communicate with the device + * + * After the device receives a Wake token, a watchdog counter starts within the + * device. After the watchdog timer expires, the device enters sleep mode + * regardless of whether some I/O transmission or command execution is in + * progress. If a command is attempted when insufficient time remains prior to + * watchdog timer execution, the device will return the watchdog timeout error + * code without attempting to execute the command. There is no way to reset the + * counter other than to put the device into sleep or idle mode and then + * wake it up again. + */ +int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + int ret; + + mutex_lock(&i2c_priv->lock); + + ret = atmel_i2c_wakeup(client); + if (ret) + goto err; + + /* send the command */ + ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE); + if (ret < 0) + goto err; + + /* delay the appropriate amount of time for command to execute */ + msleep(cmd->msecs); + + /* receive the response */ + ret = i2c_master_recv(client, cmd->data, cmd->rxsize); + if (ret < 0) + goto err; + + /* put the device into low-power mode */ + ret = atmel_i2c_sleep(client); + if (ret < 0) + goto err; + + mutex_unlock(&i2c_priv->lock); + return atmel_i2c_status(&client->dev, cmd->data); +err: + mutex_unlock(&i2c_priv->lock); + return ret; +} +EXPORT_SYMBOL(atmel_i2c_send_receive); + +static void atmel_i2c_work_handler(struct work_struct *work) +{ + struct atmel_i2c_work_data *work_data = + container_of(work, struct atmel_i2c_work_data, work); + struct atmel_i2c_cmd *cmd = &work_data->cmd; + struct i2c_client *client = work_data->client; + int status; + + status = atmel_i2c_send_receive(client, cmd); + work_data->cbk(work_data, work_data->areq, status); +} + +void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data, + void (*cbk)(struct atmel_i2c_work_data *work_data, + void *areq, int status), + void *areq) +{ + work_data->cbk = (void *)cbk; + work_data->areq = areq; + + INIT_WORK(&work_data->work, atmel_i2c_work_handler); + schedule_work(&work_data->work); +} +EXPORT_SYMBOL(atmel_i2c_enqueue); + +static inline size_t atmel_i2c_wake_token_sz(u32 bus_clk_rate) +{ + u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC); + + /* return the size of the wake_token in bytes */ + return DIV_ROUND_UP(no_of_bits, 8); +} + +static int device_sanity_check(struct i2c_client *client) +{ + struct atmel_i2c_cmd *cmd; + int ret; + + cmd = kmalloc(sizeof(*cmd), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + atmel_i2c_init_read_cmd(cmd); + + ret = atmel_i2c_send_receive(client, cmd); + if (ret) + goto free_cmd; + + /* + * It is vital that the Configuration, Data and OTP zones be locked + * prior to release into the field of the system containing the device. + * Failure to lock these zones may permit modification of any secret + * keys and may lead to other security problems. + */ + if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) { + dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n"); + ret = -ENOTSUPP; + } + + /* fall through */ +free_cmd: + kfree(cmd); + return ret; +} + +int atmel_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id) +{ + struct atmel_i2c_client_priv *i2c_priv; + struct device *dev = &client->dev; + int ret; + u32 bus_clk_rate; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { + dev_err(dev, "I2C_FUNC_I2C not supported\n"); + return -ENODEV; + } + + bus_clk_rate = i2c_acpi_find_bus_speed(&client->adapter->dev); + if (!bus_clk_rate) { + ret = device_property_read_u32(&client->adapter->dev, + "clock-frequency", &bus_clk_rate); + if (ret) { + dev_err(dev, "failed to read clock-frequency property\n"); + return ret; + } + } + + if (bus_clk_rate > 1000000L) { + dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n", + bus_clk_rate); + return -EINVAL; + } + + i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL); + if (!i2c_priv) + return -ENOMEM; + + i2c_priv->client = client; + mutex_init(&i2c_priv->lock); + + /* + * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate - + * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz + * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE. + */ + i2c_priv->wake_token_sz = atmel_i2c_wake_token_sz(bus_clk_rate); + + memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token)); + + atomic_set(&i2c_priv->tfm_count, 0); + + i2c_set_clientdata(client, i2c_priv); + + ret = device_sanity_check(client); + if (ret) + return ret; + + return 0; +} +EXPORT_SYMBOL(atmel_i2c_probe); + +MODULE_AUTHOR("Tudor Ambarus "); +MODULE_DESCRIPTION("Microchip / Atmel ECC (I2C) driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/atmel-ecc.h b/drivers/crypto/atmel-i2c.h similarity index 51% rename from drivers/crypto/atmel-ecc.h rename to drivers/crypto/atmel-i2c.h index 643a3b947338..82de5166acfa 100644 --- a/drivers/crypto/atmel-ecc.h +++ b/drivers/crypto/atmel-i2c.h @@ -4,8 +4,8 @@ * Author: Tudor Ambarus */ -#ifndef __ATMEL_ECC_H__ -#define __ATMEL_ECC_H__ +#ifndef __ATMEL_I2C_H__ +#define __ATMEL_I2C_H__ #define ATMEL_ECC_PRIORITY 300 @@ -31,7 +31,7 @@ #define MAX_RSP_SIZE GENKEY_RSP_SIZE /** - * atmel_ecc_cmd - structure used for communicating with the device. + * atmel_i2c_cmd - structure used for communicating with the device. * @word_addr: indicates the function of the packet sent to the device. This * byte should have a value of COMMAND for normal operation. * @count : number of bytes to be transferred to (or from) the device. @@ -42,7 +42,7 @@ * @rxsize : size of the data received from i2c client. * @msecs : command execution time in milliseconds */ -struct atmel_ecc_cmd { +struct atmel_i2c_cmd { u8 word_addr; u8 count; u8 opcode; @@ -113,4 +113,74 @@ static const struct { #define ECDH_COUNT 71 #define ECDH_PREFIX_MODE 0x00 -#endif /* __ATMEL_ECC_H__ */ +/* Used for binding tfm objects to i2c clients. */ +struct atmel_ecc_driver_data { + struct list_head i2c_client_list; + spinlock_t i2c_list_lock; +} ____cacheline_aligned; + +/** + * atmel_i2c_client_priv - i2c_client private data + * @client : pointer to i2c client device + * @i2c_client_list_node: part of i2c_client_list + * @lock : lock for sending i2c commands + * @wake_token : wake token array of zeros + * @wake_token_sz : size in bytes of the wake_token + * @tfm_count : number of active crypto transformations on i2c client + * + * Reads and writes from/to the i2c client are sequential. The first byte + * transmitted to the device is treated as the byte size. Any attempt to send + * more than this number of bytes will cause the device to not ACK those bytes. + * After the host writes a single command byte to the input buffer, reads are + * prohibited until after the device completes command execution. Use a mutex + * when sending i2c commands. + */ +struct atmel_i2c_client_priv { + struct i2c_client *client; + struct list_head i2c_client_list_node; + struct mutex lock; + u8 wake_token[WAKE_TOKEN_MAX_SIZE]; + size_t wake_token_sz; + atomic_t tfm_count ____cacheline_aligned; +}; + +/** + * atmel_i2c_work_data - data structure representing the work + * @ctx : transformation context. + * @cbk : pointer to a callback function to be invoked upon completion of this + * request. This has the form: + * callback(struct atmel_i2c_work_data *work_data, void *areq, u8 status) + * where: + * @work_data: data structure representing the work + * @areq : optional pointer to an argument passed with the original + * request. + * @status : status returned from the i2c client device or i2c error. + * @areq: optional pointer to a user argument for use at callback time. + * @work: describes the task to be executed. + * @cmd : structure used for communicating with the device. + */ +struct atmel_i2c_work_data { + void *ctx; + struct i2c_client *client; + void (*cbk)(struct atmel_i2c_work_data *work_data, void *areq, + int status); + void *areq; + struct work_struct work; + struct atmel_i2c_cmd cmd; +}; + +int atmel_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id); + +void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data, + void (*cbk)(struct atmel_i2c_work_data *work_data, + void *areq, int status), + void *areq); + +int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd); + +void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd); +void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid); +int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd, + struct scatterlist *pubkey); + +#endif /* __ATMEL_I2C_H__ */ From da001fb651b00e1deeaf24767dd691ae8152a4f5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 24 May 2019 18:26:49 +0200 Subject: [PATCH 057/143] crypto: atmel-i2c - add support for SHA204A random number generator The Linaro/96boards Secure96 mezzanine contains (among other things) an Atmel SHA204A symmetric crypto processor. This chip implements a number of different functionalities, but one that is highly useful for many different 96boards platforms is the random number generator. So let's implement a driver for the SHA204A, and for the time being, implement support for the random number generator only. Reviewed-by: Linus Walleij Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 14 +++ drivers/crypto/Makefile | 1 + drivers/crypto/atmel-i2c.c | 15 +++ drivers/crypto/atmel-i2c.h | 10 ++ drivers/crypto/atmel-sha204a.c | 171 +++++++++++++++++++++++++++++++++ 5 files changed, 211 insertions(+) create mode 100644 drivers/crypto/atmel-sha204a.c diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 20674842261e..bc7abad9fdff 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -536,6 +536,20 @@ config CRYPTO_DEV_ATMEL_ECC To compile this driver as a module, choose M here: the module will be called atmel-ecc. +config CRYPTO_DEV_ATMEL_SHA204A + tristate "Support for Microchip / Atmel SHA accelerator and RNG" + depends on I2C + select CRYPTO_DEV_ATMEL_I2C + select HW_RANDOM + help + Microhip / Atmel SHA accelerator and RNG. + Select this if you want to use the Microchip / Atmel SHA204A + module as a random number generator. (Other functions of the + chip are currently not exposed by this driver) + + To compile this driver as a module, choose M here: the module + will be called atmel-sha204a. + config CRYPTO_DEV_CCP bool "Support for AMD Secure Processor" depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 394e84089924..afc4753b5d28 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -4,6 +4,7 @@ obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_I2C) += atmel-i2c.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA204A) += atmel-sha204a.o obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ obj-$(CONFIG_CRYPTO_DEV_CCREE) += ccree/ diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c index 5e099368d120..be49ab7f4338 100644 --- a/drivers/crypto/atmel-i2c.c +++ b/drivers/crypto/atmel-i2c.c @@ -58,6 +58,21 @@ void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd) } EXPORT_SYMBOL(atmel_i2c_init_read_cmd); +void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd) +{ + cmd->word_addr = COMMAND; + cmd->opcode = OPCODE_RANDOM; + cmd->param1 = 0; + cmd->param2 = 0; + cmd->count = RANDOM_COUNT; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_RANDOM; + cmd->rxsize = RANDOM_RSP_SIZE; +} +EXPORT_SYMBOL(atmel_i2c_init_random_cmd); + void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid) { cmd->word_addr = COMMAND; diff --git a/drivers/crypto/atmel-i2c.h b/drivers/crypto/atmel-i2c.h index 82de5166acfa..c6bd43b78f33 100644 --- a/drivers/crypto/atmel-i2c.h +++ b/drivers/crypto/atmel-i2c.h @@ -7,6 +7,8 @@ #ifndef __ATMEL_I2C_H__ #define __ATMEL_I2C_H__ +#include + #define ATMEL_ECC_PRIORITY 300 #define COMMAND 0x03 /* packet function */ @@ -28,6 +30,7 @@ #define GENKEY_RSP_SIZE (ATMEL_ECC_PUBKEY_SIZE + \ CMD_OVERHEAD_SIZE) #define READ_RSP_SIZE (4 + CMD_OVERHEAD_SIZE) +#define RANDOM_RSP_SIZE (32 + CMD_OVERHEAD_SIZE) #define MAX_RSP_SIZE GENKEY_RSP_SIZE /** @@ -96,15 +99,20 @@ static const struct { #define MAX_EXEC_TIME_ECDH 58 #define MAX_EXEC_TIME_GENKEY 115 #define MAX_EXEC_TIME_READ 1 +#define MAX_EXEC_TIME_RANDOM 50 /* Command opcode */ #define OPCODE_ECDH 0x43 #define OPCODE_GENKEY 0x40 #define OPCODE_READ 0x02 +#define OPCODE_RANDOM 0x1b /* Definitions for the READ Command */ #define READ_COUNT 7 +/* Definitions for the RANDOM Command */ +#define RANDOM_COUNT 7 + /* Definitions for the GenKey Command */ #define GENKEY_COUNT 7 #define GENKEY_MODE_PRIVATE 0x04 @@ -142,6 +150,7 @@ struct atmel_i2c_client_priv { u8 wake_token[WAKE_TOKEN_MAX_SIZE]; size_t wake_token_sz; atomic_t tfm_count ____cacheline_aligned; + struct hwrng hwrng; }; /** @@ -179,6 +188,7 @@ void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data, int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd); void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd); +void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd); void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid); int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd, struct scatterlist *pubkey); diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c new file mode 100644 index 000000000000..ea0d2068ea4f --- /dev/null +++ b/drivers/crypto/atmel-sha204a.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip / Atmel SHA204A (I2C) driver. + * + * Copyright (c) 2019 Linaro, Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "atmel-i2c.h" + +static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data, + void *areq, int status) +{ + struct atmel_i2c_client_priv *i2c_priv = work_data->ctx; + struct hwrng *rng = areq; + + if (status) + dev_warn_ratelimited(&i2c_priv->client->dev, + "i2c transaction failed (%d)\n", + status); + + rng->priv = (unsigned long)work_data; + atomic_dec(&i2c_priv->tfm_count); +} + +static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data, + size_t max) +{ + struct atmel_i2c_client_priv *i2c_priv; + struct atmel_i2c_work_data *work_data; + + i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng); + + /* keep maximum 1 asynchronous read in flight at any time */ + if (!atomic_add_unless(&i2c_priv->tfm_count, 1, 1)) + return 0; + + if (rng->priv) { + work_data = (struct atmel_i2c_work_data *)rng->priv; + max = min(sizeof(work_data->cmd.data), max); + memcpy(data, &work_data->cmd.data, max); + rng->priv = 0; + } else { + work_data = kmalloc(sizeof(*work_data), GFP_ATOMIC); + if (!work_data) + return -ENOMEM; + + work_data->ctx = i2c_priv; + work_data->client = i2c_priv->client; + + max = 0; + } + + atmel_i2c_init_random_cmd(&work_data->cmd); + atmel_i2c_enqueue(work_data, atmel_sha204a_rng_done, rng); + + return max; +} + +static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max, + bool wait) +{ + struct atmel_i2c_client_priv *i2c_priv; + struct atmel_i2c_cmd cmd; + int ret; + + if (!wait) + return atmel_sha204a_rng_read_nonblocking(rng, data, max); + + i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng); + + atmel_i2c_init_random_cmd(&cmd); + + ret = atmel_i2c_send_receive(i2c_priv->client, &cmd); + if (ret) + return ret; + + max = min(sizeof(cmd.data), max); + memcpy(data, cmd.data, max); + + return max; +} + +static int atmel_sha204a_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct atmel_i2c_client_priv *i2c_priv; + int ret; + + ret = atmel_i2c_probe(client, id); + if (ret) + return ret; + + i2c_priv = i2c_get_clientdata(client); + + memset(&i2c_priv->hwrng, 0, sizeof(i2c_priv->hwrng)); + + i2c_priv->hwrng.name = dev_name(&client->dev); + i2c_priv->hwrng.read = atmel_sha204a_rng_read; + i2c_priv->hwrng.quality = 1024; + + ret = hwrng_register(&i2c_priv->hwrng); + if (ret) + dev_warn(&client->dev, "failed to register RNG (%d)\n", ret); + + return ret; +} + +static int atmel_sha204a_remove(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + + if (atomic_read(&i2c_priv->tfm_count)) { + dev_err(&client->dev, "Device is busy\n"); + return -EBUSY; + } + + if (i2c_priv->hwrng.priv) + kfree((void *)i2c_priv->hwrng.priv); + hwrng_unregister(&i2c_priv->hwrng); + + return 0; +} + +static const struct of_device_id atmel_sha204a_dt_ids[] = { + { .compatible = "atmel,atsha204a", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, atmel_sha204a_dt_ids); + +static const struct i2c_device_id atmel_sha204a_id[] = { + { "atsha204a", 0 }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(i2c, atmel_sha204a_id); + +static struct i2c_driver atmel_sha204a_driver = { + .probe = atmel_sha204a_probe, + .remove = atmel_sha204a_remove, + .id_table = atmel_sha204a_id, + + .driver.name = "atmel-sha204a", + .driver.of_match_table = of_match_ptr(atmel_sha204a_dt_ids), +}; + +static int __init atmel_sha204a_init(void) +{ + return i2c_add_driver(&atmel_sha204a_driver); +} + +static void __exit atmel_sha204a_exit(void) +{ + flush_scheduled_work(); + i2c_del_driver(&atmel_sha204a_driver); +} + +module_init(atmel_sha204a_init); +module_exit(atmel_sha204a_exit); + +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_LICENSE("GPL v2"); From 2af8c8927ebb474eb2af4ab3b8d7e92c3844b43f Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 24 May 2019 18:26:50 +0200 Subject: [PATCH 058/143] dt-bindings: add Atmel SHA204A I2C crypto processor Add a compatible string for the Atmel SHA204A I2C crypto processor. Signed-off-by: Ard Biesheuvel Reviewed-by: Rob Herring Signed-off-by: Herbert Xu --- Documentation/devicetree/bindings/trivial-devices.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml index 747fd3f689dc..a572c3468226 100644 --- a/Documentation/devicetree/bindings/trivial-devices.yaml +++ b/Documentation/devicetree/bindings/trivial-devices.yaml @@ -52,6 +52,8 @@ properties: - at,24c08 # i2c trusted platform module (TPM) - atmel,at97sc3204t + # i2c h/w symmetric crypto module + - atmel,atsha204a # CM32181: Ambient Light Sensor - capella,cm32181 # CM3232: Ambient Light Sensor From 0adb0c99594b73844cf9a5714faa6553ea04ba04 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 24 May 2019 18:26:51 +0200 Subject: [PATCH 059/143] dt-bindings: move Atmel ECC508A I2C crypto processor to trivial-devices Move the binding for the discrete Atmel I2C Elliptic Curve h/w crypto module to trivial-devices.yaml, as it doesn't belong in atmel-crypto which describes unrelated on-SoC peripherals. Signed-off-by: Ard Biesheuvel Reviewed-by: Rob Herring Signed-off-by: Herbert Xu --- .../devicetree/bindings/crypto/atmel-crypto.txt | 13 ------------- .../devicetree/bindings/trivial-devices.yaml | 2 ++ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/Documentation/devicetree/bindings/crypto/atmel-crypto.txt b/Documentation/devicetree/bindings/crypto/atmel-crypto.txt index 6b458bb2440d..f2aab3dc2b52 100644 --- a/Documentation/devicetree/bindings/crypto/atmel-crypto.txt +++ b/Documentation/devicetree/bindings/crypto/atmel-crypto.txt @@ -66,16 +66,3 @@ sha@f8034000 { dmas = <&dma1 2 17>; dma-names = "tx"; }; - -* Eliptic Curve Cryptography (I2C) - -Required properties: -- compatible : must be "atmel,atecc508a". -- reg: I2C bus address of the device. -- clock-frequency: must be present in the i2c controller node. - -Example: -atecc508a@c0 { - compatible = "atmel,atecc508a"; - reg = <0xC0>; -}; diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml index a572c3468226..2e742d399e87 100644 --- a/Documentation/devicetree/bindings/trivial-devices.yaml +++ b/Documentation/devicetree/bindings/trivial-devices.yaml @@ -54,6 +54,8 @@ properties: - atmel,at97sc3204t # i2c h/w symmetric crypto module - atmel,atsha204a + # i2c h/w elliptic curve crypto module + - atmel,atecc508a # CM32181: Ambient Light Sensor - capella,cm32181 # CM3232: Ambient Light Sensor From cb9877b8a8fb57f12e9d461cbbd1509074f4ce7a Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:50:53 +0200 Subject: [PATCH 060/143] crypto: inside-secure - remove empty line Cosmetic patch removing an empty line in the skcipher token creation routine. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_cipher.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index de4be10b172f..aca1cdf33362 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -73,7 +73,6 @@ static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, memcpy(cdesc->control_data.token, iv, DES3_EDE_BLOCK_SIZE); cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD; break; - case SAFEXCEL_AES: offset = AES_BLOCK_SIZE / sizeof(u32); memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE); From dd4306a6f398f49b60bd061d426e5e115234b8cc Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:50:54 +0200 Subject: [PATCH 061/143] crypto: inside-secure - move comment This cosmetic patch moves a comment before the condition it is related to. The patch does not change the driver behaviour in any way. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index ac9282c1a5ec..a7cee9ed3789 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -223,10 +223,11 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, * fit into full blocks, cache it for the next send() call. */ extra = queued & (crypto_ahash_blocksize(ahash) - 1); + + /* If this is not the last request and the queued data + * is a multiple of a block, cache the last one for now. + */ if (!extra) - /* If this is not the last request and the queued data - * is a multiple of a block, cache the last one for now. - */ extra = crypto_ahash_blocksize(ahash); if (extra) { From 0733eabadc90f148dd96e1d41c8a644312a6ab18 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:50:55 +0200 Subject: [PATCH 062/143] crypto: inside-secure - fix coding style for a condition This cosmetic patch fixes a cosmetic issue with if brackets. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 86c699c14f84..263bd4ce73c5 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -720,11 +720,10 @@ handle_results: } acknowledge: - if (i) { + if (i) writel(EIP197_xDR_PROC_xD_PKT(i) | EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset), EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); - } /* If the number of requests overflowed the counter, try to proceed more * requests. From 709ecc101aea53d3625b688697b7c9dc4235716d Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:50:56 +0200 Subject: [PATCH 063/143] crypto: inside-secure - remove useless check When sending an ahash request, the code checks for the extra variable not to be 0. This check is useless as the extra variable can't be 0 at this point (it is checked on the line just before). This patch does not modify the driver behaviour in any way. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index a7cee9ed3789..58ce480690eb 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -230,19 +230,17 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, if (!extra) extra = crypto_ahash_blocksize(ahash); - if (extra) { - sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), - req->cache_next, extra, - areq->nbytes - extra); + sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + req->cache_next, extra, + areq->nbytes - extra); - queued -= extra; - len -= extra; + queued -= extra; + len -= extra; - if (!queued) { - *commands = 0; - *results = 0; - return 0; - } + if (!queued) { + *commands = 0; + *results = 0; + return 0; } } From 942d849d32d0126442bdf7a4052bb46fbf23a3c6 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:50:57 +0200 Subject: [PATCH 064/143] crypto: inside-secure - improve the result error format when displayed The result descriptors contain errors, which are represented as a bitmap. This patch updates the error message to not treat the error as a decimal value, but as an hexadecimal one. This helps in knowing the value does not have a direct meaning (the set bits themselves have). Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 263bd4ce73c5..d5392893973c 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -589,7 +589,7 @@ inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, if (rdesc->result_data.error_code & 0x407f) { /* Fatal error (bits 0-7, 14) */ dev_err(priv->dev, - "cipher: result: result descriptor error (%d)\n", + "cipher: result: result descriptor error (0x%x)\n", rdesc->result_data.error_code); return -EIO; } else if (rdesc->result_data.error_code == BIT(9)) { From 082a9d0918d2aae48dc3b836a8e3ca6aa53d2495 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:50:58 +0200 Subject: [PATCH 065/143] crypto: inside-secure - change returned error when a descriptor reports an error This patch changes the error reported by the Inside Secure SafeXcel driver when a result descriptor reports an error, from -EIO to -EINVAL, as this is what the crypto framework expects. This was found while running the crypto extra tests. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index d5392893973c..316e5e4c1c74 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -591,7 +591,7 @@ inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, dev_err(priv->dev, "cipher: result: result descriptor error (0x%x)\n", rdesc->result_data.error_code); - return -EIO; + return -EINVAL; } else if (rdesc->result_data.error_code == BIT(9)) { /* Authentication failed */ return -EBADMSG; From 97a1440d12947293ff3f0eb9b649052927a55ed8 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:50:59 +0200 Subject: [PATCH 066/143] crypto: inside-secure - enable context reuse The context given to the crypto engine can be reused over time. While the driver was designed to allow this, the feature wasn't enabled in the hardware engine. This patch enables it. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 6 ++++++ drivers/crypto/inside-secure/safexcel.h | 7 +++++++ drivers/crypto/inside-secure/safexcel_ring.c | 3 +++ 3 files changed, 16 insertions(+) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 316e5e4c1c74..df43a2c6933b 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -398,6 +398,12 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) /* Processing Engine configuration */ + /* Token & context configuration */ + val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES | + EIP197_PE_EIP96_TOKEN_CTRL_REUSE_CTX | + EIP197_PE_EIP96_TOKEN_CTRL_POST_REUSE_CTX; + writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe)); + /* H/W capabilities selection */ val = EIP197_FUNCTION_RSVD; val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY; diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 65624a81f0fd..0bc8859f9d06 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -118,6 +118,7 @@ #define EIP197_PE_ICE_SCRATCH_CTRL(n) (0x0d04 + (0x2000 * (n))) #define EIP197_PE_ICE_FPP_CTRL(n) (0x0d80 + (0x2000 * (n))) #define EIP197_PE_ICE_RAM_CTRL(n) (0x0ff0 + (0x2000 * (n))) +#define EIP197_PE_EIP96_TOKEN_CTRL(n) (0x1000 + (0x2000 * (n))) #define EIP197_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n))) #define EIP197_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n))) #define EIP197_PE_EIP96_CONTEXT_STAT(n) (0x100c + (0x2000 * (n))) @@ -249,6 +250,11 @@ #define EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN BIT(0) #define EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN BIT(1) +/* EIP197_PE_EIP96_TOKEN_CTRL */ +#define EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES BIT(16) +#define EIP197_PE_EIP96_TOKEN_CTRL_REUSE_CTX BIT(19) +#define EIP197_PE_EIP96_TOKEN_CTRL_POST_REUSE_CTX BIT(20) + /* EIP197_PE_EIP96_FUNCTION_EN */ #define EIP197_FUNCTION_RSVD (BIT(6) | BIT(15) | BIT(20) | BIT(23)) #define EIP197_PROTOCOL_HASH_ONLY BIT(0) @@ -468,6 +474,7 @@ struct safexcel_control_data_desc { #define EIP197_OPTION_MAGIC_VALUE BIT(0) #define EIP197_OPTION_64BIT_CTX BIT(1) +#define EIP197_OPTION_RC_AUTO (0x2 << 3) #define EIP197_OPTION_CTX_CTRL_IN_CMD BIT(8) #define EIP197_OPTION_2_TOKEN_IV_CMD GENMASK(11, 10) #define EIP197_OPTION_4_TOKEN_IV_CMD GENMASK(11, 9) diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c index eb75fa684876..142bc3f5c45c 100644 --- a/drivers/crypto/inside-secure/safexcel_ring.c +++ b/drivers/crypto/inside-secure/safexcel_ring.c @@ -145,6 +145,9 @@ struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *pr (lower_32_bits(context) & GENMASK(31, 2)) >> 2; cdesc->control_data.context_hi = upper_32_bits(context); + if (priv->version == EIP197B || priv->version == EIP197D) + cdesc->control_data.options |= EIP197_OPTION_RC_AUTO; + /* TODO: large xform HMAC with SHA-384/512 uses refresh = 3 */ cdesc->control_data.refresh = 2; From aa52428698aeb1c1d4ebb7c3e9de3e7bdce25704 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:51:00 +0200 Subject: [PATCH 067/143] crypto: inside-secure - unify cache reset This patch unify the way the cache related data is zeroed when the cache buffer is DMA unmapped. It should not change the driver behaviour, but improves the code safety and readability. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 58ce480690eb..a79a73bb3969 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -183,6 +183,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz, DMA_TO_DEVICE); sreq->cache_dma = 0; + sreq->cache_sz = 0; } if (sreq->finish) @@ -344,6 +345,7 @@ unmap_cache: if (req->cache_dma) { dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz, DMA_TO_DEVICE); + req->cache_dma = 0; req->cache_sz = 0; } From b926213d6fede9c9427d7c12eaf7d9f0895deb4e Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:51:01 +0200 Subject: [PATCH 068/143] crypto: inside-secure - fix zeroing of the request in ahash_exit_inv A request is zeroed in safexcel_ahash_exit_inv(). This request total size is EIP197_AHASH_REQ_SIZE while the memset zeroing it uses sizeof(struct ahash_request), which happens to be less than EIP197_AHASH_REQ_SIZE. This patch fixes it. Fixes: f6beaea30487 ("crypto: inside-secure - authenc(hmac(sha256), cbc(aes)) support") Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index a79a73bb3969..ba0732fd4ed4 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -487,7 +487,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) struct safexcel_inv_result result = {}; int ring = ctx->base.ring; - memset(req, 0, sizeof(struct ahash_request)); + memset(req, 0, EIP197_AHASH_REQ_SIZE); /* create invalidation request */ init_completion(&result.completion); From ccd65a206a5025cf953a2e4f37e894921b131a5c Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:51:02 +0200 Subject: [PATCH 069/143] crypto: inside-secure - fix queued len computation This patch fixes the queued len computation, which could theoretically be wrong if req->len[1] - req->processed[1] > 1. Be future-proof here, and fix it. Fixes: b460edb6230a ("crypto: inside-secure - sha512 support") Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index ba0732fd4ed4..a9197d2c5a48 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -50,10 +50,12 @@ struct safexcel_ahash_req { static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) { - if (req->len[1] > req->processed[1]) - return 0xffffffff - (req->len[0] - req->processed[0]); + u64 len, processed; - return req->len[0] - req->processed[0]; + len = (0xffffffff * req->len[1]) + req->len[0]; + processed = (0xffffffff * req->processed[1]) + req->processed[0]; + + return len - processed; } static void safexcel_hash_token(struct safexcel_command_desc *cdesc, From 57660b11d5adbb18182e9388b83b7fb214c4e1a1 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:51:03 +0200 Subject: [PATCH 070/143] crypto: inside-secure - implement IV retrieval This patch adds support for retrieving intermediate IV from the crypto engine when using the CBC block mode with AES and (3)DES. The retrieved IV is copied to the request IV buffer, as requested by the kernel crypto API. This fix boot tests added by commit 8efd972ef96a ("crypto: testmgr - support checking skcipher output IV"). Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.h | 8 +++ .../crypto/inside-secure/safexcel_cipher.c | 52 ++++++++++++++++--- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 0bc8859f9d06..ca6ece5607cd 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -339,6 +339,7 @@ struct safexcel_context_record { #define CONTEXT_CONTROL_IV3 BIT(8) #define CONTEXT_CONTROL_DIGEST_CNT BIT(9) #define CONTEXT_CONTROL_COUNTER_MODE BIT(10) +#define CONTEXT_CONTROL_CRYPTO_STORE BIT(12) #define CONTEXT_CONTROL_HASH_STORE BIT(19) /* The hash counter given to the engine in the context has a granularity of @@ -431,6 +432,10 @@ struct safexcel_token { #define EIP197_TOKEN_HASH_RESULT_VERIFY BIT(16) +#define EIP197_TOKEN_CTX_OFFSET(x) (x) +#define EIP197_TOKEN_DIRECTION_EXTERNAL BIT(11) +#define EIP197_TOKEN_EXEC_IF_SUCCESSFUL (0x1 << 12) + #define EIP197_TOKEN_STAT_LAST_HASH BIT(0) #define EIP197_TOKEN_STAT_LAST_PACKET BIT(1) #define EIP197_TOKEN_OPCODE_DIRECTION 0x0 @@ -438,6 +443,7 @@ struct safexcel_token { #define EIP197_TOKEN_OPCODE_NOOP EIP197_TOKEN_OPCODE_INSERT #define EIP197_TOKEN_OPCODE_RETRIEVE 0x4 #define EIP197_TOKEN_OPCODE_VERIFY 0xd +#define EIP197_TOKEN_OPCODE_CTX_ACCESS 0xe #define EIP197_TOKEN_OPCODE_BYPASS GENMASK(3, 0) static inline void eip197_noop_token(struct safexcel_token *token) @@ -448,6 +454,8 @@ static inline void eip197_noop_token(struct safexcel_token *token) /* Instructions */ #define EIP197_TOKEN_INS_INSERT_HASH_DIGEST 0x1c +#define EIP197_TOKEN_INS_ORIGIN_IV0 0x14 +#define EIP197_TOKEN_INS_ORIGIN_LEN(x) ((x) << 5) #define EIP197_TOKEN_INS_TYPE_OUTPUT BIT(5) #define EIP197_TOKEN_INS_TYPE_HASH BIT(6) #define EIP197_TOKEN_INS_TYPE_CRYTO BIT(7) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index aca1cdf33362..cedfb121c278 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -59,26 +59,26 @@ static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, u32 length) { struct safexcel_token *token; - unsigned offset = 0; + u32 offset = 0, block_sz = 0; if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) { switch (ctx->alg) { case SAFEXCEL_DES: - offset = DES_BLOCK_SIZE / sizeof(u32); - memcpy(cdesc->control_data.token, iv, DES_BLOCK_SIZE); + block_sz = DES_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD; break; case SAFEXCEL_3DES: - offset = DES3_EDE_BLOCK_SIZE / sizeof(u32); - memcpy(cdesc->control_data.token, iv, DES3_EDE_BLOCK_SIZE); + block_sz = DES3_EDE_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD; break; case SAFEXCEL_AES: - offset = AES_BLOCK_SIZE / sizeof(u32); - memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE); + block_sz = AES_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; break; } + + offset = block_sz / sizeof(u32); + memcpy(cdesc->control_data.token, iv, block_sz); } token = (struct safexcel_token *)(cdesc->control_data.token + offset); @@ -90,6 +90,25 @@ static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, token[0].instructions = EIP197_TOKEN_INS_LAST | EIP197_TOKEN_INS_TYPE_CRYTO | EIP197_TOKEN_INS_TYPE_OUTPUT; + + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) { + u32 last = (EIP197_MAX_TOKENS - 1) - offset; + + token[last].opcode = EIP197_TOKEN_OPCODE_CTX_ACCESS; + token[last].packet_length = EIP197_TOKEN_DIRECTION_EXTERNAL | + EIP197_TOKEN_EXEC_IF_SUCCESSFUL| + EIP197_TOKEN_CTX_OFFSET(0x2); + token[last].stat = EIP197_TOKEN_STAT_LAST_HASH | + EIP197_TOKEN_STAT_LAST_PACKET; + token[last].instructions = + EIP197_TOKEN_INS_ORIGIN_LEN(block_sz / sizeof(u32)) | + EIP197_TOKEN_INS_ORIGIN_IV0; + + /* Store the updated IV values back in the internal context + * registers. + */ + cdesc->control_data.control1 |= CONTEXT_CONTROL_CRYPTO_STORE; + } } static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, @@ -559,6 +578,7 @@ static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, { struct skcipher_request *req = skcipher_request_cast(async); struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(async->tfm); int err; if (sreq->needs_inv) { @@ -569,6 +589,24 @@ static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, err = safexcel_handle_req_result(priv, ring, async, req->src, req->dst, req->cryptlen, sreq, should_complete, ret); + + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) { + u32 block_sz = 0; + + switch (ctx->alg) { + case SAFEXCEL_DES: + block_sz = DES_BLOCK_SIZE; + break; + case SAFEXCEL_3DES: + block_sz = DES3_EDE_BLOCK_SIZE; + break; + case SAFEXCEL_AES: + block_sz = AES_BLOCK_SIZE; + break; + } + + memcpy(req->iv, ctx->base.ctxr->data, block_sz); + } } return err; From 082ec2d48467b61aa89783e954645ec441714c4e Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:51:04 +0200 Subject: [PATCH 071/143] crypto: inside-secure - add support for HMAC updates This patch adds support for HMAC updates in the Inside Secure SafeXcel crypto engine driver. Updates were supported for hash algorithms, but were never enabled for HMAC ones. This fixes boot time test issues. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.h | 2 +- drivers/crypto/inside-secure/safexcel_hash.c | 58 +++++++++++++------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index ca6ece5607cd..e0c202f33674 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -644,7 +644,7 @@ struct safexcel_ahash_export_state { u32 digest; u32 state[SHA512_DIGEST_SIZE / sizeof(u32)]; - u8 cache[SHA512_BLOCK_SIZE]; + u8 cache[SHA512_BLOCK_SIZE << 1]; }; /* diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index a9197d2c5a48..20950744ea4e 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -41,11 +41,11 @@ struct safexcel_ahash_req { u64 len[2]; u64 processed[2]; - u8 cache[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); + u8 cache[SHA512_BLOCK_SIZE << 1] __aligned(sizeof(u32)); dma_addr_t cache_dma; unsigned int cache_sz; - u8 cache_next[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); + u8 cache_next[SHA512_BLOCK_SIZE << 1] __aligned(sizeof(u32)); }; static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) @@ -89,6 +89,9 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, cdesc->control_data.control0 |= ctx->alg; cdesc->control_data.control0 |= req->digest; + if (!req->finish) + cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH; + if (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) { if (req->processed[0] || req->processed[1]) { if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5) @@ -107,9 +110,6 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, cdesc->control_data.control0 |= CONTEXT_CONTROL_RESTART_HASH; } - if (!req->finish) - cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH; - /* * Copy the input digest if needed, and setup the context * fields. Do this now as we need it to setup the first command @@ -212,11 +212,15 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, struct safexcel_command_desc *cdesc, *first_cdesc = NULL; struct safexcel_result_desc *rdesc; struct scatterlist *sg; - int i, extra, n_cdesc = 0, ret = 0; - u64 queued, len, cache_len; + int i, extra = 0, n_cdesc = 0, ret = 0; + u64 queued, len, cache_len, cache_max; + + cache_max = crypto_ahash_blocksize(ahash); + if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) + cache_max <<= 1; queued = len = safexcel_queued_len(req); - if (queued <= crypto_ahash_blocksize(ahash)) + if (queued <= cache_max) cache_len = queued; else cache_len = queued - areq->nbytes; @@ -227,6 +231,10 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, */ extra = queued & (crypto_ahash_blocksize(ahash) - 1); + if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC && + extra < crypto_ahash_blocksize(ahash)) + extra += crypto_ahash_blocksize(ahash); + /* If this is not the last request and the queued data * is a multiple of a block, cache the last one for now. */ @@ -239,12 +247,6 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, queued -= extra; len -= extra; - - if (!queued) { - *commands = 0; - *results = 0; - return 0; - } } /* Add a command descriptor for the cached data, if any */ @@ -522,10 +524,9 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) /* safexcel_ahash_cache: cache data until at least one request can be sent to * the engine, aka. when there is at least 1 block size in the pipe. */ -static int safexcel_ahash_cache(struct ahash_request *areq) +static int safexcel_ahash_cache(struct ahash_request *areq, u32 cache_max) { struct safexcel_ahash_req *req = ahash_request_ctx(areq); - struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); u64 queued, cache_len; /* queued: everything accepted by the driver which will be handled by @@ -542,7 +543,7 @@ static int safexcel_ahash_cache(struct ahash_request *areq) * In case there isn't enough bytes to proceed (less than a * block size), cache the data until we have enough. */ - if (cache_len + areq->nbytes <= crypto_ahash_blocksize(ahash)) { + if (cache_len + areq->nbytes <= cache_max) { sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), req->cache + cache_len, areq->nbytes, 0); @@ -602,6 +603,7 @@ static int safexcel_ahash_update(struct ahash_request *areq) { struct safexcel_ahash_req *req = ahash_request_ctx(areq); struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + u32 cache_max; /* If the request is 0 length, do nothing */ if (!areq->nbytes) @@ -611,7 +613,11 @@ static int safexcel_ahash_update(struct ahash_request *areq) if (req->len[0] < areq->nbytes) req->len[1]++; - safexcel_ahash_cache(areq); + cache_max = crypto_ahash_blocksize(ahash); + if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) + cache_max <<= 1; + + safexcel_ahash_cache(areq, cache_max); /* * We're not doing partial updates when performing an hmac request. @@ -624,7 +630,7 @@ static int safexcel_ahash_update(struct ahash_request *areq) return safexcel_ahash_enqueue(areq); if (!req->last_req && - safexcel_queued_len(req) > crypto_ahash_blocksize(ahash)) + safexcel_queued_len(req) > cache_max) return safexcel_ahash_enqueue(areq); return 0; @@ -681,6 +687,11 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out) struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); struct safexcel_ahash_req *req = ahash_request_ctx(areq); struct safexcel_ahash_export_state *export = out; + u32 cache_sz; + + cache_sz = crypto_ahash_blocksize(ahash); + if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) + cache_sz <<= 1; export->len[0] = req->len[0]; export->len[1] = req->len[1]; @@ -690,7 +701,7 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out) export->digest = req->digest; memcpy(export->state, req->state, req->state_sz); - memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash)); + memcpy(export->cache, req->cache, cache_sz); return 0; } @@ -700,12 +711,17 @@ static int safexcel_ahash_import(struct ahash_request *areq, const void *in) struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); struct safexcel_ahash_req *req = ahash_request_ctx(areq); const struct safexcel_ahash_export_state *export = in; + u32 cache_sz; int ret; ret = crypto_ahash_init(areq); if (ret) return ret; + cache_sz = crypto_ahash_blocksize(ahash); + if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) + cache_sz <<= 1; + req->len[0] = export->len[0]; req->len[1] = export->len[1]; req->processed[0] = export->processed[0]; @@ -713,7 +729,7 @@ static int safexcel_ahash_import(struct ahash_request *areq, const void *in) req->digest = export->digest; - memcpy(req->cache, export->cache, crypto_ahash_blocksize(ahash)); + memcpy(req->cache, export->cache, cache_sz); memcpy(req->state, export->state, req->state_sz); return 0; From 583d7e195f48753c5da8cc75a8ed332cad7287b0 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:51:05 +0200 Subject: [PATCH 072/143] crypto: inside-secure - fix use of the SG list Replace sg_nents_for_len by sg_nents when DMA mapping/unmapping buffers and when looping over the SG entries. This fix cases where the SG entries aren't used fully, which would in such cases led to using fewer SG entries than needed (and thus the engine wouldn't have access to the full input data and the result would be wrong). Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- .../crypto/inside-secure/safexcel_cipher.c | 39 ++++++------------- drivers/crypto/inside-secure/safexcel_hash.c | 3 +- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index cedfb121c278..6e193baccec7 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -369,16 +369,10 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin safexcel_complete(priv, ring); if (src == dst) { - dma_unmap_sg(priv->dev, src, - sg_nents_for_len(src, cryptlen), - DMA_BIDIRECTIONAL); + dma_unmap_sg(priv->dev, src, sg_nents(src), DMA_BIDIRECTIONAL); } else { - dma_unmap_sg(priv->dev, src, - sg_nents_for_len(src, cryptlen), - DMA_TO_DEVICE); - dma_unmap_sg(priv->dev, dst, - sg_nents_for_len(dst, cryptlen), - DMA_FROM_DEVICE); + dma_unmap_sg(priv->dev, src, sg_nents(src), DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, dst, sg_nents(dst), DMA_FROM_DEVICE); } *should_complete = true; @@ -403,26 +397,21 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, int i, ret = 0; if (src == dst) { - nr_src = dma_map_sg(priv->dev, src, - sg_nents_for_len(src, totlen), + nr_src = dma_map_sg(priv->dev, src, sg_nents(src), DMA_BIDIRECTIONAL); nr_dst = nr_src; if (!nr_src) return -EINVAL; } else { - nr_src = dma_map_sg(priv->dev, src, - sg_nents_for_len(src, totlen), + nr_src = dma_map_sg(priv->dev, src, sg_nents(src), DMA_TO_DEVICE); if (!nr_src) return -EINVAL; - nr_dst = dma_map_sg(priv->dev, dst, - sg_nents_for_len(dst, totlen), + nr_dst = dma_map_sg(priv->dev, dst, sg_nents(dst), DMA_FROM_DEVICE); if (!nr_dst) { - dma_unmap_sg(priv->dev, src, - sg_nents_for_len(src, totlen), - DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, src, nr_src, DMA_TO_DEVICE); return -EINVAL; } } @@ -472,7 +461,7 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, /* result descriptors */ for_each_sg(dst, sg, nr_dst, i) { - bool first = !i, last = (i == nr_dst - 1); + bool first = !i, last = sg_is_last(sg); u32 len = sg_dma_len(sg); rdesc = safexcel_add_rdesc(priv, ring, first, last, @@ -501,16 +490,10 @@ cdesc_rollback: safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); if (src == dst) { - dma_unmap_sg(priv->dev, src, - sg_nents_for_len(src, totlen), - DMA_BIDIRECTIONAL); + dma_unmap_sg(priv->dev, src, nr_src, DMA_BIDIRECTIONAL); } else { - dma_unmap_sg(priv->dev, src, - sg_nents_for_len(src, totlen), - DMA_TO_DEVICE); - dma_unmap_sg(priv->dev, dst, - sg_nents_for_len(dst, totlen), - DMA_FROM_DEVICE); + dma_unmap_sg(priv->dev, src, nr_src, DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, dst, nr_dst, DMA_FROM_DEVICE); } return ret; diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 20950744ea4e..a80a5e757b1f 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -273,8 +273,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, } /* Now handle the current ahash request buffer(s) */ - req->nents = dma_map_sg(priv->dev, areq->src, - sg_nents_for_len(areq->src, areq->nbytes), + req->nents = dma_map_sg(priv->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); if (!req->nents) { ret = -ENOMEM; From 89332590427235680236b9470e851afc49b3caa1 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Mon, 27 May 2019 16:51:06 +0200 Subject: [PATCH 073/143] crypto: inside-secure - do not rely on the hardware last bit for result descriptors When performing a transformation the hardware is given result descriptors to save the result data. Those result descriptors are batched using a 'first' and a 'last' bit. There are cases were more descriptors than needed are given to the engine, leading to the engine only using some of them, and not setting the last bit on the last descriptor we gave. This causes issues were the driver and the hardware aren't in sync anymore about the number of result descriptors given (as the driver do not give a pool of descriptor to use for any transformation, but a pool of descriptors to use *per* transformation). This patch fixes it by attaching the number of given result descriptors to the requests, and by using this number instead of the 'last' bit found on the descriptors to process them. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- .../crypto/inside-secure/safexcel_cipher.c | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 6e193baccec7..8cdbdbe35681 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -51,6 +51,8 @@ struct safexcel_cipher_ctx { struct safexcel_cipher_req { enum safexcel_cipher_direction direction; + /* Number of result descriptors associated to the request */ + unsigned int rdescs; bool needs_inv; }; @@ -351,7 +353,10 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin *ret = 0; - do { + if (unlikely(!sreq->rdescs)) + return 0; + + while (sreq->rdescs--) { rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); if (IS_ERR(rdesc)) { dev_err(priv->dev, @@ -364,7 +369,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin *ret = safexcel_rdesc_check_errors(priv, rdesc); ndesc++; - } while (!rdesc->last_seg); + } safexcel_complete(priv, ring); @@ -502,6 +507,7 @@ cdesc_rollback: static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, int ring, struct crypto_async_request *base, + struct safexcel_cipher_req *sreq, bool *should_complete, int *ret) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); @@ -510,7 +516,10 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, *ret = 0; - do { + if (unlikely(!sreq->rdescs)) + return 0; + + while (sreq->rdescs--) { rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); if (IS_ERR(rdesc)) { dev_err(priv->dev, @@ -523,7 +532,7 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, *ret = safexcel_rdesc_check_errors(priv, rdesc); ndesc++; - } while (!rdesc->last_seg); + } safexcel_complete(priv, ring); @@ -566,7 +575,7 @@ static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, if (sreq->needs_inv) { sreq->needs_inv = false; - err = safexcel_handle_inv_result(priv, ring, async, + err = safexcel_handle_inv_result(priv, ring, async, sreq, should_complete, ret); } else { err = safexcel_handle_req_result(priv, ring, async, req->src, @@ -607,7 +616,7 @@ static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv, if (sreq->needs_inv) { sreq->needs_inv = false; - err = safexcel_handle_inv_result(priv, ring, async, + err = safexcel_handle_inv_result(priv, ring, async, sreq, should_complete, ret); } else { err = safexcel_handle_req_result(priv, ring, async, req->src, @@ -653,6 +662,8 @@ static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, ret = safexcel_send_req(async, ring, sreq, req->src, req->dst, req->cryptlen, 0, 0, req->iv, commands, results); + + sreq->rdescs = *results; return ret; } @@ -675,6 +686,7 @@ static int safexcel_aead_send(struct crypto_async_request *async, int ring, req->cryptlen, req->assoclen, crypto_aead_authsize(tfm), req->iv, commands, results); + sreq->rdescs = *results; return ret; } From c3725f7ccc8cae1fb71e97768f03bfbd846f7827 Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Tue, 28 May 2019 12:52:10 +0300 Subject: [PATCH 074/143] crypto: caam - fix pkcs1pad(rsa-caam, sha256) failure because of invalid input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The problem is with the input data size sent to CAAM for encrypt/decrypt. Pkcs1pad is failing due to pkcs1 padding done in SW starting with0x01 instead of 0x00 0x01. CAAM expects an input of modulus size. For this we strip the leading zeros in case the size is more than modulus or pad the input with zeros until the modulus size is reached. Signed-off-by: Iuliana Prodan Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caampkc.c | 87 ++++++++++++++++++++++++++++------- drivers/crypto/caam/caampkc.h | 2 + 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index 34e37f9bd828..2bd3a17fd4da 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -24,6 +24,10 @@ sizeof(struct rsa_priv_f2_pdb)) #define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \ sizeof(struct rsa_priv_f3_pdb)) +#define CAAM_RSA_MAX_INPUT_SIZE 512 /* for a 4096-bit modulus */ + +/* buffer filled with zeros, used for padding */ +static u8 *zero_buffer; static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, struct akcipher_request *req) @@ -168,6 +172,13 @@ static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err, akcipher_request_complete(req, err); } +/** + * Count leading zeros, need it to strip, from a given scatterlist + * + * @sgl : scatterlist to count zeros from + * @nbytes: number of zeros, in bytes, to strip + * @flags : operation flags + */ static int caam_rsa_count_leading_zeros(struct scatterlist *sgl, unsigned int nbytes, unsigned int flags) @@ -187,7 +198,8 @@ static int caam_rsa_count_leading_zeros(struct scatterlist *sgl, lzeros = 0; len = 0; while (nbytes > 0) { - while (len && !*buff) { + /* do not strip more than given bytes */ + while (len && !*buff && lzeros < nbytes) { lzeros++; len--; buff++; @@ -218,6 +230,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct device *dev = ctx->dev; struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + struct caam_rsa_key *key = &ctx->key; struct rsa_edesc *edesc; gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; @@ -225,21 +238,37 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, int sgc; int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; int src_nents, dst_nents; + unsigned int diff_size = 0; int lzeros; - lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len, sg_flags); - if (lzeros < 0) - return ERR_PTR(lzeros); + if (req->src_len > key->n_sz) { + /* + * strip leading zeros and + * return the number of zeros to skip + */ + lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len - + key->n_sz, sg_flags); + if (lzeros < 0) + return ERR_PTR(lzeros); - req->src_len -= lzeros; - req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros); + req->src_len -= lzeros; + req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros); + } else { + /* + * input src is less then n key modulus, + * so there will be zero padding + */ + diff_size = key->n_sz - req->src_len; + } src_nents = sg_nents_for_len(req->src, req->src_len); dst_nents = sg_nents_for_len(req->dst, req->dst_len); - if (src_nents > 1) - sec4_sg_len = src_nents; - + if (!diff_size && src_nents == 1) + sec4_sg_len = 0; /* no need for an input hw s/g table */ + else + sec4_sg_len = src_nents + !!diff_size; + sec4_sg_index = sec4_sg_len; if (dst_nents > 1) sec4_sg_len += pad_sg_nents(dst_nents); else @@ -266,12 +295,14 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, } edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen; + if (diff_size) + dma_to_sec4_sg_one(edesc->sec4_sg, ctx->padding_dma, diff_size, + 0); + + if (sec4_sg_index) + sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg + + !!diff_size, 0); - sec4_sg_index = 0; - if (src_nents > 1) { - sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); - sec4_sg_index += src_nents; - } if (dst_nents > 1) sg_to_sec4_sg_last(req->dst, dst_nents, edesc->sec4_sg + sec4_sg_index, 0); @@ -292,6 +323,10 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, edesc->sec4_sg_bytes = sec4_sg_bytes; + print_hex_dump_debug("caampkc sec4_sg@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, + edesc->sec4_sg_bytes, 1); + return edesc; sec4_sg_fail: @@ -981,6 +1016,15 @@ static int caam_rsa_init_tfm(struct crypto_akcipher *tfm) return PTR_ERR(ctx->dev); } + ctx->padding_dma = dma_map_single(ctx->dev, zero_buffer, + CAAM_RSA_MAX_INPUT_SIZE - 1, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, ctx->padding_dma)) { + dev_err(ctx->dev, "unable to map padding\n"); + caam_jr_free(ctx->dev); + return -ENOMEM; + } + return 0; } @@ -990,6 +1034,8 @@ static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm) struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct caam_rsa_key *key = &ctx->key; + dma_unmap_single(ctx->dev, ctx->padding_dma, CAAM_RSA_MAX_INPUT_SIZE - + 1, DMA_TO_DEVICE); caam_rsa_free_key(key); caam_jr_free(ctx->dev); } @@ -1030,17 +1076,26 @@ int caam_pkc_init(struct device *ctrldev) if (!pk_inst) return 0; + /* allocate zero buffer, used for padding input */ + zero_buffer = kzalloc(CAAM_RSA_MAX_INPUT_SIZE - 1, GFP_DMA | + GFP_KERNEL); + if (!zero_buffer) + return -ENOMEM; + err = crypto_register_akcipher(&caam_rsa); - if (err) + if (err) { + kfree(zero_buffer); dev_warn(ctrldev, "%s alg registration failed\n", caam_rsa.base.cra_driver_name); - else + } else { dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n"); + } return err; } void caam_pkc_exit(void) { + kfree(zero_buffer); crypto_unregister_akcipher(&caam_rsa); } diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h index 82645bcf8b27..5ac72015eba5 100644 --- a/drivers/crypto/caam/caampkc.h +++ b/drivers/crypto/caam/caampkc.h @@ -89,10 +89,12 @@ struct caam_rsa_key { * caam_rsa_ctx - per session context. * @key : RSA key in DMA zone * @dev : device structure + * @padding_dma : dma address of padding, for adding it to the input */ struct caam_rsa_ctx { struct caam_rsa_key key; struct device *dev; + dma_addr_t padding_dma; }; /** From 3b2614cb864d4114250f043ce6ff94e152198ae5 Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Tue, 28 May 2019 12:52:11 +0300 Subject: [PATCH 075/143] crypto: caam - strip input without changing crypto request For rsa and pkcs1pad, CAAM expects an input of modulus size. For this we strip the leading zeros in case the size is more than modulus. This commit avoids modifying the crypto request while stripping zeros from input, to comply with the crypto API requirement. This is done by adding a fixup input pointer and length. Signed-off-by: Iuliana Prodan Reviewed-by: Horia Geanta Signed-off-by: Herbert Xu --- drivers/crypto/caam/caampkc.c | 39 +++++++++++++++++++++++------------ drivers/crypto/caam/caampkc.h | 7 ++++++- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index 2bd3a17fd4da..19b02c1973fc 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -32,8 +32,10 @@ static u8 *zero_buffer; static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, struct akcipher_request *req) { + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE); - dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req_ctx->fixup_src, edesc->src_nents, DMA_TO_DEVICE); if (edesc->sec4_sg_bytes) dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes, @@ -251,17 +253,21 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, if (lzeros < 0) return ERR_PTR(lzeros); - req->src_len -= lzeros; - req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros); + req_ctx->fixup_src = scatterwalk_ffwd(req_ctx->src, req->src, + lzeros); + req_ctx->fixup_src_len = req->src_len - lzeros; } else { /* * input src is less then n key modulus, * so there will be zero padding */ diff_size = key->n_sz - req->src_len; + req_ctx->fixup_src = req->src; + req_ctx->fixup_src_len = req->src_len; } - src_nents = sg_nents_for_len(req->src, req->src_len); + src_nents = sg_nents_for_len(req_ctx->fixup_src, + req_ctx->fixup_src_len); dst_nents = sg_nents_for_len(req->dst, req->dst_len); if (!diff_size && src_nents == 1) @@ -282,7 +288,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, if (!edesc) return ERR_PTR(-ENOMEM); - sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + sgc = dma_map_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE); if (unlikely(!sgc)) { dev_err(dev, "unable to map source\n"); goto src_fail; @@ -300,8 +306,8 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, 0); if (sec4_sg_index) - sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg + - !!diff_size, 0); + sg_to_sec4_sg_last(req_ctx->fixup_src, src_nents, + edesc->sec4_sg + !!diff_size, 0); if (dst_nents > 1) sg_to_sec4_sg_last(req->dst, dst_nents, @@ -332,7 +338,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, sec4_sg_fail: dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE); dst_fail: - dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE); src_fail: kfree(edesc); return ERR_PTR(-ENOMEM); @@ -342,6 +348,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req, struct rsa_edesc *edesc) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct caam_rsa_key *key = &ctx->key; struct device *dev = ctx->dev; @@ -366,7 +373,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req, pdb->f_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->f_dma = sg_dma_address(req->src); + pdb->f_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -378,7 +385,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req, } pdb->sgf |= (key->e_sz << RSA_PDB_E_SHIFT) | key->n_sz; - pdb->f_len = req->src_len; + pdb->f_len = req_ctx->fixup_src_len; return 0; } @@ -411,7 +418,9 @@ static int set_rsa_priv_f1_pdb(struct akcipher_request *req, pdb->g_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->g_dma = sg_dma_address(req->src); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -474,7 +483,9 @@ static int set_rsa_priv_f2_pdb(struct akcipher_request *req, pdb->g_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->g_dma = sg_dma_address(req->src); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -561,7 +572,9 @@ static int set_rsa_priv_f3_pdb(struct akcipher_request *req, pdb->g_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->g_dma = sg_dma_address(req->src); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h index 5ac72015eba5..2c488c9a3812 100644 --- a/drivers/crypto/caam/caampkc.h +++ b/drivers/crypto/caam/caampkc.h @@ -95,14 +95,19 @@ struct caam_rsa_ctx { struct caam_rsa_key key; struct device *dev; dma_addr_t padding_dma; + }; /** * caam_rsa_req_ctx - per request context. - * @src: input scatterlist (stripped of leading zeros) + * @src : input scatterlist (stripped of leading zeros) + * @fixup_src : input scatterlist (that might be stripped of leading zeros) + * @fixup_src_len : length of the fixup_src input scatterlist */ struct caam_rsa_req_ctx { struct scatterlist src[2]; + struct scatterlist *fixup_src; + unsigned int fixup_src_len; }; /** From 1d4aaf16defa86d2665ae7db0259d6cb07e2091f Mon Sep 17 00:00:00 2001 From: Elena Petrova Date: Tue, 28 May 2019 13:41:52 +0100 Subject: [PATCH 076/143] crypto: arm64/sha1-ce - correct digest for empty data in finup The sha1-ce finup implementation for ARM64 produces wrong digest for empty input (len=0). Expected: da39a3ee..., result: 67452301... (initial value of SHA internal state). The error is in sha1_ce_finup: for empty data `finalize` will be 1, so the code is relying on sha1_ce_transform to make the final round. However, in sha1_base_do_update, the block function will not be called when len == 0. Fix it by setting finalize to 0 if data is empty. Fixes: 07eb54d306f4 ("crypto: arm64/sha1-ce - move SHA-1 ARMv8 implementation to base layer") Cc: stable@vger.kernel.org Signed-off-by: Elena Petrova Reviewed-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha1-ce-glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index eaa7a8258f1c..0652f5f07ed1 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -55,7 +55,7 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { struct sha1_ce_state *sctx = shash_desc_ctx(desc); - bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE); + bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE) && len; if (!crypto_simd_usable()) return crypto_sha1_finup(desc, data, len, out); From 6bd934de1e393466b319d29c4427598fda096c57 Mon Sep 17 00:00:00 2001 From: Elena Petrova Date: Tue, 28 May 2019 15:35:06 +0100 Subject: [PATCH 077/143] crypto: arm64/sha2-ce - correct digest for empty data in finup The sha256-ce finup implementation for ARM64 produces wrong digest for empty input (len=0). Expected: the actual digest, result: initial value of SHA internal state. The error is in sha256_ce_finup: for empty data `finalize` will be 1, so the code is relying on sha2_ce_transform to make the final round. However, in sha256_base_do_update, the block function will not be called when len == 0. Fix it by setting finalize to 0 if data is empty. Fixes: 03802f6a80b3a ("crypto: arm64/sha2-ce - move SHA-224/256 ARMv8 implementation to base layer") Cc: stable@vger.kernel.org Signed-off-by: Elena Petrova Reviewed-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/sha2-ce-glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index a725997e55f2..6a5ade974a35 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -60,7 +60,7 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { struct sha256_ce_state *sctx = shash_desc_ctx(desc); - bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE); + bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE) && len; if (!crypto_simd_usable()) { if (len) From d8ea98aa3cd4646945a2a9b647c2502b1e2dcdec Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 28 May 2019 09:40:55 -0700 Subject: [PATCH 078/143] crypto: testmgr - test the shash API For hash algorithms implemented using the "shash" algorithm type, test both the ahash and shash APIs, not just the ahash API. Testing the ahash API already tests the shash API indirectly, which is normally good enough. However, there have been corner cases where there have been shash bugs that don't get exposed through the ahash API. So, update testmgr to test the shash API too. This would have detected the arm64 SHA-1 and SHA-2 bugs for which fixes were just sent out (https://patchwork.kernel.org/patch/10964843/ and https://patchwork.kernel.org/patch/10965089/): alg: shash: sha1-ce test failed (wrong result) on test vector 0, cfg="init+finup aligned buffer" alg: shash: sha224-ce test failed (wrong result) on test vector 0, cfg="init+finup aligned buffer" alg: shash: sha256-ce test failed (wrong result) on test vector 0, cfg="init+finup aligned buffer" This also would have detected the bugs fixed by commit 307508d10729 ("crypto: crct10dif-generic - fix use via crypto_shash_digest()") and commit dec3d0b1071a ("crypto: x86/crct10dif-pcl - fix use via crypto_shash_digest()"). Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/testmgr.c | 402 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 335 insertions(+), 67 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index c9e67c2bd725..a347be142323 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1037,6 +1037,205 @@ static void crypto_reenable_simd_for_test(void) } #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int build_hash_sglist(struct test_sglist *tsgl, + const struct hash_testvec *vec, + const struct testvec_config *cfg, + unsigned int alignmask, + const struct test_sg_division *divs[XBUFSIZE]) +{ + struct kvec kv; + struct iov_iter input; + + kv.iov_base = (void *)vec->plaintext; + kv.iov_len = vec->psize; + iov_iter_kvec(&input, WRITE, &kv, 1, vec->psize); + return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize, + &input, divs); +} + +static int check_hash_result(const char *type, + const u8 *result, unsigned int digestsize, + const struct hash_testvec *vec, + const char *vec_name, + const char *driver, + const struct testvec_config *cfg) +{ + if (memcmp(result, vec->digest, digestsize) != 0) { + pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + type, driver, vec_name, cfg->name); + return -EINVAL; + } + if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) { + pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n", + type, driver, vec_name, cfg->name); + return -EOVERFLOW; + } + return 0; +} + +static inline int check_shash_op(const char *op, int err, + const char *driver, const char *vec_name, + const struct testvec_config *cfg) +{ + if (err) + pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, op, err, vec_name, cfg->name); + return err; +} + +static inline const void *sg_data(struct scatterlist *sg) +{ + return page_address(sg_page(sg)) + sg->offset; +} + +/* Test one hash test vector in one configuration, using the shash API */ +static int test_shash_vec_cfg(const char *driver, + const struct hash_testvec *vec, + const char *vec_name, + const struct testvec_config *cfg, + struct shash_desc *desc, + struct test_sglist *tsgl, + u8 *hashstate) +{ + struct crypto_shash *tfm = desc->tfm; + const unsigned int alignmask = crypto_shash_alignmask(tfm); + const unsigned int digestsize = crypto_shash_digestsize(tfm); + const unsigned int statesize = crypto_shash_statesize(tfm); + const struct test_sg_division *divs[XBUFSIZE]; + unsigned int i; + u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN]; + int err; + + /* Set the key, if specified */ + if (vec->ksize) { + err = crypto_shash_setkey(tfm, vec->key, vec->ksize); + if (err) { + if (err == vec->setkey_error) + return 0; + pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, + crypto_shash_get_flags(tfm)); + return err; + } + if (vec->setkey_error) { + pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); + return -EINVAL; + } + } + + /* Build the scatterlist for the source data */ + err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs); + if (err) { + pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); + return err; + } + + /* Do the actual hashing */ + + testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm)); + testmgr_poison(result, digestsize + TESTMGR_POISON_LEN); + + if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST || + vec->digest_error) { + /* Just using digest() */ + if (tsgl->nents != 1) + return 0; + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_digest(desc, sg_data(&tsgl->sgl[0]), + tsgl->sgl[0].length, result); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + if (err) { + if (err == vec->digest_error) + return 0; + pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, err, + cfg->name); + return err; + } + if (vec->digest_error) { + pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, cfg->name); + return -EINVAL; + } + goto result_ready; + } + + /* Using init(), zero or more update(), then final() or finup() */ + + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_init(desc); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("init", err, driver, vec_name, cfg); + if (err) + return err; + + for (i = 0; i < tsgl->nents; i++) { + if (i + 1 == tsgl->nents && + cfg->finalization_type == FINALIZATION_TYPE_FINUP) { + if (divs[i]->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_finup(desc, sg_data(&tsgl->sgl[i]), + tsgl->sgl[i].length, result); + if (divs[i]->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("finup", err, driver, vec_name, + cfg); + if (err) + return err; + goto result_ready; + } + if (divs[i]->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_update(desc, sg_data(&tsgl->sgl[i]), + tsgl->sgl[i].length); + if (divs[i]->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("update", err, driver, vec_name, cfg); + if (err) + return err; + if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) { + /* Test ->export() and ->import() */ + testmgr_poison(hashstate + statesize, + TESTMGR_POISON_LEN); + err = crypto_shash_export(desc, hashstate); + err = check_shash_op("export", err, driver, vec_name, + cfg); + if (err) + return err; + if (!testmgr_is_poison(hashstate + statesize, + TESTMGR_POISON_LEN)) { + pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); + return -EOVERFLOW; + } + testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm)); + err = crypto_shash_import(desc, hashstate); + err = check_shash_op("import", err, driver, vec_name, + cfg); + if (err) + return err; + } + } + + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_final(desc, result); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("final", err, driver, vec_name, cfg); + if (err) + return err; +result_ready: + return check_hash_result("shash", result, digestsize, vec, vec_name, + driver, cfg); +} + static int do_ahash_op(int (*op)(struct ahash_request *req), struct ahash_request *req, struct crypto_wait *wait, bool nosimd) @@ -1054,31 +1253,32 @@ static int do_ahash_op(int (*op)(struct ahash_request *req), return crypto_wait_req(err, wait); } -static int check_nonfinal_hash_op(const char *op, int err, - u8 *result, unsigned int digestsize, - const char *driver, const char *vec_name, - const struct testvec_config *cfg) +static int check_nonfinal_ahash_op(const char *op, int err, + u8 *result, unsigned int digestsize, + const char *driver, const char *vec_name, + const struct testvec_config *cfg) { if (err) { - pr_err("alg: hash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", driver, op, err, vec_name, cfg->name); return err; } if (!testmgr_is_poison(result, digestsize)) { - pr_err("alg: hash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n", driver, op, vec_name, cfg->name); return -EINVAL; } return 0; } -static int test_hash_vec_cfg(const char *driver, - const struct hash_testvec *vec, - const char *vec_name, - const struct testvec_config *cfg, - struct ahash_request *req, - struct test_sglist *tsgl, - u8 *hashstate) +/* Test one hash test vector in one configuration, using the ahash API */ +static int test_ahash_vec_cfg(const char *driver, + const struct hash_testvec *vec, + const char *vec_name, + const struct testvec_config *cfg, + struct ahash_request *req, + struct test_sglist *tsgl, + u8 *hashstate) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); const unsigned int alignmask = crypto_ahash_alignmask(tfm); @@ -1087,8 +1287,6 @@ static int test_hash_vec_cfg(const char *driver, const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags; const struct test_sg_division *divs[XBUFSIZE]; DECLARE_CRYPTO_WAIT(wait); - struct kvec _input; - struct iov_iter input; unsigned int i; struct scatterlist *pending_sgl; unsigned int pending_len; @@ -1101,26 +1299,22 @@ static int test_hash_vec_cfg(const char *driver, if (err) { if (err == vec->setkey_error) return 0; - pr_err("alg: hash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", driver, vec_name, vec->setkey_error, err, crypto_ahash_get_flags(tfm)); return err; } if (vec->setkey_error) { - pr_err("alg: hash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", driver, vec_name, vec->setkey_error); return -EINVAL; } } /* Build the scatterlist for the source data */ - _input.iov_base = (void *)vec->plaintext; - _input.iov_len = vec->psize; - iov_iter_kvec(&input, WRITE, &_input, 1, vec->psize); - err = build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize, - &input, divs); + err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs); if (err) { - pr_err("alg: hash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", driver, vec_name, cfg->name); return err; } @@ -1140,13 +1334,13 @@ static int test_hash_vec_cfg(const char *driver, if (err) { if (err == vec->digest_error) return 0; - pr_err("alg: hash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", driver, vec_name, vec->digest_error, err, cfg->name); return err; } if (vec->digest_error) { - pr_err("alg: hash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", driver, vec_name, vec->digest_error, cfg->name); return -EINVAL; } @@ -1158,8 +1352,8 @@ static int test_hash_vec_cfg(const char *driver, ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, NULL, result, 0); err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd); - err = check_nonfinal_hash_op("init", err, result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("init", err, result, digestsize, + driver, vec_name, cfg); if (err) return err; @@ -1175,9 +1369,9 @@ static int test_hash_vec_cfg(const char *driver, pending_len); err = do_ahash_op(crypto_ahash_update, req, &wait, divs[i]->nosimd); - err = check_nonfinal_hash_op("update", err, - result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("update", err, + result, digestsize, + driver, vec_name, cfg); if (err) return err; pending_sgl = NULL; @@ -1188,23 +1382,23 @@ static int test_hash_vec_cfg(const char *driver, testmgr_poison(hashstate + statesize, TESTMGR_POISON_LEN); err = crypto_ahash_export(req, hashstate); - err = check_nonfinal_hash_op("export", err, - result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("export", err, + result, digestsize, + driver, vec_name, cfg); if (err) return err; if (!testmgr_is_poison(hashstate + statesize, TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", driver, vec_name, cfg->name); return -EOVERFLOW; } testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm)); err = crypto_ahash_import(req, hashstate); - err = check_nonfinal_hash_op("import", err, - result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("import", err, + result, digestsize, + driver, vec_name, cfg); if (err) return err; } @@ -1218,13 +1412,13 @@ static int test_hash_vec_cfg(const char *driver, if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) { /* finish with update() and final() */ err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd); - err = check_nonfinal_hash_op("update", err, result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("update", err, result, digestsize, + driver, vec_name, cfg); if (err) return err; err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", driver, err, vec_name, cfg->name); return err; } @@ -1232,31 +1426,49 @@ static int test_hash_vec_cfg(const char *driver, /* finish with finup() */ err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n", driver, err, vec_name, cfg->name); return err; } } result_ready: - /* Check that the algorithm produced the correct digest */ - if (memcmp(result, vec->digest, digestsize) != 0) { - pr_err("alg: hash: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", - driver, vec_name, cfg->name); - return -EINVAL; - } - if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s overran result buffer on test vector %s, cfg=\"%s\"\n", - driver, vec_name, cfg->name); - return -EOVERFLOW; + return check_hash_result("ahash", result, digestsize, vec, vec_name, + driver, cfg); +} + +static int test_hash_vec_cfg(const char *driver, + const struct hash_testvec *vec, + const char *vec_name, + const struct testvec_config *cfg, + struct ahash_request *req, + struct shash_desc *desc, + struct test_sglist *tsgl, + u8 *hashstate) +{ + int err; + + /* + * For algorithms implemented as "shash", most bugs will be detected by + * both the shash and ahash tests. Test the shash API first so that the + * failures involve less indirection, so are easier to debug. + */ + + if (desc) { + err = test_shash_vec_cfg(driver, vec, vec_name, cfg, desc, tsgl, + hashstate); + if (err) + return err; } - return 0; + return test_ahash_vec_cfg(driver, vec, vec_name, cfg, req, tsgl, + hashstate); } static int test_hash_vec(const char *driver, const struct hash_testvec *vec, unsigned int vec_num, struct ahash_request *req, - struct test_sglist *tsgl, u8 *hashstate) + struct shash_desc *desc, struct test_sglist *tsgl, + u8 *hashstate) { char vec_name[16]; unsigned int i; @@ -1267,7 +1479,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) { err = test_hash_vec_cfg(driver, vec, vec_name, &default_hash_testvec_configs[i], - req, tsgl, hashstate); + req, desc, tsgl, hashstate); if (err) return err; } @@ -1281,7 +1493,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); err = test_hash_vec_cfg(driver, vec, vec_name, &cfg, - req, tsgl, hashstate); + req, desc, tsgl, hashstate); if (err) return err; } @@ -1343,6 +1555,7 @@ static int test_hash_vs_generic_impl(const char *driver, const char *generic_driver, unsigned int maxkeysize, struct ahash_request *req, + struct shash_desc *desc, struct test_sglist *tsgl, u8 *hashstate) { @@ -1423,7 +1636,7 @@ static int test_hash_vs_generic_impl(const char *driver, generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); err = test_hash_vec_cfg(driver, &vec, vec_name, &cfg, - req, tsgl, hashstate); + req, desc, tsgl, hashstate); if (err) goto out; cond_resched(); @@ -1441,6 +1654,7 @@ static int test_hash_vs_generic_impl(const char *driver, const char *generic_driver, unsigned int maxkeysize, struct ahash_request *req, + struct shash_desc *desc, struct test_sglist *tsgl, u8 *hashstate) { @@ -1448,26 +1662,67 @@ static int test_hash_vs_generic_impl(const char *driver, } #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int alloc_shash(const char *driver, u32 type, u32 mask, + struct crypto_shash **tfm_ret, + struct shash_desc **desc_ret) +{ + struct crypto_shash *tfm; + struct shash_desc *desc; + + tfm = crypto_alloc_shash(driver, type, mask); + if (IS_ERR(tfm)) { + if (PTR_ERR(tfm) == -ENOENT) { + /* + * This algorithm is only available through the ahash + * API, not the shash API, so skip the shash tests. + */ + return 0; + } + pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n", + driver, PTR_ERR(tfm)); + return PTR_ERR(tfm); + } + + desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL); + if (!desc) { + crypto_free_shash(tfm); + return -ENOMEM; + } + desc->tfm = tfm; + + *tfm_ret = tfm; + *desc_ret = desc; + return 0; +} + static int __alg_test_hash(const struct hash_testvec *vecs, unsigned int num_vecs, const char *driver, u32 type, u32 mask, const char *generic_driver, unsigned int maxkeysize) { - struct crypto_ahash *tfm; + struct crypto_ahash *atfm = NULL; struct ahash_request *req = NULL; + struct crypto_shash *stfm = NULL; + struct shash_desc *desc = NULL; struct test_sglist *tsgl = NULL; u8 *hashstate = NULL; + unsigned int statesize; unsigned int i; int err; - tfm = crypto_alloc_ahash(driver, type, mask); - if (IS_ERR(tfm)) { + /* + * Always test the ahash API. This works regardless of whether the + * algorithm is implemented as ahash or shash. + */ + + atfm = crypto_alloc_ahash(driver, type, mask); + if (IS_ERR(atfm)) { pr_err("alg: hash: failed to allocate transform for %s: %ld\n", - driver, PTR_ERR(tfm)); - return PTR_ERR(tfm); + driver, PTR_ERR(atfm)); + return PTR_ERR(atfm); } - req = ahash_request_alloc(tfm, GFP_KERNEL); + req = ahash_request_alloc(atfm, GFP_KERNEL); if (!req) { pr_err("alg: hash: failed to allocate request for %s\n", driver); @@ -1475,6 +1730,14 @@ static int __alg_test_hash(const struct hash_testvec *vecs, goto out; } + /* + * If available also test the shash API, to cover corner cases that may + * be missed by testing the ahash API only. + */ + err = alloc_shash(driver, type, mask, &stfm, &desc); + if (err) + goto out; + tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL); if (!tsgl || init_test_sglist(tsgl) != 0) { pr_err("alg: hash: failed to allocate test buffers for %s\n", @@ -1485,8 +1748,10 @@ static int __alg_test_hash(const struct hash_testvec *vecs, goto out; } - hashstate = kmalloc(crypto_ahash_statesize(tfm) + TESTMGR_POISON_LEN, - GFP_KERNEL); + statesize = crypto_ahash_statesize(atfm); + if (stfm) + statesize = max(statesize, crypto_shash_statesize(stfm)); + hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL); if (!hashstate) { pr_err("alg: hash: failed to allocate hash state buffer for %s\n", driver); @@ -1495,20 +1760,23 @@ static int __alg_test_hash(const struct hash_testvec *vecs, } for (i = 0; i < num_vecs; i++) { - err = test_hash_vec(driver, &vecs[i], i, req, tsgl, hashstate); + err = test_hash_vec(driver, &vecs[i], i, req, desc, tsgl, + hashstate); if (err) goto out; } err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req, - tsgl, hashstate); + desc, tsgl, hashstate); out: kfree(hashstate); if (tsgl) { destroy_test_sglist(tsgl); kfree(tsgl); } + kfree(desc); + crypto_free_shash(stfm); ahash_request_free(req); - crypto_free_ahash(tfm); + crypto_free_ahash(atfm); return err; } From d9d67c87ad37218be65f4cea3ecd7e0312735e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Wed, 29 May 2019 21:24:25 +0200 Subject: [PATCH 079/143] crypto: jitter - update implementation to 2.1.2 The Jitter RNG implementation is updated to comply with upstream version 2.1.2. The change covers the following aspects: * Time variation measurement is conducted over the LFSR operation instead of the XOR folding * Invcation of stuck test during initialization * Removal of the stirring functionality and the Von-Neumann unbiaser as the LFSR using a primitive and irreducible polynomial generates an identical distribution of random bits This implementation was successfully used in FIPS 140-2 validations as well as in German BSI evaluations. This kernel implementation was tested as follows: * The unchanged kernel code file jitterentropy.c is compiled as part of user space application to generate raw unconditioned noise data. That data is processed with the NIST SP800-90B non-IID test tool to verify that the kernel code exhibits an equal amount of noise as the upstream Jitter RNG version 2.1.2. * Using AF_ALG with the libkcapi tool of kcapi-rng the Jitter RNG was output tested with dieharder to verify that the output does not exhibit statistical weaknesses. The following command was used: kcapi-rng -n "jitterentropy_rng" -b 100000000000 | dieharder -a -g 200 * The unchanged kernel code file jitterentropy.c is compiled as part of user space application to test the LFSR implementation. The LFSR is injected a monotonically increasing counter as input and the output is fed into dieharder to verify that the LFSR operation does not exhibit statistical weaknesses. * The patch was tested on the Muen separation kernel which returns a more coarse time stamp to verify that the Jitter RNG does not cause regressions with its initialization test considering that the Jitter RNG depends on a high-resolution timer. Tested-by: Reto Buerki Signed-off-by: Stephan Mueller Signed-off-by: Herbert Xu --- crypto/jitterentropy-kcapi.c | 5 - crypto/jitterentropy.c | 305 ++++++++++------------------------- 2 files changed, 82 insertions(+), 228 deletions(-) diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 6ea1a270b8dc..699db1726ead 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -56,11 +56,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector); * Helper function ***************************************************************************/ -__u64 jent_rol64(__u64 word, unsigned int shift) -{ - return rol64(word, shift); -} - void *jent_zalloc(unsigned int len) { return kzalloc(len, GFP_KERNEL); diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index acf44b2d2d1d..77fa2120fe0c 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -2,7 +2,7 @@ * Non-physical true random number generator based on timing jitter -- * Jitter RNG standalone code. * - * Copyright Stephan Mueller , 2015 + * Copyright Stephan Mueller , 2015 - 2019 * * Design * ====== @@ -47,7 +47,7 @@ /* * This Jitterentropy RNG is based on the jitterentropy library - * version 1.1.0 provided at http://www.chronox.de/jent.html + * version 2.1.2 provided at http://www.chronox.de/jent.html */ #ifdef __OPTIMIZE__ @@ -71,10 +71,7 @@ struct rand_data { #define DATA_SIZE_BITS ((sizeof(__u64)) * 8) __u64 last_delta; /* SENSITIVE stuck test */ __s64 last_delta2; /* SENSITIVE stuck test */ - unsigned int stuck:1; /* Time measurement stuck */ unsigned int osr; /* Oversample rate */ - unsigned int stir:1; /* Post-processing stirring */ - unsigned int disable_unbias:1; /* Deactivate Von-Neuman unbias */ #define JENT_MEMORY_BLOCKS 64 #define JENT_MEMORY_BLOCKSIZE 32 #define JENT_MEMORY_ACCESSLOOPS 128 @@ -89,8 +86,6 @@ struct rand_data { }; /* Flags that can be used to initialize the RNG */ -#define JENT_DISABLE_STIR (1<<0) /* Disable stirring the entropy pool */ -#define JENT_DISABLE_UNBIAS (1<<1) /* Disable the Von-Neuman Unbiaser */ #define JENT_DISABLE_MEMORY_ACCESS (1<<2) /* Disable memory access for more * entropy, saves MEMORY_SIZE RAM for * entropy collector */ @@ -99,19 +94,16 @@ struct rand_data { #define JENT_ENOTIME 1 /* Timer service not available */ #define JENT_ECOARSETIME 2 /* Timer too coarse for RNG */ #define JENT_ENOMONOTONIC 3 /* Timer is not monotonic increasing */ -#define JENT_EMINVARIATION 4 /* Timer variations too small for RNG */ #define JENT_EVARVAR 5 /* Timer does not produce variations of * variations (2nd derivation of time is * zero). */ -#define JENT_EMINVARVAR 6 /* Timer variations of variations is tooi - * small. */ +#define JENT_ESTUCK 8 /* Too many stuck results during init. */ /*************************************************************************** * Helper functions ***************************************************************************/ void jent_get_nstime(__u64 *out); -__u64 jent_rol64(__u64 word, unsigned int shift); void *jent_zalloc(unsigned int len); void jent_zfree(void *ptr); int jent_fips_enabled(void); @@ -140,16 +132,16 @@ static __u64 jent_loop_shuffle(struct rand_data *ec, jent_get_nstime(&time); /* - * mix the current state of the random number into the shuffle - * calculation to balance that shuffle a bit more + * Mix the current state of the random number into the shuffle + * calculation to balance that shuffle a bit more. */ if (ec) time ^= ec->data; /* - * we fold the time value as much as possible to ensure that as many - * bits of the time stamp are included as possible + * We fold the time value as much as possible to ensure that as many + * bits of the time stamp are included as possible. */ - for (i = 0; (DATA_SIZE_BITS / bits) > i; i++) { + for (i = 0; ((DATA_SIZE_BITS + bits - 1) / bits) > i; i++) { shuffle ^= time & mask; time = time >> bits; } @@ -169,38 +161,28 @@ static __u64 jent_loop_shuffle(struct rand_data *ec, * CPU Jitter noise source -- this is the noise source based on the CPU * execution time jitter * - * This function folds the time into one bit units by iterating - * through the DATA_SIZE_BITS bit time value as follows: assume our time value - * is 0xabcd - * 1st loop, 1st shift generates 0xd000 - * 1st loop, 2nd shift generates 0x000d - * 2nd loop, 1st shift generates 0xcd00 - * 2nd loop, 2nd shift generates 0x000c - * 3rd loop, 1st shift generates 0xbcd0 - * 3rd loop, 2nd shift generates 0x000b - * 4th loop, 1st shift generates 0xabcd - * 4th loop, 2nd shift generates 0x000a - * Now, the values at the end of the 2nd shifts are XORed together. + * This function injects the individual bits of the time value into the + * entropy pool using an LFSR. * - * The code is deliberately inefficient and shall stay that way. This function - * is the root cause why the code shall be compiled without optimization. This - * function not only acts as folding operation, but this function's execution - * is used to measure the CPU execution time jitter. Any change to the loop in - * this function implies that careful retesting must be done. + * The code is deliberately inefficient with respect to the bit shifting + * and shall stay that way. This function is the root cause why the code + * shall be compiled without optimization. This function not only acts as + * folding operation, but this function's execution is used to measure + * the CPU execution time jitter. Any change to the loop in this function + * implies that careful retesting must be done. * * Input: * @ec entropy collector struct -- may be NULL - * @time time stamp to be folded + * @time time stamp to be injected * @loop_cnt if a value not equal to 0 is set, use the given value as number of * loops to perform the folding * * Output: - * @folded result of folding operation + * updated ec->data * * @return Number of loops the folding operation is performed */ -static __u64 jent_fold_time(struct rand_data *ec, __u64 time, - __u64 *folded, __u64 loop_cnt) +static __u64 jent_lfsr_time(struct rand_data *ec, __u64 time, __u64 loop_cnt) { unsigned int i; __u64 j = 0; @@ -217,15 +199,34 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time, if (loop_cnt) fold_loop_cnt = loop_cnt; for (j = 0; j < fold_loop_cnt; j++) { - new = 0; + new = ec->data; for (i = 1; (DATA_SIZE_BITS) >= i; i++) { __u64 tmp = time << (DATA_SIZE_BITS - i); tmp = tmp >> (DATA_SIZE_BITS - 1); + + /* + * Fibonacci LSFR with polynomial of + * x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is + * primitive according to + * http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf + * (the shift values are the polynomial values minus one + * due to counting bits from 0 to 63). As the current + * position is always the LSB, the polynomial only needs + * to shift data in from the left without wrap. + */ + tmp ^= ((new >> 63) & 1); + tmp ^= ((new >> 60) & 1); + tmp ^= ((new >> 55) & 1); + tmp ^= ((new >> 30) & 1); + tmp ^= ((new >> 27) & 1); + tmp ^= ((new >> 22) & 1); + new <<= 1; new ^= tmp; } } - *folded = new; + ec->data = new; + return fold_loop_cnt; } @@ -258,7 +259,6 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time, */ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) { - unsigned char *tmpval = NULL; unsigned int wrap = 0; __u64 i = 0; #define MAX_ACC_LOOP_BIT 7 @@ -278,7 +278,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) acc_loop_cnt = loop_cnt; for (i = 0; i < (ec->memaccessloops + acc_loop_cnt); i++) { - tmpval = ec->mem + ec->memlocation; + unsigned char *tmpval = ec->mem + ec->memlocation; /* * memory access: just add 1 to one byte, * wrap at 255 -- memory access implies read @@ -316,7 +316,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) * 0 jitter measurement not stuck (good bit) * 1 jitter measurement stuck (reject bit) */ -static void jent_stuck(struct rand_data *ec, __u64 current_delta) +static int jent_stuck(struct rand_data *ec, __u64 current_delta) { __s64 delta2 = ec->last_delta - current_delta; __s64 delta3 = delta2 - ec->last_delta2; @@ -325,14 +325,15 @@ static void jent_stuck(struct rand_data *ec, __u64 current_delta) ec->last_delta2 = delta2; if (!current_delta || !delta2 || !delta3) - ec->stuck = 1; + return 1; + + return 0; } /** * This is the heart of the entropy generation: calculate time deltas and - * use the CPU jitter in the time deltas. The jitter is folded into one - * bit. You can call this function the "random bit generator" as it - * produces one random bit per invocation. + * use the CPU jitter in the time deltas. The jitter is injected into the + * entropy pool. * * WARNING: ensure that ->prev_time is primed before using the output * of this function! This can be done by calling this function @@ -341,12 +342,11 @@ static void jent_stuck(struct rand_data *ec, __u64 current_delta) * Input: * @entropy_collector Reference to entropy collector * - * @return One random bit + * @return result of stuck test */ -static __u64 jent_measure_jitter(struct rand_data *ec) +static int jent_measure_jitter(struct rand_data *ec) { __u64 time = 0; - __u64 data = 0; __u64 current_delta = 0; /* Invoke one noise source before time measurement to add variations */ @@ -360,109 +360,11 @@ static __u64 jent_measure_jitter(struct rand_data *ec) current_delta = time - ec->prev_time; ec->prev_time = time; - /* Now call the next noise sources which also folds the data */ - jent_fold_time(ec, current_delta, &data, 0); + /* Now call the next noise sources which also injects the data */ + jent_lfsr_time(ec, current_delta, 0); - /* - * Check whether we have a stuck measurement. The enforcement - * is performed after the stuck value has been mixed into the - * entropy pool. - */ - jent_stuck(ec, current_delta); - - return data; -} - -/** - * Von Neuman unbias as explained in RFC 4086 section 4.2. As shown in the - * documentation of that RNG, the bits from jent_measure_jitter are considered - * independent which implies that the Von Neuman unbias operation is applicable. - * A proof of the Von-Neumann unbias operation to remove skews is given in the - * document "A proposal for: Functionality classes for random number - * generators", version 2.0 by Werner Schindler, section 5.4.1. - * - * Input: - * @entropy_collector Reference to entropy collector - * - * @return One random bit - */ -static __u64 jent_unbiased_bit(struct rand_data *entropy_collector) -{ - do { - __u64 a = jent_measure_jitter(entropy_collector); - __u64 b = jent_measure_jitter(entropy_collector); - - if (a == b) - continue; - if (1 == a) - return 1; - else - return 0; - } while (1); -} - -/** - * Shuffle the pool a bit by mixing some value with a bijective function (XOR) - * into the pool. - * - * The function generates a mixer value that depends on the bits set and the - * location of the set bits in the random number generated by the entropy - * source. Therefore, based on the generated random number, this mixer value - * can have 2**64 different values. That mixer value is initialized with the - * first two SHA-1 constants. After obtaining the mixer value, it is XORed into - * the random number. - * - * The mixer value is not assumed to contain any entropy. But due to the XOR - * operation, it can also not destroy any entropy present in the entropy pool. - * - * Input: - * @entropy_collector Reference to entropy collector - */ -static void jent_stir_pool(struct rand_data *entropy_collector) -{ - /* - * to shut up GCC on 32 bit, we have to initialize the 64 variable - * with two 32 bit variables - */ - union c { - __u64 u64; - __u32 u32[2]; - }; - /* - * This constant is derived from the first two 32 bit initialization - * vectors of SHA-1 as defined in FIPS 180-4 section 5.3.1 - */ - union c constant; - /* - * The start value of the mixer variable is derived from the third - * and fourth 32 bit initialization vector of SHA-1 as defined in - * FIPS 180-4 section 5.3.1 - */ - union c mixer; - unsigned int i = 0; - - /* - * Store the SHA-1 constants in reverse order to make up the 64 bit - * value -- this applies to a little endian system, on a big endian - * system, it reverses as expected. But this really does not matter - * as we do not rely on the specific numbers. We just pick the SHA-1 - * constants as they have a good mix of bit set and unset. - */ - constant.u32[1] = 0x67452301; - constant.u32[0] = 0xefcdab89; - mixer.u32[1] = 0x98badcfe; - mixer.u32[0] = 0x10325476; - - for (i = 0; i < DATA_SIZE_BITS; i++) { - /* - * get the i-th bit of the input random number and only XOR - * the constant into the mixer value when that bit is set - */ - if ((entropy_collector->data >> i) & 1) - mixer.u64 ^= constant.u64; - mixer.u64 = jent_rol64(mixer.u64, 1); - } - entropy_collector->data ^= mixer.u64; + /* Check whether we have a stuck measurement. */ + return jent_stuck(ec, current_delta); } /** @@ -480,48 +382,9 @@ static void jent_gen_entropy(struct rand_data *ec) jent_measure_jitter(ec); while (1) { - __u64 data = 0; - - if (ec->disable_unbias == 1) - data = jent_measure_jitter(ec); - else - data = jent_unbiased_bit(ec); - - /* enforcement of the jent_stuck test */ - if (ec->stuck) { - /* - * We only mix in the bit considered not appropriate - * without the LSFR. The reason is that if we apply - * the LSFR and we do not rotate, the 2nd bit with LSFR - * will cancel out the first LSFR application on the - * bad bit. - * - * And we do not rotate as we apply the next bit to the - * current bit location again. - */ - ec->data ^= data; - ec->stuck = 0; + /* If a stuck measurement is received, repeat measurement */ + if (jent_measure_jitter(ec)) continue; - } - - /* - * Fibonacci LSFR with polynom of - * x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is - * primitive according to - * http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf - * (the shift values are the polynom values minus one - * due to counting bits from 0 to 63). As the current - * position is always the LSB, the polynom only needs - * to shift data in from the left without wrap. - */ - ec->data ^= data; - ec->data ^= ((ec->data >> 63) & 1); - ec->data ^= ((ec->data >> 60) & 1); - ec->data ^= ((ec->data >> 55) & 1); - ec->data ^= ((ec->data >> 30) & 1); - ec->data ^= ((ec->data >> 27) & 1); - ec->data ^= ((ec->data >> 22) & 1); - ec->data = jent_rol64(ec->data, 1); /* * We multiply the loop value with ->osr to obtain the @@ -530,8 +393,6 @@ static void jent_gen_entropy(struct rand_data *ec) if (++k >= (DATA_SIZE_BITS * ec->osr)) break; } - if (ec->stir) - jent_stir_pool(ec); } /** @@ -639,12 +500,6 @@ struct rand_data *jent_entropy_collector_alloc(unsigned int osr, osr = 1; /* minimum sampling rate is 1 */ entropy_collector->osr = osr; - entropy_collector->stir = 1; - if (flags & JENT_DISABLE_STIR) - entropy_collector->stir = 0; - if (flags & JENT_DISABLE_UNBIAS) - entropy_collector->disable_unbias = 1; - /* fill the data pad with non-zero values */ jent_gen_entropy(entropy_collector); @@ -656,7 +511,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector) jent_zfree(entropy_collector->mem); entropy_collector->mem = NULL; jent_zfree(entropy_collector); - entropy_collector = NULL; } int jent_entropy_init(void) @@ -665,8 +519,9 @@ int jent_entropy_init(void) __u64 delta_sum = 0; __u64 old_delta = 0; int time_backwards = 0; - int count_var = 0; int count_mod = 0; + int count_stuck = 0; + struct rand_data ec = { 0 }; /* We could perform statistical tests here, but the problem is * that we only have a few loop counts to do testing. These @@ -695,12 +550,14 @@ int jent_entropy_init(void) for (i = 0; (TESTLOOPCOUNT + CLEARCACHE) > i; i++) { __u64 time = 0; __u64 time2 = 0; - __u64 folded = 0; __u64 delta = 0; unsigned int lowdelta = 0; + int stuck; + /* Invoke core entropy collection logic */ jent_get_nstime(&time); - jent_fold_time(NULL, time, &folded, 1< i) continue; + if (stuck) + count_stuck++; + /* test whether we have an increasing timer */ if (!(time2 > time)) time_backwards++; - /* - * Avoid modulo of 64 bit integer to allow code to compile - * on 32 bit architectures. - */ + /* use 32 bit value to ensure compilation on 32 bit arches */ lowdelta = time2 - time; if (!(lowdelta % 100)) count_mod++; @@ -743,14 +602,10 @@ int jent_entropy_init(void) * only after the first loop is executed as we need to prime * the old_data value */ - if (i) { - if (delta != old_delta) - count_var++; - if (delta > old_delta) - delta_sum += (delta - old_delta); - else - delta_sum += (old_delta - delta); - } + if (delta > old_delta) + delta_sum += (delta - old_delta); + else + delta_sum += (old_delta - delta); old_delta = delta; } @@ -763,25 +618,29 @@ int jent_entropy_init(void) */ if (3 < time_backwards) return JENT_ENOMONOTONIC; - /* Error if the time variances are always identical */ - if (!delta_sum) - return JENT_EVARVAR; /* * Variations of deltas of time must on average be larger * than 1 to ensure the entropy estimation * implied with 1 is preserved */ - if (delta_sum <= 1) - return JENT_EMINVARVAR; + if ((delta_sum) <= 1) + return JENT_EVARVAR; /* * Ensure that we have variations in the time stamp below 10 for at - * least 10% of all checks -- on some platforms, the counter - * increments in multiples of 100, but not always + * least 10% of all checks -- on some platforms, the counter increments + * in multiples of 100, but not always */ if ((TESTLOOPCOUNT/10 * 9) < count_mod) return JENT_ECOARSETIME; + /* + * If we have more than 90% stuck results, then this Jitter RNG is + * likely to not work well. + */ + if ((TESTLOOPCOUNT/10 * 9) < count_stuck) + return JENT_ESTUCK; + return 0; } From 67882e76492483bafa9b1b1648bb031e9abe5185 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Thu, 30 May 2019 09:52:57 +0300 Subject: [PATCH 080/143] crypto: xxhash - Implement xxhash support xxhash is currently implemented as a self-contained module in /lib. This patch enables that module to be used as part of the generic kernel crypto framework. It adds a simple wrapper to the 64bit version. I've also added test vectors (with help from Nick Terrell). The upstream xxhash code is tested by running hashing operation on random 222 byte data with seed values of 0 and a prime number. The upstream test suite can be found at https://github.com/Cyan4973/xxHash/blob/cf46e0c/xxhsum.c#L664 Essentially hashing is run on data of length 0,1,14,222 with the aforementioned seed values 0 and prime 2654435761. The particular random 222 byte string was provided to me by Nick Terrell by reading /dev/random and the checksums were calculated by the upstream xxsum utility with the following bash script: dd if=/dev/random of=TEST_VECTOR bs=1 count=222 for a in 0 1; do for l in 0 1 14 222; do for s in 0 2654435761; do echo algo $a length $l seed $s; head -c $l TEST_VECTOR | ~/projects/kernel/xxHash/xxhsum -H$a -s$s done done done This produces output as follows: algo 0 length 0 seed 0 02cc5d05 stdin algo 0 length 0 seed 2654435761 02cc5d05 stdin algo 0 length 1 seed 0 25201171 stdin algo 0 length 1 seed 2654435761 25201171 stdin algo 0 length 14 seed 0 c1d95975 stdin algo 0 length 14 seed 2654435761 c1d95975 stdin algo 0 length 222 seed 0 b38662a6 stdin algo 0 length 222 seed 2654435761 b38662a6 stdin algo 1 length 0 seed 0 ef46db3751d8e999 stdin algo 1 length 0 seed 2654435761 ac75fda2929b17ef stdin algo 1 length 1 seed 0 27c3f04c2881203a stdin algo 1 length 1 seed 2654435761 4a15ed26415dfe4d stdin algo 1 length 14 seed 0 3d33dc700231dfad stdin algo 1 length 14 seed 2654435761 ea5f7ddef9a64f80 stdin algo 1 length 222 seed 0 5f3d3c08ec2bef34 stdin algo 1 length 222 seed 2654435761 6a9df59664c7ed62 stdin algo 1 is xx64 variant, algo 0 is the 32 bit variant which is currently not hooked up. Signed-off-by: Nikolay Borisov Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 8 +++ crypto/Makefile | 1 + crypto/testmgr.c | 7 +++ crypto/testmgr.h | 106 +++++++++++++++++++++++++++++++++++++++ crypto/xxhash_generic.c | 108 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 230 insertions(+) create mode 100644 crypto/xxhash_generic.c diff --git a/crypto/Kconfig b/crypto/Kconfig index 9cdd92520f00..8c588ed36253 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -683,6 +683,14 @@ config CRYPTO_CRC32_MIPS instructions, when available. +config CRYPTO_XXHASH + tristate "xxHash hash algorithm" + select CRYPTO_HASH + select XXHASH + help + xxHash non-cryptographic hash algorithm. Extremely fast, working at + speeds close to RAM limits. + config CRYPTO_CRCT10DIF tristate "CRCT10DIF algorithm" select CRYPTO_HASH diff --git a/crypto/Makefile b/crypto/Makefile index 2adf06b178e9..9479e1a45d8c 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -129,6 +129,7 @@ obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o obj-$(CONFIG_CRYPTO_LZ4) += lz4.o obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o +obj-$(CONFIG_CRYPTO_XXHASH) += xxhash_generic.o obj-$(CONFIG_CRYPTO_842) += 842.o obj-$(CONFIG_CRYPTO_RNG2) += rng.o obj-$(CONFIG_CRYPTO_ANSI_CPRNG) += ansi_cprng.o diff --git a/crypto/testmgr.c b/crypto/testmgr.c index a347be142323..2ba0c487ea28 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -5062,6 +5062,13 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "xts512(paes)", .test = alg_test_null, .fips_allowed = 1, + }, { + .alg = "xxhash64", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = __VECS(xxhash64_tv_template) + } }, { .alg = "zlib-deflate", .test = alg_test_comp, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 2655f41d4d23..6b459a6a796b 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -33387,6 +33387,112 @@ static const struct hash_testvec crc32c_tv_template[] = { } }; +static const struct hash_testvec xxhash64_tv_template[] = { + { + .psize = 0, + .digest = "\x99\xe9\xd8\x51\x37\xdb\x46\xef", + }, + { + .plaintext = "\x40", + .psize = 1, + .digest = "\x20\x5c\x91\xaa\x88\xeb\x59\xd0", + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b", + .psize = 14, + .digest = "\xa8\xe8\x2b\xa9\x92\xa1\x37\x4a", + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b\x42\xe0" + "\xd4\x38\xa5\x2a\x26\xa5\x19\x4b" + "\x57\x65\x7f\xad\xc3\x7d\xca\x40" + "\x31\x65\x05\xbb\x31\xae\x51\x11" + "\xa8\xc0\xb3\x28\x42\xeb\x3c\x46" + "\xc8\xed\xed\x0f\x8d\x0b\xfa\x6e" + "\xbc\xe3\x88\x53\xca\x8f\xc8\xd9" + "\x41\x26\x7a\x3d\x21\xdb\x1a\x3c" + "\x01\x1d\xc9\xe9\xb7\x3a\x78\x67" + "\x57\x20\x94\xf1\x1e\xfd\xce\x39" + "\x99\x57\x69\x39\xa5\xd0\x8d\xd9" + "\x43\xfe\x1d\x66\x04\x3c\x27\x6a" + "\xe1\x0d\xe7\xc9\xfa\xc9\x07\x56" + "\xa5\xb3\xec\xd9\x1f\x42\x65\x66" + "\xaa\xbf\x87\x9b\xc5\x41\x9c\x27" + "\x3f\x2f\xa9\x55\x93\x01\x27\x33" + "\x43\x99\x4d\x81\x85\xae\x82\x00" + "\x6c\xd0\xd1\xa3\x57\x18\x06\xcc" + "\xec\x72\xf7\x8e\x87\x2d\x1f\x5e" + "\xd7\x5b\x1f\x36\x4c\xfa\xfd\x18" + "\x89\x76\xd3\x5e\xb5\x5a\xc0\x01" + "\xd2\xa1\x9a\x50\xe6\x08\xb4\x76" + "\x56\x4f\x0e\xbc\x54\xfc\x67\xe6" + "\xb9\xc0\x28\x4b\xb5\xc3\xff\x79" + "\x52\xea\xa1\x90\xc3\xaf\x08\x70" + "\x12\x02\x0c\xdb\x94\x00\x38\x95" + "\xed\xfd\x08\xf7\xe8\x04", + .psize = 222, + .digest = "\x41\xfc\xd4\x29\xfe\xe7\x85\x17", + }, + { + .psize = 0, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\xef\x17\x9b\x92\xa2\xfd\x75\xac", + }, + + { + .plaintext = "\x40", + .psize = 1, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\xd1\x70\x4f\x14\x02\xc4\x9e\x71", + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b", + .psize = 14, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\xa4\xcd\xfe\x8e\x37\xe2\x1c\x64" + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b\x42\xe0" + "\xd4\x38\xa5\x2a\x26\xa5\x19\x4b" + "\x57\x65\x7f\xad\xc3\x7d\xca\x40" + "\x31\x65\x05\xbb\x31\xae\x51\x11" + "\xa8\xc0\xb3\x28\x42\xeb\x3c\x46" + "\xc8\xed\xed\x0f\x8d\x0b\xfa\x6e" + "\xbc\xe3\x88\x53\xca\x8f\xc8\xd9" + "\x41\x26\x7a\x3d\x21\xdb\x1a\x3c" + "\x01\x1d\xc9\xe9\xb7\x3a\x78\x67" + "\x57\x20\x94\xf1\x1e\xfd\xce\x39" + "\x99\x57\x69\x39\xa5\xd0\x8d\xd9" + "\x43\xfe\x1d\x66\x04\x3c\x27\x6a" + "\xe1\x0d\xe7\xc9\xfa\xc9\x07\x56" + "\xa5\xb3\xec\xd9\x1f\x42\x65\x66" + "\xaa\xbf\x87\x9b\xc5\x41\x9c\x27" + "\x3f\x2f\xa9\x55\x93\x01\x27\x33" + "\x43\x99\x4d\x81\x85\xae\x82\x00" + "\x6c\xd0\xd1\xa3\x57\x18\x06\xcc" + "\xec\x72\xf7\x8e\x87\x2d\x1f\x5e" + "\xd7\x5b\x1f\x36\x4c\xfa\xfd\x18" + "\x89\x76\xd3\x5e\xb5\x5a\xc0\x01" + "\xd2\xa1\x9a\x50\xe6\x08\xb4\x76" + "\x56\x4f\x0e\xbc\x54\xfc\x67\xe6" + "\xb9\xc0\x28\x4b\xb5\xc3\xff\x79" + "\x52\xea\xa1\x90\xc3\xaf\x08\x70" + "\x12\x02\x0c\xdb\x94\x00\x38\x95" + "\xed\xfd\x08\xf7\xe8\x04", + .psize = 222, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\x58\xbc\x55\xf2\x42\x81\x5c\xf0" + }, +}; + static const struct comp_testvec lz4_comp_tv_template[] = { { .inlen = 255, diff --git a/crypto/xxhash_generic.c b/crypto/xxhash_generic.c new file mode 100644 index 000000000000..4aad2c0f40a9 --- /dev/null +++ b/crypto/xxhash_generic.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include + +#define XXHASH64_BLOCK_SIZE 32 +#define XXHASH64_DIGEST_SIZE 8 + +struct xxhash64_tfm_ctx { + u64 seed; +}; + +struct xxhash64_desc_ctx { + struct xxh64_state xxhstate; +}; + +static int xxhash64_setkey(struct crypto_shash *tfm, const u8 *key, + unsigned int keylen) +{ + struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(tfm); + + if (keylen != sizeof(tctx->seed)) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + tctx->seed = get_unaligned_le64(key); + return 0; +} + +static int xxhash64_init(struct shash_desc *desc) +{ + struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc); + + xxh64_reset(&dctx->xxhstate, tctx->seed); + + return 0; +} + +static int xxhash64_update(struct shash_desc *desc, const u8 *data, + unsigned int length) +{ + struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc); + + xxh64_update(&dctx->xxhstate, data, length); + + return 0; +} + +static int xxhash64_final(struct shash_desc *desc, u8 *out) +{ + struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc); + + put_unaligned_le64(xxh64_digest(&dctx->xxhstate), out); + + return 0; +} + +static int xxhash64_digest(struct shash_desc *desc, const u8 *data, + unsigned int length, u8 *out) +{ + struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + + put_unaligned_le64(xxh64(data, length, tctx->seed), out); + + return 0; +} + +static struct shash_alg alg = { + .digestsize = XXHASH64_DIGEST_SIZE, + .setkey = xxhash64_setkey, + .init = xxhash64_init, + .update = xxhash64_update, + .final = xxhash64_final, + .digest = xxhash64_digest, + .descsize = sizeof(struct xxhash64_desc_ctx), + .base = { + .cra_name = "xxhash64", + .cra_driver_name = "xxhash64-generic", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, + .cra_blocksize = XXHASH64_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct xxhash64_tfm_ctx), + .cra_module = THIS_MODULE, + } +}; + +static int __init xxhash_mod_init(void) +{ + return crypto_register_shash(&alg); +} + +static void __exit xxhash_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +subsys_initcall(xxhash_mod_init); +module_exit(xxhash_mod_fini); + +MODULE_AUTHOR("Nikolay Borisov "); +MODULE_DESCRIPTION("xxhash calculations wrapper for lib/xxhash.c"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS_CRYPTO("xxhash64"); +MODULE_ALIAS_CRYPTO("xxhash64-generic"); From 5c6bc4dfa515738149998bb0db2481a4fdead979 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 30 May 2019 10:50:39 -0700 Subject: [PATCH 081/143] crypto: ghash - fix unaligned memory access in ghash_setkey() Changing ghash_mod_init() to be subsys_initcall made it start running before the alignment fault handler has been installed on ARM. In kernel builds where the keys in the ghash test vectors happened to be misaligned in the kernel image, this exposed the longstanding bug that ghash_setkey() is incorrectly casting the key buffer (which can have any alignment) to be128 for passing to gf128mul_init_4k_lle(). Fix this by memcpy()ing the key to a temporary buffer. Don't fix it by setting an alignmask on the algorithm instead because that would unnecessarily force alignment of the data too. Fixes: 2cdc6899a88e ("crypto: ghash - Add GHASH digest algorithm for GCM") Reported-by: Peter Robinson Cc: stable@vger.kernel.org Signed-off-by: Eric Biggers Tested-by: Peter Robinson Signed-off-by: Herbert Xu --- crypto/ghash-generic.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c index e6307935413c..c8a347798eae 100644 --- a/crypto/ghash-generic.c +++ b/crypto/ghash-generic.c @@ -34,6 +34,7 @@ static int ghash_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { struct ghash_ctx *ctx = crypto_shash_ctx(tfm); + be128 k; if (keylen != GHASH_BLOCK_SIZE) { crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); @@ -42,7 +43,12 @@ static int ghash_setkey(struct crypto_shash *tfm, if (ctx->gf128) gf128mul_free_4k(ctx->gf128); - ctx->gf128 = gf128mul_init_4k_lle((be128 *)key); + + BUILD_BUG_ON(sizeof(k) != GHASH_BLOCK_SIZE); + memcpy(&k, key, GHASH_BLOCK_SIZE); /* avoid violating alignment rules */ + ctx->gf128 = gf128mul_init_4k_lle(&k); + memzero_explicit(&k, GHASH_BLOCK_SIZE); + if (!ctx->gf128) return -ENOMEM; From 20a0f9761343fba9b25ea46bd3a3e5e533d974f8 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 30 May 2019 10:53:08 -0700 Subject: [PATCH 082/143] crypto: lrw - use correct alignmask Commit c778f96bf347 ("crypto: lrw - Optimize tweak computation") incorrectly reduced the alignmask of LRW instances from '__alignof__(u64) - 1' to '__alignof__(__be32) - 1'. However, xor_tweak() and setkey() assume that the data and key, respectively, are aligned to 'be128', which has u64 alignment. Fix the alignmask to be at least '__alignof__(be128) - 1'. Fixes: c778f96bf347 ("crypto: lrw - Optimize tweak computation") Cc: # v4.20+ Cc: Ondrej Mosnacek Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/lrw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/lrw.c b/crypto/lrw.c index fa302f3f161e..b43ea285b8c7 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -388,7 +388,7 @@ static int create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.base.cra_priority = alg->base.cra_priority; inst->alg.base.cra_blocksize = LRW_BLOCK_SIZE; inst->alg.base.cra_alignmask = alg->base.cra_alignmask | - (__alignof__(__be32) - 1); + (__alignof__(be128) - 1); inst->alg.ivsize = LRW_BLOCK_SIZE; inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) + From 49d22167f826134e579179aa2217c74647e62015 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 31 May 2019 14:39:51 +0800 Subject: [PATCH 083/143] crypto: atmel - Fix sparse endianness warnings The param2 member in atmel_i2c_cmd is supposed to be little-endian but was marked as u16. This patch changes it to a __le16 which reveals a missing endian swap in atmel_i2c_init_read_cmd. Another missing little-endian marking is also added in atmel_i2c_checksum. Fixes: 11105693fa05 ("crypto: atmel-ecc - introduce Microchip...") Reported-by: kbuild test robot Signed-off-by: Herbert Xu --- drivers/crypto/atmel-i2c.c | 4 ++-- drivers/crypto/atmel-i2c.h | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c index be49ab7f4338..dc876fab2882 100644 --- a/drivers/crypto/atmel-i2c.c +++ b/drivers/crypto/atmel-i2c.c @@ -34,7 +34,7 @@ static void atmel_i2c_checksum(struct atmel_i2c_cmd *cmd) { u8 *data = &cmd->count; size_t len = cmd->count - CRC_SIZE; - u16 *__crc16 = (u16 *)(data + len); + __le16 *__crc16 = (__le16 *)(data + len); *__crc16 = cpu_to_le16(bitrev16(crc16(0, data, len))); } @@ -48,7 +48,7 @@ void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd) * (UserExtra, Selector, LockValue, LockConfig). */ cmd->param1 = CONFIG_ZONE; - cmd->param2 = DEVICE_LOCK_ADDR; + cmd->param2 = cpu_to_le16(DEVICE_LOCK_ADDR); cmd->count = READ_COUNT; atmel_i2c_checksum(cmd); diff --git a/drivers/crypto/atmel-i2c.h b/drivers/crypto/atmel-i2c.h index c6bd43b78f33..21860b99c3e3 100644 --- a/drivers/crypto/atmel-i2c.h +++ b/drivers/crypto/atmel-i2c.h @@ -8,6 +8,7 @@ #define __ATMEL_I2C_H__ #include +#include #define ATMEL_ECC_PRIORITY 300 @@ -50,7 +51,7 @@ struct atmel_i2c_cmd { u8 count; u8 opcode; u8 param1; - u16 param2; + __le16 param2; u8 data[MAX_RSP_SIZE]; u8 msecs; u16 rxsize; From ed527b13d800dd515a9e6c582f0a73eca65b2e1b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 31 May 2019 10:13:06 +0200 Subject: [PATCH 084/143] crypto: caam - limit output IV to CBC to work around CTR mode DMA issue The CAAM driver currently violates an undocumented and slightly controversial requirement imposed by the crypto stack that a buffer referred to by the request structure via its virtual address may not be modified while any scatterlists passed via the same request structure are mapped for inbound DMA. This may result in errors like alg: aead: decryption failed on test 1 for gcm_base(ctr-aes-caam,ghash-generic): ret=74 alg: aead: Failed to load transform for gcm(aes): -2 on non-cache coherent systems, due to the fact that the GCM driver passes an IV buffer by virtual address which shares a cacheline with the auth_tag buffer passed via a scatterlist, resulting in corruption of the auth_tag when the IV is updated while the DMA mapping is live. Since the IV that is returned to the caller is only valid for CBC mode, and given that the in-kernel users of CBC (such as CTS) don't trigger the same issue as the GCM driver, let's just disable the output IV generation for all modes except CBC for the time being. Fixes: 854b06f76879 ("crypto: caam - properly set IV after {en,de}crypt") Cc: Horia Geanta Cc: Iuliana Prodan Reported-by: Sascha Hauer Cc: Signed-off-by: Ard Biesheuvel Reviewed-by: Horia Geanta Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 1efa6f5b62cf..4b03c967009b 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -977,6 +977,7 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, struct skcipher_request *req = context; struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); int ivsize = crypto_skcipher_ivsize(skcipher); dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); @@ -990,9 +991,9 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. This is used e.g. by the CTS mode. + * ciphertext block when running in CBC mode. */ - if (ivsize) + if ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CBC) scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize, ivsize, 0); @@ -1836,9 +1837,9 @@ static int skcipher_decrypt(struct skcipher_request *req) /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. + * ciphertext block when running in CBC mode. */ - if (ivsize) + if ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CBC) scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize, ivsize, 0); From 385cfc84a5a80e4b1b692a3e67c018d51631abde Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Fri, 31 May 2019 14:06:34 +0300 Subject: [PATCH 085/143] crypto: caam - disable some clock checks for iMX7ULP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disabled the check and set of 'mem' and 'emi_slow' clocks, since these are not available for iMX7ULP. Signed-off-by: Iuliana Prodan Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/ctrl.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index bbde6efce8af..5d197e879899 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -540,7 +540,8 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->caam_ipg = clk; if (!of_machine_is_compatible("fsl,imx7d") && - !of_machine_is_compatible("fsl,imx7s")) { + !of_machine_is_compatible("fsl,imx7s") && + !of_machine_is_compatible("fsl,imx7ulp")) { clk = caam_drv_identify_clk(&pdev->dev, "mem"); if (IS_ERR(clk)) { ret = PTR_ERR(clk); @@ -562,7 +563,8 @@ static int caam_probe(struct platform_device *pdev) if (!of_machine_is_compatible("fsl,imx6ul") && !of_machine_is_compatible("fsl,imx7d") && - !of_machine_is_compatible("fsl,imx7s")) { + !of_machine_is_compatible("fsl,imx7s") && + !of_machine_is_compatible("fsl,imx7ulp")) { clk = caam_drv_identify_clk(&pdev->dev, "emi_slow"); if (IS_ERR(clk)) { ret = PTR_ERR(clk); From 4bb02dbd31d7430200db0e8dfb96f9d716351627 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Fri, 31 May 2019 20:17:49 +0800 Subject: [PATCH 086/143] crypto: atmel-i2c - Fix build error while CRC16 set to m If CRYPTO_DEV_ATMEL_ECC is set m, which select CRC16 to m, while CRYPTO_DEV_ATMEL_SHA204A is set to y, building fails. drivers/crypto/atmel-i2c.o: In function 'atmel_i2c_checksum': atmel-i2c.c:(.text+0x16): undefined reference to 'crc16' Add CRC16 dependency to CRYPTO_DEV_ATMEL_SHA204A Reported-by: Hulk Robot Fixes: da001fb651b0 ("crypto: atmel-i2c - add support for SHA204A random number generator") Signed-off-by: YueHaibing Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index bc7abad9fdff..67af688d7d84 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -541,6 +541,7 @@ config CRYPTO_DEV_ATMEL_SHA204A depends on I2C select CRYPTO_DEV_ATMEL_I2C select HW_RANDOM + select CRC16 help Microhip / Atmel SHA accelerator and RNG. Select this if you want to use the Microchip / Atmel SHA204A From 7545b6c2087f4ef0287c8c9b7eba6a728c67ff8e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 31 May 2019 11:12:30 -0700 Subject: [PATCH 087/143] crypto: chacha20poly1305 - fix atomic sleep when using async algorithm Clear the CRYPTO_TFM_REQ_MAY_SLEEP flag when the chacha20poly1305 operation is being continued from an async completion callback, since sleeping may not be allowed in that context. This is basically the same bug that was recently fixed in the xts and lrw templates. But, it's always been broken in chacha20poly1305 too. This was found using syzkaller in combination with the updated crypto self-tests which actually test the MAY_SLEEP flag now. Reproducer: python -c 'import socket; socket.socket(socket.AF_ALG, 5, 0).bind( ("aead", "rfc7539(cryptd(chacha20-generic),poly1305-generic)"))' Kernel output: BUG: sleeping function called from invalid context at include/crypto/algapi.h:426 in_atomic(): 1, irqs_disabled(): 0, pid: 1001, name: kworker/2:2 [...] CPU: 2 PID: 1001 Comm: kworker/2:2 Not tainted 5.2.0-rc2 #5 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-20181126_142135-anatol 04/01/2014 Workqueue: crypto cryptd_queue_worker Call Trace: __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x4d/0x6a lib/dump_stack.c:113 ___might_sleep kernel/sched/core.c:6138 [inline] ___might_sleep.cold.19+0x8e/0x9f kernel/sched/core.c:6095 crypto_yield include/crypto/algapi.h:426 [inline] crypto_hash_walk_done+0xd6/0x100 crypto/ahash.c:113 shash_ahash_update+0x41/0x60 crypto/shash.c:251 shash_async_update+0xd/0x10 crypto/shash.c:260 crypto_ahash_update include/crypto/hash.h:539 [inline] poly_setkey+0xf6/0x130 crypto/chacha20poly1305.c:337 poly_init+0x51/0x60 crypto/chacha20poly1305.c:364 async_done_continue crypto/chacha20poly1305.c:78 [inline] poly_genkey_done+0x15/0x30 crypto/chacha20poly1305.c:369 cryptd_skcipher_complete+0x29/0x70 crypto/cryptd.c:279 cryptd_skcipher_decrypt+0xcd/0x110 crypto/cryptd.c:339 cryptd_queue_worker+0x70/0xa0 crypto/cryptd.c:184 process_one_work+0x1ed/0x420 kernel/workqueue.c:2269 worker_thread+0x3e/0x3a0 kernel/workqueue.c:2415 kthread+0x11f/0x140 kernel/kthread.c:255 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:352 Fixes: 71ebc4d1b27d ("crypto: chacha20poly1305 - Add a ChaCha20-Poly1305 AEAD construction, RFC7539") Cc: # v4.2+ Cc: Martin Willi Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/chacha20poly1305.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index e38a2d61819a..acbbf010222e 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -65,6 +65,8 @@ struct chachapoly_req_ctx { unsigned int cryptlen; /* Actual AD, excluding IV */ unsigned int assoclen; + /* request flags, with MAY_SLEEP cleared if needed */ + u32 flags; union { struct poly_req poly; struct chacha_req chacha; @@ -74,8 +76,12 @@ struct chachapoly_req_ctx { static inline void async_done_continue(struct aead_request *req, int err, int (*cont)(struct aead_request *)) { - if (!err) + if (!err) { + struct chachapoly_req_ctx *rctx = aead_request_ctx(req); + + rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; err = cont(req); + } if (err != -EINPROGRESS && err != -EBUSY) aead_request_complete(req, err); @@ -142,7 +148,7 @@ static int chacha_decrypt(struct aead_request *req) dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); } - skcipher_request_set_callback(&creq->req, aead_request_flags(req), + skcipher_request_set_callback(&creq->req, rctx->flags, chacha_decrypt_done, req); skcipher_request_set_tfm(&creq->req, ctx->chacha); skcipher_request_set_crypt(&creq->req, src, dst, @@ -186,7 +192,7 @@ static int poly_tail(struct aead_request *req) memcpy(&preq->tail.cryptlen, &len, sizeof(len)); sg_set_buf(preq->src, &preq->tail, sizeof(preq->tail)); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_tail_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, @@ -217,7 +223,7 @@ static int poly_cipherpad(struct aead_request *req) sg_init_table(preq->src, 1); sg_set_buf(preq->src, &preq->pad, padlen); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_cipherpad_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen); @@ -248,7 +254,7 @@ static int poly_cipher(struct aead_request *req) sg_init_table(rctx->src, 2); crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_cipher_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen); @@ -278,7 +284,7 @@ static int poly_adpad(struct aead_request *req) sg_init_table(preq->src, 1); sg_set_buf(preq->src, preq->pad, padlen); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_adpad_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen); @@ -302,7 +308,7 @@ static int poly_ad(struct aead_request *req) struct poly_req *preq = &rctx->u.poly; int err; - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_ad_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen); @@ -329,7 +335,7 @@ static int poly_setkey(struct aead_request *req) sg_init_table(preq->src, 1); sg_set_buf(preq->src, rctx->key, sizeof(rctx->key)); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_setkey_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key)); @@ -353,7 +359,7 @@ static int poly_init(struct aead_request *req) struct poly_req *preq = &rctx->u.poly; int err; - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_init_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); @@ -391,7 +397,7 @@ static int poly_genkey(struct aead_request *req) chacha_iv(creq->iv, req, 0); - skcipher_request_set_callback(&creq->req, aead_request_flags(req), + skcipher_request_set_callback(&creq->req, rctx->flags, poly_genkey_done, req); skcipher_request_set_tfm(&creq->req, ctx->chacha); skcipher_request_set_crypt(&creq->req, creq->src, creq->src, @@ -431,7 +437,7 @@ static int chacha_encrypt(struct aead_request *req) dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); } - skcipher_request_set_callback(&creq->req, aead_request_flags(req), + skcipher_request_set_callback(&creq->req, rctx->flags, chacha_encrypt_done, req); skcipher_request_set_tfm(&creq->req, ctx->chacha); skcipher_request_set_crypt(&creq->req, src, dst, @@ -449,6 +455,7 @@ static int chachapoly_encrypt(struct aead_request *req) struct chachapoly_req_ctx *rctx = aead_request_ctx(req); rctx->cryptlen = req->cryptlen; + rctx->flags = aead_request_flags(req); /* encrypt call chain: * - chacha_encrypt/done() @@ -470,6 +477,7 @@ static int chachapoly_decrypt(struct aead_request *req) struct chachapoly_req_ctx *rctx = aead_request_ctx(req); rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE; + rctx->flags = aead_request_flags(req); /* decrypt call chain: * - poly_genkey/done() From d6ebf5286f8f94a254a8c90d4b9f2a8b076a8634 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 2 Jun 2019 22:40:57 -0700 Subject: [PATCH 088/143] crypto: make all generic algorithms set cra_driver_name Most generic crypto algorithms declare a driver name ending in "-generic". The rest don't declare a driver name and instead rely on the crypto API automagically appending "-generic" upon registration. Having multiple conventions is unnecessarily confusing and makes it harder to grep for all generic algorithms in the kernel source tree. But also, allowing NULL driver names is problematic because sometimes people fail to set it, e.g. the case fixed by commit 417980364300 ("crypto: cavium/zip - fix collision with generic cra_driver_name"). Of course, people can also incorrectly name their drivers "-generic". But that's much easier to notice / grep for. Therefore, let's make cra_driver_name mandatory. In preparation for this, this patch makes all generic algorithms set cra_driver_name. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/anubis.c | 1 + crypto/arc4.c | 2 ++ crypto/crypto_null.c | 3 +++ crypto/deflate.c | 1 + crypto/fcrypt.c | 1 + crypto/khazad.c | 1 + crypto/lz4.c | 1 + crypto/lz4hc.c | 1 + crypto/lzo-rle.c | 1 + crypto/lzo.c | 1 + crypto/md4.c | 7 ++++--- crypto/md5.c | 7 ++++--- crypto/michael_mic.c | 1 + crypto/rmd128.c | 1 + crypto/rmd160.c | 1 + crypto/rmd256.c | 1 + crypto/rmd320.c | 1 + crypto/serpent_generic.c | 1 + crypto/tea.c | 3 +++ crypto/tgr192.c | 21 ++++++++++++--------- crypto/wp512.c | 21 ++++++++++++--------- crypto/zstd.c | 1 + 22 files changed, 55 insertions(+), 24 deletions(-) diff --git a/crypto/anubis.c b/crypto/anubis.c index 673927de0eb9..f9ce78fde6ee 100644 --- a/crypto/anubis.c +++ b/crypto/anubis.c @@ -673,6 +673,7 @@ static void anubis_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static struct crypto_alg anubis_alg = { .cra_name = "anubis", + .cra_driver_name = "anubis-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = ANUBIS_BLOCK_SIZE, .cra_ctxsize = sizeof (struct anubis_ctx), diff --git a/crypto/arc4.c b/crypto/arc4.c index 2233d36456e2..b78dcb390a7e 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -115,6 +115,7 @@ static int ecb_arc4_crypt(struct skcipher_request *req) static struct crypto_alg arc4_cipher = { .cra_name = "arc4", + .cra_driver_name = "arc4-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = ARC4_BLOCK_SIZE, .cra_ctxsize = sizeof(struct arc4_ctx), @@ -132,6 +133,7 @@ static struct crypto_alg arc4_cipher = { static struct skcipher_alg arc4_skcipher = { .base.cra_name = "ecb(arc4)", + .base.cra_driver_name = "ecb(arc4)-generic", .base.cra_priority = 100, .base.cra_blocksize = ARC4_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct arc4_ctx), diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 9320d4eaa4a8..6aa9a4c29edf 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -105,6 +105,7 @@ static struct shash_alg digest_null = { .final = null_final, .base = { .cra_name = "digest_null", + .cra_driver_name = "digest_null-generic", .cra_blocksize = NULL_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -127,6 +128,7 @@ static struct skcipher_alg skcipher_null = { static struct crypto_alg null_algs[] = { { .cra_name = "cipher_null", + .cra_driver_name = "cipher_null-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = NULL_BLOCK_SIZE, .cra_ctxsize = 0, @@ -139,6 +141,7 @@ static struct crypto_alg null_algs[] = { { .cia_decrypt = null_crypt } } }, { .cra_name = "compress_null", + .cra_driver_name = "compress_null-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_blocksize = NULL_BLOCK_SIZE, .cra_ctxsize = 0, diff --git a/crypto/deflate.c b/crypto/deflate.c index aab089cde1bf..1143ccbe0296 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -279,6 +279,7 @@ static int deflate_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "deflate", + .cra_driver_name = "deflate-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct deflate_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c index 4e8704405a3b..58f935315cf8 100644 --- a/crypto/fcrypt.c +++ b/crypto/fcrypt.c @@ -391,6 +391,7 @@ static int fcrypt_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key static struct crypto_alg fcrypt_alg = { .cra_name = "fcrypt", + .cra_driver_name = "fcrypt-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = 8, .cra_ctxsize = sizeof(struct fcrypt_ctx), diff --git a/crypto/khazad.c b/crypto/khazad.c index b50aa8a3ab4c..14ca7f1631c7 100644 --- a/crypto/khazad.c +++ b/crypto/khazad.c @@ -848,6 +848,7 @@ static void khazad_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static struct crypto_alg khazad_alg = { .cra_name = "khazad", + .cra_driver_name = "khazad-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = KHAZAD_BLOCK_SIZE, .cra_ctxsize = sizeof (struct khazad_ctx), diff --git a/crypto/lz4.c b/crypto/lz4.c index 1e35134d0a98..ed9088c97c6e 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -119,6 +119,7 @@ static int lz4_decompress_crypto(struct crypto_tfm *tfm, const u8 *src, static struct crypto_alg alg_lz4 = { .cra_name = "lz4", + .cra_driver_name = "lz4-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lz4_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index 4a220b628fe7..21a342c6b9d4 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -120,6 +120,7 @@ static int lz4hc_decompress_crypto(struct crypto_tfm *tfm, const u8 *src, static struct crypto_alg alg_lz4hc = { .cra_name = "lz4hc", + .cra_driver_name = "lz4hc-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lz4hc_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index 4c82bf18440f..469814926f02 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -122,6 +122,7 @@ static int lzorle_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "lzo-rle", + .cra_driver_name = "lzo-rle-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lzorle_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/lzo.c b/crypto/lzo.c index 4a6ac8f247d0..a98ac046613a 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -122,6 +122,7 @@ static int lzo_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "lzo", + .cra_driver_name = "lzo-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lzo_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/md4.c b/crypto/md4.c index 9a1a228a0c69..2e7f2f319f95 100644 --- a/crypto/md4.c +++ b/crypto/md4.c @@ -216,9 +216,10 @@ static struct shash_alg alg = { .final = md4_final, .descsize = sizeof(struct md4_ctx), .base = { - .cra_name = "md4", - .cra_blocksize = MD4_HMAC_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "md4", + .cra_driver_name = "md4-generic", + .cra_blocksize = MD4_HMAC_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }; diff --git a/crypto/md5.c b/crypto/md5.c index 221c2c0932f8..22dc60bc0437 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -228,9 +228,10 @@ static struct shash_alg alg = { .descsize = sizeof(struct md5_state), .statesize = sizeof(struct md5_state), .base = { - .cra_name = "md5", - .cra_blocksize = MD5_HMAC_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "md5", + .cra_driver_name = "md5-generic", + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }; diff --git a/crypto/michael_mic.c b/crypto/michael_mic.c index 538ae7933795..be5c07ea6841 100644 --- a/crypto/michael_mic.c +++ b/crypto/michael_mic.c @@ -159,6 +159,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct michael_mic_desc_ctx), .base = { .cra_name = "michael_mic", + .cra_driver_name = "michael_mic-generic", .cra_blocksize = 8, .cra_alignmask = 3, .cra_ctxsize = sizeof(struct michael_mic_ctx), diff --git a/crypto/rmd128.c b/crypto/rmd128.c index faf4252c4b85..81fcae094bd5 100644 --- a/crypto/rmd128.c +++ b/crypto/rmd128.c @@ -303,6 +303,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd128_ctx), .base = { .cra_name = "rmd128", + .cra_driver_name = "rmd128-generic", .cra_blocksize = RMD128_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/rmd160.c b/crypto/rmd160.c index b33309916d4f..0c0d178dee9d 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -347,6 +347,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd160_ctx), .base = { .cra_name = "rmd160", + .cra_driver_name = "rmd160-generic", .cra_blocksize = RMD160_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/rmd256.c b/crypto/rmd256.c index 2a643250c9a5..cdbbe37266c3 100644 --- a/crypto/rmd256.c +++ b/crypto/rmd256.c @@ -322,6 +322,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd256_ctx), .base = { .cra_name = "rmd256", + .cra_driver_name = "rmd256-generic", .cra_blocksize = RMD256_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/rmd320.c b/crypto/rmd320.c index 2f062574fc8c..9327af0fe4b7 100644 --- a/crypto/rmd320.c +++ b/crypto/rmd320.c @@ -371,6 +371,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd320_ctx), .base = { .cra_name = "rmd320", + .cra_driver_name = "rmd320-generic", .cra_blocksize = RMD320_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index ec4ec89ad108..f2f549330d2b 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -641,6 +641,7 @@ static struct crypto_alg srp_algs[2] = { { .cia_decrypt = serpent_decrypt } } }, { .cra_name = "tnepres", + .cra_driver_name = "tnepres-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), diff --git a/crypto/tea.c b/crypto/tea.c index 786b589e1399..fa012589d3b0 100644 --- a/crypto/tea.c +++ b/crypto/tea.c @@ -221,6 +221,7 @@ static void xeta_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static struct crypto_alg tea_algs[3] = { { .cra_name = "tea", + .cra_driver_name = "tea-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = TEA_BLOCK_SIZE, .cra_ctxsize = sizeof (struct tea_ctx), @@ -234,6 +235,7 @@ static struct crypto_alg tea_algs[3] = { { .cia_decrypt = tea_decrypt } } }, { .cra_name = "xtea", + .cra_driver_name = "xtea-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = XTEA_BLOCK_SIZE, .cra_ctxsize = sizeof (struct xtea_ctx), @@ -247,6 +249,7 @@ static struct crypto_alg tea_algs[3] = { { .cia_decrypt = xtea_decrypt } } }, { .cra_name = "xeta", + .cra_driver_name = "xeta-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = XTEA_BLOCK_SIZE, .cra_ctxsize = sizeof (struct xtea_ctx), diff --git a/crypto/tgr192.c b/crypto/tgr192.c index 40020f8adc46..39b3ffd22f5d 100644 --- a/crypto/tgr192.c +++ b/crypto/tgr192.c @@ -635,9 +635,10 @@ static struct shash_alg tgr_algs[3] = { { .final = tgr192_final, .descsize = sizeof(struct tgr192_ctx), .base = { - .cra_name = "tgr192", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "tgr192", + .cra_driver_name = "tgr192-generic", + .cra_blocksize = TGR192_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = TGR160_DIGEST_SIZE, @@ -646,9 +647,10 @@ static struct shash_alg tgr_algs[3] = { { .final = tgr160_final, .descsize = sizeof(struct tgr192_ctx), .base = { - .cra_name = "tgr160", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "tgr160", + .cra_driver_name = "tgr160-generic", + .cra_blocksize = TGR192_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = TGR128_DIGEST_SIZE, @@ -657,9 +659,10 @@ static struct shash_alg tgr_algs[3] = { { .final = tgr128_final, .descsize = sizeof(struct tgr192_ctx), .base = { - .cra_name = "tgr128", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "tgr128", + .cra_driver_name = "tgr128-generic", + .cra_blocksize = TGR192_BLOCK_SIZE, + .cra_module = THIS_MODULE, } } }; diff --git a/crypto/wp512.c b/crypto/wp512.c index 1b8e502d999f..feadc13ccae0 100644 --- a/crypto/wp512.c +++ b/crypto/wp512.c @@ -1126,9 +1126,10 @@ static struct shash_alg wp_algs[3] = { { .final = wp512_final, .descsize = sizeof(struct wp512_ctx), .base = { - .cra_name = "wp512", - .cra_blocksize = WP512_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "wp512", + .cra_driver_name = "wp512-generic", + .cra_blocksize = WP512_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = WP384_DIGEST_SIZE, @@ -1137,9 +1138,10 @@ static struct shash_alg wp_algs[3] = { { .final = wp384_final, .descsize = sizeof(struct wp512_ctx), .base = { - .cra_name = "wp384", - .cra_blocksize = WP512_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "wp384", + .cra_driver_name = "wp384-generic", + .cra_blocksize = WP512_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = WP256_DIGEST_SIZE, @@ -1148,9 +1150,10 @@ static struct shash_alg wp_algs[3] = { { .final = wp256_final, .descsize = sizeof(struct wp512_ctx), .base = { - .cra_name = "wp256", - .cra_blocksize = WP512_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "wp256", + .cra_driver_name = "wp256-generic", + .cra_blocksize = WP512_BLOCK_SIZE, + .cra_module = THIS_MODULE, } } }; diff --git a/crypto/zstd.c b/crypto/zstd.c index 2c04055e407f..c9ff2ec8d4a3 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -214,6 +214,7 @@ static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "zstd", + .cra_driver_name = "zstd-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct zstd_ctx), .cra_module = THIS_MODULE, From 177f87d063ebc7a11a38bdafaca8fec4a9dae13e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 2 Jun 2019 22:40:58 -0700 Subject: [PATCH 089/143] crypto: algapi - require cra_name and cra_driver_name Now that all algorithms explicitly set cra_driver_name, make it required for algorithm registration and remove the code that generated a default cra_driver_name. Also add an explicit check that cra_name is set too, since that's obviously required too, yet it didn't seem to be checked anywhere. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/algapi.c | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 7c51f45d1cf1..5278e139a161 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -26,23 +26,6 @@ static LIST_HEAD(crypto_template_list); -static inline int crypto_set_driver_name(struct crypto_alg *alg) -{ - static const char suffix[] = "-generic"; - char *driver_name = alg->cra_driver_name; - int len; - - if (*driver_name) - return 0; - - len = strlcpy(driver_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); - if (len + sizeof(suffix) > CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - memcpy(driver_name + len, suffix, sizeof(suffix)); - return 0; -} - static inline void crypto_check_module_sig(struct module *mod) { if (fips_enabled && mod && !module_sig_ok(mod)) @@ -54,6 +37,9 @@ static int crypto_check_alg(struct crypto_alg *alg) { crypto_check_module_sig(alg->cra_module); + if (!alg->cra_name[0] || !alg->cra_driver_name[0]) + return -EINVAL; + if (alg->cra_alignmask & (alg->cra_alignmask + 1)) return -EINVAL; @@ -79,7 +65,7 @@ static int crypto_check_alg(struct crypto_alg *alg) refcount_set(&alg->cra_refcnt, 1); - return crypto_set_driver_name(alg); + return 0; } static void crypto_free_instance(struct crypto_instance *inst) From e63e1b0dd0003dc31f73d875907432be3a2abe5d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 2 Jun 2019 22:42:33 -0700 Subject: [PATCH 090/143] crypto: testmgr - add some more preemption points Call cond_resched() after each fuzz test iteration. This avoids stall warnings if fuzz_iterations is set very high for testing purposes. While we're at it, also call cond_resched() after finishing testing each test vector. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/testmgr.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 2ba0c487ea28..f7fdd7fe89a9 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1496,6 +1496,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, req, desc, tsgl, hashstate); if (err) return err; + cond_resched(); } } #endif @@ -1764,6 +1765,7 @@ static int __alg_test_hash(const struct hash_testvec *vecs, hashstate); if (err) goto out; + cond_resched(); } err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req, desc, tsgl, hashstate); @@ -2028,6 +2030,7 @@ static int test_aead_vec(const char *driver, int enc, &cfg, req, tsgls); if (err) return err; + cond_resched(); } } #endif @@ -2267,6 +2270,7 @@ static int test_aead(const char *driver, int enc, tsgls); if (err) return err; + cond_resched(); } return 0; } @@ -2609,6 +2613,7 @@ static int test_skcipher_vec(const char *driver, int enc, &cfg, req, tsgls); if (err) return err; + cond_resched(); } } #endif @@ -2808,6 +2813,7 @@ static int test_skcipher(const char *driver, int enc, tsgls); if (err) return err; + cond_resched(); } return 0; } From 03d66cfa2ad62cd0bb4c9aabc5a28d56af41f0cb Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 2 Jun 2019 22:44:08 -0700 Subject: [PATCH 091/143] crypto: doc - improve the skcipher API example code Rewrite the skcipher API example, changing it to encrypt a buffer with AES-256-XTS. This addresses various problems with the previous example: - It requests a specific driver "cbc-aes-aesni", which is unusual. Normally users ask for "cbc(aes)", not a specific driver. - It encrypts only a single AES block. For the reader, that doesn't clearly distinguish the "skcipher" API from the "cipher" API. - Showing how to encrypt something with bare CBC is arguably a poor choice of example, as it doesn't follow modern crypto trends. Now, usually authenticated encryption is recommended, in which case the user would use the AEAD API, not skcipher. Disk encryption is still a legitimate use for skcipher, but for that usually XTS is recommended. - Many other bugs and poor coding practices, such as not setting CRYPTO_TFM_REQ_MAY_SLEEP, unnecessarily allocating a heap buffer for the IV, unnecessary NULL checks, using a pointless wrapper struct, and forgetting to set an error code in one case. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- Documentation/crypto/api-samples.rst | 160 ++++++++++++--------------- 1 file changed, 69 insertions(+), 91 deletions(-) diff --git a/Documentation/crypto/api-samples.rst b/Documentation/crypto/api-samples.rst index f14afaaf2f32..e923f17bc2bd 100644 --- a/Documentation/crypto/api-samples.rst +++ b/Documentation/crypto/api-samples.rst @@ -4,111 +4,89 @@ Code Examples Code Example For Symmetric Key Cipher Operation ----------------------------------------------- +This code encrypts some data with AES-256-XTS. For sake of example, +all inputs are random bytes, the encryption is done in-place, and it's +assumed the code is running in a context where it can sleep. + :: - - /* tie all data structures together */ - struct skcipher_def { - struct scatterlist sg; - struct crypto_skcipher *tfm; - struct skcipher_request *req; - struct crypto_wait wait; - }; - - /* Perform cipher operation */ - static unsigned int test_skcipher_encdec(struct skcipher_def *sk, - int enc) - { - int rc; - - if (enc) - rc = crypto_wait_req(crypto_skcipher_encrypt(sk->req), &sk->wait); - else - rc = crypto_wait_req(crypto_skcipher_decrypt(sk->req), &sk->wait); - - if (rc) - pr_info("skcipher encrypt returned with result %d\n", rc); - - return rc; - } - - /* Initialize and trigger cipher operation */ static int test_skcipher(void) { - struct skcipher_def sk; - struct crypto_skcipher *skcipher = NULL; - struct skcipher_request *req = NULL; - char *scratchpad = NULL; - char *ivdata = NULL; - unsigned char key[32]; - int ret = -EFAULT; + struct crypto_skcipher *tfm = NULL; + struct skcipher_request *req = NULL; + u8 *data = NULL; + const size_t datasize = 512; /* data size in bytes */ + struct scatterlist sg; + DECLARE_CRYPTO_WAIT(wait); + u8 iv[16]; /* AES-256-XTS takes a 16-byte IV */ + u8 key[64]; /* AES-256-XTS takes a 64-byte key */ + int err; - skcipher = crypto_alloc_skcipher("cbc-aes-aesni", 0, 0); - if (IS_ERR(skcipher)) { - pr_info("could not allocate skcipher handle\n"); - return PTR_ERR(skcipher); - } + /* + * Allocate a tfm (a transformation object) and set the key. + * + * In real-world use, a tfm and key are typically used for many + * encryption/decryption operations. But in this example, we'll just do a + * single encryption operation with it (which is not very efficient). + */ - req = skcipher_request_alloc(skcipher, GFP_KERNEL); - if (!req) { - pr_info("could not allocate skcipher request\n"); - ret = -ENOMEM; - goto out; - } + tfm = crypto_alloc_skcipher("xts(aes)", 0, 0); + if (IS_ERR(tfm)) { + pr_err("Error allocating xts(aes) handle: %ld\n", PTR_ERR(tfm)); + return PTR_ERR(tfm); + } - skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, - &sk.wait); + get_random_bytes(key, sizeof(key)); + err = crypto_skcipher_setkey(tfm, key, sizeof(key)); + if (err) { + pr_err("Error setting key: %d\n", err); + goto out; + } - /* AES 256 with random key */ - get_random_bytes(&key, 32); - if (crypto_skcipher_setkey(skcipher, key, 32)) { - pr_info("key could not be set\n"); - ret = -EAGAIN; - goto out; - } + /* Allocate a request object */ + req = skcipher_request_alloc(tfm, GFP_KERNEL); + if (!req) { + err = -ENOMEM; + goto out; + } - /* IV will be random */ - ivdata = kmalloc(16, GFP_KERNEL); - if (!ivdata) { - pr_info("could not allocate ivdata\n"); - goto out; - } - get_random_bytes(ivdata, 16); + /* Prepare the input data */ + data = kmalloc(datasize, GFP_KERNEL); + if (!data) { + err = -ENOMEM; + goto out; + } + get_random_bytes(data, datasize); - /* Input data will be random */ - scratchpad = kmalloc(16, GFP_KERNEL); - if (!scratchpad) { - pr_info("could not allocate scratchpad\n"); - goto out; - } - get_random_bytes(scratchpad, 16); + /* Initialize the IV */ + get_random_bytes(iv, sizeof(iv)); - sk.tfm = skcipher; - sk.req = req; - - /* We encrypt one block */ - sg_init_one(&sk.sg, scratchpad, 16); - skcipher_request_set_crypt(req, &sk.sg, &sk.sg, 16, ivdata); - crypto_init_wait(&sk.wait); - - /* encrypt data */ - ret = test_skcipher_encdec(&sk, 1); - if (ret) - goto out; - - pr_info("Encryption triggered successfully\n"); + /* + * Encrypt the data in-place. + * + * For simplicity, in this example we wait for the request to complete + * before proceeding, even if the underlying implementation is asynchronous. + * + * To decrypt instead of encrypt, just change crypto_skcipher_encrypt() to + * crypto_skcipher_decrypt(). + */ + sg_init_one(&sg, data, datasize); + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &wait); + skcipher_request_set_crypt(req, &sg, &sg, datasize, iv); + err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); + if (err) { + pr_err("Error encrypting data: %d\n", err); + goto out; + } + pr_debug("Encryption was successful\n"); out: - if (skcipher) - crypto_free_skcipher(skcipher); - if (req) + crypto_free_skcipher(tfm); skcipher_request_free(req); - if (ivdata) - kfree(ivdata); - if (scratchpad) - kfree(scratchpad); - return ret; + kfree(data); + return err; } From 07269559ac0bf778c6eadcaa9104e7adb4d46f33 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 2 Jun 2019 22:44:50 -0700 Subject: [PATCH 092/143] crypto: x86/aesni - remove unused internal cipher algorithm Since commit 944585a64f5e ("crypto: x86/aes-ni - remove special handling of AES in PCBC mode"), the "__aes-aesni" internal cipher algorithm is no longer used. So remove it too. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_glue.c | 45 +++++------------------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 21c246799aa5..c95bd397dc07 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -375,20 +375,6 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) } } -static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); - - aesni_enc(ctx, dst, src); -} - -static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); - - aesni_dec(ctx, dst, src); -} - static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int len) { @@ -924,7 +910,7 @@ static int helper_rfc4106_decrypt(struct aead_request *req) } #endif -static struct crypto_alg aesni_algs[] = { { +static struct crypto_alg aesni_cipher_alg = { .cra_name = "aes", .cra_driver_name = "aes-aesni", .cra_priority = 300, @@ -941,24 +927,7 @@ static struct crypto_alg aesni_algs[] = { { .cia_decrypt = aes_decrypt } } -}, { - .cra_name = "__aes", - .cra_driver_name = "__aes-aesni", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = CRYPTO_AES_CTX_SIZE, - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = AES_MIN_KEY_SIZE, - .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = aes_set_key, - .cia_encrypt = __aes_encrypt, - .cia_decrypt = __aes_decrypt - } - } -} }; +}; static struct skcipher_alg aesni_skciphers[] = { { @@ -1154,7 +1123,7 @@ static int __init aesni_init(void) #endif #endif - err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); + err = crypto_register_alg(&aesni_cipher_alg); if (err) return err; @@ -1162,7 +1131,7 @@ static int __init aesni_init(void) ARRAY_SIZE(aesni_skciphers), aesni_simd_skciphers); if (err) - goto unregister_algs; + goto unregister_cipher; err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads), aesni_simd_aeads); @@ -1174,8 +1143,8 @@ static int __init aesni_init(void) unregister_skciphers: simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), aesni_simd_skciphers); -unregister_algs: - crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); +unregister_cipher: + crypto_unregister_alg(&aesni_cipher_alg); return err; } @@ -1185,7 +1154,7 @@ static void __exit aesni_exit(void) aesni_simd_aeads); simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), aesni_simd_skciphers); - crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); + crypto_unregister_alg(&aesni_cipher_alg); } late_initcall(aesni_init); From f2fe1154542e69976f859e573dc4d9aa0b76a495 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 2 Jun 2019 22:45:16 -0700 Subject: [PATCH 093/143] crypto: aead - un-inline encrypt and decrypt functions crypto_aead_encrypt() and crypto_aead_decrypt() have grown to be more than a single indirect function call. They now also check whether a key has been set, the decryption side checks whether the input is at least as long as the authentication tag length, and with CONFIG_CRYPTO_STATS=y they also update the crypto statistics. That can add up to a lot of bloat at every call site. Moreover, these always involve a function call anyway, which greatly limits the benefits of inlining. So change them to be non-inline. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/aead.c | 36 ++++++++++++++++++++++++++++++++++++ include/crypto/aead.h | 34 ++-------------------------------- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/crypto/aead.c b/crypto/aead.c index 4908b5e846f0..fc1d7ad8a487 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -89,6 +89,42 @@ int crypto_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) } EXPORT_SYMBOL_GPL(crypto_aead_setauthsize); +int crypto_aead_encrypt(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_alg *alg = aead->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else + ret = crypto_aead_alg(aead)->encrypt(req); + crypto_stats_aead_encrypt(cryptlen, alg, ret); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_aead_encrypt); + +int crypto_aead_decrypt(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_alg *alg = aead->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else if (req->cryptlen < crypto_aead_authsize(aead)) + ret = -EINVAL; + else + ret = crypto_aead_alg(aead)->decrypt(req); + crypto_stats_aead_decrypt(cryptlen, alg, ret); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_aead_decrypt); + static void crypto_aead_exit_tfm(struct crypto_tfm *tfm) { struct crypto_aead *aead = __crypto_aead_cast(tfm); diff --git a/include/crypto/aead.h b/include/crypto/aead.h index 9ad595f97c65..020d581373ab 100644 --- a/include/crypto/aead.h +++ b/include/crypto/aead.h @@ -322,21 +322,7 @@ static inline struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req) * * Return: 0 if the cipher operation was successful; < 0 if an error occurred */ -static inline int crypto_aead_encrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_alg *alg = aead->base.__crt_alg; - unsigned int cryptlen = req->cryptlen; - int ret; - - crypto_stats_get(alg); - if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) - ret = -ENOKEY; - else - ret = crypto_aead_alg(aead)->encrypt(req); - crypto_stats_aead_encrypt(cryptlen, alg, ret); - return ret; -} +int crypto_aead_encrypt(struct aead_request *req); /** * crypto_aead_decrypt() - decrypt ciphertext @@ -360,23 +346,7 @@ static inline int crypto_aead_encrypt(struct aead_request *req) * integrity of the ciphertext or the associated data was violated); * < 0 if an error occurred. */ -static inline int crypto_aead_decrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_alg *alg = aead->base.__crt_alg; - unsigned int cryptlen = req->cryptlen; - int ret; - - crypto_stats_get(alg); - if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) - ret = -ENOKEY; - else if (req->cryptlen < crypto_aead_authsize(aead)) - ret = -EINVAL; - else - ret = crypto_aead_alg(aead)->decrypt(req); - crypto_stats_aead_decrypt(cryptlen, alg, ret); - return ret; -} +int crypto_aead_decrypt(struct aead_request *req); /** * DOC: Asynchronous AEAD Request Handle From 81bcbb1ee75d44f270a2905ea09c8a5861cf0086 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 2 Jun 2019 22:45:51 -0700 Subject: [PATCH 094/143] crypto: skcipher - un-inline encrypt and decrypt functions crypto_skcipher_encrypt() and crypto_skcipher_decrypt() have grown to be more than a single indirect function call. They now also check whether a key has been set, and with CONFIG_CRYPTO_STATS=y they also update the crypto statistics. That can add up to a lot of bloat at every call site. Moreover, these always involve a function call anyway, which greatly limits the benefits of inlining. So change them to be non-inline. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/skcipher.c | 34 ++++++++++++++++++++++++++++++++++ include/crypto/skcipher.h | 32 ++------------------------------ 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 2e66f312e2c4..2828e27d7fbc 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -842,6 +842,40 @@ static int skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, return 0; } +int crypto_skcipher_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_alg *alg = tfm->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else + ret = tfm->encrypt(req); + crypto_stats_skcipher_encrypt(cryptlen, ret, alg); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_skcipher_encrypt); + +int crypto_skcipher_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_alg *alg = tfm->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else + ret = tfm->decrypt(req); + crypto_stats_skcipher_decrypt(cryptlen, ret, alg); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_skcipher_decrypt); + static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm) { struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm); diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h index e555294ed77f..98547d1f18c5 100644 --- a/include/crypto/skcipher.h +++ b/include/crypto/skcipher.h @@ -484,21 +484,7 @@ static inline struct crypto_sync_skcipher *crypto_sync_skcipher_reqtfm( * * Return: 0 if the cipher operation was successful; < 0 if an error occurred */ -static inline int crypto_skcipher_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct crypto_alg *alg = tfm->base.__crt_alg; - unsigned int cryptlen = req->cryptlen; - int ret; - - crypto_stats_get(alg); - if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) - ret = -ENOKEY; - else - ret = tfm->encrypt(req); - crypto_stats_skcipher_encrypt(cryptlen, ret, alg); - return ret; -} +int crypto_skcipher_encrypt(struct skcipher_request *req); /** * crypto_skcipher_decrypt() - decrypt ciphertext @@ -511,21 +497,7 @@ static inline int crypto_skcipher_encrypt(struct skcipher_request *req) * * Return: 0 if the cipher operation was successful; < 0 if an error occurred */ -static inline int crypto_skcipher_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct crypto_alg *alg = tfm->base.__crt_alg; - unsigned int cryptlen = req->cryptlen; - int ret; - - crypto_stats_get(alg); - if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) - ret = -ENOKEY; - else - ret = tfm->decrypt(req); - crypto_stats_skcipher_decrypt(cryptlen, ret, alg); - return ret; -} +int crypto_skcipher_decrypt(struct skcipher_request *req); /** * DOC: Symmetric Key Cipher Request Handle From 314d0f0ea69f2fb8e6fc4cb5d5ae6b1e3418e36c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 2 Jun 2019 22:46:11 -0700 Subject: [PATCH 095/143] crypto: skcipher - make chunksize and walksize accessors internal The 'chunksize' and 'walksize' properties of skcipher algorithms are implementation details that users of the skcipher API should not be looking at. So move their accessor functions from to . Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/internal/skcipher.h | 60 ++++++++++++++++++++++++++++++ include/crypto/skcipher.h | 60 ------------------------------ 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index 9de6032209cb..abb1096495c2 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -205,6 +205,66 @@ static inline unsigned int crypto_skcipher_alg_max_keysize( return alg->max_keysize; } +static inline unsigned int crypto_skcipher_alg_chunksize( + struct skcipher_alg *alg) +{ + if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) == + CRYPTO_ALG_TYPE_BLKCIPHER) + return alg->base.cra_blocksize; + + if (alg->base.cra_ablkcipher.encrypt) + return alg->base.cra_blocksize; + + return alg->chunksize; +} + +static inline unsigned int crypto_skcipher_alg_walksize( + struct skcipher_alg *alg) +{ + if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) == + CRYPTO_ALG_TYPE_BLKCIPHER) + return alg->base.cra_blocksize; + + if (alg->base.cra_ablkcipher.encrypt) + return alg->base.cra_blocksize; + + return alg->walksize; +} + +/** + * crypto_skcipher_chunksize() - obtain chunk size + * @tfm: cipher handle + * + * The block size is set to one for ciphers such as CTR. However, + * you still need to provide incremental updates in multiples of + * the underlying block size as the IV does not have sub-block + * granularity. This is known in this API as the chunk size. + * + * Return: chunk size in bytes + */ +static inline unsigned int crypto_skcipher_chunksize( + struct crypto_skcipher *tfm) +{ + return crypto_skcipher_alg_chunksize(crypto_skcipher_alg(tfm)); +} + +/** + * crypto_skcipher_walksize() - obtain walk size + * @tfm: cipher handle + * + * In some cases, algorithms can only perform optimally when operating on + * multiple blocks in parallel. This is reflected by the walksize, which + * must be a multiple of the chunksize (or equal if the concern does not + * apply) + * + * Return: walk size in bytes + */ +static inline unsigned int crypto_skcipher_walksize( + struct crypto_skcipher *tfm) +{ + return crypto_skcipher_alg_walksize(crypto_skcipher_alg(tfm)); +} + /* Helpers for simple block cipher modes of operation */ struct skcipher_ctx_simple { struct crypto_cipher *cipher; /* underlying block cipher */ diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h index 98547d1f18c5..694397fb0faa 100644 --- a/include/crypto/skcipher.h +++ b/include/crypto/skcipher.h @@ -293,66 +293,6 @@ static inline unsigned int crypto_sync_skcipher_ivsize( return crypto_skcipher_ivsize(&tfm->base); } -static inline unsigned int crypto_skcipher_alg_chunksize( - struct skcipher_alg *alg) -{ - if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER) - return alg->base.cra_blocksize; - - if (alg->base.cra_ablkcipher.encrypt) - return alg->base.cra_blocksize; - - return alg->chunksize; -} - -static inline unsigned int crypto_skcipher_alg_walksize( - struct skcipher_alg *alg) -{ - if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER) - return alg->base.cra_blocksize; - - if (alg->base.cra_ablkcipher.encrypt) - return alg->base.cra_blocksize; - - return alg->walksize; -} - -/** - * crypto_skcipher_chunksize() - obtain chunk size - * @tfm: cipher handle - * - * The block size is set to one for ciphers such as CTR. However, - * you still need to provide incremental updates in multiples of - * the underlying block size as the IV does not have sub-block - * granularity. This is known in this API as the chunk size. - * - * Return: chunk size in bytes - */ -static inline unsigned int crypto_skcipher_chunksize( - struct crypto_skcipher *tfm) -{ - return crypto_skcipher_alg_chunksize(crypto_skcipher_alg(tfm)); -} - -/** - * crypto_skcipher_walksize() - obtain walk size - * @tfm: cipher handle - * - * In some cases, algorithms can only perform optimally when operating on - * multiple blocks in parallel. This is reflected by the walksize, which - * must be a multiple of the chunksize (or equal if the concern does not - * apply) - * - * Return: walk size in bytes - */ -static inline unsigned int crypto_skcipher_walksize( - struct crypto_skcipher *tfm) -{ - return crypto_skcipher_alg_walksize(crypto_skcipher_alg(tfm)); -} - /** * crypto_skcipher_blocksize() - obtain block size of cipher * @tfm: cipher handle From 76cadf2244518d3b1610082d037b01e2651ea402 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 2 Jun 2019 22:46:34 -0700 Subject: [PATCH 096/143] crypto: chacha20poly1305 - a few cleanups - Use sg_init_one() instead of sg_init_table() then sg_set_buf(). - Remove unneeded calls to sg_init_table() prior to scatterwalk_ffwd(). - Simplify initializing the poly tail block. - Simplify computing padlen. This doesn't change any actual behavior. Cc: Martin Willi Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/chacha20poly1305.c | 43 ++++++++++++--------------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index acbbf010222e..76c13dab217d 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -139,14 +139,10 @@ static int chacha_decrypt(struct aead_request *req) chacha_iv(creq->iv, req, 1); - sg_init_table(rctx->src, 2); src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); dst = src; - - if (req->src != req->dst) { - sg_init_table(rctx->dst, 2); + if (req->src != req->dst) dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); - } skcipher_request_set_callback(&creq->req, rctx->flags, chacha_decrypt_done, req); @@ -182,15 +178,11 @@ static int poly_tail(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct poly_req *preq = &rctx->u.poly; - __le64 len; int err; - sg_init_table(preq->src, 1); - len = cpu_to_le64(rctx->assoclen); - memcpy(&preq->tail.assoclen, &len, sizeof(len)); - len = cpu_to_le64(rctx->cryptlen); - memcpy(&preq->tail.cryptlen, &len, sizeof(len)); - sg_set_buf(preq->src, &preq->tail, sizeof(preq->tail)); + preq->tail.assoclen = cpu_to_le64(rctx->assoclen); + preq->tail.cryptlen = cpu_to_le64(rctx->cryptlen); + sg_init_one(preq->src, &preq->tail, sizeof(preq->tail)); ahash_request_set_callback(&preq->req, rctx->flags, poly_tail_done, req); @@ -215,13 +207,12 @@ static int poly_cipherpad(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct poly_req *preq = &rctx->u.poly; - unsigned int padlen, bs = POLY1305_BLOCK_SIZE; + unsigned int padlen; int err; - padlen = (bs - (rctx->cryptlen % bs)) % bs; + padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE; memset(preq->pad, 0, sizeof(preq->pad)); - sg_init_table(preq->src, 1); - sg_set_buf(preq->src, &preq->pad, padlen); + sg_init_one(preq->src, preq->pad, padlen); ahash_request_set_callback(&preq->req, rctx->flags, poly_cipherpad_done, req); @@ -251,7 +242,6 @@ static int poly_cipher(struct aead_request *req) if (rctx->cryptlen == req->cryptlen) /* encrypting */ crypt = req->dst; - sg_init_table(rctx->src, 2); crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen); ahash_request_set_callback(&preq->req, rctx->flags, @@ -276,13 +266,12 @@ static int poly_adpad(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct poly_req *preq = &rctx->u.poly; - unsigned int padlen, bs = POLY1305_BLOCK_SIZE; + unsigned int padlen; int err; - padlen = (bs - (rctx->assoclen % bs)) % bs; + padlen = -rctx->assoclen % POLY1305_BLOCK_SIZE; memset(preq->pad, 0, sizeof(preq->pad)); - sg_init_table(preq->src, 1); - sg_set_buf(preq->src, preq->pad, padlen); + sg_init_one(preq->src, preq->pad, padlen); ahash_request_set_callback(&preq->req, rctx->flags, poly_adpad_done, req); @@ -332,8 +321,7 @@ static int poly_setkey(struct aead_request *req) struct poly_req *preq = &rctx->u.poly; int err; - sg_init_table(preq->src, 1); - sg_set_buf(preq->src, rctx->key, sizeof(rctx->key)); + sg_init_one(preq->src, rctx->key, sizeof(rctx->key)); ahash_request_set_callback(&preq->req, rctx->flags, poly_setkey_done, req); @@ -391,9 +379,8 @@ static int poly_genkey(struct aead_request *req) rctx->assoclen -= 8; } - sg_init_table(creq->src, 1); memset(rctx->key, 0, sizeof(rctx->key)); - sg_set_buf(creq->src, rctx->key, sizeof(rctx->key)); + sg_init_one(creq->src, rctx->key, sizeof(rctx->key)); chacha_iv(creq->iv, req, 0); @@ -428,14 +415,10 @@ static int chacha_encrypt(struct aead_request *req) chacha_iv(creq->iv, req, 1); - sg_init_table(rctx->src, 2); src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); dst = src; - - if (req->src != req->dst) { - sg_init_table(rctx->dst, 2); + if (req->src != req->dst) dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); - } skcipher_request_set_callback(&creq->req, rctx->flags, chacha_encrypt_done, req); From 860ab2e50204c41e713c54c752617d2da57f0fd7 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 2 Jun 2019 22:47:14 -0700 Subject: [PATCH 097/143] crypto: chacha - constify ctx and iv arguments Constify the ctx and iv arguments to crypto_chacha_init() and the various chacha*_stream_xor() functions. This makes it clear that they are not modified. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/chacha-neon-glue.c | 2 +- arch/arm64/crypto/chacha-neon-glue.c | 2 +- arch/x86/crypto/chacha_glue.c | 2 +- crypto/chacha_generic.c | 4 ++-- include/crypto/chacha.h | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/arm/crypto/chacha-neon-glue.c b/arch/arm/crypto/chacha-neon-glue.c index 48a89537b828..a8e9b534c8da 100644 --- a/arch/arm/crypto/chacha-neon-glue.c +++ b/arch/arm/crypto/chacha-neon-glue.c @@ -63,7 +63,7 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, } static int chacha_neon_stream_xor(struct skcipher_request *req, - struct chacha_ctx *ctx, u8 *iv) + const struct chacha_ctx *ctx, const u8 *iv) { struct skcipher_walk walk; u32 state[16]; diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index 82029cda2e77..1495d2b18518 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -60,7 +60,7 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, } static int chacha_neon_stream_xor(struct skcipher_request *req, - struct chacha_ctx *ctx, u8 *iv) + const struct chacha_ctx *ctx, const u8 *iv) { struct skcipher_walk walk; u32 state[16]; diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index 4967ad620775..7276b7ef14ec 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -128,7 +128,7 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, } static int chacha_simd_stream_xor(struct skcipher_walk *walk, - struct chacha_ctx *ctx, u8 *iv) + const struct chacha_ctx *ctx, const u8 *iv) { u32 *state, state_buf[16 + 2] __aligned(8); int next_yield = 4096; /* bytes until next FPU yield */ diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c index d2ec04997832..d283bd3bdb60 100644 --- a/crypto/chacha_generic.c +++ b/crypto/chacha_generic.c @@ -36,7 +36,7 @@ static void chacha_docrypt(u32 *state, u8 *dst, const u8 *src, } static int chacha_stream_xor(struct skcipher_request *req, - struct chacha_ctx *ctx, u8 *iv) + const struct chacha_ctx *ctx, const u8 *iv) { struct skcipher_walk walk; u32 state[16]; @@ -60,7 +60,7 @@ static int chacha_stream_xor(struct skcipher_request *req, return err; } -void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv) +void crypto_chacha_init(u32 *state, const struct chacha_ctx *ctx, const u8 *iv) { state[0] = 0x61707865; /* "expa" */ state[1] = 0x3320646e; /* "nd 3" */ diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 1fc70a69d550..d1e723c6a37d 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -41,7 +41,7 @@ static inline void chacha20_block(u32 *state, u8 *stream) } void hchacha_block(const u32 *in, u32 *out, int nrounds); -void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv); +void crypto_chacha_init(u32 *state, const struct chacha_ctx *ctx, const u8 *iv); int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keysize); From dd508618486bb573141cecd5e4e5060b35c4aefe Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Tue, 4 Jun 2019 22:53:51 +0800 Subject: [PATCH 098/143] crypto: bcm - Make some symbols static Fix sparse warnings: drivers/crypto/bcm/cipher.c:99:6: warning: symbol 'BCMHEADER' was not declared. Should it be static? drivers/crypto/bcm/cipher.c:2096:6: warning: symbol 'spu_no_incr_hash' was not declared. Should it be static? drivers/crypto/bcm/cipher.c:4823:5: warning: symbol 'bcm_spu_probe' was not declared. Should it be static? drivers/crypto/bcm/cipher.c:4867:5: warning: symbol 'bcm_spu_remove' was not declared. Should it be static? drivers/crypto/bcm/spu2.c:52:6: warning: symbol 'spu2_cipher_type_names' was not declared. Should it be static? drivers/crypto/bcm/spu2.c:56:6: warning: symbol 'spu2_cipher_mode_names' was not declared. Should it be static? drivers/crypto/bcm/spu2.c:60:6: warning: symbol 'spu2_hash_type_names' was not declared. Should it be static? drivers/crypto/bcm/spu2.c:66:6: warning: symbol 'spu2_hash_mode_names' was not declared. Should it be static? Reported-by: Hulk Robot Signed-off-by: YueHaibing Signed-off-by: Herbert Xu --- drivers/crypto/bcm/cipher.c | 8 ++++---- drivers/crypto/bcm/spu2.c | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 25f8d3913ceb..d972ffac779d 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -96,7 +96,7 @@ MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos"); * 0x70 - ring 2 * 0x78 - ring 3 */ -char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 }; +static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 }; /* * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN * is set dynamically after reading SPU type from device tree. @@ -2094,7 +2094,7 @@ static int __ahash_init(struct ahash_request *req) * Return: true if incremental hashing is not supported * false otherwise */ -bool spu_no_incr_hash(struct iproc_ctx_s *ctx) +static bool spu_no_incr_hash(struct iproc_ctx_s *ctx) { struct spu_hw *spu = &iproc_priv.spu; @@ -4820,7 +4820,7 @@ static int spu_dt_read(struct platform_device *pdev) return 0; } -int bcm_spu_probe(struct platform_device *pdev) +static int bcm_spu_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct spu_hw *spu = &iproc_priv.spu; @@ -4864,7 +4864,7 @@ failure: return err; } -int bcm_spu_remove(struct platform_device *pdev) +static int bcm_spu_remove(struct platform_device *pdev) { int i; struct device *dev = &pdev->dev; diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c index bf7ac621c591..960b472be6fd 100644 --- a/drivers/crypto/bcm/spu2.c +++ b/drivers/crypto/bcm/spu2.c @@ -49,21 +49,21 @@ enum spu2_proto_sel { SPU2_DTLS_AEAD = 10 }; -char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256", +static char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256", "DES", "3DES" }; -char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB", "XTS", - "CCM", "GCM" +static char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB", + "XTS", "CCM", "GCM" }; -char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256", +static char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256", "Reserved", "Reserved", "MD5", "SHA1", "SHA224", "SHA256", "SHA384", "SHA512", "SHA512/224", "SHA512/256", "SHA3-224", "SHA3-256", "SHA3-384", "SHA3-512" }; -char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC", +static char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC", "Rabin", "CCM", "GCM", "Reserved" }; From 98785092f279e5dff74463b311a1c7d0c10cda4d Mon Sep 17 00:00:00 2001 From: Nishad Kamdar Date: Wed, 5 Jun 2019 18:51:53 +0530 Subject: [PATCH 099/143] crypto: cavium/nitrox - Use the correct style for SPDX License Identifier This patch corrects the SPDX License Identifier style in header files related to Crypto Drivers for Cavium Nitrox family CNN55XX devices. For C header files Documentation/process/license-rules.rst mandates C-like comments (opposed to C source files where C++ style should be used) Changes made by using a script provided by Joe Perches here: https://lkml.org/lkml/2019/2/7/46 Suggested-by: Joe Perches Signed-off-by: Nishad Kamdar Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/nitrox_debugfs.h | 2 +- drivers/crypto/cavium/nitrox/nitrox_mbx.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h index f177b79bbab0..09c4cf2513fb 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h +++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NITROX_DEBUGFS_H #define __NITROX_DEBUGFS_H diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.h b/drivers/crypto/cavium/nitrox/nitrox_mbx.h index 5008399775a9..7c93d0282174 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.h +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NITROX_MBX_H #define __NITROX_MBX_H From 0b211338cf73eb7dd9a1c26721ce43076918373f Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Thu, 6 Jun 2019 11:02:55 +0300 Subject: [PATCH 100/143] ARM: dts: imx7ulp: add crypto support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add crypto node in device tree for CAAM support. Noteworthy is that on 7ulp the interrupt line is shared between the two job rings. Signed-off-by: Iuliana Prodan Signed-off-by: Franck LENORMAND Signed-off-by: Horia Geantă Acked-by: Shawn Guo Signed-off-by: Herbert Xu --- arch/arm/boot/dts/imx7ulp.dtsi | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/arch/arm/boot/dts/imx7ulp.dtsi b/arch/arm/boot/dts/imx7ulp.dtsi index d6b711011cba..e20483714be5 100644 --- a/arch/arm/boot/dts/imx7ulp.dtsi +++ b/arch/arm/boot/dts/imx7ulp.dtsi @@ -100,6 +100,29 @@ reg = <0x40000000 0x800000>; ranges; + crypto: crypto@40240000 { + compatible = "fsl,sec-v4.0"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x40240000 0x10000>; + ranges = <0 0x40240000 0x10000>; + clocks = <&pcc2 IMX7ULP_CLK_CAAM>, + <&scg1 IMX7ULP_CLK_NIC1_BUS_DIV>; + clock-names = "aclk", "ipg"; + + sec_jr0: jr0@1000 { + compatible = "fsl,sec-v4.0-job-ring"; + reg = <0x1000 0x1000>; + interrupts = ; + }; + + sec_jr1: jr1@2000 { + compatible = "fsl,sec-v4.0-job-ring"; + reg = <0x2000 0x1000>; + interrupts = ; + }; + }; + lpuart4: serial@402d0000 { compatible = "fsl,imx7ulp-lpuart"; reg = <0x402d0000 0x1000>; From 1793d1aba1941573ba9ebf75dfbb11a8c2b943b1 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 6 Jun 2019 09:57:13 -0500 Subject: [PATCH 101/143] crypto: qat - use struct_size() helper One of the more common cases of allocation size calculations is finding the size of a structure that has a zero-sized array at the end, along with memory for some number of elements for that array. For example: struct qat_alg_buf_list { ... struct qat_alg_buf bufers[]; } __packed __aligned(64); Make use of the struct_size() helper instead of an open-coded version in order to avoid any potential type mistakes. So, replace the following form: sizeof(struct qat_alg_buf_list) + ((1 + n) * sizeof(struct qat_alg_buf)) with: struct_size(bufl, bufers, n + 1) This code was detected with the help of Coccinelle. Signed-off-by: Gustavo A. R. Silva Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 2842b2cdaa90..b50eb55f8f57 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -717,8 +717,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, dma_addr_t blp; dma_addr_t bloutp = 0; struct scatterlist *sg; - size_t sz_out, sz = sizeof(struct qat_alg_buf_list) + - ((1 + n) * sizeof(struct qat_alg_buf)); + size_t sz_out, sz = struct_size(bufl, bufers, n + 1); if (unlikely(!n)) return -EINVAL; @@ -755,8 +754,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, struct qat_alg_buf *bufers; n = sg_nents(sglout); - sz_out = sizeof(struct qat_alg_buf_list) + - ((1 + n) * sizeof(struct qat_alg_buf)); + sz_out = struct_size(buflout, bufers, n + 1); sg_nctr = 0; buflout = kzalloc_node(sz_out, GFP_ATOMIC, dev_to_node(&GET_DEV(inst->accel_dev))); From b0200065cd6ee5d461e4c0af01fc3887bd755370 Mon Sep 17 00:00:00 2001 From: Nishad Kamdar Date: Thu, 6 Jun 2019 20:36:17 +0530 Subject: [PATCH 102/143] crypto: hisilicon - Use the correct style for SPDX License Identifier This patch corrects the SPDX License Identifier style in header file related to Crypto Drivers for Hisilicon SEC Engine in Hip06 and Hip07. For C header files Documentation/process/license-rules.rst mandates C-like comments (opposed to C source files where C++ style should be used) Changes made by using a script provided by Joe Perches here: https://lkml.org/lkml/2019/2/7/46 Suggested-by: Joe Perches Signed-off-by: Nishad Kamdar Acked-by: Jonathan Cameron Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec/sec_drv.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/sec/sec_drv.h b/drivers/crypto/hisilicon/sec/sec_drv.h index 2d2f186674ba..4d9063a8b10b 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.h +++ b/drivers/crypto/hisilicon/sec/sec_drv.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2016-2017 Hisilicon Limited. */ #ifndef _SEC_DRV_H_ From cec1caaf3f62f8ed104e24c8b4b9c5dc0fd44e17 Mon Sep 17 00:00:00 2001 From: Fabio Estevam Date: Thu, 6 Jun 2019 13:13:48 -0300 Subject: [PATCH 103/143] crypto: mxs-dcp - Use devm_platform_ioremap_resource() Use devm_platform_ioremap_resource() to simplify the code a bit. Signed-off-by: Fabio Estevam Signed-off-by: Herbert Xu --- drivers/crypto/mxs-dcp.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index b4429891e368..c385a746d1f4 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -992,8 +992,6 @@ static int mxs_dcp_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct dcp *sdcp = NULL; int i, ret; - - struct resource *iores; int dcp_vmi_irq, dcp_irq; if (global_sdcp) { @@ -1001,7 +999,6 @@ static int mxs_dcp_probe(struct platform_device *pdev) return -ENODEV; } - iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); dcp_vmi_irq = platform_get_irq(pdev, 0); if (dcp_vmi_irq < 0) { dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_vmi_irq); @@ -1019,7 +1016,7 @@ static int mxs_dcp_probe(struct platform_device *pdev) return -ENOMEM; sdcp->dev = dev; - sdcp->base = devm_ioremap_resource(dev, iores); + sdcp->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sdcp->base)) return PTR_ERR(sdcp->base); From b0d765219fb3308638754e28f71377d3752f1266 Mon Sep 17 00:00:00 2001 From: Fabio Estevam Date: Thu, 6 Jun 2019 13:13:49 -0300 Subject: [PATCH 104/143] crypto: sahara - Use devm_platform_ioremap_resource() Use devm_platform_ioremap_resource() to simplify the code a bit. Signed-off-by: Fabio Estevam Signed-off-by: Herbert Xu --- drivers/crypto/sahara.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index fd11162a915e..616fdc9f1816 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -1387,7 +1387,6 @@ MODULE_DEVICE_TABLE(of, sahara_dt_ids); static int sahara_probe(struct platform_device *pdev) { struct sahara_dev *dev; - struct resource *res; u32 version; int irq; int err; @@ -1401,8 +1400,7 @@ static int sahara_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dev); /* Get the base address */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dev->regs_base = devm_ioremap_resource(&pdev->dev, res); + dev->regs_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dev->regs_base)) return PTR_ERR(dev->regs_base); From 6223949a1531f31f4191a40257e6cb37af1af75f Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Fri, 10 May 2019 10:31:10 -0700 Subject: [PATCH 105/143] dt-bindings: rng: Document BCM7211 RNG compatible string BCM7211 features a RNG200 block, document its compatible string. Signed-off-by: Florian Fainelli Reviewed-by: Rob Herring Signed-off-by: Herbert Xu --- Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt b/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt index 0014da9145af..c223e54452da 100644 --- a/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt +++ b/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt @@ -2,6 +2,7 @@ HWRNG support for the iproc-rng200 driver Required properties: - compatible : Must be one of: + "brcm,bcm7211-rng200" "brcm,bcm7278-rng200" "brcm,iproc-rng200" - reg : base address and size of control register block From 1fa6d053b2a53b1979fc12000be41646be379503 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Fri, 10 May 2019 10:31:11 -0700 Subject: [PATCH 106/143] hwrng: iproc-rng200 - Add support for 7211 BCM7211 features a RNG200 hardware random number generator block, add support for this chip by matching the chip-specific compatible string. Signed-off-by: Florian Fainelli Signed-off-by: Herbert Xu --- drivers/char/hw_random/iproc-rng200.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/char/hw_random/iproc-rng200.c b/drivers/char/hw_random/iproc-rng200.c index 8b5a20b35293..92be1c0ab99f 100644 --- a/drivers/char/hw_random/iproc-rng200.c +++ b/drivers/char/hw_random/iproc-rng200.c @@ -220,6 +220,7 @@ static int iproc_rng200_probe(struct platform_device *pdev) } static const struct of_device_id iproc_rng200_of_match[] = { + { .compatible = "brcm,bcm7211-rng200", }, { .compatible = "brcm,bcm7278-rng200", }, { .compatible = "brcm,iproc-rng200", }, {}, From 059d73eea6409873446a858dd64a5bec9bf68b70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Mon, 10 Jun 2019 16:30:58 +0300 Subject: [PATCH 107/143] crypto: caam - use len instead of nents for bulding HW S/G table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, conversion of SW S/G table into HW S/G layout relies on nents returned by sg_nents_for_len(sg, len). However this leaves the possibility of HW S/G referencing more data then needed: since buffer length in HW S/G entries is filled using sg_dma_len(sg), the last entry in HW S/G table might have a length that is bigger than needed for the crypto request. This way of S/G table conversion is fine, unless after converting a table more entries have to be appended to the HW S/G table. In this case, crypto engine would access data from the S/G entry having the incorrect length, instead of advancing in the S/G table. This situation doesn't exist, but the upcoming implementation of IV update for skcipher algorithms needs to add a S/G entry after req->dst S/G (corresponding to output IV). Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 35 +++++++++--------- drivers/crypto/caam/caamalg_qi.c | 36 +++++++++---------- drivers/crypto/caam/caamalg_qi2.c | 60 +++++++++++++++---------------- drivers/crypto/caam/caamhash.c | 15 ++++---- drivers/crypto/caam/caampkc.c | 4 +-- drivers/crypto/caam/sg_sw_qm.h | 18 ++++++---- drivers/crypto/caam/sg_sw_qm2.h | 18 ++++++---- drivers/crypto/caam/sg_sw_sec4.h | 18 ++++++---- 8 files changed, 105 insertions(+), 99 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 4b03c967009b..95ef70bffd12 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1284,37 +1284,36 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; struct aead_edesc *edesc; int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; unsigned int authsize = ctx->authsize; if (unlikely(req->dst != req->src)) { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen); + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen); + src_len); return ERR_PTR(src_nents); } - dst_nents = sg_nents_for_len(req->dst, req->assoclen + - req->cryptlen + - (encrypt ? authsize : - (-authsize))); + dst_nents = sg_nents_for_len(req->dst, dst_len); if (unlikely(dst_nents < 0)) { dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : (-authsize))); + dst_len); return ERR_PTR(dst_nents); } } else { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen + - (encrypt ? authsize : 0)); + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : 0)); + src_len); return ERR_PTR(src_nents); } } @@ -1386,12 +1385,12 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, sec4_sg_index = 0; if (mapped_src_nents > 1) { - sg_to_sec4_sg_last(req->src, mapped_src_nents, + sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + sec4_sg_index, 0); sec4_sg_index += mapped_src_nents; } if (mapped_dst_nents > 1) { - sg_to_sec4_sg_last(req->dst, mapped_dst_nents, + sg_to_sec4_sg_last(req->dst, dst_len, edesc->sec4_sg + sec4_sg_index, 0); } @@ -1756,11 +1755,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); } if (dst_sg_idx) - sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg + + sg_to_sec4_sg_last(req->src, req->cryptlen, edesc->sec4_sg + !!ivsize, 0); if (mapped_dst_nents > 1) { - sg_to_sec4_sg_last(req->dst, mapped_dst_nents, + sg_to_sec4_sg_last(req->dst, req->cryptlen, edesc->sec4_sg + dst_sg_idx, 0); } diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 490a2bbdbb93..cec54344787c 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -917,6 +917,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; struct aead_edesc *edesc; dma_addr_t qm_sg_dma, iv_dma = 0; int ivsize = 0; @@ -938,13 +939,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, } if (likely(req->src == req->dst)) { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen + - (encrypt ? authsize : 0)); + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : 0)); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } @@ -957,23 +958,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, return ERR_PTR(-ENOMEM); } } else { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen); + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } - dst_nents = sg_nents_for_len(req->dst, req->assoclen + - req->cryptlen + - (encrypt ? authsize : - (-authsize))); + dst_nents = sg_nents_for_len(req->dst, dst_len); if (unlikely(dst_nents < 0)) { dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : (-authsize))); + dst_len); qi_cache_free(edesc); return ERR_PTR(dst_nents); } @@ -1082,12 +1081,11 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); qm_sg_index++; } - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); qm_sg_index += mapped_src_nents; if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + - qm_sg_index, 0); + sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(qidev, qm_sg_dma)) { @@ -1340,10 +1338,10 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, edesc->drv_req.drv_ctx = drv_ctx; dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); + sg_to_qm_sg_last(req->src, req->cryptlen, sg_table + 1, 0); if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + sg_to_qm_sg_last(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 2621d657048a..1652fa26cf96 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -371,6 +371,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; struct aead_edesc *edesc; dma_addr_t qm_sg_dma, iv_dma = 0; int ivsize = 0; @@ -387,23 +388,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, } if (unlikely(req->dst != req->src)) { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen); + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(dev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } - dst_nents = sg_nents_for_len(req->dst, req->assoclen + - req->cryptlen + - (encrypt ? authsize : - (-authsize))); + dst_nents = sg_nents_for_len(req->dst, dst_len); if (unlikely(dst_nents < 0)) { dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : (-authsize))); + dst_len); qi_cache_free(edesc); return ERR_PTR(dst_nents); } @@ -434,13 +433,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, mapped_dst_nents = 0; } } else { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen + - (encrypt ? authsize : 0)); + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(dev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : 0)); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } @@ -536,12 +535,11 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); qm_sg_index++; } - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); qm_sg_index += mapped_src_nents; if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + - qm_sg_index, 0); + sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(dev, qm_sg_dma)) { @@ -1159,10 +1157,10 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) edesc->qm_sg_bytes = qm_sg_bytes; dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); + sg_to_qm_sg_last(req->src, req->cryptlen, sg_table + 1, 0); if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + sg_to_qm_sg_last(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, @@ -3422,9 +3420,9 @@ static int ahash_update_ctx(struct ahash_request *req) if (to_hash) { struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - (*next_buflen)); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(ctx->dev, "Invalid number of src SG.\n"); return src_nents; @@ -3465,7 +3463,7 @@ static int ahash_update_ctx(struct ahash_request *req) goto unmap_ctx; if (mapped_nents) { - sg_to_qm_sg_last(req->src, mapped_nents, + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_src_index, 0); if (*next_buflen) scatterwalk_map_and_copy(next_buf, req->src, @@ -3653,7 +3651,7 @@ static int ahash_finup_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0); + sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); @@ -3739,7 +3737,7 @@ static int ahash_digest(struct ahash_request *req) struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); - sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); + sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { @@ -3882,9 +3880,9 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (to_hash) { struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - *next_buflen); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(ctx->dev, "Invalid number of src SG.\n"); return src_nents; @@ -3918,7 +3916,7 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); + sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); if (*next_buflen) scatterwalk_map_and_copy(next_buf, req->src, @@ -4037,7 +4035,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req) if (ret) goto unmap; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); + sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); @@ -4107,9 +4105,9 @@ static int ahash_update_first(struct ahash_request *req) if (to_hash) { struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - (*next_buflen)); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(ctx->dev, "Invalid number of src SG.\n"); return src_nents; @@ -4144,7 +4142,7 @@ static int ahash_update_first(struct ahash_request *req) if (mapped_nents > 1) { int qm_sg_bytes; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); + sg_to_qm_sg_last(req->src, src_len, sg_table, 0); qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index a87526f62737..e4ac5d591ad6 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -729,7 +729,7 @@ static int ahash_edesc_add_src(struct caam_hash_ctx *ctx, unsigned int sgsize = sizeof(*sg) * pad_sg_nents(first_sg + nents); - sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0); + sg_to_sec4_sg_last(req->src, to_hash, sg + first_sg, 0); src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE); if (dma_mapping_error(ctx->jrdev, src_dma)) { @@ -788,9 +788,9 @@ static int ahash_update_ctx(struct ahash_request *req) if (to_hash) { int pad_nents; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - (*next_buflen)); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(jrdev, "Invalid number of src SG.\n"); return src_nents; @@ -835,7 +835,7 @@ static int ahash_update_ctx(struct ahash_request *req) goto unmap_ctx; if (mapped_nents) - sg_to_sec4_sg_last(req->src, mapped_nents, + sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + sec4_sg_src_index, 0); else @@ -1208,9 +1208,9 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (to_hash) { int pad_nents; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - *next_buflen); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(jrdev, "Invalid number of src SG.\n"); return src_nents; @@ -1250,8 +1250,7 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - sg_to_sec4_sg_last(req->src, mapped_nents, - edesc->sec4_sg + 1, 0); + sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0); if (*next_buflen) { scatterwalk_map_and_copy(next_buf, req->src, diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index 19b02c1973fc..80574106af29 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -306,11 +306,11 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, 0); if (sec4_sg_index) - sg_to_sec4_sg_last(req_ctx->fixup_src, src_nents, + sg_to_sec4_sg_last(req_ctx->fixup_src, req_ctx->fixup_src_len, edesc->sec4_sg + !!diff_size, 0); if (dst_nents > 1) - sg_to_sec4_sg_last(req->dst, dst_nents, + sg_to_sec4_sg_last(req->dst, req->dst_len, edesc->sec4_sg + sec4_sg_index, 0); /* Save nents for later use in Job Descriptor */ diff --git a/drivers/crypto/caam/sg_sw_qm.h b/drivers/crypto/caam/sg_sw_qm.h index b3e1aaaeffea..d56cc7efbc13 100644 --- a/drivers/crypto/caam/sg_sw_qm.h +++ b/drivers/crypto/caam/sg_sw_qm.h @@ -54,15 +54,19 @@ static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr, * but does not have final bit; instead, returns last entry */ static inline struct qm_sg_entry * -sg_to_qm_sg(struct scatterlist *sg, int sg_count, +sg_to_qm_sg(struct scatterlist *sg, int len, struct qm_sg_entry *qm_sg_ptr, u16 offset) { - while (sg_count && sg) { - dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), - sg_dma_len(sg), offset); + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len, + offset); qm_sg_ptr++; sg = sg_next(sg); - sg_count--; + len -= ent_len; } return qm_sg_ptr - 1; } @@ -71,10 +75,10 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count, * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped */ -static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len, struct qm_sg_entry *qm_sg_ptr, u16 offset) { - qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); + qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset); qm_sg_entry_set_f(qm_sg_ptr, qm_sg_entry_get_len(qm_sg_ptr)); } diff --git a/drivers/crypto/caam/sg_sw_qm2.h b/drivers/crypto/caam/sg_sw_qm2.h index c9378402a5f8..b8b737d2b0ea 100644 --- a/drivers/crypto/caam/sg_sw_qm2.h +++ b/drivers/crypto/caam/sg_sw_qm2.h @@ -25,15 +25,19 @@ static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr, * but does not have final bit; instead, returns last entry */ static inline struct dpaa2_sg_entry * -sg_to_qm_sg(struct scatterlist *sg, int sg_count, +sg_to_qm_sg(struct scatterlist *sg, int len, struct dpaa2_sg_entry *qm_sg_ptr, u16 offset) { - while (sg_count && sg) { - dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), - sg_dma_len(sg), offset); + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len, + offset); qm_sg_ptr++; sg = sg_next(sg); - sg_count--; + len -= ent_len; } return qm_sg_ptr - 1; } @@ -42,11 +46,11 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count, * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped */ -static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len, struct dpaa2_sg_entry *qm_sg_ptr, u16 offset) { - qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); + qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset); dpaa2_sg_set_final(qm_sg_ptr, true); } diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index 8f9555d01011..07e1ee99273b 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -45,15 +45,19 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, * but does not have final bit; instead, returns last entry */ static inline struct sec4_sg_entry * -sg_to_sec4_sg(struct scatterlist *sg, int sg_count, +sg_to_sec4_sg(struct scatterlist *sg, int len, struct sec4_sg_entry *sec4_sg_ptr, u16 offset) { - while (sg_count) { - dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), - sg_dma_len(sg), offset); + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), ent_len, + offset); sec4_sg_ptr++; sg = sg_next(sg); - sg_count--; + len -= ent_len; } return sec4_sg_ptr - 1; } @@ -70,11 +74,11 @@ static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr) * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped */ -static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count, +static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int len, struct sec4_sg_entry *sec4_sg_ptr, u16 offset) { - sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset); + sec4_sg_ptr = sg_to_sec4_sg(sg, len, sec4_sg_ptr, offset); sg_to_sec4_set_last(sec4_sg_ptr); } From 334d37c9e26364511f0673ef155e2ad207316e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Mon, 10 Jun 2019 16:30:59 +0300 Subject: [PATCH 108/143] crypto: caam - update IV using HW support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modify drivers to perform skcipher IV update using the crypto engine, instead of performing the operation in SW. Besides being more efficient, this also fixes IV update for CTR mode. Output HW S/G table is appended with an entry pointing to the same IV buffer used as input (which is now mapped BIDIRECTIONAL). AS (Algorithm State) parameter of the OPERATION command is changed from INIFINAL to INIT in descriptors used by ctr(aes), cbc(aes). This is needed since in case FINAL bit is set, HW skips IV updating in the Context Register for the last data block. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 88 +++++++++++++++------------ drivers/crypto/caam/caamalg_desc.c | 31 ++++++++-- drivers/crypto/caam/caamalg_desc.h | 4 +- drivers/crypto/caam/caamalg_qi.c | 90 ++++++++++++---------------- drivers/crypto/caam/caamalg_qi2.c | 96 ++++++++++++++---------------- 5 files changed, 163 insertions(+), 146 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 95ef70bffd12..43f18253e5b6 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -898,7 +898,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src, } if (iv_dma) - dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL); if (sec4_sg_bytes) dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, DMA_TO_DEVICE); @@ -977,7 +977,6 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, struct skcipher_request *req = context; struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); - struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); int ivsize = crypto_skcipher_ivsize(skcipher); dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); @@ -991,16 +990,17 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block when running in CBC mode. + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. */ - if ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CBC) - scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - - ivsize, ivsize, 0); + if (ivsize) { + memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, + ivsize); - if (ivsize) print_hex_dump_debug("dstiv @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv, edesc->src_nents > 1 ? 100 : ivsize, 1); + } caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, @@ -1027,8 +1027,20 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, skcipher_unmap(jrdev, edesc, req); - print_hex_dump_debug("dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); + /* + * The crypto API expects us to set the IV (req->iv) to the last + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. + */ + if (ivsize) { + memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, + ivsize); + + print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + ivsize, 1); + } + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); @@ -1260,7 +1272,7 @@ static void init_skcipher_job(struct skcipher_request *req, if (likely(req->src == req->dst)) { dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); out_options = in_options; - } else if (edesc->mapped_dst_nents == 1) { + } else if (!ivsize && edesc->mapped_dst_nents == 1) { dst_dma = sg_dma_address(req->dst); } else { dst_dma = edesc->sec4_sg_dma + sec4_sg_index * @@ -1268,7 +1280,7 @@ static void init_skcipher_job(struct skcipher_request *req, out_options = LDST_SGF; } - append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options); + append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); } /* @@ -1699,22 +1711,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, dst_sg_idx = sec4_sg_ents; /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * * HW reads 4 S/G entries at a time; make sure the reads don't go beyond * the end of the table by allocating more S/G entries. Logic: - * if (src != dst && output S/G) + * if (output S/G) * pad output S/G, if needed - * else if (src == dst && S/G) - * overlapping S/Gs; pad one of them * else if (input S/G) ... * pad input S/G, if needed */ - if (mapped_dst_nents > 1) - sec4_sg_ents += pad_sg_nents(mapped_dst_nents); - else if ((req->src == req->dst) && (mapped_src_nents > 1)) - sec4_sg_ents = max(pad_sg_nents(sec4_sg_ents), - !!ivsize + pad_sg_nents(mapped_src_nents)); - else + if (ivsize || mapped_dst_nents > 1) { + if (req->src == req->dst) + sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents); + else + sec4_sg_ents += pad_sg_nents(mapped_dst_nents + + !!ivsize); + } else { sec4_sg_ents = pad_sg_nents(sec4_sg_ents); + } sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); @@ -1740,10 +1756,10 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, /* Make sure IV is located in a DMAable area */ if (ivsize) { - iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes; + iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes; memcpy(iv, req->iv, ivsize); - iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE); + iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(jrdev, iv_dma)) { dev_err(jrdev, "unable to map IV\n"); caam_unmap(jrdev, req->src, req->dst, src_nents, @@ -1755,13 +1771,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); } if (dst_sg_idx) - sg_to_sec4_sg_last(req->src, req->cryptlen, edesc->sec4_sg + - !!ivsize, 0); + sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg + + !!ivsize, 0); - if (mapped_dst_nents > 1) { - sg_to_sec4_sg_last(req->dst, req->cryptlen, - edesc->sec4_sg + dst_sg_idx, 0); - } + if (req->src != req->dst && (ivsize || mapped_dst_nents > 1)) + sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg + + dst_sg_idx, 0); + + if (ivsize) + dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx + + mapped_dst_nents, iv_dma, ivsize, 0); + + if (ivsize || mapped_dst_nents > 1) + sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx + + mapped_dst_nents); if (sec4_sg_bytes) { edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, @@ -1824,7 +1847,6 @@ static int skcipher_decrypt(struct skcipher_request *req) struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); - int ivsize = crypto_skcipher_ivsize(skcipher); struct device *jrdev = ctx->jrdev; u32 *desc; int ret = 0; @@ -1834,14 +1856,6 @@ static int skcipher_decrypt(struct skcipher_request *req) if (IS_ERR(edesc)) return PTR_ERR(edesc); - /* - * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block when running in CBC mode. - */ - if ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CBC) - scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - - ivsize, ivsize, 0); - /* Create and submit job descriptor*/ init_skcipher_job(req, edesc, false); desc = edesc->hw_desc; diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index a73b79c5d46c..72531837571e 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -33,12 +33,11 @@ static inline void append_dec_op1(u32 *desc, u32 type) } jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); - append_operation(desc, type | OP_ALG_AS_INITFINAL | - OP_ALG_DECRYPT); + append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT); uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); set_jump_tgt_here(desc, jump_cmd); - append_operation(desc, type | OP_ALG_AS_INITFINAL | - OP_ALG_DECRYPT | OP_ALG_AAI_DK); + append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT | + OP_ALG_AAI_DK); set_jump_tgt_here(desc, uncond_jump_cmd); } @@ -1392,12 +1391,18 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, LDST_OFFSET_SHIFT)); /* Load operation */ - append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + append_operation(desc, cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT); /* Perform operation */ skcipher_append_src_dst(desc); + /* Store IV */ + if (ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << + LDST_OFFSET_SHIFT)); + print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); @@ -1459,7 +1464,7 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, /* Choose operation */ if (ctx1_iv_off) - append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + append_operation(desc, cdata->algtype | OP_ALG_AS_INIT | OP_ALG_DECRYPT); else append_dec_op1(desc, cdata->algtype); @@ -1467,6 +1472,12 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, /* Perform operation */ skcipher_append_src_dst(desc); + /* Store IV */ + if (ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << + LDST_OFFSET_SHIFT)); + print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); @@ -1516,6 +1527,10 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) /* Perform operation */ skcipher_append_src_dst(desc); + /* Store upper 8B of IV */ + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__) ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); @@ -1564,6 +1579,10 @@ void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata) /* Perform operation */ skcipher_append_src_dst(desc); + /* Store upper 8B of IV */ + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__) ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h index d5ca42ff961a..da4a4ee60c80 100644 --- a/drivers/crypto/caam/caamalg_desc.h +++ b/drivers/crypto/caam/caamalg_desc.h @@ -44,9 +44,9 @@ #define DESC_SKCIPHER_BASE (3 * CAAM_CMD_SZ) #define DESC_SKCIPHER_ENC_LEN (DESC_SKCIPHER_BASE + \ - 20 * CAAM_CMD_SZ) + 21 * CAAM_CMD_SZ) #define DESC_SKCIPHER_DEC_LEN (DESC_SKCIPHER_BASE + \ - 15 * CAAM_CMD_SZ) + 16 * CAAM_CMD_SZ) void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, unsigned int icvsize, int era); diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index cec54344787c..32f0f8a72067 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -831,7 +831,8 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, static void caam_unmap(struct device *dev, struct scatterlist *src, struct scatterlist *dst, int src_nents, int dst_nents, dma_addr_t iv_dma, int ivsize, - dma_addr_t qm_sg_dma, int qm_sg_bytes) + enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, + int qm_sg_bytes) { if (dst != src) { if (src_nents) @@ -843,7 +844,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src, } if (iv_dma) - dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dma_unmap_single(dev, iv_dma, ivsize, iv_dir); if (qm_sg_bytes) dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); } @@ -856,7 +857,8 @@ static void aead_unmap(struct device *dev, int ivsize = crypto_aead_ivsize(aead); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, + edesc->qm_sg_bytes); dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); } @@ -867,7 +869,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, int ivsize = crypto_skcipher_ivsize(skcipher); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, + edesc->qm_sg_bytes); } static void aead_done(struct caam_drv_req *drv_req, u32 status) @@ -1036,7 +1039,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_ents, ivsize); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1051,7 +1054,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(qidev, iv_dma)) { dev_err(qidev, "unable to map IV\n"); caam_unmap(qidev, req->src, req->dst, src_nents, - dst_nents, 0, 0, 0, 0); + dst_nents, 0, 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1070,7 +1073,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(qidev, edesc->assoclen_dma)) { dev_err(qidev, "unable to map assoclen\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1092,7 +1095,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dev_err(qidev, "unable to map S/G table\n"); dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1206,11 +1209,10 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status) /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. This is used e.g. by the CTS mode. + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. */ - if (edesc->drv_req.drv_ctx->op_type == ENCRYPT) - scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - - ivsize, ivsize, 0); + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize); qi_cache_free(edesc); skcipher_request_complete(req, status); @@ -1279,22 +1281,17 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, dst_sg_idx = qm_sg_ents; /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * * HW reads 4 S/G entries at a time; make sure the reads don't go beyond - * the end of the table by allocating more S/G entries. Logic: - * if (src != dst && output S/G) - * pad output S/G, if needed - * else if (src == dst && S/G) - * overlapping S/Gs; pad one of them - * else if (input S/G) ... - * pad input S/G, if needed + * the end of the table by allocating more S/G entries. */ - if (mapped_dst_nents > 1) - qm_sg_ents += pad_sg_nents(mapped_dst_nents); - else if ((req->src == req->dst) && (mapped_src_nents > 1)) - qm_sg_ents = max(pad_sg_nents(qm_sg_ents), - 1 + pad_sg_nents(mapped_src_nents)); + if (req->src != req->dst) + qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); else - qm_sg_ents = pad_sg_nents(qm_sg_ents); + qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + @@ -1302,7 +1299,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_ents, ivsize); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1311,7 +1308,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, if (unlikely(!edesc)) { dev_err(qidev, "could not allocate extended descriptor\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1320,11 +1317,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, iv = (u8 *)(sg_table + qm_sg_ents); memcpy(iv, req->iv, ivsize); - iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); + iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(qidev, iv_dma)) { dev_err(qidev, "unable to map IV\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1338,18 +1335,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, edesc->drv_req.drv_ctx = drv_ctx; dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); - sg_to_qm_sg_last(req->src, req->cryptlen, sg_table + 1, 0); + sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); - if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, req->cryptlen, sg_table + - dst_sg_idx, 0); + if (req->src != req->dst) + sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); + + dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, + ivsize, 0); edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { dev_err(qidev, "unable to map S/G table\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1359,16 +1358,14 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, ivsize + req->cryptlen, 0); - if (req->src == req->dst) { + if (req->src == req->dst) dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + - sizeof(*sg_table), req->cryptlen, 0); - } else if (mapped_dst_nents > 1) { + sizeof(*sg_table), req->cryptlen + ivsize, + 0); + else dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * - sizeof(*sg_table), req->cryptlen, 0); - } else { - dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), - req->cryptlen, 0); - } + sizeof(*sg_table), req->cryptlen + ivsize, + 0); return edesc; } @@ -1378,7 +1375,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); - int ivsize = crypto_skcipher_ivsize(skcipher); int ret; if (unlikely(caam_congested)) @@ -1389,14 +1385,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) if (IS_ERR(edesc)) return PTR_ERR(edesc); - /* - * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. - */ - if (!encrypt) - scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - - ivsize, ivsize, 0); - ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); if (!ret) { ret = -EINPROGRESS; diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 1652fa26cf96..06bf32c32cbd 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -140,7 +140,8 @@ static struct caam_request *to_caam_req(struct crypto_async_request *areq) static void caam_unmap(struct device *dev, struct scatterlist *src, struct scatterlist *dst, int src_nents, int dst_nents, dma_addr_t iv_dma, int ivsize, - dma_addr_t qm_sg_dma, int qm_sg_bytes) + enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, + int qm_sg_bytes) { if (dst != src) { if (src_nents) @@ -152,7 +153,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src, } if (iv_dma) - dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dma_unmap_single(dev, iv_dma, ivsize, iv_dir); if (qm_sg_bytes) dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); @@ -485,7 +486,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_nents, ivsize); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -500,7 +501,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(dev, iv_dma)) { dev_err(dev, "unable to map IV\n"); caam_unmap(dev, req->src, req->dst, src_nents, - dst_nents, 0, 0, 0, 0); + dst_nents, 0, 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -524,7 +525,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(dev, edesc->assoclen_dma)) { dev_err(dev, "unable to map assoclen\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -546,7 +547,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dev_err(dev, "unable to map S/G table\n"); dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1101,22 +1102,17 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) dst_sg_idx = qm_sg_ents; /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * * HW reads 4 S/G entries at a time; make sure the reads don't go beyond - * the end of the table by allocating more S/G entries. Logic: - * if (src != dst && output S/G) - * pad output S/G, if needed - * else if (src == dst && S/G) - * overlapping S/Gs; pad one of them - * else if (input S/G) ... - * pad input S/G, if needed + * the end of the table by allocating more S/G entries. */ - if (mapped_dst_nents > 1) - qm_sg_ents += pad_sg_nents(mapped_dst_nents); - else if ((req->src == req->dst) && (mapped_src_nents > 1)) - qm_sg_ents = max(pad_sg_nents(qm_sg_ents), - 1 + pad_sg_nents(mapped_src_nents)); + if (req->src != req->dst) + qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); else - qm_sg_ents = pad_sg_nents(qm_sg_ents); + qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + @@ -1124,7 +1120,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_ents, ivsize); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1133,7 +1129,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) if (unlikely(!edesc)) { dev_err(dev, "could not allocate extended descriptor\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1142,11 +1138,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) iv = (u8 *)(sg_table + qm_sg_ents); memcpy(iv, req->iv, ivsize); - iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); + iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, iv_dma)) { dev_err(dev, "unable to map IV\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1157,18 +1153,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) edesc->qm_sg_bytes = qm_sg_bytes; dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); - sg_to_qm_sg_last(req->src, req->cryptlen, sg_table + 1, 0); + sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); - if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, req->cryptlen, sg_table + - dst_sg_idx, 0); + if (req->src != req->dst) + sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); + + dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, + ivsize, 0); edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(dev, edesc->qm_sg_dma)) { dev_err(dev, "unable to map S/G table\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1176,23 +1174,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); dpaa2_fl_set_final(in_fle, true); dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); - dpaa2_fl_set_len(out_fle, req->cryptlen); + dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); - if (req->src == req->dst) { - dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + + if (req->src == req->dst) dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + sizeof(*sg_table)); - } else if (mapped_dst_nents > 1) { - dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + else dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * sizeof(*sg_table)); - } else { - dpaa2_fl_set_format(out_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); - } return edesc; } @@ -1204,7 +1198,8 @@ static void aead_unmap(struct device *dev, struct aead_edesc *edesc, int ivsize = crypto_aead_ivsize(aead); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, + edesc->qm_sg_bytes); dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); } @@ -1215,7 +1210,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, int ivsize = crypto_skcipher_ivsize(skcipher); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, + edesc->qm_sg_bytes); } static void aead_encrypt_done(void *cbk_ctx, u32 status) @@ -1372,10 +1368,10 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status) /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. This is used e.g. by the CTS mode. + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. */ - scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize, - ivsize, 0); + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize); qi_cache_free(edesc); skcipher_request_complete(req, ecode); @@ -1407,6 +1403,14 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status) edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); skcipher_unmap(ctx->dev, edesc, req); + + /* + * The crypto API expects us to set the IV (req->iv) to the last + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. + */ + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize); + qi_cache_free(edesc); skcipher_request_complete(req, ecode); } @@ -1445,7 +1449,6 @@ static int skcipher_decrypt(struct skcipher_request *req) struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct caam_request *caam_req = skcipher_request_ctx(req); - int ivsize = crypto_skcipher_ivsize(skcipher); int ret; /* allocate extended descriptor */ @@ -1453,13 +1456,6 @@ static int skcipher_decrypt(struct skcipher_request *req) if (IS_ERR(edesc)) return PTR_ERR(edesc); - /* - * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. - */ - scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize, - ivsize, 0); - caam_req->flc = &ctx->flc[DECRYPT]; caam_req->flc_dma = ctx->flc_dma[DECRYPT]; caam_req->cbk = skcipher_decrypt_done; From f651bd96b20127d1ebc449f01aaaa79277f0574b Mon Sep 17 00:00:00 2001 From: Daniel Axtens Date: Tue, 11 Jun 2019 11:54:31 +1000 Subject: [PATCH 109/143] crypto: vmx - Document CTR mode counter width quirks The CTR code comes from OpenSSL, where it does a 32-bit counter. The kernel has a 128-bit counter. This difference has lead to issues. Document it. Signed-off-by: Daniel Axtens Signed-off-by: Herbert Xu --- drivers/crypto/vmx/aesp8-ppc.pl | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl index 9c6b5c1d6a1a..db874367b602 100644 --- a/drivers/crypto/vmx/aesp8-ppc.pl +++ b/drivers/crypto/vmx/aesp8-ppc.pl @@ -1286,6 +1286,24 @@ ___ ######################################################################### {{{ # CTR procedure[s] # + +####################### WARNING: Here be dragons! ####################### +# +# This code is written as 'ctr32', based on a 32-bit counter used +# upstream. The kernel does *not* use a 32-bit counter. The kernel uses +# a 128-bit counter. +# +# This leads to subtle changes from the upstream code: the counter +# is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in +# both the bulk (8 blocks at a time) path, and in the individual block +# path. Be aware of this when doing updates. +# +# See: +# 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug") +# 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword") +# https://github.com/openssl/openssl/pull/8942 +# +######################################################################### my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10)); my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3)); my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)= @@ -1357,7 +1375,7 @@ Loop_ctr32_enc: addi $idx,$idx,16 bdnz Loop_ctr32_enc - vadduqm $ivec,$ivec,$one + vadduqm $ivec,$ivec,$one # Kernel change for 128-bit vmr $dat,$inptail lvx $inptail,0,$inp addi $inp,$inp,16 @@ -1501,7 +1519,7 @@ Load_ctr32_enc_key: $SHL $len,$len,4 vadduqm $out1,$ivec,$one # counter values ... - vadduqm $out2,$ivec,$two + vadduqm $out2,$ivec,$two # (do all ctr adds as 128-bit) vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0] le?li $idx,8 vadduqm $out3,$out1,$two From 192125ed5ce62afba24312d8e7a0314577565b4a Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 12 Jun 2019 05:49:50 +0000 Subject: [PATCH 110/143] crypto: talitos - fix max key size for sha384 and sha512 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Below commit came with a typo in the CONFIG_ symbol, leading to a permanently reduced max key size regarless of the driver capabilities. Reported-by: Horia Geantă Fixes: b8fbdc2bc4e7 ("crypto: talitos - reduce max key size for SEC1") Signed-off-by: Christophe Leroy Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 32a7e747dc5f..c865f5d5eaba 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -823,7 +823,7 @@ static void talitos_unregister_rng(struct device *dev) * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP */ #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) -#ifdef CONFIG_CRYPTO_DEV_TALITOS_SEC2 +#ifdef CONFIG_CRYPTO_DEV_TALITOS2 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) #else #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) From dc51f25752bfcb5f1edbac1ca4ce16af7b3bd507 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 12 Jun 2019 18:19:53 +0200 Subject: [PATCH 111/143] crypto: arc4 - refactor arc4 core code into separate library Refactor the core rc4 handling so we can move most users to a library interface, permitting us to drop the cipher interface entirely in a future patch. This is part of an effort to simplify the crypto API and improve its robustness against incorrect use. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- MAINTAINERS | 1 + crypto/Kconfig | 4 +++ crypto/arc4.c | 60 +---------------------------------- include/crypto/arc4.h | 10 ++++++ lib/Makefile | 2 +- lib/crypto/Makefile | 4 +++ lib/crypto/arc4.c | 74 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 95 insertions(+), 60 deletions(-) create mode 100644 lib/crypto/Makefile create mode 100644 lib/crypto/arc4.c diff --git a/MAINTAINERS b/MAINTAINERS index 5cfbea4ce575..8d579935ffb9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4229,6 +4229,7 @@ F: crypto/ F: drivers/crypto/ F: include/crypto/ F: include/linux/crypto* +F: lib/crypto/ CRYPTOGRAPHIC RANDOM NUMBER GENERATOR M: Neil Horman diff --git a/crypto/Kconfig b/crypto/Kconfig index 8c588ed36253..e801450bcb1c 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1237,9 +1237,13 @@ config CRYPTO_ANUBIS +config CRYPTO_LIB_ARC4 + tristate + config CRYPTO_ARC4 tristate "ARC4 cipher algorithm" select CRYPTO_BLKCIPHER + select CRYPTO_LIB_ARC4 help ARC4 cipher algorithm. diff --git a/crypto/arc4.c b/crypto/arc4.c index b78dcb390a7e..d303b7ff1fed 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -18,33 +18,12 @@ #include #include -struct arc4_ctx { - u32 S[256]; - u32 x, y; -}; - static int arc4_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) { struct arc4_ctx *ctx = crypto_tfm_ctx(tfm); - int i, j = 0, k = 0; - ctx->x = 1; - ctx->y = 0; - - for (i = 0; i < 256; i++) - ctx->S[i] = i; - - for (i = 0; i < 256; i++) { - u32 a = ctx->S[i]; - j = (j + in_key[k] + a) & 0xff; - ctx->S[i] = ctx->S[j]; - ctx->S[j] = a; - if (++k >= key_len) - k = 0; - } - - return 0; + return arc4_setkey(ctx, in_key, key_len); } static int arc4_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key, @@ -53,43 +32,6 @@ static int arc4_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key, return arc4_set_key(&tfm->base, in_key, key_len); } -static void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, - unsigned int len) -{ - u32 *const S = ctx->S; - u32 x, y, a, b; - u32 ty, ta, tb; - - if (len == 0) - return; - - x = ctx->x; - y = ctx->y; - - a = S[x]; - y = (y + a) & 0xff; - b = S[y]; - - do { - S[y] = a; - a = (a + b) & 0xff; - S[x] = b; - x = (x + 1) & 0xff; - ta = S[x]; - ty = (y + ta) & 0xff; - tb = S[ty]; - *out++ = *in++ ^ S[a]; - if (--len == 0) - break; - y = ty; - a = ta; - b = tb; - } while (true); - - ctx->x = x; - ctx->y = y; -} - static void arc4_crypt_one(struct crypto_tfm *tfm, u8 *out, const u8 *in) { arc4_crypt(crypto_tfm_ctx(tfm), out, in, 1); diff --git a/include/crypto/arc4.h b/include/crypto/arc4.h index 5b2c24ab0139..f3c22fe01704 100644 --- a/include/crypto/arc4.h +++ b/include/crypto/arc4.h @@ -6,8 +6,18 @@ #ifndef _CRYPTO_ARC4_H #define _CRYPTO_ARC4_H +#include + #define ARC4_MIN_KEY_SIZE 1 #define ARC4_MAX_KEY_SIZE 256 #define ARC4_BLOCK_SIZE 1 +struct arc4_ctx { + u32 S[256]; + u32 x, y; +}; + +int arc4_setkey(struct arc4_ctx *ctx, const u8 *in_key, unsigned int key_len); +void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, unsigned int len); + #endif /* _CRYPTO_ARC4_H */ diff --git a/lib/Makefile b/lib/Makefile index fb7697031a79..d3daedf93c5a 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -102,7 +102,7 @@ endif obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any) -obj-y += math/ +obj-y += math/ crypto/ obj-$(CONFIG_GENERIC_IOMAP) += iomap.o obj-$(CONFIG_GENERIC_PCI_IOMAP) += pci_iomap.o diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile new file mode 100644 index 000000000000..88195c34932d --- /dev/null +++ b/lib/crypto/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_CRYPTO_LIB_ARC4) += libarc4.o +libarc4-y := arc4.o diff --git a/lib/crypto/arc4.c b/lib/crypto/arc4.c new file mode 100644 index 000000000000..c2020f19c652 --- /dev/null +++ b/lib/crypto/arc4.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Cryptographic API + * + * ARC4 Cipher Algorithm + * + * Jon Oberheide + */ + +#include +#include + +int arc4_setkey(struct arc4_ctx *ctx, const u8 *in_key, unsigned int key_len) +{ + int i, j = 0, k = 0; + + ctx->x = 1; + ctx->y = 0; + + for (i = 0; i < 256; i++) + ctx->S[i] = i; + + for (i = 0; i < 256; i++) { + u32 a = ctx->S[i]; + + j = (j + in_key[k] + a) & 0xff; + ctx->S[i] = ctx->S[j]; + ctx->S[j] = a; + if (++k >= key_len) + k = 0; + } + + return 0; +} +EXPORT_SYMBOL(arc4_setkey); + +void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, unsigned int len) +{ + u32 *const S = ctx->S; + u32 x, y, a, b; + u32 ty, ta, tb; + + if (len == 0) + return; + + x = ctx->x; + y = ctx->y; + + a = S[x]; + y = (y + a) & 0xff; + b = S[y]; + + do { + S[y] = a; + a = (a + b) & 0xff; + S[x] = b; + x = (x + 1) & 0xff; + ta = S[x]; + ty = (y + ta) & 0xff; + tb = S[ty]; + *out++ = *in++ ^ S[a]; + if (--len == 0) + break; + y = ty; + a = ta; + b = tb; + } while (true); + + ctx->x = x; + ctx->y = y; +} +EXPORT_SYMBOL(arc4_crypt); + +MODULE_LICENSE("GPL"); From 5fdb373570b2119abf00d909a277ebf4ea2c078f Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 12 Jun 2019 18:19:54 +0200 Subject: [PATCH 112/143] net/mac80211: move WEP handling to ARC4 library interface The WEP code in the mac80211 subsystem currently uses the crypto API to access the arc4 (RC4) cipher, which is overly complicated, and doesn't really have an upside in this particular case, since ciphers are always synchronous and therefore always implemented in software. Given that we have no accelerated software implementations either, it is much more straightforward to invoke a generic library interface directly. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- net/mac80211/Kconfig | 2 +- net/mac80211/cfg.c | 4 ++-- net/mac80211/ieee80211_i.h | 4 ++-- net/mac80211/key.h | 1 + net/mac80211/main.c | 6 ++--- net/mac80211/mlme.c | 3 ++- net/mac80211/tkip.c | 8 +++---- net/mac80211/tkip.h | 4 ++-- net/mac80211/wep.c | 49 ++++++++------------------------------ net/mac80211/wep.h | 5 ++-- net/mac80211/wpa.c | 4 ++-- 11 files changed, 30 insertions(+), 60 deletions(-) diff --git a/net/mac80211/Kconfig b/net/mac80211/Kconfig index be471fe95048..8e0b0b8d5e41 100644 --- a/net/mac80211/Kconfig +++ b/net/mac80211/Kconfig @@ -2,7 +2,7 @@ config MAC80211 tristate "Generic IEEE 802.11 Networking Stack (mac80211)" depends on CFG80211 select CRYPTO - select CRYPTO_ARC4 + select CRYPTO_LIB_ARC4 select CRYPTO_AES select CRYPTO_CCM select CRYPTO_GCM diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index 52e6a091b7e4..83c6d2844427 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include "ieee80211_i.h" @@ -403,9 +404,8 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev, case WLAN_CIPHER_SUITE_WEP40: case WLAN_CIPHER_SUITE_TKIP: case WLAN_CIPHER_SUITE_WEP104: - if (IS_ERR(local->wep_tx_tfm)) + if (WARN_ON_ONCE(fips_enabled)) return -EINVAL; - break; case WLAN_CIPHER_SUITE_CCMP: case WLAN_CIPHER_SUITE_CCMP_256: case WLAN_CIPHER_SUITE_AES_CMAC: diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index 073a8235ae1b..412da8cfbc36 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -1258,8 +1258,8 @@ struct ieee80211_local { struct rate_control_ref *rate_ctrl; - struct crypto_cipher *wep_tx_tfm; - struct crypto_cipher *wep_rx_tfm; + struct arc4_ctx wep_tx_ctx; + struct arc4_ctx wep_rx_ctx; u32 wep_iv; /* see iface.c */ diff --git a/net/mac80211/key.h b/net/mac80211/key.h index f06fbd03d235..6c5bbaebd02c 100644 --- a/net/mac80211/key.h +++ b/net/mac80211/key.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #define NUM_DEFAULT_KEYS 4 diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 2b608044ae23..93c4a2d0623e 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -733,8 +734,7 @@ EXPORT_SYMBOL(ieee80211_alloc_hw_nm); static int ieee80211_init_cipher_suites(struct ieee80211_local *local) { - bool have_wep = !(IS_ERR(local->wep_tx_tfm) || - IS_ERR(local->wep_rx_tfm)); + bool have_wep = !fips_enabled; /* FIPS does not permit the use of RC4 */ bool have_mfp = ieee80211_hw_check(&local->hw, MFP_CAPABLE); int n_suites = 0, r = 0, w = 0; u32 *suites; @@ -1301,7 +1301,6 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) fail_rate: rtnl_unlock(); ieee80211_led_exit(local); - ieee80211_wep_free(local); fail_flows: destroy_workqueue(local->workqueue); fail_workqueue: @@ -1358,7 +1357,6 @@ void ieee80211_unregister_hw(struct ieee80211_hw *hw) destroy_workqueue(local->workqueue); wiphy_unregister(local->hw.wiphy); - ieee80211_wep_free(local); ieee80211_led_exit(local); kfree(local->int_scan_req); } diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index b7a9fe3d5fcb..048a07b101b4 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -15,6 +15,7 @@ */ #include +#include #include #include #include @@ -5038,7 +5039,7 @@ int ieee80211_mgd_auth(struct ieee80211_sub_if_data *sdata, auth_alg = WLAN_AUTH_OPEN; break; case NL80211_AUTHTYPE_SHARED_KEY: - if (IS_ERR(local->wep_tx_tfm)) + if (fips_enabled) return -EOPNOTSUPP; auth_alg = WLAN_AUTH_SHARED_KEY; break; diff --git a/net/mac80211/tkip.c b/net/mac80211/tkip.c index b3622823bad2..96b87fc7122e 100644 --- a/net/mac80211/tkip.c +++ b/net/mac80211/tkip.c @@ -222,7 +222,7 @@ EXPORT_SYMBOL(ieee80211_get_tkip_p2k); * @payload_len is the length of payload (_not_ including IV/ICV length). * @ta is the transmitter addresses. */ -int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm, +int ieee80211_tkip_encrypt_data(struct arc4_ctx *ctx, struct ieee80211_key *key, struct sk_buff *skb, u8 *payload, size_t payload_len) @@ -231,7 +231,7 @@ int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm, ieee80211_get_tkip_p2k(&key->conf, skb, rc4key); - return ieee80211_wep_encrypt_data(tfm, rc4key, 16, + return ieee80211_wep_encrypt_data(ctx, rc4key, 16, payload, payload_len); } @@ -239,7 +239,7 @@ int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm, * beginning of the buffer containing IEEE 802.11 header payload, i.e., * including IV, Ext. IV, real data, Michael MIC, ICV. @payload_len is the * length of payload, including IV, Ext. IV, MIC, ICV. */ -int ieee80211_tkip_decrypt_data(struct crypto_cipher *tfm, +int ieee80211_tkip_decrypt_data(struct arc4_ctx *ctx, struct ieee80211_key *key, u8 *payload, size_t payload_len, u8 *ta, u8 *ra, int only_iv, int queue, @@ -297,7 +297,7 @@ int ieee80211_tkip_decrypt_data(struct crypto_cipher *tfm, tkip_mixing_phase2(tk, &rx_ctx->ctx, iv16, rc4key); - res = ieee80211_wep_decrypt_data(tfm, rc4key, 16, pos, payload_len - 12); + res = ieee80211_wep_decrypt_data(ctx, rc4key, 16, pos, payload_len - 12); done: if (res == TKIP_DECRYPT_OK) { /* diff --git a/net/mac80211/tkip.h b/net/mac80211/tkip.h index a1bcbfbefe7c..798583056201 100644 --- a/net/mac80211/tkip.h +++ b/net/mac80211/tkip.h @@ -13,7 +13,7 @@ #include #include "key.h" -int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm, +int ieee80211_tkip_encrypt_data(struct arc4_ctx *ctx, struct ieee80211_key *key, struct sk_buff *skb, u8 *payload, size_t payload_len); @@ -24,7 +24,7 @@ enum { TKIP_DECRYPT_INVALID_KEYIDX = -2, TKIP_DECRYPT_REPLAY = -3, }; -int ieee80211_tkip_decrypt_data(struct crypto_cipher *tfm, +int ieee80211_tkip_decrypt_data(struct arc4_ctx *ctx, struct ieee80211_key *key, u8 *payload, size_t payload_len, u8 *ta, u8 *ra, int only_iv, int queue, diff --git a/net/mac80211/wep.c b/net/mac80211/wep.c index bfe9ed9f4c48..9f5673736967 100644 --- a/net/mac80211/wep.c +++ b/net/mac80211/wep.c @@ -30,30 +30,9 @@ int ieee80211_wep_init(struct ieee80211_local *local) /* start WEP IV from a random value */ get_random_bytes(&local->wep_iv, IEEE80211_WEP_IV_LEN); - local->wep_tx_tfm = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(local->wep_tx_tfm)) { - local->wep_rx_tfm = ERR_PTR(-EINVAL); - return PTR_ERR(local->wep_tx_tfm); - } - - local->wep_rx_tfm = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(local->wep_rx_tfm)) { - crypto_free_cipher(local->wep_tx_tfm); - local->wep_tx_tfm = ERR_PTR(-EINVAL); - return PTR_ERR(local->wep_rx_tfm); - } - return 0; } -void ieee80211_wep_free(struct ieee80211_local *local) -{ - if (!IS_ERR(local->wep_tx_tfm)) - crypto_free_cipher(local->wep_tx_tfm); - if (!IS_ERR(local->wep_rx_tfm)) - crypto_free_cipher(local->wep_rx_tfm); -} - static inline bool ieee80211_wep_weak_iv(u32 iv, int keylen) { /* @@ -131,21 +110,17 @@ static void ieee80211_wep_remove_iv(struct ieee80211_local *local, /* Perform WEP encryption using given key. data buffer must have tailroom * for 4-byte ICV. data_len must not include this ICV. Note: this function * does _not_ add IV. data = RC4(data | CRC32(data)) */ -int ieee80211_wep_encrypt_data(struct crypto_cipher *tfm, u8 *rc4key, +int ieee80211_wep_encrypt_data(struct arc4_ctx *ctx, u8 *rc4key, size_t klen, u8 *data, size_t data_len) { __le32 icv; - int i; - - if (IS_ERR(tfm)) - return -1; icv = cpu_to_le32(~crc32_le(~0, data, data_len)); put_unaligned(icv, (__le32 *)(data + data_len)); - crypto_cipher_setkey(tfm, rc4key, klen); - for (i = 0; i < data_len + IEEE80211_WEP_ICV_LEN; i++) - crypto_cipher_encrypt_one(tfm, data + i, data + i); + arc4_setkey(ctx, rc4key, klen); + arc4_crypt(ctx, data, data, data_len + IEEE80211_WEP_ICV_LEN); + memzero_explicit(ctx, sizeof(*ctx)); return 0; } @@ -184,7 +159,7 @@ int ieee80211_wep_encrypt(struct ieee80211_local *local, /* Add room for ICV */ skb_put(skb, IEEE80211_WEP_ICV_LEN); - return ieee80211_wep_encrypt_data(local->wep_tx_tfm, rc4key, keylen + 3, + return ieee80211_wep_encrypt_data(&local->wep_tx_ctx, rc4key, keylen + 3, iv + IEEE80211_WEP_IV_LEN, len); } @@ -192,18 +167,14 @@ int ieee80211_wep_encrypt(struct ieee80211_local *local, /* Perform WEP decryption using given key. data buffer includes encrypted * payload, including 4-byte ICV, but _not_ IV. data_len must not include ICV. * Return 0 on success and -1 on ICV mismatch. */ -int ieee80211_wep_decrypt_data(struct crypto_cipher *tfm, u8 *rc4key, +int ieee80211_wep_decrypt_data(struct arc4_ctx *ctx, u8 *rc4key, size_t klen, u8 *data, size_t data_len) { __le32 crc; - int i; - if (IS_ERR(tfm)) - return -1; - - crypto_cipher_setkey(tfm, rc4key, klen); - for (i = 0; i < data_len + IEEE80211_WEP_ICV_LEN; i++) - crypto_cipher_decrypt_one(tfm, data + i, data + i); + arc4_setkey(ctx, rc4key, klen); + arc4_crypt(ctx, data, data, data_len + IEEE80211_WEP_ICV_LEN); + memzero_explicit(ctx, sizeof(*ctx)); crc = cpu_to_le32(~crc32_le(~0, data, data_len)); if (memcmp(&crc, data + data_len, IEEE80211_WEP_ICV_LEN) != 0) @@ -256,7 +227,7 @@ static int ieee80211_wep_decrypt(struct ieee80211_local *local, /* Copy rest of the WEP key (the secret part) */ memcpy(rc4key + 3, key->conf.key, key->conf.keylen); - if (ieee80211_wep_decrypt_data(local->wep_rx_tfm, rc4key, klen, + if (ieee80211_wep_decrypt_data(&local->wep_rx_ctx, rc4key, klen, skb->data + hdrlen + IEEE80211_WEP_IV_LEN, len)) ret = -1; diff --git a/net/mac80211/wep.h b/net/mac80211/wep.h index 9615749d1f65..3644f4a5bb87 100644 --- a/net/mac80211/wep.h +++ b/net/mac80211/wep.h @@ -17,13 +17,12 @@ #include "key.h" int ieee80211_wep_init(struct ieee80211_local *local); -void ieee80211_wep_free(struct ieee80211_local *local); -int ieee80211_wep_encrypt_data(struct crypto_cipher *tfm, u8 *rc4key, +int ieee80211_wep_encrypt_data(struct arc4_ctx *ctx, u8 *rc4key, size_t klen, u8 *data, size_t data_len); int ieee80211_wep_encrypt(struct ieee80211_local *local, struct sk_buff *skb, const u8 *key, int keylen, int keyidx); -int ieee80211_wep_decrypt_data(struct crypto_cipher *tfm, u8 *rc4key, +int ieee80211_wep_decrypt_data(struct arc4_ctx *ctx, u8 *rc4key, size_t klen, u8 *data, size_t data_len); ieee80211_rx_result diff --git a/net/mac80211/wpa.c b/net/mac80211/wpa.c index 58d0b258b684..02e8ab7b2b4c 100644 --- a/net/mac80211/wpa.c +++ b/net/mac80211/wpa.c @@ -242,7 +242,7 @@ static int tkip_encrypt_skb(struct ieee80211_tx_data *tx, struct sk_buff *skb) /* Add room for ICV */ skb_put(skb, IEEE80211_TKIP_ICV_LEN); - return ieee80211_tkip_encrypt_data(tx->local->wep_tx_tfm, + return ieee80211_tkip_encrypt_data(&tx->local->wep_tx_ctx, key, skb, pos, len); } @@ -293,7 +293,7 @@ ieee80211_crypto_tkip_decrypt(struct ieee80211_rx_data *rx) if (status->flag & RX_FLAG_DECRYPTED) hwaccel = 1; - res = ieee80211_tkip_decrypt_data(rx->local->wep_rx_tfm, + res = ieee80211_tkip_decrypt_data(&rx->local->wep_rx_ctx, key, skb->data + hdrlen, skb->len - hdrlen, rx->sta->sta.addr, hdr->addr1, hwaccel, rx->security_idx, From af1f3d327d49d9bb94d9c173023e6cbdc5453d38 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 12 Jun 2019 18:19:55 +0200 Subject: [PATCH 113/143] net/lib80211: move WEP handling to ARC4 library code The crypto API abstraction is not very useful for invoking ciphers directly, especially in the case of arc4, which only has a generic implementation in C. So let's invoke the library code directly. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- net/wireless/Kconfig | 1 + net/wireless/lib80211_crypt_wep.c | 51 ++++++++----------------------- 2 files changed, 14 insertions(+), 38 deletions(-) diff --git a/net/wireless/Kconfig b/net/wireless/Kconfig index 41722046b937..abff3c6a5b50 100644 --- a/net/wireless/Kconfig +++ b/net/wireless/Kconfig @@ -212,6 +212,7 @@ config LIB80211 config LIB80211_CRYPT_WEP tristate + select CRYPTO_LIB_ARC4 config LIB80211_CRYPT_CCMP tristate diff --git a/net/wireless/lib80211_crypt_wep.c b/net/wireless/lib80211_crypt_wep.c index 20c1ad63ad44..04e4d66ea19d 100644 --- a/net/wireless/lib80211_crypt_wep.c +++ b/net/wireless/lib80211_crypt_wep.c @@ -11,6 +11,7 @@ */ #include +#include #include #include #include @@ -22,7 +23,7 @@ #include -#include +#include #include MODULE_AUTHOR("Jouni Malinen"); @@ -35,52 +36,31 @@ struct lib80211_wep_data { u8 key[WEP_KEY_LEN + 1]; u8 key_len; u8 key_idx; - struct crypto_cipher *tx_tfm; - struct crypto_cipher *rx_tfm; + struct arc4_ctx tx_ctx; + struct arc4_ctx rx_ctx; }; static void *lib80211_wep_init(int keyidx) { struct lib80211_wep_data *priv; + if (fips_enabled) + return NULL; + priv = kzalloc(sizeof(*priv), GFP_ATOMIC); if (priv == NULL) - goto fail; + return NULL; priv->key_idx = keyidx; - priv->tx_tfm = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(priv->tx_tfm)) { - priv->tx_tfm = NULL; - goto fail; - } - - priv->rx_tfm = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(priv->rx_tfm)) { - priv->rx_tfm = NULL; - goto fail; - } /* start WEP IV from a random value */ get_random_bytes(&priv->iv, 4); return priv; - - fail: - if (priv) { - crypto_free_cipher(priv->tx_tfm); - crypto_free_cipher(priv->rx_tfm); - kfree(priv); - } - return NULL; } static void lib80211_wep_deinit(void *priv) { - struct lib80211_wep_data *_priv = priv; - if (_priv) { - crypto_free_cipher(_priv->tx_tfm); - crypto_free_cipher(_priv->rx_tfm); - } - kfree(priv); + kzfree(priv); } /* Add WEP IV/key info to a frame that has at least 4 bytes of headroom */ @@ -132,7 +112,6 @@ static int lib80211_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv) u32 crc, klen, len; u8 *pos, *icv; u8 key[WEP_KEY_LEN + 3]; - int i; /* other checks are in lib80211_wep_build_iv */ if (skb_tailroom(skb) < 4) @@ -160,10 +139,8 @@ static int lib80211_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv) icv[2] = crc >> 16; icv[3] = crc >> 24; - crypto_cipher_setkey(wep->tx_tfm, key, klen); - - for (i = 0; i < len + 4; i++) - crypto_cipher_encrypt_one(wep->tx_tfm, pos + i, pos + i); + arc4_setkey(&wep->tx_ctx, key, klen); + arc4_crypt(&wep->tx_ctx, pos, pos, len + 4); return 0; } @@ -181,7 +158,6 @@ static int lib80211_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv) u32 crc, klen, plen; u8 key[WEP_KEY_LEN + 3]; u8 keyidx, *pos, icv[4]; - int i; if (skb->len < hdr_len + 8) return -1; @@ -202,9 +178,8 @@ static int lib80211_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv) /* Apply RC4 to data and compute CRC32 over decrypted data */ plen = skb->len - hdr_len - 8; - crypto_cipher_setkey(wep->rx_tfm, key, klen); - for (i = 0; i < plen + 4; i++) - crypto_cipher_decrypt_one(wep->rx_tfm, pos + i, pos + i); + arc4_setkey(&wep->rx_ctx, key, klen); + arc4_crypt(&wep->rx_ctx, pos, pos, plen + 4); crc = ~crc32_le(~0, pos, plen); icv[0] = crc; From 4be297016cd3a6aedbcc3b053da16cdd42546f04 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 12 Jun 2019 18:19:56 +0200 Subject: [PATCH 114/143] net/lib80211: move TKIP handling to ARC4 library code The crypto API abstraction is not very useful for invoking ciphers directly, especially in the case of arc4, which only has a generic implementation in C. So let's invoke the library code directly. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- net/wireless/Kconfig | 1 + net/wireless/lib80211_crypt_tkip.c | 48 +++++++++++------------------- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/net/wireless/Kconfig b/net/wireless/Kconfig index abff3c6a5b50..2cfc5b7800e9 100644 --- a/net/wireless/Kconfig +++ b/net/wireless/Kconfig @@ -219,6 +219,7 @@ config LIB80211_CRYPT_CCMP config LIB80211_CRYPT_TKIP tristate + select CRYPTO_LIB_ARC4 config LIB80211_DEBUG bool "lib80211 debugging messages" diff --git a/net/wireless/lib80211_crypt_tkip.c b/net/wireless/lib80211_crypt_tkip.c index 11eaa5956f00..0fd155c4e0a6 100644 --- a/net/wireless/lib80211_crypt_tkip.c +++ b/net/wireless/lib80211_crypt_tkip.c @@ -13,6 +13,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include +#include #include #include #include @@ -64,9 +66,9 @@ struct lib80211_tkip_data { int key_idx; - struct crypto_cipher *rx_tfm_arc4; + struct arc4_ctx rx_ctx_arc4; + struct arc4_ctx tx_ctx_arc4; struct crypto_shash *rx_tfm_michael; - struct crypto_cipher *tx_tfm_arc4; struct crypto_shash *tx_tfm_michael; /* scratch buffers for virt_to_page() (crypto API) */ @@ -93,30 +95,21 @@ static void *lib80211_tkip_init(int key_idx) { struct lib80211_tkip_data *priv; + if (fips_enabled) + return NULL; + priv = kzalloc(sizeof(*priv), GFP_ATOMIC); if (priv == NULL) goto fail; priv->key_idx = key_idx; - priv->tx_tfm_arc4 = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(priv->tx_tfm_arc4)) { - priv->tx_tfm_arc4 = NULL; - goto fail; - } - priv->tx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0); if (IS_ERR(priv->tx_tfm_michael)) { priv->tx_tfm_michael = NULL; goto fail; } - priv->rx_tfm_arc4 = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(priv->rx_tfm_arc4)) { - priv->rx_tfm_arc4 = NULL; - goto fail; - } - priv->rx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0); if (IS_ERR(priv->rx_tfm_michael)) { priv->rx_tfm_michael = NULL; @@ -128,9 +121,7 @@ static void *lib80211_tkip_init(int key_idx) fail: if (priv) { crypto_free_shash(priv->tx_tfm_michael); - crypto_free_cipher(priv->tx_tfm_arc4); crypto_free_shash(priv->rx_tfm_michael); - crypto_free_cipher(priv->rx_tfm_arc4); kfree(priv); } @@ -142,11 +133,9 @@ static void lib80211_tkip_deinit(void *priv) struct lib80211_tkip_data *_priv = priv; if (_priv) { crypto_free_shash(_priv->tx_tfm_michael); - crypto_free_cipher(_priv->tx_tfm_arc4); crypto_free_shash(_priv->rx_tfm_michael); - crypto_free_cipher(_priv->rx_tfm_arc4); } - kfree(priv); + kzfree(priv); } static inline u16 RotR1(u16 val) @@ -345,7 +334,6 @@ static int lib80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv) int len; u8 rc4key[16], *pos, *icv; u32 crc; - int i; if (tkey->flags & IEEE80211_CRYPTO_TKIP_COUNTERMEASURES) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; @@ -370,9 +358,9 @@ static int lib80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv) icv[2] = crc >> 16; icv[3] = crc >> 24; - crypto_cipher_setkey(tkey->tx_tfm_arc4, rc4key, 16); - for (i = 0; i < len + 4; i++) - crypto_cipher_encrypt_one(tkey->tx_tfm_arc4, pos + i, pos + i); + arc4_setkey(&tkey->tx_ctx_arc4, rc4key, 16); + arc4_crypt(&tkey->tx_ctx_arc4, pos, pos, len + 4); + return 0; } @@ -400,7 +388,6 @@ static int lib80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) u8 icv[4]; u32 crc; int plen; - int i; hdr = (struct ieee80211_hdr *)skb->data; @@ -453,9 +440,8 @@ static int lib80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) plen = skb->len - hdr_len - 12; - crypto_cipher_setkey(tkey->rx_tfm_arc4, rc4key, 16); - for (i = 0; i < plen + 4; i++) - crypto_cipher_decrypt_one(tkey->rx_tfm_arc4, pos + i, pos + i); + arc4_setkey(&tkey->rx_ctx_arc4, rc4key, 16); + arc4_crypt(&tkey->rx_ctx_arc4, pos, pos, plen + 4); crc = ~crc32_le(~0, pos, plen); icv[0] = crc; @@ -640,17 +626,17 @@ static int lib80211_tkip_set_key(void *key, int len, u8 * seq, void *priv) struct lib80211_tkip_data *tkey = priv; int keyidx; struct crypto_shash *tfm = tkey->tx_tfm_michael; - struct crypto_cipher *tfm2 = tkey->tx_tfm_arc4; + struct arc4_ctx *tfm2 = &tkey->tx_ctx_arc4; struct crypto_shash *tfm3 = tkey->rx_tfm_michael; - struct crypto_cipher *tfm4 = tkey->rx_tfm_arc4; + struct arc4_ctx *tfm4 = &tkey->rx_ctx_arc4; keyidx = tkey->key_idx; memset(tkey, 0, sizeof(*tkey)); tkey->key_idx = keyidx; tkey->tx_tfm_michael = tfm; - tkey->tx_tfm_arc4 = tfm2; + tkey->tx_ctx_arc4 = *tfm2; tkey->rx_tfm_michael = tfm3; - tkey->rx_tfm_arc4 = tfm4; + tkey->rx_ctx_arc4 = *tfm4; if (len == TKIP_KEY_LEN) { memcpy(tkey->key, key, TKIP_KEY_LEN); tkey->key_set = 1; From 611a23c2d3961d2ec72f42582ee88755f9a03cee Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 12 Jun 2019 18:19:57 +0200 Subject: [PATCH 115/143] crypto: arc4 - remove cipher implementation There are no remaining users of the cipher implementation, and there are no meaningful ways in which the arc4 cipher can be combined with templates other than ECB (and the way we do provide that combination is highly dubious to begin with). So let's drop the arc4 cipher altogether, and only keep the ecb(arc4) skcipher, which is used in various places in the kernel. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/arc4.c | 65 +++++++++++------------------------------------- crypto/testmgr.c | 1 + 2 files changed, 16 insertions(+), 50 deletions(-) diff --git a/crypto/arc4.c b/crypto/arc4.c index d303b7ff1fed..dbb1f8b65396 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -18,26 +18,15 @@ #include #include -static int arc4_set_key(struct crypto_tfm *tfm, const u8 *in_key, - unsigned int key_len) +static int crypto_arc4_setkey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) { - struct arc4_ctx *ctx = crypto_tfm_ctx(tfm); + struct arc4_ctx *ctx = crypto_skcipher_ctx(tfm); return arc4_setkey(ctx, in_key, key_len); } -static int arc4_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key, - unsigned int key_len) -{ - return arc4_set_key(&tfm->base, in_key, key_len); -} - -static void arc4_crypt_one(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - arc4_crypt(crypto_tfm_ctx(tfm), out, in, 1); -} - -static int ecb_arc4_crypt(struct skcipher_request *req) +static int crypto_arc4_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct arc4_ctx *ctx = crypto_skcipher_ctx(tfm); @@ -55,25 +44,11 @@ static int ecb_arc4_crypt(struct skcipher_request *req) return err; } -static struct crypto_alg arc4_cipher = { - .cra_name = "arc4", - .cra_driver_name = "arc4-generic", - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = ARC4_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct arc4_ctx), - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = ARC4_MIN_KEY_SIZE, - .cia_max_keysize = ARC4_MAX_KEY_SIZE, - .cia_setkey = arc4_set_key, - .cia_encrypt = arc4_crypt_one, - .cia_decrypt = arc4_crypt_one, - }, - }, -}; - -static struct skcipher_alg arc4_skcipher = { +static struct skcipher_alg arc4_alg = { + /* + * For legacy reasons, this is named "ecb(arc4)", not "arc4". + * Nevertheless it's actually a stream cipher, not a block cipher. + */ .base.cra_name = "ecb(arc4)", .base.cra_driver_name = "ecb(arc4)-generic", .base.cra_priority = 100, @@ -82,29 +57,19 @@ static struct skcipher_alg arc4_skcipher = { .base.cra_module = THIS_MODULE, .min_keysize = ARC4_MIN_KEY_SIZE, .max_keysize = ARC4_MAX_KEY_SIZE, - .setkey = arc4_set_key_skcipher, - .encrypt = ecb_arc4_crypt, - .decrypt = ecb_arc4_crypt, + .setkey = crypto_arc4_setkey, + .encrypt = crypto_arc4_crypt, + .decrypt = crypto_arc4_crypt, }; static int __init arc4_init(void) { - int err; - - err = crypto_register_alg(&arc4_cipher); - if (err) - return err; - - err = crypto_register_skcipher(&arc4_skcipher); - if (err) - crypto_unregister_alg(&arc4_cipher); - return err; + return crypto_register_skcipher(&arc4_alg); } static void __exit arc4_exit(void) { - crypto_unregister_alg(&arc4_cipher); - crypto_unregister_skcipher(&arc4_skcipher); + crypto_unregister_skcipher(&arc4_alg); } subsys_initcall(arc4_init); @@ -113,4 +78,4 @@ module_exit(arc4_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("ARC4 Cipher Algorithm"); MODULE_AUTHOR("Jon Oberheide "); -MODULE_ALIAS_CRYPTO("arc4"); +MODULE_ALIAS_CRYPTO("ecb(arc4)"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index f7fdd7fe89a9..5d163dd2ffac 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4404,6 +4404,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "ecb(arc4)", + .generic_driver = "ecb(arc4)-generic", .test = alg_test_skcipher, .suite = { .cipher = __VECS(arc4_tv_template) From 0e5a610b5ca52ca4fa0705c87114310a95cfbe34 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 12 Jun 2019 18:19:58 +0200 Subject: [PATCH 116/143] ppp: mppe: switch to RC4 library interface The MPPE code uses the sync skcipher to invoke the ecb(arc4) skcipher, of which only a single generic C code implementation exists. This means that going through all the trouble of using scatterlists etc buys us very little, and we're better off just invoking the arc4 library directly. Note that the SHA1 shash used by this driver has several accelerated implementations for various architectures, so retaining that part does make sense. Cc: linux-ppp@vger.kernel.org Cc: Paul Mackerras Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- drivers/net/ppp/Kconfig | 3 +- drivers/net/ppp/ppp_mppe.c | 97 ++++++-------------------------------- 2 files changed, 15 insertions(+), 85 deletions(-) diff --git a/drivers/net/ppp/Kconfig b/drivers/net/ppp/Kconfig index 1373c6d7278d..dc3f231cfd97 100644 --- a/drivers/net/ppp/Kconfig +++ b/drivers/net/ppp/Kconfig @@ -86,8 +86,7 @@ config PPP_MPPE depends on PPP select CRYPTO select CRYPTO_SHA1 - select CRYPTO_ARC4 - select CRYPTO_ECB + select CRYPTO_LIB_ARC4 ---help--- Support for the MPPE Encryption protocol, as employed by the Microsoft Point-to-Point Tunneling Protocol. diff --git a/drivers/net/ppp/ppp_mppe.c b/drivers/net/ppp/ppp_mppe.c index ff61dd8748de..de3b57d09d0c 100644 --- a/drivers/net/ppp/ppp_mppe.c +++ b/drivers/net/ppp/ppp_mppe.c @@ -42,9 +42,10 @@ * deprecated in 2.6 */ +#include #include -#include #include +#include #include #include #include @@ -65,13 +66,6 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_ALIAS("ppp-compress-" __stringify(CI_MPPE)); MODULE_VERSION("1.0.2"); -static unsigned int -setup_sg(struct scatterlist *sg, const void *address, unsigned int length) -{ - sg_set_buf(sg, address, length); - return length; -} - #define SHA1_PAD_SIZE 40 /* @@ -95,7 +89,7 @@ static inline void sha_pad_init(struct sha_pad *shapad) * State for an MPPE (de)compressor. */ struct ppp_mppe_state { - struct crypto_sync_skcipher *arc4; + struct arc4_ctx arc4; struct shash_desc *sha1; unsigned char *sha1_digest; unsigned char master_key[MPPE_MAX_KEY_LEN]; @@ -154,24 +148,11 @@ static void get_new_key_from_sha(struct ppp_mppe_state * state) */ static void mppe_rekey(struct ppp_mppe_state * state, int initial_key) { - struct scatterlist sg_in[1], sg_out[1]; - SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4); - - skcipher_request_set_sync_tfm(req, state->arc4); - skcipher_request_set_callback(req, 0, NULL, NULL); - get_new_key_from_sha(state); if (!initial_key) { - crypto_sync_skcipher_setkey(state->arc4, state->sha1_digest, - state->keylen); - sg_init_table(sg_in, 1); - sg_init_table(sg_out, 1); - setup_sg(sg_in, state->sha1_digest, state->keylen); - setup_sg(sg_out, state->session_key, state->keylen); - skcipher_request_set_crypt(req, sg_in, sg_out, state->keylen, - NULL); - if (crypto_skcipher_encrypt(req)) - printk(KERN_WARNING "mppe_rekey: cipher_encrypt failed\n"); + arc4_setkey(&state->arc4, state->sha1_digest, state->keylen); + arc4_crypt(&state->arc4, state->session_key, state->sha1_digest, + state->keylen); } else { memcpy(state->session_key, state->sha1_digest, state->keylen); } @@ -181,9 +162,7 @@ static void mppe_rekey(struct ppp_mppe_state * state, int initial_key) state->session_key[1] = 0x26; state->session_key[2] = 0x9e; } - crypto_sync_skcipher_setkey(state->arc4, state->session_key, - state->keylen); - skcipher_request_zero(req); + arc4_setkey(&state->arc4, state->session_key, state->keylen); } /* @@ -196,7 +175,8 @@ static void *mppe_alloc(unsigned char *options, int optlen) unsigned int digestsize; if (optlen != CILEN_MPPE + sizeof(state->master_key) || - options[0] != CI_MPPE || options[1] != CILEN_MPPE) + options[0] != CI_MPPE || options[1] != CILEN_MPPE || + fips_enabled) goto out; state = kzalloc(sizeof(*state), GFP_KERNEL); @@ -204,12 +184,6 @@ static void *mppe_alloc(unsigned char *options, int optlen) goto out; - state->arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0); - if (IS_ERR(state->arc4)) { - state->arc4 = NULL; - goto out_free; - } - shash = crypto_alloc_shash("sha1", 0, 0); if (IS_ERR(shash)) goto out_free; @@ -250,7 +224,6 @@ out_free: crypto_free_shash(state->sha1->tfm); kzfree(state->sha1); } - crypto_free_sync_skcipher(state->arc4); kfree(state); out: return NULL; @@ -266,8 +239,7 @@ static void mppe_free(void *arg) kfree(state->sha1_digest); crypto_free_shash(state->sha1->tfm); kzfree(state->sha1); - crypto_free_sync_skcipher(state->arc4); - kfree(state); + kzfree(state); } } @@ -366,10 +338,7 @@ mppe_compress(void *arg, unsigned char *ibuf, unsigned char *obuf, int isize, int osize) { struct ppp_mppe_state *state = (struct ppp_mppe_state *) arg; - SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4); int proto; - int err; - struct scatterlist sg_in[1], sg_out[1]; /* * Check that the protocol is in the range we handle. @@ -420,21 +389,7 @@ mppe_compress(void *arg, unsigned char *ibuf, unsigned char *obuf, ibuf += 2; /* skip to proto field */ isize -= 2; - /* Encrypt packet */ - sg_init_table(sg_in, 1); - sg_init_table(sg_out, 1); - setup_sg(sg_in, ibuf, isize); - setup_sg(sg_out, obuf, osize); - - skcipher_request_set_sync_tfm(req, state->arc4); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, sg_in, sg_out, isize, NULL); - err = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); - if (err) { - printk(KERN_DEBUG "crypto_cypher_encrypt failed\n"); - return -1; - } + arc4_crypt(&state->arc4, obuf, ibuf, isize); state->stats.unc_bytes += isize; state->stats.unc_packets++; @@ -480,10 +435,8 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf, int osize) { struct ppp_mppe_state *state = (struct ppp_mppe_state *) arg; - SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4); unsigned ccount; int flushed = MPPE_BITS(ibuf) & MPPE_BIT_FLUSHED; - struct scatterlist sg_in[1], sg_out[1]; if (isize <= PPP_HDRLEN + MPPE_OVHD) { if (state->debug) @@ -610,19 +563,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf, * Decrypt the first byte in order to check if it is * a compressed or uncompressed protocol field. */ - sg_init_table(sg_in, 1); - sg_init_table(sg_out, 1); - setup_sg(sg_in, ibuf, 1); - setup_sg(sg_out, obuf, 1); - - skcipher_request_set_sync_tfm(req, state->arc4); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, sg_in, sg_out, 1, NULL); - if (crypto_skcipher_decrypt(req)) { - printk(KERN_DEBUG "crypto_cypher_decrypt failed\n"); - osize = DECOMP_ERROR; - goto out_zap_req; - } + arc4_crypt(&state->arc4, obuf, ibuf, 1); /* * Do PFC decompression. @@ -637,14 +578,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf, } /* And finally, decrypt the rest of the packet. */ - setup_sg(sg_in, ibuf + 1, isize - 1); - setup_sg(sg_out, obuf + 1, osize - 1); - skcipher_request_set_crypt(req, sg_in, sg_out, isize - 1, NULL); - if (crypto_skcipher_decrypt(req)) { - printk(KERN_DEBUG "crypto_cypher_decrypt failed\n"); - osize = DECOMP_ERROR; - goto out_zap_req; - } + arc4_crypt(&state->arc4, obuf + 1, ibuf + 1, isize - 1); state->stats.unc_bytes += osize; state->stats.unc_packets++; @@ -654,8 +588,6 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf, /* good packet credit */ state->sanity_errors >>= 1; -out_zap_req: - skcipher_request_zero(req); return osize; sanity_error: @@ -728,8 +660,7 @@ static struct compressor ppp_mppe = { static int __init ppp_mppe_init(void) { int answer; - if (!(crypto_has_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC) && - crypto_has_ahash("sha1", 0, CRYPTO_ALG_ASYNC))) + if (fips_enabled || !crypto_has_ahash("sha1", 0, CRYPTO_ALG_ASYNC)) return -ENODEV; sha_pad = kmalloc(sizeof(struct sha_pad), GFP_KERNEL); From 97a5fee2bd70df9e48cdfdbbacc15d45089228a3 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 12 Jun 2019 18:19:59 +0200 Subject: [PATCH 117/143] fs: cifs: switch to RC4 library interface The CIFS code uses the sync skcipher API to invoke the ecb(arc4) skcipher, of which only a single generic C code implementation exists. This means that going through all the trouble of using scatterlists etc buys us very little, and we're better off just invoking the arc4 library directly. This also reverts commit 5f4b55699aaf ("CIFS: Fix BUG() in calc_seckey()"), since it is no longer necessary to allocate sec_key on the heap. Cc: linux-cifs@vger.kernel.org Cc: Steve French Signed-off-by: Ard Biesheuvel Acked-by: Steve French Signed-off-by: Herbert Xu --- fs/cifs/Kconfig | 2 +- fs/cifs/cifsencrypt.c | 62 +++++++++++-------------------------------- fs/cifs/cifsfs.c | 1 - 3 files changed, 17 insertions(+), 48 deletions(-) diff --git a/fs/cifs/Kconfig b/fs/cifs/Kconfig index 76724efc831c..3da294231dcc 100644 --- a/fs/cifs/Kconfig +++ b/fs/cifs/Kconfig @@ -9,7 +9,7 @@ config CIFS select CRYPTO_SHA512 select CRYPTO_CMAC select CRYPTO_HMAC - select CRYPTO_ARC4 + select CRYPTO_LIB_ARC4 select CRYPTO_AEAD2 select CRYPTO_CCM select CRYPTO_ECB diff --git a/fs/cifs/cifsencrypt.c b/fs/cifs/cifsencrypt.c index d2a05e46d6f5..97b7497c13ef 100644 --- a/fs/cifs/cifsencrypt.c +++ b/fs/cifs/cifsencrypt.c @@ -33,7 +33,8 @@ #include #include #include -#include +#include +#include #include int __cifs_calc_signature(struct smb_rqst *rqst, @@ -772,63 +773,32 @@ setup_ntlmv2_rsp_ret: int calc_seckey(struct cifs_ses *ses) { - int rc; - struct crypto_skcipher *tfm_arc4; - struct scatterlist sgin, sgout; - struct skcipher_request *req; - unsigned char *sec_key; + unsigned char sec_key[CIFS_SESS_KEY_SIZE]; /* a nonce */ + struct arc4_ctx *ctx_arc4; - sec_key = kmalloc(CIFS_SESS_KEY_SIZE, GFP_KERNEL); - if (sec_key == NULL) - return -ENOMEM; + if (fips_enabled) + return -ENODEV; get_random_bytes(sec_key, CIFS_SESS_KEY_SIZE); - tfm_arc4 = crypto_alloc_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC); - if (IS_ERR(tfm_arc4)) { - rc = PTR_ERR(tfm_arc4); - cifs_dbg(VFS, "could not allocate crypto API arc4\n"); - goto out; + ctx_arc4 = kmalloc(sizeof(*ctx_arc4), GFP_KERNEL); + if (!ctx_arc4) { + cifs_dbg(VFS, "could not allocate arc4 context\n"); + return -ENOMEM; } - rc = crypto_skcipher_setkey(tfm_arc4, ses->auth_key.response, - CIFS_SESS_KEY_SIZE); - if (rc) { - cifs_dbg(VFS, "%s: Could not set response as a key\n", - __func__); - goto out_free_cipher; - } - - req = skcipher_request_alloc(tfm_arc4, GFP_KERNEL); - if (!req) { - rc = -ENOMEM; - cifs_dbg(VFS, "could not allocate crypto API arc4 request\n"); - goto out_free_cipher; - } - - sg_init_one(&sgin, sec_key, CIFS_SESS_KEY_SIZE); - sg_init_one(&sgout, ses->ntlmssp->ciphertext, CIFS_CPHTXT_SIZE); - - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sgin, &sgout, CIFS_CPHTXT_SIZE, NULL); - - rc = crypto_skcipher_encrypt(req); - skcipher_request_free(req); - if (rc) { - cifs_dbg(VFS, "could not encrypt session key rc: %d\n", rc); - goto out_free_cipher; - } + arc4_setkey(ctx_arc4, ses->auth_key.response, CIFS_SESS_KEY_SIZE); + arc4_crypt(ctx_arc4, ses->ntlmssp->ciphertext, sec_key, + CIFS_CPHTXT_SIZE); /* make secondary_key/nonce as session key */ memcpy(ses->auth_key.response, sec_key, CIFS_SESS_KEY_SIZE); /* and make len as that of session key only */ ses->auth_key.len = CIFS_SESS_KEY_SIZE; -out_free_cipher: - crypto_free_skcipher(tfm_arc4); -out: - kfree(sec_key); - return rc; + memzero_explicit(sec_key, CIFS_SESS_KEY_SIZE); + kzfree(ctx_arc4); + return 0; } void diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index f5fcd6360056..e55afaf9e5a3 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -1590,7 +1590,6 @@ MODULE_DESCRIPTION ("VFS to access SMB3 servers e.g. Samba, Macs, Azure and Windows (and " "also older servers complying with the SNIA CIFS Specification)"); MODULE_VERSION(CIFS_VERSION); -MODULE_SOFTDEP("pre: arc4"); MODULE_SOFTDEP("pre: des"); MODULE_SOFTDEP("pre: ecb"); MODULE_SOFTDEP("pre: hmac"); From 576d152ccc56574475faa7460d4b507e8440e9aa Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 14 Jun 2019 16:29:04 +0200 Subject: [PATCH 118/143] crypto: nx - no need to check return value of debugfs_create functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When calling debugfs functions, there is no need to ever check the return value. The function can work or not, but the code logic should never do something different based on this. Also, there is no need to store the individual debugfs file names, especially as the whole directiry is deleted at once, so remove the unneeded structure entirely. Cc: "Breno Leitão" Cc: Nayna Jain Cc: Paulo Flabiano Smorigo Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Michael Ellerman Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Cc: linuxppc-dev@lists.ozlabs.org Signed-off-by: Greg Kroah-Hartman Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx.c | 4 +- drivers/crypto/nx/nx.h | 12 +----- drivers/crypto/nx/nx_debugfs.c | 71 +++++++++++----------------------- 3 files changed, 26 insertions(+), 61 deletions(-) diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 3a5e31be4764..20b5e276f184 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -581,9 +581,7 @@ static int nx_register_algs(void) memset(&nx_driver.stats, 0, sizeof(struct nx_stats)); - rc = NX_DEBUGFS_INIT(&nx_driver); - if (rc) - goto out; + NX_DEBUGFS_INIT(&nx_driver); nx_driver.of.status = NX_OKAY; diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h index c3e54af18645..c6b5a3be02be 100644 --- a/drivers/crypto/nx/nx.h +++ b/drivers/crypto/nx/nx.h @@ -76,20 +76,12 @@ struct nx_stats { atomic_t last_error_pid; }; -struct nx_debugfs { - struct dentry *dfs_root; - struct dentry *dfs_aes_ops, *dfs_aes_bytes; - struct dentry *dfs_sha256_ops, *dfs_sha256_bytes; - struct dentry *dfs_sha512_ops, *dfs_sha512_bytes; - struct dentry *dfs_errors, *dfs_last_error, *dfs_last_error_pid; -}; - struct nx_crypto_driver { struct nx_stats stats; struct nx_of of; struct vio_dev *viodev; struct vio_driver viodriver; - struct nx_debugfs dfs; + struct dentry *dfs_root; }; #define NX_GCM4106_NONCE_LEN (4) @@ -177,7 +169,7 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *, unsigned int, #define NX_DEBUGFS_INIT(drv) nx_debugfs_init(drv) #define NX_DEBUGFS_FINI(drv) nx_debugfs_fini(drv) -int nx_debugfs_init(struct nx_crypto_driver *); +void nx_debugfs_init(struct nx_crypto_driver *); void nx_debugfs_fini(struct nx_crypto_driver *); #else #define NX_DEBUGFS_INIT(drv) (0) diff --git a/drivers/crypto/nx/nx_debugfs.c b/drivers/crypto/nx/nx_debugfs.c index 7ab2e8dcd9b4..add1d8d0d23c 100644 --- a/drivers/crypto/nx/nx_debugfs.c +++ b/drivers/crypto/nx/nx_debugfs.c @@ -42,62 +42,37 @@ * Documentation/ABI/testing/debugfs-pfo-nx-crypto */ -int nx_debugfs_init(struct nx_crypto_driver *drv) +void nx_debugfs_init(struct nx_crypto_driver *drv) { - struct nx_debugfs *dfs = &drv->dfs; + struct dentry *root; - dfs->dfs_root = debugfs_create_dir(NX_NAME, NULL); + root = debugfs_create_dir(NX_NAME, NULL); + drv->dfs_root = root; - dfs->dfs_aes_ops = - debugfs_create_u32("aes_ops", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, (u32 *)&drv->stats.aes_ops); - dfs->dfs_sha256_ops = - debugfs_create_u32("sha256_ops", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u32 *)&drv->stats.sha256_ops); - dfs->dfs_sha512_ops = - debugfs_create_u32("sha512_ops", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u32 *)&drv->stats.sha512_ops); - dfs->dfs_aes_bytes = - debugfs_create_u64("aes_bytes", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u64 *)&drv->stats.aes_bytes); - dfs->dfs_sha256_bytes = - debugfs_create_u64("sha256_bytes", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u64 *)&drv->stats.sha256_bytes); - dfs->dfs_sha512_bytes = - debugfs_create_u64("sha512_bytes", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u64 *)&drv->stats.sha512_bytes); - dfs->dfs_errors = - debugfs_create_u32("errors", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, (u32 *)&drv->stats.errors); - dfs->dfs_last_error = - debugfs_create_u32("last_error", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u32 *)&drv->stats.last_error); - dfs->dfs_last_error_pid = - debugfs_create_u32("last_error_pid", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u32 *)&drv->stats.last_error_pid); - return 0; + debugfs_create_u32("aes_ops", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.aes_ops); + debugfs_create_u32("sha256_ops", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.sha256_ops); + debugfs_create_u32("sha512_ops", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.sha512_ops); + debugfs_create_u64("aes_bytes", S_IRUSR | S_IRGRP | S_IROTH, + root, (u64 *)&drv->stats.aes_bytes); + debugfs_create_u64("sha256_bytes", S_IRUSR | S_IRGRP | S_IROTH, + root, (u64 *)&drv->stats.sha256_bytes); + debugfs_create_u64("sha512_bytes", S_IRUSR | S_IRGRP | S_IROTH, + root, (u64 *)&drv->stats.sha512_bytes); + debugfs_create_u32("errors", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.errors); + debugfs_create_u32("last_error", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.last_error); + debugfs_create_u32("last_error_pid", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.last_error_pid); } void nx_debugfs_fini(struct nx_crypto_driver *drv) { - debugfs_remove_recursive(drv->dfs.dfs_root); + debugfs_remove_recursive(drv->dfs_root); } #endif From 0b970d54772f08f56326b89db74cba2ca3c7c278 Mon Sep 17 00:00:00 2001 From: "ofir.drang@arm.com" Date: Mon, 17 Jun 2019 11:46:27 +0300 Subject: [PATCH 119/143] crypto: ccree - Relocate driver irq registration after clk init Signed-off-by: Ofir Drang Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_driver.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 86ac7b443355..0f80cb4f79fb 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -315,15 +315,6 @@ static int init_cc_resources(struct platform_device *plat_dev) return new_drvdata->irq; } - rc = devm_request_irq(dev, new_drvdata->irq, cc_isr, - IRQF_SHARED, "ccree", new_drvdata); - if (rc) { - dev_err(dev, "Could not register to interrupt %d\n", - new_drvdata->irq); - return rc; - } - dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq); - init_completion(&new_drvdata->hw_queue_avail); if (!plat_dev->dev.dma_mask) @@ -401,6 +392,15 @@ static int init_cc_resources(struct platform_device *plat_dev) /* Display HW versions */ dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X/0x%8X, Driver version %s\n", hw_rev->name, hw_rev_pidr, sig_cidr, DRV_MODULE_VERSION); + /* register the driver isr function */ + rc = devm_request_irq(dev, new_drvdata->irq, cc_isr, + IRQF_SHARED, "ccree", new_drvdata); + if (rc) { + dev_err(dev, "Could not register to interrupt %d\n", + new_drvdata->irq); + return rc; + } + dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq); rc = init_cc_regs(new_drvdata, true); if (rc) { From d84f6269ce24eb4c468e246b24fc0fdce34ab6f6 Mon Sep 17 00:00:00 2001 From: Ofir Drang Date: Mon, 17 Jun 2019 11:46:28 +0300 Subject: [PATCH 120/143] crypto: ccree - check that cryptocell reset completed In case of driver probe and pm resume we need to check that the cryptocell hardware reset cycle is completed. during the reset cycle that Cryptocell provide read only access to the APB interface which allows to verify through the CC registers that the reset is completed. Until reset completion we assume that any write/crypto operation is blocked. Signed-off-by: Ofir Drang Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_driver.c | 33 ++++++++++++++++++++++++++++- drivers/crypto/ccree/cc_driver.h | 3 +++ drivers/crypto/ccree/cc_host_regs.h | 3 +++ drivers/crypto/ccree/cc_pm.c | 5 +++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 0f80cb4f79fb..1e73d32148dd 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -48,6 +48,7 @@ struct cc_hw_data { }; #define CC_NUM_IDRS 4 +#define CC_HW_RESET_LOOP_COUNT 10 /* Note: PIDR3 holds CMOD/Rev so ignored for HW identification purposes */ static const u32 pidr_0124_offsets[CC_NUM_IDRS] = { @@ -188,6 +189,31 @@ static irqreturn_t cc_isr(int irq, void *dev_id) return IRQ_HANDLED; } +bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata) +{ + unsigned int val; + unsigned int i; + + /* 712/710/63 has no reset completion indication, always return true */ + if (drvdata->hw_rev <= CC_HW_REV_712) + return true; + + for (i = 0; i < CC_HW_RESET_LOOP_COUNT; i++) { + /* in cc7x3 NVM_IS_IDLE indicates that CC reset is + * completed and device is fully functional + */ + val = cc_ioread(drvdata, CC_REG(NVM_IS_IDLE)); + if (val & CC_NVM_IS_IDLE_MASK) { + /* hw indicate reset completed */ + return true; + } + /* allow scheduling other process on the processor */ + schedule(); + } + /* reset not completed */ + return false; +} + int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe) { unsigned int val, cache_params; @@ -343,6 +369,11 @@ static int init_cc_resources(struct platform_device *plat_dev) new_drvdata->sec_disabled = cc_sec_disable; + /* wait for Crytpcell reset completion */ + if (!cc_wait_for_reset_completion(new_drvdata)) { + dev_err(dev, "Cryptocell reset not completed"); + } + if (hw_rev->rev <= CC_HW_REV_712) { /* Verify correct mapping */ val = cc_ioread(new_drvdata, new_drvdata->sig_offset); @@ -398,7 +429,7 @@ static int init_cc_resources(struct platform_device *plat_dev) if (rc) { dev_err(dev, "Could not register to interrupt %d\n", new_drvdata->irq); - return rc; + goto post_clk_err; } dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq); diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index b76181335c08..579eecae4825 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -67,6 +67,8 @@ enum cc_std_body { #define CC_SECURITY_DISABLED_MASK BIT(CC_SECURITY_DISABLED_VALUE_BIT_SHIFT) +#define CC_NVM_IS_IDLE_MASK BIT(CC_NVM_IS_IDLE_VALUE_BIT_SHIFT) + #define AXIM_MON_COMP_VALUE GENMASK(CC_AXIM_MON_COMP_VALUE_BIT_SIZE + \ CC_AXIM_MON_COMP_VALUE_BIT_SHIFT, \ CC_AXIM_MON_COMP_VALUE_BIT_SHIFT) @@ -216,6 +218,7 @@ static inline void dump_byte_array(const char *name, const u8 *the_array, __dump_byte_array(name, the_array, size); } +bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata); int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe); void fini_cc_regs(struct cc_drvdata *drvdata); int cc_clk_on(struct cc_drvdata *drvdata); diff --git a/drivers/crypto/ccree/cc_host_regs.h b/drivers/crypto/ccree/cc_host_regs.h index d0764147573f..ad1acb105f2d 100644 --- a/drivers/crypto/ccree/cc_host_regs.h +++ b/drivers/crypto/ccree/cc_host_regs.h @@ -114,6 +114,9 @@ #define CC_HOST_ICR_DSCRPTR_WATERMARK_QUEUE0_CLEAR_BIT_SIZE 0x1UL #define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SHIFT 0x17UL #define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SIZE 0x1UL +#define CC_NVM_IS_IDLE_REG_OFFSET 0x0A10UL +#define CC_NVM_IS_IDLE_VALUE_BIT_SHIFT 0x0UL +#define CC_NVM_IS_IDLE_VALUE_BIT_SIZE 0x1UL #define CC_SECURITY_DISABLED_REG_OFFSET 0x0A1CUL #define CC_SECURITY_DISABLED_VALUE_BIT_SHIFT 0x0UL #define CC_SECURITY_DISABLED_VALUE_BIT_SIZE 0x1UL diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index 2dad9c9543c6..20f7f3d34e27 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -49,6 +49,11 @@ int cc_pm_resume(struct device *dev) dev_err(dev, "failed getting clock back on. We're toast.\n"); return rc; } + /* wait for Crytpcell reset completion */ + if (!cc_wait_for_reset_completion(drvdata)) { + dev_err(dev, "Cryptocell reset not completed"); + return -EBUSY; + } cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE); rc = init_cc_regs(drvdata, false); From 3db617e77ae2f8051fd6f5d3fff0e15f086d5b4c Mon Sep 17 00:00:00 2001 From: Ofir Drang Date: Mon, 17 Jun 2019 11:46:29 +0300 Subject: [PATCH 121/143] crypto: ccree - prevent isr handling in case driver is suspended ccree irq may be shared with other devices, in order to prevent ccree isr handling while device maybe suspended we added a check to verify that the device is not suspended. Signed-off-by: Ofir Drang Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_driver.c | 3 +++ drivers/crypto/ccree/cc_pm.c | 6 ++++++ drivers/crypto/ccree/cc_pm.h | 7 +++++++ 3 files changed, 16 insertions(+) diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 1e73d32148dd..667bc73d7a00 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -134,6 +134,9 @@ static irqreturn_t cc_isr(int irq, void *dev_id) u32 imr; /* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */ + /* if driver suspended return, probebly shared interrupt */ + if (cc_pm_is_dev_suspended(dev)) + return IRQ_NONE; /* read the interrupt status */ irr = cc_ioread(drvdata, CC_REG(HOST_IRR)); diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index 20f7f3d34e27..899a52f05b7a 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -106,6 +106,12 @@ int cc_pm_put_suspend(struct device *dev) return rc; } +bool cc_pm_is_dev_suspended(struct device *dev) +{ + /* check device state using runtime api */ + return pm_runtime_suspended(dev); +} + int cc_pm_init(struct cc_drvdata *drvdata) { struct device *dev = drvdata_to_dev(drvdata); diff --git a/drivers/crypto/ccree/cc_pm.h b/drivers/crypto/ccree/cc_pm.h index 6190cdba5dad..a7d98a5da2e1 100644 --- a/drivers/crypto/ccree/cc_pm.h +++ b/drivers/crypto/ccree/cc_pm.h @@ -22,6 +22,7 @@ int cc_pm_suspend(struct device *dev); int cc_pm_resume(struct device *dev); int cc_pm_get(struct device *dev); int cc_pm_put_suspend(struct device *dev); +bool cc_pm_is_dev_suspended(struct device *dev); #else @@ -54,6 +55,12 @@ static inline int cc_pm_put_suspend(struct device *dev) return 0; } +static inline bool cc_pm_is_dev_suspended(struct device *dev) +{ + /* if PM not supported device is never suspend */ + return false; +} + #endif #endif /*__POWER_MGR_H__*/ From 303f99ac9470a92c06fec1dfb5f263fbc26eb8f4 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Mon, 17 Jun 2019 11:46:30 +0300 Subject: [PATCH 122/143] crypto: ccree - add HW engine config check Add check to verify the stated device tree HW configuration matches the HW. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_driver.c | 18 ++++++++++++++++++ drivers/crypto/ccree/cc_driver.h | 3 +++ drivers/crypto/ccree/cc_host_regs.h | 17 +++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 667bc73d7a00..980aa04b655b 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -408,6 +408,24 @@ static int init_cc_resources(struct platform_device *plat_dev) } sig_cidr = val; + /* Check HW engine configuration */ + val = cc_ioread(new_drvdata, CC_REG(HOST_REMOVE_INPUT_PINS)); + switch (val) { + case CC_PINS_FULL: + /* This is fine */ + break; + case CC_PINS_SLIM: + if (new_drvdata->std_bodies & CC_STD_NIST) { + dev_warn(dev, "703 mode forced due to HW configuration.\n"); + new_drvdata->std_bodies = CC_STD_OSCCA; + } + break; + default: + dev_err(dev, "Unsupported engines configration.\n"); + rc = -EINVAL; + goto post_clk_err; + } + /* Check security disable state */ val = cc_ioread(new_drvdata, CC_REG(SECURITY_DISABLED)); val &= CC_SECURITY_DISABLED_MASK; diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index 579eecae4825..7cd99380bf1f 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -53,6 +53,9 @@ enum cc_std_body { #define CC_COHERENT_CACHE_PARAMS 0xEEE +#define CC_PINS_FULL 0x0 +#define CC_PINS_SLIM 0x9F + /* Maximum DMA mask supported by IP */ #define DMA_BIT_MASK_LEN 48 diff --git a/drivers/crypto/ccree/cc_host_regs.h b/drivers/crypto/ccree/cc_host_regs.h index ad1acb105f2d..efe3e1d8b87b 100644 --- a/drivers/crypto/ccree/cc_host_regs.h +++ b/drivers/crypto/ccree/cc_host_regs.h @@ -206,6 +206,23 @@ #define CC_HOST_POWER_DOWN_EN_REG_OFFSET 0xA78UL #define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SHIFT 0x0UL #define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REG_OFFSET 0x0A7CUL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_ENGINE_BIT_SHIFT 0x0UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_MAC_ENGINE_BIT_SHIFT 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_MAC_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_GHASH_ENGINE_BIT_SHIFT 0x2UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_GHASH_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_DES_ENGINE_BIT_SHIFT 0x3UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_DES_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_HASH_ENGINE_BIT_SHIFT 0x4UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_HASH_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM3_ENGINE_BIT_SHIFT 0x5UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM3_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM4_ENGINE_BIT_SHIFT 0x6UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM4_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_OTP_DISCONNECTED_BIT_SHIFT 0x7UL +#define CC_HOST_REMOVE_INPUT_PINS_OTP_DISCONNECTED_BIT_SIZE 0x1UL // -------------------------------------- // BLOCK: ID_REGISTERS // -------------------------------------- From 8d3bcb9900ca5a193088c1f2d20c92865482368b Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 17 Jun 2019 15:25:17 +0200 Subject: [PATCH 123/143] crypto: sun4i-ss - reduce stack usage After the latest addition, the stack usage of sun4i_ss_cipher_poll grew beyond the warning limit when KASAN is enabled: drivers/crypto/sunxi-ss/sun4i-ss-cipher.c:118:12: error: stack frame size of 1152 bytes in function 'sun4i_ss_cipher_poll' [-Werror,-Wframe-larger-than=] static int sun4i_ss_cipher_poll(struct skcipher_request *areq) Reduce it in three ways: - split out the new code into a separate function so its stack usage can overlap that of the sun4i_ss_opti_poll() code path - mark both special cases as noinline_for_stack, which should ideally result in a tail call that frees the rest of the stack - move the buf and obuf variables into the code blocks in which they are used. The three separate functions now use 144, 640 and 304 bytes of kernel stack, respectively. Fixes: 0ae1f46c55f8 ("crypto: sun4i-ss - fallback when length is not multiple of blocksize") Signed-off-by: Arnd Bergmann Tested-by: Corentin LABBE Signed-off-by: Herbert Xu --- drivers/crypto/sunxi-ss/sun4i-ss-cipher.c | 47 +++++++++++++++-------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c index b060a0810934..e87717e1e00a 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -16,7 +16,7 @@ */ #include "sun4i-ss.h" -static int sun4i_ss_opti_poll(struct skcipher_request *areq) +static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); @@ -118,6 +118,29 @@ release_ss: return err; } + +static int noinline_for_stack sun4i_ss_cipher_poll_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm); + int err; + + skcipher_request_set_sync_tfm(subreq, op->fallback_tfm); + skcipher_request_set_callback(subreq, areq->base.flags, NULL, + NULL); + skcipher_request_set_crypt(subreq, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (ctx->mode & SS_DECRYPTION) + err = crypto_skcipher_decrypt(subreq); + else + err = crypto_skcipher_encrypt(subreq); + skcipher_request_zero(subreq); + + return err; +} + /* Generic function that support SG with size not multiple of 4 */ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) { @@ -144,8 +167,6 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) unsigned int todo; struct sg_mapping_iter mi, mo; unsigned int oi, oo; /* offset for in and out */ - char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ - char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ unsigned int ob = 0; /* offset in buf */ unsigned int obo = 0; /* offset in bufo*/ unsigned int obl = 0; /* length of data in bufo */ @@ -182,20 +203,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) if (no_chunk == 1 && !need_fallback) return sun4i_ss_opti_poll(areq); - if (need_fallback) { - SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm); - skcipher_request_set_sync_tfm(subreq, op->fallback_tfm); - skcipher_request_set_callback(subreq, areq->base.flags, NULL, - NULL); - skcipher_request_set_crypt(subreq, areq->src, areq->dst, - areq->cryptlen, areq->iv); - if (ctx->mode & SS_DECRYPTION) - err = crypto_skcipher_decrypt(subreq); - else - err = crypto_skcipher_encrypt(subreq); - skcipher_request_zero(subreq); - return err; - } + if (need_fallback) + return sun4i_ss_cipher_poll_fallback(areq); spin_lock_irqsave(&ss->slock, flags); @@ -228,6 +237,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) while (oleft) { if (ileft) { + char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ + /* * todo is the number of consecutive 4byte word that we * can read from current SG @@ -285,6 +296,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) oo = 0; } } else { + char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ + /* * read obl bytes in bufo, we read at maximum for * emptying the device From c8c74647b2945e3522b77b659917766d7e1c6ed9 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 17 Jun 2019 21:14:45 +0000 Subject: [PATCH 124/143] crypto: talitos - eliminate unneeded 'done' functions at build time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building for SEC1 only, talitos2_done functions are unneeded and should go away. For this, use has_ftr_sec1() which will always return true when only SEC1 support is being built, allowing GCC to drop TALITOS2 functions. Signed-off-by: Christophe Leroy Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index c865f5d5eaba..cb2da0921392 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -3414,7 +3414,7 @@ static int talitos_probe(struct platform_device *ofdev) if (err) goto err_out; - if (of_device_is_compatible(np, "fsl,sec1.0")) { + if (has_ftr_sec1(priv)) { if (priv->num_channels == 1) tasklet_init(&priv->done_task[0], talitos1_done_ch0, (unsigned long)dev); From 6b5ca646ca9d99611e30f3c8f6b4837b9890eb73 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 18 Jun 2019 11:21:52 +0200 Subject: [PATCH 125/143] crypto: testmgr - dynamically allocate testvec_config On arm32, we get warnings about high stack usage in some of the functions: crypto/testmgr.c:2269:12: error: stack frame size of 1032 bytes in function 'alg_test_aead' [-Werror,-Wframe-larger-than=] static int alg_test_aead(const struct alg_test_desc *desc, const char *driver, ^ crypto/testmgr.c:1693:12: error: stack frame size of 1312 bytes in function '__alg_test_hash' [-Werror,-Wframe-larger-than=] static int __alg_test_hash(const struct hash_testvec *vecs, ^ On of the larger objects on the stack here is struct testvec_config, so change that to dynamic allocation. Fixes: 40153b10d91c ("crypto: testmgr - fuzz AEADs against their generic implementation") Fixes: d435e10e67be ("crypto: testmgr - fuzz skciphers against their generic implementation") Fixes: 9a8a6b3f0950 ("crypto: testmgr - fuzz hashes against their generic implementation") Signed-off-by: Arnd Bergmann Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 5d163dd2ffac..ace4c260ea5d 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1570,7 +1570,7 @@ static int test_hash_vs_generic_impl(const char *driver, unsigned int i; struct hash_testvec vec = { 0 }; char vec_name[64]; - struct testvec_config cfg; + struct testvec_config *cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; @@ -1600,6 +1600,12 @@ static int test_hash_vs_generic_impl(const char *driver, return err; } + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) { + err = -ENOMEM; + goto out; + } + /* Check the algorithm properties for consistency. */ if (digestsize != crypto_shash_digestsize(generic_tfm)) { @@ -1634,9 +1640,9 @@ static int test_hash_vs_generic_impl(const char *driver, generate_random_hash_testvec(generic_tfm, &vec, maxkeysize, maxdatasize, vec_name, sizeof(vec_name)); - generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); - err = test_hash_vec_cfg(driver, &vec, vec_name, &cfg, + err = test_hash_vec_cfg(driver, &vec, vec_name, cfg, req, desc, tsgl, hashstate); if (err) goto out; @@ -1644,6 +1650,7 @@ static int test_hash_vs_generic_impl(const char *driver, } err = 0; out: + kfree(cfg); kfree(vec.key); kfree(vec.plaintext); kfree(vec.digest); @@ -2140,7 +2147,7 @@ static int test_aead_vs_generic_impl(const char *driver, unsigned int i; struct aead_testvec vec = { 0 }; char vec_name[64]; - struct testvec_config cfg; + struct testvec_config *cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; @@ -2170,6 +2177,12 @@ static int test_aead_vs_generic_impl(const char *driver, return err; } + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) { + err = -ENOMEM; + goto out; + } + generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL); if (!generic_req) { err = -ENOMEM; @@ -2224,13 +2237,13 @@ static int test_aead_vs_generic_impl(const char *driver, generate_random_aead_testvec(generic_req, &vec, maxkeysize, maxdatasize, vec_name, sizeof(vec_name)); - generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); - err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, &cfg, + err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, cfg, req, tsgls); if (err) goto out; - err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, &cfg, + err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, cfg, req, tsgls); if (err) goto out; @@ -2238,6 +2251,7 @@ static int test_aead_vs_generic_impl(const char *driver, } err = 0; out: + kfree(cfg); kfree(vec.key); kfree(vec.iv); kfree(vec.assoc); @@ -2687,7 +2701,7 @@ static int test_skcipher_vs_generic_impl(const char *driver, unsigned int i; struct cipher_testvec vec = { 0 }; char vec_name[64]; - struct testvec_config cfg; + struct testvec_config *cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; @@ -2721,6 +2735,12 @@ static int test_skcipher_vs_generic_impl(const char *driver, return err; } + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) { + err = -ENOMEM; + goto out; + } + generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL); if (!generic_req) { err = -ENOMEM; @@ -2768,20 +2788,21 @@ static int test_skcipher_vs_generic_impl(const char *driver, for (i = 0; i < fuzz_iterations * 8; i++) { generate_random_cipher_testvec(generic_req, &vec, maxdatasize, vec_name, sizeof(vec_name)); - generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name, - &cfg, req, tsgls); + cfg, req, tsgls); if (err) goto out; err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name, - &cfg, req, tsgls); + cfg, req, tsgls); if (err) goto out; cond_resched(); } err = 0; out: + kfree(cfg); kfree(vec.key); kfree(vec.iv); kfree(vec.ptext); From 149c4e6ef7788d58b9c05eed9fb85e0f5a2c3456 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 18 Jun 2019 11:21:53 +0200 Subject: [PATCH 126/143] crypto: testmgr - dynamically allocate crypto_shash The largest stack object in this file is now the shash descriptor. Since there are many other stack variables, this can push it over the 1024 byte warning limit, in particular with clang and KASAN: crypto/testmgr.c:1693:12: error: stack frame size of 1312 bytes in function '__alg_test_hash' [-Werror,-Wframe-larger-than=] Make test_hash_vs_generic_impl() do the same thing as the corresponding eaed and skcipher functions by allocating the descriptor dynamically. We can still do better than this, but it brings us well below the 1024 byte limit. Suggested-by: Eric Biggers Fixes: 9a8a6b3f0950 ("crypto: testmgr - fuzz hashes against their generic implementation") Signed-off-by: Arnd Bergmann Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index ace4c260ea5d..d760f5cd35b2 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1508,14 +1508,12 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, * Generate a hash test vector from the given implementation. * Assumes the buffers in 'vec' were already allocated. */ -static void generate_random_hash_testvec(struct crypto_shash *tfm, +static void generate_random_hash_testvec(struct shash_desc *desc, struct hash_testvec *vec, unsigned int maxkeysize, unsigned int maxdatasize, char *name, size_t max_namelen) { - SHASH_DESC_ON_STACK(desc, tfm); - /* Data */ vec->psize = generate_random_length(maxdatasize); generate_random_bytes((u8 *)vec->plaintext, vec->psize); @@ -1532,7 +1530,7 @@ static void generate_random_hash_testvec(struct crypto_shash *tfm, vec->ksize = 1 + (prandom_u32() % maxkeysize); generate_random_bytes((u8 *)vec->key, vec->ksize); - vec->setkey_error = crypto_shash_setkey(tfm, vec->key, + vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key, vec->ksize); /* If the key couldn't be set, no need to continue to digest. */ if (vec->setkey_error) @@ -1540,7 +1538,6 @@ static void generate_random_hash_testvec(struct crypto_shash *tfm, } /* Digest */ - desc->tfm = tfm; vec->digest_error = crypto_shash_digest(desc, vec->plaintext, vec->psize, (u8 *)vec->digest); done: @@ -1567,6 +1564,7 @@ static int test_hash_vs_generic_impl(const char *driver, const char *algname = crypto_hash_alg_common(tfm)->base.cra_name; char _generic_driver[CRYPTO_MAX_ALG_NAME]; struct crypto_shash *generic_tfm = NULL; + struct shash_desc *generic_desc = NULL; unsigned int i; struct hash_testvec vec = { 0 }; char vec_name[64]; @@ -1606,6 +1604,14 @@ static int test_hash_vs_generic_impl(const char *driver, goto out; } + generic_desc = kzalloc(sizeof(*desc) + + crypto_shash_descsize(generic_tfm), GFP_KERNEL); + if (!generic_desc) { + err = -ENOMEM; + goto out; + } + generic_desc->tfm = generic_tfm; + /* Check the algorithm properties for consistency. */ if (digestsize != crypto_shash_digestsize(generic_tfm)) { @@ -1637,7 +1643,7 @@ static int test_hash_vs_generic_impl(const char *driver, } for (i = 0; i < fuzz_iterations * 8; i++) { - generate_random_hash_testvec(generic_tfm, &vec, + generate_random_hash_testvec(generic_desc, &vec, maxkeysize, maxdatasize, vec_name, sizeof(vec_name)); generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); @@ -1655,6 +1661,7 @@ out: kfree(vec.plaintext); kfree(vec.digest); crypto_free_shash(generic_tfm); + kzfree(generic_desc); return err; } #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ From 473971187d6727609951858c63bf12b0307ef015 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 18 Jun 2019 13:19:42 +0200 Subject: [PATCH 127/143] crypto: serpent - mark __serpent_setkey_sbox noinline The same bug that gcc hit in the past is apparently now showing up with clang, which decides to inline __serpent_setkey_sbox: crypto/serpent_generic.c:268:5: error: stack frame size of 2112 bytes in function '__serpent_setkey' [-Werror,-Wframe-larger-than=] Marking it 'noinline' reduces the stack usage from 2112 bytes to 192 and 96 bytes, respectively, and seems to generate more useful object code. Fixes: c871c10e4ea7 ("crypto: serpent - improve __serpent_setkey with UBSAN") Signed-off-by: Arnd Bergmann Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/serpent_generic.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index f2f549330d2b..e133006b2217 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -229,7 +229,13 @@ x4 ^= x2; \ }) -static void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2, u32 r3, u32 r4, u32 *k) +/* + * both gcc and clang have misoptimized this function in the past, + * producing horrible object code from spilling temporary variables + * on the stack. Forcing this part out of line avoids that. + */ +static noinline void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2, + u32 r3, u32 r4, u32 *k) { k += 100; S3(r3, r4, r0, r1, r2); store_and_load_keys(r1, r2, r4, r3, 28, 24); From 90acc0653d2bee203174e66d519fbaaa513502de Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 18 Jun 2019 14:13:47 +0200 Subject: [PATCH 128/143] crypto: asymmetric_keys - select CRYPTO_HASH where needed Build testing with some core crypto options disabled revealed a few modules that are missing CRYPTO_HASH: crypto/asymmetric_keys/x509_public_key.o: In function `x509_get_sig_params': x509_public_key.c:(.text+0x4c7): undefined reference to `crypto_alloc_shash' x509_public_key.c:(.text+0x5e5): undefined reference to `crypto_shash_digest' crypto/asymmetric_keys/pkcs7_verify.o: In function `pkcs7_digest.isra.0': pkcs7_verify.c:(.text+0xab): undefined reference to `crypto_alloc_shash' pkcs7_verify.c:(.text+0x1b2): undefined reference to `crypto_shash_digest' pkcs7_verify.c:(.text+0x3c1): undefined reference to `crypto_shash_update' pkcs7_verify.c:(.text+0x411): undefined reference to `crypto_shash_finup' This normally doesn't show up in randconfig tests because there is a large number of other options that select CRYPTO_HASH. Signed-off-by: Arnd Bergmann Signed-off-by: Herbert Xu --- crypto/asymmetric_keys/Kconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/asymmetric_keys/Kconfig b/crypto/asymmetric_keys/Kconfig index be70ca6c85d3..1f1f004dc757 100644 --- a/crypto/asymmetric_keys/Kconfig +++ b/crypto/asymmetric_keys/Kconfig @@ -15,6 +15,7 @@ config ASYMMETRIC_PUBLIC_KEY_SUBTYPE select MPILIB select CRYPTO_HASH_INFO select CRYPTO_AKCIPHER + select CRYPTO_HASH help This option provides support for asymmetric public key type handling. If signature generation and/or verification are to be used, @@ -65,6 +66,7 @@ config TPM_KEY_PARSER config PKCS7_MESSAGE_PARSER tristate "PKCS#7 message parser" depends on X509_CERTIFICATE_PARSER + select CRYPTO_HASH select ASN1 select OID_REGISTRY help @@ -87,6 +89,7 @@ config SIGNED_PE_FILE_VERIFICATION bool "Support for PE file signature verification" depends on PKCS7_MESSAGE_PARSER=y depends on SYSTEM_DATA_VERIFICATION + select CRYPTO_HASH select ASN1 select OID_REGISTRY help From e52d484d9869eb291140545746ccbe5ffc7c9306 Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Tue, 18 Jun 2019 12:09:22 -0700 Subject: [PATCH 129/143] crypto/NX: Set receive window credits to max number of CRBs in RxFIFO System gets checkstop if RxFIFO overruns with more requests than the maximum possible number of CRBs in FIFO at the same time. The max number of requests per window is controlled by window credits. So find max CRBs from FIFO size and set it to receive window credits. Fixes: b0d6c9bab5e4 ("crypto/nx: Add P9 NX support for 842 compression engine") CC: stable@vger.kernel.org # v4.14+ Signed-off-by:Haren Myneni Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx-842-powernv.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index c68df7e8bee1..7ce2467c771e 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -36,8 +36,6 @@ MODULE_ALIAS_CRYPTO("842-nx"); #define WORKMEM_ALIGN (CRB_ALIGN) #define CSB_WAIT_MAX (5000) /* ms */ #define VAS_RETRIES (10) -/* # of requests allowed per RxFIFO at a time. 0 for unlimited */ -#define MAX_CREDITS_PER_RXFIFO (1024) struct nx842_workmem { /* Below fields must be properly aligned */ @@ -821,7 +819,11 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id, rxattr.lnotify_lpid = lpid; rxattr.lnotify_pid = pid; rxattr.lnotify_tid = tid; - rxattr.wcreds_max = MAX_CREDITS_PER_RXFIFO; + /* + * Maximum RX window credits can not be more than #CRBs in + * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns. + */ + rxattr.wcreds_max = fifo_size / CRB_SIZE; /* * Open a VAS receice window which is used to configure RxFIFO From aeb87246537a83c2aff482f3f34a2e0991e02cbc Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 24 Jun 2019 07:20:14 +0000 Subject: [PATCH 130/143] lib/scatterlist: Fix mapping iterator when sg->offset is greater than PAGE_SIZE All mapping iterator logic is based on the assumption that sg->offset is always lower than PAGE_SIZE. But there are situations where sg->offset is such that the SG item is on the second page. In that case sg_copy_to_buffer() fails properly copying the data into the buffer. One of the reason is that the data will be outside the kmapped area used to access that data. This patch fixes the issue by adjusting the mapping iterator offset and pgoffset fields such that offset is always lower than PAGE_SIZE. Signed-off-by: Christophe Leroy Fixes: 4225fc8555a9 ("lib/scatterlist: use page iterator in the mapping iterator") Cc: stable@vger.kernel.org Signed-off-by: Herbert Xu --- lib/scatterlist.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/scatterlist.c b/lib/scatterlist.c index 739dc9fe2c55..f0757a67affe 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -678,17 +678,18 @@ static bool sg_miter_get_next_page(struct sg_mapping_iter *miter) { if (!miter->__remaining) { struct scatterlist *sg; - unsigned long pgoffset; if (!__sg_page_iter_next(&miter->piter)) return false; sg = miter->piter.sg; - pgoffset = miter->piter.sg_pgoffset; - miter->__offset = pgoffset ? 0 : sg->offset; + miter->__offset = miter->piter.sg_pgoffset ? 0 : sg->offset; + miter->piter.sg_pgoffset += miter->__offset >> PAGE_SHIFT; + miter->__offset &= PAGE_SIZE - 1; miter->__remaining = sg->offset + sg->length - - (pgoffset << PAGE_SHIFT) - miter->__offset; + (miter->piter.sg_pgoffset << PAGE_SHIFT) - + miter->__offset; miter->__remaining = min_t(unsigned long, miter->__remaining, PAGE_SIZE - miter->__offset); } From d44769e4ccb636e8238adbc151f25467a536711b Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 24 Jun 2019 07:20:15 +0000 Subject: [PATCH 131/143] crypto: talitos - move struct talitos_edesc into talitos.h Moves struct talitos_edesc into talitos.h so that it can be used from any place in talitos.c It will be required for next patch ("crypto: talitos - fix hash on SEC1") Signed-off-by: Christophe Leroy Cc: stable@vger.kernel.org Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 30 ------------------------------ drivers/crypto/talitos.h | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index cb2da0921392..3918b3dbe684 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -951,36 +951,6 @@ badkey: goto out; } -/* - * talitos_edesc - s/w-extended descriptor - * @src_nents: number of segments in input scatterlist - * @dst_nents: number of segments in output scatterlist - * @icv_ool: whether ICV is out-of-line - * @iv_dma: dma address of iv for checking continuity and link table - * @dma_len: length of dma mapped link_tbl space - * @dma_link_tbl: bus physical address of link_tbl/buf - * @desc: h/w descriptor - * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) - * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) - * - * if decrypting (with authcheck), or either one of src_nents or dst_nents - * is greater than 1, an integrity check value is concatenated to the end - * of link_tbl data - */ -struct talitos_edesc { - int src_nents; - int dst_nents; - bool icv_ool; - dma_addr_t iv_dma; - int dma_len; - dma_addr_t dma_link_tbl; - struct talitos_desc desc; - union { - struct talitos_ptr link_tbl[0]; - u8 buf[0]; - }; -}; - static void talitos_sg_unmap(struct device *dev, struct talitos_edesc *edesc, struct scatterlist *src, diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 32ad4fc679ed..95f78c6d9206 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -42,6 +42,36 @@ struct talitos_desc { #define TALITOS_DESC_SIZE (sizeof(struct talitos_desc) - sizeof(__be32)) +/* + * talitos_edesc - s/w-extended descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @icv_ool: whether ICV is out-of-line + * @iv_dma: dma address of iv for checking continuity and link table + * @dma_len: length of dma mapped link_tbl space + * @dma_link_tbl: bus physical address of link_tbl/buf + * @desc: h/w descriptor + * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) + * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) + * + * if decrypting (with authcheck), or either one of src_nents or dst_nents + * is greater than 1, an integrity check value is concatenated to the end + * of link_tbl data + */ +struct talitos_edesc { + int src_nents; + int dst_nents; + bool icv_ool; + dma_addr_t iv_dma; + int dma_len; + dma_addr_t dma_link_tbl; + struct talitos_desc desc; + union { + struct talitos_ptr link_tbl[0]; + u8 buf[0]; + }; +}; + /** * talitos_request - descriptor submission request * @desc: descriptor pointer (kernel virtual) From 58cdbc6d2263beb36954408522762bbe73169306 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 24 Jun 2019 07:20:16 +0000 Subject: [PATCH 132/143] crypto: talitos - fix hash on SEC1. On SEC1, hash provides wrong result when performing hashing in several steps with input data SG list has more than one element. This was detected with CONFIG_CRYPTO_MANAGER_EXTRA_TESTS: [ 44.185947] alg: hash: md5-talitos test failed (wrong result) on test vector 6, cfg="random: may_sleep use_finup src_divs=[25.88%@+8063, 24.19%@+9588, 28.63%@+16333, 4.60%@+6756, 16.70%@+16281] dst_divs=[71.61%@alignmask+16361, 14.36%@+7756, 14.3%@+" [ 44.325122] alg: hash: sha1-talitos test failed (wrong result) on test vector 3, cfg="random: inplace use_final src_divs=[16.56%@+16378, 52.0%@+16329, 21.42%@alignmask+16380, 10.2%@alignmask+16380] iv_offset=39" [ 44.493500] alg: hash: sha224-talitos test failed (wrong result) on test vector 4, cfg="random: use_final nosimd src_divs=[52.27%@+7401, 17.34%@+16285, 17.71%@+26, 12.68%@+10644] iv_offset=43" [ 44.673262] alg: hash: sha256-talitos test failed (wrong result) on test vector 4, cfg="random: may_sleep use_finup src_divs=[60.6%@+12790, 17.86%@+1329, 12.64%@alignmask+16300, 8.29%@+15, 0.40%@+13506, 0.51%@+16322, 0.24%@+16339] dst_divs" This is due to two issues: - We have an overlap between the buffer used for copying the input data (SEC1 doesn't do scatter/gather) and the chained descriptor. - Data copy is wrong when the previous hash left less than one blocksize of data to hash, implying a complement of the previous block with a few bytes from the new request. Fix it by: - Moving the second descriptor after the buffer, as moving the buffer after the descriptor would make it more complex for other cipher operations (AEAD, ABLKCIPHER) - Skip the bytes taken from the new request to complete the previous one by moving the SG list forward. Fixes: 37b5e8897eb5 ("crypto: talitos - chain in buffered data for ahash on SEC1") Cc: stable@vger.kernel.org Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 69 ++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 3918b3dbe684..c431d96e5596 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -320,6 +320,21 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, return -EINPROGRESS; } +static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1) +{ + struct talitos_edesc *edesc; + + if (!is_sec1) + return request->desc->hdr; + + if (!request->desc->next_desc) + return request->desc->hdr1; + + edesc = container_of(request->desc, struct talitos_edesc, desc); + + return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1; +} + /* * process what was done, notify callback of error if not */ @@ -341,12 +356,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) /* descriptors with their done bits set don't get the error */ rmb(); - if (!is_sec1) - hdr = request->desc->hdr; - else if (request->desc->next_desc) - hdr = (request->desc + 1)->hdr1; - else - hdr = request->desc->hdr1; + hdr = get_request_hdr(request, is_sec1); if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; @@ -476,8 +486,14 @@ static u32 current_desc_hdr(struct device *dev, int ch) } } - if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) - return (priv->chan[ch].fifo[iter].desc + 1)->hdr; + if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) { + struct talitos_edesc *edesc; + + edesc = container_of(priv->chan[ch].fifo[iter].desc, + struct talitos_edesc, desc); + return ((struct talitos_desc *) + (edesc->buf + edesc->dma_len))->hdr; + } return priv->chan[ch].fifo[iter].desc->hdr; } @@ -1402,15 +1418,11 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; edesc->dma_len = dma_len; - if (dma_len) { - void *addr = &edesc->link_tbl[0]; - - if (is_sec1 && !dst) - addr += sizeof(struct talitos_desc); - edesc->dma_link_tbl = dma_map_single(dev, addr, + if (dma_len) + edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], edesc->dma_len, DMA_BIDIRECTIONAL); - } + return edesc; } @@ -1722,14 +1734,16 @@ static void common_nonsnoop_hash_unmap(struct device *dev, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); struct talitos_desc *desc = &edesc->desc; - struct talitos_desc *desc2 = desc + 1; + struct talitos_desc *desc2 = (struct talitos_desc *) + (edesc->buf + edesc->dma_len); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); if (desc->next_desc && desc->ptr[5].ptr != desc2->ptr[5].ptr) unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE); - talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); + if (req_ctx->psrc) + talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); /* When using hashctx-in, must unmap it. */ if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) @@ -1796,7 +1810,6 @@ static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, static int common_nonsnoop_hash(struct talitos_edesc *edesc, struct ahash_request *areq, unsigned int length, - unsigned int offset, void (*callback) (struct device *dev, struct talitos_desc *desc, void *context, int error)) @@ -1835,9 +1848,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) - sg_pcopy_to_buffer(req_ctx->psrc, sg_count, - edesc->buf + sizeof(struct talitos_desc), - length, req_ctx->nbuf); + sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length); else if (length) sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, DMA_TO_DEVICE); @@ -1850,7 +1861,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, DMA_TO_DEVICE); } else { sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc->ptr[3], sg_count, offset, 0); + &desc->ptr[3], sg_count, 0, 0); if (sg_count > 1) sync_needed = true; } @@ -1874,7 +1885,8 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); if (is_sec1 && req_ctx->nbuf && length) { - struct talitos_desc *desc2 = desc + 1; + struct talitos_desc *desc2 = (struct talitos_desc *) + (edesc->buf + edesc->dma_len); dma_addr_t next_desc; memset(desc2, 0, sizeof(*desc2)); @@ -1895,7 +1907,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, DMA_TO_DEVICE); copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1); sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc2->ptr[3], sg_count, offset, 0); + &desc2->ptr[3], sg_count, 0, 0); if (sg_count > 1) sync_needed = true; copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1); @@ -2006,7 +2018,6 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) struct device *dev = ctx->dev; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); - int offset = 0; u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { @@ -2046,6 +2057,8 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { + int offset; + if (nbytes_to_hash > blocksize) offset = blocksize - req_ctx->nbuf; else @@ -2058,7 +2071,8 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) sg_copy_to_buffer(areq->src, nents, ctx_buf + req_ctx->nbuf, offset); req_ctx->nbuf += offset; - req_ctx->psrc = areq->src; + req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src, + offset); } else req_ctx->psrc = areq->src; @@ -2098,8 +2112,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) if (ctx->keylen && (req_ctx->first || req_ctx->last)) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; - return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset, - ahash_done); + return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done); } static int ahash_update(struct ahash_request *areq) From d45b1714e25e39b94df3da10ba8e95e63865b278 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 24 Jun 2019 07:20:17 +0000 Subject: [PATCH 133/143] crypto: talitos - drop icv_ool icv_ool is not used anymore, drop it. Fixes: e345177ded17 ("crypto: talitos - fix AEAD processing.") Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 3 --- drivers/crypto/talitos.h | 2 -- 2 files changed, 5 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index c431d96e5596..0683a09a9f55 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1285,9 +1285,6 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, is_ipsec_esp && !encrypt); tbl_off += ret; - /* ICV data */ - edesc->icv_ool = !encrypt; - if (!encrypt && is_ipsec_esp) { struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 95f78c6d9206..1469b956948a 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -46,7 +46,6 @@ struct talitos_desc { * talitos_edesc - s/w-extended descriptor * @src_nents: number of segments in input scatterlist * @dst_nents: number of segments in output scatterlist - * @icv_ool: whether ICV is out-of-line * @iv_dma: dma address of iv for checking continuity and link table * @dma_len: length of dma mapped link_tbl space * @dma_link_tbl: bus physical address of link_tbl/buf @@ -61,7 +60,6 @@ struct talitos_desc { struct talitos_edesc { int src_nents; int dst_nents; - bool icv_ool; dma_addr_t iv_dma; int dma_len; dma_addr_t dma_link_tbl; From e217413964a453fc2eeb437c32deb00581cf899d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jun 2019 19:38:30 +0200 Subject: [PATCH 134/143] crypto: arm64/aes-ce - add 5 way interleave routines In preparation of tweaking the accelerated AES chaining mode routines to be able to use a 5-way stride, implement the core routines to support processing 5 blocks of input at a time. While at it, drop the 2 way versions, which have been unused for a while now. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-ce.S | 58 +++++++++++++++++++++-------------- arch/arm64/crypto/aes-modes.S | 16 ++++++++++ arch/arm64/crypto/aes-neon.S | 46 +-------------------------- 3 files changed, 52 insertions(+), 68 deletions(-) diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S index 143070510809..0fca5f463406 100644 --- a/arch/arm64/crypto/aes-ce.S +++ b/arch/arm64/crypto/aes-ce.S @@ -52,7 +52,7 @@ load_round_keys \rounds, \temp .endm - .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3 + .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4 aes\de \i0\().16b, \k\().16b aes\mc \i0\().16b, \i0\().16b .ifnb \i1 @@ -63,27 +63,34 @@ aes\mc \i2\().16b, \i2\().16b aes\de \i3\().16b, \k\().16b aes\mc \i3\().16b, \i3\().16b + .ifnb \i4 + aes\de \i4\().16b, \k\().16b + aes\mc \i4\().16b, \i4\().16b + .endif .endif .endif .endm - /* up to 4 interleaved encryption rounds with the same round key */ - .macro round_Nx, enc, k, i0, i1, i2, i3 + /* up to 5 interleaved encryption rounds with the same round key */ + .macro round_Nx, enc, k, i0, i1, i2, i3, i4 .ifc \enc, e - do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3 + do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3, \i4 .else - do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3 + do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3, \i4 .endif .endm - /* up to 4 interleaved final rounds */ - .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3 + /* up to 5 interleaved final rounds */ + .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4 aes\de \i0\().16b, \k\().16b .ifnb \i1 aes\de \i1\().16b, \k\().16b .ifnb \i3 aes\de \i2\().16b, \k\().16b aes\de \i3\().16b, \k\().16b + .ifnb \i4 + aes\de \i4\().16b, \k\().16b + .endif .endif .endif eor \i0\().16b, \i0\().16b, \k2\().16b @@ -92,47 +99,52 @@ .ifnb \i3 eor \i2\().16b, \i2\().16b, \k2\().16b eor \i3\().16b, \i3\().16b, \k2\().16b + .ifnb \i4 + eor \i4\().16b, \i4\().16b, \k2\().16b + .endif .endif .endif .endm - /* up to 4 interleaved blocks */ - .macro do_block_Nx, enc, rounds, i0, i1, i2, i3 + /* up to 5 interleaved blocks */ + .macro do_block_Nx, enc, rounds, i0, i1, i2, i3, i4 cmp \rounds, #12 blo 2222f /* 128 bits */ beq 1111f /* 192 bits */ - round_Nx \enc, v17, \i0, \i1, \i2, \i3 - round_Nx \enc, v18, \i0, \i1, \i2, \i3 -1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3 - round_Nx \enc, v20, \i0, \i1, \i2, \i3 + round_Nx \enc, v17, \i0, \i1, \i2, \i3, \i4 + round_Nx \enc, v18, \i0, \i1, \i2, \i3, \i4 +1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3, \i4 + round_Nx \enc, v20, \i0, \i1, \i2, \i3, \i4 2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29 - round_Nx \enc, \key, \i0, \i1, \i2, \i3 + round_Nx \enc, \key, \i0, \i1, \i2, \i3, \i4 .endr - fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3 + fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3, \i4 .endm .macro encrypt_block, in, rounds, t0, t1, t2 do_block_Nx e, \rounds, \in .endm - .macro encrypt_block2x, i0, i1, rounds, t0, t1, t2 - do_block_Nx e, \rounds, \i0, \i1 - .endm - .macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 do_block_Nx e, \rounds, \i0, \i1, \i2, \i3 .endm + .macro encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \i0, \i1, \i2, \i3, \i4 + .endm + .macro decrypt_block, in, rounds, t0, t1, t2 do_block_Nx d, \rounds, \in .endm - .macro decrypt_block2x, i0, i1, rounds, t0, t1, t2 - do_block_Nx d, \rounds, \i0, \i1 - .endm - .macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 do_block_Nx d, \rounds, \i0, \i1, \i2, \i3 .endm + .macro decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \i0, \i1, \i2, \i3, \i4 + .endm + +#define MAX_STRIDE 5 + #include "aes-modes.S" diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index 4c7ce231963c..add6267f9e3a 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -13,6 +13,10 @@ .text .align 4 +#ifndef MAX_STRIDE +#define MAX_STRIDE 4 +#endif + aes_encrypt_block4x: encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 ret @@ -23,6 +27,18 @@ aes_decrypt_block4x: ret ENDPROC(aes_decrypt_block4x) +#if MAX_STRIDE == 5 +aes_encrypt_block5x: + encrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7 + ret +ENDPROC(aes_encrypt_block5x) + +aes_decrypt_block5x: + decrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7 + ret +ENDPROC(aes_decrypt_block5x) +#endif + /* * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, * int blocks) diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S index 29100f692e8a..33bb6af309a3 100644 --- a/arch/arm64/crypto/aes-neon.S +++ b/arch/arm64/crypto/aes-neon.S @@ -117,26 +117,9 @@ /* * Interleaved versions: functionally equivalent to the - * ones above, but applied to 2 or 4 AES states in parallel. + * ones above, but applied to AES states in parallel. */ - .macro sub_bytes_2x, in0, in1 - sub v8.16b, \in0\().16b, v15.16b - tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b - sub v9.16b, \in1\().16b, v15.16b - tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b - sub v10.16b, v8.16b, v15.16b - tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b - sub v11.16b, v9.16b, v15.16b - tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b - sub v8.16b, v10.16b, v15.16b - tbx \in0\().16b, {v24.16b-v27.16b}, v10.16b - sub v9.16b, v11.16b, v15.16b - tbx \in1\().16b, {v24.16b-v27.16b}, v11.16b - tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b - tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b - .endm - .macro sub_bytes_4x, in0, in1, in2, in3 sub v8.16b, \in0\().16b, v15.16b tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b @@ -215,25 +198,6 @@ eor \in1\().16b, \in1\().16b, v11.16b .endm - .macro do_block_2x, enc, in0, in1, rounds, rk, rkp, i - ld1 {v15.4s}, [\rk] - add \rkp, \rk, #16 - mov \i, \rounds -1111: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ - eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ - movi v15.16b, #0x40 - tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */ - tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */ - sub_bytes_2x \in0, \in1 - subs \i, \i, #1 - ld1 {v15.4s}, [\rkp], #16 - beq 2222f - mix_columns_2x \in0, \in1, \enc - b 1111b -2222: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ - eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ - .endm - .macro do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i ld1 {v15.4s}, [\rk] add \rkp, \rk, #16 @@ -260,14 +224,6 @@ eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */ .endm - .macro encrypt_block2x, in0, in1, rounds, rk, rkp, i - do_block_2x 1, \in0, \in1, \rounds, \rk, \rkp, \i - .endm - - .macro decrypt_block2x, in0, in1, rounds, rk, rkp, i - do_block_2x 0, \in0, \in1, \rounds, \rk, \rkp, \i - .endm - .macro encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i do_block_4x 1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i .endm From 7367bfeb2c141fb3ddff6b09bb5dfeb739b3d245 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jun 2019 19:38:31 +0200 Subject: [PATCH 135/143] crypto: arm64/aes-ce - implement 5 way interleave for ECB, CBC and CTR This implements 5-way interleaving for ECB, CBC decryption and CTR, resulting in a speedup of ~11% on Marvell ThunderX2, which has a very deep pipeline and therefore a high issue latency for NEON instructions operating on the same registers. Note that XTS is left alone: implementing 5-way interleave there would either involve spilling of the calculated tweaks to the stack, or recalculating them after the encryption operation, and doing either of those would most likely penalize low end cores. For ECB, this is not a concern at all, given that we have plenty of spare registers. For CTR and CBC decryption, we take advantage of the fact that v16 is not used by the CE version of the code (which is the only one targeted by the optimization), and so we can reshuffle the code a bit and avoid having to spill to memory (with the exception of one extra reload in the CBC routine) Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-ce.S | 2 + arch/arm64/crypto/aes-modes.S | 102 +++++++++++++++++++++++----------- arch/arm64/crypto/aes-neon.S | 2 + 3 files changed, 75 insertions(+), 31 deletions(-) diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S index 0fca5f463406..dbfc04ea20c7 100644 --- a/arch/arm64/crypto/aes-ce.S +++ b/arch/arm64/crypto/aes-ce.S @@ -18,6 +18,8 @@ .arch armv8-a+crypto xtsmask .req v16 + cbciv .req v16 + vctr .req v16 .macro xts_reload_mask, tmp .endm diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index add6267f9e3a..3f60f7b152bf 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -17,6 +17,14 @@ #define MAX_STRIDE 4 #endif +#if MAX_STRIDE == 4 +#define ST4(x...) x +#define ST5(x...) +#else +#define ST4(x...) +#define ST5(x...) x +#endif + aes_encrypt_block4x: encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 ret @@ -53,14 +61,17 @@ AES_ENTRY(aes_ecb_encrypt) enc_prepare w3, x2, x5 .LecbencloopNx: - subs w4, w4, #4 + subs w4, w4, #MAX_STRIDE bmi .Lecbenc1x ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ - bl aes_encrypt_block4x +ST4( bl aes_encrypt_block4x ) +ST5( ld1 {v4.16b}, [x1], #16 ) +ST5( bl aes_encrypt_block5x ) st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) b .LecbencloopNx .Lecbenc1x: - adds w4, w4, #4 + adds w4, w4, #MAX_STRIDE beq .Lecbencout .Lecbencloop: ld1 {v0.16b}, [x1], #16 /* get next pt block */ @@ -81,14 +92,17 @@ AES_ENTRY(aes_ecb_decrypt) dec_prepare w3, x2, x5 .LecbdecloopNx: - subs w4, w4, #4 + subs w4, w4, #MAX_STRIDE bmi .Lecbdec1x ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ - bl aes_decrypt_block4x +ST4( bl aes_decrypt_block4x ) +ST5( ld1 {v4.16b}, [x1], #16 ) +ST5( bl aes_decrypt_block5x ) st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) b .LecbdecloopNx .Lecbdec1x: - adds w4, w4, #4 + adds w4, w4, #MAX_STRIDE beq .Lecbdecout .Lecbdecloop: ld1 {v0.16b}, [x1], #16 /* get next ct block */ @@ -148,39 +162,56 @@ AES_ENTRY(aes_cbc_decrypt) stp x29, x30, [sp, #-16]! mov x29, sp - ld1 {v7.16b}, [x5] /* get iv */ + ld1 {cbciv.16b}, [x5] /* get iv */ dec_prepare w3, x2, x6 .LcbcdecloopNx: - subs w4, w4, #4 + subs w4, w4, #MAX_STRIDE bmi .Lcbcdec1x ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ +#if MAX_STRIDE == 5 + ld1 {v4.16b}, [x1], #16 /* get 1 ct block */ + mov v5.16b, v0.16b + mov v6.16b, v1.16b + mov v7.16b, v2.16b + bl aes_decrypt_block5x + sub x1, x1, #32 + eor v0.16b, v0.16b, cbciv.16b + eor v1.16b, v1.16b, v5.16b + ld1 {v5.16b}, [x1], #16 /* reload 1 ct block */ + ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */ + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + eor v4.16b, v4.16b, v5.16b +#else mov v4.16b, v0.16b mov v5.16b, v1.16b mov v6.16b, v2.16b bl aes_decrypt_block4x sub x1, x1, #16 - eor v0.16b, v0.16b, v7.16b + eor v0.16b, v0.16b, cbciv.16b eor v1.16b, v1.16b, v4.16b - ld1 {v7.16b}, [x1], #16 /* reload 1 ct block */ + ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */ eor v2.16b, v2.16b, v5.16b eor v3.16b, v3.16b, v6.16b +#endif st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) b .LcbcdecloopNx .Lcbcdec1x: - adds w4, w4, #4 + adds w4, w4, #MAX_STRIDE beq .Lcbcdecout .Lcbcdecloop: ld1 {v1.16b}, [x1], #16 /* get next ct block */ mov v0.16b, v1.16b /* ...and copy to v0 */ decrypt_block v0, w3, x2, x6, w7 - eor v0.16b, v0.16b, v7.16b /* xor with iv => pt */ - mov v7.16b, v1.16b /* ct is next iv */ + eor v0.16b, v0.16b, cbciv.16b /* xor with iv => pt */ + mov cbciv.16b, v1.16b /* ct is next iv */ st1 {v0.16b}, [x0], #16 subs w4, w4, #1 bne .Lcbcdecloop .Lcbcdecout: - st1 {v7.16b}, [x5] /* return iv */ + st1 {cbciv.16b}, [x5] /* return iv */ ldp x29, x30, [sp], #16 ret AES_ENDPROC(aes_cbc_decrypt) @@ -274,51 +305,60 @@ AES_ENTRY(aes_ctr_encrypt) mov x29, sp enc_prepare w3, x2, x6 - ld1 {v4.16b}, [x5] + ld1 {vctr.16b}, [x5] - umov x6, v4.d[1] /* keep swabbed ctr in reg */ + umov x6, vctr.d[1] /* keep swabbed ctr in reg */ rev x6, x6 cmn w6, w4 /* 32 bit overflow? */ bcs .Lctrloop .LctrloopNx: - subs w4, w4, #4 + subs w4, w4, #MAX_STRIDE bmi .Lctr1x add w7, w6, #1 - mov v0.16b, v4.16b + mov v0.16b, vctr.16b add w8, w6, #2 - mov v1.16b, v4.16b + mov v1.16b, vctr.16b + add w9, w6, #3 + mov v2.16b, vctr.16b add w9, w6, #3 - mov v2.16b, v4.16b rev w7, w7 - mov v3.16b, v4.16b + mov v3.16b, vctr.16b rev w8, w8 +ST5( mov v4.16b, vctr.16b ) mov v1.s[3], w7 rev w9, w9 +ST5( add w10, w6, #4 ) mov v2.s[3], w8 +ST5( rev w10, w10 ) mov v3.s[3], w9 +ST5( mov v4.s[3], w10 ) ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */ - bl aes_encrypt_block4x +ST4( bl aes_encrypt_block4x ) +ST5( bl aes_encrypt_block5x ) eor v0.16b, v5.16b, v0.16b - ld1 {v5.16b}, [x1], #16 /* get 1 input block */ +ST4( ld1 {v5.16b}, [x1], #16 ) eor v1.16b, v6.16b, v1.16b +ST5( ld1 {v5.16b-v6.16b}, [x1], #32 ) eor v2.16b, v7.16b, v2.16b eor v3.16b, v5.16b, v3.16b +ST5( eor v4.16b, v6.16b, v4.16b ) st1 {v0.16b-v3.16b}, [x0], #64 - add x6, x6, #4 +ST5( st1 {v4.16b}, [x0], #16 ) + add x6, x6, #MAX_STRIDE rev x7, x6 - ins v4.d[1], x7 + ins vctr.d[1], x7 cbz w4, .Lctrout b .LctrloopNx .Lctr1x: - adds w4, w4, #4 + adds w4, w4, #MAX_STRIDE beq .Lctrout .Lctrloop: - mov v0.16b, v4.16b + mov v0.16b, vctr.16b encrypt_block v0, w3, x2, x8, w7 adds x6, x6, #1 /* increment BE ctr */ rev x7, x6 - ins v4.d[1], x7 + ins vctr.d[1], x7 bcs .Lctrcarry /* overflow? */ .Lctrcarrydone: @@ -330,7 +370,7 @@ AES_ENTRY(aes_ctr_encrypt) bne .Lctrloop .Lctrout: - st1 {v4.16b}, [x5] /* return next CTR value */ + st1 {vctr.16b}, [x5] /* return next CTR value */ ldp x29, x30, [sp], #16 ret @@ -339,11 +379,11 @@ AES_ENTRY(aes_ctr_encrypt) b .Lctrout .Lctrcarry: - umov x7, v4.d[0] /* load upper word of ctr */ + umov x7, vctr.d[0] /* load upper word of ctr */ rev x7, x7 /* ... to handle the carry */ add x7, x7, #1 rev x7, x7 - ins v4.d[0], x7 + ins vctr.d[0], x7 b .Lctrcarrydone AES_ENDPROC(aes_ctr_encrypt) diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S index 33bb6af309a3..8bd66a6c4749 100644 --- a/arch/arm64/crypto/aes-neon.S +++ b/arch/arm64/crypto/aes-neon.S @@ -15,6 +15,8 @@ #define AES_ENDPROC(func) ENDPROC(neon_ ## func) xtsmask .req v7 + cbciv .req v7 + vctr .req v4 .macro xts_reload_mask, tmp xts_load_mask \tmp From 5a35316d97914d56d6be8b3748b2437785e74790 Mon Sep 17 00:00:00 2001 From: "Hook, Gary" Date: Tue, 25 Jun 2019 23:43:43 +0000 Subject: [PATCH 136/143] crypto: doc - Add parameter documentation Fill in missing parameter descriptions for the compression algorithm, then pick them up to document for the compression_alg structure. Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- Documentation/crypto/api-skcipher.rst | 2 +- include/linux/crypto.h | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Documentation/crypto/api-skcipher.rst b/Documentation/crypto/api-skcipher.rst index 4eec4a93f7e3..20ba08dddf2e 100644 --- a/Documentation/crypto/api-skcipher.rst +++ b/Documentation/crypto/api-skcipher.rst @@ -5,7 +5,7 @@ Block Cipher Algorithm Definitions :doc: Block Cipher Algorithm Definitions .. kernel-doc:: include/linux/crypto.h - :functions: crypto_alg ablkcipher_alg blkcipher_alg cipher_alg + :functions: crypto_alg ablkcipher_alg blkcipher_alg cipher_alg compress_alg Symmetric Key Cipher API ------------------------ diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 311237b1dab0..4b4e2ffbee74 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -327,6 +327,17 @@ struct cipher_alg { void (*cia_decrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src); }; +/** + * struct compress_alg - compression/decompression algorithm + * @coa_compress: Compress a buffer of specified length, storing the resulting + * data in the specified buffer. Return the length of the + * compressed data in dlen. + * @coa_decompress: Decompress the source buffer, storing the uncompressed + * data in the specified buffer. The length of the data is + * returned in dlen. + * + * All fields are mandatory. + */ struct compress_alg { int (*coa_compress)(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen); From ae400be94b84206e8344e8f10d896aeb3ea2f505 Mon Sep 17 00:00:00 2001 From: "Hook, Gary" Date: Tue, 25 Jun 2019 23:43:50 +0000 Subject: [PATCH 137/143] crypto: doc - Fix formatting of new crypto engine content Tidy up the formatting/grammar in crypto_engine.rst. Use bulleted lists where appropriate. Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- Documentation/crypto/crypto_engine.rst | 103 +++++++++++++++++-------- 1 file changed, 69 insertions(+), 34 deletions(-) diff --git a/Documentation/crypto/crypto_engine.rst b/Documentation/crypto/crypto_engine.rst index 1d56221dfe35..236c674d6897 100644 --- a/Documentation/crypto/crypto_engine.rst +++ b/Documentation/crypto/crypto_engine.rst @@ -1,50 +1,85 @@ -============= -CRYPTO ENGINE +.. SPDX-License-Identifier: GPL-2.0 +Crypto Engine ============= Overview -------- -The crypto engine API (CE), is a crypto queue manager. +The crypto engine (CE) API is a crypto queue manager. Requirement ----------- -You have to put at start of your tfm_ctx the struct crypto_engine_ctx:: +You must put, at the start of your transform context your_tfm_ctx, the structure +crypto_engine: - struct your_tfm_ctx { - struct crypto_engine_ctx enginectx; - ... - }; +:: -Why: Since CE manage only crypto_async_request, it cannot know the underlying -request_type and so have access only on the TFM. -So using container_of for accessing __ctx is impossible. -Furthermore, the crypto engine cannot know the "struct your_tfm_ctx", -so it must assume that crypto_engine_ctx is at start of it. + struct your_tfm_ctx { + struct crypto_engine engine; + ... + }; + +The crypto engine only manages asynchronous requests in the form of +crypto_async_request. It cannot know the underlying request type and thus only +has access to the transform structure. It is not possible to access the context +using container_of. In addition, the engine knows nothing about your +structure "``struct your_tfm_ctx``". The engine assumes (requires) the placement +of the known member ``struct crypto_engine`` at the beginning. Order of operations ------------------- -You have to obtain a struct crypto_engine via crypto_engine_alloc_init(). -And start it via crypto_engine_start(). +You are required to obtain a struct crypto_engine via ``crypto_engine_alloc_init()``. +Start it via ``crypto_engine_start()``. When finished with your work, shut down the +engine using ``crypto_engine_stop()`` and destroy the engine with +``crypto_engine_exit()``. -Before transferring any request, you have to fill the enginectx. -- prepare_request: (taking a function pointer) If you need to do some processing before doing the request -- unprepare_request: (taking a function pointer) Undoing what's done in prepare_request -- do_one_request: (taking a function pointer) Do encryption for current request +Before transferring any request, you have to fill the context enginectx by +providing functions for the following: -Note: that those three functions get the crypto_async_request associated with the received request. -So your need to get the original request via container_of(areq, struct yourrequesttype_request, base); +* ``prepare_crypt_hardware``: Called once before any prepare functions are + called. -When your driver receive a crypto_request, you have to transfer it to -the cryptoengine via one of: -- crypto_transfer_ablkcipher_request_to_engine() -- crypto_transfer_aead_request_to_engine() -- crypto_transfer_akcipher_request_to_engine() -- crypto_transfer_hash_request_to_engine() -- crypto_transfer_skcipher_request_to_engine() +* ``unprepare_crypt_hardware``: Called once after all unprepare functions have + been called. -At the end of the request process, a call to one of the following function is needed: -- crypto_finalize_ablkcipher_request -- crypto_finalize_aead_request -- crypto_finalize_akcipher_request -- crypto_finalize_hash_request -- crypto_finalize_skcipher_request +* ``prepare_cipher_request``/``prepare_hash_request``: Called before each + corresponding request is performed. If some processing or other preparatory + work is required, do it here. + +* ``unprepare_cipher_request``/``unprepare_hash_request``: Called after each + request is handled. Clean up / undo what was done in the prepare function. + +* ``cipher_one_request``/``hash_one_request``: Handle the current request by + performing the operation. + +Note that these functions access the crypto_async_request structure +associated with the received request. You are able to retrieve the original +request by using: + +:: + + container_of(areq, struct yourrequesttype_request, base); + +When your driver receives a crypto_request, you must to transfer it to +the crypto engine via one of: + +* crypto_transfer_ablkcipher_request_to_engine() + +* crypto_transfer_aead_request_to_engine() + +* crypto_transfer_akcipher_request_to_engine() + +* crypto_transfer_hash_request_to_engine() + +* crypto_transfer_skcipher_request_to_engine() + +At the end of the request process, a call to one of the following functions is needed: + +* crypto_finalize_ablkcipher_request() + +* crypto_finalize_aead_request() + +* crypto_finalize_akcipher_request() + +* crypto_finalize_hash_request() + +* crypto_finalize_skcipher_request() From 52393d617af7b554f03531e6756facf2ea687d2e Mon Sep 17 00:00:00 2001 From: "Hook, Gary" Date: Thu, 27 Jun 2019 16:16:23 +0000 Subject: [PATCH 138/143] crypto: ccp - Validate the the error value used to index error messages The error code read from the queue status register is only 6 bits wide, but we need to verify its value is within range before indexing the error messages. Fixes: 81422badb3907 ("crypto: ccp - Make syslog errors human-readable") Cc: Reported-by: Cfir Cohen Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev.c | 96 +++++++++++++++++++----------------- drivers/crypto/ccp/ccp-dev.h | 2 +- 2 files changed, 52 insertions(+), 46 deletions(-) diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index 1b5035d56288..9b6d8972a565 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -35,56 +35,62 @@ struct ccp_tasklet_data { }; /* Human-readable error strings */ +#define CCP_MAX_ERROR_CODE 64 static char *ccp_error_codes[] = { "", - "ERR 01: ILLEGAL_ENGINE", - "ERR 02: ILLEGAL_KEY_ID", - "ERR 03: ILLEGAL_FUNCTION_TYPE", - "ERR 04: ILLEGAL_FUNCTION_MODE", - "ERR 05: ILLEGAL_FUNCTION_ENCRYPT", - "ERR 06: ILLEGAL_FUNCTION_SIZE", - "ERR 07: Zlib_MISSING_INIT_EOM", - "ERR 08: ILLEGAL_FUNCTION_RSVD", - "ERR 09: ILLEGAL_BUFFER_LENGTH", - "ERR 10: VLSB_FAULT", - "ERR 11: ILLEGAL_MEM_ADDR", - "ERR 12: ILLEGAL_MEM_SEL", - "ERR 13: ILLEGAL_CONTEXT_ID", - "ERR 14: ILLEGAL_KEY_ADDR", - "ERR 15: 0xF Reserved", - "ERR 16: Zlib_ILLEGAL_MULTI_QUEUE", - "ERR 17: Zlib_ILLEGAL_JOBID_CHANGE", - "ERR 18: CMD_TIMEOUT", - "ERR 19: IDMA0_AXI_SLVERR", - "ERR 20: IDMA0_AXI_DECERR", - "ERR 21: 0x15 Reserved", - "ERR 22: IDMA1_AXI_SLAVE_FAULT", - "ERR 23: IDMA1_AIXI_DECERR", - "ERR 24: 0x18 Reserved", - "ERR 25: ZLIBVHB_AXI_SLVERR", - "ERR 26: ZLIBVHB_AXI_DECERR", - "ERR 27: 0x1B Reserved", - "ERR 27: ZLIB_UNEXPECTED_EOM", - "ERR 27: ZLIB_EXTRA_DATA", - "ERR 30: ZLIB_BTYPE", - "ERR 31: ZLIB_UNDEFINED_SYMBOL", - "ERR 32: ZLIB_UNDEFINED_DISTANCE_S", - "ERR 33: ZLIB_CODE_LENGTH_SYMBOL", - "ERR 34: ZLIB _VHB_ILLEGAL_FETCH", - "ERR 35: ZLIB_UNCOMPRESSED_LEN", - "ERR 36: ZLIB_LIMIT_REACHED", - "ERR 37: ZLIB_CHECKSUM_MISMATCH0", - "ERR 38: ODMA0_AXI_SLVERR", - "ERR 39: ODMA0_AXI_DECERR", - "ERR 40: 0x28 Reserved", - "ERR 41: ODMA1_AXI_SLVERR", - "ERR 42: ODMA1_AXI_DECERR", - "ERR 43: LSB_PARITY_ERR", + "ILLEGAL_ENGINE", + "ILLEGAL_KEY_ID", + "ILLEGAL_FUNCTION_TYPE", + "ILLEGAL_FUNCTION_MODE", + "ILLEGAL_FUNCTION_ENCRYPT", + "ILLEGAL_FUNCTION_SIZE", + "Zlib_MISSING_INIT_EOM", + "ILLEGAL_FUNCTION_RSVD", + "ILLEGAL_BUFFER_LENGTH", + "VLSB_FAULT", + "ILLEGAL_MEM_ADDR", + "ILLEGAL_MEM_SEL", + "ILLEGAL_CONTEXT_ID", + "ILLEGAL_KEY_ADDR", + "0xF Reserved", + "Zlib_ILLEGAL_MULTI_QUEUE", + "Zlib_ILLEGAL_JOBID_CHANGE", + "CMD_TIMEOUT", + "IDMA0_AXI_SLVERR", + "IDMA0_AXI_DECERR", + "0x15 Reserved", + "IDMA1_AXI_SLAVE_FAULT", + "IDMA1_AIXI_DECERR", + "0x18 Reserved", + "ZLIBVHB_AXI_SLVERR", + "ZLIBVHB_AXI_DECERR", + "0x1B Reserved", + "ZLIB_UNEXPECTED_EOM", + "ZLIB_EXTRA_DATA", + "ZLIB_BTYPE", + "ZLIB_UNDEFINED_SYMBOL", + "ZLIB_UNDEFINED_DISTANCE_S", + "ZLIB_CODE_LENGTH_SYMBOL", + "ZLIB _VHB_ILLEGAL_FETCH", + "ZLIB_UNCOMPRESSED_LEN", + "ZLIB_LIMIT_REACHED", + "ZLIB_CHECKSUM_MISMATCH0", + "ODMA0_AXI_SLVERR", + "ODMA0_AXI_DECERR", + "0x28 Reserved", + "ODMA1_AXI_SLVERR", + "ODMA1_AXI_DECERR", }; -void ccp_log_error(struct ccp_device *d, int e) +void ccp_log_error(struct ccp_device *d, unsigned int e) { - dev_err(d->dev, "CCP error: %s (0x%x)\n", ccp_error_codes[e], e); + if (WARN_ON(e >= CCP_MAX_ERROR_CODE)) + return; + + if (e < ARRAY_SIZE(ccp_error_codes)) + dev_err(d->dev, "CCP error %d: %s\n", e, ccp_error_codes[e]); + else + dev_err(d->dev, "CCP error %d: Unknown Error\n", e); } /* List of CCPs, CCP count, read-write access lock, and access functions diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 6810b65c1939..7442b0422f8a 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -632,7 +632,7 @@ struct ccp5_desc { void ccp_add_device(struct ccp_device *ccp); void ccp_del_device(struct ccp_device *ccp); -extern void ccp_log_error(struct ccp_device *, int); +extern void ccp_log_error(struct ccp_device *, unsigned int); struct ccp_device *ccp_alloc_struct(struct sp_device *sp); bool ccp_queues_suspended(struct ccp_device *ccp); From 600bac009ea9822d371aae03d33fc05991c5b5a2 Mon Sep 17 00:00:00 2001 From: "Hook, Gary" Date: Thu, 27 Jun 2019 16:23:29 +0000 Subject: [PATCH 139/143] crypto: ccp - Switch to SPDX license identifiers Add an SPDX identifier and remove any specific statements. Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-crypto-aes-cmac.c | 5 +---- drivers/crypto/ccp/ccp-crypto-aes-galois.c | 5 +---- drivers/crypto/ccp/ccp-crypto-aes-xts.c | 5 +---- drivers/crypto/ccp/ccp-crypto-aes.c | 4 ---- drivers/crypto/ccp/ccp-crypto-des3.c | 5 +---- drivers/crypto/ccp/ccp-crypto-main.c | 5 +---- drivers/crypto/ccp/ccp-crypto-rsa.c | 5 +---- drivers/crypto/ccp/ccp-crypto-sha.c | 5 +---- drivers/crypto/ccp/ccp-crypto.h | 5 +---- drivers/crypto/ccp/ccp-debugfs.c | 5 +---- drivers/crypto/ccp/ccp-dev-v3.c | 5 +---- drivers/crypto/ccp/ccp-dev-v5.c | 5 +---- drivers/crypto/ccp/ccp-dev.c | 5 +---- drivers/crypto/ccp/ccp-dev.h | 5 +---- drivers/crypto/ccp/ccp-dmaengine.c | 5 +---- drivers/crypto/ccp/ccp-ops.c | 4 ---- drivers/crypto/ccp/psp-dev.c | 5 +---- drivers/crypto/ccp/psp-dev.h | 5 +---- drivers/crypto/ccp/sp-dev.c | 5 +---- drivers/crypto/ccp/sp-dev.h | 5 +---- drivers/crypto/ccp/sp-pci.c | 5 +---- drivers/crypto/ccp/sp-platform.c | 5 +---- 22 files changed, 20 insertions(+), 88 deletions(-) diff --git a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c index f6e252c1d6fb..7b3ac61b67df 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c +++ b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) AES CMAC crypto API support * * Copyright (C) 2013,2018 Advanced Micro Devices, Inc. * * Author: Tom Lendacky - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c index ca1f0d780b61..d22631cb2bb3 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes-galois.c +++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) AES GCM crypto API support * * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. * * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c index ca4630b8395f..7a733e2c4e90 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes-xts.c +++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support * @@ -5,10 +6,6 @@ * * Author: Gary R Hook * Author: Tom Lendacky - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c index 3f768699332b..114fa0587d72 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes.c +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -5,10 +5,6 @@ * Copyright (C) 2013-2019 Advanced Micro Devices, Inc. * * Author: Tom Lendacky - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c index 91482ffcac59..6ff9de1b4546 100644 --- a/drivers/crypto/ccp/ccp-crypto-des3.c +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) DES3 crypto API support * * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. * * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c index b95d19974aa6..44a9917a4a6a 100644 --- a/drivers/crypto/ccp/ccp-crypto-main.c +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) crypto API support * * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. * * Author: Tom Lendacky - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-crypto-rsa.c b/drivers/crypto/ccp/ccp-crypto-rsa.c index a2570c0c8cdc..d9ee8d38c7fc 100644 --- a/drivers/crypto/ccp/ccp-crypto-rsa.c +++ b/drivers/crypto/ccp/ccp-crypto-rsa.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) RSA crypto API support * * Copyright (C) 2017 Advanced Micro Devices, Inc. * * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c index 3e10573f589e..bf2bfa268bec 100644 --- a/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) SHA crypto API support * @@ -5,10 +6,6 @@ * * Author: Tom Lendacky * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index 28819e11db96..a4a593dddfd6 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * AMD Cryptographic Coprocessor (CCP) crypto API support * * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. * * Author: Tom Lendacky - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef __CCP_CRYPTO_H__ diff --git a/drivers/crypto/ccp/ccp-debugfs.c b/drivers/crypto/ccp/ccp-debugfs.c index 4bd26af7098d..51ac76cbbbc9 100644 --- a/drivers/crypto/ccp/ccp-debugfs.c +++ b/drivers/crypto/ccp/ccp-debugfs.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) driver * * Copyright (C) 2017 Advanced Micro Devices, Inc. * * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c index 240bebbcb8ac..2339a8101a52 100644 --- a/drivers/crypto/ccp/ccp-dev-v3.c +++ b/drivers/crypto/ccp/ccp-dev-v3.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) driver * @@ -5,10 +6,6 @@ * * Author: Tom Lendacky * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index c9bfd4f439ce..c76a9fa115b8 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) driver * * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. * * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index 9b6d8972a565..f3ff36f93207 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) driver * @@ -5,10 +6,6 @@ * * Author: Tom Lendacky * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 7442b0422f8a..4a54e731f836 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * AMD Cryptographic Coprocessor (CCP) driver * @@ -5,10 +6,6 @@ * * Author: Tom Lendacky * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef __CCP_DEV_H__ diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c index 67155cb21636..9aee619db6e4 100644 --- a/drivers/crypto/ccp/ccp-dmaengine.c +++ b/drivers/crypto/ccp/ccp-dmaengine.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Cryptographic Coprocessor (CCP) driver * * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. * * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index b116d62991c6..a817f2755c58 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -6,10 +6,6 @@ * * Author: Tom Lendacky * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c index 656838433f2f..3e712f385bc1 100644 --- a/drivers/crypto/ccp/psp-dev.c +++ b/drivers/crypto/ccp/psp-dev.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Platform Security Processor (PSP) interface * * Copyright (C) 2016,2018 Advanced Micro Devices, Inc. * * Author: Brijesh Singh - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/psp-dev.h b/drivers/crypto/ccp/psp-dev.h index f5afeccf42a1..6ba59faab3a5 100644 --- a/drivers/crypto/ccp/psp-dev.h +++ b/drivers/crypto/ccp/psp-dev.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * AMD Platform Security Processor (PSP) interface driver * * Copyright (C) 2017-2018 Advanced Micro Devices, Inc. * * Author: Brijesh Singh - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef __PSP_DEV_H__ diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c index b2879767fc98..ec4f3baba071 100644 --- a/drivers/crypto/ccp/sp-dev.c +++ b/drivers/crypto/ccp/sp-dev.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Secure Processor driver * @@ -6,10 +7,6 @@ * Author: Tom Lendacky * Author: Gary R Hook * Author: Brijesh Singh - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h index 5b0790025db3..a54325f0d6a4 100644 --- a/drivers/crypto/ccp/sp-dev.h +++ b/drivers/crypto/ccp/sp-dev.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * AMD Secure Processor driver * @@ -6,10 +7,6 @@ * Author: Tom Lendacky * Author: Gary R Hook * Author: Brijesh Singh - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef __SP_DEV_H__ diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index 41bce0a3f4bb..2d189595bfbe 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Secure Processor device driver * @@ -5,10 +6,6 @@ * * Author: Tom Lendacky * Author: Gary R Hook - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c index d24228efbaaa..5525544df2f1 100644 --- a/drivers/crypto/ccp/sp-platform.c +++ b/drivers/crypto/ccp/sp-platform.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * AMD Secure Processor device driver * * Copyright (C) 2014,2018 Advanced Micro Devices, Inc. * * Author: Tom Lendacky - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include From 66d9c1cf9f95d96aa211d9f732cd42e93e2c1508 Mon Sep 17 00:00:00 2001 From: Fuqian Huang Date: Fri, 28 Jun 2019 10:46:31 +0800 Subject: [PATCH 140/143] crypto: amcc - remove memset after dma_alloc_coherent In commit af7ddd8a627c ("Merge tag 'dma-mapping-4.21' of git://git.infradead.org/users/hch/dma-mapping"), dma_alloc_coherent has already zeroed the memory. So memset is not needed. Signed-off-by: Fuqian Huang Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 45f65d638caf..50fa2683b8a1 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -191,7 +191,6 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) dev->pdr_pa); return -ENOMEM; } - memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD); dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device, sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, &dev->shadow_sa_pool_pa, From fdbd643a74efb353a20ecde94a2200e13a283b66 Mon Sep 17 00:00:00 2001 From: Lionel Debieve Date: Fri, 28 Jun 2019 13:26:41 +0200 Subject: [PATCH 141/143] crypto: stm32/crc32 - rename driver file Use the same naming convention for all stm32 crypto drivers. Signed-off-by: Lionel Debieve Signed-off-by: Herbert Xu --- drivers/crypto/stm32/Makefile | 2 +- drivers/crypto/stm32/{stm32_crc32.c => stm32-crc32.c} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename drivers/crypto/stm32/{stm32_crc32.c => stm32-crc32.c} (100%) diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile index 53d1bb94b221..23ce3bc46989 100644 --- a/drivers/crypto/stm32/Makefile +++ b/drivers/crypto/stm32/Makefile @@ -1,3 +1,3 @@ -obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32_crc32.o +obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32-crc32.o obj-$(CONFIG_CRYPTO_DEV_STM32_HASH) += stm32-hash.o obj-$(CONFIG_CRYPTO_DEV_STM32_CRYP) += stm32-cryp.o diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32-crc32.c similarity index 100% rename from drivers/crypto/stm32/stm32_crc32.c rename to drivers/crypto/stm32/stm32-crc32.c From 0acabecebc912b3ba06289e4ef40476acc499a37 Mon Sep 17 00:00:00 2001 From: Lionel Debieve Date: Fri, 28 Jun 2019 13:26:54 +0200 Subject: [PATCH 142/143] crypto: stm32/hash - Fix hmac issue more than 256 bytes Correct condition for the second hmac loop. Key must be only set in the first loop. Initial condition was wrong, HMAC_KEY flag was not properly checked. Signed-off-by: Lionel Debieve Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index bfc49e67124b..573ff3fad054 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -363,7 +363,7 @@ static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev, return -ETIMEDOUT; if ((hdev->flags & HASH_FLAGS_HMAC) && - (hdev->flags & ~HASH_FLAGS_HMAC_KEY)) { + (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) { hdev->flags |= HASH_FLAGS_HMAC_KEY; stm32_hash_write_key(hdev); if (stm32_hash_wait_busy(hdev)) From f3880a23564e3172437285ebcb5b8a124539fdae Mon Sep 17 00:00:00 2001 From: Lionel Debieve Date: Fri, 28 Jun 2019 13:26:55 +0200 Subject: [PATCH 143/143] crypto: stm32/hash - remove interruptible condition for dma When DMA is used, waiting for completion must not be interruptible as it can generate an error that is not handle by the driver. There is no need to put the completion interruptible in this driver. Signed-off-by: Lionel Debieve Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-hash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index 573ff3fad054..33a0612efa57 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -461,8 +461,8 @@ static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev, dma_async_issue_pending(hdev->dma_lch); - if (!wait_for_completion_interruptible_timeout(&hdev->dma_completion, - msecs_to_jiffies(100))) + if (!wait_for_completion_timeout(&hdev->dma_completion, + msecs_to_jiffies(100))) err = -ETIMEDOUT; if (dma_async_is_tx_complete(hdev->dma_lch, cookie,