redonkable
/
alistair23-linux
1
0
Fork 0
Code Releases Activity

chcr: Support for Chelsio's Crypto Hardware 

The Chelsio's Crypto Hardware can perform the following operations:
SHA1, SHA224, SHA256, SHA384 and SHA512, HMAC(SHA1), HMAC(SHA224),
HMAC(SHA256), HMAC(SHA384), HAMC(SHA512), AES-128-CBC, AES-192-CBC,
AES-256-CBC, AES-128-XTS, AES-256-XTS

This patch implements the driver for above mentioned features. This
driver is an Upper Layer Driver which is attached to Chelsio's LLD
(cxgb4) and uses the queue allocated by the LLD for sending the crypto
requests to the Hardware and receiving the responses from it.

The crypto operations can be performed by Chelsio's hardware from the
userspace applications and/or from within the kernel space using the
kernel's crypto API.

The above mentioned crypto features have been tested using kernel's
tests mentioned in testmgr.h. They also have been tested from user
space using libkcapi and Openssl.

Signed-off-by: Atul Gupta <atul.gupta@chelsio.com>
Signed-off-by: Hariprasad Shenai <hariprasad@chelsio.com>
Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>
hifive-unleashed-5.1
Hariprasad Shenai 2016-08-17 12:33:05 +05:30 committed by David S. Miller
parent d6657781b5
commit 324429d741
5 changed files with 2519 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1525
drivers/crypto/chelsio/chcr_algo.c 100644
View File

File diff suppressed because it is too large Load Diff

471
drivers/crypto/chelsio/chcr_algo.h 100644
View File

@ -0,0 +1,471 @@
/*
* This file is part of the Chelsio T6 Crypto driver for Linux.
*
* Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
#ifndef __CHCR_ALGO_H__
#define __CHCR_ALGO_H__
/* Crypto key context */
#define KEY_CONTEXT_CTX_LEN_S 24
#define KEY_CONTEXT_CTX_LEN_M 0xff
#define KEY_CONTEXT_CTX_LEN_V(x) ((x) << KEY_CONTEXT_CTX_LEN_S)
#define KEY_CONTEXT_CTX_LEN_G(x) \
(((x) >> KEY_CONTEXT_CTX_LEN_S) & KEY_CONTEXT_CTX_LEN_M)
#define KEY_CONTEXT_DUAL_CK_S 12
#define KEY_CONTEXT_DUAL_CK_M 0x1
#define KEY_CONTEXT_DUAL_CK_V(x) ((x) << KEY_CONTEXT_DUAL_CK_S)
#define KEY_CONTEXT_DUAL_CK_G(x) \
(((x) >> KEY_CONTEXT_DUAL_CK_S) & KEY_CONTEXT_DUAL_CK_M)
#define KEY_CONTEXT_DUAL_CK_F KEY_CONTEXT_DUAL_CK_V(1U)
#define KEY_CONTEXT_SALT_PRESENT_S 10
#define KEY_CONTEXT_SALT_PRESENT_M 0x1
#define KEY_CONTEXT_SALT_PRESENT_V(x) ((x) << KEY_CONTEXT_SALT_PRESENT_S)
#define KEY_CONTEXT_SALT_PRESENT_G(x) \
(((x) >> KEY_CONTEXT_SALT_PRESENT_S) & \
KEY_CONTEXT_SALT_PRESENT_M)
#define KEY_CONTEXT_SALT_PRESENT_F KEY_CONTEXT_SALT_PRESENT_V(1U)
#define KEY_CONTEXT_VALID_S 0
#define KEY_CONTEXT_VALID_M 0x1
#define KEY_CONTEXT_VALID_V(x) ((x) << KEY_CONTEXT_VALID_S)
#define KEY_CONTEXT_VALID_G(x) \
(((x) >> KEY_CONTEXT_VALID_S) & \
KEY_CONTEXT_VALID_M)
#define KEY_CONTEXT_VALID_F KEY_CONTEXT_VALID_V(1U)
#define KEY_CONTEXT_CK_SIZE_S 6
#define KEY_CONTEXT_CK_SIZE_M 0xf
#define KEY_CONTEXT_CK_SIZE_V(x) ((x) << KEY_CONTEXT_CK_SIZE_S)
#define KEY_CONTEXT_CK_SIZE_G(x) \
(((x) >> KEY_CONTEXT_CK_SIZE_S) & KEY_CONTEXT_CK_SIZE_M)
#define KEY_CONTEXT_MK_SIZE_S 2
#define KEY_CONTEXT_MK_SIZE_M 0xf
#define KEY_CONTEXT_MK_SIZE_V(x) ((x) << KEY_CONTEXT_MK_SIZE_S)
#define KEY_CONTEXT_MK_SIZE_G(x) \
(((x) >> KEY_CONTEXT_MK_SIZE_S) & KEY_CONTEXT_MK_SIZE_M)
#define KEY_CONTEXT_OPAD_PRESENT_S 11
#define KEY_CONTEXT_OPAD_PRESENT_M 0x1
#define KEY_CONTEXT_OPAD_PRESENT_V(x) ((x) << KEY_CONTEXT_OPAD_PRESENT_S)
#define KEY_CONTEXT_OPAD_PRESENT_G(x) \
(((x) >> KEY_CONTEXT_OPAD_PRESENT_S) & \
KEY_CONTEXT_OPAD_PRESENT_M)
#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U)
#define CHCR_HASH_MAX_DIGEST_SIZE 64
#define CHCR_MAX_SHA_DIGEST_SIZE 64
#define IPSEC_TRUNCATED_ICV_SIZE 12
#define TLS_TRUNCATED_HMAC_SIZE 10
#define CBCMAC_DIGEST_SIZE 16
#define MAX_HASH_NAME 20
#define SHA1_INIT_STATE_5X4B 5
#define SHA256_INIT_STATE_8X4B 8
#define SHA512_INIT_STATE_8X8B 8
#define SHA1_INIT_STATE SHA1_INIT_STATE_5X4B
#define SHA224_INIT_STATE SHA256_INIT_STATE_8X4B
#define SHA256_INIT_STATE SHA256_INIT_STATE_8X4B
#define SHA384_INIT_STATE SHA512_INIT_STATE_8X8B
#define SHA512_INIT_STATE SHA512_INIT_STATE_8X8B
#define DUMMY_BYTES 16
#define IPAD_DATA 0x36363636
#define OPAD_DATA 0x5c5c5c5c
#define TRANSHDR_SIZE(alignedkctx_len)\
(sizeof(struct ulptx_idata) +\
sizeof(struct ulp_txpkt) +\
sizeof(struct fw_crypto_lookaside_wr) +\
sizeof(struct cpl_tx_sec_pdu) +\
(alignedkctx_len))
#define CIPHER_TRANSHDR_SIZE(alignedkctx_len, sge_pairs) \
(TRANSHDR_SIZE(alignedkctx_len) + sge_pairs +\
sizeof(struct cpl_rx_phys_dsgl))
#define HASH_TRANSHDR_SIZE(alignedkctx_len)\
(TRANSHDR_SIZE(alignedkctx_len) + DUMMY_BYTES)
#define SEC_CPL_OFFSET (sizeof(struct fw_crypto_lookaside_wr) + \
sizeof(struct ulp_txpkt) + \
sizeof(struct ulptx_idata))
#define FILL_SEC_CPL_OP_IVINSR(id, len, hldr, ofst) \
htonl( \
CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | \
CPL_TX_SEC_PDU_RXCHID_V((id)) | \
CPL_TX_SEC_PDU_ACKFOLLOWS_V(0) | \
CPL_TX_SEC_PDU_ULPTXLPBK_V(1) | \
CPL_TX_SEC_PDU_CPLLEN_V((len)) | \
CPL_TX_SEC_PDU_PLACEHOLDER_V((hldr)) | \
CPL_TX_SEC_PDU_IVINSRTOFST_V((ofst)))
#define FILL_SEC_CPL_CIPHERSTOP_HI(a_start, a_stop, c_start, c_stop_hi) \
htonl( \
CPL_TX_SEC_PDU_AADSTART_V((a_start)) | \
CPL_TX_SEC_PDU_AADSTOP_V((a_stop)) | \
CPL_TX_SEC_PDU_CIPHERSTART_V((c_start)) | \
CPL_TX_SEC_PDU_CIPHERSTOP_HI_V((c_stop_hi)))
#define FILL_SEC_CPL_AUTHINSERT(c_stop_lo, a_start, a_stop, a_inst) \
htonl( \
CPL_TX_SEC_PDU_CIPHERSTOP_LO_V((c_stop_lo)) | \
CPL_TX_SEC_PDU_AUTHSTART_V((a_start)) | \
CPL_TX_SEC_PDU_AUTHSTOP_V((a_stop)) | \
CPL_TX_SEC_PDU_AUTHINSERT_V((a_inst)))
#define FILL_SEC_CPL_SCMD0_SEQNO(ctrl, seq, cmode, amode, opad, size, nivs) \
htonl( \
SCMD_SEQ_NO_CTRL_V(0) | \
SCMD_STATUS_PRESENT_V(0) | \
SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) | \
SCMD_ENC_DEC_CTRL_V((ctrl)) | \
SCMD_CIPH_AUTH_SEQ_CTRL_V((seq)) | \
SCMD_CIPH_MODE_V((cmode)) | \
SCMD_AUTH_MODE_V((amode)) | \
SCMD_HMAC_CTRL_V((opad)) | \
SCMD_IV_SIZE_V((size)) | \
SCMD_NUM_IVS_V((nivs)))
#define FILL_SEC_CPL_IVGEN_HDRLEN(last, more, ctx_in, mac, ivdrop, len) htonl( \
SCMD_ENB_DBGID_V(0) | \
SCMD_IV_GEN_CTRL_V(0) | \
SCMD_LAST_FRAG_V((last)) | \
SCMD_MORE_FRAGS_V((more)) | \
SCMD_TLS_COMPPDU_V(0) | \
SCMD_KEY_CTX_INLINE_V((ctx_in)) | \
SCMD_TLS_FRAG_ENABLE_V(0) | \
SCMD_MAC_ONLY_V((mac)) | \
SCMD_AADIVDROP_V((ivdrop)) | \
SCMD_HDR_LEN_V((len)))
#define FILL_KEY_CTX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \
htonl(KEY_CONTEXT_VALID_V(1) | \
KEY_CONTEXT_CK_SIZE_V((ck_size)) | \
KEY_CONTEXT_MK_SIZE_V(mk_size) | \
KEY_CONTEXT_DUAL_CK_V((d_ck)) | \
KEY_CONTEXT_OPAD_PRESENT_V((opad)) | \
KEY_CONTEXT_SALT_PRESENT_V(1) | \
KEY_CONTEXT_CTX_LEN_V((ctx_len)))
#define FILL_WR_OP_CCTX_SIZE(len, ctx_len) \
htonl( \
FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \
FW_CRYPTO_LOOKASIDE_WR) | \
FW_CRYPTO_LOOKASIDE_WR_COMPL_V(0) | \
FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((len)) | \
FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \
FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len)))
#define FILL_WR_RX_Q_ID(cid, qid, wr_iv) \
htonl( \
FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \
FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \
FW_CRYPTO_LOOKASIDE_WR_LCB_V(0) | \
FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)))
#define FILL_ULPTX_CMD_DEST(cid) \
htonl(ULPTX_CMD_V(ULP_TX_PKT) | \
ULP_TXPKT_DEST_V(0) | \
ULP_TXPKT_DATAMODIFY_V(0) | \
ULP_TXPKT_CHANNELID_V((cid)) | \
ULP_TXPKT_RO_V(1) | \
ULP_TXPKT_FID_V(0))
#define KEYCTX_ALIGN_PAD(bs) ({unsigned int _bs = (bs);\
_bs == SHA1_DIGEST_SIZE ? 12 : 0; })
#define FILL_PLD_SIZE_HASH_SIZE(payload_sgl_len, sgl_lengths, total_frags) \
htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(payload_sgl_len ? \
sgl_lengths[total_frags] : 0) |\
FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(0))
#define FILL_LEN_PKD(calc_tx_flits_ofld, skb) \
htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP((\
calc_tx_flits_ofld(skb) * 8), 16)))
#define FILL_CMD_MORE(immdatalen) htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) |\
ULP_TX_SC_MORE_V((immdatalen) ? 0 : 1))
#define MAX_NK 8
#define CRYPTO_MAX_IMM_TX_PKT_LEN 256
struct algo_param {
unsigned int auth_mode;
unsigned int mk_size;
unsigned int result_size;
};
struct hash_wr_param {
unsigned int opad_needed;
unsigned int more;
unsigned int last;
struct algo_param alg_prm;
unsigned int sg_len;
unsigned int bfr_len;
u64 scmd1;
};
enum {
AES_KEYLENGTH_128BIT = 128,
AES_KEYLENGTH_192BIT = 192,
AES_KEYLENGTH_256BIT = 256
};
enum {
KEYLENGTH_3BYTES = 3,
KEYLENGTH_4BYTES = 4,
KEYLENGTH_6BYTES = 6,
KEYLENGTH_8BYTES = 8
};
enum {
NUMBER_OF_ROUNDS_10 = 10,
NUMBER_OF_ROUNDS_12 = 12,
NUMBER_OF_ROUNDS_14 = 14,
};
/*
* CCM defines values of 4, 6, 8, 10, 12, 14, and 16 octets,
* where they indicate the size of the integrity check value (ICV)
*/
enum {
AES_CCM_ICV_4 = 4,
AES_CCM_ICV_6 = 6,
AES_CCM_ICV_8 = 8,
AES_CCM_ICV_10 = 10,
AES_CCM_ICV_12 = 12,
AES_CCM_ICV_14 = 14,
AES_CCM_ICV_16 = 16
};
struct hash_op_params {
unsigned char mk_size;
unsigned char pad_align;
unsigned char auth_mode;
char hash_name[MAX_HASH_NAME];
unsigned short block_size;
unsigned short word_size;
unsigned short ipad_size;
};
struct phys_sge_pairs {
__be16 len[8];
__be64 addr[8];
};
struct phys_sge_parm {
unsigned int nents;
unsigned int obsize;
unsigned short qid;
unsigned char align;
};
struct crypto_result {
struct completion completion;
int err;
};
static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
};
static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
};
static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
};
static const u64 sha384_init[SHA512_DIGEST_SIZE / 8] = {
SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3,
SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7,
};
static const u64 sha512_init[SHA512_DIGEST_SIZE / 8] = {
SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3,
SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7,
};
static inline void copy_hash_init_values(char *key, int digestsize)
{
u8 i;
__be32 *dkey = (__be32 *)key;
u64 *ldkey = (u64 *)key;
__be64 *sha384 = (__be64 *)sha384_init;
__be64 *sha512 = (__be64 *)sha512_init;
switch (digestsize) {
case SHA1_DIGEST_SIZE:
for (i = 0; i < SHA1_INIT_STATE; i++)
dkey[i] = cpu_to_be32(sha1_init[i]);
break;
case SHA224_DIGEST_SIZE:
for (i = 0; i < SHA224_INIT_STATE; i++)
dkey[i] = cpu_to_be32(sha224_init[i]);
break;
case SHA256_DIGEST_SIZE:
for (i = 0; i < SHA256_INIT_STATE; i++)
dkey[i] = cpu_to_be32(sha256_init[i]);
break;
case SHA384_DIGEST_SIZE:
for (i = 0; i < SHA384_INIT_STATE; i++)
ldkey[i] = be64_to_cpu(sha384[i]);
break;
case SHA512_DIGEST_SIZE:
for (i = 0; i < SHA512_INIT_STATE; i++)
ldkey[i] = be64_to_cpu(sha512[i]);
break;
}
}
static const u8 sgl_lengths[20] = {
0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15
};
/* Number of len fields(8) * size of one addr field */
#define PHYSDSGL_MAX_LEN_SIZE 16
static inline u16 get_space_for_phys_dsgl(unsigned int sgl_entr)
{
/* len field size + addr field size */
return ((sgl_entr >> 3) + ((sgl_entr % 8) ?
1 : 0)) * PHYSDSGL_MAX_LEN_SIZE +
(sgl_entr << 3) + ((sgl_entr % 2 ? 1 : 0) << 3);
}
/* The AES s-transform matrix (s-box). */
static const u8 aes_sbox[256] = {
99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215,
171, 118, 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175,
156, 164, 114, 192, 183, 253, 147, 38, 54, 63, 247, 204, 52, 165,
229, 241, 113, 216, 49, 21, 4, 199, 35, 195, 24, 150, 5, 154, 7,
18, 128, 226, 235, 39, 178, 117, 9, 131, 44, 26, 27, 110, 90,
160, 82, 59, 214, 179, 41, 227, 47, 132, 83, 209, 0, 237, 32,
252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207, 208, 239, 170,
251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168, 81,
163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243,
210, 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100,
93, 25, 115, 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184,
20, 222, 94, 11, 219, 224, 50, 58, 10, 73, 6, 36, 92, 194,
211, 172, 98, 145, 149, 228, 121, 231, 200, 55, 109, 141, 213, 78,
169, 108, 86, 244, 234, 101, 122, 174, 8, 186, 120, 37, 46, 28, 166,
180, 198, 232, 221, 116, 31, 75, 189, 139, 138, 112, 62, 181, 102,
72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158, 225, 248,
152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84,
187, 22
};
static u32 aes_ks_subword(const u32 w)
{
u8 bytes[4];
*(u32 *)(&bytes[0]) = w;
bytes[0] = aes_sbox[bytes[0]];
bytes[1] = aes_sbox[bytes[1]];
bytes[2] = aes_sbox[bytes[2]];
bytes[3] = aes_sbox[bytes[3]];
return *(u32 *)(&bytes[0]);
}
static u32 round_constant[11] = {
0x01000000, 0x02000000, 0x04000000, 0x08000000,
0x10000000, 0x20000000, 0x40000000, 0x80000000,
0x1B000000, 0x36000000, 0x6C000000
};
/* dec_key - OUTPUT - Reverse round key
* key - INPUT - key
* keylength - INPUT - length of the key in number of bits
*/
static inline void get_aes_decrypt_key(unsigned char *dec_key,
const unsigned char *key,
unsigned int keylength)
{
u32 temp;
u32 w_ring[MAX_NK];
int i, j, k = 0;
u8 nr, nk;
switch (keylength) {
case AES_KEYLENGTH_128BIT:
nk = KEYLENGTH_4BYTES;
nr = NUMBER_OF_ROUNDS_10;
break;
case AES_KEYLENGTH_192BIT:
nk = KEYLENGTH_6BYTES;
nr = NUMBER_OF_ROUNDS_12;
break;
case AES_KEYLENGTH_256BIT:
nk = KEYLENGTH_8BYTES;
nr = NUMBER_OF_ROUNDS_14;
break;
default:
return;
}
for (i = 0; i < nk; i++ )
w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
i = 0;
temp = w_ring[nk - 1];
while(i + nk < (nr + 1) * 4) {
if(!(i % nk)) {
/* RotWord(temp) */
temp = (temp << 8) | (temp >> 24);
temp = aes_ks_subword(temp);
temp ^= round_constant[i / nk];
}
else if (nk == 8 && (i % 4 == 0))
temp = aes_ks_subword(temp);
w_ring[i % nk] ^= temp;
temp = w_ring[i % nk];
i++;
}
for (k = 0, j = i % nk; k < nk; k++) {
*((u32 *)dec_key + k) = htonl(w_ring[j]);
j--;
if(j < 0)
j += nk;
}
}
#endif /* __CHCR_ALGO_H__ */

240
drivers/crypto/chelsio/chcr_core.c 100644
View File

@ -0,0 +1,240 @@
/**
* This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux.
*
* Copyright (C) 2011-2016 Chelsio Communications. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation.
*
* Written and Maintained by:
* Manoj Malviya (manojmalviya@chelsio.com)
* Atul Gupta (atul.gupta@chelsio.com)
* Jitendra Lulla (jlulla@chelsio.com)
* Yeshaswi M R Gowda (yeshaswi@chelsio.com)
* Harsh Jain (harsh@chelsio.com)
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <crypto/aes.h>
#include <crypto/hash.h>
#include "t4_msg.h"
#include "chcr_core.h"
#include "cxgb4_uld.h"
static LIST_HEAD(uld_ctx_list);
static DEFINE_MUTEX(dev_mutex);
static atomic_t dev_count;
typedef int (*chcr_handler_func)(struct chcr_dev *dev, unsigned char *input);
static int cpl_fw6_pld_handler(struct chcr_dev *dev, unsigned char *input);
static void *chcr_uld_add(const struct cxgb4_lld_info *lld);
static int chcr_uld_state_change(void *handle, enum cxgb4_state state);
static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
[CPL_FW6_PLD] = cpl_fw6_pld_handler,
};
static struct cxgb4_pci_uld_info chcr_uld_info = {
.name = DRV_MODULE_NAME,
.nrxq = 4,
.rxq_size = 1024,
.nciq = 0,
.ciq_size = 0,
.add = chcr_uld_add,
.state_change = chcr_uld_state_change,
.rx_handler = chcr_uld_rx_handler,
};
int assign_chcr_device(struct chcr_dev **dev)
{
struct uld_ctx *u_ctx;
/*
* Which device to use if multiple devices are available TODO
* May be select the device based on round robin. One session
* must go to the same device to maintain the ordering.
*/
mutex_lock(&dev_mutex); /* TODO ? */
u_ctx = list_first_entry(&uld_ctx_list, struct uld_ctx, entry);
if (!u_ctx) {
mutex_unlock(&dev_mutex);
return -ENXIO;
}
*dev = u_ctx->dev;
mutex_unlock(&dev_mutex);
return 0;
}
static int chcr_dev_add(struct uld_ctx *u_ctx)
{
struct chcr_dev *dev;
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (!dev)
return -ENXIO;
spin_lock_init(&dev->lock_chcr_dev);
u_ctx->dev = dev;
dev->u_ctx = u_ctx;
atomic_inc(&dev_count);
return 0;
}
static int chcr_dev_remove(struct uld_ctx *u_ctx)
{
kfree(u_ctx->dev);
u_ctx->dev = NULL;
atomic_dec(&dev_count);
return 0;
}
static int cpl_fw6_pld_handler(struct chcr_dev *dev,
unsigned char *input)
{
struct crypto_async_request *req;
struct cpl_fw6_pld *fw6_pld;
u32 ack_err_status = 0;
int error_status = 0;
fw6_pld = (struct cpl_fw6_pld *)input;
req = (struct crypto_async_request *)(uintptr_t)be64_to_cpu(
fw6_pld->data[1]);
ack_err_status =
ntohl(*(__be32 *)((unsigned char *)&fw6_pld->data[0] + 4));
if (ack_err_status) {
if (CHK_MAC_ERR_BIT(ack_err_status) ||
CHK_PAD_ERR_BIT(ack_err_status))
error_status = -EINVAL;
}
/* call completion callback with failure status */
if (req) {
if (!chcr_handle_resp(req, input, error_status))
req->complete(req, error_status);
else
return -EINVAL;
} else {
pr_err("Incorrect request address from the firmware\n");
return -EFAULT;
}
return 0;
}
int chcr_send_wr(struct sk_buff *skb)
{
return cxgb4_ofld_send(skb->dev, skb);
}
static void *chcr_uld_add(const struct cxgb4_lld_info *lld)
{
struct uld_ctx *u_ctx;
/* Create the device and add it in the device list */
u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
if (!u_ctx) {
u_ctx = ERR_PTR(-ENOMEM);
goto out;
}
u_ctx->lldi = *lld;
mutex_lock(&dev_mutex);
list_add_tail(&u_ctx->entry, &uld_ctx_list);
mutex_unlock(&dev_mutex);
out:
return u_ctx;
}
int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
const struct pkt_gl *pgl)
{
struct uld_ctx *u_ctx = (struct uld_ctx *)handle;
struct chcr_dev *dev = u_ctx->dev;
const struct cpl_act_establish *rpl = (struct cpl_act_establish
*)rsp;
if (rpl->ot.opcode != CPL_FW6_PLD) {
pr_err("Unsupported opcode\n");
return 0;
}
if (!pgl)
work_handlers[rpl->ot.opcode](dev, (unsigned char *)&rsp[1]);
else
work_handlers[rpl->ot.opcode](dev, pgl->va);
return 0;
}
static int chcr_uld_state_change(void *handle, enum cxgb4_state state)
{
struct uld_ctx *u_ctx = handle;
int ret = 0;
switch (state) {
case CXGB4_STATE_UP:
if (!u_ctx->dev) {
ret = chcr_dev_add(u_ctx);
if (ret != 0)
return ret;
}
if (atomic_read(&dev_count) == 1)
ret = start_crypto();
break;
case CXGB4_STATE_DETACH:
if (u_ctx->dev) {
mutex_lock(&dev_mutex);
chcr_dev_remove(u_ctx);
mutex_unlock(&dev_mutex);
}
if (!atomic_read(&dev_count))
stop_crypto();
break;
case CXGB4_STATE_START_RECOVERY:
case CXGB4_STATE_DOWN:
default:
break;
}
return ret;
}
static int __init chcr_crypto_init(void)
{
if (cxgb4_register_pci_uld(CXGB4_PCI_ULD1, &chcr_uld_info)) {
pr_err("ULD register fail: No chcr crypto support in cxgb4");
return -1;
}
return 0;
}
static void __exit chcr_crypto_exit(void)
{
struct uld_ctx *u_ctx, *tmp;
if (atomic_read(&dev_count))
stop_crypto();
/* Remove all devices from list */
mutex_lock(&dev_mutex);
list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
if (u_ctx->dev)
chcr_dev_remove(u_ctx);
kfree(u_ctx);
}
mutex_unlock(&dev_mutex);
cxgb4_unregister_pci_uld(CXGB4_PCI_ULD1);
}
module_init(chcr_crypto_init);
module_exit(chcr_crypto_exit);
MODULE_DESCRIPTION("Crypto Co-processor for Chelsio Terminator cards.");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Chelsio Communications");
MODULE_VERSION(DRV_VERSION);

80
drivers/crypto/chelsio/chcr_core.h 100644
View File

@ -0,0 +1,80 @@
/*
* This file is part of the Chelsio T6 Crypto driver for Linux.
*
* Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
#ifndef __CHCR_CORE_H__
#define __CHCR_CORE_H__
#include <crypto/algapi.h>
#include "t4_hw.h"
#include "cxgb4.h"
#include "cxgb4_uld.h"
#define DRV_MODULE_NAME "chcr"
#define DRV_VERSION "1.0.0.0"
#define MAX_PENDING_REQ_TO_HW 20
#define CHCR_TEST_RESPONSE_TIMEOUT 1000
#define PAD_ERROR_BIT 1
#define CHK_PAD_ERR_BIT(x) (((x) >> PAD_ERROR_BIT) & 1)
#define MAC_ERROR_BIT 0
#define CHK_MAC_ERR_BIT(x) (((x) >> MAC_ERROR_BIT) & 1)
struct uld_ctx;
struct chcr_dev {
/* Request submited to h/w and waiting for response. */
spinlock_t lock_chcr_dev;
struct crypto_queue pending_queue;
struct uld_ctx *u_ctx;
unsigned char tx_channel_id;
};
struct uld_ctx {
struct list_head entry;
struct cxgb4_lld_info lldi;
struct chcr_dev *dev;
};
int assign_chcr_device(struct chcr_dev **dev);
int chcr_send_wr(struct sk_buff *skb);
int start_crypto(void);
int stop_crypto(void);
int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
const struct pkt_gl *pgl);
int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
int err);
#endif /* __CHCR_CORE_H__ */

203
drivers/crypto/chelsio/chcr_crypto.h 100644
View File

@ -0,0 +1,203 @@
/*
* This file is part of the Chelsio T6 Crypto driver for Linux.
*
* Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
#ifndef __CHCR_CRYPTO_H__
#define __CHCR_CRYPTO_H__
/* Define following if h/w is not dropping the AAD and IV data before
* giving the processed data
*/
#define CHCR_CRA_PRIORITY 300
#define CHCR_AES_MAX_KEY_LEN (2 * (AES_MAX_KEY_SIZE)) /* consider xts */
#define CHCR_MAX_CRYPTO_IV_LEN 16 /* AES IV len */
#define CHCR_MAX_AUTHENC_AES_KEY_LEN 32 /* max aes key length*/
#define CHCR_MAX_AUTHENC_SHA_KEY_LEN 128 /* max sha key length*/
#define CHCR_GIVENCRYPT_OP 2
/* CPL/SCMD parameters */
#define CHCR_ENCRYPT_OP 0
#define CHCR_DECRYPT_OP 1
#define CHCR_SCMD_SEQ_NO_CTRL_32BIT 1
#define CHCR_SCMD_SEQ_NO_CTRL_48BIT 2
#define CHCR_SCMD_SEQ_NO_CTRL_64BIT 3
#define CHCR_SCMD_PROTO_VERSION_GENERIC 4
#define CHCR_SCMD_AUTH_CTRL_AUTH_CIPHER 0
#define CHCR_SCMD_AUTH_CTRL_CIPHER_AUTH 1
#define CHCR_SCMD_CIPHER_MODE_NOP 0
#define CHCR_SCMD_CIPHER_MODE_AES_CBC 1
#define CHCR_SCMD_CIPHER_MODE_GENERIC_AES 4
#define CHCR_SCMD_CIPHER_MODE_AES_XTS 6
#define CHCR_SCMD_AUTH_MODE_NOP 0
#define CHCR_SCMD_AUTH_MODE_SHA1 1
#define CHCR_SCMD_AUTH_MODE_SHA224 2
#define CHCR_SCMD_AUTH_MODE_SHA256 3
#define CHCR_SCMD_AUTH_MODE_SHA512_224 5
#define CHCR_SCMD_AUTH_MODE_SHA512_256 6
#define CHCR_SCMD_AUTH_MODE_SHA512_384 7
#define CHCR_SCMD_AUTH_MODE_SHA512_512 8
#define CHCR_SCMD_HMAC_CTRL_NOP 0
#define CHCR_SCMD_HMAC_CTRL_NO_TRUNC 1
#define CHCR_SCMD_IVGEN_CTRL_HW 0
#define CHCR_SCMD_IVGEN_CTRL_SW 1
/* This are not really mac key size. They are intermediate values
* of sha engine and its size
*/
#define CHCR_KEYCTX_MAC_KEY_SIZE_128 0
#define CHCR_KEYCTX_MAC_KEY_SIZE_160 1
#define CHCR_KEYCTX_MAC_KEY_SIZE_192 2
#define CHCR_KEYCTX_MAC_KEY_SIZE_256 3
#define CHCR_KEYCTX_MAC_KEY_SIZE_512 4
#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128 0
#define CHCR_KEYCTX_CIPHER_KEY_SIZE_192 1
#define CHCR_KEYCTX_CIPHER_KEY_SIZE_256 2
#define CHCR_KEYCTX_NO_KEY 15
#define CHCR_CPL_FW4_PLD_IV_OFFSET (5 * 64) /* bytes. flt #5 and #6 */
#define CHCR_CPL_FW4_PLD_HASH_RESULT_OFFSET (7 * 64) /* bytes. flt #7 */
#define CHCR_CPL_FW4_PLD_DATA_SIZE (4 * 64) /* bytes. flt #4 to #7 */
#define KEY_CONTEXT_HDR_SALT_AND_PAD 16
#define flits_to_bytes(x) (x * 8)
#define IV_NOP 0
#define IV_IMMEDIATE 1
#define IV_DSGL 2
#define CRYPTO_ALG_SUB_TYPE_MASK 0x0f000000
#define CRYPTO_ALG_SUB_TYPE_HASH_HMAC 0x01000000
#define CRYPTO_ALG_TYPE_HMAC (CRYPTO_ALG_TYPE_AHASH |\
CRYPTO_ALG_SUB_TYPE_HASH_HMAC)
#define MAX_SALT 4
#define MAX_SCRATCH_PAD_SIZE 32
#define CHCR_HASH_MAX_BLOCK_SIZE_64 64
#define CHCR_HASH_MAX_BLOCK_SIZE_128 128
/* Aligned to 128 bit boundary */
struct _key_ctx {
__be32 ctx_hdr;
u8 salt[MAX_SALT];
__be64 reserverd;
unsigned char key[0];
};
struct ablk_ctx {
u8 enc;
unsigned int processed_len;
__be32 key_ctx_hdr;
unsigned int enckey_len;
unsigned int dst_nents;
struct scatterlist iv_sg;
u8 key[CHCR_AES_MAX_KEY_LEN];
u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
unsigned char ciph_mode;
};
struct hmac_ctx {
struct shash_desc *desc;
u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128];
u8 opad[CHCR_HASH_MAX_BLOCK_SIZE_128];
};
struct __crypto_ctx {
struct hmac_ctx hmacctx[0];
struct ablk_ctx ablkctx[0];
};
struct chcr_context {
struct chcr_dev *dev;
unsigned char tx_channel_id;
struct __crypto_ctx crypto_ctx[0];
};
struct chcr_ahash_req_ctx {
u32 result;
char bfr[CHCR_HASH_MAX_BLOCK_SIZE_128];
u8 bfr_len;
/* DMA the partial hash in it */
u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE];
u64 data_len; /* Data len till time */
void *dummy_payload_ptr;
/* SKB which is being sent to the hardware for processing */
struct sk_buff *skb;
};
struct chcr_blkcipher_req_ctx {
struct sk_buff *skb;
};
struct chcr_alg_template {
u32 type;
u32 is_registered;
union {
struct crypto_alg crypto;
struct ahash_alg hash;
} alg;
};
struct chcr_req_ctx {
union {
struct ahash_request *ahash_req;
struct ablkcipher_request *ablk_req;
} req;
union {
struct chcr_ahash_req_ctx *ahash_ctx;
struct chcr_blkcipher_req_ctx *ablk_ctx;
} ctx;
};
struct sge_opaque_hdr {
void *dev;
dma_addr_t addr[MAX_SKB_FRAGS + 1];
};
typedef struct sk_buff *(*create_wr_t)(struct crypto_async_request *req,
struct chcr_context *ctx,
unsigned short qid,
unsigned short op_type);
#endif /* __CHCR_CRYPTO_H__ */