crypto: x86/sha512_ssse3 - move SHA-384/512 SSSE3 implementation to base layer
This removes all the boilerplate from the existing implementation, and replaces it with calls into the base layer. It also changes the prototypes of the core asm functions to be compatible with the base prototype void (sha512_block_fn)(struct sha256_state *sst, u8 const *src, int blocks) so that they can be passed to the base layer directly. Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>wifi-calibration
Ard Biesheuvel
Herbert Xu
parent
1631030ae6
commit
e68410ebf6
|
|
@ -54,9 +54,9 @@
|
|||
|
||||
# Virtual Registers
|
||||
# ARG1
|
||||
msg = %rdi
|
||||
digest = %rdi
|
||||
# ARG2
|
||||
digest = %rsi
|
||||
msg = %rsi
|
||||
# ARG3
|
||||
msglen = %rdx
|
||||
T1 = %rcx
|
||||
|
|
@ -271,7 +271,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE
|
|||
.endm
|
||||
|
||||
########################################################################
|
||||
# void sha512_transform_avx(const void* M, void* D, u64 L)
|
||||
# void sha512_transform_avx(void* D, const void* M, u64 L)
|
||||
# Purpose: Updates the SHA512 digest stored at D with the message stored in M.
|
||||
# The size of the message pointed to by M must be an integer multiple of SHA512
|
||||
# message blocks.
|
||||
|
|
|
|||
|
|
@ -70,9 +70,9 @@ XFER = YTMP0
|
|||
BYTE_FLIP_MASK = %ymm9
|
||||
|
||||
# 1st arg
|
||||
INP = %rdi
|
||||
CTX = %rdi
|
||||
# 2nd arg
|
||||
CTX = %rsi
|
||||
INP = %rsi
|
||||
# 3rd arg
|
||||
NUM_BLKS = %rdx
|
||||
|
||||
|
|
@ -562,7 +562,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE
|
|||
.endm
|
||||
|
||||
########################################################################
|
||||
# void sha512_transform_rorx(const void* M, void* D, uint64_t L)#
|
||||
# void sha512_transform_rorx(void* D, const void* M, uint64_t L)#
|
||||
# Purpose: Updates the SHA512 digest stored at D with the message stored in M.
|
||||
# The size of the message pointed to by M must be an integer multiple of SHA512
|
||||
# message blocks.
|
||||
|
|
|
|||
|
|
@ -53,9 +53,9 @@
|
|||
|
||||
# Virtual Registers
|
||||
# ARG1
|
||||
msg = %rdi
|
||||
digest = %rdi
|
||||
# ARG2
|
||||
digest = %rsi
|
||||
msg = %rsi
|
||||
# ARG3
|
||||
msglen = %rdx
|
||||
T1 = %rcx
|
||||
|
|
@ -269,7 +269,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE
|
|||
.endm
|
||||
|
||||
########################################################################
|
||||
# void sha512_transform_ssse3(const void* M, void* D, u64 L)#
|
||||
# void sha512_transform_ssse3(void* D, const void* M, u64 L)#
|
||||
# Purpose: Updates the SHA512 digest stored at D with the message stored in M.
|
||||
# The size of the message pointed to by M must be an integer multiple of SHA512
|
||||
# message blocks.
|
||||
|
|
|
|||
|
|
@ -34,205 +34,75 @@
|
|||
#include <linux/cryptohash.h>
|
||||
#include <linux/types.h>
|
||||
#include <crypto/sha.h>
|
||||
#include <asm/byteorder.h>
|
||||
#include <crypto/sha512_base.h>
|
||||
#include <asm/i387.h>
|
||||
#include <asm/xcr.h>
|
||||
#include <asm/xsave.h>
|
||||
|
||||
#include <linux/string.h>
|
||||
|
||||
asmlinkage void sha512_transform_ssse3(const char *data, u64 *digest,
|
||||
u64 rounds);
|
||||
asmlinkage void sha512_transform_ssse3(u64 *digest, const char *data,
|
||||
u64 rounds);
|
||||
#ifdef CONFIG_AS_AVX
|
||||
asmlinkage void sha512_transform_avx(const char *data, u64 *digest,
|
||||
asmlinkage void sha512_transform_avx(u64 *digest, const char *data,
|
||||
u64 rounds);
|
||||
#endif
|
||||
#ifdef CONFIG_AS_AVX2
|
||||
asmlinkage void sha512_transform_rorx(const char *data, u64 *digest,
|
||||
u64 rounds);
|
||||
asmlinkage void sha512_transform_rorx(u64 *digest, const char *data,
|
||||
u64 rounds);
|
||||
#endif
|
||||
|
||||
static asmlinkage void (*sha512_transform_asm)(const char *, u64 *, u64);
|
||||
|
||||
|
||||
static int sha512_ssse3_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
sctx->state[0] = SHA512_H0;
|
||||
sctx->state[1] = SHA512_H1;
|
||||
sctx->state[2] = SHA512_H2;
|
||||
sctx->state[3] = SHA512_H3;
|
||||
sctx->state[4] = SHA512_H4;
|
||||
sctx->state[5] = SHA512_H5;
|
||||
sctx->state[6] = SHA512_H6;
|
||||
sctx->state[7] = SHA512_H7;
|
||||
sctx->count[0] = sctx->count[1] = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __sha512_ssse3_update(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len, unsigned int partial)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
unsigned int done = 0;
|
||||
|
||||
sctx->count[0] += len;
|
||||
if (sctx->count[0] < len)
|
||||
sctx->count[1]++;
|
||||
|
||||
if (partial) {
|
||||
done = SHA512_BLOCK_SIZE - partial;
|
||||
memcpy(sctx->buf + partial, data, done);
|
||||
sha512_transform_asm(sctx->buf, sctx->state, 1);
|
||||
}
|
||||
|
||||
if (len - done >= SHA512_BLOCK_SIZE) {
|
||||
const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE;
|
||||
|
||||
sha512_transform_asm(data + done, sctx->state, (u64) rounds);
|
||||
|
||||
done += rounds * SHA512_BLOCK_SIZE;
|
||||
}
|
||||
|
||||
memcpy(sctx->buf, data + done, len - done);
|
||||
|
||||
return 0;
|
||||
}
|
||||
static void (*sha512_transform_asm)(u64 *, const char *, u64);
|
||||
|
||||
static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len)
|
||||
unsigned int len)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE;
|
||||
int res;
|
||||
|
||||
/* Handle the fast case right here */
|
||||
if (partial + len < SHA512_BLOCK_SIZE) {
|
||||
sctx->count[0] += len;
|
||||
if (sctx->count[0] < len)
|
||||
sctx->count[1]++;
|
||||
memcpy(sctx->buf + partial, data, len);
|
||||
if (!irq_fpu_usable() ||
|
||||
(sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE)
|
||||
return crypto_sha512_update(desc, data, len);
|
||||
|
||||
return 0;
|
||||
}
|
||||
/* make sure casting to sha512_block_fn() is safe */
|
||||
BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
|
||||
|
||||
if (!irq_fpu_usable()) {
|
||||
res = crypto_sha512_update(desc, data, len);
|
||||
} else {
|
||||
kernel_fpu_begin();
|
||||
res = __sha512_ssse3_update(desc, data, len, partial);
|
||||
kernel_fpu_end();
|
||||
}
|
||||
kernel_fpu_begin();
|
||||
sha512_base_do_update(desc, data, len,
|
||||
(sha512_block_fn *)sha512_transform_asm);
|
||||
kernel_fpu_end();
|
||||
|
||||
return res;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len, u8 *out)
|
||||
{
|
||||
if (!irq_fpu_usable())
|
||||
return crypto_sha512_finup(desc, data, len, out);
|
||||
|
||||
kernel_fpu_begin();
|
||||
if (len)
|
||||
sha512_base_do_update(desc, data, len,
|
||||
(sha512_block_fn *)sha512_transform_asm);
|
||||
sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_transform_asm);
|
||||
kernel_fpu_end();
|
||||
|
||||
return sha512_base_finish(desc, out);
|
||||
}
|
||||
|
||||
/* Add padding and return the message digest. */
|
||||
static int sha512_ssse3_final(struct shash_desc *desc, u8 *out)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
unsigned int i, index, padlen;
|
||||
__be64 *dst = (__be64 *)out;
|
||||
__be64 bits[2];
|
||||
static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, };
|
||||
|
||||
/* save number of bits */
|
||||
bits[1] = cpu_to_be64(sctx->count[0] << 3);
|
||||
bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61);
|
||||
|
||||
/* Pad out to 112 mod 128 and append length */
|
||||
index = sctx->count[0] & 0x7f;
|
||||
padlen = (index < 112) ? (112 - index) : ((128+112) - index);
|
||||
|
||||
if (!irq_fpu_usable()) {
|
||||
crypto_sha512_update(desc, padding, padlen);
|
||||
crypto_sha512_update(desc, (const u8 *)&bits, sizeof(bits));
|
||||
} else {
|
||||
kernel_fpu_begin();
|
||||
/* We need to fill a whole block for __sha512_ssse3_update() */
|
||||
if (padlen <= 112) {
|
||||
sctx->count[0] += padlen;
|
||||
if (sctx->count[0] < padlen)
|
||||
sctx->count[1]++;
|
||||
memcpy(sctx->buf + index, padding, padlen);
|
||||
} else {
|
||||
__sha512_ssse3_update(desc, padding, padlen, index);
|
||||
}
|
||||
__sha512_ssse3_update(desc, (const u8 *)&bits,
|
||||
sizeof(bits), 112);
|
||||
kernel_fpu_end();
|
||||
}
|
||||
|
||||
/* Store state in digest */
|
||||
for (i = 0; i < 8; i++)
|
||||
dst[i] = cpu_to_be64(sctx->state[i]);
|
||||
|
||||
/* Wipe context */
|
||||
memset(sctx, 0, sizeof(*sctx));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sha512_ssse3_export(struct shash_desc *desc, void *out)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
memcpy(out, sctx, sizeof(*sctx));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sha512_ssse3_import(struct shash_desc *desc, const void *in)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
memcpy(sctx, in, sizeof(*sctx));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sha384_ssse3_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
sctx->state[0] = SHA384_H0;
|
||||
sctx->state[1] = SHA384_H1;
|
||||
sctx->state[2] = SHA384_H2;
|
||||
sctx->state[3] = SHA384_H3;
|
||||
sctx->state[4] = SHA384_H4;
|
||||
sctx->state[5] = SHA384_H5;
|
||||
sctx->state[6] = SHA384_H6;
|
||||
sctx->state[7] = SHA384_H7;
|
||||
|
||||
sctx->count[0] = sctx->count[1] = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sha384_ssse3_final(struct shash_desc *desc, u8 *hash)
|
||||
{
|
||||
u8 D[SHA512_DIGEST_SIZE];
|
||||
|
||||
sha512_ssse3_final(desc, D);
|
||||
|
||||
memcpy(hash, D, SHA384_DIGEST_SIZE);
|
||||
memzero_explicit(D, SHA512_DIGEST_SIZE);
|
||||
|
||||
return 0;
|
||||
return sha512_ssse3_finup(desc, NULL, 0, out);
|
||||
}
|
||||
|
||||
static struct shash_alg algs[] = { {
|
||||
.digestsize = SHA512_DIGEST_SIZE,
|
||||
.init = sha512_ssse3_init,
|
||||
.init = sha512_base_init,
|
||||
.update = sha512_ssse3_update,
|
||||
.final = sha512_ssse3_final,
|
||||
.export = sha512_ssse3_export,
|
||||
.import = sha512_ssse3_import,
|
||||
.finup = sha512_ssse3_finup,
|
||||
.descsize = sizeof(struct sha512_state),
|
||||
.statesize = sizeof(struct sha512_state),
|
||||
.base = {
|
||||
.cra_name = "sha512",
|
||||
.cra_driver_name = "sha512-ssse3",
|
||||
|
|
@ -243,13 +113,11 @@ static struct shash_alg algs[] = { {
|
|||
}
|
||||
}, {
|
||||
.digestsize = SHA384_DIGEST_SIZE,
|
||||
.init = sha384_ssse3_init,
|
||||
.init = sha384_base_init,
|
||||
.update = sha512_ssse3_update,
|
||||
.final = sha384_ssse3_final,
|
||||
.export = sha512_ssse3_export,
|
||||
.import = sha512_ssse3_import,
|
||||
.final = sha512_ssse3_final,
|
||||
.finup = sha512_ssse3_finup,
|
||||
.descsize = sizeof(struct sha512_state),
|
||||
.statesize = sizeof(struct sha512_state),
|
||||
.base = {
|
||||
.cra_name = "sha384",
|
||||
.cra_driver_name = "sha384-ssse3",
|
||||
|
|
|
|||
Loading…
Reference in New Issue