866cd902e8
Currently we reset the key for each segment fed to the xcrypt instructions. This patch optimises this for CBC and ECB so that we only do this once for each encrypt/decrypt operation. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
717 lines
19 KiB
C
717 lines
19 KiB
C
/*
|
|
* Cryptographic API.
|
|
*
|
|
* Support for VIA PadLock hardware crypto engine.
|
|
*
|
|
* Copyright (c) 2004 Michal Ludvig <michal@logix.cz>
|
|
*
|
|
* Key expansion routine taken from crypto/aes_generic.c
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* ---------------------------------------------------------------------------
|
|
* Copyright (c) 2002, Dr Brian Gladman <brg@gladman.me.uk>, Worcester, UK.
|
|
* All rights reserved.
|
|
*
|
|
* LICENSE TERMS
|
|
*
|
|
* The free distribution and use of this software in both source and binary
|
|
* form is allowed (with or without changes) provided that:
|
|
*
|
|
* 1. distributions of this source code include the above copyright
|
|
* notice, this list of conditions and the following disclaimer;
|
|
*
|
|
* 2. distributions in binary form include the above copyright
|
|
* notice, this list of conditions and the following disclaimer
|
|
* in the documentation and/or other associated materials;
|
|
*
|
|
* 3. the copyright holder's name is not used to endorse products
|
|
* built using this software without specific written permission.
|
|
*
|
|
* ALTERNATIVELY, provided that this notice is retained in full, this product
|
|
* may be distributed under the terms of the GNU General Public License (GPL),
|
|
* in which case the provisions of the GPL apply INSTEAD OF those given above.
|
|
*
|
|
* DISCLAIMER
|
|
*
|
|
* This software is provided 'as is' with no explicit or implied warranties
|
|
* in respect of its properties, including, but not limited to, correctness
|
|
* and/or fitness for purpose.
|
|
* ---------------------------------------------------------------------------
|
|
*/
|
|
|
|
#include <crypto/algapi.h>
|
|
#include <crypto/aes.h>
|
|
#include <linux/module.h>
|
|
#include <linux/init.h>
|
|
#include <linux/types.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/kernel.h>
|
|
#include <asm/byteorder.h>
|
|
#include "padlock.h"
|
|
|
|
#define AES_EXTENDED_KEY_SIZE 64 /* in uint32_t units */
|
|
#define AES_EXTENDED_KEY_SIZE_B (AES_EXTENDED_KEY_SIZE * sizeof(uint32_t))
|
|
|
|
/* Control word. */
|
|
struct cword {
|
|
unsigned int __attribute__ ((__packed__))
|
|
rounds:4,
|
|
algo:3,
|
|
keygen:1,
|
|
interm:1,
|
|
encdec:1,
|
|
ksize:2;
|
|
} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
|
|
|
|
/* Whenever making any changes to the following
|
|
* structure *make sure* you keep E, d_data
|
|
* and cword aligned on 16 Bytes boundaries!!! */
|
|
struct aes_ctx {
|
|
struct {
|
|
struct cword encrypt;
|
|
struct cword decrypt;
|
|
} cword;
|
|
u32 *D;
|
|
int key_length;
|
|
u32 E[AES_EXTENDED_KEY_SIZE]
|
|
__attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
|
|
u32 d_data[AES_EXTENDED_KEY_SIZE]
|
|
__attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
|
|
};
|
|
|
|
/* ====== Key management routines ====== */
|
|
|
|
static inline uint32_t
|
|
generic_rotr32 (const uint32_t x, const unsigned bits)
|
|
{
|
|
const unsigned n = bits % 32;
|
|
return (x >> n) | (x << (32 - n));
|
|
}
|
|
|
|
static inline uint32_t
|
|
generic_rotl32 (const uint32_t x, const unsigned bits)
|
|
{
|
|
const unsigned n = bits % 32;
|
|
return (x << n) | (x >> (32 - n));
|
|
}
|
|
|
|
#define rotl generic_rotl32
|
|
#define rotr generic_rotr32
|
|
|
|
/*
|
|
* #define byte(x, nr) ((unsigned char)((x) >> (nr*8)))
|
|
*/
|
|
static inline uint8_t
|
|
byte(const uint32_t x, const unsigned n)
|
|
{
|
|
return x >> (n << 3);
|
|
}
|
|
|
|
#define E_KEY ctx->E
|
|
#define D_KEY ctx->D
|
|
|
|
static uint8_t pow_tab[256];
|
|
static uint8_t log_tab[256];
|
|
static uint8_t sbx_tab[256];
|
|
static uint8_t isb_tab[256];
|
|
static uint32_t rco_tab[10];
|
|
static uint32_t ft_tab[4][256];
|
|
static uint32_t it_tab[4][256];
|
|
|
|
static uint32_t fl_tab[4][256];
|
|
static uint32_t il_tab[4][256];
|
|
|
|
static inline uint8_t
|
|
f_mult (uint8_t a, uint8_t b)
|
|
{
|
|
uint8_t aa = log_tab[a], cc = aa + log_tab[b];
|
|
|
|
return pow_tab[cc + (cc < aa ? 1 : 0)];
|
|
}
|
|
|
|
#define ff_mult(a,b) (a && b ? f_mult(a, b) : 0)
|
|
|
|
#define f_rn(bo, bi, n, k) \
|
|
bo[n] = ft_tab[0][byte(bi[n],0)] ^ \
|
|
ft_tab[1][byte(bi[(n + 1) & 3],1)] ^ \
|
|
ft_tab[2][byte(bi[(n + 2) & 3],2)] ^ \
|
|
ft_tab[3][byte(bi[(n + 3) & 3],3)] ^ *(k + n)
|
|
|
|
#define i_rn(bo, bi, n, k) \
|
|
bo[n] = it_tab[0][byte(bi[n],0)] ^ \
|
|
it_tab[1][byte(bi[(n + 3) & 3],1)] ^ \
|
|
it_tab[2][byte(bi[(n + 2) & 3],2)] ^ \
|
|
it_tab[3][byte(bi[(n + 1) & 3],3)] ^ *(k + n)
|
|
|
|
#define ls_box(x) \
|
|
( fl_tab[0][byte(x, 0)] ^ \
|
|
fl_tab[1][byte(x, 1)] ^ \
|
|
fl_tab[2][byte(x, 2)] ^ \
|
|
fl_tab[3][byte(x, 3)] )
|
|
|
|
#define f_rl(bo, bi, n, k) \
|
|
bo[n] = fl_tab[0][byte(bi[n],0)] ^ \
|
|
fl_tab[1][byte(bi[(n + 1) & 3],1)] ^ \
|
|
fl_tab[2][byte(bi[(n + 2) & 3],2)] ^ \
|
|
fl_tab[3][byte(bi[(n + 3) & 3],3)] ^ *(k + n)
|
|
|
|
#define i_rl(bo, bi, n, k) \
|
|
bo[n] = il_tab[0][byte(bi[n],0)] ^ \
|
|
il_tab[1][byte(bi[(n + 3) & 3],1)] ^ \
|
|
il_tab[2][byte(bi[(n + 2) & 3],2)] ^ \
|
|
il_tab[3][byte(bi[(n + 1) & 3],3)] ^ *(k + n)
|
|
|
|
static void
|
|
gen_tabs (void)
|
|
{
|
|
uint32_t i, t;
|
|
uint8_t p, q;
|
|
|
|
/* log and power tables for GF(2**8) finite field with
|
|
0x011b as modular polynomial - the simplest prmitive
|
|
root is 0x03, used here to generate the tables */
|
|
|
|
for (i = 0, p = 1; i < 256; ++i) {
|
|
pow_tab[i] = (uint8_t) p;
|
|
log_tab[p] = (uint8_t) i;
|
|
|
|
p ^= (p << 1) ^ (p & 0x80 ? 0x01b : 0);
|
|
}
|
|
|
|
log_tab[1] = 0;
|
|
|
|
for (i = 0, p = 1; i < 10; ++i) {
|
|
rco_tab[i] = p;
|
|
|
|
p = (p << 1) ^ (p & 0x80 ? 0x01b : 0);
|
|
}
|
|
|
|
for (i = 0; i < 256; ++i) {
|
|
p = (i ? pow_tab[255 - log_tab[i]] : 0);
|
|
q = ((p >> 7) | (p << 1)) ^ ((p >> 6) | (p << 2));
|
|
p ^= 0x63 ^ q ^ ((q >> 6) | (q << 2));
|
|
sbx_tab[i] = p;
|
|
isb_tab[p] = (uint8_t) i;
|
|
}
|
|
|
|
for (i = 0; i < 256; ++i) {
|
|
p = sbx_tab[i];
|
|
|
|
t = p;
|
|
fl_tab[0][i] = t;
|
|
fl_tab[1][i] = rotl (t, 8);
|
|
fl_tab[2][i] = rotl (t, 16);
|
|
fl_tab[3][i] = rotl (t, 24);
|
|
|
|
t = ((uint32_t) ff_mult (2, p)) |
|
|
((uint32_t) p << 8) |
|
|
((uint32_t) p << 16) | ((uint32_t) ff_mult (3, p) << 24);
|
|
|
|
ft_tab[0][i] = t;
|
|
ft_tab[1][i] = rotl (t, 8);
|
|
ft_tab[2][i] = rotl (t, 16);
|
|
ft_tab[3][i] = rotl (t, 24);
|
|
|
|
p = isb_tab[i];
|
|
|
|
t = p;
|
|
il_tab[0][i] = t;
|
|
il_tab[1][i] = rotl (t, 8);
|
|
il_tab[2][i] = rotl (t, 16);
|
|
il_tab[3][i] = rotl (t, 24);
|
|
|
|
t = ((uint32_t) ff_mult (14, p)) |
|
|
((uint32_t) ff_mult (9, p) << 8) |
|
|
((uint32_t) ff_mult (13, p) << 16) |
|
|
((uint32_t) ff_mult (11, p) << 24);
|
|
|
|
it_tab[0][i] = t;
|
|
it_tab[1][i] = rotl (t, 8);
|
|
it_tab[2][i] = rotl (t, 16);
|
|
it_tab[3][i] = rotl (t, 24);
|
|
}
|
|
}
|
|
|
|
#define star_x(x) (((x) & 0x7f7f7f7f) << 1) ^ ((((x) & 0x80808080) >> 7) * 0x1b)
|
|
|
|
#define imix_col(y,x) \
|
|
u = star_x(x); \
|
|
v = star_x(u); \
|
|
w = star_x(v); \
|
|
t = w ^ (x); \
|
|
(y) = u ^ v ^ w; \
|
|
(y) ^= rotr(u ^ t, 8) ^ \
|
|
rotr(v ^ t, 16) ^ \
|
|
rotr(t,24)
|
|
|
|
/* initialise the key schedule from the user supplied key */
|
|
|
|
#define loop4(i) \
|
|
{ t = rotr(t, 8); t = ls_box(t) ^ rco_tab[i]; \
|
|
t ^= E_KEY[4 * i]; E_KEY[4 * i + 4] = t; \
|
|
t ^= E_KEY[4 * i + 1]; E_KEY[4 * i + 5] = t; \
|
|
t ^= E_KEY[4 * i + 2]; E_KEY[4 * i + 6] = t; \
|
|
t ^= E_KEY[4 * i + 3]; E_KEY[4 * i + 7] = t; \
|
|
}
|
|
|
|
#define loop6(i) \
|
|
{ t = rotr(t, 8); t = ls_box(t) ^ rco_tab[i]; \
|
|
t ^= E_KEY[6 * i]; E_KEY[6 * i + 6] = t; \
|
|
t ^= E_KEY[6 * i + 1]; E_KEY[6 * i + 7] = t; \
|
|
t ^= E_KEY[6 * i + 2]; E_KEY[6 * i + 8] = t; \
|
|
t ^= E_KEY[6 * i + 3]; E_KEY[6 * i + 9] = t; \
|
|
t ^= E_KEY[6 * i + 4]; E_KEY[6 * i + 10] = t; \
|
|
t ^= E_KEY[6 * i + 5]; E_KEY[6 * i + 11] = t; \
|
|
}
|
|
|
|
#define loop8(i) \
|
|
{ t = rotr(t, 8); ; t = ls_box(t) ^ rco_tab[i]; \
|
|
t ^= E_KEY[8 * i]; E_KEY[8 * i + 8] = t; \
|
|
t ^= E_KEY[8 * i + 1]; E_KEY[8 * i + 9] = t; \
|
|
t ^= E_KEY[8 * i + 2]; E_KEY[8 * i + 10] = t; \
|
|
t ^= E_KEY[8 * i + 3]; E_KEY[8 * i + 11] = t; \
|
|
t = E_KEY[8 * i + 4] ^ ls_box(t); \
|
|
E_KEY[8 * i + 12] = t; \
|
|
t ^= E_KEY[8 * i + 5]; E_KEY[8 * i + 13] = t; \
|
|
t ^= E_KEY[8 * i + 6]; E_KEY[8 * i + 14] = t; \
|
|
t ^= E_KEY[8 * i + 7]; E_KEY[8 * i + 15] = t; \
|
|
}
|
|
|
|
/* Tells whether the ACE is capable to generate
|
|
the extended key for a given key_len. */
|
|
static inline int
|
|
aes_hw_extkey_available(uint8_t key_len)
|
|
{
|
|
/* TODO: We should check the actual CPU model/stepping
|
|
as it's possible that the capability will be
|
|
added in the next CPU revisions. */
|
|
if (key_len == 16)
|
|
return 1;
|
|
return 0;
|
|
}
|
|
|
|
static inline struct aes_ctx *aes_ctx_common(void *ctx)
|
|
{
|
|
unsigned long addr = (unsigned long)ctx;
|
|
unsigned long align = PADLOCK_ALIGNMENT;
|
|
|
|
if (align <= crypto_tfm_ctx_alignment())
|
|
align = 1;
|
|
return (struct aes_ctx *)ALIGN(addr, align);
|
|
}
|
|
|
|
static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
|
|
{
|
|
return aes_ctx_common(crypto_tfm_ctx(tfm));
|
|
}
|
|
|
|
static inline struct aes_ctx *blk_aes_ctx(struct crypto_blkcipher *tfm)
|
|
{
|
|
return aes_ctx_common(crypto_blkcipher_ctx(tfm));
|
|
}
|
|
|
|
static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
|
|
unsigned int key_len)
|
|
{
|
|
struct aes_ctx *ctx = aes_ctx(tfm);
|
|
const __le32 *key = (const __le32 *)in_key;
|
|
u32 *flags = &tfm->crt_flags;
|
|
uint32_t i, t, u, v, w;
|
|
uint32_t P[AES_EXTENDED_KEY_SIZE];
|
|
uint32_t rounds;
|
|
|
|
if (key_len % 8) {
|
|
*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
|
|
return -EINVAL;
|
|
}
|
|
|
|
ctx->key_length = key_len;
|
|
|
|
/*
|
|
* If the hardware is capable of generating the extended key
|
|
* itself we must supply the plain key for both encryption
|
|
* and decryption.
|
|
*/
|
|
ctx->D = ctx->E;
|
|
|
|
E_KEY[0] = le32_to_cpu(key[0]);
|
|
E_KEY[1] = le32_to_cpu(key[1]);
|
|
E_KEY[2] = le32_to_cpu(key[2]);
|
|
E_KEY[3] = le32_to_cpu(key[3]);
|
|
|
|
/* Prepare control words. */
|
|
memset(&ctx->cword, 0, sizeof(ctx->cword));
|
|
|
|
ctx->cword.decrypt.encdec = 1;
|
|
ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4;
|
|
ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds;
|
|
ctx->cword.encrypt.ksize = (key_len - 16) / 8;
|
|
ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;
|
|
|
|
/* Don't generate extended keys if the hardware can do it. */
|
|
if (aes_hw_extkey_available(key_len))
|
|
return 0;
|
|
|
|
ctx->D = ctx->d_data;
|
|
ctx->cword.encrypt.keygen = 1;
|
|
ctx->cword.decrypt.keygen = 1;
|
|
|
|
switch (key_len) {
|
|
case 16:
|
|
t = E_KEY[3];
|
|
for (i = 0; i < 10; ++i)
|
|
loop4 (i);
|
|
break;
|
|
|
|
case 24:
|
|
E_KEY[4] = le32_to_cpu(key[4]);
|
|
t = E_KEY[5] = le32_to_cpu(key[5]);
|
|
for (i = 0; i < 8; ++i)
|
|
loop6 (i);
|
|
break;
|
|
|
|
case 32:
|
|
E_KEY[4] = le32_to_cpu(key[4]);
|
|
E_KEY[5] = le32_to_cpu(key[5]);
|
|
E_KEY[6] = le32_to_cpu(key[6]);
|
|
t = E_KEY[7] = le32_to_cpu(key[7]);
|
|
for (i = 0; i < 7; ++i)
|
|
loop8 (i);
|
|
break;
|
|
}
|
|
|
|
D_KEY[0] = E_KEY[0];
|
|
D_KEY[1] = E_KEY[1];
|
|
D_KEY[2] = E_KEY[2];
|
|
D_KEY[3] = E_KEY[3];
|
|
|
|
for (i = 4; i < key_len + 24; ++i) {
|
|
imix_col (D_KEY[i], E_KEY[i]);
|
|
}
|
|
|
|
/* PadLock needs a different format of the decryption key. */
|
|
rounds = 10 + (key_len - 16) / 4;
|
|
|
|
for (i = 0; i < rounds; i++) {
|
|
P[((i + 1) * 4) + 0] = D_KEY[((rounds - i - 1) * 4) + 0];
|
|
P[((i + 1) * 4) + 1] = D_KEY[((rounds - i - 1) * 4) + 1];
|
|
P[((i + 1) * 4) + 2] = D_KEY[((rounds - i - 1) * 4) + 2];
|
|
P[((i + 1) * 4) + 3] = D_KEY[((rounds - i - 1) * 4) + 3];
|
|
}
|
|
|
|
P[0] = E_KEY[(rounds * 4) + 0];
|
|
P[1] = E_KEY[(rounds * 4) + 1];
|
|
P[2] = E_KEY[(rounds * 4) + 2];
|
|
P[3] = E_KEY[(rounds * 4) + 3];
|
|
|
|
memcpy(D_KEY, P, AES_EXTENDED_KEY_SIZE_B);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* ====== Encryption/decryption routines ====== */
|
|
|
|
/* These are the real call to PadLock. */
|
|
static inline void padlock_reset_key(void)
|
|
{
|
|
asm volatile ("pushfl; popfl");
|
|
}
|
|
|
|
static inline void padlock_xcrypt(const u8 *input, u8 *output, void *key,
|
|
void *control_word)
|
|
{
|
|
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
|
|
: "+S"(input), "+D"(output)
|
|
: "d"(control_word), "b"(key), "c"(1));
|
|
}
|
|
|
|
static void aes_crypt_copy(const u8 *in, u8 *out, u32 *key, struct cword *cword)
|
|
{
|
|
u8 buf[AES_BLOCK_SIZE * 2 + PADLOCK_ALIGNMENT - 1];
|
|
u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
|
|
|
|
memcpy(tmp, in, AES_BLOCK_SIZE);
|
|
padlock_xcrypt(tmp, out, key, cword);
|
|
}
|
|
|
|
static inline void aes_crypt(const u8 *in, u8 *out, u32 *key,
|
|
struct cword *cword)
|
|
{
|
|
/* padlock_xcrypt requires at least two blocks of data. */
|
|
if (unlikely(!(((unsigned long)in ^ (PAGE_SIZE - AES_BLOCK_SIZE)) &
|
|
(PAGE_SIZE - 1)))) {
|
|
aes_crypt_copy(in, out, key, cword);
|
|
return;
|
|
}
|
|
|
|
padlock_xcrypt(in, out, key, cword);
|
|
}
|
|
|
|
static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
|
|
void *control_word, u32 count)
|
|
{
|
|
if (count == 1) {
|
|
aes_crypt(input, output, key, control_word);
|
|
return;
|
|
}
|
|
|
|
asm volatile ("test $1, %%cl;"
|
|
"je 1f;"
|
|
"lea -1(%%ecx), %%eax;"
|
|
"mov $1, %%ecx;"
|
|
".byte 0xf3,0x0f,0xa7,0xc8;" /* rep xcryptecb */
|
|
"mov %%eax, %%ecx;"
|
|
"1:"
|
|
".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
|
|
: "+S"(input), "+D"(output)
|
|
: "d"(control_word), "b"(key), "c"(count)
|
|
: "ax");
|
|
}
|
|
|
|
static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
|
|
u8 *iv, void *control_word, u32 count)
|
|
{
|
|
/* rep xcryptcbc */
|
|
asm volatile (".byte 0xf3,0x0f,0xa7,0xd0"
|
|
: "+S" (input), "+D" (output), "+a" (iv)
|
|
: "d" (control_word), "b" (key), "c" (count));
|
|
return iv;
|
|
}
|
|
|
|
static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
|
|
{
|
|
struct aes_ctx *ctx = aes_ctx(tfm);
|
|
padlock_reset_key();
|
|
aes_crypt(in, out, ctx->E, &ctx->cword.encrypt);
|
|
}
|
|
|
|
static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
|
|
{
|
|
struct aes_ctx *ctx = aes_ctx(tfm);
|
|
padlock_reset_key();
|
|
aes_crypt(in, out, ctx->D, &ctx->cword.decrypt);
|
|
}
|
|
|
|
static struct crypto_alg aes_alg = {
|
|
.cra_name = "aes",
|
|
.cra_driver_name = "aes-padlock",
|
|
.cra_priority = PADLOCK_CRA_PRIORITY,
|
|
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
|
|
.cra_blocksize = AES_BLOCK_SIZE,
|
|
.cra_ctxsize = sizeof(struct aes_ctx),
|
|
.cra_alignmask = PADLOCK_ALIGNMENT - 1,
|
|
.cra_module = THIS_MODULE,
|
|
.cra_list = LIST_HEAD_INIT(aes_alg.cra_list),
|
|
.cra_u = {
|
|
.cipher = {
|
|
.cia_min_keysize = AES_MIN_KEY_SIZE,
|
|
.cia_max_keysize = AES_MAX_KEY_SIZE,
|
|
.cia_setkey = aes_set_key,
|
|
.cia_encrypt = aes_encrypt,
|
|
.cia_decrypt = aes_decrypt,
|
|
}
|
|
}
|
|
};
|
|
|
|
static int ecb_aes_encrypt(struct blkcipher_desc *desc,
|
|
struct scatterlist *dst, struct scatterlist *src,
|
|
unsigned int nbytes)
|
|
{
|
|
struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
|
|
struct blkcipher_walk walk;
|
|
int err;
|
|
|
|
padlock_reset_key();
|
|
|
|
blkcipher_walk_init(&walk, dst, src, nbytes);
|
|
err = blkcipher_walk_virt(desc, &walk);
|
|
|
|
while ((nbytes = walk.nbytes)) {
|
|
padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
|
|
ctx->E, &ctx->cword.encrypt,
|
|
nbytes / AES_BLOCK_SIZE);
|
|
nbytes &= AES_BLOCK_SIZE - 1;
|
|
err = blkcipher_walk_done(desc, &walk, nbytes);
|
|
}
|
|
|
|
return err;
|
|
}
|
|
|
|
static int ecb_aes_decrypt(struct blkcipher_desc *desc,
|
|
struct scatterlist *dst, struct scatterlist *src,
|
|
unsigned int nbytes)
|
|
{
|
|
struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
|
|
struct blkcipher_walk walk;
|
|
int err;
|
|
|
|
padlock_reset_key();
|
|
|
|
blkcipher_walk_init(&walk, dst, src, nbytes);
|
|
err = blkcipher_walk_virt(desc, &walk);
|
|
|
|
while ((nbytes = walk.nbytes)) {
|
|
padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
|
|
ctx->D, &ctx->cword.decrypt,
|
|
nbytes / AES_BLOCK_SIZE);
|
|
nbytes &= AES_BLOCK_SIZE - 1;
|
|
err = blkcipher_walk_done(desc, &walk, nbytes);
|
|
}
|
|
|
|
return err;
|
|
}
|
|
|
|
static struct crypto_alg ecb_aes_alg = {
|
|
.cra_name = "ecb(aes)",
|
|
.cra_driver_name = "ecb-aes-padlock",
|
|
.cra_priority = PADLOCK_COMPOSITE_PRIORITY,
|
|
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
|
|
.cra_blocksize = AES_BLOCK_SIZE,
|
|
.cra_ctxsize = sizeof(struct aes_ctx),
|
|
.cra_alignmask = PADLOCK_ALIGNMENT - 1,
|
|
.cra_type = &crypto_blkcipher_type,
|
|
.cra_module = THIS_MODULE,
|
|
.cra_list = LIST_HEAD_INIT(ecb_aes_alg.cra_list),
|
|
.cra_u = {
|
|
.blkcipher = {
|
|
.min_keysize = AES_MIN_KEY_SIZE,
|
|
.max_keysize = AES_MAX_KEY_SIZE,
|
|
.setkey = aes_set_key,
|
|
.encrypt = ecb_aes_encrypt,
|
|
.decrypt = ecb_aes_decrypt,
|
|
}
|
|
}
|
|
};
|
|
|
|
static int cbc_aes_encrypt(struct blkcipher_desc *desc,
|
|
struct scatterlist *dst, struct scatterlist *src,
|
|
unsigned int nbytes)
|
|
{
|
|
struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
|
|
struct blkcipher_walk walk;
|
|
int err;
|
|
|
|
padlock_reset_key();
|
|
|
|
blkcipher_walk_init(&walk, dst, src, nbytes);
|
|
err = blkcipher_walk_virt(desc, &walk);
|
|
|
|
while ((nbytes = walk.nbytes)) {
|
|
u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
|
|
walk.dst.virt.addr, ctx->E,
|
|
walk.iv, &ctx->cword.encrypt,
|
|
nbytes / AES_BLOCK_SIZE);
|
|
memcpy(walk.iv, iv, AES_BLOCK_SIZE);
|
|
nbytes &= AES_BLOCK_SIZE - 1;
|
|
err = blkcipher_walk_done(desc, &walk, nbytes);
|
|
}
|
|
|
|
return err;
|
|
}
|
|
|
|
static int cbc_aes_decrypt(struct blkcipher_desc *desc,
|
|
struct scatterlist *dst, struct scatterlist *src,
|
|
unsigned int nbytes)
|
|
{
|
|
struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
|
|
struct blkcipher_walk walk;
|
|
int err;
|
|
|
|
padlock_reset_key();
|
|
|
|
blkcipher_walk_init(&walk, dst, src, nbytes);
|
|
err = blkcipher_walk_virt(desc, &walk);
|
|
|
|
while ((nbytes = walk.nbytes)) {
|
|
padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
|
|
ctx->D, walk.iv, &ctx->cword.decrypt,
|
|
nbytes / AES_BLOCK_SIZE);
|
|
nbytes &= AES_BLOCK_SIZE - 1;
|
|
err = blkcipher_walk_done(desc, &walk, nbytes);
|
|
}
|
|
|
|
return err;
|
|
}
|
|
|
|
static struct crypto_alg cbc_aes_alg = {
|
|
.cra_name = "cbc(aes)",
|
|
.cra_driver_name = "cbc-aes-padlock",
|
|
.cra_priority = PADLOCK_COMPOSITE_PRIORITY,
|
|
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
|
|
.cra_blocksize = AES_BLOCK_SIZE,
|
|
.cra_ctxsize = sizeof(struct aes_ctx),
|
|
.cra_alignmask = PADLOCK_ALIGNMENT - 1,
|
|
.cra_type = &crypto_blkcipher_type,
|
|
.cra_module = THIS_MODULE,
|
|
.cra_list = LIST_HEAD_INIT(cbc_aes_alg.cra_list),
|
|
.cra_u = {
|
|
.blkcipher = {
|
|
.min_keysize = AES_MIN_KEY_SIZE,
|
|
.max_keysize = AES_MAX_KEY_SIZE,
|
|
.ivsize = AES_BLOCK_SIZE,
|
|
.setkey = aes_set_key,
|
|
.encrypt = cbc_aes_encrypt,
|
|
.decrypt = cbc_aes_decrypt,
|
|
}
|
|
}
|
|
};
|
|
|
|
static int __init padlock_init(void)
|
|
{
|
|
int ret;
|
|
|
|
if (!cpu_has_xcrypt) {
|
|
printk(KERN_ERR PFX "VIA PadLock not detected.\n");
|
|
return -ENODEV;
|
|
}
|
|
|
|
if (!cpu_has_xcrypt_enabled) {
|
|
printk(KERN_ERR PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
|
|
return -ENODEV;
|
|
}
|
|
|
|
gen_tabs();
|
|
if ((ret = crypto_register_alg(&aes_alg)))
|
|
goto aes_err;
|
|
|
|
if ((ret = crypto_register_alg(&ecb_aes_alg)))
|
|
goto ecb_aes_err;
|
|
|
|
if ((ret = crypto_register_alg(&cbc_aes_alg)))
|
|
goto cbc_aes_err;
|
|
|
|
printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
|
|
|
|
out:
|
|
return ret;
|
|
|
|
cbc_aes_err:
|
|
crypto_unregister_alg(&ecb_aes_alg);
|
|
ecb_aes_err:
|
|
crypto_unregister_alg(&aes_alg);
|
|
aes_err:
|
|
printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
|
|
goto out;
|
|
}
|
|
|
|
static void __exit padlock_fini(void)
|
|
{
|
|
crypto_unregister_alg(&cbc_aes_alg);
|
|
crypto_unregister_alg(&ecb_aes_alg);
|
|
crypto_unregister_alg(&aes_alg);
|
|
}
|
|
|
|
module_init(padlock_init);
|
|
module_exit(padlock_fini);
|
|
|
|
MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
|
|
MODULE_LICENSE("GPL");
|
|
MODULE_AUTHOR("Michal Ludvig");
|
|
|
|
MODULE_ALIAS("aes");
|