crypto: scompress - Use per-CPU struct instead multiple variables
Two per-CPU variables are allocated as pointer to per-CPU memory which then are used as scratch buffers. We could be smart about this and use instead a per-CPU struct which contains the pointers already and then we need to allocate just the scratch buffers. Add a lock to the struct. By doing so we can avoid the get_cpu() statement and gain lockdep coverage (if enabled) to ensure that the lock is always acquired in the right context. On non-preemptible kernels the lock vanishes. It is okay to use raw_cpu_ptr() in order to get a pointer to the struct since it is protected by the spinlock. The diffstat of this is negative and according to size scompress.o: text data bss dec hex filename 1847 160 24 2031 7ef dbg_before.o 1754 232 4 1990 7c6 dbg_after.o 1799 64 24 1887 75f no_dbg-before.o 1703 88 4 1795 703 no_dbg-after.o The overall size increase difference is also negative. The increase in the data section is only four bytes without lockdep. Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>hifive-unleashed-5.2
Sebastian Andrzej Siewior
Herbert Xu
parent
6a4d1b18ef
commit
71052dcf4b
|
|
@ -29,9 +29,17 @@
|
|||
#include <crypto/internal/scompress.h>
|
||||
#include "internal.h"
|
||||
|
||||
struct scomp_scratch {
|
||||
spinlock_t lock;
|
||||
void *src;
|
||||
void *dst;
|
||||
};
|
||||
|
||||
static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = {
|
||||
.lock = __SPIN_LOCK_UNLOCKED(scomp_scratch.lock),
|
||||
};
|
||||
|
||||
static const struct crypto_type crypto_scomp_type;
|
||||
static void * __percpu *scomp_src_scratches;
|
||||
static void * __percpu *scomp_dst_scratches;
|
||||
static int scomp_scratch_users;
|
||||
static DEFINE_MUTEX(scomp_lock);
|
||||
|
||||
|
|
@ -62,76 +70,53 @@ static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
|
|||
seq_puts(m, "type : scomp\n");
|
||||
}
|
||||
|
||||
static void crypto_scomp_free_scratches(void * __percpu *scratches)
|
||||
static void crypto_scomp_free_scratches(void)
|
||||
{
|
||||
struct scomp_scratch *scratch;
|
||||
int i;
|
||||
|
||||
if (!scratches)
|
||||
return;
|
||||
|
||||
for_each_possible_cpu(i)
|
||||
vfree(*per_cpu_ptr(scratches, i));
|
||||
|
||||
free_percpu(scratches);
|
||||
}
|
||||
|
||||
static void * __percpu *crypto_scomp_alloc_scratches(void)
|
||||
{
|
||||
void * __percpu *scratches;
|
||||
int i;
|
||||
|
||||
scratches = alloc_percpu(void *);
|
||||
if (!scratches)
|
||||
return NULL;
|
||||
|
||||
for_each_possible_cpu(i) {
|
||||
void *scratch;
|
||||
scratch = raw_cpu_ptr(&scomp_scratch);
|
||||
|
||||
scratch = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i));
|
||||
if (!scratch)
|
||||
vfree(scratch->src);
|
||||
vfree(scratch->dst);
|
||||
scratch->src = NULL;
|
||||
scratch->dst = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static int crypto_scomp_alloc_scratches(void)
|
||||
{
|
||||
struct scomp_scratch *scratch;
|
||||
int i;
|
||||
|
||||
for_each_possible_cpu(i) {
|
||||
void *mem;
|
||||
|
||||
scratch = raw_cpu_ptr(&scomp_scratch);
|
||||
|
||||
mem = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i));
|
||||
if (!mem)
|
||||
goto error;
|
||||
*per_cpu_ptr(scratches, i) = scratch;
|
||||
}
|
||||
|
||||
return scratches;
|
||||
|
||||
error:
|
||||
crypto_scomp_free_scratches(scratches);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void crypto_scomp_free_all_scratches(void)
|
||||
{
|
||||
if (!--scomp_scratch_users) {
|
||||
crypto_scomp_free_scratches(scomp_src_scratches);
|
||||
crypto_scomp_free_scratches(scomp_dst_scratches);
|
||||
scomp_src_scratches = NULL;
|
||||
scomp_dst_scratches = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static int crypto_scomp_alloc_all_scratches(void)
|
||||
{
|
||||
if (!scomp_scratch_users++) {
|
||||
scomp_src_scratches = crypto_scomp_alloc_scratches();
|
||||
if (!scomp_src_scratches)
|
||||
return -ENOMEM;
|
||||
scomp_dst_scratches = crypto_scomp_alloc_scratches();
|
||||
if (!scomp_dst_scratches) {
|
||||
crypto_scomp_free_scratches(scomp_src_scratches);
|
||||
scomp_src_scratches = NULL;
|
||||
return -ENOMEM;
|
||||
}
|
||||
scratch->src = mem;
|
||||
mem = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i));
|
||||
if (!mem)
|
||||
goto error;
|
||||
scratch->dst = mem;
|
||||
}
|
||||
return 0;
|
||||
error:
|
||||
crypto_scomp_free_scratches();
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
static int crypto_scomp_init_tfm(struct crypto_tfm *tfm)
|
||||
{
|
||||
int ret;
|
||||
int ret = 0;
|
||||
|
||||
mutex_lock(&scomp_lock);
|
||||
ret = crypto_scomp_alloc_all_scratches();
|
||||
if (!scomp_scratch_users++)
|
||||
ret = crypto_scomp_alloc_scratches();
|
||||
mutex_unlock(&scomp_lock);
|
||||
|
||||
return ret;
|
||||
|
|
@ -143,31 +128,28 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
|
|||
void **tfm_ctx = acomp_tfm_ctx(tfm);
|
||||
struct crypto_scomp *scomp = *tfm_ctx;
|
||||
void **ctx = acomp_request_ctx(req);
|
||||
const int cpu = get_cpu();
|
||||
u8 *scratch_src = *per_cpu_ptr(scomp_src_scratches, cpu);
|
||||
u8 *scratch_dst = *per_cpu_ptr(scomp_dst_scratches, cpu);
|
||||
struct scomp_scratch *scratch;
|
||||
int ret;
|
||||
|
||||
if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE) {
|
||||
ret = -EINVAL;
|
||||
goto out;
|
||||
}
|
||||
if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE)
|
||||
return -EINVAL;
|
||||
|
||||
if (req->dst && !req->dlen) {
|
||||
ret = -EINVAL;
|
||||
goto out;
|
||||
}
|
||||
if (req->dst && !req->dlen)
|
||||
return -EINVAL;
|
||||
|
||||
if (!req->dlen || req->dlen > SCOMP_SCRATCH_SIZE)
|
||||
req->dlen = SCOMP_SCRATCH_SIZE;
|
||||
|
||||
scatterwalk_map_and_copy(scratch_src, req->src, 0, req->slen, 0);
|
||||
scratch = raw_cpu_ptr(&scomp_scratch);
|
||||
spin_lock(&scratch->lock);
|
||||
|
||||
scatterwalk_map_and_copy(scratch->src, req->src, 0, req->slen, 0);
|
||||
if (dir)
|
||||
ret = crypto_scomp_compress(scomp, scratch_src, req->slen,
|
||||
scratch_dst, &req->dlen, *ctx);
|
||||
ret = crypto_scomp_compress(scomp, scratch->src, req->slen,
|
||||
scratch->dst, &req->dlen, *ctx);
|
||||
else
|
||||
ret = crypto_scomp_decompress(scomp, scratch_src, req->slen,
|
||||
scratch_dst, &req->dlen, *ctx);
|
||||
ret = crypto_scomp_decompress(scomp, scratch->src, req->slen,
|
||||
scratch->dst, &req->dlen, *ctx);
|
||||
if (!ret) {
|
||||
if (!req->dst) {
|
||||
req->dst = sgl_alloc(req->dlen, GFP_ATOMIC, NULL);
|
||||
|
|
@ -176,11 +158,11 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
|
|||
goto out;
|
||||
}
|
||||
}
|
||||
scatterwalk_map_and_copy(scratch_dst, req->dst, 0, req->dlen,
|
||||
scatterwalk_map_and_copy(scratch->dst, req->dst, 0, req->dlen,
|
||||
1);
|
||||
}
|
||||
out:
|
||||
put_cpu();
|
||||
spin_unlock(&scratch->lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -201,7 +183,8 @@ static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm)
|
|||
crypto_free_scomp(*ctx);
|
||||
|
||||
mutex_lock(&scomp_lock);
|
||||
crypto_scomp_free_all_scratches();
|
||||
if (!--scomp_scratch_users)
|
||||
crypto_scomp_free_scratches();
|
||||
mutex_unlock(&scomp_lock);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue