![Randy Dunlap](/assets/img/avatar_default.png)
Drop the doubled word "a". Signed-off-by: Randy Dunlap <rdunlap@infradead.org> Cc: Herbert Xu <herbert@gondor.apana.org.au> Cc: "David S. Miller" <davem@davemloft.net> Cc: linux-crypto@vger.kernel.org Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
568 lines
15 KiB
C
568 lines
15 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
/*
|
|
* Handle async block request by crypto hardware engine.
|
|
*
|
|
* Copyright (C) 2016 Linaro, Inc.
|
|
*
|
|
* Author: Baolin Wang <baolin.wang@linaro.org>
|
|
*/
|
|
|
|
#include <linux/err.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/device.h>
|
|
#include <crypto/engine.h>
|
|
#include <uapi/linux/sched/types.h>
|
|
#include "internal.h"
|
|
|
|
#define CRYPTO_ENGINE_MAX_QLEN 10
|
|
|
|
/**
|
|
* crypto_finalize_request - finalize one request if the request is done
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be finalized
|
|
* @err: error number
|
|
*/
|
|
static void crypto_finalize_request(struct crypto_engine *engine,
|
|
struct crypto_async_request *req, int err)
|
|
{
|
|
unsigned long flags;
|
|
bool finalize_req = false;
|
|
int ret;
|
|
struct crypto_engine_ctx *enginectx;
|
|
|
|
/*
|
|
* If hardware cannot enqueue more requests
|
|
* and retry mechanism is not supported
|
|
* make sure we are completing the current request
|
|
*/
|
|
if (!engine->retry_support) {
|
|
spin_lock_irqsave(&engine->queue_lock, flags);
|
|
if (engine->cur_req == req) {
|
|
finalize_req = true;
|
|
engine->cur_req = NULL;
|
|
}
|
|
spin_unlock_irqrestore(&engine->queue_lock, flags);
|
|
}
|
|
|
|
if (finalize_req || engine->retry_support) {
|
|
enginectx = crypto_tfm_ctx(req->tfm);
|
|
if (enginectx->op.prepare_request &&
|
|
enginectx->op.unprepare_request) {
|
|
ret = enginectx->op.unprepare_request(engine, req);
|
|
if (ret)
|
|
dev_err(engine->dev, "failed to unprepare request\n");
|
|
}
|
|
}
|
|
req->complete(req, err);
|
|
|
|
kthread_queue_work(engine->kworker, &engine->pump_requests);
|
|
}
|
|
|
|
/**
|
|
* crypto_pump_requests - dequeue one request from engine queue to process
|
|
* @engine: the hardware engine
|
|
* @in_kthread: true if we are in the context of the request pump thread
|
|
*
|
|
* This function checks if there is any request in the engine queue that
|
|
* needs processing and if so call out to the driver to initialize hardware
|
|
* and handle each request.
|
|
*/
|
|
static void crypto_pump_requests(struct crypto_engine *engine,
|
|
bool in_kthread)
|
|
{
|
|
struct crypto_async_request *async_req, *backlog;
|
|
unsigned long flags;
|
|
bool was_busy = false;
|
|
int ret;
|
|
struct crypto_engine_ctx *enginectx;
|
|
|
|
spin_lock_irqsave(&engine->queue_lock, flags);
|
|
|
|
/* Make sure we are not already running a request */
|
|
if (!engine->retry_support && engine->cur_req)
|
|
goto out;
|
|
|
|
/* If another context is idling then defer */
|
|
if (engine->idling) {
|
|
kthread_queue_work(engine->kworker, &engine->pump_requests);
|
|
goto out;
|
|
}
|
|
|
|
/* Check if the engine queue is idle */
|
|
if (!crypto_queue_len(&engine->queue) || !engine->running) {
|
|
if (!engine->busy)
|
|
goto out;
|
|
|
|
/* Only do teardown in the thread */
|
|
if (!in_kthread) {
|
|
kthread_queue_work(engine->kworker,
|
|
&engine->pump_requests);
|
|
goto out;
|
|
}
|
|
|
|
engine->busy = false;
|
|
engine->idling = true;
|
|
spin_unlock_irqrestore(&engine->queue_lock, flags);
|
|
|
|
if (engine->unprepare_crypt_hardware &&
|
|
engine->unprepare_crypt_hardware(engine))
|
|
dev_err(engine->dev, "failed to unprepare crypt hardware\n");
|
|
|
|
spin_lock_irqsave(&engine->queue_lock, flags);
|
|
engine->idling = false;
|
|
goto out;
|
|
}
|
|
|
|
start_request:
|
|
/* Get the fist request from the engine queue to handle */
|
|
backlog = crypto_get_backlog(&engine->queue);
|
|
async_req = crypto_dequeue_request(&engine->queue);
|
|
if (!async_req)
|
|
goto out;
|
|
|
|
/*
|
|
* If hardware doesn't support the retry mechanism,
|
|
* keep track of the request we are processing now.
|
|
* We'll need it on completion (crypto_finalize_request).
|
|
*/
|
|
if (!engine->retry_support)
|
|
engine->cur_req = async_req;
|
|
|
|
if (backlog)
|
|
backlog->complete(backlog, -EINPROGRESS);
|
|
|
|
if (engine->busy)
|
|
was_busy = true;
|
|
else
|
|
engine->busy = true;
|
|
|
|
spin_unlock_irqrestore(&engine->queue_lock, flags);
|
|
|
|
/* Until here we get the request need to be encrypted successfully */
|
|
if (!was_busy && engine->prepare_crypt_hardware) {
|
|
ret = engine->prepare_crypt_hardware(engine);
|
|
if (ret) {
|
|
dev_err(engine->dev, "failed to prepare crypt hardware\n");
|
|
goto req_err_2;
|
|
}
|
|
}
|
|
|
|
enginectx = crypto_tfm_ctx(async_req->tfm);
|
|
|
|
if (enginectx->op.prepare_request) {
|
|
ret = enginectx->op.prepare_request(engine, async_req);
|
|
if (ret) {
|
|
dev_err(engine->dev, "failed to prepare request: %d\n",
|
|
ret);
|
|
goto req_err_2;
|
|
}
|
|
}
|
|
if (!enginectx->op.do_one_request) {
|
|
dev_err(engine->dev, "failed to do request\n");
|
|
ret = -EINVAL;
|
|
goto req_err_1;
|
|
}
|
|
|
|
ret = enginectx->op.do_one_request(engine, async_req);
|
|
|
|
/* Request unsuccessfully executed by hardware */
|
|
if (ret < 0) {
|
|
/*
|
|
* If hardware queue is full (-ENOSPC), requeue request
|
|
* regardless of backlog flag.
|
|
* Otherwise, unprepare and complete the request.
|
|
*/
|
|
if (!engine->retry_support ||
|
|
(ret != -ENOSPC)) {
|
|
dev_err(engine->dev,
|
|
"Failed to do one request from queue: %d\n",
|
|
ret);
|
|
goto req_err_1;
|
|
}
|
|
/*
|
|
* If retry mechanism is supported,
|
|
* unprepare current request and
|
|
* enqueue it back into crypto-engine queue.
|
|
*/
|
|
if (enginectx->op.unprepare_request) {
|
|
ret = enginectx->op.unprepare_request(engine,
|
|
async_req);
|
|
if (ret)
|
|
dev_err(engine->dev,
|
|
"failed to unprepare request\n");
|
|
}
|
|
spin_lock_irqsave(&engine->queue_lock, flags);
|
|
/*
|
|
* If hardware was unable to execute request, enqueue it
|
|
* back in front of crypto-engine queue, to keep the order
|
|
* of requests.
|
|
*/
|
|
crypto_enqueue_request_head(&engine->queue, async_req);
|
|
|
|
kthread_queue_work(engine->kworker, &engine->pump_requests);
|
|
goto out;
|
|
}
|
|
|
|
goto retry;
|
|
|
|
req_err_1:
|
|
if (enginectx->op.unprepare_request) {
|
|
ret = enginectx->op.unprepare_request(engine, async_req);
|
|
if (ret)
|
|
dev_err(engine->dev, "failed to unprepare request\n");
|
|
}
|
|
|
|
req_err_2:
|
|
async_req->complete(async_req, ret);
|
|
|
|
retry:
|
|
/* If retry mechanism is supported, send new requests to engine */
|
|
if (engine->retry_support) {
|
|
spin_lock_irqsave(&engine->queue_lock, flags);
|
|
goto start_request;
|
|
}
|
|
return;
|
|
|
|
out:
|
|
spin_unlock_irqrestore(&engine->queue_lock, flags);
|
|
|
|
/*
|
|
* Batch requests is possible only if
|
|
* hardware can enqueue multiple requests
|
|
*/
|
|
if (engine->do_batch_requests) {
|
|
ret = engine->do_batch_requests(engine);
|
|
if (ret)
|
|
dev_err(engine->dev, "failed to do batch requests: %d\n",
|
|
ret);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
static void crypto_pump_work(struct kthread_work *work)
|
|
{
|
|
struct crypto_engine *engine =
|
|
container_of(work, struct crypto_engine, pump_requests);
|
|
|
|
crypto_pump_requests(engine, true);
|
|
}
|
|
|
|
/**
|
|
* crypto_transfer_request - transfer the new request into the engine queue
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be listed into the engine queue
|
|
*/
|
|
static int crypto_transfer_request(struct crypto_engine *engine,
|
|
struct crypto_async_request *req,
|
|
bool need_pump)
|
|
{
|
|
unsigned long flags;
|
|
int ret;
|
|
|
|
spin_lock_irqsave(&engine->queue_lock, flags);
|
|
|
|
if (!engine->running) {
|
|
spin_unlock_irqrestore(&engine->queue_lock, flags);
|
|
return -ESHUTDOWN;
|
|
}
|
|
|
|
ret = crypto_enqueue_request(&engine->queue, req);
|
|
|
|
if (!engine->busy && need_pump)
|
|
kthread_queue_work(engine->kworker, &engine->pump_requests);
|
|
|
|
spin_unlock_irqrestore(&engine->queue_lock, flags);
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* crypto_transfer_request_to_engine - transfer one request to list
|
|
* into the engine queue
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be listed into the engine queue
|
|
*/
|
|
static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
|
|
struct crypto_async_request *req)
|
|
{
|
|
return crypto_transfer_request(engine, req, true);
|
|
}
|
|
|
|
/**
|
|
* crypto_transfer_aead_request_to_engine - transfer one aead_request
|
|
* to list into the engine queue
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be listed into the engine queue
|
|
*/
|
|
int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
|
|
struct aead_request *req)
|
|
{
|
|
return crypto_transfer_request_to_engine(engine, &req->base);
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);
|
|
|
|
/**
|
|
* crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
|
|
* to list into the engine queue
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be listed into the engine queue
|
|
*/
|
|
int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
|
|
struct akcipher_request *req)
|
|
{
|
|
return crypto_transfer_request_to_engine(engine, &req->base);
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);
|
|
|
|
/**
|
|
* crypto_transfer_hash_request_to_engine - transfer one ahash_request
|
|
* to list into the engine queue
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be listed into the engine queue
|
|
*/
|
|
int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
|
|
struct ahash_request *req)
|
|
{
|
|
return crypto_transfer_request_to_engine(engine, &req->base);
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
|
|
|
|
/**
|
|
* crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
|
|
* to list into the engine queue
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be listed into the engine queue
|
|
*/
|
|
int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
|
|
struct skcipher_request *req)
|
|
{
|
|
return crypto_transfer_request_to_engine(engine, &req->base);
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);
|
|
|
|
/**
|
|
* crypto_finalize_aead_request - finalize one aead_request if
|
|
* the request is done
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be finalized
|
|
* @err: error number
|
|
*/
|
|
void crypto_finalize_aead_request(struct crypto_engine *engine,
|
|
struct aead_request *req, int err)
|
|
{
|
|
return crypto_finalize_request(engine, &req->base, err);
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);
|
|
|
|
/**
|
|
* crypto_finalize_akcipher_request - finalize one akcipher_request if
|
|
* the request is done
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be finalized
|
|
* @err: error number
|
|
*/
|
|
void crypto_finalize_akcipher_request(struct crypto_engine *engine,
|
|
struct akcipher_request *req, int err)
|
|
{
|
|
return crypto_finalize_request(engine, &req->base, err);
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);
|
|
|
|
/**
|
|
* crypto_finalize_hash_request - finalize one ahash_request if
|
|
* the request is done
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be finalized
|
|
* @err: error number
|
|
*/
|
|
void crypto_finalize_hash_request(struct crypto_engine *engine,
|
|
struct ahash_request *req, int err)
|
|
{
|
|
return crypto_finalize_request(engine, &req->base, err);
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
|
|
|
|
/**
|
|
* crypto_finalize_skcipher_request - finalize one skcipher_request if
|
|
* the request is done
|
|
* @engine: the hardware engine
|
|
* @req: the request need to be finalized
|
|
* @err: error number
|
|
*/
|
|
void crypto_finalize_skcipher_request(struct crypto_engine *engine,
|
|
struct skcipher_request *req, int err)
|
|
{
|
|
return crypto_finalize_request(engine, &req->base, err);
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);
|
|
|
|
/**
|
|
* crypto_engine_start - start the hardware engine
|
|
* @engine: the hardware engine need to be started
|
|
*
|
|
* Return 0 on success, else on fail.
|
|
*/
|
|
int crypto_engine_start(struct crypto_engine *engine)
|
|
{
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(&engine->queue_lock, flags);
|
|
|
|
if (engine->running || engine->busy) {
|
|
spin_unlock_irqrestore(&engine->queue_lock, flags);
|
|
return -EBUSY;
|
|
}
|
|
|
|
engine->running = true;
|
|
spin_unlock_irqrestore(&engine->queue_lock, flags);
|
|
|
|
kthread_queue_work(engine->kworker, &engine->pump_requests);
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_engine_start);
|
|
|
|
/**
|
|
* crypto_engine_stop - stop the hardware engine
|
|
* @engine: the hardware engine need to be stopped
|
|
*
|
|
* Return 0 on success, else on fail.
|
|
*/
|
|
int crypto_engine_stop(struct crypto_engine *engine)
|
|
{
|
|
unsigned long flags;
|
|
unsigned int limit = 500;
|
|
int ret = 0;
|
|
|
|
spin_lock_irqsave(&engine->queue_lock, flags);
|
|
|
|
/*
|
|
* If the engine queue is not empty or the engine is on busy state,
|
|
* we need to wait for a while to pump the requests of engine queue.
|
|
*/
|
|
while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
|
|
spin_unlock_irqrestore(&engine->queue_lock, flags);
|
|
msleep(20);
|
|
spin_lock_irqsave(&engine->queue_lock, flags);
|
|
}
|
|
|
|
if (crypto_queue_len(&engine->queue) || engine->busy)
|
|
ret = -EBUSY;
|
|
else
|
|
engine->running = false;
|
|
|
|
spin_unlock_irqrestore(&engine->queue_lock, flags);
|
|
|
|
if (ret)
|
|
dev_warn(engine->dev, "could not stop engine\n");
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_engine_stop);
|
|
|
|
/**
|
|
* crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
|
|
* and initialize it by setting the maximum number of entries in the software
|
|
* crypto-engine queue.
|
|
* @dev: the device attached with one hardware engine
|
|
* @retry_support: whether hardware has support for retry mechanism
|
|
* @cbk_do_batch: pointer to a callback function to be invoked when executing
|
|
* a batch of requests.
|
|
* This has the form:
|
|
* callback(struct crypto_engine *engine)
|
|
* where:
|
|
* @engine: the crypto engine structure.
|
|
* @rt: whether this queue is set to run as a realtime task
|
|
* @qlen: maximum size of the crypto-engine queue
|
|
*
|
|
* This must be called from context that can sleep.
|
|
* Return: the crypto engine structure on success, else NULL.
|
|
*/
|
|
struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
|
|
bool retry_support,
|
|
int (*cbk_do_batch)(struct crypto_engine *engine),
|
|
bool rt, int qlen)
|
|
{
|
|
struct crypto_engine *engine;
|
|
|
|
if (!dev)
|
|
return NULL;
|
|
|
|
engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
|
|
if (!engine)
|
|
return NULL;
|
|
|
|
engine->dev = dev;
|
|
engine->rt = rt;
|
|
engine->running = false;
|
|
engine->busy = false;
|
|
engine->idling = false;
|
|
engine->retry_support = retry_support;
|
|
engine->priv_data = dev;
|
|
/*
|
|
* Batch requests is possible only if
|
|
* hardware has support for retry mechanism.
|
|
*/
|
|
engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;
|
|
|
|
snprintf(engine->name, sizeof(engine->name),
|
|
"%s-engine", dev_name(dev));
|
|
|
|
crypto_init_queue(&engine->queue, qlen);
|
|
spin_lock_init(&engine->queue_lock);
|
|
|
|
engine->kworker = kthread_create_worker(0, "%s", engine->name);
|
|
if (IS_ERR(engine->kworker)) {
|
|
dev_err(dev, "failed to create crypto request pump task\n");
|
|
return NULL;
|
|
}
|
|
kthread_init_work(&engine->pump_requests, crypto_pump_work);
|
|
|
|
if (engine->rt) {
|
|
dev_info(dev, "will run requests pump with realtime priority\n");
|
|
sched_set_fifo(engine->kworker->task);
|
|
}
|
|
|
|
return engine;
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);
|
|
|
|
/**
|
|
* crypto_engine_alloc_init - allocate crypto hardware engine structure and
|
|
* initialize it.
|
|
* @dev: the device attached with one hardware engine
|
|
* @rt: whether this queue is set to run as a realtime task
|
|
*
|
|
* This must be called from context that can sleep.
|
|
* Return: the crypto engine structure on success, else NULL.
|
|
*/
|
|
struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
|
|
{
|
|
return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
|
|
CRYPTO_ENGINE_MAX_QLEN);
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
|
|
|
|
/**
|
|
* crypto_engine_exit - free the resources of hardware engine when exit
|
|
* @engine: the hardware engine need to be freed
|
|
*
|
|
* Return 0 for success.
|
|
*/
|
|
int crypto_engine_exit(struct crypto_engine *engine)
|
|
{
|
|
int ret;
|
|
|
|
ret = crypto_engine_stop(engine);
|
|
if (ret)
|
|
return ret;
|
|
|
|
kthread_destroy_worker(engine->kworker);
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(crypto_engine_exit);
|
|
|
|
MODULE_LICENSE("GPL");
|
|
MODULE_DESCRIPTION("Crypto hardware engine framework");
|