![Gilad Ben-Yossef](/assets/img/avatar_default.png)
This sacrificial copyright header update is offered to the legal department as atonement for any changes made in this driver files in the course of the current year which have not been duly recorded as such. Signed-off-by: Gilad Ben-Yossef <gilad@benyossef.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
764 lines
21 KiB
C
764 lines
21 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/nospec.h>
|
|
#include "cc_driver.h"
|
|
#include "cc_buffer_mgr.h"
|
|
#include "cc_request_mgr.h"
|
|
#include "cc_ivgen.h"
|
|
#include "cc_pm.h"
|
|
|
|
#define CC_MAX_POLL_ITER 10
|
|
/* The highest descriptor count in used */
|
|
#define CC_MAX_DESC_SEQ_LEN 23
|
|
|
|
struct cc_req_mgr_handle {
|
|
/* Request manager resources */
|
|
unsigned int hw_queue_size; /* HW capability */
|
|
unsigned int min_free_hw_slots;
|
|
unsigned int max_used_sw_slots;
|
|
struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE];
|
|
u32 req_queue_head;
|
|
u32 req_queue_tail;
|
|
u32 axi_completed;
|
|
u32 q_free_slots;
|
|
/* This lock protects access to HW register
|
|
* that must be single request at a time
|
|
*/
|
|
spinlock_t hw_lock;
|
|
struct cc_hw_desc compl_desc;
|
|
u8 *dummy_comp_buff;
|
|
dma_addr_t dummy_comp_buff_dma;
|
|
|
|
/* backlog queue */
|
|
struct list_head backlog;
|
|
unsigned int bl_len;
|
|
spinlock_t bl_lock; /* protect backlog queue */
|
|
|
|
#ifdef COMP_IN_WQ
|
|
struct workqueue_struct *workq;
|
|
struct delayed_work compwork;
|
|
#else
|
|
struct tasklet_struct comptask;
|
|
#endif
|
|
bool is_runtime_suspended;
|
|
};
|
|
|
|
struct cc_bl_item {
|
|
struct cc_crypto_req creq;
|
|
struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN];
|
|
unsigned int len;
|
|
struct list_head list;
|
|
bool notif;
|
|
};
|
|
|
|
static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = {
|
|
{ BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) },
|
|
{ BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT),
|
|
BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) }
|
|
};
|
|
|
|
static void comp_handler(unsigned long devarg);
|
|
#ifdef COMP_IN_WQ
|
|
static void comp_work_handler(struct work_struct *work);
|
|
#endif
|
|
|
|
static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot)
|
|
{
|
|
alg = array_index_nospec(alg, CC_CPP_NUM_ALGS);
|
|
slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS);
|
|
|
|
return cc_cpp_int_masks[alg][slot];
|
|
}
|
|
|
|
void cc_req_mgr_fini(struct cc_drvdata *drvdata)
|
|
{
|
|
struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
|
|
if (!req_mgr_h)
|
|
return; /* Not allocated */
|
|
|
|
if (req_mgr_h->dummy_comp_buff_dma) {
|
|
dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff,
|
|
req_mgr_h->dummy_comp_buff_dma);
|
|
}
|
|
|
|
dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size -
|
|
req_mgr_h->min_free_hw_slots));
|
|
dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
|
|
|
|
#ifdef COMP_IN_WQ
|
|
flush_workqueue(req_mgr_h->workq);
|
|
destroy_workqueue(req_mgr_h->workq);
|
|
#else
|
|
/* Kill tasklet */
|
|
tasklet_kill(&req_mgr_h->comptask);
|
|
#endif
|
|
kzfree(req_mgr_h);
|
|
drvdata->request_mgr_handle = NULL;
|
|
}
|
|
|
|
int cc_req_mgr_init(struct cc_drvdata *drvdata)
|
|
{
|
|
struct cc_req_mgr_handle *req_mgr_h;
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
int rc = 0;
|
|
|
|
req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL);
|
|
if (!req_mgr_h) {
|
|
rc = -ENOMEM;
|
|
goto req_mgr_init_err;
|
|
}
|
|
|
|
drvdata->request_mgr_handle = req_mgr_h;
|
|
|
|
spin_lock_init(&req_mgr_h->hw_lock);
|
|
spin_lock_init(&req_mgr_h->bl_lock);
|
|
INIT_LIST_HEAD(&req_mgr_h->backlog);
|
|
|
|
#ifdef COMP_IN_WQ
|
|
dev_dbg(dev, "Initializing completion workqueue\n");
|
|
req_mgr_h->workq = create_singlethread_workqueue("ccree");
|
|
if (!req_mgr_h->workq) {
|
|
dev_err(dev, "Failed creating work queue\n");
|
|
rc = -ENOMEM;
|
|
goto req_mgr_init_err;
|
|
}
|
|
INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler);
|
|
#else
|
|
dev_dbg(dev, "Initializing completion tasklet\n");
|
|
tasklet_init(&req_mgr_h->comptask, comp_handler,
|
|
(unsigned long)drvdata);
|
|
#endif
|
|
req_mgr_h->hw_queue_size = cc_ioread(drvdata,
|
|
CC_REG(DSCRPTR_QUEUE_SRAM_SIZE));
|
|
dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);
|
|
if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {
|
|
dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n",
|
|
req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
|
|
rc = -ENOMEM;
|
|
goto req_mgr_init_err;
|
|
}
|
|
req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size;
|
|
req_mgr_h->max_used_sw_slots = 0;
|
|
|
|
/* Allocate DMA word for "dummy" completion descriptor use */
|
|
req_mgr_h->dummy_comp_buff =
|
|
dma_alloc_coherent(dev, sizeof(u32),
|
|
&req_mgr_h->dummy_comp_buff_dma,
|
|
GFP_KERNEL);
|
|
if (!req_mgr_h->dummy_comp_buff) {
|
|
dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n",
|
|
sizeof(u32));
|
|
rc = -ENOMEM;
|
|
goto req_mgr_init_err;
|
|
}
|
|
|
|
/* Init. "dummy" completion descriptor */
|
|
hw_desc_init(&req_mgr_h->compl_desc);
|
|
set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32));
|
|
set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma,
|
|
sizeof(u32), NS_BIT, 1);
|
|
set_flow_mode(&req_mgr_h->compl_desc, BYPASS);
|
|
set_queue_last_ind(drvdata, &req_mgr_h->compl_desc);
|
|
|
|
return 0;
|
|
|
|
req_mgr_init_err:
|
|
cc_req_mgr_fini(drvdata);
|
|
return rc;
|
|
}
|
|
|
|
static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[],
|
|
unsigned int seq_len)
|
|
{
|
|
int i, w;
|
|
void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0);
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
|
|
/*
|
|
* We do indeed write all 6 command words to the same
|
|
* register. The HW supports this.
|
|
*/
|
|
|
|
for (i = 0; i < seq_len; i++) {
|
|
for (w = 0; w <= 5; w++)
|
|
writel_relaxed(seq[i].word[w], reg);
|
|
|
|
if (cc_dump_desc)
|
|
dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
|
|
i, seq[i].word[0], seq[i].word[1],
|
|
seq[i].word[2], seq[i].word[3],
|
|
seq[i].word[4], seq[i].word[5]);
|
|
}
|
|
}
|
|
|
|
/*!
|
|
* Completion will take place if and only if user requested completion
|
|
* by cc_send_sync_request().
|
|
*
|
|
* \param dev
|
|
* \param dx_compl_h The completion event to signal
|
|
*/
|
|
static void request_mgr_complete(struct device *dev, void *dx_compl_h,
|
|
int dummy)
|
|
{
|
|
struct completion *this_compl = dx_compl_h;
|
|
|
|
complete(this_compl);
|
|
}
|
|
|
|
static int cc_queues_status(struct cc_drvdata *drvdata,
|
|
struct cc_req_mgr_handle *req_mgr_h,
|
|
unsigned int total_seq_len)
|
|
{
|
|
unsigned long poll_queue;
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
|
|
/* SW queue is checked only once as it will not
|
|
* be chaned during the poll because the spinlock_bh
|
|
* is held by the thread
|
|
*/
|
|
if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) ==
|
|
req_mgr_h->req_queue_tail) {
|
|
dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",
|
|
req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
|
|
return -ENOSPC;
|
|
}
|
|
|
|
if (req_mgr_h->q_free_slots >= total_seq_len)
|
|
return 0;
|
|
|
|
/* Wait for space in HW queue. Poll constant num of iterations. */
|
|
for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) {
|
|
req_mgr_h->q_free_slots =
|
|
cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
|
|
if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots)
|
|
req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;
|
|
|
|
if (req_mgr_h->q_free_slots >= total_seq_len) {
|
|
/* If there is enough place return */
|
|
return 0;
|
|
}
|
|
|
|
dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",
|
|
req_mgr_h->q_free_slots, total_seq_len);
|
|
}
|
|
/* No room in the HW queue try again later */
|
|
dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n",
|
|
req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE,
|
|
req_mgr_h->q_free_slots, total_seq_len);
|
|
return -ENOSPC;
|
|
}
|
|
|
|
/*!
|
|
* Enqueue caller request to crypto hardware.
|
|
* Need to be called with HW lock held and PM running
|
|
*
|
|
* \param drvdata
|
|
* \param cc_req The request to enqueue
|
|
* \param desc The crypto sequence
|
|
* \param len The crypto sequence length
|
|
* \param add_comp If "true": add an artificial dout DMA to mark completion
|
|
*
|
|
* \return int Returns -EINPROGRESS or error code
|
|
*/
|
|
static int cc_do_send_request(struct cc_drvdata *drvdata,
|
|
struct cc_crypto_req *cc_req,
|
|
struct cc_hw_desc *desc, unsigned int len,
|
|
bool add_comp, bool ivgen)
|
|
{
|
|
struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
|
|
unsigned int used_sw_slots;
|
|
unsigned int iv_seq_len = 0;
|
|
unsigned int total_seq_len = len; /*initial sequence length*/
|
|
struct cc_hw_desc iv_seq[CC_IVPOOL_SEQ_LEN];
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
int rc;
|
|
|
|
if (ivgen) {
|
|
dev_dbg(dev, "Acquire IV from pool into %d DMA addresses %pad, %pad, %pad, IV-size=%u\n",
|
|
cc_req->ivgen_dma_addr_len,
|
|
&cc_req->ivgen_dma_addr[0],
|
|
&cc_req->ivgen_dma_addr[1],
|
|
&cc_req->ivgen_dma_addr[2],
|
|
cc_req->ivgen_size);
|
|
|
|
/* Acquire IV from pool */
|
|
rc = cc_get_iv(drvdata, cc_req->ivgen_dma_addr,
|
|
cc_req->ivgen_dma_addr_len,
|
|
cc_req->ivgen_size, iv_seq, &iv_seq_len);
|
|
|
|
if (rc) {
|
|
dev_err(dev, "Failed to generate IV (rc=%d)\n", rc);
|
|
return rc;
|
|
}
|
|
|
|
total_seq_len += iv_seq_len;
|
|
}
|
|
|
|
used_sw_slots = ((req_mgr_h->req_queue_head -
|
|
req_mgr_h->req_queue_tail) &
|
|
(MAX_REQUEST_QUEUE_SIZE - 1));
|
|
if (used_sw_slots > req_mgr_h->max_used_sw_slots)
|
|
req_mgr_h->max_used_sw_slots = used_sw_slots;
|
|
|
|
/* Enqueue request - must be locked with HW lock*/
|
|
req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req;
|
|
req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) &
|
|
(MAX_REQUEST_QUEUE_SIZE - 1);
|
|
/* TODO: Use circ_buf.h ? */
|
|
|
|
dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head);
|
|
|
|
/*
|
|
* We are about to push command to the HW via the command registers
|
|
* that may refernece hsot memory. We need to issue a memory barrier
|
|
* to make sure there are no outstnading memory writes
|
|
*/
|
|
wmb();
|
|
|
|
/* STAT_PHASE_4: Push sequence */
|
|
if (ivgen)
|
|
enqueue_seq(drvdata, iv_seq, iv_seq_len);
|
|
|
|
enqueue_seq(drvdata, desc, len);
|
|
|
|
if (add_comp) {
|
|
enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1);
|
|
total_seq_len++;
|
|
}
|
|
|
|
if (req_mgr_h->q_free_slots < total_seq_len) {
|
|
/* This situation should never occur. Maybe indicating problem
|
|
* with resuming power. Set the free slot count to 0 and hope
|
|
* for the best.
|
|
*/
|
|
dev_err(dev, "HW free slot count mismatch.");
|
|
req_mgr_h->q_free_slots = 0;
|
|
} else {
|
|
/* Update the free slots in HW queue */
|
|
req_mgr_h->q_free_slots -= total_seq_len;
|
|
}
|
|
|
|
/* Operation still in process */
|
|
return -EINPROGRESS;
|
|
}
|
|
|
|
static void cc_enqueue_backlog(struct cc_drvdata *drvdata,
|
|
struct cc_bl_item *bli)
|
|
{
|
|
struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
|
|
spin_lock_bh(&mgr->bl_lock);
|
|
list_add_tail(&bli->list, &mgr->backlog);
|
|
++mgr->bl_len;
|
|
dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len);
|
|
spin_unlock_bh(&mgr->bl_lock);
|
|
tasklet_schedule(&mgr->comptask);
|
|
}
|
|
|
|
static void cc_proc_backlog(struct cc_drvdata *drvdata)
|
|
{
|
|
struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
|
|
struct cc_bl_item *bli;
|
|
struct cc_crypto_req *creq;
|
|
void *req;
|
|
bool ivgen;
|
|
unsigned int total_len;
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
int rc;
|
|
|
|
spin_lock(&mgr->bl_lock);
|
|
|
|
while (mgr->bl_len) {
|
|
bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list);
|
|
dev_dbg(dev, "---bl len: %d\n", mgr->bl_len);
|
|
|
|
spin_unlock(&mgr->bl_lock);
|
|
|
|
|
|
creq = &bli->creq;
|
|
req = creq->user_arg;
|
|
|
|
/*
|
|
* Notify the request we're moving out of the backlog
|
|
* but only if we haven't done so already.
|
|
*/
|
|
if (!bli->notif) {
|
|
creq->user_cb(dev, req, -EINPROGRESS);
|
|
bli->notif = true;
|
|
}
|
|
|
|
ivgen = !!creq->ivgen_dma_addr_len;
|
|
total_len = bli->len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0);
|
|
|
|
spin_lock(&mgr->hw_lock);
|
|
|
|
rc = cc_queues_status(drvdata, mgr, total_len);
|
|
if (rc) {
|
|
/*
|
|
* There is still not room in the FIFO for
|
|
* this request. Bail out. We'll return here
|
|
* on the next completion irq.
|
|
*/
|
|
spin_unlock(&mgr->hw_lock);
|
|
return;
|
|
}
|
|
|
|
rc = cc_do_send_request(drvdata, &bli->creq, bli->desc,
|
|
bli->len, false, ivgen);
|
|
|
|
spin_unlock(&mgr->hw_lock);
|
|
|
|
if (rc != -EINPROGRESS) {
|
|
cc_pm_put_suspend(dev);
|
|
creq->user_cb(dev, req, rc);
|
|
}
|
|
|
|
/* Remove ourselves from the backlog list */
|
|
spin_lock(&mgr->bl_lock);
|
|
list_del(&bli->list);
|
|
--mgr->bl_len;
|
|
}
|
|
|
|
spin_unlock(&mgr->bl_lock);
|
|
}
|
|
|
|
int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
|
|
struct cc_hw_desc *desc, unsigned int len,
|
|
struct crypto_async_request *req)
|
|
{
|
|
int rc;
|
|
struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
|
|
bool ivgen = !!cc_req->ivgen_dma_addr_len;
|
|
unsigned int total_len = len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0);
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
|
|
gfp_t flags = cc_gfp_flags(req);
|
|
struct cc_bl_item *bli;
|
|
|
|
rc = cc_pm_get(dev);
|
|
if (rc) {
|
|
dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc);
|
|
return rc;
|
|
}
|
|
|
|
spin_lock_bh(&mgr->hw_lock);
|
|
rc = cc_queues_status(drvdata, mgr, total_len);
|
|
|
|
#ifdef CC_DEBUG_FORCE_BACKLOG
|
|
if (backlog_ok)
|
|
rc = -ENOSPC;
|
|
#endif /* CC_DEBUG_FORCE_BACKLOG */
|
|
|
|
if (rc == -ENOSPC && backlog_ok) {
|
|
spin_unlock_bh(&mgr->hw_lock);
|
|
|
|
bli = kmalloc(sizeof(*bli), flags);
|
|
if (!bli) {
|
|
cc_pm_put_suspend(dev);
|
|
return -ENOMEM;
|
|
}
|
|
|
|
memcpy(&bli->creq, cc_req, sizeof(*cc_req));
|
|
memcpy(&bli->desc, desc, len * sizeof(*desc));
|
|
bli->len = len;
|
|
bli->notif = false;
|
|
cc_enqueue_backlog(drvdata, bli);
|
|
return -EBUSY;
|
|
}
|
|
|
|
if (!rc)
|
|
rc = cc_do_send_request(drvdata, cc_req, desc, len, false,
|
|
ivgen);
|
|
|
|
spin_unlock_bh(&mgr->hw_lock);
|
|
return rc;
|
|
}
|
|
|
|
int cc_send_sync_request(struct cc_drvdata *drvdata,
|
|
struct cc_crypto_req *cc_req, struct cc_hw_desc *desc,
|
|
unsigned int len)
|
|
{
|
|
int rc;
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
|
|
|
|
init_completion(&cc_req->seq_compl);
|
|
cc_req->user_cb = request_mgr_complete;
|
|
cc_req->user_arg = &cc_req->seq_compl;
|
|
|
|
rc = cc_pm_get(dev);
|
|
if (rc) {
|
|
dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc);
|
|
return rc;
|
|
}
|
|
|
|
while (true) {
|
|
spin_lock_bh(&mgr->hw_lock);
|
|
rc = cc_queues_status(drvdata, mgr, len + 1);
|
|
|
|
if (!rc)
|
|
break;
|
|
|
|
spin_unlock_bh(&mgr->hw_lock);
|
|
if (rc != -EAGAIN) {
|
|
cc_pm_put_suspend(dev);
|
|
return rc;
|
|
}
|
|
wait_for_completion_interruptible(&drvdata->hw_queue_avail);
|
|
reinit_completion(&drvdata->hw_queue_avail);
|
|
}
|
|
|
|
rc = cc_do_send_request(drvdata, cc_req, desc, len, true, false);
|
|
spin_unlock_bh(&mgr->hw_lock);
|
|
|
|
if (rc != -EINPROGRESS) {
|
|
cc_pm_put_suspend(dev);
|
|
return rc;
|
|
}
|
|
|
|
wait_for_completion(&cc_req->seq_compl);
|
|
return 0;
|
|
}
|
|
|
|
/*!
|
|
* Enqueue caller request to crypto hardware during init process.
|
|
* assume this function is not called in middle of a flow,
|
|
* since we set QUEUE_LAST_IND flag in the last descriptor.
|
|
*
|
|
* \param drvdata
|
|
* \param desc The crypto sequence
|
|
* \param len The crypto sequence length
|
|
*
|
|
* \return int Returns "0" upon success
|
|
*/
|
|
int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc,
|
|
unsigned int len)
|
|
{
|
|
struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
|
|
unsigned int total_seq_len = len; /*initial sequence length*/
|
|
int rc = 0;
|
|
|
|
/* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT.
|
|
*/
|
|
rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len);
|
|
if (rc)
|
|
return rc;
|
|
|
|
set_queue_last_ind(drvdata, &desc[(len - 1)]);
|
|
|
|
/*
|
|
* We are about to push command to the HW via the command registers
|
|
* that may refernece hsot memory. We need to issue a memory barrier
|
|
* to make sure there are no outstnading memory writes
|
|
*/
|
|
wmb();
|
|
enqueue_seq(drvdata, desc, len);
|
|
|
|
/* Update the free slots in HW queue */
|
|
req_mgr_h->q_free_slots =
|
|
cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
|
|
|
|
return 0;
|
|
}
|
|
|
|
void complete_request(struct cc_drvdata *drvdata)
|
|
{
|
|
struct cc_req_mgr_handle *request_mgr_handle =
|
|
drvdata->request_mgr_handle;
|
|
|
|
complete(&drvdata->hw_queue_avail);
|
|
#ifdef COMP_IN_WQ
|
|
queue_delayed_work(request_mgr_handle->workq,
|
|
&request_mgr_handle->compwork, 0);
|
|
#else
|
|
tasklet_schedule(&request_mgr_handle->comptask);
|
|
#endif
|
|
}
|
|
|
|
#ifdef COMP_IN_WQ
|
|
static void comp_work_handler(struct work_struct *work)
|
|
{
|
|
struct cc_drvdata *drvdata =
|
|
container_of(work, struct cc_drvdata, compwork.work);
|
|
|
|
comp_handler((unsigned long)drvdata);
|
|
}
|
|
#endif
|
|
|
|
static void proc_completions(struct cc_drvdata *drvdata)
|
|
{
|
|
struct cc_crypto_req *cc_req;
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
struct cc_req_mgr_handle *request_mgr_handle =
|
|
drvdata->request_mgr_handle;
|
|
unsigned int *tail = &request_mgr_handle->req_queue_tail;
|
|
unsigned int *head = &request_mgr_handle->req_queue_head;
|
|
int rc;
|
|
u32 mask;
|
|
|
|
while (request_mgr_handle->axi_completed) {
|
|
request_mgr_handle->axi_completed--;
|
|
|
|
/* Dequeue request */
|
|
if (*head == *tail) {
|
|
/* We are supposed to handle a completion but our
|
|
* queue is empty. This is not normal. Return and
|
|
* hope for the best.
|
|
*/
|
|
dev_err(dev, "Request queue is empty head == tail %u\n",
|
|
*head);
|
|
break;
|
|
}
|
|
|
|
cc_req = &request_mgr_handle->req_queue[*tail];
|
|
|
|
if (cc_req->cpp.is_cpp) {
|
|
|
|
dev_dbg(dev, "CPP request completion slot: %d alg:%d\n",
|
|
cc_req->cpp.slot, cc_req->cpp.alg);
|
|
mask = cc_cpp_int_mask(cc_req->cpp.alg,
|
|
cc_req->cpp.slot);
|
|
rc = (drvdata->irq & mask ? -EPERM : 0);
|
|
dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask,
|
|
drvdata->irq, rc);
|
|
} else {
|
|
dev_dbg(dev, "None CPP request completion\n");
|
|
rc = 0;
|
|
}
|
|
|
|
if (cc_req->user_cb)
|
|
cc_req->user_cb(dev, cc_req->user_arg, rc);
|
|
*tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);
|
|
dev_dbg(dev, "Dequeue request tail=%u\n", *tail);
|
|
dev_dbg(dev, "Request completed. axi_completed=%d\n",
|
|
request_mgr_handle->axi_completed);
|
|
cc_pm_put_suspend(dev);
|
|
}
|
|
}
|
|
|
|
static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata)
|
|
{
|
|
return FIELD_GET(AXIM_MON_COMP_VALUE,
|
|
cc_ioread(drvdata, drvdata->axim_mon_offset));
|
|
}
|
|
|
|
/* Deferred service handler, run as interrupt-fired tasklet */
|
|
static void comp_handler(unsigned long devarg)
|
|
{
|
|
struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg;
|
|
struct cc_req_mgr_handle *request_mgr_handle =
|
|
drvdata->request_mgr_handle;
|
|
struct device *dev = drvdata_to_dev(drvdata);
|
|
u32 irq;
|
|
|
|
dev_dbg(dev, "Completion handler called!\n");
|
|
irq = (drvdata->irq & drvdata->comp_mask);
|
|
|
|
/* To avoid the interrupt from firing as we unmask it,
|
|
* we clear it now
|
|
*/
|
|
cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
|
|
|
|
/* Avoid race with above clear: Test completion counter once more */
|
|
|
|
request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
|
|
|
|
dev_dbg(dev, "AXI completion after updated: %d\n",
|
|
request_mgr_handle->axi_completed);
|
|
|
|
while (request_mgr_handle->axi_completed) {
|
|
do {
|
|
drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR));
|
|
irq = (drvdata->irq & drvdata->comp_mask);
|
|
proc_completions(drvdata);
|
|
|
|
/* At this point (after proc_completions()),
|
|
* request_mgr_handle->axi_completed is 0.
|
|
*/
|
|
request_mgr_handle->axi_completed +=
|
|
cc_axi_comp_count(drvdata);
|
|
} while (request_mgr_handle->axi_completed > 0);
|
|
|
|
cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
|
|
|
|
request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
|
|
}
|
|
|
|
/* after verifing that there is nothing to do,
|
|
* unmask AXI completion interrupt
|
|
*/
|
|
cc_iowrite(drvdata, CC_REG(HOST_IMR),
|
|
cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask);
|
|
|
|
cc_proc_backlog(drvdata);
|
|
dev_dbg(dev, "Comp. handler done.\n");
|
|
}
|
|
|
|
/*
|
|
* resume the queue configuration - no need to take the lock as this happens
|
|
* inside the spin lock protection
|
|
*/
|
|
#if defined(CONFIG_PM)
|
|
int cc_resume_req_queue(struct cc_drvdata *drvdata)
|
|
{
|
|
struct cc_req_mgr_handle *request_mgr_handle =
|
|
drvdata->request_mgr_handle;
|
|
|
|
spin_lock_bh(&request_mgr_handle->hw_lock);
|
|
request_mgr_handle->is_runtime_suspended = false;
|
|
spin_unlock_bh(&request_mgr_handle->hw_lock);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* suspend the queue configuration. Since it is used for the runtime suspend
|
|
* only verify that the queue can be suspended.
|
|
*/
|
|
int cc_suspend_req_queue(struct cc_drvdata *drvdata)
|
|
{
|
|
struct cc_req_mgr_handle *request_mgr_handle =
|
|
drvdata->request_mgr_handle;
|
|
|
|
/* lock the send_request */
|
|
spin_lock_bh(&request_mgr_handle->hw_lock);
|
|
if (request_mgr_handle->req_queue_head !=
|
|
request_mgr_handle->req_queue_tail) {
|
|
spin_unlock_bh(&request_mgr_handle->hw_lock);
|
|
return -EBUSY;
|
|
}
|
|
request_mgr_handle->is_runtime_suspended = true;
|
|
spin_unlock_bh(&request_mgr_handle->hw_lock);
|
|
|
|
return 0;
|
|
}
|
|
|
|
bool cc_req_queue_suspended(struct cc_drvdata *drvdata)
|
|
{
|
|
struct cc_req_mgr_handle *request_mgr_handle =
|
|
drvdata->request_mgr_handle;
|
|
|
|
return request_mgr_handle->is_runtime_suspended;
|
|
}
|
|
|
|
#endif
|