![Satya Tangirala](/assets/img/avatar_default.png)
Whenever a device supports blk-integrity, make the kernel pretend that the device doesn't support inline encryption (essentially by setting the keyslot manager in the request queue to NULL). There's no hardware currently that supports both integrity and inline encryption. However, it seems possible that there will be such hardware in the near future (like the NVMe key per I/O support that might support both inline encryption and PI). But properly integrating both features is not trivial, and without real hardware that implements both, it is difficult to tell if it will be done correctly by the majority of hardware that support both. So it seems best not to support both features together right now, and to decide what to do at probe time. Signed-off-by: Satya Tangirala <satyat@google.com> Reviewed-by: Eric Biggers <ebiggers@google.com> Reviewed-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Jens Axboe <axboe@kernel.dk>
398 lines
11 KiB
C
398 lines
11 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright 2019 Google LLC
|
|
*/
|
|
|
|
/**
|
|
* DOC: The Keyslot Manager
|
|
*
|
|
* Many devices with inline encryption support have a limited number of "slots"
|
|
* into which encryption contexts may be programmed, and requests can be tagged
|
|
* with a slot number to specify the key to use for en/decryption.
|
|
*
|
|
* As the number of slots is limited, and programming keys is expensive on
|
|
* many inline encryption hardware, we don't want to program the same key into
|
|
* multiple slots - if multiple requests are using the same key, we want to
|
|
* program just one slot with that key and use that slot for all requests.
|
|
*
|
|
* The keyslot manager manages these keyslots appropriately, and also acts as
|
|
* an abstraction between the inline encryption hardware and the upper layers.
|
|
*
|
|
* Lower layer devices will set up a keyslot manager in their request queue
|
|
* and tell it how to perform device specific operations like programming/
|
|
* evicting keys from keyslots.
|
|
*
|
|
* Upper layers will call blk_ksm_get_slot_for_key() to program a
|
|
* key into some slot in the inline encryption hardware.
|
|
*/
|
|
|
|
#define pr_fmt(fmt) "blk-crypto: " fmt
|
|
|
|
#include <linux/keyslot-manager.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/mutex.h>
|
|
#include <linux/pm_runtime.h>
|
|
#include <linux/wait.h>
|
|
#include <linux/blkdev.h>
|
|
|
|
struct blk_ksm_keyslot {
|
|
atomic_t slot_refs;
|
|
struct list_head idle_slot_node;
|
|
struct hlist_node hash_node;
|
|
const struct blk_crypto_key *key;
|
|
struct blk_keyslot_manager *ksm;
|
|
};
|
|
|
|
static inline void blk_ksm_hw_enter(struct blk_keyslot_manager *ksm)
|
|
{
|
|
/*
|
|
* Calling into the driver requires ksm->lock held and the device
|
|
* resumed. But we must resume the device first, since that can acquire
|
|
* and release ksm->lock via blk_ksm_reprogram_all_keys().
|
|
*/
|
|
if (ksm->dev)
|
|
pm_runtime_get_sync(ksm->dev);
|
|
down_write(&ksm->lock);
|
|
}
|
|
|
|
static inline void blk_ksm_hw_exit(struct blk_keyslot_manager *ksm)
|
|
{
|
|
up_write(&ksm->lock);
|
|
if (ksm->dev)
|
|
pm_runtime_put_sync(ksm->dev);
|
|
}
|
|
|
|
/**
|
|
* blk_ksm_init() - Initialize a keyslot manager
|
|
* @ksm: The keyslot_manager to initialize.
|
|
* @num_slots: The number of key slots to manage.
|
|
*
|
|
* Allocate memory for keyslots and initialize a keyslot manager. Called by
|
|
* e.g. storage drivers to set up a keyslot manager in their request_queue.
|
|
*
|
|
* Return: 0 on success, or else a negative error code.
|
|
*/
|
|
int blk_ksm_init(struct blk_keyslot_manager *ksm, unsigned int num_slots)
|
|
{
|
|
unsigned int slot;
|
|
unsigned int i;
|
|
unsigned int slot_hashtable_size;
|
|
|
|
memset(ksm, 0, sizeof(*ksm));
|
|
|
|
if (num_slots == 0)
|
|
return -EINVAL;
|
|
|
|
ksm->slots = kvcalloc(num_slots, sizeof(ksm->slots[0]), GFP_KERNEL);
|
|
if (!ksm->slots)
|
|
return -ENOMEM;
|
|
|
|
ksm->num_slots = num_slots;
|
|
|
|
init_rwsem(&ksm->lock);
|
|
|
|
init_waitqueue_head(&ksm->idle_slots_wait_queue);
|
|
INIT_LIST_HEAD(&ksm->idle_slots);
|
|
|
|
for (slot = 0; slot < num_slots; slot++) {
|
|
ksm->slots[slot].ksm = ksm;
|
|
list_add_tail(&ksm->slots[slot].idle_slot_node,
|
|
&ksm->idle_slots);
|
|
}
|
|
|
|
spin_lock_init(&ksm->idle_slots_lock);
|
|
|
|
slot_hashtable_size = roundup_pow_of_two(num_slots);
|
|
ksm->log_slot_ht_size = ilog2(slot_hashtable_size);
|
|
ksm->slot_hashtable = kvmalloc_array(slot_hashtable_size,
|
|
sizeof(ksm->slot_hashtable[0]),
|
|
GFP_KERNEL);
|
|
if (!ksm->slot_hashtable)
|
|
goto err_destroy_ksm;
|
|
for (i = 0; i < slot_hashtable_size; i++)
|
|
INIT_HLIST_HEAD(&ksm->slot_hashtable[i]);
|
|
|
|
return 0;
|
|
|
|
err_destroy_ksm:
|
|
blk_ksm_destroy(ksm);
|
|
return -ENOMEM;
|
|
}
|
|
EXPORT_SYMBOL_GPL(blk_ksm_init);
|
|
|
|
static inline struct hlist_head *
|
|
blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager *ksm,
|
|
const struct blk_crypto_key *key)
|
|
{
|
|
return &ksm->slot_hashtable[hash_ptr(key, ksm->log_slot_ht_size)];
|
|
}
|
|
|
|
static void blk_ksm_remove_slot_from_lru_list(struct blk_ksm_keyslot *slot)
|
|
{
|
|
struct blk_keyslot_manager *ksm = slot->ksm;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(&ksm->idle_slots_lock, flags);
|
|
list_del(&slot->idle_slot_node);
|
|
spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);
|
|
}
|
|
|
|
static struct blk_ksm_keyslot *blk_ksm_find_keyslot(
|
|
struct blk_keyslot_manager *ksm,
|
|
const struct blk_crypto_key *key)
|
|
{
|
|
const struct hlist_head *head = blk_ksm_hash_bucket_for_key(ksm, key);
|
|
struct blk_ksm_keyslot *slotp;
|
|
|
|
hlist_for_each_entry(slotp, head, hash_node) {
|
|
if (slotp->key == key)
|
|
return slotp;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
static struct blk_ksm_keyslot *blk_ksm_find_and_grab_keyslot(
|
|
struct blk_keyslot_manager *ksm,
|
|
const struct blk_crypto_key *key)
|
|
{
|
|
struct blk_ksm_keyslot *slot;
|
|
|
|
slot = blk_ksm_find_keyslot(ksm, key);
|
|
if (!slot)
|
|
return NULL;
|
|
if (atomic_inc_return(&slot->slot_refs) == 1) {
|
|
/* Took first reference to this slot; remove it from LRU list */
|
|
blk_ksm_remove_slot_from_lru_list(slot);
|
|
}
|
|
return slot;
|
|
}
|
|
|
|
unsigned int blk_ksm_get_slot_idx(struct blk_ksm_keyslot *slot)
|
|
{
|
|
return slot - slot->ksm->slots;
|
|
}
|
|
EXPORT_SYMBOL_GPL(blk_ksm_get_slot_idx);
|
|
|
|
/**
|
|
* blk_ksm_get_slot_for_key() - Program a key into a keyslot.
|
|
* @ksm: The keyslot manager to program the key into.
|
|
* @key: Pointer to the key object to program, including the raw key, crypto
|
|
* mode, and data unit size.
|
|
* @slot_ptr: A pointer to return the pointer of the allocated keyslot.
|
|
*
|
|
* Get a keyslot that's been programmed with the specified key. If one already
|
|
* exists, return it with incremented refcount. Otherwise, wait for a keyslot
|
|
* to become idle and program it.
|
|
*
|
|
* Context: Process context. Takes and releases ksm->lock.
|
|
* Return: BLK_STS_OK on success (and keyslot is set to the pointer of the
|
|
* allocated keyslot), or some other blk_status_t otherwise (and
|
|
* keyslot is set to NULL).
|
|
*/
|
|
blk_status_t blk_ksm_get_slot_for_key(struct blk_keyslot_manager *ksm,
|
|
const struct blk_crypto_key *key,
|
|
struct blk_ksm_keyslot **slot_ptr)
|
|
{
|
|
struct blk_ksm_keyslot *slot;
|
|
int slot_idx;
|
|
int err;
|
|
|
|
*slot_ptr = NULL;
|
|
down_read(&ksm->lock);
|
|
slot = blk_ksm_find_and_grab_keyslot(ksm, key);
|
|
up_read(&ksm->lock);
|
|
if (slot)
|
|
goto success;
|
|
|
|
for (;;) {
|
|
blk_ksm_hw_enter(ksm);
|
|
slot = blk_ksm_find_and_grab_keyslot(ksm, key);
|
|
if (slot) {
|
|
blk_ksm_hw_exit(ksm);
|
|
goto success;
|
|
}
|
|
|
|
/*
|
|
* If we're here, that means there wasn't a slot that was
|
|
* already programmed with the key. So try to program it.
|
|
*/
|
|
if (!list_empty(&ksm->idle_slots))
|
|
break;
|
|
|
|
blk_ksm_hw_exit(ksm);
|
|
wait_event(ksm->idle_slots_wait_queue,
|
|
!list_empty(&ksm->idle_slots));
|
|
}
|
|
|
|
slot = list_first_entry(&ksm->idle_slots, struct blk_ksm_keyslot,
|
|
idle_slot_node);
|
|
slot_idx = blk_ksm_get_slot_idx(slot);
|
|
|
|
err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot_idx);
|
|
if (err) {
|
|
wake_up(&ksm->idle_slots_wait_queue);
|
|
blk_ksm_hw_exit(ksm);
|
|
return errno_to_blk_status(err);
|
|
}
|
|
|
|
/* Move this slot to the hash list for the new key. */
|
|
if (slot->key)
|
|
hlist_del(&slot->hash_node);
|
|
slot->key = key;
|
|
hlist_add_head(&slot->hash_node, blk_ksm_hash_bucket_for_key(ksm, key));
|
|
|
|
atomic_set(&slot->slot_refs, 1);
|
|
|
|
blk_ksm_remove_slot_from_lru_list(slot);
|
|
|
|
blk_ksm_hw_exit(ksm);
|
|
success:
|
|
*slot_ptr = slot;
|
|
return BLK_STS_OK;
|
|
}
|
|
|
|
/**
|
|
* blk_ksm_put_slot() - Release a reference to a slot
|
|
* @slot: The keyslot to release the reference of.
|
|
*
|
|
* Context: Any context.
|
|
*/
|
|
void blk_ksm_put_slot(struct blk_ksm_keyslot *slot)
|
|
{
|
|
struct blk_keyslot_manager *ksm;
|
|
unsigned long flags;
|
|
|
|
if (!slot)
|
|
return;
|
|
|
|
ksm = slot->ksm;
|
|
|
|
if (atomic_dec_and_lock_irqsave(&slot->slot_refs,
|
|
&ksm->idle_slots_lock, flags)) {
|
|
list_add_tail(&slot->idle_slot_node, &ksm->idle_slots);
|
|
spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);
|
|
wake_up(&ksm->idle_slots_wait_queue);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* blk_ksm_crypto_cfg_supported() - Find out if a crypto configuration is
|
|
* supported by a ksm.
|
|
* @ksm: The keyslot manager to check
|
|
* @cfg: The crypto configuration to check for.
|
|
*
|
|
* Checks for crypto_mode/data unit size/dun bytes support.
|
|
*
|
|
* Return: Whether or not this ksm supports the specified crypto config.
|
|
*/
|
|
bool blk_ksm_crypto_cfg_supported(struct blk_keyslot_manager *ksm,
|
|
const struct blk_crypto_config *cfg)
|
|
{
|
|
if (!ksm)
|
|
return false;
|
|
if (!(ksm->crypto_modes_supported[cfg->crypto_mode] &
|
|
cfg->data_unit_size))
|
|
return false;
|
|
if (ksm->max_dun_bytes_supported < cfg->dun_bytes)
|
|
return false;
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* blk_ksm_evict_key() - Evict a key from the lower layer device.
|
|
* @ksm: The keyslot manager to evict from
|
|
* @key: The key to evict
|
|
*
|
|
* Find the keyslot that the specified key was programmed into, and evict that
|
|
* slot from the lower layer device. The slot must not be in use by any
|
|
* in-flight IO when this function is called.
|
|
*
|
|
* Context: Process context. Takes and releases ksm->lock.
|
|
* Return: 0 on success or if there's no keyslot with the specified key, -EBUSY
|
|
* if the keyslot is still in use, or another -errno value on other
|
|
* error.
|
|
*/
|
|
int blk_ksm_evict_key(struct blk_keyslot_manager *ksm,
|
|
const struct blk_crypto_key *key)
|
|
{
|
|
struct blk_ksm_keyslot *slot;
|
|
int err = 0;
|
|
|
|
blk_ksm_hw_enter(ksm);
|
|
slot = blk_ksm_find_keyslot(ksm, key);
|
|
if (!slot)
|
|
goto out_unlock;
|
|
|
|
if (WARN_ON_ONCE(atomic_read(&slot->slot_refs) != 0)) {
|
|
err = -EBUSY;
|
|
goto out_unlock;
|
|
}
|
|
err = ksm->ksm_ll_ops.keyslot_evict(ksm, key,
|
|
blk_ksm_get_slot_idx(slot));
|
|
if (err)
|
|
goto out_unlock;
|
|
|
|
hlist_del(&slot->hash_node);
|
|
slot->key = NULL;
|
|
err = 0;
|
|
out_unlock:
|
|
blk_ksm_hw_exit(ksm);
|
|
return err;
|
|
}
|
|
|
|
/**
|
|
* blk_ksm_reprogram_all_keys() - Re-program all keyslots.
|
|
* @ksm: The keyslot manager
|
|
*
|
|
* Re-program all keyslots that are supposed to have a key programmed. This is
|
|
* intended only for use by drivers for hardware that loses its keys on reset.
|
|
*
|
|
* Context: Process context. Takes and releases ksm->lock.
|
|
*/
|
|
void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager *ksm)
|
|
{
|
|
unsigned int slot;
|
|
|
|
/* This is for device initialization, so don't resume the device */
|
|
down_write(&ksm->lock);
|
|
for (slot = 0; slot < ksm->num_slots; slot++) {
|
|
const struct blk_crypto_key *key = ksm->slots[slot].key;
|
|
int err;
|
|
|
|
if (!key)
|
|
continue;
|
|
|
|
err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot);
|
|
WARN_ON(err);
|
|
}
|
|
up_write(&ksm->lock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(blk_ksm_reprogram_all_keys);
|
|
|
|
void blk_ksm_destroy(struct blk_keyslot_manager *ksm)
|
|
{
|
|
if (!ksm)
|
|
return;
|
|
kvfree(ksm->slot_hashtable);
|
|
memzero_explicit(ksm->slots, sizeof(ksm->slots[0]) * ksm->num_slots);
|
|
kvfree(ksm->slots);
|
|
memzero_explicit(ksm, sizeof(*ksm));
|
|
}
|
|
EXPORT_SYMBOL_GPL(blk_ksm_destroy);
|
|
|
|
bool blk_ksm_register(struct blk_keyslot_manager *ksm, struct request_queue *q)
|
|
{
|
|
if (blk_integrity_queue_supports_integrity(q)) {
|
|
pr_warn("Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n");
|
|
return false;
|
|
}
|
|
q->ksm = ksm;
|
|
return true;
|
|
}
|
|
EXPORT_SYMBOL_GPL(blk_ksm_register);
|
|
|
|
void blk_ksm_unregister(struct request_queue *q)
|
|
{
|
|
q->ksm = NULL;
|
|
}
|