diff options
Diffstat (limited to 'crypto/crypto_engine.c')
| -rw-r--r-- | crypto/crypto_engine.c | 745 |
1 files changed, 745 insertions, 0 deletions
diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c new file mode 100644 index 0000000000..108d9d55c5 --- /dev/null +++ b/crypto/crypto_engine.c @@ -0,0 +1,745 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Handle async block request by crypto hardware engine. + * + * Copyright (C) 2016 Linaro, Inc. + * + * Author: Baolin Wang <baolin.wang@linaro.org> + */ + +#include <crypto/internal/aead.h> +#include <crypto/internal/akcipher.h> +#include <crypto/internal/engine.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/kpp.h> +#include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <uapi/linux/sched/types.h> +#include "internal.h" + +#define CRYPTO_ENGINE_MAX_QLEN 10 + +/* Temporary algorithm flag used to indicate an updated driver. */ +#define CRYPTO_ALG_ENGINE 0x200 + +struct crypto_engine_alg { + struct crypto_alg base; + struct crypto_engine_op op; +}; + +/** + * crypto_finalize_request - finalize one request if the request is done + * @engine: the hardware engine + * @req: the request need to be finalized + * @err: error number + */ +static void crypto_finalize_request(struct crypto_engine *engine, + struct crypto_async_request *req, int err) +{ + unsigned long flags; + + /* + * If hardware cannot enqueue more requests + * and retry mechanism is not supported + * make sure we are completing the current request + */ + if (!engine->retry_support) { + spin_lock_irqsave(&engine->queue_lock, flags); + if (engine->cur_req == req) { + engine->cur_req = NULL; + } + spin_unlock_irqrestore(&engine->queue_lock, flags); + } + + lockdep_assert_in_softirq(); + crypto_request_complete(req, err); + + kthread_queue_work(engine->kworker, &engine->pump_requests); +} + +/** + * crypto_pump_requests - dequeue one request from engine queue to process + * @engine: the hardware engine + * @in_kthread: true if we are in the context of the request pump thread + * + * This function checks if there is any request in the engine queue that + * needs processing and if so call out to the driver to initialize hardware + * and handle each request. + */ +static void crypto_pump_requests(struct crypto_engine *engine, + bool in_kthread) +{ + struct crypto_async_request *async_req, *backlog; + struct crypto_engine_alg *alg; + struct crypto_engine_op *op; + unsigned long flags; + bool was_busy = false; + int ret; + + spin_lock_irqsave(&engine->queue_lock, flags); + + /* Make sure we are not already running a request */ + if (!engine->retry_support && engine->cur_req) + goto out; + + /* If another context is idling then defer */ + if (engine->idling) { + kthread_queue_work(engine->kworker, &engine->pump_requests); + goto out; + } + + /* Check if the engine queue is idle */ + if (!crypto_queue_len(&engine->queue) || !engine->running) { + if (!engine->busy) + goto out; + + /* Only do teardown in the thread */ + if (!in_kthread) { + kthread_queue_work(engine->kworker, + &engine->pump_requests); + goto out; + } + + engine->busy = false; + engine->idling = true; + spin_unlock_irqrestore(&engine->queue_lock, flags); + + if (engine->unprepare_crypt_hardware && + engine->unprepare_crypt_hardware(engine)) + dev_err(engine->dev, "failed to unprepare crypt hardware\n"); + + spin_lock_irqsave(&engine->queue_lock, flags); + engine->idling = false; + goto out; + } + +start_request: + /* Get the fist request from the engine queue to handle */ + backlog = crypto_get_backlog(&engine->queue); + async_req = crypto_dequeue_request(&engine->queue); + if (!async_req) + goto out; + + /* + * If hardware doesn't support the retry mechanism, + * keep track of the request we are processing now. + * We'll need it on completion (crypto_finalize_request). + */ + if (!engine->retry_support) + engine->cur_req = async_req; + + if (engine->busy) + was_busy = true; + else + engine->busy = true; + + spin_unlock_irqrestore(&engine->queue_lock, flags); + + /* Until here we get the request need to be encrypted successfully */ + if (!was_busy && engine->prepare_crypt_hardware) { + ret = engine->prepare_crypt_hardware(engine); + if (ret) { + dev_err(engine->dev, "failed to prepare crypt hardware\n"); + goto req_err_1; + } + } + + if (async_req->tfm->__crt_alg->cra_flags & CRYPTO_ALG_ENGINE) { + alg = container_of(async_req->tfm->__crt_alg, + struct crypto_engine_alg, base); + op = &alg->op; + } else { + dev_err(engine->dev, "failed to do request\n"); + ret = -EINVAL; + goto req_err_1; + } + + ret = op->do_one_request(engine, async_req); + + /* Request unsuccessfully executed by hardware */ + if (ret < 0) { + /* + * If hardware queue is full (-ENOSPC), requeue request + * regardless of backlog flag. + * Otherwise, unprepare and complete the request. + */ + if (!engine->retry_support || + (ret != -ENOSPC)) { + dev_err(engine->dev, + "Failed to do one request from queue: %d\n", + ret); + goto req_err_1; + } + spin_lock_irqsave(&engine->queue_lock, flags); + /* + * If hardware was unable to execute request, enqueue it + * back in front of crypto-engine queue, to keep the order + * of requests. + */ + crypto_enqueue_request_head(&engine->queue, async_req); + + kthread_queue_work(engine->kworker, &engine->pump_requests); + goto out; + } + + goto retry; + +req_err_1: + crypto_request_complete(async_req, ret); + +retry: + if (backlog) + crypto_request_complete(backlog, -EINPROGRESS); + + /* If retry mechanism is supported, send new requests to engine */ + if (engine->retry_support) { + spin_lock_irqsave(&engine->queue_lock, flags); + goto start_request; + } + return; + +out: + spin_unlock_irqrestore(&engine->queue_lock, flags); + + /* + * Batch requests is possible only if + * hardware can enqueue multiple requests + */ + if (engine->do_batch_requests) { + ret = engine->do_batch_requests(engine); + if (ret) + dev_err(engine->dev, "failed to do batch requests: %d\n", + ret); + } + + return; +} + +static void crypto_pump_work(struct kthread_work *work) +{ + struct crypto_engine *engine = + container_of(work, struct crypto_engine, pump_requests); + + crypto_pump_requests(engine, true); +} + +/** + * crypto_transfer_request - transfer the new request into the engine queue + * @engine: the hardware engine + * @req: the request need to be listed into the engine queue + * @need_pump: indicates whether queue the pump of request to kthread_work + */ +static int crypto_transfer_request(struct crypto_engine *engine, + struct crypto_async_request *req, + bool need_pump) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&engine->queue_lock, flags); + + if (!engine->running) { + spin_unlock_irqrestore(&engine->queue_lock, flags); + return -ESHUTDOWN; + } + + ret = crypto_enqueue_request(&engine->queue, req); + + if (!engine->busy && need_pump) + kthread_queue_work(engine->kworker, &engine->pump_requests); + + spin_unlock_irqrestore(&engine->queue_lock, flags); + return ret; +} + +/** + * crypto_transfer_request_to_engine - transfer one request to list + * into the engine queue + * @engine: the hardware engine + * @req: the request need to be listed into the engine queue + */ +static int crypto_transfer_request_to_engine(struct crypto_engine *engine, + struct crypto_async_request *req) +{ + return crypto_transfer_request(engine, req, true); +} + +/** + * crypto_transfer_aead_request_to_engine - transfer one aead_request + * to list into the engine queue + * @engine: the hardware engine + * @req: the request need to be listed into the engine queue + */ +int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine, + struct aead_request *req) +{ + return crypto_transfer_request_to_engine(engine, &req->base); +} +EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine); + +/** + * crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request + * to list into the engine queue + * @engine: the hardware engine + * @req: the request need to be listed into the engine queue + */ +int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine, + struct akcipher_request *req) +{ + return crypto_transfer_request_to_engine(engine, &req->base); +} +EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine); + +/** + * crypto_transfer_hash_request_to_engine - transfer one ahash_request + * to list into the engine queue + * @engine: the hardware engine + * @req: the request need to be listed into the engine queue + */ +int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine, + struct ahash_request *req) +{ + return crypto_transfer_request_to_engine(engine, &req->base); +} +EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine); + +/** + * crypto_transfer_kpp_request_to_engine - transfer one kpp_request to list + * into the engine queue + * @engine: the hardware engine + * @req: the request need to be listed into the engine queue + */ +int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine, + struct kpp_request *req) +{ + return crypto_transfer_request_to_engine(engine, &req->base); +} +EXPORT_SYMBOL_GPL(crypto_transfer_kpp_request_to_engine); + +/** + * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request + * to list into the engine queue + * @engine: the hardware engine + * @req: the request need to be listed into the engine queue + */ +int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine, + struct skcipher_request *req) +{ + return crypto_transfer_request_to_engine(engine, &req->base); +} +EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine); + +/** + * crypto_finalize_aead_request - finalize one aead_request if + * the request is done + * @engine: the hardware engine + * @req: the request need to be finalized + * @err: error number + */ +void crypto_finalize_aead_request(struct crypto_engine *engine, + struct aead_request *req, int err) +{ + return crypto_finalize_request(engine, &req->base, err); +} +EXPORT_SYMBOL_GPL(crypto_finalize_aead_request); + +/** + * crypto_finalize_akcipher_request - finalize one akcipher_request if + * the request is done + * @engine: the hardware engine + * @req: the request need to be finalized + * @err: error number + */ +void crypto_finalize_akcipher_request(struct crypto_engine *engine, + struct akcipher_request *req, int err) +{ + return crypto_finalize_request(engine, &req->base, err); +} +EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request); + +/** + * crypto_finalize_hash_request - finalize one ahash_request if + * the request is done + * @engine: the hardware engine + * @req: the request need to be finalized + * @err: error number + */ +void crypto_finalize_hash_request(struct crypto_engine *engine, + struct ahash_request *req, int err) +{ + return crypto_finalize_request(engine, &req->base, err); +} +EXPORT_SYMBOL_GPL(crypto_finalize_hash_request); + +/** + * crypto_finalize_kpp_request - finalize one kpp_request if the request is done + * @engine: the hardware engine + * @req: the request need to be finalized + * @err: error number + */ +void crypto_finalize_kpp_request(struct crypto_engine *engine, + struct kpp_request *req, int err) +{ + return crypto_finalize_request(engine, &req->base, err); +} +EXPORT_SYMBOL_GPL(crypto_finalize_kpp_request); + +/** + * crypto_finalize_skcipher_request - finalize one skcipher_request if + * the request is done + * @engine: the hardware engine + * @req: the request need to be finalized + * @err: error number + */ +void crypto_finalize_skcipher_request(struct crypto_engine *engine, + struct skcipher_request *req, int err) +{ + return crypto_finalize_request(engine, &req->base, err); +} +EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request); + +/** + * crypto_engine_start - start the hardware engine + * @engine: the hardware engine need to be started + * + * Return 0 on success, else on fail. + */ +int crypto_engine_start(struct crypto_engine *engine) +{ + unsigned long flags; + + spin_lock_irqsave(&engine->queue_lock, flags); + + if (engine->running || engine->busy) { + spin_unlock_irqrestore(&engine->queue_lock, flags); + return -EBUSY; + } + + engine->running = true; + spin_unlock_irqrestore(&engine->queue_lock, flags); + + kthread_queue_work(engine->kworker, &engine->pump_requests); + + return 0; +} +EXPORT_SYMBOL_GPL(crypto_engine_start); + +/** + * crypto_engine_stop - stop the hardware engine + * @engine: the hardware engine need to be stopped + * + * Return 0 on success, else on fail. + */ +int crypto_engine_stop(struct crypto_engine *engine) +{ + unsigned long flags; + unsigned int limit = 500; + int ret = 0; + + spin_lock_irqsave(&engine->queue_lock, flags); + + /* + * If the engine queue is not empty or the engine is on busy state, + * we need to wait for a while to pump the requests of engine queue. + */ + while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) { + spin_unlock_irqrestore(&engine->queue_lock, flags); + msleep(20); + spin_lock_irqsave(&engine->queue_lock, flags); + } + + if (crypto_queue_len(&engine->queue) || engine->busy) + ret = -EBUSY; + else + engine->running = false; + + spin_unlock_irqrestore(&engine->queue_lock, flags); + + if (ret) + dev_warn(engine->dev, "could not stop engine\n"); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_engine_stop); + +/** + * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure + * and initialize it by setting the maximum number of entries in the software + * crypto-engine queue. + * @dev: the device attached with one hardware engine + * @retry_support: whether hardware has support for retry mechanism + * @cbk_do_batch: pointer to a callback function to be invoked when executing + * a batch of requests. + * This has the form: + * callback(struct crypto_engine *engine) + * where: + * engine: the crypto engine structure. + * @rt: whether this queue is set to run as a realtime task + * @qlen: maximum size of the crypto-engine queue + * + * This must be called from context that can sleep. + * Return: the crypto engine structure on success, else NULL. + */ +struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev, + bool retry_support, + int (*cbk_do_batch)(struct crypto_engine *engine), + bool rt, int qlen) +{ + struct crypto_engine *engine; + + if (!dev) + return NULL; + + engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL); + if (!engine) + return NULL; + + engine->dev = dev; + engine->rt = rt; + engine->running = false; + engine->busy = false; + engine->idling = false; + engine->retry_support = retry_support; + engine->priv_data = dev; + /* + * Batch requests is possible only if + * hardware has support for retry mechanism. + */ + engine->do_batch_requests = retry_support ? cbk_do_batch : NULL; + + snprintf(engine->name, sizeof(engine->name), + "%s-engine", dev_name(dev)); + + crypto_init_queue(&engine->queue, qlen); + spin_lock_init(&engine->queue_lock); + + engine->kworker = kthread_create_worker(0, "%s", engine->name); + if (IS_ERR(engine->kworker)) { + dev_err(dev, "failed to create crypto request pump task\n"); + return NULL; + } + kthread_init_work(&engine->pump_requests, crypto_pump_work); + + if (engine->rt) { + dev_info(dev, "will run requests pump with realtime priority\n"); + sched_set_fifo(engine->kworker->task); + } + + return engine; +} +EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set); + +/** + * crypto_engine_alloc_init - allocate crypto hardware engine structure and + * initialize it. + * @dev: the device attached with one hardware engine + * @rt: whether this queue is set to run as a realtime task + * + * This must be called from context that can sleep. + * Return: the crypto engine structure on success, else NULL. + */ +struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) +{ + return crypto_engine_alloc_init_and_set(dev, false, NULL, rt, + CRYPTO_ENGINE_MAX_QLEN); +} +EXPORT_SYMBOL_GPL(crypto_engine_alloc_init); + +/** + * crypto_engine_exit - free the resources of hardware engine when exit + * @engine: the hardware engine need to be freed + * + * Return 0 for success. + */ +int crypto_engine_exit(struct crypto_engine *engine) +{ + int ret; + + ret = crypto_engine_stop(engine); + if (ret) + return ret; + + kthread_destroy_worker(engine->kworker); + + return 0; +} +EXPORT_SYMBOL_GPL(crypto_engine_exit); + +int crypto_engine_register_aead(struct aead_engine_alg *alg) +{ + if (!alg->op.do_one_request) + return -EINVAL; + + alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE; + + return crypto_register_aead(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_register_aead); + +void crypto_engine_unregister_aead(struct aead_engine_alg *alg) +{ + crypto_unregister_aead(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_aead); + +int crypto_engine_register_aeads(struct aead_engine_alg *algs, int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_engine_register_aead(&algs[i]); + if (ret) + goto err; + } + + return 0; + +err: + crypto_engine_unregister_aeads(algs, i); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_engine_register_aeads); + +void crypto_engine_unregister_aeads(struct aead_engine_alg *algs, int count) +{ + int i; + + for (i = count - 1; i >= 0; --i) + crypto_engine_unregister_aead(&algs[i]); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_aeads); + +int crypto_engine_register_ahash(struct ahash_engine_alg *alg) +{ + if (!alg->op.do_one_request) + return -EINVAL; + + alg->base.halg.base.cra_flags |= CRYPTO_ALG_ENGINE; + + return crypto_register_ahash(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_register_ahash); + +void crypto_engine_unregister_ahash(struct ahash_engine_alg *alg) +{ + crypto_unregister_ahash(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_ahash); + +int crypto_engine_register_ahashes(struct ahash_engine_alg *algs, int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_engine_register_ahash(&algs[i]); + if (ret) + goto err; + } + + return 0; + +err: + crypto_engine_unregister_ahashes(algs, i); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_engine_register_ahashes); + +void crypto_engine_unregister_ahashes(struct ahash_engine_alg *algs, + int count) +{ + int i; + + for (i = count - 1; i >= 0; --i) + crypto_engine_unregister_ahash(&algs[i]); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_ahashes); + +int crypto_engine_register_akcipher(struct akcipher_engine_alg *alg) +{ + if (!alg->op.do_one_request) + return -EINVAL; + + alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE; + + return crypto_register_akcipher(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_register_akcipher); + +void crypto_engine_unregister_akcipher(struct akcipher_engine_alg *alg) +{ + crypto_unregister_akcipher(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_akcipher); + +int crypto_engine_register_kpp(struct kpp_engine_alg *alg) +{ + if (!alg->op.do_one_request) + return -EINVAL; + + alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE; + + return crypto_register_kpp(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_register_kpp); + +void crypto_engine_unregister_kpp(struct kpp_engine_alg *alg) +{ + crypto_unregister_kpp(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_kpp); + +int crypto_engine_register_skcipher(struct skcipher_engine_alg *alg) +{ + if (!alg->op.do_one_request) + return -EINVAL; + + alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE; + + return crypto_register_skcipher(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_register_skcipher); + +void crypto_engine_unregister_skcipher(struct skcipher_engine_alg *alg) +{ + return crypto_unregister_skcipher(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_skcipher); + +int crypto_engine_register_skciphers(struct skcipher_engine_alg *algs, + int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_engine_register_skcipher(&algs[i]); + if (ret) + goto err; + } + + return 0; + +err: + crypto_engine_unregister_skciphers(algs, i); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_engine_register_skciphers); + +void crypto_engine_unregister_skciphers(struct skcipher_engine_alg *algs, + int count) +{ + int i; + + for (i = count - 1; i >= 0; --i) + crypto_engine_unregister_skcipher(&algs[i]); +} +EXPORT_SYMBOL_GPL(crypto_engine_unregister_skciphers); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Crypto hardware engine framework"); |