summaryrefslogtreecommitdiffstats
path: root/crypto/crypto_engine.c
diff options
context:
space:
mode:
Diffstat (limited to 'crypto/crypto_engine.c')
-rw-r--r--crypto/crypto_engine.c745
1 files changed, 745 insertions, 0 deletions
diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c
new file mode 100644
index 0000000000..108d9d55c5
--- /dev/null
+++ b/crypto/crypto_engine.c
@@ -0,0 +1,745 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Handle async block request by crypto hardware engine.
+ *
+ * Copyright (C) 2016 Linaro, Inc.
+ *
+ * Author: Baolin Wang <baolin.wang@linaro.org>
+ */
+
+#include <crypto/internal/aead.h>
+#include <crypto/internal/akcipher.h>
+#include <crypto/internal/engine.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/kpp.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <uapi/linux/sched/types.h>
+#include "internal.h"
+
+#define CRYPTO_ENGINE_MAX_QLEN 10
+
+/* Temporary algorithm flag used to indicate an updated driver. */
+#define CRYPTO_ALG_ENGINE 0x200
+
+struct crypto_engine_alg {
+ struct crypto_alg base;
+ struct crypto_engine_op op;
+};
+
+/**
+ * crypto_finalize_request - finalize one request if the request is done
+ * @engine: the hardware engine
+ * @req: the request need to be finalized
+ * @err: error number
+ */
+static void crypto_finalize_request(struct crypto_engine *engine,
+ struct crypto_async_request *req, int err)
+{
+ unsigned long flags;
+
+ /*
+ * If hardware cannot enqueue more requests
+ * and retry mechanism is not supported
+ * make sure we are completing the current request
+ */
+ if (!engine->retry_support) {
+ spin_lock_irqsave(&engine->queue_lock, flags);
+ if (engine->cur_req == req) {
+ engine->cur_req = NULL;
+ }
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+ }
+
+ lockdep_assert_in_softirq();
+ crypto_request_complete(req, err);
+
+ kthread_queue_work(engine->kworker, &engine->pump_requests);
+}
+
+/**
+ * crypto_pump_requests - dequeue one request from engine queue to process
+ * @engine: the hardware engine
+ * @in_kthread: true if we are in the context of the request pump thread
+ *
+ * This function checks if there is any request in the engine queue that
+ * needs processing and if so call out to the driver to initialize hardware
+ * and handle each request.
+ */
+static void crypto_pump_requests(struct crypto_engine *engine,
+ bool in_kthread)
+{
+ struct crypto_async_request *async_req, *backlog;
+ struct crypto_engine_alg *alg;
+ struct crypto_engine_op *op;
+ unsigned long flags;
+ bool was_busy = false;
+ int ret;
+
+ spin_lock_irqsave(&engine->queue_lock, flags);
+
+ /* Make sure we are not already running a request */
+ if (!engine->retry_support && engine->cur_req)
+ goto out;
+
+ /* If another context is idling then defer */
+ if (engine->idling) {
+ kthread_queue_work(engine->kworker, &engine->pump_requests);
+ goto out;
+ }
+
+ /* Check if the engine queue is idle */
+ if (!crypto_queue_len(&engine->queue) || !engine->running) {
+ if (!engine->busy)
+ goto out;
+
+ /* Only do teardown in the thread */
+ if (!in_kthread) {
+ kthread_queue_work(engine->kworker,
+ &engine->pump_requests);
+ goto out;
+ }
+
+ engine->busy = false;
+ engine->idling = true;
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+
+ if (engine->unprepare_crypt_hardware &&
+ engine->unprepare_crypt_hardware(engine))
+ dev_err(engine->dev, "failed to unprepare crypt hardware\n");
+
+ spin_lock_irqsave(&engine->queue_lock, flags);
+ engine->idling = false;
+ goto out;
+ }
+
+start_request:
+ /* Get the fist request from the engine queue to handle */
+ backlog = crypto_get_backlog(&engine->queue);
+ async_req = crypto_dequeue_request(&engine->queue);
+ if (!async_req)
+ goto out;
+
+ /*
+ * If hardware doesn't support the retry mechanism,
+ * keep track of the request we are processing now.
+ * We'll need it on completion (crypto_finalize_request).
+ */
+ if (!engine->retry_support)
+ engine->cur_req = async_req;
+
+ if (engine->busy)
+ was_busy = true;
+ else
+ engine->busy = true;
+
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+
+ /* Until here we get the request need to be encrypted successfully */
+ if (!was_busy && engine->prepare_crypt_hardware) {
+ ret = engine->prepare_crypt_hardware(engine);
+ if (ret) {
+ dev_err(engine->dev, "failed to prepare crypt hardware\n");
+ goto req_err_1;
+ }
+ }
+
+ if (async_req->tfm->__crt_alg->cra_flags & CRYPTO_ALG_ENGINE) {
+ alg = container_of(async_req->tfm->__crt_alg,
+ struct crypto_engine_alg, base);
+ op = &alg->op;
+ } else {
+ dev_err(engine->dev, "failed to do request\n");
+ ret = -EINVAL;
+ goto req_err_1;
+ }
+
+ ret = op->do_one_request(engine, async_req);
+
+ /* Request unsuccessfully executed by hardware */
+ if (ret < 0) {
+ /*
+ * If hardware queue is full (-ENOSPC), requeue request
+ * regardless of backlog flag.
+ * Otherwise, unprepare and complete the request.
+ */
+ if (!engine->retry_support ||
+ (ret != -ENOSPC)) {
+ dev_err(engine->dev,
+ "Failed to do one request from queue: %d\n",
+ ret);
+ goto req_err_1;
+ }
+ spin_lock_irqsave(&engine->queue_lock, flags);
+ /*
+ * If hardware was unable to execute request, enqueue it
+ * back in front of crypto-engine queue, to keep the order
+ * of requests.
+ */
+ crypto_enqueue_request_head(&engine->queue, async_req);
+
+ kthread_queue_work(engine->kworker, &engine->pump_requests);
+ goto out;
+ }
+
+ goto retry;
+
+req_err_1:
+ crypto_request_complete(async_req, ret);
+
+retry:
+ if (backlog)
+ crypto_request_complete(backlog, -EINPROGRESS);
+
+ /* If retry mechanism is supported, send new requests to engine */
+ if (engine->retry_support) {
+ spin_lock_irqsave(&engine->queue_lock, flags);
+ goto start_request;
+ }
+ return;
+
+out:
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+
+ /*
+ * Batch requests is possible only if
+ * hardware can enqueue multiple requests
+ */
+ if (engine->do_batch_requests) {
+ ret = engine->do_batch_requests(engine);
+ if (ret)
+ dev_err(engine->dev, "failed to do batch requests: %d\n",
+ ret);
+ }
+
+ return;
+}
+
+static void crypto_pump_work(struct kthread_work *work)
+{
+ struct crypto_engine *engine =
+ container_of(work, struct crypto_engine, pump_requests);
+
+ crypto_pump_requests(engine, true);
+}
+
+/**
+ * crypto_transfer_request - transfer the new request into the engine queue
+ * @engine: the hardware engine
+ * @req: the request need to be listed into the engine queue
+ * @need_pump: indicates whether queue the pump of request to kthread_work
+ */
+static int crypto_transfer_request(struct crypto_engine *engine,
+ struct crypto_async_request *req,
+ bool need_pump)
+{
+ unsigned long flags;
+ int ret;
+
+ spin_lock_irqsave(&engine->queue_lock, flags);
+
+ if (!engine->running) {
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+ return -ESHUTDOWN;
+ }
+
+ ret = crypto_enqueue_request(&engine->queue, req);
+
+ if (!engine->busy && need_pump)
+ kthread_queue_work(engine->kworker, &engine->pump_requests);
+
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+ return ret;
+}
+
+/**
+ * crypto_transfer_request_to_engine - transfer one request to list
+ * into the engine queue
+ * @engine: the hardware engine
+ * @req: the request need to be listed into the engine queue
+ */
+static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
+ struct crypto_async_request *req)
+{
+ return crypto_transfer_request(engine, req, true);
+}
+
+/**
+ * crypto_transfer_aead_request_to_engine - transfer one aead_request
+ * to list into the engine queue
+ * @engine: the hardware engine
+ * @req: the request need to be listed into the engine queue
+ */
+int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
+ struct aead_request *req)
+{
+ return crypto_transfer_request_to_engine(engine, &req->base);
+}
+EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);
+
+/**
+ * crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
+ * to list into the engine queue
+ * @engine: the hardware engine
+ * @req: the request need to be listed into the engine queue
+ */
+int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
+ struct akcipher_request *req)
+{
+ return crypto_transfer_request_to_engine(engine, &req->base);
+}
+EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);
+
+/**
+ * crypto_transfer_hash_request_to_engine - transfer one ahash_request
+ * to list into the engine queue
+ * @engine: the hardware engine
+ * @req: the request need to be listed into the engine queue
+ */
+int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
+ struct ahash_request *req)
+{
+ return crypto_transfer_request_to_engine(engine, &req->base);
+}
+EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
+
+/**
+ * crypto_transfer_kpp_request_to_engine - transfer one kpp_request to list
+ * into the engine queue
+ * @engine: the hardware engine
+ * @req: the request need to be listed into the engine queue
+ */
+int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine,
+ struct kpp_request *req)
+{
+ return crypto_transfer_request_to_engine(engine, &req->base);
+}
+EXPORT_SYMBOL_GPL(crypto_transfer_kpp_request_to_engine);
+
+/**
+ * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
+ * to list into the engine queue
+ * @engine: the hardware engine
+ * @req: the request need to be listed into the engine queue
+ */
+int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
+ struct skcipher_request *req)
+{
+ return crypto_transfer_request_to_engine(engine, &req->base);
+}
+EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);
+
+/**
+ * crypto_finalize_aead_request - finalize one aead_request if
+ * the request is done
+ * @engine: the hardware engine
+ * @req: the request need to be finalized
+ * @err: error number
+ */
+void crypto_finalize_aead_request(struct crypto_engine *engine,
+ struct aead_request *req, int err)
+{
+ return crypto_finalize_request(engine, &req->base, err);
+}
+EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);
+
+/**
+ * crypto_finalize_akcipher_request - finalize one akcipher_request if
+ * the request is done
+ * @engine: the hardware engine
+ * @req: the request need to be finalized
+ * @err: error number
+ */
+void crypto_finalize_akcipher_request(struct crypto_engine *engine,
+ struct akcipher_request *req, int err)
+{
+ return crypto_finalize_request(engine, &req->base, err);
+}
+EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);
+
+/**
+ * crypto_finalize_hash_request - finalize one ahash_request if
+ * the request is done
+ * @engine: the hardware engine
+ * @req: the request need to be finalized
+ * @err: error number
+ */
+void crypto_finalize_hash_request(struct crypto_engine *engine,
+ struct ahash_request *req, int err)
+{
+ return crypto_finalize_request(engine, &req->base, err);
+}
+EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
+
+/**
+ * crypto_finalize_kpp_request - finalize one kpp_request if the request is done
+ * @engine: the hardware engine
+ * @req: the request need to be finalized
+ * @err: error number
+ */
+void crypto_finalize_kpp_request(struct crypto_engine *engine,
+ struct kpp_request *req, int err)
+{
+ return crypto_finalize_request(engine, &req->base, err);
+}
+EXPORT_SYMBOL_GPL(crypto_finalize_kpp_request);
+
+/**
+ * crypto_finalize_skcipher_request - finalize one skcipher_request if
+ * the request is done
+ * @engine: the hardware engine
+ * @req: the request need to be finalized
+ * @err: error number
+ */
+void crypto_finalize_skcipher_request(struct crypto_engine *engine,
+ struct skcipher_request *req, int err)
+{
+ return crypto_finalize_request(engine, &req->base, err);
+}
+EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);
+
+/**
+ * crypto_engine_start - start the hardware engine
+ * @engine: the hardware engine need to be started
+ *
+ * Return 0 on success, else on fail.
+ */
+int crypto_engine_start(struct crypto_engine *engine)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&engine->queue_lock, flags);
+
+ if (engine->running || engine->busy) {
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+ return -EBUSY;
+ }
+
+ engine->running = true;
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+
+ kthread_queue_work(engine->kworker, &engine->pump_requests);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_engine_start);
+
+/**
+ * crypto_engine_stop - stop the hardware engine
+ * @engine: the hardware engine need to be stopped
+ *
+ * Return 0 on success, else on fail.
+ */
+int crypto_engine_stop(struct crypto_engine *engine)
+{
+ unsigned long flags;
+ unsigned int limit = 500;
+ int ret = 0;
+
+ spin_lock_irqsave(&engine->queue_lock, flags);
+
+ /*
+ * If the engine queue is not empty or the engine is on busy state,
+ * we need to wait for a while to pump the requests of engine queue.
+ */
+ while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+ msleep(20);
+ spin_lock_irqsave(&engine->queue_lock, flags);
+ }
+
+ if (crypto_queue_len(&engine->queue) || engine->busy)
+ ret = -EBUSY;
+ else
+ engine->running = false;
+
+ spin_unlock_irqrestore(&engine->queue_lock, flags);
+
+ if (ret)
+ dev_warn(engine->dev, "could not stop engine\n");
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_engine_stop);
+
+/**
+ * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
+ * and initialize it by setting the maximum number of entries in the software
+ * crypto-engine queue.
+ * @dev: the device attached with one hardware engine
+ * @retry_support: whether hardware has support for retry mechanism
+ * @cbk_do_batch: pointer to a callback function to be invoked when executing
+ * a batch of requests.
+ * This has the form:
+ * callback(struct crypto_engine *engine)
+ * where:
+ * engine: the crypto engine structure.
+ * @rt: whether this queue is set to run as a realtime task
+ * @qlen: maximum size of the crypto-engine queue
+ *
+ * This must be called from context that can sleep.
+ * Return: the crypto engine structure on success, else NULL.
+ */
+struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
+ bool retry_support,
+ int (*cbk_do_batch)(struct crypto_engine *engine),
+ bool rt, int qlen)
+{
+ struct crypto_engine *engine;
+
+ if (!dev)
+ return NULL;
+
+ engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
+ if (!engine)
+ return NULL;
+
+ engine->dev = dev;
+ engine->rt = rt;
+ engine->running = false;
+ engine->busy = false;
+ engine->idling = false;
+ engine->retry_support = retry_support;
+ engine->priv_data = dev;
+ /*
+ * Batch requests is possible only if
+ * hardware has support for retry mechanism.
+ */
+ engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;
+
+ snprintf(engine->name, sizeof(engine->name),
+ "%s-engine", dev_name(dev));
+
+ crypto_init_queue(&engine->queue, qlen);
+ spin_lock_init(&engine->queue_lock);
+
+ engine->kworker = kthread_create_worker(0, "%s", engine->name);
+ if (IS_ERR(engine->kworker)) {
+ dev_err(dev, "failed to create crypto request pump task\n");
+ return NULL;
+ }
+ kthread_init_work(&engine->pump_requests, crypto_pump_work);
+
+ if (engine->rt) {
+ dev_info(dev, "will run requests pump with realtime priority\n");
+ sched_set_fifo(engine->kworker->task);
+ }
+
+ return engine;
+}
+EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);
+
+/**
+ * crypto_engine_alloc_init - allocate crypto hardware engine structure and
+ * initialize it.
+ * @dev: the device attached with one hardware engine
+ * @rt: whether this queue is set to run as a realtime task
+ *
+ * This must be called from context that can sleep.
+ * Return: the crypto engine structure on success, else NULL.
+ */
+struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
+{
+ return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
+ CRYPTO_ENGINE_MAX_QLEN);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
+
+/**
+ * crypto_engine_exit - free the resources of hardware engine when exit
+ * @engine: the hardware engine need to be freed
+ *
+ * Return 0 for success.
+ */
+int crypto_engine_exit(struct crypto_engine *engine)
+{
+ int ret;
+
+ ret = crypto_engine_stop(engine);
+ if (ret)
+ return ret;
+
+ kthread_destroy_worker(engine->kworker);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_engine_exit);
+
+int crypto_engine_register_aead(struct aead_engine_alg *alg)
+{
+ if (!alg->op.do_one_request)
+ return -EINVAL;
+
+ alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE;
+
+ return crypto_register_aead(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_register_aead);
+
+void crypto_engine_unregister_aead(struct aead_engine_alg *alg)
+{
+ crypto_unregister_aead(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_unregister_aead);
+
+int crypto_engine_register_aeads(struct aead_engine_alg *algs, int count)
+{
+ int i, ret;
+
+ for (i = 0; i < count; i++) {
+ ret = crypto_engine_register_aead(&algs[i]);
+ if (ret)
+ goto err;
+ }
+
+ return 0;
+
+err:
+ crypto_engine_unregister_aeads(algs, i);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_engine_register_aeads);
+
+void crypto_engine_unregister_aeads(struct aead_engine_alg *algs, int count)
+{
+ int i;
+
+ for (i = count - 1; i >= 0; --i)
+ crypto_engine_unregister_aead(&algs[i]);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_unregister_aeads);
+
+int crypto_engine_register_ahash(struct ahash_engine_alg *alg)
+{
+ if (!alg->op.do_one_request)
+ return -EINVAL;
+
+ alg->base.halg.base.cra_flags |= CRYPTO_ALG_ENGINE;
+
+ return crypto_register_ahash(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_register_ahash);
+
+void crypto_engine_unregister_ahash(struct ahash_engine_alg *alg)
+{
+ crypto_unregister_ahash(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_unregister_ahash);
+
+int crypto_engine_register_ahashes(struct ahash_engine_alg *algs, int count)
+{
+ int i, ret;
+
+ for (i = 0; i < count; i++) {
+ ret = crypto_engine_register_ahash(&algs[i]);
+ if (ret)
+ goto err;
+ }
+
+ return 0;
+
+err:
+ crypto_engine_unregister_ahashes(algs, i);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_engine_register_ahashes);
+
+void crypto_engine_unregister_ahashes(struct ahash_engine_alg *algs,
+ int count)
+{
+ int i;
+
+ for (i = count - 1; i >= 0; --i)
+ crypto_engine_unregister_ahash(&algs[i]);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_unregister_ahashes);
+
+int crypto_engine_register_akcipher(struct akcipher_engine_alg *alg)
+{
+ if (!alg->op.do_one_request)
+ return -EINVAL;
+
+ alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE;
+
+ return crypto_register_akcipher(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_register_akcipher);
+
+void crypto_engine_unregister_akcipher(struct akcipher_engine_alg *alg)
+{
+ crypto_unregister_akcipher(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_unregister_akcipher);
+
+int crypto_engine_register_kpp(struct kpp_engine_alg *alg)
+{
+ if (!alg->op.do_one_request)
+ return -EINVAL;
+
+ alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE;
+
+ return crypto_register_kpp(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_register_kpp);
+
+void crypto_engine_unregister_kpp(struct kpp_engine_alg *alg)
+{
+ crypto_unregister_kpp(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_unregister_kpp);
+
+int crypto_engine_register_skcipher(struct skcipher_engine_alg *alg)
+{
+ if (!alg->op.do_one_request)
+ return -EINVAL;
+
+ alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE;
+
+ return crypto_register_skcipher(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_register_skcipher);
+
+void crypto_engine_unregister_skcipher(struct skcipher_engine_alg *alg)
+{
+ return crypto_unregister_skcipher(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_unregister_skcipher);
+
+int crypto_engine_register_skciphers(struct skcipher_engine_alg *algs,
+ int count)
+{
+ int i, ret;
+
+ for (i = 0; i < count; i++) {
+ ret = crypto_engine_register_skcipher(&algs[i]);
+ if (ret)
+ goto err;
+ }
+
+ return 0;
+
+err:
+ crypto_engine_unregister_skciphers(algs, i);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_engine_register_skciphers);
+
+void crypto_engine_unregister_skciphers(struct skcipher_engine_alg *algs,
+ int count)
+{
+ int i;
+
+ for (i = count - 1; i >= 0; --i)
+ crypto_engine_unregister_skcipher(&algs[i]);
+}
+EXPORT_SYMBOL_GPL(crypto_engine_unregister_skciphers);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Crypto hardware engine framework");