summaryrefslogtreecommitdiffstats
path: root/crypto/shash.c
diff options
context:
space:
mode:
Diffstat (limited to 'crypto/shash.c')
-rw-r--r--crypto/shash.c379
1 files changed, 61 insertions, 318 deletions
diff --git a/crypto/shash.c b/crypto/shash.c
index 1fadb6b59b..d5194221c8 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -10,17 +10,12 @@
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
-#include <linux/slab.h>
#include <linux/seq_file.h>
#include <linux/string.h>
#include <net/netlink.h>
#include "hash.h"
-#define MAX_SHASH_ALIGNMASK 63
-
-static const struct crypto_type crypto_shash_type;
-
static inline struct crypto_istat_hash *shash_get_stat(struct shash_alg *alg)
{
return hash_get_stat(&alg->halg);
@@ -28,7 +23,13 @@ static inline struct crypto_istat_hash *shash_get_stat(struct shash_alg *alg)
static inline int crypto_shash_errstat(struct shash_alg *alg, int err)
{
- return crypto_hash_errstat(&alg->halg, err);
+ if (!IS_ENABLED(CONFIG_CRYPTO_STATS))
+ return err;
+
+ if (err && err != -EINPROGRESS && err != -EBUSY)
+ atomic64_inc(&shash_get_stat(alg)->err_cnt);
+
+ return err;
}
int shash_no_setkey(struct crypto_shash *tfm, const u8 *key,
@@ -38,27 +39,6 @@ int shash_no_setkey(struct crypto_shash *tfm, const u8 *key,
}
EXPORT_SYMBOL_GPL(shash_no_setkey);
-static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key,
- unsigned int keylen)
-{
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
- unsigned long absize;
- u8 *buffer, *alignbuffer;
- int err;
-
- absize = keylen + (alignmask & ~(crypto_tfm_ctx_alignment() - 1));
- buffer = kmalloc(absize, GFP_ATOMIC);
- if (!buffer)
- return -ENOMEM;
-
- alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1);
- memcpy(alignbuffer, key, keylen);
- err = shash->setkey(tfm, alignbuffer, keylen);
- kfree_sensitive(buffer);
- return err;
-}
-
static void shash_set_needkey(struct crypto_shash *tfm, struct shash_alg *alg)
{
if (crypto_shash_alg_needs_key(alg))
@@ -69,14 +49,9 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key,
unsigned int keylen)
{
struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
int err;
- if ((unsigned long)key & alignmask)
- err = shash_setkey_unaligned(tfm, key, keylen);
- else
- err = shash->setkey(tfm, key, keylen);
-
+ err = shash->setkey(tfm, key, keylen);
if (unlikely(err)) {
shash_set_needkey(tfm, shash);
return err;
@@ -87,108 +62,42 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key,
}
EXPORT_SYMBOL_GPL(crypto_shash_setkey);
-static int shash_update_unaligned(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- struct crypto_shash *tfm = desc->tfm;
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
- unsigned int unaligned_len = alignmask + 1 -
- ((unsigned long)data & alignmask);
- /*
- * We cannot count on __aligned() working for large values:
- * https://patchwork.kernel.org/patch/9507697/
- */
- u8 ubuf[MAX_SHASH_ALIGNMASK * 2];
- u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1);
- int err;
-
- if (WARN_ON(buf + unaligned_len > ubuf + sizeof(ubuf)))
- return -EINVAL;
-
- if (unaligned_len > len)
- unaligned_len = len;
-
- memcpy(buf, data, unaligned_len);
- err = shash->update(desc, buf, unaligned_len);
- memset(buf, 0, unaligned_len);
-
- return err ?:
- shash->update(desc, data + unaligned_len, len - unaligned_len);
-}
-
int crypto_shash_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
- struct crypto_shash *tfm = desc->tfm;
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
+ struct shash_alg *shash = crypto_shash_alg(desc->tfm);
int err;
if (IS_ENABLED(CONFIG_CRYPTO_STATS))
atomic64_add(len, &shash_get_stat(shash)->hash_tlen);
- if ((unsigned long)data & alignmask)
- err = shash_update_unaligned(desc, data, len);
- else
- err = shash->update(desc, data, len);
+ err = shash->update(desc, data, len);
return crypto_shash_errstat(shash, err);
}
EXPORT_SYMBOL_GPL(crypto_shash_update);
-static int shash_final_unaligned(struct shash_desc *desc, u8 *out)
-{
- struct crypto_shash *tfm = desc->tfm;
- unsigned long alignmask = crypto_shash_alignmask(tfm);
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned int ds = crypto_shash_digestsize(tfm);
- /*
- * We cannot count on __aligned() working for large values:
- * https://patchwork.kernel.org/patch/9507697/
- */
- u8 ubuf[MAX_SHASH_ALIGNMASK + HASH_MAX_DIGESTSIZE];
- u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1);
- int err;
-
- if (WARN_ON(buf + ds > ubuf + sizeof(ubuf)))
- return -EINVAL;
-
- err = shash->final(desc, buf);
- if (err)
- goto out;
-
- memcpy(out, buf, ds);
-
-out:
- memset(buf, 0, ds);
- return err;
-}
-
int crypto_shash_final(struct shash_desc *desc, u8 *out)
{
- struct crypto_shash *tfm = desc->tfm;
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
+ struct shash_alg *shash = crypto_shash_alg(desc->tfm);
int err;
if (IS_ENABLED(CONFIG_CRYPTO_STATS))
atomic64_inc(&shash_get_stat(shash)->hash_cnt);
- if ((unsigned long)out & alignmask)
- err = shash_final_unaligned(desc, out);
- else
- err = shash->final(desc, out);
+ err = shash->final(desc, out);
return crypto_shash_errstat(shash, err);
}
EXPORT_SYMBOL_GPL(crypto_shash_final);
-static int shash_finup_unaligned(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
+static int shash_default_finup(struct shash_desc *desc, const u8 *data,
+ unsigned int len, u8 *out)
{
- return shash_update_unaligned(desc, data, len) ?:
- shash_final_unaligned(desc, out);
+ struct shash_alg *shash = crypto_shash_alg(desc->tfm);
+
+ return shash->update(desc, data, len) ?:
+ shash->final(desc, out);
}
int crypto_shash_finup(struct shash_desc *desc, const u8 *data,
@@ -196,7 +105,6 @@ int crypto_shash_finup(struct shash_desc *desc, const u8 *data,
{
struct crypto_shash *tfm = desc->tfm;
struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
int err;
if (IS_ENABLED(CONFIG_CRYPTO_STATS)) {
@@ -206,22 +114,19 @@ int crypto_shash_finup(struct shash_desc *desc, const u8 *data,
atomic64_add(len, &istat->hash_tlen);
}
- if (((unsigned long)data | (unsigned long)out) & alignmask)
- err = shash_finup_unaligned(desc, data, len, out);
- else
- err = shash->finup(desc, data, len, out);
-
+ err = shash->finup(desc, data, len, out);
return crypto_shash_errstat(shash, err);
}
EXPORT_SYMBOL_GPL(crypto_shash_finup);
-static int shash_digest_unaligned(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
+static int shash_default_digest(struct shash_desc *desc, const u8 *data,
+ unsigned int len, u8 *out)
{
- return crypto_shash_init(desc) ?:
- shash_update_unaligned(desc, data, len) ?:
- shash_final_unaligned(desc, out);
+ struct shash_alg *shash = crypto_shash_alg(desc->tfm);
+
+ return shash->init(desc) ?:
+ shash->finup(desc, data, len, out);
}
int crypto_shash_digest(struct shash_desc *desc, const u8 *data,
@@ -229,7 +134,6 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data,
{
struct crypto_shash *tfm = desc->tfm;
struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
int err;
if (IS_ENABLED(CONFIG_CRYPTO_STATS)) {
@@ -241,8 +145,6 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data,
if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
err = -ENOKEY;
- else if (((unsigned long)data | (unsigned long)out) & alignmask)
- err = shash_digest_unaligned(desc, data, len, out);
else
err = shash->digest(desc, data, len, out);
@@ -266,202 +168,34 @@ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data,
}
EXPORT_SYMBOL_GPL(crypto_shash_tfm_digest);
-static int shash_default_export(struct shash_desc *desc, void *out)
-{
- memcpy(out, shash_desc_ctx(desc), crypto_shash_descsize(desc->tfm));
- return 0;
-}
-
-static int shash_default_import(struct shash_desc *desc, const void *in)
-{
- memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(desc->tfm));
- return 0;
-}
-
-static int shash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
- unsigned int keylen)
-{
- struct crypto_shash **ctx = crypto_ahash_ctx(tfm);
-
- return crypto_shash_setkey(*ctx, key, keylen);
-}
-
-static int shash_async_init(struct ahash_request *req)
-{
- struct crypto_shash **ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
- struct shash_desc *desc = ahash_request_ctx(req);
-
- desc->tfm = *ctx;
-
- return crypto_shash_init(desc);
-}
-
-int shash_ahash_update(struct ahash_request *req, struct shash_desc *desc)
-{
- struct crypto_hash_walk walk;
- int nbytes;
-
- for (nbytes = crypto_hash_walk_first(req, &walk); nbytes > 0;
- nbytes = crypto_hash_walk_done(&walk, nbytes))
- nbytes = crypto_shash_update(desc, walk.data, nbytes);
-
- return nbytes;
-}
-EXPORT_SYMBOL_GPL(shash_ahash_update);
-
-static int shash_async_update(struct ahash_request *req)
-{
- return shash_ahash_update(req, ahash_request_ctx(req));
-}
-
-static int shash_async_final(struct ahash_request *req)
-{
- return crypto_shash_final(ahash_request_ctx(req), req->result);
-}
-
-int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc)
-{
- struct crypto_hash_walk walk;
- int nbytes;
-
- nbytes = crypto_hash_walk_first(req, &walk);
- if (!nbytes)
- return crypto_shash_final(desc, req->result);
-
- do {
- nbytes = crypto_hash_walk_last(&walk) ?
- crypto_shash_finup(desc, walk.data, nbytes,
- req->result) :
- crypto_shash_update(desc, walk.data, nbytes);
- nbytes = crypto_hash_walk_done(&walk, nbytes);
- } while (nbytes > 0);
-
- return nbytes;
-}
-EXPORT_SYMBOL_GPL(shash_ahash_finup);
-
-static int shash_async_finup(struct ahash_request *req)
-{
- struct crypto_shash **ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
- struct shash_desc *desc = ahash_request_ctx(req);
-
- desc->tfm = *ctx;
-
- return shash_ahash_finup(req, desc);
-}
-
-int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc)
-{
- unsigned int nbytes = req->nbytes;
- struct scatterlist *sg;
- unsigned int offset;
- int err;
-
- if (nbytes &&
- (sg = req->src, offset = sg->offset,
- nbytes <= min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset))) {
- void *data;
-
- data = kmap_local_page(sg_page(sg));
- err = crypto_shash_digest(desc, data + offset, nbytes,
- req->result);
- kunmap_local(data);
- } else
- err = crypto_shash_init(desc) ?:
- shash_ahash_finup(req, desc);
-
- return err;
-}
-EXPORT_SYMBOL_GPL(shash_ahash_digest);
-
-static int shash_async_digest(struct ahash_request *req)
-{
- struct crypto_shash **ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
- struct shash_desc *desc = ahash_request_ctx(req);
-
- desc->tfm = *ctx;
-
- return shash_ahash_digest(req, desc);
-}
-
-static int shash_async_export(struct ahash_request *req, void *out)
-{
- return crypto_shash_export(ahash_request_ctx(req), out);
-}
-
-static int shash_async_import(struct ahash_request *req, const void *in)
-{
- struct crypto_shash **ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
- struct shash_desc *desc = ahash_request_ctx(req);
-
- desc->tfm = *ctx;
-
- return crypto_shash_import(desc, in);
-}
-
-static void crypto_exit_shash_ops_async(struct crypto_tfm *tfm)
+int crypto_shash_export(struct shash_desc *desc, void *out)
{
- struct crypto_shash **ctx = crypto_tfm_ctx(tfm);
-
- crypto_free_shash(*ctx);
-}
-
-int crypto_init_shash_ops_async(struct crypto_tfm *tfm)
-{
- struct crypto_alg *calg = tfm->__crt_alg;
- struct shash_alg *alg = __crypto_shash_alg(calg);
- struct crypto_ahash *crt = __crypto_ahash_cast(tfm);
- struct crypto_shash **ctx = crypto_tfm_ctx(tfm);
- struct crypto_shash *shash;
-
- if (!crypto_mod_get(calg))
- return -EAGAIN;
-
- shash = crypto_create_tfm(calg, &crypto_shash_type);
- if (IS_ERR(shash)) {
- crypto_mod_put(calg);
- return PTR_ERR(shash);
- }
-
- *ctx = shash;
- tfm->exit = crypto_exit_shash_ops_async;
-
- crt->init = shash_async_init;
- crt->update = shash_async_update;
- crt->final = shash_async_final;
- crt->finup = shash_async_finup;
- crt->digest = shash_async_digest;
- if (crypto_shash_alg_has_setkey(alg))
- crt->setkey = shash_async_setkey;
-
- crypto_ahash_set_flags(crt, crypto_shash_get_flags(shash) &
- CRYPTO_TFM_NEED_KEY);
-
- crt->export = shash_async_export;
- crt->import = shash_async_import;
+ struct crypto_shash *tfm = desc->tfm;
+ struct shash_alg *shash = crypto_shash_alg(tfm);
- crt->reqsize = sizeof(struct shash_desc) + crypto_shash_descsize(shash);
+ if (shash->export)
+ return shash->export(desc, out);
+ memcpy(out, shash_desc_ctx(desc), crypto_shash_descsize(tfm));
return 0;
}
+EXPORT_SYMBOL_GPL(crypto_shash_export);
-struct crypto_ahash *crypto_clone_shash_ops_async(struct crypto_ahash *nhash,
- struct crypto_ahash *hash)
+int crypto_shash_import(struct shash_desc *desc, const void *in)
{
- struct crypto_shash **nctx = crypto_ahash_ctx(nhash);
- struct crypto_shash **ctx = crypto_ahash_ctx(hash);
- struct crypto_shash *shash;
+ struct crypto_shash *tfm = desc->tfm;
+ struct shash_alg *shash = crypto_shash_alg(tfm);
- shash = crypto_clone_shash(*ctx);
- if (IS_ERR(shash)) {
- crypto_free_ahash(nhash);
- return ERR_CAST(shash);
- }
+ if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
+ return -ENOKEY;
- *nctx = shash;
+ if (shash->import)
+ return shash->import(desc, in);
- return nhash;
+ memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(tfm));
+ return 0;
}
+EXPORT_SYMBOL_GPL(crypto_shash_import);
static void crypto_shash_exit_tfm(struct crypto_tfm *tfm)
{
@@ -541,7 +275,7 @@ static int __maybe_unused crypto_shash_report_stat(
return crypto_hash_report_stat(skb, alg, "shash");
}
-static const struct crypto_type crypto_shash_type = {
+const struct crypto_type crypto_shash_type = {
.extsize = crypto_alg_extsize,
.init_tfm = crypto_shash_init_tfm,
.free = crypto_shash_free_instance,
@@ -626,6 +360,10 @@ int hash_prepare_alg(struct hash_alg_common *alg)
if (alg->digestsize > HASH_MAX_DIGESTSIZE)
return -EINVAL;
+ /* alignmask is not useful for hashes, so it is not supported. */
+ if (base->cra_alignmask)
+ return -EINVAL;
+
base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
if (IS_ENABLED(CONFIG_CRYPTO_STATS))
@@ -642,9 +380,6 @@ static int shash_prepare_alg(struct shash_alg *alg)
if (alg->descsize > HASH_MAX_DESCSIZE)
return -EINVAL;
- if (base->cra_alignmask > MAX_SHASH_ALIGNMASK)
- return -EINVAL;
-
if ((alg->export && !alg->import) || (alg->import && !alg->export))
return -EINVAL;
@@ -655,15 +390,23 @@ static int shash_prepare_alg(struct shash_alg *alg)
base->cra_type = &crypto_shash_type;
base->cra_flags |= CRYPTO_ALG_TYPE_SHASH;
+ /*
+ * Handle missing optional functions. For each one we can either
+ * install a default here, or we can leave the pointer as NULL and check
+ * the pointer for NULL in crypto_shash_*(), avoiding an indirect call
+ * when the default behavior is desired. For ->finup and ->digest we
+ * install defaults, since for optimal performance algorithms should
+ * implement these anyway. On the other hand, for ->import and
+ * ->export the common case and best performance comes from the simple
+ * memcpy of the shash_desc_ctx, so when those pointers are NULL we
+ * leave them NULL and provide the memcpy with no indirect call.
+ */
if (!alg->finup)
- alg->finup = shash_finup_unaligned;
+ alg->finup = shash_default_finup;
if (!alg->digest)
- alg->digest = shash_digest_unaligned;
- if (!alg->export) {
- alg->export = shash_default_export;
- alg->import = shash_default_import;
+ alg->digest = shash_default_digest;
+ if (!alg->export)
alg->halg.statesize = alg->descsize;
- }
if (!alg->setkey)
alg->setkey = shash_no_setkey;