diff options
322 files changed, 5973 insertions, 4248 deletions
diff --git a/Documentation/crypto/api-samples.rst b/Documentation/crypto/api-samples.rst index 0f6ca8b7261e..f14afaaf2f32 100644 --- a/Documentation/crypto/api-samples.rst +++ b/Documentation/crypto/api-samples.rst @@ -133,7 +133,6 @@ Code Example For Use of Operational State Memory With SHASH if (!sdesc) return ERR_PTR(-ENOMEM); sdesc->shash.tfm = alg; - sdesc->shash.flags = 0x0; return sdesc; } diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index 07e31941dc67..617c2c99ebfb 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -278,6 +278,8 @@ static int __xts_crypt(struct skcipher_request *req, int err; err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; crypto_cipher_encrypt_one(ctx->tweak_tfm, walk.iv, walk.iv); diff --git a/arch/arm/crypto/chacha-neon-glue.c b/arch/arm/crypto/chacha-neon-glue.c index 9d6fda81986d..48a89537b828 100644 --- a/arch/arm/crypto/chacha-neon-glue.c +++ b/arch/arm/crypto/chacha-neon-glue.c @@ -21,6 +21,7 @@ #include <crypto/algapi.h> #include <crypto/chacha.h> +#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <linux/kernel.h> #include <linux/module.h> @@ -93,7 +94,7 @@ static int chacha_neon(struct skcipher_request *req) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - if (req->cryptlen <= CHACHA_BLOCK_SIZE || !may_use_simd()) + if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable()) return crypto_chacha_crypt(req); return chacha_neon_stream_xor(req, ctx, req->iv); @@ -107,7 +108,7 @@ static int xchacha_neon(struct skcipher_request *req) u32 state[16]; u8 real_iv[16]; - if (req->cryptlen <= CHACHA_BLOCK_SIZE || !may_use_simd()) + if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable()) return crypto_xchacha_crypt(req); crypto_chacha_init(state, ctx, req->iv); diff --git a/arch/arm/crypto/crc32-ce-glue.c b/arch/arm/crypto/crc32-ce-glue.c index cd9e93b46c2d..e712c2a7d387 100644 --- a/arch/arm/crypto/crc32-ce-glue.c +++ b/arch/arm/crypto/crc32-ce-glue.c @@ -16,6 +16,7 @@ #include <linux/string.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <asm/hwcap.h> #include <asm/neon.h> @@ -113,7 +114,7 @@ static int crc32_pmull_update(struct shash_desc *desc, const u8 *data, u32 *crc = shash_desc_ctx(desc); unsigned int l; - if (may_use_simd()) { + if (crypto_simd_usable()) { if ((u32)data % SCALE_F) { l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); @@ -147,7 +148,7 @@ static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data, u32 *crc = shash_desc_ctx(desc); unsigned int l; - if (may_use_simd()) { + if (crypto_simd_usable()) { if ((u32)data % SCALE_F) { l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); diff --git a/arch/arm/crypto/crct10dif-ce-glue.c b/arch/arm/crypto/crct10dif-ce-glue.c index 3d6b800b8396..3b24f2872592 100644 --- a/arch/arm/crypto/crct10dif-ce-glue.c +++ b/arch/arm/crypto/crct10dif-ce-glue.c @@ -15,6 +15,7 @@ #include <linux/string.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <asm/neon.h> #include <asm/simd.h> @@ -36,7 +37,7 @@ static int crct10dif_update(struct shash_desc *desc, const u8 *data, { u16 *crc = shash_desc_ctx(desc); - if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && may_use_simd()) { + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { kernel_neon_begin(); *crc = crc_t10dif_pmull(*crc, data, length); kernel_neon_end(); diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index b7d30b6cf49c..39d1ccec1aab 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -14,6 +14,7 @@ #include <asm/unaligned.h> #include <crypto/cryptd.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/gf128mul.h> #include <linux/cpufeature.h> #include <linux/crypto.h> @@ -185,7 +186,6 @@ static int ghash_async_init(struct ahash_request *req) struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); desc->tfm = child; - desc->flags = req->base.flags; return crypto_shash_init(desc); } @@ -196,7 +196,7 @@ static int ghash_async_update(struct ahash_request *req) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!may_use_simd() || + if (!crypto_simd_usable() || (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); @@ -214,7 +214,7 @@ static int ghash_async_final(struct ahash_request *req) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!may_use_simd() || + if (!crypto_simd_usable() || (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); @@ -232,7 +232,7 @@ static int ghash_async_digest(struct ahash_request *req) struct ahash_request *cryptd_req = ahash_request_ctx(req); struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!may_use_simd() || + if (!crypto_simd_usable() || (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); @@ -242,7 +242,6 @@ static int ghash_async_digest(struct ahash_request *req) struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); desc->tfm = child; - desc->flags = req->base.flags; return shash_ahash_digest(req, desc); } } @@ -255,7 +254,6 @@ static int ghash_async_import(struct ahash_request *req, const void *in) struct shash_desc *desc = cryptd_shash_desc(cryptd_req); desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm); - desc->flags = req->base.flags; return crypto_shash_import(desc, in); } diff --git a/arch/arm/crypto/nhpoly1305-neon-glue.c b/arch/arm/crypto/nhpoly1305-neon-glue.c index 49aae87cb2bc..ae5aefc44a4d 100644 --- a/arch/arm/crypto/nhpoly1305-neon-glue.c +++ b/arch/arm/crypto/nhpoly1305-neon-glue.c @@ -9,6 +9,7 @@ #include <asm/neon.h> #include <asm/simd.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/nhpoly1305.h> #include <linux/module.h> @@ -25,7 +26,7 @@ static void _nh_neon(const u32 *key, const u8 *message, size_t message_len, static int nhpoly1305_neon_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { - if (srclen < 64 || !may_use_simd()) + if (srclen < 64 || !crypto_simd_usable()) return crypto_nhpoly1305_update(desc, src, srclen); do { diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c index b732522e20f8..4c6c6900853c 100644 --- a/arch/arm/crypto/sha1-ce-glue.c +++ b/arch/arm/crypto/sha1-ce-glue.c @@ -9,6 +9,7 @@ */ #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/sha.h> #include <crypto/sha1_base.h> #include <linux/cpufeature.h> @@ -33,7 +34,7 @@ static int sha1_ce_update(struct shash_desc *desc, const u8 *data, { struct sha1_state *sctx = shash_desc_ctx(desc); - if (!may_use_simd() || + if (!crypto_simd_usable() || (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) return sha1_update_arm(desc, data, len); @@ -47,7 +48,7 @@ static int sha1_ce_update(struct shash_desc *desc, const u8 *data, static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!may_use_simd()) + if (!crypto_simd_usable()) return sha1_finup_arm(desc, data, len, out); kernel_neon_begin(); diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index d15e0ea2c95e..d6c95c213d42 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -19,6 +19,7 @@ */ #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <linux/init.h> #include <linux/module.h> #include <linux/mm.h> @@ -39,7 +40,7 @@ static int sha1_neon_update(struct shash_desc *desc, const u8 *data, { struct sha1_state *sctx = shash_desc_ctx(desc); - if (!may_use_simd() || + if (!crypto_simd_usable() || (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) return sha1_update_arm(desc, data, len); @@ -54,7 +55,7 @@ static int sha1_neon_update(struct shash_desc *desc, const u8 *data, static int sha1_neon_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!may_use_simd()) + if (!crypto_simd_usable()) return sha1_finup_arm(desc, data, len, out); kernel_neon_begin(); diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c index 1211a5c129fc..a47a9d4b663e 100644 --- a/arch/arm/crypto/sha2-ce-glue.c +++ b/arch/arm/crypto/sha2-ce-glue.c @@ -9,6 +9,7 @@ */ #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/sha.h> #include <crypto/sha256_base.h> #include <linux/cpufeature.h> @@ -34,7 +35,7 @@ static int sha2_ce_update(struct shash_desc *desc, const u8 *data, { struct sha256_state *sctx = shash_desc_ctx(desc); - if (!may_use_simd() || + if (!crypto_simd_usable() || (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) return crypto_sha256_arm_update(desc, data, len); @@ -49,7 +50,7 @@ static int sha2_ce_update(struct shash_desc *desc, const u8 *data, static int sha2_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!may_use_simd()) + if (!crypto_simd_usable()) return crypto_sha256_arm_finup(desc, data, len, out); kernel_neon_begin(); diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c index 1d82c6cd31a4..f3f6b1624fc3 100644 --- a/arch/arm/crypto/sha256_neon_glue.c +++ b/arch/arm/crypto/sha256_neon_glue.c @@ -15,6 +15,7 @@ */ #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <linux/cryptohash.h> #include <linux/types.h> #include <linux/string.h> @@ -34,7 +35,7 @@ static int sha256_update(struct shash_desc *desc, const u8 *data, { struct sha256_state *sctx = shash_desc_ctx(desc); - if (!may_use_simd() || + if (!crypto_simd_usable() || (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) return crypto_sha256_arm_update(desc, data, len); @@ -49,7 +50,7 @@ static int sha256_update(struct shash_desc *desc, const u8 *data, static int sha256_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!may_use_simd()) + if (!crypto_simd_usable()) return crypto_sha256_arm_finup(desc, data, len, out); kernel_neon_begin(); diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-neon-glue.c index 8a5642b41fd6..d33ab59c26c0 100644 --- a/arch/arm/crypto/sha512-neon-glue.c +++ b/arch/arm/crypto/sha512-neon-glue.c @@ -9,6 +9,7 @@ */ #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/sha.h> #include <crypto/sha512_base.h> #include <linux/crypto.h> @@ -30,7 +31,7 @@ static int sha512_neon_update(struct shash_desc *desc, const u8 *data, { struct sha512_state *sctx = shash_desc_ctx(desc); - if (!may_use_simd() || + if (!crypto_simd_usable() || (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) return sha512_arm_update(desc, data, len); @@ -45,7 +46,7 @@ static int sha512_neon_update(struct shash_desc *desc, const u8 *data, static int sha512_neon_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!may_use_simd()) + if (!crypto_simd_usable()) return sha512_arm_finup(desc, data, len, out); kernel_neon_begin(); diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index 036ea77f83bc..cb89c80800b5 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -14,6 +14,7 @@ #include <crypto/aes.h> #include <crypto/scatterwalk.h> #include <crypto/internal/aead.h> +#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <linux/module.h> @@ -109,7 +110,7 @@ static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen) static void ccm_update_mac(struct crypto_aes_ctx *key, u8 mac[], u8 const in[], u32 abytes, u32 *macp) { - if (may_use_simd()) { + if (crypto_simd_usable()) { kernel_neon_begin(); ce_aes_ccm_auth_data(mac, in, abytes, macp, key->key_enc, num_rounds(key)); @@ -255,7 +256,7 @@ static int ccm_encrypt(struct aead_request *req) err = skcipher_walk_aead_encrypt(&walk, req, false); - if (may_use_simd()) { + if (crypto_simd_usable()) { while (walk.nbytes) { u32 tail = walk.nbytes % AES_BLOCK_SIZE; @@ -313,7 +314,7 @@ static int ccm_decrypt(struct aead_request *req) err = skcipher_walk_aead_decrypt(&walk, req, false); - if (may_use_simd()) { + if (crypto_simd_usable()) { while (walk.nbytes) { u32 tail = walk.nbytes % AES_BLOCK_SIZE; diff --git a/arch/arm64/crypto/aes-ce-glue.c b/arch/arm64/crypto/aes-ce-glue.c index e6b3227bbf57..3213843fcb46 100644 --- a/arch/arm64/crypto/aes-ce-glue.c +++ b/arch/arm64/crypto/aes-ce-glue.c @@ -12,6 +12,7 @@ #include <asm/simd.h> #include <asm/unaligned.h> #include <crypto/aes.h> +#include <crypto/internal/simd.h> #include <linux/cpufeature.h> #include <linux/crypto.h> #include <linux/module.h> @@ -52,7 +53,7 @@ static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) { struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); - if (!may_use_simd()) { + if (!crypto_simd_usable()) { __aes_arm64_encrypt(ctx->key_enc, dst, src, num_rounds(ctx)); return; } @@ -66,7 +67,7 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) { struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); - if (!may_use_simd()) { + if (!crypto_simd_usable()) { __aes_arm64_decrypt(ctx->key_dec, dst, src, num_rounds(ctx)); return; } diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 1e676625ef33..f0ceb545bd1e 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -405,7 +405,7 @@ static int ctr_encrypt_sync(struct skcipher_request *req) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); - if (!may_use_simd()) + if (!crypto_simd_usable()) return aes_ctr_encrypt_fallback(ctx, req); return ctr_encrypt(req); @@ -642,7 +642,7 @@ static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, { int rounds = 6 + ctx->key_length / 4; - if (may_use_simd()) { + if (crypto_simd_usable()) { kernel_neon_begin(); aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before, enc_after); @@ -707,7 +707,7 @@ static int cbcmac_final(struct shash_desc *desc, u8 *out) struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1, 0); + mac_do_update(&tctx->key, NULL, 0, ctx->dg, (ctx->len != 0), 0); memcpy(out, ctx->dg, AES_BLOCK_SIZE); diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c index bf1b321ff4c1..02b65d9eb947 100644 --- a/arch/arm64/crypto/aes-neonbs-glue.c +++ b/arch/arm64/crypto/aes-neonbs-glue.c @@ -288,7 +288,7 @@ static int ctr_encrypt_sync(struct skcipher_request *req) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - if (!may_use_simd()) + if (!crypto_simd_usable()) return aes_ctr_encrypt_fallback(&ctx->fallback, req); return ctr_encrypt(req); @@ -304,6 +304,8 @@ static int __xts_crypt(struct skcipher_request *req, int err; err = skcipher_walk_virt(&walk, req, false); + if (err) + return err; kernel_neon_begin(); neon_aes_ecb_encrypt(walk.iv, walk.iv, ctx->twkey, ctx->key.rounds, 1); diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index cb054f51c917..82029cda2e77 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -21,6 +21,7 @@ #include <crypto/algapi.h> #include <crypto/chacha.h> +#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <linux/kernel.h> #include <linux/module.h> @@ -90,7 +91,7 @@ static int chacha_neon(struct skcipher_request *req) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - if (req->cryptlen <= CHACHA_BLOCK_SIZE || !may_use_simd()) + if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable()) return crypto_chacha_crypt(req); return chacha_neon_stream_xor(req, ctx, req->iv); @@ -104,7 +105,7 @@ static int xchacha_neon(struct skcipher_request *req) u32 state[16]; u8 real_iv[16]; - if (req->cryptlen <= CHACHA_BLOCK_SIZE || !may_use_simd()) + if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable()) return crypto_xchacha_crypt(req); crypto_chacha_init(state, ctx, req->iv); diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c index e81d5bd555c0..2e0a7d2eee24 100644 --- a/arch/arm64/crypto/crct10dif-ce-glue.c +++ b/arch/arm64/crypto/crct10dif-ce-glue.c @@ -16,6 +16,7 @@ #include <linux/string.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <asm/neon.h> #include <asm/simd.h> @@ -38,7 +39,7 @@ static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, { u16 *crc = shash_desc_ctx(desc); - if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && may_use_simd()) { + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { kernel_neon_begin(); *crc = crc_t10dif_pmull_p8(*crc, data, length); kernel_neon_end(); @@ -54,7 +55,7 @@ static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, { u16 *crc = shash_desc_ctx(desc); - if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && may_use_simd()) { + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { kernel_neon_begin(); *crc = crc_t10dif_pmull_p64(*crc, data, length); kernel_neon_end(); diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 4e69bb78ea89..b39ed99b06fb 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -17,6 +17,7 @@ #include <crypto/gf128mul.h> #include <crypto/internal/aead.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/cpufeature.h> @@ -89,7 +90,7 @@ static void ghash_do_update(int blocks, u64 dg[], const char *src, struct ghash_key const *k, const char *head)) { - if (likely(may_use_simd())) { + if (likely(crypto_simd_usable())) { kernel_neon_begin(); simd_update(blocks, dg, src, key, head); kernel_neon_end(); @@ -441,7 +442,7 @@ static int gcm_encrypt(struct aead_request *req) err = skcipher_walk_aead_encrypt(&walk, req, false); - if (likely(may_use_simd() && walk.total >= 2 * AES_BLOCK_SIZE)) { + if (likely(crypto_simd_usable() && walk.total >= 2 * AES_BLOCK_SIZE)) { u32 const *rk = NULL; kernel_neon_begin(); @@ -473,9 +474,11 @@ static int gcm_encrypt(struct aead_request *req) put_unaligned_be32(2, iv + GCM_IV_SIZE); while (walk.nbytes >= (2 * AES_BLOCK_SIZE)) { - int blocks = walk.nbytes / AES_BLOCK_SIZE; + const int blocks = + walk.nbytes / (2 * AES_BLOCK_SIZE) * 2; u8 *dst = walk.dst.virt.addr; u8 *src = walk.src.virt.addr; + int remaining = blocks; do { __aes_arm64_encrypt(ctx->aes_key.key_enc, @@ -485,9 +488,9 @@ static int gcm_encrypt(struct aead_request *req) dst += AES_BLOCK_SIZE; src += AES_BLOCK_SIZE; - } while (--blocks > 0); + } while (--remaining > 0); - ghash_do_update(walk.nbytes / AES_BLOCK_SIZE, dg, + ghash_do_update(blocks, dg, walk.dst.virt.addr, &ctx->ghash_key, NULL, pmull_ghash_update_p64); @@ -563,7 +566,7 @@ static int gcm_decrypt(struct aead_request *req) err = skcipher_walk_aead_decrypt(&walk, req, false); - if (likely(may_use_simd() && walk.total >= 2 * AES_BLOCK_SIZE)) { + if (likely(crypto_simd_usable() && walk.total >= 2 * AES_BLOCK_SIZE)) { u32 const *rk = NULL; kernel_neon_begin(); @@ -609,7 +612,7 @@ static int gcm_decrypt(struct aead_request *req) put_unaligned_be32(2, iv + GCM_IV_SIZE); while (walk.nbytes >= (2 * AES_BLOCK_SIZE)) { - int blocks = walk.nbytes / AES_BLOCK_SIZE; + int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2; u8 *dst = walk.dst.virt.addr; u8 *src = walk.src.virt.addr; diff --git a/arch/arm64/crypto/nhpoly1305-neon-glue.c b/arch/arm64/crypto/nhpoly1305-neon-glue.c index 38a589044b6c..895d3727c1fb 100644 --- a/arch/arm64/crypto/nhpoly1305-neon-glue.c +++ b/arch/arm64/crypto/nhpoly1305-neon-glue.c @@ -9,6 +9,7 @@ #include <asm/neon.h> #include <asm/simd.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/nhpoly1305.h> #include <linux/module.h> @@ -25,7 +26,7 @@ static void _nh_neon(const u32 *key, const u8 *message, size_t message_len, static int nhpoly1305_neon_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { - if (srclen < 64 || !may_use_simd()) + if (srclen < 64 || !crypto_simd_usable()) return crypto_nhpoly1305_update(desc, src, srclen); do { diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index 17fac2889f56..eaa7a8258f1c 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -12,6 +12,7 @@ #include <asm/simd.h> #include <asm/unaligned.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/sha.h> #include <crypto/sha1_base.h> #include <linux/cpufeature.h> @@ -38,7 +39,7 @@ static int sha1_ce_update(struct shash_desc *desc, const u8 *data, { struct sha1_ce_state *sctx = shash_desc_ctx(desc); - if (!may_use_simd()) + if (!crypto_simd_usable()) return crypto_sha1_update(desc, data, len); sctx->finalize = 0; @@ -56,7 +57,7 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, struct sha1_ce_state *sctx = shash_desc_ctx(desc); bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE); - if (!may_use_simd()) + if (!crypto_simd_usable()) return crypto_sha1_finup(desc, data, len, out); /* @@ -78,7 +79,7 @@ static int sha1_ce_final(struct shash_desc *desc, u8 *out) { struct sha1_ce_state *sctx = shash_desc_ctx(desc); - if (!may_use_simd()) + if (!crypto_simd_usable()) return crypto_sha1_finup(desc, NULL, 0, out); sctx->finalize = 0; diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 261f5195cab7..a725997e55f2 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -12,6 +12,7 @@ #include <asm/simd.h> #include <asm/unaligned.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/sha.h> #include <crypto/sha256_base.h> #include <linux/cpufeature.h> @@ -42,7 +43,7 @@ static int sha256_ce_update(struct shash_desc *desc, const u8 *data, { struct sha256_ce_state *sctx = shash_desc_ctx(desc); - if (!may_use_simd()) + if (!crypto_simd_usable()) return sha256_base_do_update(desc, data, len, (sha256_block_fn *)sha256_block_data_order); @@ -61,7 +62,7 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, struct sha256_ce_state *sctx = shash_desc_ctx(desc); bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE); - if (!may_use_simd()) { + if (!crypto_simd_usable()) { if (len) sha256_base_do_update(desc, data, len, (sha256_block_fn *)sha256_block_data_order); @@ -90,7 +91,7 @@ static int sha256_ce_final(struct shash_desc *desc, u8 *out) { struct sha256_ce_state *sctx = shash_desc_ctx(desc); - if (!may_use_simd()) { + if (!crypto_simd_usable()) { sha256_base_do_finalize(desc, (sha256_block_fn *)sha256_block_data_order); return sha256_base_finish(desc, out); diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c index 0cccdb9cc2c0..e62298740e31 100644 --- a/arch/arm64/crypto/sha256-glue.c +++ b/arch/arm64/crypto/sha256-glue.c @@ -14,6 +14,7 @@ #include <asm/neon.h> #include <asm/simd.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/sha.h> #include <crypto/sha256_base.h> #include <linux/cryptohash.h> @@ -89,7 +90,7 @@ static int sha256_update_neon(struct shash_desc *desc, const u8 *data, { struct sha256_state *sctx = shash_desc_ctx(desc); - if (!may_use_simd()) + if (!crypto_simd_usable()) return sha256_base_do_update(desc, data, len, (sha256_block_fn *)sha256_block_data_order); @@ -119,7 +120,7 @@ static int sha256_update_neon(struct shash_desc *desc, const u8 *data, static int sha256_finup_neon(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!may_use_simd()) { + if (!crypto_simd_usable()) { if (len) sha256_base_do_update(desc, data, len, (sha256_block_fn *)sha256_block_data_order); diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c index a336feac0f59..9a4bbfc45f40 100644 --- a/arch/arm64/crypto/sha3-ce-glue.c +++ b/arch/arm64/crypto/sha3-ce-glue.c @@ -14,6 +14,7 @@ #include <asm/simd.h> #include <asm/unaligned.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/sha3.h> #include <linux/cpufeature.h> #include <linux/crypto.h> @@ -32,7 +33,7 @@ static int sha3_update(struct shash_desc *desc, const u8 *data, struct sha3_state *sctx = shash_desc_ctx(desc); unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - if (!may_use_simd()) + if (!crypto_simd_usable()) return crypto_sha3_update(desc, data, len); if ((sctx->partial + len) >= sctx->rsiz) { @@ -76,7 +77,7 @@ static int sha3_final(struct shash_desc *desc, u8 *out) __le64 *digest = (__le64 *)out; int i; - if (!may_use_simd()) + if (!crypto_simd_usable()) return crypto_sha3_final(desc, out); sctx->buf[sctx->partial++] = 0x06; diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c index f2c5f28c622a..2369540040aa 100644 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -13,6 +13,7 @@ #include <asm/simd.h> #include <asm/unaligned.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/sha.h> #include <crypto/sha512_base.h> #include <linux/cpufeature.h> @@ -31,7 +32,7 @@ asmlinkage void sha512_block_data_order(u64 *digest, u8 const *src, int blocks); static int sha512_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - if (!may_use_simd()) + if (!crypto_simd_usable()) return sha512_base_do_update(desc, data, len, (sha512_block_fn *)sha512_block_data_order); @@ -46,7 +47,7 @@ static int sha512_ce_update(struct shash_desc *desc, const u8 *data, static int sha512_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!may_use_simd()) { + if (!crypto_simd_usable()) { if (len) sha512_base_do_update(desc, data, len, (sha512_block_fn *)sha512_block_data_order); @@ -65,7 +66,7 @@ static int sha512_ce_finup(struct shash_desc *desc, const u8 *data, static int sha512_ce_final(struct shash_desc *desc, u8 *out) { - if (!may_use_simd()) { + if (!crypto_simd_usable()) { sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_block_data_order); return sha512_base_finish(desc, out); diff --git a/arch/arm64/crypto/sm3-ce-glue.c b/arch/arm64/crypto/sm3-ce-glue.c index 88938a20d9b2..5d15533799a2 100644 --- a/arch/arm64/crypto/sm3-ce-glue.c +++ b/arch/arm64/crypto/sm3-ce-glue.c @@ -12,6 +12,7 @@ #include <asm/simd.h> #include <asm/unaligned.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/sm3.h> #include <crypto/sm3_base.h> #include <linux/cpufeature.h> @@ -28,7 +29,7 @@ asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src, static int sm3_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - if (!may_use_simd()) + if (!crypto_simd_usable()) return crypto_sm3_update(desc, data, len); kernel_neon_begin(); @@ -40,7 +41,7 @@ static int sm3_ce_update(struct shash_desc *desc, const u8 *data, static int sm3_ce_final(struct shash_desc *desc, u8 *out) { - if (!may_use_simd()) + if (!crypto_simd_usable()) return crypto_sm3_finup(desc, NULL, 0, out); kernel_neon_begin(); @@ -53,7 +54,7 @@ static int sm3_ce_final(struct shash_desc *desc, u8 *out) static int sm3_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!may_use_simd()) + if (!crypto_simd_usable()) return crypto_sm3_finup(desc, data, len, out); kernel_neon_begin(); diff --git a/arch/arm64/crypto/sm4-ce-glue.c b/arch/arm64/crypto/sm4-ce-glue.c index 0c4fc223f225..2754c875d39c 100644 --- a/arch/arm64/crypto/sm4-ce-glue.c +++ b/arch/arm64/crypto/sm4-ce-glue.c @@ -3,6 +3,7 @@ #include <asm/neon.h> #include <asm/simd.h> #include <crypto/sm4.h> +#include <crypto/internal/simd.h> #include <linux/module.h> #include <linux/cpufeature.h> #include <linux/crypto.h> @@ -20,7 +21,7 @@ static void sm4_ce_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { const struct crypto_sm4_ctx *ctx = crypto_tfm_ctx(tfm); - if (!may_use_simd()) { + if (!crypto_simd_usable()) { crypto_sm4_encrypt(tfm, out, in); } else { kernel_neon_begin(); @@ -33,7 +34,7 @@ static void sm4_ce_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { const struct crypto_sm4_ctx *ctx = crypto_tfm_ctx(tfm); - if (!may_use_simd()) { + if (!crypto_simd_usable()) { crypto_sm4_decrypt(tfm, out, in); } else { kernel_neon_begin(); diff --git a/arch/powerpc/crypto/crc32c-vpmsum_glue.c b/arch/powerpc/crypto/crc32c-vpmsum_glue.c index fd1d6c83f0c0..c4fa242dd652 100644 --- a/arch/powerpc/crypto/crc32c-vpmsum_glue.c +++ b/arch/powerpc/crypto/crc32c-vpmsum_glue.c @@ -1,10 +1,12 @@ #include <linux/crc32.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <linux/init.h> #include <linux/module.h> #include <linux/string.h> #include <linux/kernel.h> #include <linux/cpufeature.h> +#include <asm/simd.h> #include <asm/switch_to.h> #define CHKSUM_BLOCK_SIZE 1 @@ -22,7 +24,7 @@ static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len) unsigned int prealign; unsigned int tail; - if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || in_interrupt()) + if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable()) return __crc32c_le(crc, p, len); if ((unsigned long)p & VMX_ALIGN_MASK) { diff --git a/arch/powerpc/crypto/crct10dif-vpmsum_glue.c b/arch/powerpc/crypto/crct10dif-vpmsum_glue.c index 02ea277863d1..e27ff16573b5 100644 --- a/arch/powerpc/crypto/crct10dif-vpmsum_glue.c +++ b/arch/powerpc/crypto/crct10dif-vpmsum_glue.c @@ -12,11 +12,13 @@ #include <linux/crc-t10dif.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <linux/init.h> #include <linux/module.h> #include <linux/string.h> #include <linux/kernel.h> #include <linux/cpufeature.h> +#include <asm/simd.h> #include <asm/switch_to.h> #define VMX_ALIGN 16 @@ -32,7 +34,7 @@ static u16 crct10dif_vpmsum(u16 crci, unsigned char const *p, size_t len) unsigned int tail; u32 crc = crci; - if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || in_interrupt()) + if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable()) return crc_t10dif_generic(crc, p, len); if ((unsigned long)p & VMX_ALIGN_MASK) { diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild index 36bda391e549..b9f6e72bf4e5 100644 --- a/arch/powerpc/include/asm/Kbuild +++ b/arch/powerpc/include/asm/Kbuild @@ -10,3 +10,4 @@ generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += vtime.h generic-y += msi.h +generic-y += simd.h diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index 0d15383d0ff1..1f9ab24dc048 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c @@ -224,24 +224,11 @@ static int des3_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key_len) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); + int err; - if (!(crypto_memneq(key, &key[DES_KEY_SIZE], DES_KEY_SIZE) && - crypto_memneq(&key[DES_KEY_SIZE], &key[DES_KEY_SIZE * 2], - DES_KEY_SIZE)) && - (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { - tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } - - /* in fips mode, ensure k1 != k2 and k2 != k3 and k1 != k3 */ - if (fips_enabled && - !(crypto_memneq(key, &key[DES_KEY_SIZE], DES_KEY_SIZE) && - crypto_memneq(&key[DES_KEY_SIZE], &key[DES_KEY_SIZE * 2], - DES_KEY_SIZE) && - crypto_memneq(key, &key[DES_KEY_SIZE * 2], DES_KEY_SIZE))) { - tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } + err = __des3_verify_key(&tfm->crt_flags, key); + if (unlikely(err)) + return err; memcpy(ctx->key, key, key_len); return 0; diff --git a/arch/sparc/crypto/des_glue.c b/arch/sparc/crypto/des_glue.c index 4884315daff4..453a4cf5492a 100644 --- a/arch/sparc/crypto/des_glue.c +++ b/arch/sparc/crypto/des_glue.c @@ -201,18 +201,15 @@ static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); - const u32 *K = (const u32 *)key; u32 *flags = &tfm->crt_flags; u64 k1[DES_EXPKEY_WORDS / 2]; u64 k2[DES_EXPKEY_WORDS / 2]; u64 k3[DES_EXPKEY_WORDS / 2]; + int err; - if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } + err = __des3_verify_key(flags, key); + if (unlikely(err)) + return err; des_sparc64_key_expand((const u32 *)key, k1); key += DES_KEY_SIZE; diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c index 3ea71b871813..bdeee1b830be 100644 --- a/arch/x86/crypto/aegis128-aesni-glue.c +++ b/arch/x86/crypto/aegis128-aesni-glue.c @@ -11,8 +11,8 @@ * any later version. */ -#include <crypto/cryptd.h> #include <crypto/internal/aead.h> +#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/module.h> @@ -242,131 +242,35 @@ static void crypto_aegis128_aesni_exit_tfm(struct crypto_aead *aead) { } -static int cryptd_aegis128_aesni_setkey(struct crypto_aead *aead, - const u8 *key, unsigned int keylen) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setkey(&cryptd_tfm->base, key, keylen); -} - -static int cryptd_aegis128_aesni_setauthsize(struct crypto_aead *aead, - unsigned int authsize) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); -} - -static int cryptd_aegis128_aesni_encrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - aead = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - aead = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, aead); - - return crypto_aead_encrypt(req); -} - -static int cryptd_aegis128_aesni_decrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - aead = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - aead = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, aead); - - return crypto_aead_decrypt(req); -} - -static int cryptd_aegis128_aesni_init_tfm(struct crypto_aead *aead) -{ - struct cryptd_aead *cryptd_tfm; - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_tfm = cryptd_alloc_aead("__aegis128-aesni", CRYPTO_ALG_INTERNAL, - CRYPTO_ALG_INTERNAL); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - - *ctx = cryptd_tfm; - crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); - return 0; -} - -static void cryptd_aegis128_aesni_exit_tfm(struct crypto_aead *aead) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_free_aead(*ctx); -} - -static struct aead_alg crypto_aegis128_aesni_alg[] = { - { - .setkey = crypto_aegis128_aesni_setkey, - .setauthsize = crypto_aegis128_aesni_setauthsize, - .encrypt = crypto_aegis128_aesni_encrypt, - .decrypt = crypto_aegis128_aesni_decrypt, - .init = crypto_aegis128_aesni_init_tfm, - .exit = crypto_aegis128_aesni_exit_tfm, - - .ivsize = AEGIS128_NONCE_SIZE, - .maxauthsize = AEGIS128_MAX_AUTH_SIZE, - .chunksize = AEGIS128_BLOCK_SIZE, - - .base = { - .cra_flags = CRYPTO_ALG_INTERNAL, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct aegis_ctx) + - __alignof__(struct aegis_ctx), - .cra_alignmask = 0, - - .cra_name = "__aegis128", - .cra_driver_name = "__aegis128-aesni", - - .cra_module = THIS_MODULE, - } - }, { - .setkey = cryptd_aegis128_aesni_setkey, - .setauthsize = cryptd_aegis128_aesni_setauthsize, - .encrypt = cryptd_aegis128_aesni_encrypt, - .decrypt = cryptd_aegis128_aesni_decrypt, - .init = cryptd_aegis128_aesni_init_tfm, - .exit = cryptd_aegis128_aesni_exit_tfm, - - .ivsize = AEGIS128_NONCE_SIZE, - .maxauthsize = AEGIS128_MAX_AUTH_SIZE, - .chunksize = AEGIS128_BLOCK_SIZE, - - .base = { - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct cryptd_aead *), - .cra_alignmask = 0, - - .cra_priority = 400, - - .cra_name = "aegis128", - .cra_driver_name = "aegis128-aesni", - - .cra_module = THIS_MODULE, - } +static struct aead_alg crypto_aegis128_aesni_alg = { + .setkey = crypto_aegis128_aesni_setkey, + .setauthsize = crypto_aegis128_aesni_setauthsize, + .encrypt = crypto_aegis128_aesni_encrypt, + .decrypt = crypto_aegis128_aesni_decrypt, + .init = crypto_aegis128_aesni_init_tfm, + .exit = crypto_aegis128_aesni_exit_tfm, + + .ivsize = AEGIS128_NONCE_SIZE, + .maxauthsize = AEGIS128_MAX_AUTH_SIZE, + .chunksize = AEGIS128_BLOCK_SIZE, + + .base = { + .cra_flags = CRYPTO_ALG_INTERNAL, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aegis_ctx) + + __alignof__(struct aegis_ctx), + .cra_alignmask = 0, + .cra_priority = 400, + + .cra_name = "__aegis128", + .cra_driver_name = "__aegis128-aesni", + + .cra_module = THIS_MODULE, } }; +static struct simd_aead_alg *simd_alg; + static int __init crypto_aegis128_aesni_module_init(void) { if (!boot_cpu_has(X86_FEATURE_XMM2) || @@ -374,14 +278,13 @@ static int __init crypto_aegis128_aesni_module_init(void) !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; - return crypto_register_aeads(crypto_aegis128_aesni_alg, - ARRAY_SIZE(crypto_aegis128_aesni_alg)); + return simd_register_aeads_compat(&crypto_aegis128_aesni_alg, 1, + &simd_alg); } static void __exit crypto_aegis128_aesni_module_exit(void) { - crypto_unregister_aeads(crypto_aegis128_aesni_alg, - ARRAY_SIZE(crypto_aegis128_aesni_alg)); + simd_unregister_aeads(&crypto_aegis128_aesni_alg, 1, &simd_alg); } module_init(crypto_aegis128_aesni_module_init); diff --git a/arch/x86/crypto/aegis128l-aesni-glue.c b/arch/x86/crypto/aegis128l-aesni-glue.c index 1b1b39c66c5e..80d917f7e467 100644 --- a/arch/x86/crypto/aegis128l-aesni-glue.c +++ b/arch/x86/crypto/aegis128l-aesni-glue.c @@ -11,8 +11,8 @@ * any later version. */ -#include <crypto/cryptd.h> #include <crypto/internal/aead.h> +#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/module.h> @@ -242,131 +242,35 @@ static void crypto_aegis128l_aesni_exit_tfm(struct crypto_aead *aead) { } -static int cryptd_aegis128l_aesni_setkey(struct crypto_aead *aead, - const u8 *key, unsigned int keylen) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setkey(&cryptd_tfm->base, key, keylen); -} - -static int cryptd_aegis128l_aesni_setauthsize(struct crypto_aead *aead, - unsigned int authsize) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); -} - -static int cryptd_aegis128l_aesni_encrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - aead = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - aead = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, aead); - - return crypto_aead_encrypt(req); -} - -static int cryptd_aegis128l_aesni_decrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - aead = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - aead = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, aead); - - return crypto_aead_decrypt(req); -} - -static int cryptd_aegis128l_aesni_init_tfm(struct crypto_aead *aead) -{ - struct cryptd_aead *cryptd_tfm; - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_tfm = cryptd_alloc_aead("__aegis128l-aesni", CRYPTO_ALG_INTERNAL, - CRYPTO_ALG_INTERNAL); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - - *ctx = cryptd_tfm; - crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); - return 0; -} - -static void cryptd_aegis128l_aesni_exit_tfm(struct crypto_aead *aead) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_free_aead(*ctx); -} - -static struct aead_alg crypto_aegis128l_aesni_alg[] = { - { - .setkey = crypto_aegis128l_aesni_setkey, - .setauthsize = crypto_aegis128l_aesni_setauthsize, - .encrypt = crypto_aegis128l_aesni_encrypt, - .decrypt = crypto_aegis128l_aesni_decrypt, - .init = crypto_aegis128l_aesni_init_tfm, - .exit = crypto_aegis128l_aesni_exit_tfm, - - .ivsize = AEGIS128L_NONCE_SIZE, - .maxauthsize = AEGIS128L_MAX_AUTH_SIZE, - .chunksize = AEGIS128L_BLOCK_SIZE, - - .base = { - .cra_flags = CRYPTO_ALG_INTERNAL, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct aegis_ctx) + - __alignof__(struct aegis_ctx), - .cra_alignmask = 0, - - .cra_name = "__aegis128l", - .cra_driver_name = "__aegis128l-aesni", - - .cra_module = THIS_MODULE, - } - }, { - .setkey = cryptd_aegis128l_aesni_setkey, - .setauthsize = cryptd_aegis128l_aesni_setauthsize, - .encrypt = cryptd_aegis128l_aesni_encrypt, - .decrypt = cryptd_aegis128l_aesni_decrypt, - .init = cryptd_aegis128l_aesni_init_tfm, - .exit = cryptd_aegis128l_aesni_exit_tfm, - - .ivsize = AEGIS128L_NONCE_SIZE, - .maxauthsize = AEGIS128L_MAX_AUTH_SIZE, - .chunksize = AEGIS128L_BLOCK_SIZE, - - .base = { - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct cryptd_aead *), - .cra_alignmask = 0, - - .cra_priority = 400, - - .cra_name = "aegis128l", - .cra_driver_name = "aegis128l-aesni", - - .cra_module = THIS_MODULE, - } +static struct aead_alg crypto_aegis128l_aesni_alg = { + .setkey = crypto_aegis128l_aesni_setkey, + .setauthsize = crypto_aegis128l_aesni_setauthsize, + .encrypt = crypto_aegis128l_aesni_encrypt, + .decrypt = crypto_aegis128l_aesni_decrypt, + .init = crypto_aegis128l_aesni_init_tfm, + .exit = crypto_aegis128l_aesni_exit_tfm, + + .ivsize = AEGIS128L_NONCE_SIZE, + .maxauthsize = AEGIS128L_MAX_AUTH_SIZE, + .chunksize = AEGIS128L_BLOCK_SIZE, + + .base = { + .cra_flags = CRYPTO_ALG_INTERNAL, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aegis_ctx) + + __alignof__(struct aegis_ctx), + .cra_alignmask = 0, + .cra_priority = 400, + + .cra_name = "__aegis128l", + .cra_driver_name = "__aegis128l-aesni", + + .cra_module = THIS_MODULE, } }; +static struct simd_aead_alg *simd_alg; + static int __init crypto_aegis128l_aesni_module_init(void) { if (!boot_cpu_has(X86_FEATURE_XMM2) || @@ -374,14 +278,13 @@ static int __init crypto_aegis128l_aesni_module_init(void) !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; - return crypto_register_aeads(crypto_aegis128l_aesni_alg, - ARRAY_SIZE(crypto_aegis128l_aesni_alg)); + return simd_register_aeads_compat(&crypto_aegis128l_aesni_alg, 1, + &simd_alg); } static void __exit crypto_aegis128l_aesni_module_exit(void) { - crypto_unregister_aeads(crypto_aegis128l_aesni_alg, - ARRAY_SIZE(crypto_aegis128l_aesni_alg)); + simd_unregister_aeads(&crypto_aegis128l_aesni_alg, 1, &simd_alg); } module_init(crypto_aegis128l_aesni_module_init); diff --git a/arch/x86/crypto/aegis256-aesni-glue.c b/arch/x86/crypto/aegis256-aesni-glue.c index 6227ca3220a0..716eecb66bd5 100644 --- a/arch/x86/crypto/aegis256-aesni-glue.c +++ b/arch/x86/crypto/aegis256-aesni-glue.c @@ -11,8 +11,8 @@ * any later version. */ -#include <crypto/cryptd.h> #include <crypto/internal/aead.h> +#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/module.h> @@ -242,131 +242,35 @@ static void crypto_aegis256_aesni_exit_tfm(struct crypto_aead *aead) { } -static int cryptd_aegis256_aesni_setkey(struct crypto_aead *aead, - const u8 *key, unsigned int keylen) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setkey(&cryptd_tfm->base, key, keylen); -} - -static int cryptd_aegis256_aesni_setauthsize(struct crypto_aead *aead, - unsigned int authsize) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); -} - -static int cryptd_aegis256_aesni_encrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - aead = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - aead = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, aead); - - return crypto_aead_encrypt(req); -} - -static int cryptd_aegis256_aesni_decrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - aead = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - aead = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, aead); - - return crypto_aead_decrypt(req); -} - -static int cryptd_aegis256_aesni_init_tfm(struct crypto_aead *aead) -{ - struct cryptd_aead *cryptd_tfm; - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_tfm = cryptd_alloc_aead("__aegis256-aesni", CRYPTO_ALG_INTERNAL, - CRYPTO_ALG_INTERNAL); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - - *ctx = cryptd_tfm; - crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); - return 0; -} - -static void cryptd_aegis256_aesni_exit_tfm(struct crypto_aead *aead) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_free_aead(*ctx); -} - -static struct aead_alg crypto_aegis256_aesni_alg[] = { - { - .setkey = crypto_aegis256_aesni_setkey, - .setauthsize = crypto_aegis256_aesni_setauthsize, - .encrypt = crypto_aegis256_aesni_encrypt, - .decrypt = crypto_aegis256_aesni_decrypt, - .init = crypto_aegis256_aesni_init_tfm, - .exit = crypto_aegis256_aesni_exit_tfm, - - .ivsize = AEGIS256_NONCE_SIZE, - .maxauthsize = AEGIS256_MAX_AUTH_SIZE, - .chunksize = AEGIS256_BLOCK_SIZE, - - .base = { - .cra_flags = CRYPTO_ALG_INTERNAL, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct aegis_ctx) + - __alignof__(struct aegis_ctx), - .cra_alignmask = 0, - - .cra_name = "__aegis256", - .cra_driver_name = "__aegis256-aesni", - - .cra_module = THIS_MODULE, - } - }, { - .setkey = cryptd_aegis256_aesni_setkey, - .setauthsize = cryptd_aegis256_aesni_setauthsize, - .encrypt = cryptd_aegis256_aesni_encrypt, - .decrypt = cryptd_aegis256_aesni_decrypt, - .init = cryptd_aegis256_aesni_init_tfm, - .exit = cryptd_aegis256_aesni_exit_tfm, - - .ivsize = AEGIS256_NONCE_SIZE, - .maxauthsize = AEGIS256_MAX_AUTH_SIZE, - .chunksize = AEGIS256_BLOCK_SIZE, - - .base = { - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct cryptd_aead *), - .cra_alignmask = 0, - - .cra_priority = 400, - - .cra_name = "aegis256", - .cra_driver_name = "aegis256-aesni", - - .cra_module = THIS_MODULE, - } +static struct aead_alg crypto_aegis256_aesni_alg = { + .setkey = crypto_aegis256_aesni_setkey, + .setauthsize = crypto_aegis256_aesni_setauthsize, + .encrypt = crypto_aegis256_aesni_encrypt, + .decrypt = crypto_aegis256_aesni_decrypt, + .init = crypto_aegis256_aesni_init_tfm, + .exit = crypto_aegis256_aesni_exit_tfm, + + .ivsize = AEGIS256_NONCE_SIZE, + .maxauthsize = AEGIS256_MAX_AUTH_SIZE, + .chunksize = AEGIS256_BLOCK_SIZE, + + .base = { + .cra_flags = CRYPTO_ALG_INTERNAL, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aegis_ctx) + + __alignof__(struct aegis_ctx), + .cra_alignmask = 0, + .cra_priority = 400, + + .cra_name = "__aegis256", + .cra_driver_name = "__aegis256-aesni", + + .cra_module = THIS_MODULE, } }; +static struct simd_aead_alg *simd_alg; + static int __init crypto_aegis256_aesni_module_init(void) { if (!boot_cpu_has(X86_FEATURE_XMM2) || @@ -374,14 +278,13 @@ static int __init crypto_aegis256_aesni_module_init(void) !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; - return crypto_register_aeads(crypto_aegis256_aesni_alg, - ARRAY_SIZE(crypto_aegis256_aesni_alg)); + return simd_register_aeads_compat(&crypto_aegis256_aesni_alg, 1, + &simd_alg); } static void __exit crypto_aegis256_aesni_module_exit(void) { - crypto_unregister_aeads(crypto_aegis256_aesni_alg, - ARRAY_SIZE(crypto_aegis256_aesni_alg)); + simd_unregister_aeads(&crypto_aegis256_aesni_alg, 1, &simd_alg); } module_init(crypto_aegis256_aesni_module_init); diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 1e3d2102033a..21c246799aa5 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -25,14 +25,13 @@ #include <linux/err.h> #include <crypto/algapi.h> #include <crypto/aes.h> -#include <crypto/cryptd.h> #include <crypto/ctr.h> #include <crypto/b128ops.h> #include <crypto/gcm.h> #include <crypto/xts.h> #include <asm/cpu_device_id.h> -#include <asm/fpu/api.h> #include <asm/crypto/aes.h> +#include <asm/simd.h> #include <crypto/scatterwalk.h> #include <crypto/internal/aead.h> #include <crypto/internal/simd.h> @@ -333,7 +332,7 @@ static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, return -EINVAL; } - if (!irq_fpu_usable()) + if (!crypto_simd_usable()) err = crypto_aes_expand_key(ctx, in_key, key_len); else { kernel_fpu_begin(); @@ -354,7 +353,7 @@ static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); - if (!irq_fpu_usable()) + if (!crypto_simd_usable()) crypto_aes_encrypt_x86(ctx, dst, src); else { kernel_fpu_begin(); @@ -367,7 +366,7 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); - if (!irq_fpu_usable()) + if (!crypto_simd_usable()) crypto_aes_decrypt_x86(ctx, dst, src); else { kernel_fpu_begin(); @@ -643,29 +642,6 @@ static int xts_decrypt(struct skcipher_request *req) aes_ctx(ctx->raw_crypt_ctx)); } -static int rfc4106_init(struct crypto_aead *aead) -{ - struct cryptd_aead *cryptd_tfm; - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", - CRYPTO_ALG_INTERNAL, - CRYPTO_ALG_INTERNAL); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - - *ctx = cryptd_tfm; - crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); - return 0; -} - -static void rfc4106_exit(struct crypto_aead *aead) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_free_aead(*ctx); -} - static int rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) { @@ -710,15 +686,8 @@ static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); } -static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key, - unsigned int key_len) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(parent); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setkey(&cryptd_tfm->base, key, key_len); -} - +/* This is the Integrity Check Value (aka the authentication tag) length and can + * be 8, 12 or 16 bytes long. */ static int common_rfc4106_set_authsize(struct crypto_aead *aead, unsigned int authsize) { @@ -734,17 +703,6 @@ static int common_rfc4106_set_authsize(struct crypto_aead *aead, return 0; } -/* This is the Integrity Check Value (aka the authentication tag length and can - * be 8, 12 or 16 bytes long. */ -static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent, - unsigned int authsize) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(parent); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); -} - static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, unsigned int authsize) { @@ -964,38 +922,6 @@ static int helper_rfc4106_decrypt(struct aead_request *req) return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, aes_ctx); } - -static int gcmaes_wrapper_encrypt(struct aead_request *req) -{ - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(tfm); - struct cryptd_aead *cryptd_tfm = *ctx; - - tfm = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - tfm = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, tfm); - - return crypto_aead_encrypt(req); -} - -static int gcmaes_wrapper_decrypt(struct aead_request *req) -{ - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(tfm); - struct cryptd_aead *cryptd_tfm = *ctx; - - tfm = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - tfm = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, tfm); - - return crypto_aead_decrypt(req); -} #endif static struct crypto_alg aesni_algs[] = { { @@ -1148,31 +1074,7 @@ static int generic_gcmaes_decrypt(struct aead_request *req) aes_ctx); } -static int generic_gcmaes_init(struct crypto_aead *aead) -{ - struct cryptd_aead *cryptd_tfm; - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni", - CRYPTO_ALG_INTERNAL, - CRYPTO_ALG_INTERNAL); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - - *ctx = cryptd_tfm; - crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); - - return 0; -} - -static void generic_gcmaes_exit(struct crypto_aead *aead) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_free_aead(*ctx); -} - -static struct aead_alg aesni_aead_algs[] = { { +static struct aead_alg aesni_aeads[] = { { .setkey = common_rfc4106_set_key, .setauthsize = common_rfc4106_set_authsize, .encrypt = helper_rfc4106_encrypt, @@ -1180,8 +1082,9 @@ static struct aead_alg aesni_aead_algs[] = { { .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = 16, .base = { - .cra_name = "__gcm-aes-aesni", - .cra_driver_name = "__driver-gcm-aes-aesni", + .cra_name = "__rfc4106(gcm(aes))", + .cra_driver_name = "__rfc4106-gcm-aesni", + .cra_priority = 400, .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx), @@ -1189,24 +1092,6 @@ static struct aead_alg aesni_aead_algs[] = { { .cra_module = THIS_MODULE, }, }, { - .init = rfc4106_init, - .exit = rfc4106_exit, - .setkey = gcmaes_wrapper_set_key, - .setauthsize = gcmaes_wrapper_set_authsize, - .encrypt = gcmaes_wrapper_encrypt, - .decrypt = gcmaes_wrapper_decrypt, - .ivsize = GCM_RFC4106_IV_SIZE, - .maxauthsize = 16, - .base = { - .cra_name = "rfc4106(gcm(aes))", - .cra_driver_name = "rfc4106-gcm-aesni", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct cryptd_aead *), - .cra_module = THIS_MODULE, - }, -}, { .setkey = generic_gcmaes_set_key, .setauthsize = generic_gcmaes_set_authsize, .encrypt = generic_gcmaes_encrypt, @@ -1214,38 +1099,21 @@ static struct aead_alg aesni_aead_algs[] = { { .ivsize = GCM_AES_IV_SIZE, .maxauthsize = 16, .base = { - .cra_name = "__generic-gcm-aes-aesni", - .cra_driver_name = "__driver-generic-gcm-aes-aesni", - .cra_priority = 0, + .cra_name = "__gcm(aes)", + .cra_driver_name = "__generic-gcm-aesni", + .cra_priority = 400, .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), .cra_alignmask = AESNI_ALIGN - 1, .cra_module = THIS_MODULE, }, -}, { - .init = generic_gcmaes_init, - .exit = generic_gcmaes_exit, - .setkey = gcmaes_wrapper_set_key, - .setauthsize = gcmaes_wrapper_set_authsize, - .encrypt = gcmaes_wrapper_encrypt, - .decrypt = gcmaes_wrapper_decrypt, - .ivsize = GCM_AES_IV_SIZE, - .maxauthsize = 16, - .base = { - .cra_name = "gcm(aes)", - .cra_driver_name = "generic-gcm-aesni", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct cryptd_aead *), - .cra_module = THIS_MODULE, - }, } }; #else -static struct aead_alg aesni_aead_algs[0]; +static struct aead_alg aesni_aeads[0]; #endif +static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)]; static const struct x86_cpu_id aesni_cpu_id[] = { X86_FEATURE_MATCH(X86_FEATURE_AES), @@ -1253,23 +1121,9 @@ static const struct x86_cpu_id aesni_cpu_id[] = { }; MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); -static void aesni_free_simds(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) && - aesni_simd_skciphers[i]; i++) - simd_skcipher_free(aesni_simd_skciphers[i]); -} - static int __init aesni_init(void) { - struct simd_skcipher_alg *simd; - const char *basename; - const char *algname; - const char *drvname; int err; - int i; if (!x86_match_cpu(aesni_cpu_id)) return -ENODEV; @@ -1304,36 +1158,22 @@ static int __init aesni_init(void) if (err) return err; - err = crypto_register_skciphers(aesni_skciphers, - ARRAY_SIZE(aesni_skciphers)); + err = simd_register_skciphers_compat(aesni_skciphers, + ARRAY_SIZE(aesni_skciphers), + aesni_simd_skciphers); if (err) goto unregister_algs; - err = crypto_register_aeads(aesni_aead_algs, - ARRAY_SIZE(aesni_aead_algs)); + err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads), + aesni_simd_aeads); if (err) goto unregister_skciphers; - for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) { - algname = aesni_skciphers[i].base.cra_name + 2; - drvname = aesni_skciphers[i].base.cra_driver_name + 2; - basename = aesni_skciphers[i].base.cra_driver_name; - simd = simd_skcipher_create_compat(algname, drvname, basename); - err = PTR_ERR(simd); - if (IS_ERR(simd)) - goto unregister_simds; - - aesni_simd_skciphers[i] = simd; - } - return 0; -unregister_simds: - aesni_free_simds(); - crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs)); unregister_skciphers: - crypto_unregister_skciphers(aesni_skciphers, - ARRAY_SIZE(aesni_skciphers)); + simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), + aesni_simd_skciphers); unregister_algs: crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); return err; @@ -1341,10 +1181,10 @@ unregister_algs: static void __exit aesni_exit(void) { - aesni_free_simds(); - crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs)); - crypto_unregister_skciphers(aesni_skciphers, - ARRAY_SIZE(aesni_skciphers)); + simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads), + aesni_simd_aeads); + simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), + aesni_simd_skciphers); crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); } diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index 45c1c4143176..4967ad620775 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -12,10 +12,10 @@ #include <crypto/algapi.h> #include <crypto/chacha.h> +#include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <linux/kernel.h> #include <linux/module.h> -#include <asm/fpu/api.h> #include <asm/simd.h> #define CHACHA_STATE_ALIGN 16 @@ -170,7 +170,7 @@ static int chacha_simd(struct skcipher_request *req) struct skcipher_walk walk; int err; - if (req->cryptlen <= CHACHA_BLOCK_SIZE || !irq_fpu_usable()) + if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable()) return crypto_chacha_crypt(req); err = skcipher_walk_virt(&walk, req, true); @@ -193,7 +193,7 @@ static int xchacha_simd(struct skcipher_request *req) u8 real_iv[16]; int err; - if (req->cryptlen <= CHACHA_BLOCK_SIZE || !irq_fpu_usable()) + if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable()) return crypto_xchacha_crypt(req); err = skcipher_walk_virt(&walk, req, true); diff --git a/arch/x86/crypto/crc32-pclmul_glue.c b/arch/x86/crypto/crc32-pclmul_glue.c index c8d9cdacbf10..cb4ab6645106 100644 --- a/arch/x86/crypto/crc32-pclmul_glue.c +++ b/arch/x86/crypto/crc32-pclmul_glue.c @@ -32,10 +32,11 @@ #include <linux/kernel.h> #include <linux/crc32.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <asm/cpufeatures.h> #include <asm/cpu_device_id.h> -#include <asm/fpu/api.h> +#include <asm/simd.h> #define CHKSUM_BLOCK_SIZE 1 #define CHKSUM_DIGEST_SIZE 4 @@ -54,7 +55,7 @@ static u32 __attribute__((pure)) unsigned int iremainder; unsigned int prealign; - if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !irq_fpu_usable()) + if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !crypto_simd_usable()) return crc32_le(crc, p, len); if ((long)p & SCALE_F_MASK) { diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c index 5773e1161072..a58fe217c856 100644 --- a/arch/x86/crypto/crc32c-intel_glue.c +++ b/arch/x86/crypto/crc32c-intel_glue.c @@ -29,10 +29,11 @@ #include <linux/string.h> #include <linux/kernel.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <asm/cpufeatures.h> #include <asm/cpu_device_id.h> -#include <asm/fpu/internal.h> +#include <asm/simd.h> #define CHKSUM_BLOCK_SIZE 1 #define CHKSUM_DIGEST_SIZE 4 @@ -177,7 +178,7 @@ static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data, * use faster PCL version if datasize is large enough to * overcome kernel fpu state save/restore overhead */ - if (len >= CRC32C_PCL_BREAKEVEN && irq_fpu_usable()) { + if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) { kernel_fpu_begin(); *crcp = crc_pcl(data, len, *crcp); kernel_fpu_end(); @@ -189,7 +190,7 @@ static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data, static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out) { - if (len >= CRC32C_PCL_BREAKEVEN && irq_fpu_usable()) { + if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) { kernel_fpu_begin(); *(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp)); kernel_fpu_end(); diff --git a/arch/x86/crypto/crct10dif-pclmul_glue.c b/arch/x86/crypto/crct10dif-pclmul_glue.c index 0e785c0b2354..3c81e15b0873 100644 --- a/arch/x86/crypto/crct10dif-pclmul_glue.c +++ b/arch/x86/crypto/crct10dif-pclmul_glue.c @@ -26,12 +26,13 @@ #include <linux/module.h> #include <linux/crc-t10dif.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <linux/init.h> #include <linux/string.h> #include <linux/kernel.h> -#include <asm/fpu/api.h> #include <asm/cpufeatures.h> #include <asm/cpu_device_id.h> +#include <asm/simd.h> asmlinkage u16 crc_t10dif_pcl(u16 init_crc, const u8 *buf, size_t len); @@ -53,7 +54,7 @@ static int chksum_update(struct shash_desc *desc, const u8 *data, { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); - if (length >= 16 && irq_fpu_usable()) { + if (length >= 16 && crypto_simd_usable()) { kernel_fpu_begin(); ctx->crc = crc_t10dif_pcl(ctx->crc, data, length); kernel_fpu_end(); @@ -70,15 +71,14 @@ static int chksum_final(struct shash_desc *desc, u8 *out) return 0; } -static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len, - u8 *out) +static int __chksum_finup(__u16 crc, const u8 *data, unsigned int len, u8 *out) { - if (len >= 16 && irq_fpu_usable()) { + if (len >= 16 && crypto_simd_usable()) { kernel_fpu_begin(); - *(__u16 *)out = crc_t10dif_pcl(*crcp, data, len); + *(__u16 *)out = crc_t10dif_pcl(crc, data, len); kernel_fpu_end(); } else - *(__u16 *)out = crc_t10dif_generic(*crcp, data, len); + *(__u16 *)out = crc_t10dif_generic(crc, data, len); return 0; } @@ -87,15 +87,13 @@ static int chksum_finup(struct shash_desc *desc, const u8 *data, { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); - return __chksum_finup(&ctx->crc, data, len, out); + return __chksum_finup(ctx->crc, data, len, out); } static int chksum_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) { - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); - - return __chksum_finup(&ctx->crc, data, length, out); + return __chksum_finup(0, data, length, out); } static struct shash_alg alg = { diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c index 3582ae885ee1..e3f3e6fd9d65 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c @@ -19,8 +19,9 @@ #include <crypto/cryptd.h> #include <crypto/gf128mul.h> #include <crypto/internal/hash.h> -#include <asm/fpu/api.h> +#include <crypto/internal/simd.h> #include <asm/cpu_device_id.h> +#include <asm/simd.h> #define GHASH_BLOCK_SIZE 16 #define GHASH_DIGEST_SIZE 16 @@ -171,7 +172,6 @@ static int ghash_async_init(struct ahash_request *req) struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); desc->tfm = child; - desc->flags = req->base.flags; return crypto_shash_init(desc); } @@ -182,7 +182,7 @@ static int ghash_async_update(struct ahash_request *req) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!irq_fpu_usable() || + if (!crypto_simd_usable() || (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); @@ -200,7 +200,7 @@ static int ghash_async_final(struct ahash_request *req) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!irq_fpu_usable() || + if (!crypto_simd_usable() || (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); @@ -241,7 +241,7 @@ static int ghash_async_digest(struct ahash_request *req) struct ahash_request *cryptd_req = ahash_request_ctx(req); struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; - if (!irq_fpu_usable() || + if (!crypto_simd_usable() || (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { memcpy(cryptd_req, req, sizeof(*req)); ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); @@ -251,7 +251,6 @@ static int ghash_async_digest(struct ahash_request *req) struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); desc->tfm = child; - desc->flags = req->base.flags; return shash_ahash_digest(req, desc); } } diff --git a/arch/x86/crypto/morus1280-avx2-glue.c b/arch/x86/crypto/morus1280-avx2-glue.c index 6634907d6ccd..679627a2a824 100644 --- a/arch/x86/crypto/morus1280-avx2-glue.c +++ b/arch/x86/crypto/morus1280-avx2-glue.c @@ -12,6 +12,7 @@ */ #include <crypto/internal/aead.h> +#include <crypto/internal/simd.h> #include <crypto/morus1280_glue.h> #include <linux/module.h> #include <asm/fpu/api.h> @@ -35,7 +36,9 @@ asmlinkage void crypto_morus1280_avx2_dec_tail(void *state, const void *src, asmlinkage void crypto_morus1280_avx2_final(void *state, void *tag_xor, u64 assoclen, u64 cryptlen); -MORUS1280_DECLARE_ALGS(avx2, "morus1280-avx2", 400); +MORUS1280_DECLARE_ALG(avx2, "morus1280-avx2", 400); + +static struct simd_aead_alg *simd_alg; static int __init crypto_morus1280_avx2_module_init(void) { @@ -44,14 +47,13 @@ static int __init crypto_morus1280_avx2_module_init(void) !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) return -ENODEV; - return crypto_register_aeads(crypto_morus1280_avx2_algs, - ARRAY_SIZE(crypto_morus1280_avx2_algs)); + return simd_register_aeads_compat(&crypto_morus1280_avx2_alg, 1, + &simd_alg); } static void __exit crypto_morus1280_avx2_module_exit(void) { - crypto_unregister_aeads(crypto_morus1280_avx2_algs, - ARRAY_SIZE(crypto_morus1280_avx2_algs)); + simd_unregister_aeads(&crypto_morus1280_avx2_alg, 1, &simd_alg); } module_init(crypto_morus1280_avx2_module_init); diff --git a/arch/x86/crypto/morus1280-sse2-glue.c b/arch/x86/crypto/morus1280-sse2-glue.c index f40244eaf14d..c35c0638d0bb 100644 --- a/arch/x86/crypto/morus1280-sse2-glue.c +++ b/arch/x86/crypto/morus1280-sse2-glue.c @@ -12,6 +12,7 @@ */ #include <crypto/internal/aead.h> +#include <crypto/internal/simd.h> #include <crypto/morus1280_glue.h> #include <linux/module.h> #include <asm/fpu/api.h> @@ -35,7 +36,9 @@ asmlinkage void crypto_morus1280_sse2_dec_tail(void *state, const void *src, asmlinkage void crypto_morus1280_sse2_final(void *state, void *tag_xor, u64 assoclen, u64 cryptlen); -MORUS1280_DECLARE_ALGS(sse2, "morus1280-sse2", 350); +MORUS1280_DECLARE_ALG(sse2, "morus1280-sse2", 350); + +static struct simd_aead_alg *simd_alg; static int __init crypto_morus1280_sse2_module_init(void) { @@ -43,14 +46,13 @@ static int __init crypto_morus1280_sse2_module_init(void) !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; - return crypto_register_aeads(crypto_morus1280_sse2_algs, - ARRAY_SIZE(crypto_morus1280_sse2_algs)); + return simd_register_aeads_compat(&crypto_morus1280_sse2_alg, 1, + &simd_alg); } static void __exit crypto_morus1280_sse2_module_exit(void) { - crypto_unregister_aeads(crypto_morus1280_sse2_algs, - ARRAY_SIZE(crypto_morus1280_sse2_algs)); + simd_unregister_aeads(&crypto_morus1280_sse2_alg, 1, &simd_alg); } module_init(crypto_morus1280_sse2_module_init); diff --git a/arch/x86/crypto/morus1280_glue.c b/arch/x86/crypto/morus1280_glue.c index 7e600f8bcdad..30fc1bd98ec3 100644 --- a/arch/x86/crypto/morus1280_glue.c +++ b/arch/x86/crypto/morus1280_glue.c @@ -11,7 +11,6 @@ * any later version. */ -#include <crypto/cryptd.h> #include <crypto/internal/aead.h> #include <crypto/internal/skcipher.h> #include <crypto/morus1280_glue.h> @@ -205,90 +204,6 @@ void crypto_morus1280_glue_init_ops(struct crypto_aead *aead, } EXPORT_SYMBOL_GPL(crypto_morus1280_glue_init_ops); -int cryptd_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int keylen) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setkey(&cryptd_tfm->base, key, keylen); -} -EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_setkey); - -int cryptd_morus1280_glue_setauthsize(struct crypto_aead *aead, - unsigned int authsize) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); -} -EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_setauthsize); - -int cryptd_morus1280_glue_encrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - aead = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - aead = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, aead); - - return crypto_aead_encrypt(req); -} -EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_encrypt); - -int cryptd_morus1280_glue_decrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - aead = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - aead = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, aead); - - return crypto_aead_decrypt(req); -} -EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_decrypt); - -int cryptd_morus1280_glue_init_tfm(struct crypto_aead *aead) -{ - struct cryptd_aead *cryptd_tfm; - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - const char *name = crypto_aead_alg(aead)->base.cra_driver_name; - char internal_name[CRYPTO_MAX_ALG_NAME]; - - if (snprintf(internal_name, CRYPTO_MAX_ALG_NAME, "__%s", name) - >= CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - cryptd_tfm = cryptd_alloc_aead(internal_name, CRYPTO_ALG_INTERNAL, - CRYPTO_ALG_INTERNAL); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - - *ctx = cryptd_tfm; - crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); - return 0; -} -EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_init_tfm); - -void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_free_aead(*ctx); -} -EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_exit_tfm); - MODULE_LICENSE("GPL"); MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>"); MODULE_DESCRIPTION("MORUS-1280 AEAD mode -- glue for x86 optimizations"); diff --git a/arch/x86/crypto/morus640-sse2-glue.c b/arch/x86/crypto/morus640-sse2-glue.c index 9afaf8f8565a..32da56b3bdad 100644 --- a/arch/x86/crypto/morus640-sse2-glue.c +++ b/arch/x86/crypto/morus640-sse2-glue.c @@ -12,6 +12,7 @@ */ #include <crypto/internal/aead.h> +#include <crypto/internal/simd.h> #include <crypto/morus640_glue.h> #include <linux/module.h> #include <asm/fpu/api.h> @@ -35,7 +36,9 @@ asmlinkage void crypto_morus640_sse2_dec_tail(void *state, const void *src, asmlinkage void crypto_morus640_sse2_final(void *state, void *tag_xor, u64 assoclen, u64 cryptlen); -MORUS640_DECLARE_ALGS(sse2, "morus640-sse2", 400); +MORUS640_DECLARE_ALG(sse2, "morus640-sse2", 400); + +static struct simd_aead_alg *simd_alg; static int __init crypto_morus640_sse2_module_init(void) { @@ -43,14 +46,13 @@ static int __init crypto_morus640_sse2_module_init(void) !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; - return crypto_register_aeads(crypto_morus640_sse2_algs, - ARRAY_SIZE(crypto_morus640_sse2_algs)); + return simd_register_aeads_compat(&crypto_morus640_sse2_alg, 1, + &simd_alg); } static void __exit crypto_morus640_sse2_module_exit(void) { - crypto_unregister_aeads(crypto_morus640_sse2_algs, - ARRAY_SIZE(crypto_morus640_sse2_algs)); + simd_unregister_aeads(&crypto_morus640_sse2_alg, 1, &simd_alg); } module_init(crypto_morus640_sse2_module_init); diff --git a/arch/x86/crypto/morus640_glue.c b/arch/x86/crypto/morus640_glue.c index cb3a81732016..1dea33d84426 100644 --- a/arch/x86/crypto/morus640_glue.c +++ b/arch/x86/crypto/morus640_glue.c @@ -11,7 +11,6 @@ * any later version. */ -#include <crypto/cryptd.h> #include <crypto/internal/aead.h> #include <crypto/internal/skcipher.h> #include <crypto/morus640_glue.h> @@ -200,90 +199,6 @@ void crypto_morus640_glue_init_ops(struct crypto_aead *aead, } EXPORT_SYMBOL_GPL(crypto_morus640_glue_init_ops); -int cryptd_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int keylen) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setkey(&cryptd_tfm->base, key, keylen); -} -EXPORT_SYMBOL_GPL(cryptd_morus640_glue_setkey); - -int cryptd_morus640_glue_setauthsize(struct crypto_aead *aead, - unsigned int authsize) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); -} -EXPORT_SYMBOL_GPL(cryptd_morus640_glue_setauthsize); - -int cryptd_morus640_glue_encrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - aead = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - aead = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, aead); - - return crypto_aead_encrypt(req); -} -EXPORT_SYMBOL_GPL(cryptd_morus640_glue_encrypt); - -int cryptd_morus640_glue_decrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - struct cryptd_aead *cryptd_tfm = *ctx; - - aead = &cryptd_tfm->base; - if (irq_fpu_usable() && (!in_atomic() || - !cryptd_aead_queued(cryptd_tfm))) - aead = cryptd_aead_child(cryptd_tfm); - - aead_request_set_tfm(req, aead); - - return crypto_aead_decrypt(req); -} -EXPORT_SYMBOL_GPL(cryptd_morus640_glue_decrypt); - -int cryptd_morus640_glue_init_tfm(struct crypto_aead *aead) -{ - struct cryptd_aead *cryptd_tfm; - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - const char *name = crypto_aead_alg(aead)->base.cra_driver_name; - char internal_name[CRYPTO_MAX_ALG_NAME]; - - if (snprintf(internal_name, CRYPTO_MAX_ALG_NAME, "__%s", name) - >= CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - cryptd_tfm = cryptd_alloc_aead(internal_name, CRYPTO_ALG_INTERNAL, - CRYPTO_ALG_INTERNAL); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - - *ctx = cryptd_tfm; - crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); - return 0; -} -EXPORT_SYMBOL_GPL(cryptd_morus640_glue_init_tfm); - -void cryptd_morus640_glue_exit_tfm(struct crypto_aead *aead) -{ - struct cryptd_aead **ctx = crypto_aead_ctx(aead); - - cryptd_free_aead(*ctx); -} -EXPORT_SYMBOL_GPL(cryptd_morus640_glue_exit_tfm); - MODULE_LICENSE("GPL"); MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>"); MODULE_DESCRIPTION("MORUS-640 AEAD mode -- glue for x86 optimizations"); diff --git a/arch/x86/crypto/nhpoly1305-avx2-glue.c b/arch/x86/crypto/nhpoly1305-avx2-glue.c index 20d815ea4b6a..f7567cbd35b6 100644 --- a/arch/x86/crypto/nhpoly1305-avx2-glue.c +++ b/arch/x86/crypto/nhpoly1305-avx2-glue.c @@ -7,9 +7,10 @@ */ #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/nhpoly1305.h> #include <linux/module.h> -#include <asm/fpu/api.h> +#include <asm/simd.h> asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len, u8 hash[NH_HASH_BYTES]); @@ -24,7 +25,7 @@ static void _nh_avx2(const u32 *key, const u8 *message, size_t message_len, static int nhpoly1305_avx2_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { - if (srclen < 64 || !irq_fpu_usable()) + if (srclen < 64 || !crypto_simd_usable()) return crypto_nhpoly1305_update(desc, src, srclen); do { diff --git a/arch/x86/crypto/nhpoly1305-sse2-glue.c b/arch/x86/crypto/nhpoly1305-sse2-glue.c index ed68d164ce14..a661ede3b5cf 100644 --- a/arch/x86/crypto/nhpoly1305-sse2-glue.c +++ b/arch/x86/crypto/nhpoly1305-sse2-glue.c @@ -7,9 +7,10 @@ */ #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/nhpoly1305.h> #include <linux/module.h> -#include <asm/fpu/api.h> +#include <asm/simd.h> asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len, u8 hash[NH_HASH_BYTES]); @@ -24,7 +25,7 @@ static void _nh_sse2(const u32 *key, const u8 *message, size_t message_len, static int nhpoly1305_sse2_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { - if (srclen < 64 || !irq_fpu_usable()) + if (srclen < 64 || !crypto_simd_usable()) return crypto_nhpoly1305_update(desc, src, srclen); do { diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index 88cc01506c84..6eb65b237b3c 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -11,11 +11,11 @@ #include <crypto/algapi.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/poly1305.h> #include <linux/crypto.h> #include <linux/kernel.h> #include <linux/module.h> -#include <asm/fpu/api.h> #include <asm/simd.h> struct poly1305_simd_desc_ctx { @@ -126,7 +126,7 @@ static int poly1305_simd_update(struct shash_desc *desc, unsigned int bytes; /* kernel_fpu_begin/end is costly, use fallback for small updates */ - if (srclen <= 288 || !may_use_simd()) + if (srclen <= 288 || !crypto_simd_usable()) return crypto_poly1305_update(desc, src, srclen); kernel_fpu_begin(); diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index 7391c7de72c7..42f177afc33a 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -22,6 +22,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <linux/init.h> #include <linux/module.h> #include <linux/mm.h> @@ -29,7 +30,7 @@ #include <linux/types.h> #include <crypto/sha.h> #include <crypto/sha1_base.h> -#include <asm/fpu/api.h> +#include <asm/simd.h> typedef void (sha1_transform_fn)(u32 *digest, const char *data, unsigned int rounds); @@ -39,7 +40,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, { struct sha1_state *sctx = shash_desc_ctx(desc); - if (!irq_fpu_usable() || + if (!crypto_simd_usable() || (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) return crypto_sha1_update(desc, data, len); @@ -57,7 +58,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, static int sha1_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out, sha1_transform_fn *sha1_xform) { - if (!irq_fpu_usable()) + if (!crypto_simd_usable()) return crypto_sha1_finup(desc, data, len, out); kernel_fpu_begin(); diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index 773a873d2b28..73867da3cbee 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -30,6 +30,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <linux/init.h> #include <linux/module.h> #include <linux/mm.h> @@ -37,8 +38,8 @@ #include <linux/types.h> #include <crypto/sha.h> #include <crypto/sha256_base.h> -#include <asm/fpu/api.h> #include <linux/string.h> +#include <asm/simd.h> asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data, u64 rounds); @@ -49,7 +50,7 @@ static int sha256_update(struct shash_desc *desc, const u8 *data, { struct sha256_state *sctx = shash_desc_ctx(desc); - if (!irq_fpu_usable() || + if (!crypto_simd_usable() || (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) return crypto_sha256_update(desc, data, len); @@ -67,7 +68,7 @@ static int sha256_update(struct shash_desc *desc, const u8 *data, static int sha256_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out, sha256_transform_fn *sha256_xform) { - if (!irq_fpu_usable()) + if (!crypto_simd_usable()) return crypto_sha256_finup(desc, data, len, out); kernel_fpu_begin(); diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index f1b811b60ba6..458356a3f124 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -28,16 +28,16 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <linux/init.h> #include <linux/module.h> #include <linux/mm.h> #include <linux/cryptohash.h> +#include <linux/string.h> #include <linux/types.h> #include <crypto/sha.h> #include <crypto/sha512_base.h> -#include <asm/fpu/api.h> - -#include <linux/string.h> +#include <asm/simd.h> asmlinkage void sha512_transform_ssse3(u64 *digest, const char *data, u64 rounds); @@ -49,7 +49,7 @@ static int sha512_update(struct shash_desc *desc, const u8 *data, { struct sha512_state *sctx = shash_desc_ctx(desc); - if (!irq_fpu_usable() || + if (!crypto_simd_usable() || (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) return crypto_sha512_update(desc, data, len); @@ -67,7 +67,7 @@ static int sha512_update(struct shash_desc *desc, const u8 *data, static int sha512_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out, sha512_transform_fn *sha512_xform) { - if (!irq_fpu_usable()) + if (!crypto_simd_usable()) return crypto_sha512_finup(desc, data, len, out); kernel_fpu_begin(); diff --git a/arch/x86/power/hibernate.c b/arch/x86/power/hibernate.c index bcddf09b5aa3..4845b8c7be7f 100644 --- a/arch/x86/power/hibernate.c +++ b/arch/x86/power/hibernate.c @@ -90,7 +90,6 @@ static int get_e820_md5(struct e820_table *table, void *buf) } desc->tfm = tfm; - desc->flags = 0; size = offsetof(struct e820_table, entries) + sizeof(struct e820_entry) * table->nr_entries; diff --git a/crypto/842.c b/crypto/842.c index bc26dc942821..5f98393b65d1 100644 --- a/crypto/842.c +++ b/crypto/842.c @@ -144,7 +144,7 @@ static int __init crypto842_mod_init(void) return ret; } -module_init(crypto842_mod_init); +subsys_initcall(crypto842_mod_init); static void __exit crypto842_mod_exit(void) { diff --git a/crypto/Kconfig b/crypto/Kconfig index bbab6bf33519..3d056e7da65f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -27,8 +27,8 @@ config CRYPTO_FIPS depends on (CRYPTO_ANSI_CPRNG || CRYPTO_DRBG) && !CRYPTO_MANAGER_DISABLE_TESTS depends on (MODULE_SIG || !MODULES) help - This options enables the fips boot option which is - required if you want to system to operate in a FIPS 200 + This option enables the fips boot option which is + required if you want the system to operate in a FIPS 200 certification. You should say no unless you know what this is. @@ -113,29 +113,6 @@ config CRYPTO_ACOMP select CRYPTO_ALGAPI select CRYPTO_ACOMP2 -config CRYPTO_RSA - tristate "RSA algorithm" - select CRYPTO_AKCIPHER - select CRYPTO_MANAGER - select MPILIB - select ASN1 - help - Generic implementation of the RSA public key algorithm. - -config CRYPTO_DH - tristate "Diffie-Hellman algorithm" - select CRYPTO_KPP - select MPILIB - help - Generic implementation of the Diffie-Hellman algorithm. - -config CRYPTO_ECDH - tristate "ECDH algorithm" - select CRYPTO_KPP - select CRYPTO_RNG_DEFAULT - help - Generic implementation of the ECDH algorithm - config CRYPTO_MANAGER tristate "Cryptographic algorithm manager" select CRYPTO_MANAGER2 @@ -253,6 +230,48 @@ config CRYPTO_GLUE_HELPER_X86 config CRYPTO_ENGINE tristate +comment "Public-key cryptography" + +config CRYPTO_RSA + tristate "RSA algorithm" + select CRYPTO_AKCIPHER + select CRYPTO_MANAGER + select MPILIB + select ASN1 + help + Generic implementation of the RSA public key algorithm. + +config CRYPTO_DH + tristate "Diffie-Hellman algorithm" + select CRYPTO_KPP + select MPILIB + help + Generic implementation of the Diffie-Hellman algorithm. + +config CRYPTO_ECC + tristate + +config CRYPTO_ECDH + tristate "ECDH algorithm" + select CRYPTO_ECC + select CRYPTO_KPP + select CRYPTO_RNG_DEFAULT + help + Generic implementation of the ECDH algorithm + +config CRYPTO_ECRDSA + tristate "EC-RDSA (GOST 34.10) algorithm" + select CRYPTO_ECC + select CRYPTO_AKCIPHER + select CRYPTO_STREEBOG + select OID_REGISTRY + select ASN1 + help + Elliptic Curve Russian Digital Signature Algorithm (GOST R 34.10-2012, + RFC 7091, ISO/IEC 14888-3:2018) is one of the Russian cryptographic + standard algorithms (called GOST algorithms). Only signature verification + is implemented. + comment "Authenticated Encryption with Associated Data" config CRYPTO_CCM @@ -310,25 +329,25 @@ config CRYPTO_AEGIS128_AESNI_SSE2 tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)" depends on X86 && 64BIT select CRYPTO_AEAD - select CRYPTO_CRYPTD + select CRYPTO_SIMD help - AESNI+SSE2 implementation of the AEGSI-128 dedicated AEAD algorithm. + AESNI+SSE2 implementation of the AEGIS-128 dedicated AEAD algorithm. config CRYPTO_AEGIS128L_AESNI_SSE2 tristate "AEGIS-128L AEAD algorithm (x86_64 AESNI+SSE2 implementation)" depends on X86 && 64BIT select CRYPTO_AEAD - select CRYPTO_CRYPTD + select CRYPTO_SIMD help - AESNI+SSE2 implementation of the AEGSI-128L dedicated AEAD algorithm. + AESNI+SSE2 implementation of the AEGIS-128L dedicated AEAD algorithm. config CRYPTO_AEGIS256_AESNI_SSE2 tristate "AEGIS-256 AEAD algorithm (x86_64 AESNI+SSE2 implementation)" depends on X86 && 64BIT select CRYPTO_AEAD - select CRYPTO_CRYPTD + select CRYPTO_SIMD help - AESNI+SSE2 implementation of the AEGSI-256 dedicated AEAD algorithm. + AESNI+SSE2 implementation of the AEGIS-256 dedicated AEAD algorithm. config CRYPTO_MORUS640 tristate "MORUS-640 AEAD algorithm" @@ -340,7 +359,7 @@ config CRYPTO_MORUS640_GLUE tristate depends on X86 select CRYPTO_AEAD - select CRYPTO_CRYPTD + select CRYPTO_SIMD help Common glue for SIMD optimizations of the MORUS-640 dedicated AEAD algorithm. @@ -363,7 +382,7 @@ config CRYPTO_MORUS1280_GLUE tristate depends on X86 select CRYPTO_AEAD - select CRYPTO_CRYPTD + select CRYPTO_SIMD help Common glue for SIMD optimizations of the MORUS-1280 dedicated AEAD algorithm. diff --git a/crypto/Makefile b/crypto/Makefile index fb5bf2a3a666..266a4cdbb9e2 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -147,12 +147,20 @@ obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o obj-$(CONFIG_CRYPTO_OFB) += ofb.o +obj-$(CONFIG_CRYPTO_ECC) += ecc.o -ecdh_generic-y := ecc.o ecdh_generic-y += ecdh.o ecdh_generic-y += ecdh_helper.o obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o +$(obj)/ecrdsa_params.asn1.o: $(obj)/ecrdsa_params.asn1.c $(obj)/ecrdsa_params.asn1.h +$(obj)/ecrdsa_pub_key.asn1.o: $(obj)/ecrdsa_pub_key.asn1.c $(obj)/ecrdsa_pub_key.asn1.h +$(obj)/ecrdsa.o: $(obj)/ecrdsa_params.asn1.h $(obj)/ecrdsa_pub_key.asn1.h +ecrdsa_generic-y += ecrdsa.o +ecrdsa_generic-y += ecrdsa_params.asn1.o +ecrdsa_generic-y += ecrdsa_pub_key.asn1.o +obj-$(CONFIG_CRYPTO_ECRDSA) += ecrdsa_generic.o + # # generic algorithms and the async_tx api # diff --git a/crypto/adiantum.c b/crypto/adiantum.c index 5564e73266a6..395a3ddd3707 100644 --- a/crypto/adiantum.c +++ b/crypto/adiantum.c @@ -265,7 +265,6 @@ static int adiantum_hash_message(struct skcipher_request *req, int err; hash_desc->tfm = tctx->hash; - hash_desc->flags = 0; err = crypto_shash_init(hash_desc); if (err) @@ -659,7 +658,7 @@ static void __exit adiantum_module_exit(void) crypto_unregister_template(&adiantum_tmpl); } -module_init(adiantum_module_init); +subsys_initcall(adiantum_module_init); module_exit(adiantum_module_exit); MODULE_DESCRIPTION("Adiantum length-preserving encryption mode"); diff --git a/crypto/aegis128.c b/crypto/aegis128.c index 3718a8341303..d78f77fc5dd1 100644 --- a/crypto/aegis128.c +++ b/crypto/aegis128.c @@ -448,7 +448,7 @@ static void __exit crypto_aegis128_module_exit(void) crypto_unregister_aead(&crypto_aegis128_alg); } -module_init(crypto_aegis128_module_init); +subsys_initcall(crypto_aegis128_module_init); module_exit(crypto_aegis128_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/aegis128l.c b/crypto/aegis128l.c index 275a8616d71b..9bca3d619a22 100644 --- a/crypto/aegis128l.c +++ b/crypto/aegis128l.c @@ -512,7 +512,7 @@ static void __exit crypto_aegis128l_module_exit(void) crypto_unregister_aead(&crypto_aegis128l_alg); } -module_init(crypto_aegis128l_module_init); +subsys_initcall(crypto_aegis128l_module_init); module_exit(crypto_aegis128l_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/aegis256.c b/crypto/aegis256.c index ecd6b7f34a2d..b47fd39595ad 100644 --- a/crypto/aegis256.c +++ b/crypto/aegis256.c @@ -463,7 +463,7 @@ static void __exit crypto_aegis256_module_exit(void) crypto_unregister_aead(&crypto_aegis256_alg); } -module_init(crypto_aegis256_module_init); +subsys_initcall(crypto_aegis256_module_init); module_exit(crypto_aegis256_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c index 13df33aca463..f217568917e4 100644 --- a/crypto/aes_generic.c +++ b/crypto/aes_generic.c @@ -64,7 +64,7 @@ static inline u8 byte(const u32 x, const unsigned n) static const u32 rco_tab[10] = { 1, 2, 4, 8, 16, 32, 64, 128, 27, 54 }; /* cacheline-aligned to facilitate prefetching into cache */ -__visible const u32 crypto_ft_tab[4][256] __cacheline_aligned = { +__visible const u32 crypto_ft_tab[4][256] ____cacheline_aligned = { { 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, 0xb16f6fde, 0x54c5c591, @@ -328,7 +328,7 @@ __visible const u32 crypto_ft_tab[4][256] __cacheline_aligned = { } }; -__visible const u32 crypto_fl_tab[4][256] __cacheline_aligned = { +__visible const u32 crypto_fl_tab[4][256] ____cacheline_aligned = { { 0x00000063, 0x0000007c, 0x00000077, 0x0000007b, 0x000000f2, 0x0000006b, 0x0000006f, 0x000000c5, @@ -592,7 +592,7 @@ __visible const u32 crypto_fl_tab[4][256] __cacheline_aligned = { } }; -__visible const u32 crypto_it_tab[4][256] __cacheline_aligned = { +__visible const u32 crypto_it_tab[4][256] ____cacheline_aligned = { { 0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f, 0xab58faac, 0x9303e34b, @@ -856,7 +856,7 @@ __visible const u32 crypto_it_tab[4][256] __cacheline_aligned = { } }; -__visible const u32 crypto_il_tab[4][256] __cacheline_aligned = { +__visible const u32 crypto_il_tab[4][256] ____cacheline_aligned = { { 0x00000052, 0x00000009, 0x0000006a, 0x000000d5, 0x00000030, 0x00000036, 0x000000a5, 0x00000038, @@ -1470,7 +1470,7 @@ static void __exit aes_fini(void) crypto_unregister_alg(&aes_alg); } -module_init(aes_init); +subsys_initcall(aes_init); module_exit(aes_fini); MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm"); diff --git a/crypto/akcipher.c b/crypto/akcipher.c index 0cbeae137e0a..780daa436dac 100644 --- a/crypto/akcipher.c +++ b/crypto/akcipher.c @@ -119,10 +119,24 @@ static void akcipher_prepare_alg(struct akcipher_alg *alg) base->cra_flags |= CRYPTO_ALG_TYPE_AKCIPHER; } +static int akcipher_default_op(struct akcipher_request *req) +{ + return -ENOSYS; +} + int crypto_register_akcipher(struct akcipher_alg *alg) { struct crypto_alg *base = &alg->base; + if (!alg->sign) + alg->sign = akcipher_default_op; + if (!alg->verify) + alg->verify = akcipher_default_op; + if (!alg->encrypt) + alg->encrypt = akcipher_default_op; + if (!alg->decrypt) + alg->decrypt = akcipher_default_op; + akcipher_prepare_alg(alg); return crypto_register_alg(base); } diff --git a/crypto/algboss.c b/crypto/algboss.c index 527b44d0af21..bb97cfb38836 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -296,7 +296,13 @@ static void __exit cryptomgr_exit(void) BUG_ON(err); } -subsys_initcall(cryptomgr_init); +/* + * This is arch_initcall() so that the crypto self-tests are run on algorithms + * registered early by subsys_initcall(). subsys_initcall() is needed for + * generic implementations so that they're available for comparison tests when + * other implementations are registered later by module_init(). + */ +arch_initcall(cryptomgr_init); module_exit(cryptomgr_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c index eff337ce9003..e7c43ea4ce9d 100644 --- a/crypto/ansi_cprng.c +++ b/crypto/ansi_cprng.c @@ -472,7 +472,7 @@ MODULE_DESCRIPTION("Software Pseudo Random Number Generator"); MODULE_AUTHOR("Neil Horman <nhorman@tuxdriver.com>"); module_param(dbg, int, 0); MODULE_PARM_DESC(dbg, "Boolean to enable debugging (0/1 == off/on)"); -module_init(prng_mod_init); +subsys_initcall(prng_mod_init); module_exit(prng_mod_fini); MODULE_ALIAS_CRYPTO("stdrng"); MODULE_ALIAS_CRYPTO("ansi_cprng"); diff --git a/crypto/anubis.c b/crypto/anubis.c index 4bb187c2a902..673927de0eb9 100644 --- a/crypto/anubis.c +++ b/crypto/anubis.c @@ -699,7 +699,7 @@ static void __exit anubis_mod_fini(void) crypto_unregister_alg(&anubis_alg); } -module_init(anubis_mod_init); +subsys_initcall(anubis_mod_init); module_exit(anubis_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/arc4.c b/crypto/arc4.c index 6c93342e3405..2233d36456e2 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -163,7 +163,7 @@ static void __exit arc4_exit(void) crypto_unregister_skcipher(&arc4_skcipher); } -module_init(arc4_init); +subsys_initcall(arc4_init); module_exit(arc4_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/asymmetric_keys/asym_tpm.c b/crypto/asymmetric_keys/asym_tpm.c index 5d4c270463f6..76d2ce3a1b5b 100644 --- a/crypto/asymmetric_keys/asym_tpm.c +++ b/crypto/asymmetric_keys/asym_tpm.c @@ -276,6 +276,10 @@ static int tpm_sign(struct tpm_buf *tb, return datalen; } + +/* Room to fit two u32 zeros for algo id and parameters length. */ +#define SETKEY_PARAMS_SIZE (sizeof(u32) * 2) + /* * Maximum buffer size for the BER/DER encoded public key. The public key * is of the form SEQUENCE { INTEGER n, INTEGER e } where n is a maximum 2048 @@ -286,8 +290,9 @@ static int tpm_sign(struct tpm_buf *tb, * - 257 bytes of n * - max 2 bytes for INTEGER e type/length * - 3 bytes of e + * - 4+4 of zeros for set_pub_key parameters (SETKEY_PARAMS_SIZE) */ -#define PUB_KEY_BUF_SIZE (4 + 4 + 257 + 2 + 3) +#define PUB_KEY_BUF_SIZE (4 + 4 + 257 + 2 + 3 + SETKEY_PARAMS_SIZE) /* * Provide a part of a description of the key for /proc/keys. @@ -364,6 +369,8 @@ static uint32_t derive_pub_key(const void *pub_key, uint32_t len, uint8_t *buf) cur = encode_tag_length(cur, 0x02, sizeof(e)); memcpy(cur, e, sizeof(e)); cur += sizeof(e); + /* Zero parameters to satisfy set_pub_key ABI. */ + memset(cur, 0, SETKEY_PARAMS_SIZE); return cur - buf; } @@ -744,12 +751,10 @@ static int tpm_key_verify_signature(const struct key *key, struct crypto_wait cwait; struct crypto_akcipher *tfm; struct akcipher_request *req; - struct scatterlist sig_sg, digest_sg; + struct scatterlist src_sg[2]; char alg_name[CRYPTO_MAX_ALG_NAME]; uint8_t der_pub_key[PUB_KEY_BUF_SIZE]; uint32_t der_pub_key_len; - void *output; - unsigned int outlen; int ret; pr_devel("==>%s()\n", __func__); @@ -781,37 +786,17 @@ static int tpm_key_verify_signature(const struct key *key, if (!req) goto error_free_tfm; - ret = -ENOMEM; - outlen = crypto_akcipher_maxsize(tfm); - output = kmalloc(outlen, GFP_KERNEL); - if (!output) - goto error_free_req; - - sg_init_one(&sig_sg, sig->s, sig->s_size); - sg_init_one(&digest_sg, output, outlen); - akcipher_request_set_crypt(req, &sig_sg, &digest_sg, sig->s_size, - outlen); + sg_init_table(src_sg, 2); + sg_set_buf(&src_sg[0], sig->s, sig->s_size); + sg_set_buf(&src_sg[1], sig->digest, sig->digest_size); + akcipher_request_set_crypt(req, src_sg, NULL, sig->s_size, + sig->digest_size); crypto_init_wait(&cwait); akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &cwait); - - /* Perform the verification calculation. This doesn't actually do the - * verification, but rather calculates the hash expected by the - * signature and returns that to us. - */ ret = crypto_wait_req(crypto_akcipher_verify(req), &cwait); - if (ret) - goto out_free_output; - - /* Do the actual verification step. */ - if (req->dst_len != sig->digest_size || - memcmp(sig->digest, output, sig->digest_size) != 0) - ret = -EKEYREJECTED; -out_free_output: - kfree(output); -error_free_req: akcipher_request_free(req); error_free_tfm: crypto_free_akcipher(tfm); diff --git a/crypto/asymmetric_keys/pkcs7_verify.c b/crypto/asymmetric_keys/pkcs7_verify.c index 97c77f66b20d..f7b0980bf02d 100644 --- a/crypto/asymmetric_keys/pkcs7_verify.c +++ b/crypto/asymmetric_keys/pkcs7_verify.c @@ -56,7 +56,6 @@ static int pkcs7_digest(struct pkcs7_message *pkcs7, goto error_no_desc; desc->tfm = tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; /* Digest the message [RFC2315 9.3] */ ret = crypto_shash_digest(desc, pkcs7->data, pkcs7->data_len, diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index f5d85b47fcc6..77e0ae7840ff 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -45,6 +45,7 @@ void public_key_free(struct public_key *key) { if (key) { kfree(key->key); + kfree(key->params); kfree(key); } } @@ -94,6 +95,12 @@ int software_key_determine_akcipher(const char *encoding, return -ENOPKG; } +static u8 *pkey_pack_u32(u8 *dst, u32 val) +{ + memcpy(dst, &val, sizeof(val)); + return dst + sizeof(val); +} + /* * Query information about a key. */ @@ -103,6 +110,7 @@ static int software_key_query(const struct kernel_pkey_params *params, struct crypto_akcipher *tfm; struct public_key *pkey = params->key->payload.data[asym_crypto]; char alg_name[CRYPTO_MAX_ALG_NAME]; + u8 *key, *ptr; int ret, len; ret = software_key_determine_akcipher(params->encoding, @@ -115,14 +123,22 @@ static int software_key_query(const struct kernel_pkey_params *params, if (IS_ERR(tfm)) return PTR_ERR(tfm); + key = kmalloc(pkey->keylen + sizeof(u32) * 2 + pkey->paramlen, + GFP_KERNEL); + if (!key) + goto error_free_tfm; + memcpy(key, pkey->key, pkey->keylen); + ptr = key + pkey->keylen; + ptr = pkey_pack_u32(ptr, pkey->algo); + ptr = pkey_pack_u32(ptr, pkey->paramlen); + memcpy(ptr, pkey->params, pkey->paramlen); + if (pkey->key_is_private) - ret = crypto_akcipher_set_priv_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_priv_key(tfm, key, pkey->keylen); else - ret = crypto_akcipher_set_pub_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_pub_key(tfm, key, pkey->keylen); if (ret < 0) - goto error_free_tfm; + goto error_free_key; len = crypto_akcipher_maxsize(tfm); info->key_size = len * 8; @@ -137,6 +153,8 @@ static int software_key_query(const struct kernel_pkey_params *params, KEYCTL_SUPPORTS_SIGN); ret = 0; +error_free_key: + kfree(key); error_free_tfm: crypto_free_akcipher(tfm); pr_devel("<==%s() = %d\n", __func__, ret); @@ -155,6 +173,7 @@ static int software_key_eds_op(struct kernel_pkey_params *params, struct crypto_wait cwait; struct scatterlist in_sg, out_sg; char alg_name[CRYPTO_MAX_ALG_NAME]; + char *key, *ptr; int ret; pr_devel("==>%s()\n", __func__); @@ -173,14 +192,23 @@ static int software_key_eds_op(struct kernel_pkey_params *params, if (!req) goto error_free_tfm; + key = kmalloc(pkey->keylen + sizeof(u32) * 2 + pkey->paramlen, + GFP_KERNEL); + if (!key) + goto error_free_req; + + memcpy(key, pkey->key, pkey->keylen); + ptr = key + pkey->keylen; + ptr = pkey_pack_u32(ptr, pkey->algo); + ptr = pkey_pack_u32(ptr, pkey->paramlen); + memcpy(ptr, pkey->params, pkey->paramlen); + if (pkey->key_is_private) - ret = crypto_akcipher_set_priv_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_priv_key(tfm, key, pkey->keylen); else - ret = crypto_akcipher_set_pub_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_pub_key(tfm, key, pkey->keylen); if (ret) - goto error_free_req; + goto error_free_key; sg_init_one(&in_sg, in, params->in_len); sg_init_one(&out_sg, out, params->out_len); @@ -210,6 +238,8 @@ static int software_key_eds_op(struct kernel_pkey_params *params, if (ret == 0) ret = req->dst_len; +error_free_key: + kfree(key); error_free_req: akcipher_request_free(req); error_free_tfm: @@ -227,10 +257,9 @@ int public_key_verify_signature(const struct public_key *pkey, struct crypto_wait cwait; struct crypto_akcipher *tfm; struct akcipher_request *req; - struct scatterlist sig_sg, digest_sg; + struct scatterlist src_sg[2]; char alg_name[CRYPTO_MAX_ALG_NAME]; - void *output; - unsigned int outlen; + char *key, *ptr; int ret; pr_devel("==>%s()\n", __func__); @@ -254,45 +283,37 @@ int public_key_verify_signature(const struct public_key *pkey, if (!req) goto error_free_tfm; + key = kmalloc(pkey->keylen + sizeof(u32) * 2 + pkey->paramlen, + GFP_KERNEL); + if (!key) + goto error_free_req; + + memcpy(key, pkey->key, pkey->keylen); + ptr = key + pkey->keylen; + ptr = pkey_pack_u32(ptr, pkey->algo); + ptr = pkey_pack_u32(ptr, pkey->paramlen); + memcpy(ptr, pkey->params, pkey->paramlen); + if (pkey->key_is_private) - ret = crypto_akcipher_set_priv_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_priv_key(tfm, key, pkey->keylen); else - ret = crypto_akcipher_set_pub_key(tfm, - pkey->key, pkey->keylen); + ret = crypto_akcipher_set_pub_key(tfm, key, pkey->keylen); if (ret) - goto error_free_req; - - ret = -ENOMEM; - outlen = crypto_akcipher_maxsize(tfm); - output = kmalloc(outlen, GFP_KERNEL); - if (!output) - goto error_free_req; + goto error_free_key; - sg_init_one(&sig_sg, sig->s, sig->s_size); - sg_init_one(&digest_sg, output, outlen); - akcipher_request_set_crypt(req, &sig_sg, &digest_sg, sig->s_size, - outlen); + sg_init_table(src_sg, 2); + sg_set_buf(&src_sg[0], sig->s, sig->s_size); + sg_set_buf(&src_sg[1], sig->digest, sig->digest_size); + akcipher_request_set_crypt(req, src_sg, NULL, sig->s_size, + sig->digest_size); crypto_init_wait(&cwait); akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &cwait); - - /* Perform the verification calculation. This doesn't actually do the - * verification, but rather calculates the hash expected by the - * signature and returns that to us. - */ ret = crypto_wait_req(crypto_akcipher_verify(req), &cwait); - if (ret) - goto out_free_output; - - /* Do the actual verification step. */ - if (req->dst_len != sig->digest_size || - memcmp(sig->digest, output, sig->digest_size) != 0) - ret = -EKEYREJECTED; -out_free_output: - kfree(output); +error_free_key: + kfree(key); error_free_req: akcipher_request_free(req); error_free_tfm: diff --git a/crypto/asymmetric_keys/verify_pefile.c b/crypto/asymmetric_keys/verify_pefile.c index d178650fd524..f8e4a932bcfb 100644 --- a/crypto/asymmetric_keys/verify_pefile.c +++ b/crypto/asymmetric_keys/verify_pefile.c @@ -354,7 +354,6 @@ static int pefile_digest_pe(const void *pebuf, unsigned int pelen, goto error_no_desc; desc->tfm = tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; ret = crypto_shash_init(desc); if (ret < 0) goto error; diff --git a/crypto/asymmetric_keys/x509.asn1 b/crypto/asymmetric_keys/x509.asn1 index aae0cde414e2..5c9f4e4a5231 100644 --- a/crypto/asymmetric_keys/x509.asn1 +++ b/crypto/asymmetric_keys/x509.asn1 @@ -22,7 +22,7 @@ CertificateSerialNumber ::= INTEGER AlgorithmIdentifier ::= SEQUENCE { algorithm OBJECT IDENTIFIER ({ x509_note_OID }), - parameters ANY OPTIONAL + parameters ANY OPTIONAL ({ x509_note_params }) } Name ::= SEQUENCE OF RelativeDistinguishedName diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index 991f4d735a4e..5b7bfd95c334 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -26,6 +26,9 @@ struct x509_parse_context { const void *cert_start; /* Start of cert content */ const void *key; /* Key data */ size_t key_size; /* Size of key data */ + const void *params; /* Key parameters */ + size_t params_size; /* Size of key parameters */ + enum OID key_algo; /* Public key algorithm */ enum OID last_oid; /* Last OID encountered */ enum OID algo_oid; /* Algorithm OID */ unsigned char nr_mpi; /* Number of MPIs stored */ @@ -109,6 +112,13 @@ struct x509_certificate *x509_cert_parse(const void *data, size_t datalen) cert->pub->keylen = ctx->key_size; + cert->pub->params = kmemdup(ctx->params, ctx->params_size, GFP_KERNEL); + if (!cert->pub->params) + goto error_decode; + + cert->pub->paramlen = ctx->params_size; + cert->pub->algo = ctx->key_algo; + /* Grab the signature bits */ ret = x509_get_sig_params(cert); if (ret < 0) @@ -220,6 +230,14 @@ int x509_note_pkey_algo(void *context, size_t hdrlen, case OID_sha224WithRSAEncryption: ctx->cert->sig->hash_algo = "sha224"; goto rsa_pkcs1; + + case OID_gost2012Signature256: + ctx->cert->sig->hash_algo = "streebog256"; + goto ecrdsa; + + case OID_gost2012Signature512: + ctx->cert->sig->hash_algo = "streebog512"; + goto ecrdsa; } rsa_pkcs1: @@ -227,6 +245,11 @@ rsa_pkcs1: ctx->cert->sig->encoding = "pkcs1"; ctx->algo_oid = ctx->last_oid; return 0; +ecrdsa: + ctx->cert->sig->pkey_algo = "ecrdsa"; + ctx->cert->sig->encoding = "raw"; + ctx->algo_oid = ctx->last_oid; + return 0; } /* @@ -246,7 +269,8 @@ int x509_note_signature(void *context, size_t hdrlen, return -EINVAL; } - if (strcmp(ctx->cert->sig->pkey_algo, "rsa") == 0) { + if (strcmp(ctx->cert->sig->pkey_algo, "rsa") == 0 || + strcmp(ctx->cert->sig->pkey_algo, "ecrdsa") == 0) { /* Discard the BIT STRING metadata */ if (vlen < 1 || *(const u8 *)value != 0) return -EBADMSG; @@ -401,6 +425,27 @@ int x509_note_subject(void *context, size_t hdrlen, } /* + * Extract the parameters for the public key + */ +int x509_note_params(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct x509_parse_context *ctx = context; + + /* + * AlgorithmIdentifier is used three times in the x509, we should skip + * first and ignore third, using second one which is after subject and + * before subjectPublicKey. + */ + if (!ctx->cert->raw_subject || ctx->key) + return 0; + ctx->params = value - hdrlen; + ctx->params_size = vlen + hdrlen; + return 0; +} + +/* * Extract the data for the public key algorithm */ int x509_extract_key_data(void *context, size_t hdrlen, @@ -409,11 +454,15 @@ int x509_extract_key_data(void *context, size_t hdrlen, { struct x509_parse_context *ctx = context; - if (ctx->last_oid != OID_rsaEncryption) + ctx->key_algo = ctx->last_oid; + if (ctx->last_oid == OID_rsaEncryption) + ctx->cert->pub->pkey_algo = "rsa"; + else if (ctx->last_oid == OID_gost2012PKey256 || + ctx->last_oid == OID_gost2012PKey512) + ctx->cert->pub->pkey_algo = "ecrdsa"; + else return -ENOPKG; - ctx->cert->pub->pkey_algo = "rsa"; - /* Discard the BIT STRING metadata */ if (vlen < 1 || *(const u8 *)value != 0) return -EBADMSG; diff --git a/crypto/asymmetric_keys/x509_public_key.c b/crypto/asymmetric_keys/x509_public_key.c index 9338b4558cdc..bd96683d8cde 100644 --- a/crypto/asymmetric_keys/x509_public_key.c +++ b/crypto/asymmetric_keys/x509_public_key.c @@ -77,7 +77,6 @@ int x509_get_sig_params(struct x509_certificate *cert) goto error; desc->tfm = tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; ret = crypto_shash_digest(desc, cert->tbs, cert->tbs_size, sig->digest); if (ret < 0) diff --git a/crypto/authenc.c b/crypto/authenc.c index 4be293a4b5f0..b3eddac7fa3a 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -508,7 +508,7 @@ static void __exit crypto_authenc_module_exit(void) crypto_unregister_template(&crypto_authenc_tmpl); } -module_init(crypto_authenc_module_init); +subsys_initcall(crypto_authenc_module_init); module_exit(crypto_authenc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/authencesn.c b/crypto/authencesn.c index 4741fe89ba2c..58074308e535 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -523,7 +523,7 @@ static void __exit crypto_authenc_esn_module_exit(void) crypto_unregister_template(&crypto_authenc_esn_tmpl); } -module_init(crypto_authenc_esn_module_init); +subsys_initcall(crypto_authenc_esn_module_init); module_exit(crypto_authenc_esn_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/blowfish_generic.c b/crypto/blowfish_generic.c index 87b392a77a93..8548ced8b074 100644 --- a/crypto/blowfish_generic.c +++ b/crypto/blowfish_generic.c @@ -133,7 +133,7 @@ static void __exit blowfish_mod_fini(void) crypto_unregister_alg(&alg); } -module_init(blowfish_mod_init); +subsys_initcall(blowfish_mod_init); module_exit(blowfish_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/camellia_generic.c b/crypto/camellia_generic.c index 32ddd4836ff5..15ce1281f5d9 100644 --- a/crypto/camellia_generic.c +++ b/crypto/camellia_generic.c @@ -1092,7 +1092,7 @@ static void __exit camellia_fini(void) crypto_unregister_alg(&camellia_alg); } -module_init(camellia_init); +subsys_initcall(camellia_init); module_exit(camellia_fini); MODULE_DESCRIPTION("Camellia Cipher Algorithm"); diff --git a/crypto/cast5_generic.c b/crypto/cast5_generic.c index 66169c178314..24bc7d4e33be 100644 --- a/crypto/cast5_generic.c +++ b/crypto/cast5_generic.c @@ -543,7 +543,7 @@ static void __exit cast5_mod_fini(void) crypto_unregister_alg(&alg); } -module_init(cast5_mod_init); +subsys_initcall(cast5_mod_init); module_exit(cast5_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/cast6_generic.c b/crypto/cast6_generic.c index c8e5ec69790e..edd59cc34991 100644 --- a/crypto/cast6_generic.c +++ b/crypto/cast6_generic.c @@ -285,7 +285,7 @@ static void __exit cast6_mod_fini(void) crypto_unregister_alg(&alg); } -module_init(cast6_mod_init); +subsys_initcall(cast6_mod_init); module_exit(cast6_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/cbc.c b/crypto/cbc.c index d12efaac9230..129f79d03365 100644 --- a/crypto/cbc.c +++ b/crypto/cbc.c @@ -98,7 +98,7 @@ static void __exit crypto_cbc_module_exit(void) crypto_unregister_template(&crypto_cbc_tmpl); } -module_init(crypto_cbc_module_init); +subsys_initcall(crypto_cbc_module_init); module_exit(crypto_cbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ccm.c b/crypto/ccm.c index 50df8f001c1c..c1ef9d0b4271 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -458,7 +458,6 @@ static void crypto_ccm_free(struct aead_instance *inst) static int crypto_ccm_create_common(struct crypto_template *tmpl, struct rtattr **tb, - const char *full_name, const char *ctr_name, const char *mac_name) { @@ -486,7 +485,8 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl, mac = __crypto_hash_alg_common(mac_alg); err = -EINVAL; - if (mac->digestsize != 16) + if (strncmp(mac->base.cra_name, "cbcmac(", 7) != 0 || + mac->digestsize != 16) goto out_put_mac; inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL); @@ -509,23 +509,27 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl, ctr = crypto_spawn_skcipher_alg(&ictx->ctr); - /* Not a stream cipher? */ + /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */ err = -EINVAL; - if (ctr->base.cra_blocksize != 1) + if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 || + crypto_skcipher_alg_ivsize(ctr) != 16 || + ctr->base.cra_blocksize != 1) goto err_drop_ctr; - /* We want the real thing! */ - if (crypto_skcipher_alg_ivsize(ctr) != 16) + /* ctr and cbcmac must use the same underlying block cipher. */ + if (strcmp(ctr->base.cra_name + 4, mac->base.cra_name + 7) != 0) goto err_drop_ctr; err = -ENAMETOOLONG; + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "ccm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME) + goto err_drop_ctr; + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "ccm_base(%s,%s)", ctr->base.cra_driver_name, mac->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_ctr; - memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME); - inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC; inst->alg.base.cra_priority = (mac->base.cra_priority + ctr->base.cra_priority) / 2; @@ -567,7 +571,6 @@ static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb) const char *cipher_name; char ctr_name[CRYPTO_MAX_ALG_NAME]; char mac_name[CRYPTO_MAX_ALG_NAME]; - char full_name[CRYPTO_MAX_ALG_NAME]; cipher_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(cipher_name)) @@ -581,35 +584,24 @@ static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb) cipher_name) >= CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; - if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm(%s)", cipher_name) >= - CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name, - mac_name); + return crypto_ccm_create_common(tmpl, tb, ctr_name, mac_name); } static int crypto_ccm_base_create(struct crypto_template *tmpl, struct rtattr **tb) { const char *ctr_name; - const char *cipher_name; - char full_name[CRYPTO_MAX_ALG_NAME]; + const char *mac_name; ctr_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(ctr_name)) return PTR_ERR(ctr_name); - cipher_name = crypto_attr_alg_name(tb[2]); - if (IS_ERR(cipher_name)) - return PTR_ERR(cipher_name); - - if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm_base(%s,%s)", - ctr_name, cipher_name) >= CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; + mac_name = crypto_attr_alg_name(tb[2]); + if (IS_ERR(mac_name)) + return PTR_ERR(mac_name); - return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name, - cipher_name); + return crypto_ccm_create_common(tmpl, tb, ctr_name, mac_name); } static int crypto_rfc4309_setkey(struct crypto_aead *parent, const u8 *key, @@ -1014,7 +1006,7 @@ static void __exit crypto_ccm_module_exit(void) ARRAY_SIZE(crypto_ccm_tmpls)); } -module_init(crypto_ccm_module_init); +subsys_initcall(crypto_ccm_module_init); module_exit(crypto_ccm_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/cfb.c b/crypto/cfb.c index 03ac847f6d6a..7b68fbb61732 100644 --- a/crypto/cfb.c +++ b/crypto/cfb.c @@ -243,7 +243,7 @@ static void __exit crypto_cfb_module_exit(void) crypto_unregister_template(&crypto_cfb_tmpl); } -module_init(crypto_cfb_module_init); +subsys_initcall(crypto_cfb_module_init); module_exit(crypto_cfb_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index ed2e12e26dd8..e38a2d61819a 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -645,8 +645,8 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, - "%s(%s,%s)", name, chacha_name, - poly_name) >= CRYPTO_MAX_ALG_NAME) + "%s(%s,%s)", name, chacha->base.cra_name, + poly->cra_name) >= CRYPTO_MAX_ALG_NAME) goto out_drop_chacha; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s,%s)", name, chacha->base.cra_driver_name, @@ -725,7 +725,7 @@ static void __exit chacha20poly1305_module_exit(void) ARRAY_SIZE(rfc7539_tmpls)); } -module_init(chacha20poly1305_module_init); +subsys_initcall(chacha20poly1305_module_init); module_exit(chacha20poly1305_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c index 35b583101f4f..d2ec04997832 100644 --- a/crypto/chacha_generic.c +++ b/crypto/chacha_generic.c @@ -22,18 +22,16 @@ static void chacha_docrypt(u32 *state, u8 *dst, const u8 *src, /* aligned to potentially speed up crypto_xor() */ u8 stream[CHACHA_BLOCK_SIZE] __aligned(sizeof(long)); - if (dst != src) - memcpy(dst, src, bytes); - while (bytes >= CHACHA_BLOCK_SIZE) { chacha_block(state, stream, nrounds); - crypto_xor(dst, stream, CHACHA_BLOCK_SIZE); + crypto_xor_cpy(dst, src, stream, CHACHA_BLOCK_SIZE); bytes -= CHACHA_BLOCK_SIZE; dst += CHACHA_BLOCK_SIZE; + src += CHACHA_BLOCK_SIZE; } if (bytes) { chacha_block(state, stream, nrounds); - crypto_xor(dst, stream, bytes); + crypto_xor_cpy(dst, src, stream, bytes); } } @@ -52,7 +50,7 @@ static int chacha_stream_xor(struct skcipher_request *req, unsigned int nbytes = walk.nbytes; if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); + nbytes = round_down(nbytes, CHACHA_BLOCK_SIZE); chacha_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr, nbytes, ctx->nrounds); @@ -203,7 +201,7 @@ static void __exit chacha_generic_mod_fini(void) crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); } -module_init(chacha_generic_mod_init); +subsys_initcall(chacha_generic_mod_init); module_exit(chacha_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/cmac.c b/crypto/cmac.c index 16301f52858c..c60b6c011ec6 100644 --- a/crypto/cmac.c +++ b/crypto/cmac.c @@ -313,7 +313,7 @@ static void __exit crypto_cmac_module_exit(void) crypto_unregister_template(&crypto_cmac_tmpl); } -module_init(crypto_cmac_module_init); +subsys_initcall(crypto_cmac_module_init); module_exit(crypto_cmac_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/crc32_generic.c b/crypto/crc32_generic.c index 00facd27bcc2..9e97912280bd 100644 --- a/crypto/crc32_generic.c +++ b/crypto/crc32_generic.c @@ -146,7 +146,7 @@ static void __exit crc32_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(crc32_mod_init); +subsys_initcall(crc32_mod_init); module_exit(crc32_mod_fini); MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>"); diff --git a/crypto/crc32c_generic.c b/crypto/crc32c_generic.c index 7283066ecc98..ad26f15d4c7b 100644 --- a/crypto/crc32c_generic.c +++ b/crypto/crc32c_generic.c @@ -165,7 +165,7 @@ static void __exit crc32c_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(crc32c_mod_init); +subsys_initcall(crc32c_mod_init); module_exit(crc32c_mod_fini); MODULE_AUTHOR("Clay Haapala <chaapala@cisco.com>"); diff --git a/crypto/crct10dif_generic.c b/crypto/crct10dif_generic.c index 8e94e29dc6fc..d90c0070710e 100644 --- a/crypto/crct10dif_generic.c +++ b/crypto/crct10dif_generic.c @@ -65,10 +65,9 @@ static int chksum_final(struct shash_desc *desc, u8 *out) return 0; } -static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len, - u8 *out) +static int __chksum_finup(__u16 crc, const u8 *data, unsigned int len, u8 *out) { - *(__u16 *)out = crc_t10dif_generic(*crcp, data, len); + *(__u16 *)out = crc_t10dif_generic(crc, data, len); return 0; } @@ -77,15 +76,13 @@ static int chksum_finup(struct shash_desc *desc, const u8 *data, { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); - return __chksum_finup(&ctx->crc, data, len, out); + return __chksum_finup(ctx->crc, data, len, out); } static int chksum_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) { - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); - - return __chksum_finup(&ctx->crc, data, length, out); + return __chksum_finup(0, data, length, out); } static struct shash_alg alg = { @@ -115,7 +112,7 @@ static void __exit crct10dif_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(crct10dif_mod_init); +subsys_initcall(crct10dif_mod_init); module_exit(crct10dif_mod_fini); MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>"); diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 5640e5db7bdb..b3bb99390ae7 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -65,15 +65,6 @@ struct aead_instance_ctx { struct cryptd_queue *queue; }; -struct cryptd_blkcipher_ctx { - atomic_t refcnt; - struct crypto_blkcipher *child; -}; - -struct cryptd_blkcipher_request_ctx { - crypto_completion_t complete; -}; - struct cryptd_skcipher_ctx { atomic_t refcnt; struct crypto_sync_skcipher *child; @@ -216,129 +207,6 @@ static inline void cryptd_check_internal(struct rtattr **tb, u32 *type, *mask |= algt->mask & CRYPTO_ALG_INTERNAL; } -static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent, - const u8 *key, unsigned int keylen) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent); - struct crypto_blkcipher *child = ctx->child; - int err; - - crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) & - CRYPTO_TFM_REQ_MASK); - err = crypto_blkcipher_setkey(child, key, keylen); - crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) & - CRYPTO_TFM_RES_MASK); - return err; -} - -static void cryptd_blkcipher_crypt(struct ablkcipher_request *req, - struct crypto_blkcipher *child, - int err, - int (*crypt)(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, - unsigned int len)) -{ - struct cryptd_blkcipher_request_ctx *rctx; - struct cryptd_blkcipher_ctx *ctx; - struct crypto_ablkcipher *tfm; - struct blkcipher_desc desc; - int refcnt; - - rctx = ablkcipher_request_ctx(req); - - if (unlikely(err == -EINPROGRESS)) - goto out; - - desc.tfm = child; - desc.info = req->info; - desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; - - err = crypt(&desc, req->dst, req->src, req->nbytes); - - req->base.complete = rctx->complete; - -out: - tfm = crypto_ablkcipher_reqtfm(req); - ctx = crypto_ablkcipher_ctx(tfm); - refcnt = atomic_read(&ctx->refcnt); - - local_bh_disable(); - rctx->complete(&req->base, err); - local_bh_enable(); - - if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt)) - crypto_free_ablkcipher(tfm); -} - -static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); - struct crypto_blkcipher *child = ctx->child; - - cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, - crypto_blkcipher_crt(child)->encrypt); -} - -static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); - struct crypto_blkcipher *child = ctx->child; - - cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, - crypto_blkcipher_crt(child)->decrypt); -} - -static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req, - crypto_completion_t compl) -{ - struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req); - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct cryptd_queue *queue; - - queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm)); - rctx->complete = req->base.complete; - req->base.complete = compl; - - return cryptd_enqueue_request(queue, &req->base); -} - -static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req) -{ - return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt); -} - -static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req) -{ - return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt); -} - -static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm) -{ - struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); - struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); - struct crypto_spawn *spawn = &ictx->spawn; - struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_blkcipher *cipher; - - cipher = crypto_spawn_blkcipher(spawn); - if (IS_ERR(cipher)) - return PTR_ERR(cipher); - - ctx->child = cipher; - tfm->crt_ablkcipher.reqsize = - sizeof(struct cryptd_blkcipher_request_ctx); - return 0; -} - -static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto_free_blkcipher(ctx->child); -} - static int cryptd_init_instance(struct crypto_instance *inst, struct crypto_alg *alg) { @@ -382,67 +250,6 @@ out_free_inst: goto out; } -static int cryptd_create_blkcipher(struct crypto_template *tmpl, - struct rtattr **tb, - struct cryptd_queue *queue) -{ - struct cryptd_instance_ctx *ctx; - struct crypto_instance *inst; - struct crypto_alg *alg; - u32 type = CRYPTO_ALG_TYPE_BLKCIPHER; - u32 mask = CRYPTO_ALG_TYPE_MASK; - int err; - - cryptd_check_internal(tb, &type, &mask); - - alg = crypto_get_attr_alg(tb, type, mask); - if (IS_ERR(alg)) - return PTR_ERR(alg); - - inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx)); - err = PTR_ERR(inst); - if (IS_ERR(inst)) - goto out_put_alg; - - ctx = crypto_instance_ctx(inst); - ctx->queue = queue; - - err = crypto_init_spawn(&ctx->spawn, alg, inst, - CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); - if (err) - goto out_free_inst; - - type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; - if (alg->cra_flags & CRYPTO_ALG_INTERNAL) - type |= CRYPTO_ALG_INTERNAL; - inst->alg.cra_flags = type; - inst->alg.cra_type = &crypto_ablkcipher_type; - - inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize; - inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize; - inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize; - - inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx); - - inst->alg.cra_init = cryptd_blkcipher_init_tfm; - inst->alg.cra_exit = cryptd_blkcipher_exit_tfm; - - inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey; - inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue; - inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue; - - err = crypto_register_instance(tmpl, inst); - if (err) { - crypto_drop_spawn(&ctx->spawn); -out_free_inst: - kfree(inst); - } - -out_put_alg: - crypto_mod_put(alg); - return err; -} - static int cryptd_skcipher_setkey(struct crypto_skcipher *parent, const u8 *key, unsigned int keylen) { @@ -738,7 +545,6 @@ static void cryptd_hash_init(struct crypto_async_request *req_async, int err) goto out; desc->tfm = child; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; err = crypto_shash_init(desc); @@ -830,7 +636,6 @@ static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) goto out; desc->tfm = child; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; err = shash_ahash_digest(req, desc); @@ -859,7 +664,6 @@ static int cryptd_hash_import(struct ahash_request *req, const void *in) struct shash_desc *desc = cryptd_shash_desc(req); desc->tfm = ctx->child; - desc->flags = req->base.flags; return crypto_shash_import(desc, in); } @@ -1118,10 +922,6 @@ static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_BLKCIPHER: - if ((algt->type & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER) - return cryptd_create_blkcipher(tmpl, tb, &queue); - return cryptd_create_skcipher(tmpl, tb, &queue); case CRYPTO_ALG_TYPE_DIGEST: return cryptd_create_hash(tmpl, tb, &queue); @@ -1160,58 +960,6 @@ static struct crypto_template cryptd_tmpl = { .module = THIS_MODULE, }; -struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, - u32 type, u32 mask) -{ - char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; - struct cryptd_blkcipher_ctx *ctx; - struct crypto_tfm *tfm; - - if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, - "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) - return ERR_PTR(-EINVAL); - type = crypto_skcipher_type(type); - mask &= ~CRYPTO_ALG_TYPE_MASK; - mask |= CRYPTO_ALG_TYPE_BLKCIPHER_MASK; - tfm = crypto_alloc_base(cryptd_alg_name, type, mask); - if (IS_ERR(tfm)) - return ERR_CAST(tfm); - if (tfm->__crt_alg->cra_module != THIS_MODULE) { - crypto_free_tfm(tfm); - return ERR_PTR(-EINVAL); - } - - ctx = crypto_tfm_ctx(tfm); - atomic_set(&ctx->refcnt, 1); - - return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm)); -} -EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher); - -struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); - return ctx->child; -} -EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child); - -bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); - - return atomic_read(&ctx->refcnt) - 1; -} -EXPORT_SYMBOL_GPL(cryptd_ablkcipher_queued); - -void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm) -{ - struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); - - if (atomic_dec_and_test(&ctx->refcnt)) - crypto_free_ablkcipher(&tfm->base); -} -EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher); - struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name, u32 type, u32 mask) { diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 01630a9c7e01..9320d4eaa4a8 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -220,7 +220,7 @@ static void __exit crypto_null_mod_fini(void) crypto_unregister_skcipher(&skcipher_null); } -module_init(crypto_null_mod_init); +subsys_initcall(crypto_null_mod_init); module_exit(crypto_null_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/ctr.c b/crypto/ctr.c index ec8f8b67473a..52cdf2c5605f 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -384,7 +384,7 @@ static void __exit crypto_ctr_module_exit(void) ARRAY_SIZE(crypto_ctr_tmpls)); } -module_init(crypto_ctr_module_init); +subsys_initcall(crypto_ctr_module_init); module_exit(crypto_ctr_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/cts.c b/crypto/cts.c index 4e28d83ae37d..6b6087dbb62a 100644 --- a/crypto/cts.c +++ b/crypto/cts.c @@ -152,12 +152,14 @@ static int crypto_cts_encrypt(struct skcipher_request *req) struct skcipher_request *subreq = &rctx->subreq; int bsize = crypto_skcipher_blocksize(tfm); unsigned int nbytes = req->cryptlen; - int cbc_blocks = (nbytes + bsize - 1) / bsize - 1; unsigned int offset; skcipher_request_set_tfm(subreq, ctx->child); - if (cbc_blocks <= 0) { + if (nbytes < bsize) + return -EINVAL; + + if (nbytes == bsize) { skcipher_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data); @@ -166,7 +168,7 @@ static int crypto_cts_encrypt(struct skcipher_request *req) return crypto_skcipher_encrypt(subreq); } - offset = cbc_blocks * bsize; + offset = rounddown(nbytes - 1, bsize); rctx->offset = offset; skcipher_request_set_callback(subreq, req->base.flags, @@ -244,13 +246,15 @@ static int crypto_cts_decrypt(struct skcipher_request *req) struct skcipher_request *subreq = &rctx->subreq; int bsize = crypto_skcipher_blocksize(tfm); unsigned int nbytes = req->cryptlen; - int cbc_blocks = (nbytes + bsize - 1) / bsize - 1; unsigned int offset; u8 *space; skcipher_request_set_tfm(subreq, ctx->child); - if (cbc_blocks <= 0) { + if (nbytes < bsize) + return -EINVAL; + + if (nbytes == bsize) { skcipher_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data); @@ -264,10 +268,10 @@ static int crypto_cts_decrypt(struct skcipher_request *req) space = crypto_cts_reqctx_space(req); - offset = cbc_blocks * bsize; + offset = rounddown(nbytes - 1, bsize); rctx->offset = offset; - if (cbc_blocks <= 1) + if (offset <= bsize) memcpy(space, req->iv, bsize); else scatterwalk_map_and_copy(space, req->src, offset - 2 * bsize, @@ -419,7 +423,7 @@ static void __exit crypto_cts_module_exit(void) crypto_unregister_template(&crypto_cts_tmpl); } -module_init(crypto_cts_module_init); +subsys_initcall(crypto_cts_module_init); module_exit(crypto_cts_module_exit); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/deflate.c b/crypto/deflate.c index 94ec3b36a8e8..aab089cde1bf 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -334,7 +334,7 @@ static void __exit deflate_mod_fini(void) crypto_unregister_scomps(scomp, ARRAY_SIZE(scomp)); } -module_init(deflate_mod_init); +subsys_initcall(deflate_mod_init); module_exit(deflate_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/des_generic.c b/crypto/des_generic.c index 1e6621665dd9..d7a88b4fa611 100644 --- a/crypto/des_generic.c +++ b/crypto/des_generic.c @@ -862,14 +862,11 @@ static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) int __des3_ede_setkey(u32 *expkey, u32 *flags, const u8 *key, unsigned int keylen) { - const u32 *K = (const u32 *)key; + int err; - if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } + err = __des3_verify_key(flags, key); + if (unlikely(err)) + return err; des_ekey(expkey, key); expkey += DES_EXPKEY_WORDS; key += DES_KEY_SIZE; dkey(expkey, key); expkey += DES_EXPKEY_WORDS; key += DES_KEY_SIZE; @@ -993,7 +990,7 @@ static void __exit des_generic_mod_fini(void) crypto_unregister_algs(des_algs, ARRAY_SIZE(des_algs)); } -module_init(des_generic_mod_init); +subsys_initcall(des_generic_mod_init); module_exit(des_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/dh.c b/crypto/dh.c index 09a44de4209d..ce77fb4ee8b3 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -236,7 +236,7 @@ static void dh_exit(void) crypto_unregister_kpp(&dh); } -module_init(dh_init); +subsys_initcall(dh_init); module_exit(dh_exit); MODULE_ALIAS_CRYPTO("dh"); MODULE_LICENSE("GPL"); diff --git a/crypto/drbg.c b/crypto/drbg.c index bc52d9562611..2a5b16bb000c 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1587,7 +1587,6 @@ static int drbg_init_hash_kernel(struct drbg_state *drbg) } sdesc->shash.tfm = tfm; - sdesc->shash.flags = 0; drbg->priv_data = sdesc; return crypto_shash_alignmask(tfm); @@ -2039,7 +2038,7 @@ static void __exit drbg_exit(void) crypto_unregister_rngs(drbg_algs, (ARRAY_SIZE(drbg_cores) * 2)); } -module_init(drbg_init); +subsys_initcall(drbg_init); module_exit(drbg_exit); #ifndef CRYPTO_DRBG_HASH_STRING #define CRYPTO_DRBG_HASH_STRING "" diff --git a/crypto/ecb.c b/crypto/ecb.c index 0732715c8d91..de839129d151 100644 --- a/crypto/ecb.c +++ b/crypto/ecb.c @@ -101,7 +101,7 @@ static void __exit crypto_ecb_module_exit(void) crypto_unregister_template(&crypto_ecb_tmpl); } -module_init(crypto_ecb_module_init); +subsys_initcall(crypto_ecb_module_init); module_exit(crypto_ecb_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ecc.c b/crypto/ecc.c index ed1237115066..dfe114bc0c4a 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -1,6 +1,6 @@ /* - * Copyright (c) 2013, Kenneth MacKay - * All rights reserved. + * Copyright (c) 2013, 2014 Kenneth MacKay. All rights reserved. + * Copyright (c) 2019 Vitaly Chikunov <vt@altlinux.org> * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are @@ -24,12 +24,15 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <linux/module.h> #include <linux/random.h> #include <linux/slab.h> #include <linux/swab.h> #include <linux/fips.h> #include <crypto/ecdh.h> #include <crypto/rng.h> +#include <asm/unaligned.h> +#include <linux/ratelimit.h> #include "ecc.h" #include "ecc_curve_defs.h" @@ -112,7 +115,7 @@ static void vli_clear(u64 *vli, unsigned int ndigits) } /* Returns true if vli == 0, false otherwise. */ -static bool vli_is_zero(const u64 *vli, unsigned int ndigits) +bool vli_is_zero(const u64 *vli, unsigned int ndigits) { int i; @@ -123,6 +126,7 @@ static bool vli_is_zero(const u64 *vli, unsigned int ndigits) return true; } +EXPORT_SYMBOL(vli_is_zero); /* Returns nonzero if bit bit of vli is set. */ static u64 vli_test_bit(const u64 *vli, unsigned int bit) @@ -130,6 +134,11 @@ static u64 vli_test_bit(const u64 *vli, unsigned int bit) return (vli[bit / 64] & ((u64)1 << (bit % 64))); } +static bool vli_is_negative(const u64 *vli, unsigned int ndigits) +{ + return vli_test_bit(vli, ndigits * 64 - 1); +} + /* Counts the number of 64-bit "digits" in vli. */ static unsigned int vli_num_digits(const u64 *vli, unsigned int ndigits) { @@ -161,6 +170,27 @@ static unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits) return ((num_digits - 1) * 64 + i); } +/* Set dest from unaligned bit string src. */ +void vli_from_be64(u64 *dest, const void *src, unsigned int ndigits) +{ + int i; + const u64 *from = src; + + for (i = 0; i < ndigits; i++) + dest[i] = get_unaligned_be64(&from[ndigits - 1 - i]); +} +EXPORT_SYMBOL(vli_from_be64); + +void vli_from_le64(u64 *dest, const void *src, unsigned int ndigits) +{ + int i; + const u64 *from = src; + + for (i = 0; i < ndigits; i++) + dest[i] = get_unaligned_le64(&from[i]); +} +EXPORT_SYMBOL(vli_from_le64); + /* Sets dest = src. */ static void vli_set(u64 *dest, const u64 *src, unsigned int ndigits) { @@ -171,7 +201,7 @@ static void vli_set(u64 *dest, const u64 *src, unsigned int ndigits) } /* Returns sign of left - right. */ -static int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits) +int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits) { int i; @@ -184,6 +214,7 @@ static int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits) return 0; } +EXPORT_SYMBOL(vli_cmp); /* Computes result = in << c, returning carry. Can modify in place * (if result == in). 0 < shift < 64. @@ -239,8 +270,30 @@ static u64 vli_add(u64 *result, const u64 *left, const u64 *right, return carry; } +/* Computes result = left + right, returning carry. Can modify in place. */ +static u64 vli_uadd(u64 *result, const u64 *left, u64 right, + unsigned int ndigits) +{ + u64 carry = right; + int i; + + for (i = 0; i < ndigits; i++) { + u64 sum; + + sum = left[i] + carry; + if (sum != left[i]) + carry = (sum < left[i]); + else + carry = !!carry; + + result[i] = sum; + } + + return carry; +} + /* Computes result = left - right, returning borrow. Can modify in place. */ -static u64 vli_sub(u64 *result, const u64 *left, const u64 *right, +u64 vli_sub(u64 *result, const u64 *left, const u64 *right, unsigned int ndigits) { u64 borrow = 0; @@ -258,9 +311,37 @@ static u64 vli_sub(u64 *result, const u64 *left, const u64 *right, return borrow; } +EXPORT_SYMBOL(vli_sub); + +/* Computes result = left - right, returning borrow. Can modify in place. */ +static u64 vli_usub(u64 *result, const u64 *left, u64 right, + unsigned int ndigits) +{ + u64 borrow = right; + int i; + + for (i = 0; i < ndigits; i++) { + u64 diff; + + diff = left[i] - borrow; + if (diff != left[i]) + borrow = (diff > left[i]); + + result[i] = diff; + } + + return borrow; +} static uint128_t mul_64_64(u64 left, u64 right) { + uint128_t result; +#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) + unsigned __int128 m = (unsigned __int128)left * right; + + result.m_low = m; + result.m_high = m >> 64; +#else u64 a0 = left & 0xffffffffull; u64 a1 = left >> 32; u64 b0 = right & 0xffffffffull; @@ -269,7 +350,6 @@ static uint128_t mul_64_64(u64 left, u64 right) u64 m1 = a0 * b1; u64 m2 = a1 * b0; u64 m3 = a1 * b1; - uint128_t result; m2 += (m0 >> 32); m2 += m1; @@ -280,7 +360,7 @@ static uint128_t mul_64_64(u64 left, u64 right) result.m_low = (m0 & 0xffffffffull) | (m2 << 32); result.m_high = m3 + (m2 >> 32); - +#endif return result; } @@ -330,6 +410,28 @@ static void vli_mult(u64 *result, const u64 *left, const u64 *right, result[ndigits * 2 - 1] = r01.m_low; } +/* Compute product = left * right, for a small right value. */ +static void vli_umult(u64 *result, const u64 *left, u32 right, + unsigned int ndigits) +{ + uint128_t r01 = { 0 }; + unsigned int k; + + for (k = 0; k < ndigits; k++) { + uint128_t product; + + product = mul_64_64(left[k], right); + r01 = add_128_128(r01, product); + /* no carry */ + result[k] = r01.m_low; + r01.m_low = r01.m_high; + r01.m_high = 0; + } + result[k] = r01.m_low; + for (++k; k < ndigits * 2; k++) + result[k] = 0; +} + static void vli_square(u64 *result, const u64 *left, unsigned int ndigits) { uint128_t r01 = { 0, 0 }; @@ -402,6 +504,170 @@ static void vli_mod_sub(u64 *result, const u64 *left, const u64 *right, vli_add(result, result, mod, ndigits); } +/* + * Computes result = product % mod + * for special form moduli: p = 2^k-c, for small c (note the minus sign) + * + * References: + * R. Crandall, C. Pomerance. Prime Numbers: A Computational Perspective. + * 9 Fast Algorithms for Large-Integer Arithmetic. 9.2.3 Moduli of special form + * Algorithm 9.2.13 (Fast mod operation for special-form moduli). + */ +static void vli_mmod_special(u64 *result, const u64 *product, + const u64 *mod, unsigned int ndigits) +{ + u64 c = -mod[0]; + u64 t[ECC_MAX_DIGITS * 2]; + u64 r[ECC_MAX_DIGITS * 2]; + + vli_set(r, product, ndigits * 2); + while (!vli_is_zero(r + ndigits, ndigits)) { + vli_umult(t, r + ndigits, c, ndigits); + vli_clear(r + ndigits, ndigits); + vli_add(r, r, t, ndigits * 2); + } + vli_set(t, mod, ndigits); + vli_clear(t + ndigits, ndigits); + while (vli_cmp(r, t, ndigits * 2) >= 0) + vli_sub(r, r, t, ndigits * 2); + vli_set(result, r, ndigits); +} + +/* + * Computes result = product % mod + * for special form moduli: p = 2^{k-1}+c, for small c (note the plus sign) + * where k-1 does not fit into qword boundary by -1 bit (such as 255). + + * References (loosely based on): + * A. Menezes, P. van Oorschot, S. Vanstone. Handbook of Applied Cryptography. + * 14.3.4 Reduction methods for moduli of special form. Algorithm 14.47. + * URL: http://cacr.uwaterloo.ca/hac/about/chap14.pdf + * + * H. Cohen, G. Frey, R. Avanzi, C. Doche, T. Lange, K. Nguyen, F. Vercauteren. + * Handbook of Elliptic and Hyperelliptic Curve Cryptography. + * Algorithm 10.25 Fast reduction for special form moduli + */ +static void vli_mmod_special2(u64 *result, const u64 *product, + const u64 *mod, unsigned int ndigits) +{ + u64 c2 = mod[0] * 2; + u64 q[ECC_MAX_DIGITS]; + u64 r[ECC_MAX_DIGITS * 2]; + u64 m[ECC_MAX_DIGITS * 2]; /* expanded mod */ + int carry; /* last bit that doesn't fit into q */ + int i; + + vli_set(m, mod, ndigits); + vli_clear(m + ndigits, ndigits); + + vli_set(r, product, ndigits); + /* q and carry are top bits */ + vli_set(q, product + ndigits, ndigits); + vli_clear(r + ndigits, ndigits); + carry = vli_is_negative(r, ndigits); + if (carry) + r[ndigits - 1] &= (1ull << 63) - 1; + for (i = 1; carry || !vli_is_zero(q, ndigits); i++) { + u64 qc[ECC_MAX_DIGITS * 2]; + + vli_umult(qc, q, c2, ndigits); + if (carry) + vli_uadd(qc, qc, mod[0], ndigits * 2); + vli_set(q, qc + ndigits, ndigits); + vli_clear(qc + ndigits, ndigits); + carry = vli_is_negative(qc, ndigits); + if (carry) + qc[ndigits - 1] &= (1ull << 63) - 1; + if (i & 1) + vli_sub(r, r, qc, ndigits * 2); + else + vli_add(r, r, qc, ndigits * 2); + } + while (vli_is_negative(r, ndigits * 2)) + vli_add(r, r, m, ndigits * 2); + while (vli_cmp(r, m, ndigits * 2) >= 0) + vli_sub(r, r, m, ndigits * 2); + + vli_set(result, r, ndigits); +} + +/* + * Computes result = product % mod, where product is 2N words long. + * Reference: Ken MacKay's micro-ecc. + * Currently only designed to work for curve_p or curve_n. + */ +static void vli_mmod_slow(u64 *result, u64 *product, const u64 *mod, + unsigned int ndigits) +{ + u64 mod_m[2 * ECC_MAX_DIGITS]; + u64 tmp[2 * ECC_MAX_DIGITS]; + u64 *v[2] = { tmp, product }; + u64 carry = 0; + unsigned int i; + /* Shift mod so its highest set bit is at the maximum position. */ + int shift = (ndigits * 2 * 64) - vli_num_bits(mod, ndigits); + int word_shift = shift / 64; + int bit_shift = shift % 64; + + vli_clear(mod_m, word_shift); + if (bit_shift > 0) { + for (i = 0; i < ndigits; ++i) { + mod_m[word_shift + i] = (mod[i] << bit_shift) | carry; + carry = mod[i] >> (64 - bit_shift); + } + } else + vli_set(mod_m + word_shift, mod, ndigits); + + for (i = 1; shift >= 0; --shift) { + u64 borrow = 0; + unsigned int j; + + for (j = 0; j < ndigits * 2; ++j) { + u64 diff = v[i][j] - mod_m[j] - borrow; + + if (diff != v[i][j]) + borrow = (diff > v[i][j]); + v[1 - i][j] = diff; + } + i = !(i ^ borrow); /* Swap the index if there was no borrow */ + vli_rshift1(mod_m, ndigits); + mod_m[ndigits - 1] |= mod_m[ndigits] << (64 - 1); + vli_rshift1(mod_m + ndigits, ndigits); + } + vli_set(result, v[i], ndigits); +} + +/* Computes result = product % mod using Barrett's reduction with precomputed + * value mu appended to the mod after ndigits, mu = (2^{2w} / mod) and have + * length ndigits + 1, where mu * (2^w - 1) should not overflow ndigits + * boundary. + * + * Reference: + * R. Brent, P. Zimmermann. Modern Computer Arithmetic. 2010. + * 2.4.1 Barrett's algorithm. Algorithm 2.5. + */ +static void vli_mmod_barrett(u64 *result, u64 *product, const u64 *mod, + unsigned int ndigits) +{ + u64 q[ECC_MAX_DIGITS * 2]; + u64 r[ECC_MAX_DIGITS * 2]; + const u64 *mu = mod + ndigits; + + vli_mult(q, product + ndigits, mu, ndigits); + if (mu[ndigits]) + vli_add(q + ndigits, q + ndigits, product + ndigits, ndigits); + vli_mult(r, mod, q + ndigits, ndigits); + vli_sub(r, product, r, ndigits * 2); + while (!vli_is_zero(r + ndigits, ndigits) || + vli_cmp(r, mod, ndigits) != -1) { + u64 carry; + + carry = vli_sub(r, r, mod, ndigits); + vli_usub(r + ndigits, r + ndigits, carry, ndigits); + } + vli_set(result, r, ndigits); +} + /* Computes p_result = p_product % curve_p. * See algorithm 5 and 6 from * http://www.isys.uni-klu.ac.at/PDF/2001-0126-MT.pdf @@ -509,14 +775,33 @@ static void vli_mmod_fast_256(u64 *result, const u64 *product, } } -/* Computes result = product % curve_prime - * from http://www.nsa.gov/ia/_files/nist-routines.pdf -*/ +/* Computes result = product % curve_prime for different curve_primes. + * + * Note that curve_primes are distinguished just by heuristic check and + * not by complete conformance check. + */ static bool vli_mmod_fast(u64 *result, u64 *product, const u64 *curve_prime, unsigned int ndigits) { u64 tmp[2 * ECC_MAX_DIGITS]; + /* Currently, both NIST primes have -1 in lowest qword. */ + if (curve_prime[0] != -1ull) { + /* Try to handle Pseudo-Marsenne primes. */ + if (curve_prime[ndigits - 1] == -1ull) { + vli_mmod_special(result, product, curve_prime, + ndigits); + return true; + } else if (curve_prime[ndigits - 1] == 1ull << 63 && + curve_prime[ndigits - 2] == 0) { + vli_mmod_special2(result, product, curve_prime, + ndigits); + return true; + } + vli_mmod_barrett(result, product, curve_prime, ndigits); + return true; + } + switch (ndigits) { case 3: vli_mmod_fast_192(result, product, curve_prime, tmp); @@ -525,13 +810,26 @@ static bool vli_mmod_fast(u64 *result, u64 *product, vli_mmod_fast_256(result, product, curve_prime, tmp); break; default: - pr_err("unsupports digits size!\n"); + pr_err_ratelimited("ecc: unsupported digits size!\n"); return false; } return true; } +/* Computes result = (left * right) % mod. + * Assumes that mod is big enough curve order. + */ +void vli_mod_mult_slow(u64 *result, const u64 *left, const u64 *right, + const u64 *mod, unsigned int ndigits) +{ + u64 product[ECC_MAX_DIGITS * 2]; + + vli_mult(product, left, right, ndigits); + vli_mmod_slow(result, product, mod, ndigits); +} +EXPORT_SYMBOL(vli_mod_mult_slow); + /* Computes result = (left * right) % curve_prime. */ static void vli_mod_mult_fast(u64 *result, const u64 *left, const u64 *right, const u64 *curve_prime, unsigned int ndigits) @@ -557,7 +855,7 @@ static void vli_mod_square_fast(u64 *result, const u64 *left, * See "From Euclid's GCD to Montgomery Multiplication to the Great Divide" * https://labs.oracle.com/techrep/2001/smli_tr-2001-95.pdf */ -static void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod, +void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod, unsigned int ndigits) { u64 a[ECC_MAX_DIGITS], b[ECC_MAX_DIGITS]; @@ -630,6 +928,7 @@ static void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod, vli_set(result, u, ndigits); } +EXPORT_SYMBOL(vli_mod_inv); /* ------ Point operations ------ */ @@ -903,6 +1202,85 @@ static void ecc_point_mult(struct ecc_point *result, vli_set(result->y, ry[0], ndigits); } +/* Computes R = P + Q mod p */ +static void ecc_point_add(const struct ecc_point *result, + const struct ecc_point *p, const struct ecc_point *q, + const struct ecc_curve *curve) +{ + u64 z[ECC_MAX_DIGITS]; + u64 px[ECC_MAX_DIGITS]; + u64 py[ECC_MAX_DIGITS]; + unsigned int ndigits = curve->g.ndigits; + + vli_set(result->x, q->x, ndigits); + vli_set(result->y, q->y, ndigits); + vli_mod_sub(z, result->x, p->x, curve->p, ndigits); + vli_set(px, p->x, ndigits); + vli_set(py, p->y, ndigits); + xycz_add(px, py, result->x, result->y, curve->p, ndigits); + vli_mod_inv(z, z, curve->p, ndigits); + apply_z(result->x, result->y, z, curve->p, ndigits); +} + +/* Computes R = u1P + u2Q mod p using Shamir's trick. + * Based on: Kenneth MacKay's micro-ecc (2014). + */ +void ecc_point_mult_shamir(const struct ecc_point *result, + const u64 *u1, const struct ecc_point *p, + const u64 *u2, const struct ecc_point *q, + const struct ecc_curve *curve) +{ + u64 z[ECC_MAX_DIGITS]; + u64 sump[2][ECC_MAX_DIGITS]; + u64 *rx = result->x; + u64 *ry = result->y; + unsigned int ndigits = curve->g.ndigits; + unsigned int num_bits; + struct ecc_point sum = ECC_POINT_INIT(sump[0], sump[1], ndigits); + const struct ecc_point *points[4]; + const struct ecc_point *point; + unsigned int idx; + int i; + + ecc_point_add(&sum, p, q, curve); + points[0] = NULL; + points[1] = p; + points[2] = q; + points[3] = ∑ + + num_bits = max(vli_num_bits(u1, ndigits), + vli_num_bits(u2, ndigits)); + i = num_bits - 1; + idx = (!!vli_test_bit(u1, i)) | ((!!vli_test_bit(u2, i)) << 1); + point = points[idx]; + + vli_set(rx, point->x, ndigits); + vli_set(ry, point->y, ndigits); + vli_clear(z + 1, ndigits - 1); + z[0] = 1; + + for (--i; i >= 0; i--) { + ecc_point_double_jacobian(rx, ry, z, curve->p, ndigits); + idx = (!!vli_test_bit(u1, i)) | ((!!vli_test_bit(u2, i)) << 1); + point = points[idx]; + if (point) { + u64 tx[ECC_MAX_DIGITS]; + u64 ty[ECC_MAX_DIGITS]; + u64 tz[ECC_MAX_DIGITS]; + + vli_set(tx, point->x, ndigits); + vli_set(ty, point->y, ndigits); + apply_z(tx, ty, z, curve->p, ndigits); + vli_mod_sub(tz, rx, tx, curve->p, ndigits); + xycz_add(tx, ty, rx, ry, curve->p, ndigits); + vli_mod_mult_fast(z, z, tz, curve->p, ndigits); + } + } + vli_mod_inv(z, z, curve->p, ndigits); + apply_z(rx, ry, z, curve->p, ndigits); +} +EXPORT_SYMBOL(ecc_point_mult_shamir); + static inline void ecc_swap_digits(const u64 *in, u64 *out, unsigned int ndigits) { @@ -948,6 +1326,7 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, return __ecc_is_key_valid(curve, private_key, ndigits); } +EXPORT_SYMBOL(ecc_is_key_valid); /* * ECC private keys are generated using the method of extra random bits, @@ -1000,6 +1379,7 @@ int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, u64 *privkey) return 0; } +EXPORT_SYMBOL(ecc_gen_privkey); int ecc_make_pub_key(unsigned int curve_id, unsigned int ndigits, const u64 *private_key, u64 *public_key) @@ -1036,13 +1416,17 @@ err_free_point: out: return ret; } +EXPORT_SYMBOL(ecc_make_pub_key); /* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */ -static int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve, - struct ecc_point *pk) +int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve, + struct ecc_point *pk) { u64 yy[ECC_MAX_DIGITS], xxx[ECC_MAX_DIGITS], w[ECC_MAX_DIGITS]; + if (WARN_ON(pk->ndigits != curve->g.ndigits)) + return -EINVAL; + /* Check 1: Verify key is not the zero point. */ if (ecc_point_is_zero(pk)) return -EINVAL; @@ -1064,8 +1448,8 @@ static int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve, return -EINVAL; return 0; - } +EXPORT_SYMBOL(ecc_is_pubkey_valid_partial); int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, const u64 *private_key, const u64 *public_key, @@ -1121,3 +1505,6 @@ err_alloc_product: out: return ret; } +EXPORT_SYMBOL(crypto_ecdh_shared_secret); + +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/ecc.h b/crypto/ecc.h index f75a86baa3bd..ab0eb70b9c09 100644 --- a/crypto/ecc.h +++ b/crypto/ecc.h @@ -26,13 +26,51 @@ #ifndef _CRYPTO_ECC_H #define _CRYPTO_ECC_H +/* One digit is u64 qword. */ #define ECC_CURVE_NIST_P192_DIGITS 3 #define ECC_CURVE_NIST_P256_DIGITS 4 -#define ECC_MAX_DIGITS ECC_CURVE_NIST_P256_DIGITS +#define ECC_MAX_DIGITS (512 / 64) #define ECC_DIGITS_TO_BYTES_SHIFT 3 /** + * struct ecc_point - elliptic curve point in affine coordinates + * + * @x: X coordinate in vli form. + * @y: Y coordinate in vli form. + * @ndigits: Length of vlis in u64 qwords. + */ +struct ecc_point { + u64 *x; + u64 *y; + u8 ndigits; +}; + +#define ECC_POINT_INIT(x, y, ndigits) (struct ecc_point) { x, y, ndigits } + +/** + * struct ecc_curve - definition of elliptic curve + * + * @name: Short name of the curve. + * @g: Generator point of the curve. + * @p: Prime number, if Barrett's reduction is used for this curve + * pre-calculated value 'mu' is appended to the @p after ndigits. + * Use of Barrett's reduction is heuristically determined in + * vli_mmod_fast(). + * @n: Order of the curve group. + * @a: Curve parameter a. + * @b: Curve parameter b. + */ +struct ecc_curve { + char *name; + struct ecc_point g; + u64 *p; + u64 *n; + u64 *a; + u64 *b; +}; + +/** * ecc_is_key_valid() - Validate a given ECDH private key * * @curve_id: id representing the curve to use @@ -91,4 +129,117 @@ int ecc_make_pub_key(const unsigned int curve_id, unsigned int ndigits, int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, const u64 *private_key, const u64 *public_key, u64 *secret); + +/** + * ecc_is_pubkey_valid_partial() - Partial public key validation + * + * @curve: elliptic curve domain parameters + * @pk: public key as a point + * + * Valdiate public key according to SP800-56A section 5.6.2.3.4 ECC Partial + * Public-Key Validation Routine. + * + * Note: There is no check that the public key is in the correct elliptic curve + * subgroup. + * + * Return: 0 if validation is successful, -EINVAL if validation is failed. + */ +int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve, + struct ecc_point *pk); + +/** + * vli_is_zero() - Determine is vli is zero + * + * @vli: vli to check. + * @ndigits: length of the @vli + */ +bool vli_is_zero(const u64 *vli, unsigned int ndigits); + +/** + * vli_cmp() - compare left and right vlis + * + * @left: vli + * @right: vli + * @ndigits: length of both vlis + * + * Returns sign of @left - @right, i.e. -1 if @left < @right, + * 0 if @left == @right, 1 if @left > @right. + */ +int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits); + +/** + * vli_sub() - Subtracts right from left + * + * @result: where to write result + * @left: vli + * @right vli + * @ndigits: length of all vlis + * + * Note: can modify in-place. + * + * Return: carry bit. + */ +u64 vli_sub(u64 *result, const u64 *left, const u64 *right, + unsigned int ndigits); + +/** + * vli_from_be64() - Load vli from big-endian u64 array + * + * @dest: destination vli + * @src: source array of u64 BE values + * @ndigits: length of both vli and array + */ +void vli_from_be64(u64 *dest, const void *src, unsigned int ndigits); + +/** + * vli_from_le64() - Load vli from little-endian u64 array + * + * @dest: destination vli + * @src: source array of u64 LE values + * @ndigits: length of both vli and array + */ +void vli_from_le64(u64 *dest, const void *src, unsigned int ndigits); + +/** + * vli_mod_inv() - Modular inversion + * + * @result: where to write vli number + * @input: vli value to operate on + * @mod: modulus + * @ndigits: length of all vlis + */ +void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod, + unsigned int ndigits); + +/** + * vli_mod_mult_slow() - Modular multiplication + * + * @result: where to write result value + * @left: vli number to multiply with @right + * @right: vli number to multiply with @left + * @mod: modulus + * @ndigits: length of all vlis + * + * Note: Assumes that mod is big enough curve order. + */ +void vli_mod_mult_slow(u64 *result, const u64 *left, const u64 *right, + const u64 *mod, unsigned int ndigits); + +/** + * ecc_point_mult_shamir() - Add two points multiplied by scalars + * + * @result: resulting point + * @x: scalar to multiply with @p + * @p: point to multiply with @x + * @y: scalar to multiply with @q + * @q: point to multiply with @y + * @curve: curve + * + * Returns result = x * p + x * q over the curve. + * This works faster than two multiplications and addition. + */ +void ecc_point_mult_shamir(const struct ecc_point *result, + const u64 *x, const struct ecc_point *p, + const u64 *y, const struct ecc_point *q, + const struct ecc_curve *curve); #endif diff --git a/crypto/ecc_curve_defs.h b/crypto/ecc_curve_defs.h index 336ab1805639..69be6c7d228f 100644 --- a/crypto/ecc_curve_defs.h +++ b/crypto/ecc_curve_defs.h @@ -2,21 +2,6 @@ #ifndef _CRYTO_ECC_CURVE_DEFS_H #define _CRYTO_ECC_CURVE_DEFS_H -struct ecc_point { - u64 *x; - u64 *y; - u8 ndigits; -}; - -struct ecc_curve { - char *name; - struct ecc_point g; - u64 *p; - u64 *n; - u64 *a; - u64 *b; -}; - /* NIST P-192: a = p - 3 */ static u64 nist_p192_g_x[] = { 0xF4FF0AFD82FF1012ull, 0x7CBF20EB43A18800ull, 0x188DA80EB03090F6ull }; diff --git a/crypto/ecdh.c b/crypto/ecdh.c index bf6300175b9c..890092bd8989 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -166,7 +166,7 @@ static void ecdh_exit(void) crypto_unregister_kpp(&ecdh); } -module_init(ecdh_init); +subsys_initcall(ecdh_init); module_exit(ecdh_exit); MODULE_ALIAS_CRYPTO("ecdh"); MODULE_LICENSE("GPL"); diff --git a/crypto/echainiv.c b/crypto/echainiv.c index 77e607fdbfb7..e71d1bc8d850 100644 --- a/crypto/echainiv.c +++ b/crypto/echainiv.c @@ -174,7 +174,7 @@ static void __exit echainiv_module_exit(void) crypto_unregister_template(&echainiv_tmpl); } -module_init(echainiv_module_init); +subsys_initcall(echainiv_module_init); module_exit(echainiv_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ecrdsa.c b/crypto/ecrdsa.c new file mode 100644 index 000000000000..887ec21aee49 --- /dev/null +++ b/crypto/ecrdsa.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Elliptic Curve (Russian) Digital Signature Algorithm for Cryptographic API + * + * Copyright (c) 2019 Vitaly Chikunov <vt@altlinux.org> + * + * References: + * GOST 34.10-2018, GOST R 34.10-2012, RFC 7091, ISO/IEC 14888-3:2018. + * + * Historical references: + * GOST R 34.10-2001, RFC 4357, ISO/IEC 14888-3:2006/Amd 1:2010. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include <linux/module.h> +#include <linux/crypto.h> +#include <crypto/streebog.h> +#include <crypto/internal/akcipher.h> +#include <crypto/akcipher.h> +#include <linux/oid_registry.h> +#include "ecrdsa_params.asn1.h" +#include "ecrdsa_pub_key.asn1.h" +#include "ecc.h" +#include "ecrdsa_defs.h" + +#define ECRDSA_MAX_SIG_SIZE (2 * 512 / 8) +#define ECRDSA_MAX_DIGITS (512 / 64) + +struct ecrdsa_ctx { + enum OID algo_oid; /* overall public key oid */ + enum OID curve_oid; /* parameter */ + enum OID digest_oid; /* parameter */ + const struct ecc_curve *curve; /* curve from oid */ + unsigned int digest_len; /* parameter (bytes) */ + const char *digest; /* digest name from oid */ + unsigned int key_len; /* @key length (bytes) */ + const char *key; /* raw public key */ + struct ecc_point pub_key; + u64 _pubp[2][ECRDSA_MAX_DIGITS]; /* point storage for @pub_key */ +}; + +static const struct ecc_curve *get_curve_by_oid(enum OID oid) +{ + switch (oid) { + case OID_gostCPSignA: + case OID_gostTC26Sign256B: + return &gost_cp256a; + case OID_gostCPSignB: + case OID_gostTC26Sign256C: + return &gost_cp256b; + case OID_gostCPSignC: + case OID_gostTC26Sign256D: + return &gost_cp256c; + case OID_gostTC26Sign512A: + return &gost_tc512a; + case OID_gostTC26Sign512B: + return &gost_tc512b; + /* The following two aren't implemented: */ + case OID_gostTC26Sign256A: + case OID_gostTC26Sign512C: + default: + return NULL; + } +} + +static int ecrdsa_verify(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct ecrdsa_ctx *ctx = akcipher_tfm_ctx(tfm); + unsigned char sig[ECRDSA_MAX_SIG_SIZE]; + unsigned char digest[STREEBOG512_DIGEST_SIZE]; + unsigned int ndigits = req->dst_len / sizeof(u64); + u64 r[ECRDSA_MAX_DIGITS]; /* witness (r) */ + u64 _r[ECRDSA_MAX_DIGITS]; /* -r */ + u64 s[ECRDSA_MAX_DIGITS]; /* second part of sig (s) */ + u64 e[ECRDSA_MAX_DIGITS]; /* h \mod q */ + u64 *v = e; /* e^{-1} \mod q */ + u64 z1[ECRDSA_MAX_DIGITS]; + u64 *z2 = _r; + struct ecc_point cc = ECC_POINT_INIT(s, e, ndigits); /* reuse s, e */ + + /* + * Digest value, digest algorithm, and curve (modulus) should have the + * same length (256 or 512 bits), public key and signature should be + * twice bigger. + */ + if (!ctx->curve || + !ctx->digest || + !req->src || + !ctx->pub_key.x || + req->dst_len != ctx->digest_len || + req->dst_len != ctx->curve->g.ndigits * sizeof(u64) || + ctx->pub_key.ndigits != ctx->curve->g.ndigits || + req->dst_len * 2 != req->src_len || + WARN_ON(req->src_len > sizeof(sig)) || + WARN_ON(req->dst_len > sizeof(digest))) + return -EBADMSG; + + sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, req->src_len), + sig, req->src_len); + sg_pcopy_to_buffer(req->src, + sg_nents_for_len(req->src, + req->src_len + req->dst_len), + digest, req->dst_len, req->src_len); + + vli_from_be64(s, sig, ndigits); + vli_from_be64(r, sig + ndigits * sizeof(u64), ndigits); + + /* Step 1: verify that 0 < r < q, 0 < s < q */ + if (vli_is_zero(r, ndigits) || + vli_cmp(r, ctx->curve->n, ndigits) == 1 || + vli_is_zero(s, ndigits) || + vli_cmp(s, ctx->curve->n, ndigits) == 1) + return -EKEYREJECTED; + + /* Step 2: calculate hash (h) of the message (passed as input) */ + /* Step 3: calculate e = h \mod q */ + vli_from_le64(e, digest, ndigits); + if (vli_cmp(e, ctx->curve->n, ndigits) == 1) + vli_sub(e, e, ctx->curve->n, ndigits); + if (vli_is_zero(e, ndigits)) + e[0] = 1; + + /* Step 4: calculate v = e^{-1} \mod q */ + vli_mod_inv(v, e, ctx->curve->n, ndigits); + + /* Step 5: calculate z_1 = sv \mod q, z_2 = -rv \mod q */ + vli_mod_mult_slow(z1, s, v, ctx->curve->n, ndigits); + vli_sub(_r, ctx->curve->n, r, ndigits); + vli_mod_mult_slow(z2, _r, v, ctx->curve->n, ndigits); + + /* Step 6: calculate point C = z_1P + z_2Q, and R = x_c \mod q */ + ecc_point_mult_shamir(&cc, z1, &ctx->curve->g, z2, &ctx->pub_key, + ctx->curve); + if (vli_cmp(cc.x, ctx->curve->n, ndigits) == 1) + vli_sub(cc.x, cc.x, ctx->curve->n, ndigits); + + /* Step 7: if R == r signature is valid */ + if (!vli_cmp(cc.x, r, ndigits)) + return 0; + else + return -EKEYREJECTED; +} + +int ecrdsa_param_curve(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct ecrdsa_ctx *ctx = context; + + ctx->curve_oid = look_up_OID(value, vlen); + if (!ctx->curve_oid) + return -EINVAL; + ctx->curve = get_curve_by_oid(ctx->curve_oid); + return 0; +} + +/* Optional. If present should match expected digest algo OID. */ +int ecrdsa_param_digest(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct ecrdsa_ctx *ctx = context; + int digest_oid = look_up_OID(value, vlen); + + if (digest_oid != ctx->digest_oid) + return -EINVAL; + return 0; +} + +int ecrdsa_parse_pub_key(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct ecrdsa_ctx *ctx = context; + + ctx->key = value; + ctx->key_len = vlen; + return 0; +} + +static u8 *ecrdsa_unpack_u32(u32 *dst, void *src) +{ + memcpy(dst, src, sizeof(u32)); + return src + sizeof(u32); +} + +/* Parse BER encoded subjectPublicKey. */ +static int ecrdsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct ecrdsa_ctx *ctx = akcipher_tfm_ctx(tfm); + unsigned int ndigits; + u32 algo, paramlen; + u8 *params; + int err; + + err = asn1_ber_decoder(&ecrdsa_pub_key_decoder, ctx, key, keylen); + if (err < 0) + return err; + + /* Key parameters is in the key after keylen. */ + params = ecrdsa_unpack_u32(¶mlen, + ecrdsa_unpack_u32(&algo, (u8 *)key + keylen)); + + if (algo == OID_gost2012PKey256) { + ctx->digest = "streebog256"; + ctx->digest_oid = OID_gost2012Digest256; + ctx->digest_len = 256 / 8; + } else if (algo == OID_gost2012PKey512) { + ctx->digest = "streebog512"; + ctx->digest_oid = OID_gost2012Digest512; + ctx->digest_len = 512 / 8; + } else + return -ENOPKG; + ctx->algo_oid = algo; + + /* Parse SubjectPublicKeyInfo.AlgorithmIdentifier.parameters. */ + err = asn1_ber_decoder(&ecrdsa_params_decoder, ctx, params, paramlen); + if (err < 0) + return err; + /* + * Sizes of algo (set in digest_len) and curve should match + * each other. + */ + if (!ctx->curve || + ctx->curve->g.ndigits * sizeof(u64) != ctx->digest_len) + return -ENOPKG; + /* + * Key is two 256- or 512-bit coordinates which should match + * curve size. + */ + if ((ctx->key_len != (2 * 256 / 8) && + ctx->key_len != (2 * 512 / 8)) || + ctx->key_len != ctx->curve->g.ndigits * sizeof(u64) * 2) + return -ENOPKG; + + ndigits = ctx->key_len / sizeof(u64) / 2; + ctx->pub_key = ECC_POINT_INIT(ctx->_pubp[0], ctx->_pubp[1], ndigits); + vli_from_le64(ctx->pub_key.x, ctx->key, ndigits); + vli_from_le64(ctx->pub_key.y, ctx->key + ndigits * sizeof(u64), + ndigits); + + if (ecc_is_pubkey_valid_partial(ctx->curve, &ctx->pub_key)) + return -EKEYREJECTED; + + return 0; +} + +static unsigned int ecrdsa_max_size(struct crypto_akcipher *tfm) +{ + struct ecrdsa_ctx *ctx = akcipher_tfm_ctx(tfm); + + /* + * Verify doesn't need any output, so it's just informational + * for keyctl to determine the key bit size. + */ + return ctx->pub_key.ndigits * sizeof(u64); +} + +static void ecrdsa_exit_tfm(struct crypto_akcipher *tfm) +{ +} + +static struct akcipher_alg ecrdsa_alg = { + .verify = ecrdsa_verify, + .set_pub_key = ecrdsa_set_pub_key, + .max_size = ecrdsa_max_size, + .exit = ecrdsa_exit_tfm, + .base = { + .cra_name = "ecrdsa", + .cra_driver_name = "ecrdsa-generic", + .cra_priority = 100, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct ecrdsa_ctx), + }, +}; + +static int __init ecrdsa_mod_init(void) +{ + return crypto_register_akcipher(&ecrdsa_alg); +} + +static void __exit ecrdsa_mod_fini(void) +{ + crypto_unregister_akcipher(&ecrdsa_alg); +} + +module_init(ecrdsa_mod_init); +module_exit(ecrdsa_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Vitaly Chikunov <vt@altlinux.org>"); +MODULE_DESCRIPTION("EC-RDSA generic algorithm"); +MODULE_ALIAS_CRYPTO("ecrdsa-generic"); diff --git a/crypto/ecrdsa_defs.h b/crypto/ecrdsa_defs.h new file mode 100644 index 000000000000..170baf039007 --- /dev/null +++ b/crypto/ecrdsa_defs.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Definitions of EC-RDSA Curve Parameters + * + * Copyright (c) 2019 Vitaly Chikunov <vt@altlinux.org> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#ifndef _CRYTO_ECRDSA_DEFS_H +#define _CRYTO_ECRDSA_DEFS_H + +#include "ecc.h" + +#define ECRDSA_MAX_SIG_SIZE (2 * 512 / 8) +#define ECRDSA_MAX_DIGITS (512 / 64) + +/* + * EC-RDSA uses its own set of curves. + * + * cp256{a,b,c} curves first defined for GOST R 34.10-2001 in RFC 4357 (as + * 256-bit {A,B,C}-ParamSet), but inherited for GOST R 34.10-2012 and + * proposed for use in R 50.1.114-2016 and RFC 7836 as the 256-bit curves. + */ +/* OID_gostCPSignA 1.2.643.2.2.35.1 */ +static u64 cp256a_g_x[] = { + 0x0000000000000001ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; +static u64 cp256a_g_y[] = { + 0x22ACC99C9E9F1E14ull, 0x35294F2DDF23E3B1ull, + 0x27DF505A453F2B76ull, 0x8D91E471E0989CDAull, }; +static u64 cp256a_p[] = { /* p = 2^256 - 617 */ + 0xFFFFFFFFFFFFFD97ull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull }; +static u64 cp256a_n[] = { + 0x45841B09B761B893ull, 0x6C611070995AD100ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull }; +static u64 cp256a_a[] = { /* a = p - 3 */ + 0xFFFFFFFFFFFFFD94ull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull }; +static u64 cp256a_b[] = { + 0x00000000000000a6ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull }; + +static struct ecc_curve gost_cp256a = { + .name = "cp256a", + .g = { + .x = cp256a_g_x, + .y = cp256a_g_y, + .ndigits = 256 / 64, + }, + .p = cp256a_p, + .n = cp256a_n, + .a = cp256a_a, + .b = cp256a_b +}; + +/* OID_gostCPSignB 1.2.643.2.2.35.2 */ +static u64 cp256b_g_x[] = { + 0x0000000000000001ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; +static u64 cp256b_g_y[] = { + 0x744BF8D717717EFCull, 0xC545C9858D03ECFBull, + 0xB83D1C3EB2C070E5ull, 0x3FA8124359F96680ull, }; +static u64 cp256b_p[] = { /* p = 2^255 + 3225 */ + 0x0000000000000C99ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x8000000000000000ull, }; +static u64 cp256b_n[] = { + 0xE497161BCC8A198Full, 0x5F700CFFF1A624E5ull, + 0x0000000000000001ull, 0x8000000000000000ull, }; +static u64 cp256b_a[] = { /* a = p - 3 */ + 0x0000000000000C96ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x8000000000000000ull, }; +static u64 cp256b_b[] = { + 0x2F49D4CE7E1BBC8Bull, 0xE979259373FF2B18ull, + 0x66A7D3C25C3DF80Aull, 0x3E1AF419A269A5F8ull, }; + +static struct ecc_curve gost_cp256b = { + .name = "cp256b", + .g = { + .x = cp256b_g_x, + .y = cp256b_g_y, + .ndigits = 256 / 64, + }, + .p = cp256b_p, + .n = cp256b_n, + .a = cp256b_a, + .b = cp256b_b +}; + +/* OID_gostCPSignC 1.2.643.2.2.35.3 */ +static u64 cp256c_g_x[] = { + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; +static u64 cp256c_g_y[] = { + 0x366E550DFDB3BB67ull, 0x4D4DC440D4641A8Full, + 0x3CBF3783CD08C0EEull, 0x41ECE55743711A8Cull, }; +static u64 cp256c_p[] = { + 0x7998F7B9022D759Bull, 0xCF846E86789051D3ull, + 0xAB1EC85E6B41C8AAull, 0x9B9F605F5A858107ull, + /* pre-computed value for Barrett's reduction */ + 0xedc283cdd217b5a2ull, 0xbac48fc06398ae59ull, + 0x405384d55f9f3b73ull, 0xa51f176161f1d734ull, + 0x0000000000000001ull, }; +static u64 cp256c_n[] = { + 0xF02F3A6598980BB9ull, 0x582CA3511EDDFB74ull, + 0xAB1EC85E6B41C8AAull, 0x9B9F605F5A858107ull, }; +static u64 cp256c_a[] = { /* a = p - 3 */ + 0x7998F7B9022D7598ull, 0xCF846E86789051D3ull, + 0xAB1EC85E6B41C8AAull, 0x9B9F605F5A858107ull, }; +static u64 cp256c_b[] = { + 0x000000000000805aull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; + +static struct ecc_curve gost_cp256c = { + .name = "cp256c", + .g = { + .x = cp256c_g_x, + .y = cp256c_g_y, + .ndigits = 256 / 64, + }, + .p = cp256c_p, + .n = cp256c_n, + .a = cp256c_a, + .b = cp256c_b +}; + +/* tc512{a,b} curves first recommended in 2013 and then standardized in + * R 50.1.114-2016 and RFC 7836 for use with GOST R 34.10-2012 (as TC26 + * 512-bit ParamSet{A,B}). + */ +/* OID_gostTC26Sign512A 1.2.643.7.1.2.1.2.1 */ +static u64 tc512a_g_x[] = { + 0x0000000000000003ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; +static u64 tc512a_g_y[] = { + 0x89A589CB5215F2A4ull, 0x8028FE5FC235F5B8ull, + 0x3D75E6A50E3A41E9ull, 0xDF1626BE4FD036E9ull, + 0x778064FDCBEFA921ull, 0xCE5E1C93ACF1ABC1ull, + 0xA61B8816E25450E6ull, 0x7503CFE87A836AE3ull, }; +static u64 tc512a_p[] = { /* p = 2^512 - 569 */ + 0xFFFFFFFFFFFFFDC7ull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, }; +static u64 tc512a_n[] = { + 0xCACDB1411F10B275ull, 0x9B4B38ABFAD2B85Dull, + 0x6FF22B8D4E056060ull, 0x27E69532F48D8911ull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, }; +static u64 tc512a_a[] = { /* a = p - 3 */ + 0xFFFFFFFFFFFFFDC4ull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, + 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, }; +static u64 tc512a_b[] = { + 0x503190785A71C760ull, 0x862EF9D4EBEE4761ull, + 0x4CB4574010DA90DDull, 0xEE3CB090F30D2761ull, + 0x79BD081CFD0B6265ull, 0x34B82574761CB0E8ull, + 0xC1BD0B2B6667F1DAull, 0xE8C2505DEDFC86DDull, }; + +static struct ecc_curve gost_tc512a = { + .name = "tc512a", + .g = { + .x = tc512a_g_x, + .y = tc512a_g_y, + .ndigits = 512 / 64, + }, + .p = tc512a_p, + .n = tc512a_n, + .a = tc512a_a, + .b = tc512a_b +}; + +/* OID_gostTC26Sign512B 1.2.643.7.1.2.1.2.2 */ +static u64 tc512b_g_x[] = { + 0x0000000000000002ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, }; +static u64 tc512b_g_y[] = { + 0x7E21340780FE41BDull, 0x28041055F94CEEECull, + 0x152CBCAAF8C03988ull, 0xDCB228FD1EDF4A39ull, + 0xBE6DD9E6C8EC7335ull, 0x3C123B697578C213ull, + 0x2C071E3647A8940Full, 0x1A8F7EDA389B094Cull, }; +static u64 tc512b_p[] = { /* p = 2^511 + 111 */ + 0x000000000000006Full, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x8000000000000000ull, }; +static u64 tc512b_n[] = { + 0xC6346C54374F25BDull, 0x8B996712101BEA0Eull, + 0xACFDB77BD9D40CFAull, 0x49A1EC142565A545ull, + 0x0000000000000001ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x8000000000000000ull, }; +static u64 tc512b_a[] = { /* a = p - 3 */ + 0x000000000000006Cull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x0000000000000000ull, + 0x0000000000000000ull, 0x8000000000000000ull, }; +static u64 tc512b_b[] = { + 0xFB8CCBC7C5140116ull, 0x50F78BEE1FA3106Eull, + 0x7F8B276FAD1AB69Cull, 0x3E965D2DB1416D21ull, + 0xBF85DC806C4B289Full, 0xB97C7D614AF138BCull, + 0x7E3E06CF6F5E2517ull, 0x687D1B459DC84145ull, }; + +static struct ecc_curve gost_tc512b = { + .name = "tc512b", + .g = { + .x = tc512b_g_x, + .y = tc512b_g_y, + .ndigits = 512 / 64, + }, + .p = tc512b_p, + .n = tc512b_n, + .a = tc512b_a, + .b = tc512b_b +}; + +#endif diff --git a/crypto/ecrdsa_params.asn1 b/crypto/ecrdsa_params.asn1 new file mode 100644 index 000000000000..aba99c3763cf --- /dev/null +++ b/crypto/ecrdsa_params.asn1 @@ -0,0 +1,4 @@ +EcrdsaParams ::= SEQUENCE { + curve OBJECT IDENTIFIER ({ ecrdsa_param_curve }), + digest OBJECT IDENTIFIER OPTIONAL ({ ecrdsa_param_digest }) +} diff --git a/crypto/ecrdsa_pub_key.asn1 b/crypto/ecrdsa_pub_key.asn1 new file mode 100644 index 000000000000..048cb646bce4 --- /dev/null +++ b/crypto/ecrdsa_pub_key.asn1 @@ -0,0 +1 @@ +EcrdsaPubKey ::= OCTET STRING ({ ecrdsa_parse_pub_key }) diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c index 77286ea28865..4e8704405a3b 100644 --- a/crypto/fcrypt.c +++ b/crypto/fcrypt.c @@ -414,7 +414,7 @@ static void __exit fcrypt_mod_fini(void) crypto_unregister_alg(&fcrypt_alg); } -module_init(fcrypt_mod_init); +subsys_initcall(fcrypt_mod_init); module_exit(fcrypt_mod_fini); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/fips.c b/crypto/fips.c index 9d627c1cf8bc..9dfed122d6da 100644 --- a/crypto/fips.c +++ b/crypto/fips.c @@ -74,5 +74,5 @@ static void __exit fips_exit(void) crypto_proc_fips_exit(); } -module_init(fips_init); +subsys_initcall(fips_init); module_exit(fips_exit); diff --git a/crypto/gcm.c b/crypto/gcm.c index e1a11f529d25..33f45a980967 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -597,7 +597,6 @@ static void crypto_gcm_free(struct aead_instance *inst) static int crypto_gcm_create_common(struct crypto_template *tmpl, struct rtattr **tb, - const char *full_name, const char *ctr_name, const char *ghash_name) { @@ -638,7 +637,8 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, goto err_free_inst; err = -EINVAL; - if (ghash->digestsize != 16) + if (strcmp(ghash->base.cra_name, "ghash") != 0 || + ghash->digestsize != 16) goto err_drop_ghash; crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst)); @@ -650,24 +650,24 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, ctr = crypto_spawn_skcipher_alg(&ctx->ctr); - /* We only support 16-byte blocks. */ + /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */ err = -EINVAL; - if (crypto_skcipher_alg_ivsize(ctr) != 16) + if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 || + crypto_skcipher_alg_ivsize(ctr) != 16 || + ctr->base.cra_blocksize != 1) goto out_put_ctr; - /* Not a stream cipher? */ - if (ctr->base.cra_blocksize != 1) + err = -ENAMETOOLONG; + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME) goto out_put_ctr; - err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", ctr->base.cra_driver_name, ghash_alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto out_put_ctr; - memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME); - inst->alg.base.cra_flags = (ghash->base.cra_flags | ctr->base.cra_flags) & CRYPTO_ALG_ASYNC; inst->alg.base.cra_priority = (ghash->base.cra_priority + @@ -709,7 +709,6 @@ static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) { const char *cipher_name; char ctr_name[CRYPTO_MAX_ALG_NAME]; - char full_name[CRYPTO_MAX_ALG_NAME]; cipher_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(cipher_name)) @@ -719,12 +718,7 @@ static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; - if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >= - CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - return crypto_gcm_create_common(tmpl, tb, full_name, - ctr_name, "ghash"); + return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash"); } static int crypto_gcm_base_create(struct crypto_template *tmpl, @@ -732,7 +726,6 @@ static int crypto_gcm_base_create(struct crypto_template *tmpl, { const char *ctr_name; const char *ghash_name; - char full_name[CRYPTO_MAX_ALG_NAME]; ctr_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(ctr_name)) @@ -742,12 +735,7 @@ static int crypto_gcm_base_create(struct crypto_template *tmpl, if (IS_ERR(ghash_name)) return PTR_ERR(ghash_name); - if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", - ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - return crypto_gcm_create_common(tmpl, tb, full_name, - ctr_name, ghash_name); + return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name); } static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, @@ -1258,7 +1246,7 @@ static void __exit crypto_gcm_module_exit(void) ARRAY_SIZE(crypto_gcm_tmpls)); } -module_init(crypto_gcm_module_init); +subsys_initcall(crypto_gcm_module_init); module_exit(crypto_gcm_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c index d9f192b953b2..e6307935413c 100644 --- a/crypto/ghash-generic.c +++ b/crypto/ghash-generic.c @@ -149,7 +149,7 @@ static void __exit ghash_mod_exit(void) crypto_unregister_shash(&ghash_alg); } -module_init(ghash_mod_init); +subsys_initcall(ghash_mod_init); module_exit(ghash_mod_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/hmac.c b/crypto/hmac.c index e74730224f0a..a68c1266121f 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -57,8 +57,6 @@ static int hmac_setkey(struct crypto_shash *parent, unsigned int i; shash->tfm = hash; - shash->flags = crypto_shash_get_flags(parent) - & CRYPTO_TFM_REQ_MAY_SLEEP; if (keylen > bs) { int err; @@ -91,8 +89,6 @@ static int hmac_export(struct shash_desc *pdesc, void *out) { struct shash_desc *desc = shash_desc_ctx(pdesc); - desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_shash_export(desc, out); } @@ -102,7 +98,6 @@ static int hmac_import(struct shash_desc *pdesc, const void *in) struct hmac_ctx *ctx = hmac_ctx(pdesc->tfm); desc->tfm = ctx->hash; - desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; return crypto_shash_import(desc, in); } @@ -117,8 +112,6 @@ static int hmac_update(struct shash_desc *pdesc, { struct shash_desc *desc = shash_desc_ctx(pdesc); - desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_shash_update(desc, data, nbytes); } @@ -130,8 +123,6 @@ static int hmac_final(struct shash_desc *pdesc, u8 *out) char *opad = crypto_shash_ctx_aligned(parent) + ss; struct shash_desc *desc = shash_desc_ctx(pdesc); - desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_shash_final(desc, out) ?: crypto_shash_import(desc, opad) ?: crypto_shash_finup(desc, out, ds, out); @@ -147,8 +138,6 @@ static int hmac_finup(struct shash_desc *pdesc, const u8 *data, char *opad = crypto_shash_ctx_aligned(parent) + ss; struct shash_desc *desc = shash_desc_ctx(pdesc); - desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_shash_finup(desc, data, nbytes, out) ?: crypto_shash_import(desc, opad) ?: crypto_shash_finup(desc, out, ds, out); @@ -268,7 +257,7 @@ static void __exit hmac_module_exit(void) crypto_unregister_template(&hmac_tmpl); } -module_init(hmac_module_init); +subsys_initcall(hmac_module_init); module_exit(hmac_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 787dccca3715..6ea1a270b8dc 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -198,7 +198,7 @@ static void __exit jent_mod_exit(void) crypto_unregister_rng(&jent_alg); } -module_init(jent_mod_init); +subsys_initcall(jent_mod_init); module_exit(jent_mod_exit); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/keywrap.c b/crypto/keywrap.c index a5cfe610d8f4..a155c88105ea 100644 --- a/crypto/keywrap.c +++ b/crypto/keywrap.c @@ -310,7 +310,7 @@ static void __exit crypto_kw_exit(void) crypto_unregister_template(&crypto_kw_tmpl); } -module_init(crypto_kw_init); +subsys_initcall(crypto_kw_init); module_exit(crypto_kw_exit); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/khazad.c b/crypto/khazad.c index 873eb5ded6d7..b50aa8a3ab4c 100644 --- a/crypto/khazad.c +++ b/crypto/khazad.c @@ -875,7 +875,7 @@ static void __exit khazad_mod_fini(void) } -module_init(khazad_mod_init); +subsys_initcall(khazad_mod_init); module_exit(khazad_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lrw.c b/crypto/lrw.c index 08a0e458bc3e..fa302f3f161e 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -162,8 +162,10 @@ static int xor_tweak(struct skcipher_request *req, bool second_pass) } err = skcipher_walk_virt(&w, req, false); - iv = (__be32 *)w.iv; + if (err) + return err; + iv = (__be32 *)w.iv; counter[0] = be32_to_cpu(iv[3]); counter[1] = be32_to_cpu(iv[2]); counter[2] = be32_to_cpu(iv[1]); @@ -435,7 +437,7 @@ static void __exit crypto_module_exit(void) crypto_unregister_template(&crypto_tmpl); } -module_init(crypto_module_init); +subsys_initcall(crypto_module_init); module_exit(crypto_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/lz4.c b/crypto/lz4.c index c160dfdbf2e0..1e35134d0a98 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -164,7 +164,7 @@ static void __exit lz4_mod_fini(void) crypto_unregister_scomp(&scomp); } -module_init(lz4_mod_init); +subsys_initcall(lz4_mod_init); module_exit(lz4_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index 583b5e013d7a..4a220b628fe7 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -165,7 +165,7 @@ static void __exit lz4hc_mod_fini(void) crypto_unregister_scomp(&scomp); } -module_init(lz4hc_mod_init); +subsys_initcall(lz4hc_mod_init); module_exit(lz4hc_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index ea9c75b1db49..4c82bf18440f 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -167,7 +167,7 @@ static void __exit lzorle_mod_fini(void) crypto_unregister_scomp(&scomp); } -module_init(lzorle_mod_init); +subsys_initcall(lzorle_mod_init); module_exit(lzorle_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lzo.c b/crypto/lzo.c index 218567d717d6..4a6ac8f247d0 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -167,7 +167,7 @@ static void __exit lzo_mod_fini(void) crypto_unregister_scomp(&scomp); } -module_init(lzo_mod_init); +subsys_initcall(lzo_mod_init); module_exit(lzo_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/md4.c b/crypto/md4.c index 9965ec40d9f9..9a1a228a0c69 100644 --- a/crypto/md4.c +++ b/crypto/md4.c @@ -232,7 +232,7 @@ static void __exit md4_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(md4_mod_init); +subsys_initcall(md4_mod_init); module_exit(md4_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/md5.c b/crypto/md5.c index 94dd78144ba3..221c2c0932f8 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -244,7 +244,7 @@ static void __exit md5_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(md5_mod_init); +subsys_initcall(md5_mod_init); module_exit(md5_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/michael_mic.c b/crypto/michael_mic.c index 46195e0d0f4d..538ae7933795 100644 --- a/crypto/michael_mic.c +++ b/crypto/michael_mic.c @@ -178,7 +178,7 @@ static void __exit michael_mic_exit(void) } -module_init(michael_mic_init); +subsys_initcall(michael_mic_init); module_exit(michael_mic_exit); MODULE_LICENSE("GPL v2"); diff --git a/crypto/morus1280.c b/crypto/morus1280.c index 0747732d5b78..f8734c6576af 100644 --- a/crypto/morus1280.c +++ b/crypto/morus1280.c @@ -532,7 +532,7 @@ static void __exit crypto_morus1280_module_exit(void) crypto_unregister_aead(&crypto_morus1280_alg); } -module_init(crypto_morus1280_module_init); +subsys_initcall(crypto_morus1280_module_init); module_exit(crypto_morus1280_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/morus640.c b/crypto/morus640.c index 1617a1eb8be1..ae5aa9482cb4 100644 --- a/crypto/morus640.c +++ b/crypto/morus640.c @@ -523,7 +523,7 @@ static void __exit crypto_morus640_module_exit(void) crypto_unregister_aead(&crypto_morus640_alg); } -module_init(crypto_morus640_module_init); +subsys_initcall(crypto_morus640_module_init); module_exit(crypto_morus640_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/nhpoly1305.c b/crypto/nhpoly1305.c index ec831a5594d8..9ab4e07cde4d 100644 --- a/crypto/nhpoly1305.c +++ b/crypto/nhpoly1305.c @@ -244,7 +244,7 @@ static void __exit nhpoly1305_mod_exit(void) crypto_unregister_shash(&nhpoly1305_alg); } -module_init(nhpoly1305_mod_init); +subsys_initcall(nhpoly1305_mod_init); module_exit(nhpoly1305_mod_exit); MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function"); diff --git a/crypto/ofb.c b/crypto/ofb.c index 34b6e1f426f7..133ff4c7f2c6 100644 --- a/crypto/ofb.c +++ b/crypto/ofb.c @@ -95,7 +95,7 @@ static void __exit crypto_ofb_module_exit(void) crypto_unregister_template(&crypto_ofb_tmpl); } -module_init(crypto_ofb_module_init); +subsys_initcall(crypto_ofb_module_init); module_exit(crypto_ofb_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/pcbc.c b/crypto/pcbc.c index 2fa03fc576fe..31b3ce948474 100644 --- a/crypto/pcbc.c +++ b/crypto/pcbc.c @@ -191,7 +191,7 @@ static void __exit crypto_pcbc_module_exit(void) crypto_unregister_template(&crypto_pcbc_tmpl); } -module_init(crypto_pcbc_module_init); +subsys_initcall(crypto_pcbc_module_init); module_exit(crypto_pcbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index d47cfc47b1b1..0e9ce329fd47 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -512,7 +512,7 @@ static void __exit pcrypt_exit(void) crypto_unregister_template(&pcrypt_tmpl); } -module_init(pcrypt_init); +subsys_initcall(pcrypt_init); module_exit(pcrypt_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c index 2a06874204e8..adc40298c749 100644 --- a/crypto/poly1305_generic.c +++ b/crypto/poly1305_generic.c @@ -318,7 +318,7 @@ static void __exit poly1305_mod_exit(void) crypto_unregister_shash(&poly1305_alg); } -module_init(poly1305_mod_init); +subsys_initcall(poly1305_mod_init); module_exit(poly1305_mod_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/rmd128.c b/crypto/rmd128.c index 5f4472256e27..faf4252c4b85 100644 --- a/crypto/rmd128.c +++ b/crypto/rmd128.c @@ -318,7 +318,7 @@ static void __exit rmd128_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(rmd128_mod_init); +subsys_initcall(rmd128_mod_init); module_exit(rmd128_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/rmd160.c b/crypto/rmd160.c index 737645344d1c..b33309916d4f 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -362,7 +362,7 @@ static void __exit rmd160_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(rmd160_mod_init); +subsys_initcall(rmd160_mod_init); module_exit(rmd160_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/rmd256.c b/crypto/rmd256.c index 0e9d30676a01..2a643250c9a5 100644 --- a/crypto/rmd256.c +++ b/crypto/rmd256.c @@ -337,7 +337,7 @@ static void __exit rmd256_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(rmd256_mod_init); +subsys_initcall(rmd256_mod_init); module_exit(rmd256_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/rmd320.c b/crypto/rmd320.c index 3ae1df5bb48c..2f062574fc8c 100644 --- a/crypto/rmd320.c +++ b/crypto/rmd320.c @@ -386,7 +386,7 @@ static void __exit rmd320_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(rmd320_mod_init); +subsys_initcall(rmd320_mod_init); module_exit(rmd320_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c index 0a6680ca8cb6..29c336068dc0 100644 --- a/crypto/rsa-pkcs1pad.c +++ b/crypto/rsa-pkcs1pad.c @@ -429,7 +429,7 @@ static int pkcs1pad_sign(struct akcipher_request *req) akcipher_request_set_crypt(&req_ctx->child_req, req_ctx->in_sg, req->dst, ctx->key_size - 1, req->dst_len); - err = crypto_akcipher_sign(&req_ctx->child_req); + err = crypto_akcipher_decrypt(&req_ctx->child_req); if (err != -EINPROGRESS && err != -EBUSY) return pkcs1pad_encrypt_sign_complete(req, err); @@ -488,14 +488,21 @@ static int pkcs1pad_verify_complete(struct akcipher_request *req, int err) err = 0; - if (req->dst_len < dst_len - pos) - err = -EOVERFLOW; - req->dst_len = dst_len - pos; - - if (!err) - sg_copy_from_buffer(req->dst, - sg_nents_for_len(req->dst, req->dst_len), - out_buf + pos, req->dst_len); + if (req->dst_len != dst_len - pos) { + err = -EKEYREJECTED; + req->dst_len = dst_len - pos; + goto done; + } + /* Extract appended digest. */ + sg_pcopy_to_buffer(req->src, + sg_nents_for_len(req->src, + req->src_len + req->dst_len), + req_ctx->out_buf + ctx->key_size, + req->dst_len, ctx->key_size); + /* Do the actual verification step. */ + if (memcmp(req_ctx->out_buf + ctx->key_size, out_buf + pos, + req->dst_len) != 0) + err = -EKEYREJECTED; done: kzfree(req_ctx->out_buf); @@ -532,10 +539,12 @@ static int pkcs1pad_verify(struct akcipher_request *req) struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req); int err; - if (!ctx->key_size || req->src_len < ctx->key_size) + if (WARN_ON(req->dst) || + WARN_ON(!req->dst_len) || + !ctx->key_size || req->src_len < ctx->key_size) return -EINVAL; - req_ctx->out_buf = kmalloc(ctx->key_size, GFP_KERNEL); + req_ctx->out_buf = kmalloc(ctx->key_size + req->dst_len, GFP_KERNEL); if (!req_ctx->out_buf) return -ENOMEM; @@ -551,7 +560,7 @@ static int pkcs1pad_verify(struct akcipher_request *req) req_ctx->out_sg, req->src_len, ctx->key_size); - err = crypto_akcipher_verify(&req_ctx->child_req); + err = crypto_akcipher_encrypt(&req_ctx->child_req); if (err != -EINPROGRESS && err != -EBUSY) return pkcs1pad_verify_complete(req, err); diff --git a/crypto/rsa.c b/crypto/rsa.c index 4167980c243d..dcbb03431778 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -50,34 +50,6 @@ static int _rsa_dec(const struct rsa_mpi_key *key, MPI m, MPI c) return mpi_powm(m, c, key->d, key->n); } -/* - * RSASP1 function [RFC3447 sec 5.2.1] - * s = m^d mod n - */ -static int _rsa_sign(const struct rsa_mpi_key *key, MPI s, MPI m) -{ - /* (1) Validate 0 <= m < n */ - if (mpi_cmp_ui(m, 0) < 0 || mpi_cmp(m, key->n) >= 0) - return -EINVAL; - - /* (2) s = m^d mod n */ - return mpi_powm(s, m, key->d, key->n); -} - -/* - * RSAVP1 function [RFC3447 sec 5.2.2] - * m = s^e mod n; - */ -static int _rsa_verify(const struct rsa_mpi_key *key, MPI m, MPI s) -{ - /* (1) Validate 0 <= s < n */ - if (mpi_cmp_ui(s, 0) < 0 || mpi_cmp(s, key->n) >= 0) - return -EINVAL; - - /* (2) m = s^e mod n */ - return mpi_powm(m, s, key->e, key->n); -} - static inline struct rsa_mpi_key *rsa_get_key(struct crypto_akcipher *tfm) { return akcipher_tfm_ctx(tfm); @@ -160,85 +132,6 @@ err_free_m: return ret; } -static int rsa_sign(struct akcipher_request *req) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - const struct rsa_mpi_key *pkey = rsa_get_key(tfm); - MPI m, s = mpi_alloc(0); - int ret = 0; - int sign; - - if (!s) - return -ENOMEM; - - if (unlikely(!pkey->n || !pkey->d)) { - ret = -EINVAL; - goto err_free_s; - } - - ret = -ENOMEM; - m = mpi_read_raw_from_sgl(req->src, req->src_len); - if (!m) - goto err_free_s; - - ret = _rsa_sign(pkey, s, m); - if (ret) - goto err_free_m; - - ret = mpi_write_to_sgl(s, req->dst, req->dst_len, &sign); - if (ret) - goto err_free_m; - - if (sign < 0) - ret = -EBADMSG; - -err_free_m: - mpi_free(m); -err_free_s: - mpi_free(s); - return ret; -} - -static int rsa_verify(struct akcipher_request *req) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - const struct rsa_mpi_key *pkey = rsa_get_key(tfm); - MPI s, m = mpi_alloc(0); - int ret = 0; - int sign; - - if (!m) - return -ENOMEM; - - if (unlikely(!pkey->n || !pkey->e)) { - ret = -EINVAL; - goto err_free_m; - } - - s = mpi_read_raw_from_sgl(req->src, req->src_len); - if (!s) { - ret = -ENOMEM; - goto err_free_m; - } - - ret = _rsa_verify(pkey, m, s); - if (ret) - goto err_free_s; - - ret = mpi_write_to_sgl(m, req->dst, req->dst_len, &sign); - if (ret) - goto err_free_s; - - if (sign < 0) - ret = -EBADMSG; - -err_free_s: - mpi_free(s); -err_free_m: - mpi_free(m); - return ret; -} - static void rsa_free_mpi_key(struct rsa_mpi_key *key) { mpi_free(key->d); @@ -353,8 +246,6 @@ static void rsa_exit_tfm(struct crypto_akcipher *tfm) static struct akcipher_alg rsa = { .encrypt = rsa_enc, .decrypt = rsa_dec, - .sign = rsa_sign, - .verify = rsa_verify, .set_priv_key = rsa_set_priv_key, .set_pub_key = rsa_set_pub_key, .max_size = rsa_max_size, @@ -391,7 +282,7 @@ static void rsa_exit(void) crypto_unregister_akcipher(&rsa); } -module_init(rsa_init); +subsys_initcall(rsa_init); module_exit(rsa_exit); MODULE_ALIAS_CRYPTO("rsa"); MODULE_LICENSE("GPL"); diff --git a/crypto/salsa20_generic.c b/crypto/salsa20_generic.c index 00fce32ae17a..c81a44404086 100644 --- a/crypto/salsa20_generic.c +++ b/crypto/salsa20_generic.c @@ -86,18 +86,17 @@ static void salsa20_docrypt(u32 *state, u8 *dst, const u8 *src, { __le32 stream[SALSA20_BLOCK_SIZE / sizeof(__le32)]; - if (dst != src) - memcpy(dst, src, bytes); - while (bytes >= SALSA20_BLOCK_SIZE) { salsa20_block(state, stream); - crypto_xor(dst, (const u8 *)stream, SALSA20_BLOCK_SIZE); + crypto_xor_cpy(dst, src, (const u8 *)stream, + SALSA20_BLOCK_SIZE); bytes -= SALSA20_BLOCK_SIZE; dst += SALSA20_BLOCK_SIZE; + src += SALSA20_BLOCK_SIZE; } if (bytes) { salsa20_block(state, stream); - crypto_xor(dst, (const u8 *)stream, bytes); + crypto_xor_cpy(dst, src, (const u8 *)stream, bytes); } } @@ -161,7 +160,7 @@ static int salsa20_crypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, false); - salsa20_init(state, ctx, walk.iv); + salsa20_init(state, ctx, req->iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -204,7 +203,7 @@ static void __exit salsa20_generic_mod_fini(void) crypto_unregister_skcipher(&alg); } -module_init(salsa20_generic_mod_init); +subsys_initcall(salsa20_generic_mod_init); module_exit(salsa20_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/scompress.c b/crypto/scompress.c index 6f8305f8c300..712b4c2ea021 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -29,9 +29,17 @@ #include <crypto/internal/scompress.h> #include "internal.h" +struct scomp_scratch { + spinlock_t lock; + void *src; + void *dst; +}; + +static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = { + .lock = __SPIN_LOCK_UNLOCKED(scomp_scratch.lock), +}; + static const struct crypto_type crypto_scomp_type; -static void * __percpu *scomp_src_scratches; -static void * __percpu *scomp_dst_scratches; static int scomp_scratch_users; static DEFINE_MUTEX(scomp_lock); @@ -62,76 +70,53 @@ static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) seq_puts(m, "type : scomp\n"); } -static void crypto_scomp_free_scratches(void * __percpu *scratches) +static void crypto_scomp_free_scratches(void) { + struct scomp_scratch *scratch; int i; - if (!scratches) - return; - - for_each_possible_cpu(i) - vfree(*per_cpu_ptr(scratches, i)); + for_each_possible_cpu(i) { + scratch = per_cpu_ptr(&scomp_scratch, i); - free_percpu(scratches); + vfree(scratch->src); + vfree(scratch->dst); + scratch->src = NULL; + scratch->dst = NULL; + } } -static void * __percpu *crypto_scomp_alloc_scratches(void) +static int crypto_scomp_alloc_scratches(void) { - void * __percpu *scratches; + struct scomp_scratch *scratch; int i; - scratches = alloc_percpu(void *); - if (!scratches) - return NULL; - for_each_possible_cpu(i) { - void *scratch; - - scratch = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); - if (!scratch) - goto error; - *per_cpu_ptr(scratches, i) = scratch; - } - - return scratches; - -error: - crypto_scomp_free_scratches(scratches); - return NULL; -} + void *mem; -static void crypto_scomp_free_all_scratches(void) -{ - if (!--scomp_scratch_users) { - crypto_scomp_free_scratches(scomp_src_scratches); - crypto_scomp_free_scratches(scomp_dst_scratches); - scomp_src_scratches = NULL; - scomp_dst_scratches = NULL; - } -} + scratch = per_cpu_ptr(&scomp_scratch, i); -static int crypto_scomp_alloc_all_scratches(void) -{ - if (!scomp_scratch_users++) { - scomp_src_scratches = crypto_scomp_alloc_scratches(); - if (!scomp_src_scratches) - return -ENOMEM; - scomp_dst_scratches = crypto_scomp_alloc_scratches(); - if (!scomp_dst_scratches) { - crypto_scomp_free_scratches(scomp_src_scratches); - scomp_src_scratches = NULL; - return -ENOMEM; - } + mem = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); + if (!mem) + goto error; + scratch->src = mem; + mem = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); + if (!mem) + goto error; + scratch->dst = mem; } return 0; +error: + crypto_scomp_free_scratches(); + return -ENOMEM; } static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) { - int ret; + int ret = 0; mutex_lock(&scomp_lock); - ret = crypto_scomp_alloc_all_scratches(); + if (!scomp_scratch_users++) + ret = crypto_scomp_alloc_scratches(); mutex_unlock(&scomp_lock); return ret; @@ -143,42 +128,41 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) void **tfm_ctx = acomp_tfm_ctx(tfm); struct crypto_scomp *scomp = *tfm_ctx; void **ctx = acomp_request_ctx(req); - const int cpu = get_cpu(); - u8 *scratch_src = *per_cpu_ptr(scomp_src_scratches, cpu); - u8 *scratch_dst = *per_cpu_ptr(scomp_dst_scratches, cpu); + struct scomp_scratch *scratch; int ret; - if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE) { - ret = -EINVAL; - goto out; - } + if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE) + return -EINVAL; - if (req->dst && !req->dlen) { - ret = -EINVAL; - goto out; - } + if (req->dst && !req->dlen) + return -EINVAL; if (!req->dlen || req->dlen > SCOMP_SCRATCH_SIZE) req->dlen = SCOMP_SCRATCH_SIZE; - scatterwalk_map_and_copy(scratch_src, req->src, 0, req->slen, 0); + scratch = raw_cpu_ptr(&scomp_scratch); + spin_lock(&scratch->lock); + + scatterwalk_map_and_copy(scratch->src, req->src, 0, req->slen, 0); if (dir) - ret = crypto_scomp_compress(scomp, scratch_src, req->slen, - scratch_dst, &req->dlen, *ctx); + ret = crypto_scomp_compress(scomp, scratch->src, req->slen, + scratch->dst, &req->dlen, *ctx); else - ret = crypto_scomp_decompress(scomp, scratch_src, req->slen, - scratch_dst, &req->dlen, *ctx); + ret = crypto_scomp_decompress(scomp, scratch->src, req->slen, + scratch->dst, &req->dlen, *ctx); if (!ret) { if (!req->dst) { req->dst = sgl_alloc(req->dlen, GFP_ATOMIC, NULL); - if (!req->dst) + if (!req->dst) { + ret = -ENOMEM; goto out; + } } - scatterwalk_map_and_copy(scratch_dst, req->dst, 0, req->dlen, + scatterwalk_map_and_copy(scratch->dst, req->dst, 0, req->dlen, 1); } out: - put_cpu(); + spin_unlock(&scratch->lock); return ret; } @@ -199,7 +183,8 @@ static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) crypto_free_scomp(*ctx); mutex_lock(&scomp_lock); - crypto_scomp_free_all_scratches(); + if (!--scomp_scratch_users) + crypto_scomp_free_scratches(); mutex_unlock(&scomp_lock); } diff --git a/crypto/seed.c b/crypto/seed.c index c6ba8438be43..a75ac50fa4fd 100644 --- a/crypto/seed.c +++ b/crypto/seed.c @@ -470,7 +470,7 @@ static void __exit seed_fini(void) crypto_unregister_alg(&seed_alg); } -module_init(seed_init); +subsys_initcall(seed_init); module_exit(seed_fini); MODULE_DESCRIPTION("SEED Cipher Algorithm"); diff --git a/crypto/seqiv.c b/crypto/seqiv.c index ed1b0e9f2436..3f2fad615d26 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -211,7 +211,7 @@ static void __exit seqiv_module_exit(void) crypto_unregister_template(&seqiv_tmpl); } -module_init(seqiv_module_init); +subsys_initcall(seqiv_module_init); module_exit(seqiv_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index 7c3382facc82..ec4ec89ad108 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -664,7 +664,7 @@ static void __exit serpent_mod_fini(void) crypto_unregister_algs(srp_algs, ARRAY_SIZE(srp_algs)); } -module_init(serpent_mod_init); +subsys_initcall(serpent_mod_init); module_exit(serpent_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 2af64ef81f40..1b806d4584b2 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -92,7 +92,7 @@ static void __exit sha1_generic_mod_fini(void) crypto_unregister_shash(&alg); } -module_init(sha1_generic_mod_init); +subsys_initcall(sha1_generic_mod_init); module_exit(sha1_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index 1e5ba6649e8d..5844e9a469e8 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -301,7 +301,7 @@ static void __exit sha256_generic_mod_fini(void) crypto_unregister_shashes(sha256_algs, ARRAY_SIZE(sha256_algs)); } -module_init(sha256_generic_mod_init); +subsys_initcall(sha256_generic_mod_init); module_exit(sha256_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha3_generic.c b/crypto/sha3_generic.c index 7ed98367d4fb..60fd2be609d8 100644 --- a/crypto/sha3_generic.c +++ b/crypto/sha3_generic.c @@ -294,7 +294,7 @@ static void __exit sha3_generic_mod_fini(void) crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } -module_init(sha3_generic_mod_init); +subsys_initcall(sha3_generic_mod_init); module_exit(sha3_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index 4097cd555eb6..0193ecb8ae10 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -223,7 +223,7 @@ static void __exit sha512_generic_mod_fini(void) crypto_unregister_shashes(sha512_algs, ARRAY_SIZE(sha512_algs)); } -module_init(sha512_generic_mod_init); +subsys_initcall(sha512_generic_mod_init); module_exit(sha512_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/shash.c b/crypto/shash.c index 15b369c4745f..e55c1f558bc3 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -238,7 +238,6 @@ static int shash_async_init(struct ahash_request *req) struct shash_desc *desc = ahash_request_ctx(req); desc->tfm = *ctx; - desc->flags = req->base.flags; return crypto_shash_init(desc); } @@ -293,7 +292,6 @@ static int shash_async_finup(struct ahash_request *req) struct shash_desc *desc = ahash_request_ctx(req); desc->tfm = *ctx; - desc->flags = req->base.flags; return shash_ahash_finup(req, desc); } @@ -307,14 +305,13 @@ int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc) if (nbytes && (sg = req->src, offset = sg->offset, - nbytes < min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset))) { + nbytes <= min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset))) { void *data; data = kmap_atomic(sg_page(sg)); err = crypto_shash_digest(desc, data + offset, nbytes, req->result); kunmap_atomic(data); - crypto_yield(desc->flags); } else err = crypto_shash_init(desc) ?: shash_ahash_finup(req, desc); @@ -329,7 +326,6 @@ static int shash_async_digest(struct ahash_request *req) struct shash_desc *desc = ahash_request_ctx(req); desc->tfm = *ctx; - desc->flags = req->base.flags; return shash_ahash_digest(req, desc); } @@ -345,7 +341,6 @@ static int shash_async_import(struct ahash_request *req, const void *in) struct shash_desc *desc = ahash_request_ctx(req); desc->tfm = *ctx; - desc->flags = req->base.flags; return crypto_shash_import(desc, in); } diff --git a/crypto/simd.c b/crypto/simd.c index 78e8d037ae2b..3e3b1d1a6b1f 100644 --- a/crypto/simd.c +++ b/crypto/simd.c @@ -3,6 +3,7 @@ * * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> * Copyright (c) 2016 Herbert Xu <herbert@gondor.apana.org.au> + * Copyright (c) 2019 Google LLC * * Based on aesni-intel_glue.c by: * Copyright (C) 2008, Intel Corp. @@ -20,10 +21,26 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/* + * Shared crypto SIMD helpers. These functions dynamically create and register + * an skcipher or AEAD algorithm that wraps another, internal algorithm. The + * wrapper ensures that the internal algorithm is only executed in a context + * where SIMD instructions are usable, i.e. where may_use_simd() returns true. + * If SIMD is already usable, the wrapper directly calls the internal algorithm. + * Otherwise it defers execution to a workqueue via cryptd. * + * This is an alternative to the internal algorithm implementing a fallback for + * the !may_use_simd() case itself. + * + * Note that the wrapper algorithm is asynchronous, i.e. it has the + * CRYPTO_ALG_ASYNC flag set. Therefore it won't be found by users who + * explicitly allocate a synchronous algorithm. */ #include <crypto/cryptd.h> +#include <crypto/internal/aead.h> #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <linux/kernel.h> @@ -31,6 +48,8 @@ #include <linux/preempt.h> #include <asm/simd.h> +/* skcipher support */ + struct simd_skcipher_alg { const char *ialg_name; struct skcipher_alg alg; @@ -66,7 +85,7 @@ static int simd_skcipher_encrypt(struct skcipher_request *req) subreq = skcipher_request_ctx(req); *subreq = *req; - if (!may_use_simd() || + if (!crypto_simd_usable() || (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm))) child = &ctx->cryptd_tfm->base; else @@ -87,7 +106,7 @@ static int simd_skcipher_decrypt(struct skcipher_request *req) subreq = skcipher_request_ctx(req); *subreq = *req; - if (!may_use_simd() || + if (!crypto_simd_usable() || (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm))) child = &ctx->cryptd_tfm->base; else @@ -272,4 +291,254 @@ void simd_unregister_skciphers(struct skcipher_alg *algs, int count, } EXPORT_SYMBOL_GPL(simd_unregister_skciphers); +/* AEAD support */ + +struct simd_aead_alg { + const char *ialg_name; + struct aead_alg alg; +}; + +struct simd_aead_ctx { + struct cryptd_aead *cryptd_tfm; +}; + +static int simd_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int key_len) +{ + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_aead *child = &ctx->cryptd_tfm->base; + int err; + + crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(child, crypto_aead_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(child, key, key_len); + crypto_aead_set_flags(tfm, crypto_aead_get_flags(child) & + CRYPTO_TFM_RES_MASK); + return err; +} + +static int simd_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_aead *child = &ctx->cryptd_tfm->base; + + return crypto_aead_setauthsize(child, authsize); +} + +static int simd_aead_encrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_request *subreq; + struct crypto_aead *child; + + subreq = aead_request_ctx(req); + *subreq = *req; + + if (!crypto_simd_usable() || + (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm))) + child = &ctx->cryptd_tfm->base; + else + child = cryptd_aead_child(ctx->cryptd_tfm); + + aead_request_set_tfm(subreq, child); + + return crypto_aead_encrypt(subreq); +} + +static int simd_aead_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_request *subreq; + struct crypto_aead *child; + + subreq = aead_request_ctx(req); + *subreq = *req; + + if (!crypto_simd_usable() || + (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm))) + child = &ctx->cryptd_tfm->base; + else + child = cryptd_aead_child(ctx->cryptd_tfm); + + aead_request_set_tfm(subreq, child); + + return crypto_aead_decrypt(subreq); +} + +static void simd_aead_exit(struct crypto_aead *tfm) +{ + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + + cryptd_free_aead(ctx->cryptd_tfm); +} + +static int simd_aead_init(struct crypto_aead *tfm) +{ + struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct cryptd_aead *cryptd_tfm; + struct simd_aead_alg *salg; + struct aead_alg *alg; + unsigned reqsize; + + alg = crypto_aead_alg(tfm); + salg = container_of(alg, struct simd_aead_alg, alg); + + cryptd_tfm = cryptd_alloc_aead(salg->ialg_name, CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL); + if (IS_ERR(cryptd_tfm)) + return PTR_ERR(cryptd_tfm); + + ctx->cryptd_tfm = cryptd_tfm; + + reqsize = crypto_aead_reqsize(cryptd_aead_child(cryptd_tfm)); + reqsize = max(reqsize, crypto_aead_reqsize(&cryptd_tfm->base)); + reqsize += sizeof(struct aead_request); + + crypto_aead_set_reqsize(tfm, reqsize); + + return 0; +} + +struct simd_aead_alg *simd_aead_create_compat(const char *algname, + const char *drvname, + const char *basename) +{ + struct simd_aead_alg *salg; + struct crypto_aead *tfm; + struct aead_alg *ialg; + struct aead_alg *alg; + int err; + + tfm = crypto_alloc_aead(basename, CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC); + if (IS_ERR(tfm)) + return ERR_CAST(tfm); + + ialg = crypto_aead_alg(tfm); + + salg = kzalloc(sizeof(*salg), GFP_KERNEL); + if (!salg) { + salg = ERR_PTR(-ENOMEM); + goto out_put_tfm; + } + + salg->ialg_name = basename; + alg = &salg->alg; + + err = -ENAMETOOLONG; + if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >= + CRYPTO_MAX_ALG_NAME) + goto out_free_salg; + + if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + drvname) >= CRYPTO_MAX_ALG_NAME) + goto out_free_salg; + + alg->base.cra_flags = CRYPTO_ALG_ASYNC; + alg->base.cra_priority = ialg->base.cra_priority; + alg->base.cra_blocksize = ialg->base.cra_blocksize; + alg->base.cra_alignmask = ialg->base.cra_alignmask; + alg->base.cra_module = ialg->base.cra_module; + alg->base.cra_ctxsize = sizeof(struct simd_aead_ctx); + + alg->ivsize = ialg->ivsize; + alg->maxauthsize = ialg->maxauthsize; + alg->chunksize = ialg->chunksize; + + alg->init = simd_aead_init; + alg->exit = simd_aead_exit; + + alg->setkey = simd_aead_setkey; + alg->setauthsize = simd_aead_setauthsize; + alg->encrypt = simd_aead_encrypt; + alg->decrypt = simd_aead_decrypt; + + err = crypto_register_aead(alg); + if (err) + goto out_free_salg; + +out_put_tfm: + crypto_free_aead(tfm); + return salg; + +out_free_salg: + kfree(salg); + salg = ERR_PTR(err); + goto out_put_tfm; +} +EXPORT_SYMBOL_GPL(simd_aead_create_compat); + +struct simd_aead_alg *simd_aead_create(const char *algname, + const char *basename) +{ + char drvname[CRYPTO_MAX_ALG_NAME]; + + if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >= + CRYPTO_MAX_ALG_NAME) + return ERR_PTR(-ENAMETOOLONG); + + return simd_aead_create_compat(algname, drvname, basename); +} +EXPORT_SYMBOL_GPL(simd_aead_create); + +void simd_aead_free(struct simd_aead_alg *salg) +{ + crypto_unregister_aead(&salg->alg); + kfree(salg); +} +EXPORT_SYMBOL_GPL(simd_aead_free); + +int simd_register_aeads_compat(struct aead_alg *algs, int count, + struct simd_aead_alg **simd_algs) +{ + int err; + int i; + const char *algname; + const char *drvname; + const char *basename; + struct simd_aead_alg *simd; + + err = crypto_register_aeads(algs, count); + if (err) + return err; + + for (i = 0; i < count; i++) { + WARN_ON(strncmp(algs[i].base.cra_name, "__", 2)); + WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2)); + algname = algs[i].base.cra_name + 2; + drvname = algs[i].base.cra_driver_name + 2; + basename = algs[i].base.cra_driver_name; + simd = simd_aead_create_compat(algname, drvname, basename); + err = PTR_ERR(simd); + if (IS_ERR(simd)) + goto err_unregister; + simd_algs[i] = simd; + } + return 0; + +err_unregister: + simd_unregister_aeads(algs, count, simd_algs); + return err; +} +EXPORT_SYMBOL_GPL(simd_register_aeads_compat); + +void simd_unregister_aeads(struct aead_alg *algs, int count, + struct simd_aead_alg **simd_algs) +{ + int i; + + crypto_unregister_aeads(algs, count); + + for (i = 0; i < count; i++) { + if (simd_algs[i]) { + simd_aead_free(simd_algs[i]); + simd_algs[i] = NULL; + } + } +} +EXPORT_SYMBOL_GPL(simd_unregister_aeads); + MODULE_LICENSE("GPL"); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index bcf13d95f54a..2e66f312e2c4 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -131,8 +131,13 @@ unmap_src: memcpy(walk->dst.virt.addr, walk->page, n); skcipher_unmap_dst(walk); } else if (unlikely(walk->flags & SKCIPHER_WALK_SLOW)) { - if (WARN_ON(err)) { - /* unexpected case; didn't process all bytes */ + if (err) { + /* + * Didn't process all bytes. Either the algorithm is + * broken, or this was the last step and it turned out + * the message wasn't evenly divisible into blocks but + * the algorithm requires it. + */ err = -EINVAL; goto finish; } diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c index c0cf87ae7ef6..e227bcada2a2 100644 --- a/crypto/sm3_generic.c +++ b/crypto/sm3_generic.c @@ -199,7 +199,7 @@ static void __exit sm3_generic_mod_fini(void) crypto_unregister_shash(&sm3_alg); } -module_init(sm3_generic_mod_init); +subsys_initcall(sm3_generic_mod_init); module_exit(sm3_generic_mod_fini); MODULE_LICENSE("GPL v2"); diff --git a/crypto/sm4_generic.c b/crypto/sm4_generic.c index c18eebfd5edd..71ffb343709a 100644 --- a/crypto/sm4_generic.c +++ b/crypto/sm4_generic.c @@ -237,7 +237,7 @@ static void __exit sm4_fini(void) crypto_unregister_alg(&sm4_alg); } -module_init(sm4_init); +subsys_initcall(sm4_init); module_exit(sm4_fini); MODULE_DESCRIPTION("SM4 Cipher Algorithm"); diff --git a/crypto/streebog_generic.c b/crypto/streebog_generic.c index 5a2eafed9c29..63663c3bab7e 100644 --- a/crypto/streebog_generic.c +++ b/crypto/streebog_generic.c @@ -996,7 +996,7 @@ static void streebog_add512(const struct streebog_uint512 *x, static void streebog_g(struct streebog_uint512 *h, const struct streebog_uint512 *N, - const u8 *m) + const struct streebog_uint512 *m) { struct streebog_uint512 Ki, data; unsigned int i; @@ -1005,7 +1005,7 @@ static void streebog_g(struct streebog_uint512 *h, /* Starting E() */ Ki = data; - streebog_xlps(&Ki, (const struct streebog_uint512 *)&m[0], &data); + streebog_xlps(&Ki, m, &data); for (i = 0; i < 11; i++) streebog_round(i, &Ki, &data); @@ -1015,16 +1015,19 @@ static void streebog_g(struct streebog_uint512 *h, /* E() done */ streebog_xor(&data, h, &data); - streebog_xor(&data, (const struct streebog_uint512 *)&m[0], h); + streebog_xor(&data, m, h); } static void streebog_stage2(struct streebog_state *ctx, const u8 *data) { - streebog_g(&ctx->h, &ctx->N, data); + struct streebog_uint512 m; + + memcpy(&m, data, sizeof(m)); + + streebog_g(&ctx->h, &ctx->N, &m); streebog_add512(&ctx->N, &buffer512, &ctx->N); - streebog_add512(&ctx->Sigma, (const struct streebog_uint512 *)data, - &ctx->Sigma); + streebog_add512(&ctx->Sigma, &m, &ctx->Sigma); } static void streebog_stage3(struct streebog_state *ctx) @@ -1034,13 +1037,11 @@ static void streebog_stage3(struct streebog_state *ctx) buf.qword[0] = cpu_to_le64(ctx->fillsize << 3); streebog_pad(ctx); - streebog_g(&ctx->h, &ctx->N, (const u8 *)&ctx->buffer); + streebog_g(&ctx->h, &ctx->N, &ctx->m); streebog_add512(&ctx->N, &buf, &ctx->N); - streebog_add512(&ctx->Sigma, - (const struct streebog_uint512 *)&ctx->buffer[0], - &ctx->Sigma); - streebog_g(&ctx->h, &buffer0, (const u8 *)&ctx->N); - streebog_g(&ctx->h, &buffer0, (const u8 *)&ctx->Sigma); + streebog_add512(&ctx->Sigma, &ctx->m, &ctx->Sigma); + streebog_g(&ctx->h, &buffer0, &ctx->N); + streebog_g(&ctx->h, &buffer0, &ctx->Sigma); memcpy(&ctx->hash, &ctx->h, sizeof(struct streebog_uint512)); } @@ -1127,7 +1128,7 @@ static void __exit streebog_mod_fini(void) crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } -module_init(streebog_mod_init); +subsys_initcall(streebog_mod_init); module_exit(streebog_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 1ea2d5007ff5..798253f05203 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -3053,7 +3053,7 @@ err_free_tv: */ static void __exit tcrypt_mod_fini(void) { } -module_init(tcrypt_mod_init); +subsys_initcall(tcrypt_mod_init); module_exit(tcrypt_mod_fini); module_param(alg, charp, 0); diff --git a/crypto/tea.c b/crypto/tea.c index b70b441c7d1e..786b589e1399 100644 --- a/crypto/tea.c +++ b/crypto/tea.c @@ -274,7 +274,7 @@ MODULE_ALIAS_CRYPTO("tea"); MODULE_ALIAS_CRYPTO("xtea"); MODULE_ALIAS_CRYPTO("xeta"); -module_init(tea_mod_init); +subsys_initcall(tea_mod_init); module_exit(tea_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 8386038d67c7..c9e67c2bd725 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -37,6 +37,7 @@ #include <crypto/akcipher.h> #include <crypto/kpp.h> #include <crypto/acompress.h> +#include <crypto/internal/simd.h> #include "internal.h" @@ -44,6 +45,9 @@ static bool notests; module_param(notests, bool, 0644); MODULE_PARM_DESC(notests, "disable crypto self-tests"); +static bool panic_on_fail; +module_param(panic_on_fail, bool, 0444); + #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS static bool noextratests; module_param(noextratests, bool, 0644); @@ -52,6 +56,9 @@ MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests"); static unsigned int fuzz_iterations = 100; module_param(fuzz_iterations, uint, 0644); MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations"); + +DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test); +EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test); #endif #ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS @@ -121,6 +128,7 @@ struct kpp_test_suite { struct alg_test_desc { const char *alg; + const char *generic_driver; int (*test)(const struct alg_test_desc *desc, const char *driver, u32 type, u32 mask); int fips_allowed; /* set if alg is allowed in fips mode */ @@ -230,12 +238,14 @@ enum finalization_type { * @offset * @flush_type: for hashes, whether an update() should be done now vs. * continuing to accumulate data + * @nosimd: if doing the pending update(), do it with SIMD disabled? */ struct test_sg_division { unsigned int proportion_of_total; unsigned int offset; bool offset_relative_to_alignmask; enum flush_type flush_type; + bool nosimd; }; /** @@ -255,6 +265,7 @@ struct test_sg_division { * @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to * the @iv_offset * @finalization_type: what finalization function to use for hashes + * @nosimd: execute with SIMD disabled? Requires !CRYPTO_TFM_REQ_MAY_SLEEP. */ struct testvec_config { const char *name; @@ -265,6 +276,7 @@ struct testvec_config { unsigned int iv_offset; bool iv_offset_relative_to_alignmask; enum finalization_type finalization_type; + bool nosimd; }; #define TESTVEC_CONFIG_NAMELEN 192 @@ -416,8 +428,11 @@ static unsigned int count_test_sg_divisions(const struct test_sg_division *divs) return ndivs; } +#define SGDIVS_HAVE_FLUSHES BIT(0) +#define SGDIVS_HAVE_NOSIMD BIT(1) + static bool valid_sg_divisions(const struct test_sg_division *divs, - unsigned int count, bool *any_flushes_ret) + unsigned int count, int *flags_ret) { unsigned int total = 0; unsigned int i; @@ -428,7 +443,9 @@ static bool valid_sg_divisions(const struct test_sg_division *divs, return false; total += divs[i].proportion_of_total; if (divs[i].flush_type != FLUSH_TYPE_NONE) - *any_flushes_ret = true; + *flags_ret |= SGDIVS_HAVE_FLUSHES; + if (divs[i].nosimd) + *flags_ret |= SGDIVS_HAVE_NOSIMD; } return total == TEST_SG_TOTAL && memchr_inv(&divs[i], 0, (count - i) * sizeof(divs[0])) == NULL; @@ -441,19 +458,18 @@ static bool valid_sg_divisions(const struct test_sg_division *divs, */ static bool valid_testvec_config(const struct testvec_config *cfg) { - bool any_flushes = false; + int flags = 0; if (cfg->name == NULL) return false; if (!valid_sg_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs), - &any_flushes)) + &flags)) return false; if (cfg->dst_divs[0].proportion_of_total) { if (!valid_sg_divisions(cfg->dst_divs, - ARRAY_SIZE(cfg->dst_divs), - &any_flushes)) + ARRAY_SIZE(cfg->dst_divs), &flags)) return false; } else { if (memchr_inv(cfg->dst_divs, 0, sizeof(cfg->dst_divs))) @@ -466,7 +482,12 @@ static bool valid_testvec_config(const struct testvec_config *cfg) MAX_ALGAPI_ALIGNMASK + 1) return false; - if (any_flushes && cfg->finalization_type == FINALIZATION_TYPE_DIGEST) + if ((flags & (SGDIVS_HAVE_FLUSHES | SGDIVS_HAVE_NOSIMD)) && + cfg->finalization_type == FINALIZATION_TYPE_DIGEST) + return false; + + if ((cfg->nosimd || (flags & SGDIVS_HAVE_NOSIMD)) && + (cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP)) return false; return true; @@ -725,15 +746,101 @@ static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls, } #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS + +/* Generate a random length in range [0, max_len], but prefer smaller values */ +static unsigned int generate_random_length(unsigned int max_len) +{ + unsigned int len = prandom_u32() % (max_len + 1); + + switch (prandom_u32() % 4) { + case 0: + return len % 64; + case 1: + return len % 256; + case 2: + return len % 1024; + default: + return len; + } +} + +/* Sometimes make some random changes to the given data buffer */ +static void mutate_buffer(u8 *buf, size_t count) +{ + size_t num_flips; + size_t i; + size_t pos; + + /* Sometimes flip some bits */ + if (prandom_u32() % 4 == 0) { + num_flips = min_t(size_t, 1 << (prandom_u32() % 8), count * 8); + for (i = 0; i < num_flips; i++) { + pos = prandom_u32() % (count * 8); + buf[pos / 8] ^= 1 << (pos % 8); + } + } + + /* Sometimes flip some bytes */ + if (prandom_u32() % 4 == 0) { + num_flips = min_t(size_t, 1 << (prandom_u32() % 8), count); + for (i = 0; i < num_flips; i++) + buf[prandom_u32() % count] ^= 0xff; + } +} + +/* Randomly generate 'count' bytes, but sometimes make them "interesting" */ +static void generate_random_bytes(u8 *buf, size_t count) +{ + u8 b; + u8 increment; + size_t i; + + if (count == 0) + return; + + switch (prandom_u32() % 8) { /* Choose a generation strategy */ + case 0: + case 1: + /* All the same byte, plus optional mutations */ + switch (prandom_u32() % 4) { + case 0: + b = 0x00; + break; + case 1: + b = 0xff; + break; + default: + b = (u8)prandom_u32(); + break; + } + memset(buf, b, count); + mutate_buffer(buf, count); + break; + case 2: + /* Ascending or descending bytes, plus optional mutations */ + increment = (u8)prandom_u32(); + b = (u8)prandom_u32(); + for (i = 0; i < count; i++, b += increment) + buf[i] = b; + mutate_buffer(buf, count); + break; + default: + /* Fully random bytes */ + for (i = 0; i < count; i++) + buf[i] = (u8)prandom_u32(); + } +} + static char *generate_random_sgl_divisions(struct test_sg_division *divs, size_t max_divs, char *p, char *end, - bool gen_flushes) + bool gen_flushes, u32 req_flags) { struct test_sg_division *div = divs; unsigned int remaining = TEST_SG_TOTAL; do { unsigned int this_len; + const char *flushtype_str; if (div == &divs[max_divs - 1] || prandom_u32() % 2 == 0) this_len = remaining; @@ -762,11 +869,31 @@ static char *generate_random_sgl_divisions(struct test_sg_division *divs, } } + if (div->flush_type != FLUSH_TYPE_NONE && + !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) && + prandom_u32() % 2 == 0) + div->nosimd = true; + + switch (div->flush_type) { + case FLUSH_TYPE_FLUSH: + if (div->nosimd) + flushtype_str = "<flush,nosimd>"; + else + flushtype_str = "<flush>"; + break; + case FLUSH_TYPE_REIMPORT: + if (div->nosimd) + flushtype_str = "<reimport,nosimd>"; + else + flushtype_str = "<reimport>"; + break; + default: + flushtype_str = ""; + break; + } + BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */ - p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", - div->flush_type == FLUSH_TYPE_NONE ? "" : - div->flush_type == FLUSH_TYPE_FLUSH ? - "<flush> " : "<reimport> ", + p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", flushtype_str, this_len / 100, this_len % 100, div->offset_relative_to_alignmask ? "alignmask" : "", @@ -816,18 +943,26 @@ static void generate_random_testvec_config(struct testvec_config *cfg, break; } + if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) && + prandom_u32() % 2 == 0) { + cfg->nosimd = true; + p += scnprintf(p, end - p, " nosimd"); + } + p += scnprintf(p, end - p, " src_divs=["); p = generate_random_sgl_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs), p, end, (cfg->finalization_type != - FINALIZATION_TYPE_DIGEST)); + FINALIZATION_TYPE_DIGEST), + cfg->req_flags); p += scnprintf(p, end - p, "]"); if (!cfg->inplace && prandom_u32() % 2 == 0) { p += scnprintf(p, end - p, " dst_divs=["); p = generate_random_sgl_divisions(cfg->dst_divs, ARRAY_SIZE(cfg->dst_divs), - p, end, false); + p, end, false, + cfg->req_flags); p += scnprintf(p, end - p, "]"); } @@ -838,21 +973,100 @@ static void generate_random_testvec_config(struct testvec_config *cfg, WARN_ON_ONCE(!valid_testvec_config(cfg)); } -#endif /* CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + +static void crypto_disable_simd_for_test(void) +{ + preempt_disable(); + __this_cpu_write(crypto_simd_disabled_for_test, true); +} + +static void crypto_reenable_simd_for_test(void) +{ + __this_cpu_write(crypto_simd_disabled_for_test, false); + preempt_enable(); +} + +/* + * Given an algorithm name, build the name of the generic implementation of that + * algorithm, assuming the usual naming convention. Specifically, this appends + * "-generic" to every part of the name that is not a template name. Examples: + * + * aes => aes-generic + * cbc(aes) => cbc(aes-generic) + * cts(cbc(aes)) => cts(cbc(aes-generic)) + * rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic) + * + * Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long + */ +static int build_generic_driver_name(const char *algname, + char driver_name[CRYPTO_MAX_ALG_NAME]) +{ + const char *in = algname; + char *out = driver_name; + size_t len = strlen(algname); + + if (len >= CRYPTO_MAX_ALG_NAME) + goto too_long; + do { + const char *in_saved = in; + + while (*in && *in != '(' && *in != ')' && *in != ',') + *out++ = *in++; + if (*in != '(' && in > in_saved) { + len += 8; + if (len >= CRYPTO_MAX_ALG_NAME) + goto too_long; + memcpy(out, "-generic", 8); + out += 8; + } + } while ((*out++ = *in++) != '\0'); + return 0; + +too_long: + pr_err("alg: generic driver name for \"%s\" would be too long\n", + algname); + return -ENAMETOOLONG; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static void crypto_disable_simd_for_test(void) +{ +} + +static void crypto_reenable_simd_for_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + +static int do_ahash_op(int (*op)(struct ahash_request *req), + struct ahash_request *req, + struct crypto_wait *wait, bool nosimd) +{ + int err; + + if (nosimd) + crypto_disable_simd_for_test(); + + err = op(req); + + if (nosimd) + crypto_reenable_simd_for_test(); + + return crypto_wait_req(err, wait); +} static int check_nonfinal_hash_op(const char *op, int err, u8 *result, unsigned int digestsize, - const char *driver, unsigned int vec_num, + const char *driver, const char *vec_name, const struct testvec_config *cfg) { if (err) { - pr_err("alg: hash: %s %s() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, op, err, vec_num, cfg->name); + pr_err("alg: hash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, op, err, vec_name, cfg->name); return err; } if (!testmgr_is_poison(result, digestsize)) { - pr_err("alg: hash: %s %s() used result buffer on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: hash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return -EINVAL; } return 0; @@ -860,7 +1074,7 @@ static int check_nonfinal_hash_op(const char *op, int err, static int test_hash_vec_cfg(const char *driver, const struct hash_testvec *vec, - unsigned int vec_num, + const char *vec_name, const struct testvec_config *cfg, struct ahash_request *req, struct test_sglist *tsgl, @@ -885,11 +1099,18 @@ static int test_hash_vec_cfg(const char *driver, if (vec->ksize) { err = crypto_ahash_setkey(tfm, vec->key, vec->ksize); if (err) { - pr_err("alg: hash: %s setkey failed with err %d on test vector %u; flags=%#x\n", - driver, err, vec_num, + if (err == vec->setkey_error) + return 0; + pr_err("alg: hash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, crypto_ahash_get_flags(tfm)); return err; } + if (vec->setkey_error) { + pr_err("alg: hash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); + return -EINVAL; + } } /* Build the scatterlist for the source data */ @@ -899,8 +1120,8 @@ static int test_hash_vec_cfg(const char *driver, err = build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize, &input, divs); if (err) { - pr_err("alg: hash: %s: error preparing scatterlist for test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return err; } @@ -909,17 +1130,26 @@ static int test_hash_vec_cfg(const char *driver, testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm)); testmgr_poison(result, digestsize + TESTMGR_POISON_LEN); - if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST) { + if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST || + vec->digest_error) { /* Just using digest() */ ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize); - err = crypto_wait_req(crypto_ahash_digest(req), &wait); + err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s digest() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, err, vec_num, cfg->name); + if (err == vec->digest_error) + return 0; + pr_err("alg: hash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, err, + cfg->name); return err; } + if (vec->digest_error) { + pr_err("alg: hash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, cfg->name); + return -EINVAL; + } goto result_ready; } @@ -927,9 +1157,9 @@ static int test_hash_vec_cfg(const char *driver, ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, NULL, result, 0); - err = crypto_wait_req(crypto_ahash_init(req), &wait); + err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd); err = check_nonfinal_hash_op("init", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; @@ -943,10 +1173,11 @@ static int test_hash_vec_cfg(const char *driver, crypto_req_done, &wait); ahash_request_set_crypt(req, pending_sgl, result, pending_len); - err = crypto_wait_req(crypto_ahash_update(req), &wait); + err = do_ahash_op(crypto_ahash_update, req, &wait, + divs[i]->nosimd); err = check_nonfinal_hash_op("update", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; pending_sgl = NULL; @@ -959,13 +1190,13 @@ static int test_hash_vec_cfg(const char *driver, err = crypto_ahash_export(req, hashstate); err = check_nonfinal_hash_op("export", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; if (!testmgr_is_poison(hashstate + statesize, TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s export() overran state buffer on test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return -EOVERFLOW; } @@ -973,7 +1204,7 @@ static int test_hash_vec_cfg(const char *driver, err = crypto_ahash_import(req, hashstate); err = check_nonfinal_hash_op("import", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; } @@ -986,23 +1217,23 @@ static int test_hash_vec_cfg(const char *driver, ahash_request_set_crypt(req, pending_sgl, result, pending_len); if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) { /* finish with update() and final() */ - err = crypto_wait_req(crypto_ahash_update(req), &wait); + err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd); err = check_nonfinal_hash_op("update", err, result, digestsize, - driver, vec_num, cfg); + driver, vec_name, cfg); if (err) return err; - err = crypto_wait_req(crypto_ahash_final(req), &wait); + err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s final() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, err, vec_num, cfg->name); + pr_err("alg: hash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, err, vec_name, cfg->name); return err; } } else { /* finish with finup() */ - err = crypto_wait_req(crypto_ahash_finup(req), &wait); + err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s finup() failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, err, vec_num, cfg->name); + pr_err("alg: hash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, err, vec_name, cfg->name); return err; } } @@ -1010,13 +1241,13 @@ static int test_hash_vec_cfg(const char *driver, result_ready: /* Check that the algorithm produced the correct digest */ if (memcmp(result, vec->digest, digestsize) != 0) { - pr_err("alg: hash: %s test failed (wrong result) on test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return -EINVAL; } if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s overran result buffer on test vector %u, cfg=\"%s\"\n", - driver, vec_num, cfg->name); + pr_err("alg: hash: %s overran result buffer on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); return -EOVERFLOW; } @@ -1027,11 +1258,14 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, unsigned int vec_num, struct ahash_request *req, struct test_sglist *tsgl, u8 *hashstate) { + char vec_name[16]; unsigned int i; int err; + sprintf(vec_name, "%u", vec_num); + for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) { - err = test_hash_vec_cfg(driver, vec, vec_num, + err = test_hash_vec_cfg(driver, vec, vec_name, &default_hash_testvec_configs[i], req, tsgl, hashstate); if (err) @@ -1046,7 +1280,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_hash_vec_cfg(driver, vec, vec_num, &cfg, + err = test_hash_vec_cfg(driver, vec, vec_name, &cfg, req, tsgl, hashstate); if (err) return err; @@ -1056,9 +1290,168 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, return 0; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +/* + * Generate a hash test vector from the given implementation. + * Assumes the buffers in 'vec' were already allocated. + */ +static void generate_random_hash_testvec(struct crypto_shash *tfm, + struct hash_testvec *vec, + unsigned int maxkeysize, + unsigned int maxdatasize, + char *name, size_t max_namelen) +{ + SHASH_DESC_ON_STACK(desc, tfm); + + /* Data */ + vec->psize = generate_random_length(maxdatasize); + generate_random_bytes((u8 *)vec->plaintext, vec->psize); + + /* + * Key: length in range [1, maxkeysize], but usually choose maxkeysize. + * If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0. + */ + vec->setkey_error = 0; + vec->ksize = 0; + if (maxkeysize) { + vec->ksize = maxkeysize; + if (prandom_u32() % 4 == 0) + vec->ksize = 1 + (prandom_u32() % maxkeysize); + generate_random_bytes((u8 *)vec->key, vec->ksize); + + vec->setkey_error = crypto_shash_setkey(tfm, vec->key, + vec->ksize); + /* If the key couldn't be set, no need to continue to digest. */ + if (vec->setkey_error) + goto done; + } + + /* Digest */ + desc->tfm = tfm; + vec->digest_error = crypto_shash_digest(desc, vec->plaintext, + vec->psize, (u8 *)vec->digest); +done: + snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"", + vec->psize, vec->ksize); +} + +/* + * Test the hash algorithm represented by @req against the corresponding generic + * implementation, if one is available. + */ +static int test_hash_vs_generic_impl(const char *driver, + const char *generic_driver, + unsigned int maxkeysize, + struct ahash_request *req, + struct test_sglist *tsgl, + u8 *hashstate) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + const unsigned int digestsize = crypto_ahash_digestsize(tfm); + const unsigned int blocksize = crypto_ahash_blocksize(tfm); + const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; + const char *algname = crypto_hash_alg_common(tfm)->base.cra_name; + char _generic_driver[CRYPTO_MAX_ALG_NAME]; + struct crypto_shash *generic_tfm = NULL; + unsigned int i; + struct hash_testvec vec = { 0 }; + char vec_name[64]; + struct testvec_config cfg; + char cfgname[TESTVEC_CONFIG_NAMELEN]; + int err; + + if (noextratests) + return 0; + + if (!generic_driver) { /* Use default naming convention? */ + err = build_generic_driver_name(algname, _generic_driver); + if (err) + return err; + generic_driver = _generic_driver; + } + + if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */ + return 0; + + generic_tfm = crypto_alloc_shash(generic_driver, 0, 0); + if (IS_ERR(generic_tfm)) { + err = PTR_ERR(generic_tfm); + if (err == -ENOENT) { + pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n", + driver, generic_driver); + return 0; + } + pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n", + generic_driver, algname, err); + return err; + } + + /* Check the algorithm properties for consistency. */ + + if (digestsize != crypto_shash_digestsize(generic_tfm)) { + pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n", + driver, digestsize, + crypto_shash_digestsize(generic_tfm)); + err = -EINVAL; + goto out; + } + + if (blocksize != crypto_shash_blocksize(generic_tfm)) { + pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n", + driver, blocksize, crypto_shash_blocksize(generic_tfm)); + err = -EINVAL; + goto out; + } + + /* + * Now generate test vectors using the generic implementation, and test + * the other implementation against them. + */ + + vec.key = kmalloc(maxkeysize, GFP_KERNEL); + vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL); + vec.digest = kmalloc(digestsize, GFP_KERNEL); + if (!vec.key || !vec.plaintext || !vec.digest) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < fuzz_iterations * 8; i++) { + generate_random_hash_testvec(generic_tfm, &vec, + maxkeysize, maxdatasize, + vec_name, sizeof(vec_name)); + generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + + err = test_hash_vec_cfg(driver, &vec, vec_name, &cfg, + req, tsgl, hashstate); + if (err) + goto out; + cond_resched(); + } + err = 0; +out: + kfree(vec.key); + kfree(vec.plaintext); + kfree(vec.digest); + crypto_free_shash(generic_tfm); + return err; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int test_hash_vs_generic_impl(const char *driver, + const char *generic_driver, + unsigned int maxkeysize, + struct ahash_request *req, + struct test_sglist *tsgl, + u8 *hashstate) +{ + return 0; +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + static int __alg_test_hash(const struct hash_testvec *vecs, unsigned int num_vecs, const char *driver, - u32 type, u32 mask) + u32 type, u32 mask, + const char *generic_driver, unsigned int maxkeysize) { struct crypto_ahash *tfm; struct ahash_request *req = NULL; @@ -1106,7 +1499,8 @@ static int __alg_test_hash(const struct hash_testvec *vecs, if (err) goto out; } - err = 0; + err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req, + tsgl, hashstate); out: kfree(hashstate); if (tsgl) { @@ -1124,6 +1518,7 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver, const struct hash_testvec *template = desc->suite.hash.vecs; unsigned int tcount = desc->suite.hash.count; unsigned int nr_unkeyed, nr_keyed; + unsigned int maxkeysize = 0; int err; /* @@ -1142,23 +1537,27 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver, "unkeyed ones must come first\n", desc->alg); return -EINVAL; } + maxkeysize = max_t(unsigned int, maxkeysize, + template[nr_unkeyed + nr_keyed].ksize); } err = 0; if (nr_unkeyed) { - err = __alg_test_hash(template, nr_unkeyed, driver, type, mask); + err = __alg_test_hash(template, nr_unkeyed, driver, type, mask, + desc->generic_driver, maxkeysize); template += nr_unkeyed; } if (!err && nr_keyed) - err = __alg_test_hash(template, nr_keyed, driver, type, mask); + err = __alg_test_hash(template, nr_keyed, driver, type, mask, + desc->generic_driver, maxkeysize); return err; } static int test_aead_vec_cfg(const char *driver, int enc, const struct aead_testvec *vec, - unsigned int vec_num, + const char *vec_name, const struct testvec_config *cfg, struct aead_request *req, struct cipher_test_sglists *tsgls) @@ -1175,6 +1574,7 @@ static int test_aead_vec_cfg(const char *driver, int enc, cfg->iv_offset + (cfg->iv_offset_relative_to_alignmask ? alignmask : 0); struct kvec input[2]; + int expected_error; int err; /* Set the key */ @@ -1183,26 +1583,33 @@ static int test_aead_vec_cfg(const char *driver, int enc, else crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); err = crypto_aead_setkey(tfm, vec->key, vec->klen); - if (err) { - if (vec->fail) /* expectedly failed to set key? */ - return 0; - pr_err("alg: aead: %s setkey failed with err %d on test vector %u; flags=%#x\n", - driver, err, vec_num, crypto_aead_get_flags(tfm)); + if (err && err != vec->setkey_error) { + pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, + crypto_aead_get_flags(tfm)); return err; } - if (vec->fail) { - pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %u\n", - driver, vec_num); + if (!err && vec->setkey_error) { + pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); return -EINVAL; } /* Set the authentication tag size */ err = crypto_aead_setauthsize(tfm, authsize); - if (err) { - pr_err("alg: aead: %s setauthsize failed with err %d on test vector %u\n", - driver, err, vec_num); + if (err && err != vec->setauthsize_error) { + pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n", + driver, vec_name, vec->setauthsize_error, err); return err; } + if (!err && vec->setauthsize_error) { + pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setauthsize_error); + return -EINVAL; + } + + if (vec->setkey_error || vec->setauthsize_error) + return 0; /* The IV must be copied to a buffer, as the algorithm may modify it */ if (WARN_ON(ivsize > MAX_IVLEN)) @@ -1224,8 +1631,8 @@ static int test_aead_vec_cfg(const char *driver, int enc, vec->plen), input, 2); if (err) { - pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } @@ -1235,23 +1642,12 @@ static int test_aead_vec_cfg(const char *driver, int enc, aead_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr, enc ? vec->plen : vec->clen, iv); aead_request_set_ad(req, vec->alen); - err = crypto_wait_req(enc ? crypto_aead_encrypt(req) : - crypto_aead_decrypt(req), &wait); - - aead_request_set_tfm(req, tfm); /* TODO: get rid of this */ - - if (err) { - if (err == -EBADMSG && vec->novrfy) - return 0; - pr_err("alg: aead: %s %s failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, op, err, vec_num, cfg->name); - return err; - } - if (vec->novrfy) { - pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); - return -EINVAL; - } + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = crypto_wait_req(err, &wait); /* Check that the algorithm didn't overwrite things it shouldn't have */ if (req->cryptlen != (enc ? vec->plen : vec->clen) || @@ -1263,8 +1659,8 @@ static int test_aead_vec_cfg(const char *driver, int enc, req->base.complete != crypto_req_done || req->base.flags != req_flags || req->base.data != &wait) { - pr_err("alg: aead: %s %s corrupted request struct on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); if (req->cryptlen != (enc ? vec->plen : vec->clen)) pr_err("alg: aead: changed 'req->cryptlen'\n"); if (req->assoclen != vec->alen) @@ -1286,14 +1682,29 @@ static int test_aead_vec_cfg(const char *driver, int enc, return -EINVAL; } if (is_test_sglist_corrupted(&tsgls->src)) { - pr_err("alg: aead: %s %s corrupted src sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return -EINVAL; } if (tsgls->dst.sgl_ptr != tsgls->src.sgl && is_test_sglist_corrupted(&tsgls->dst)) { - pr_err("alg: aead: %s %s corrupted dst sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); + return -EINVAL; + } + + /* Check for success or failure */ + expected_error = vec->novrfy ? -EBADMSG : vec->crypt_error; + if (err) { + if (err == expected_error) + return 0; + pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, expected_error, err, cfg->name); + return err; + } + if (expected_error) { + pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, expected_error, cfg->name); return -EINVAL; } @@ -1302,13 +1713,13 @@ static int test_aead_vec_cfg(const char *driver, int enc, enc ? vec->clen : vec->plen, vec->alen, enc || !cfg->inplace); if (err == -EOVERFLOW) { - pr_err("alg: aead: %s %s overran dst buffer on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } if (err) { - pr_err("alg: aead: %s %s test failed (wrong result) on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } @@ -1320,14 +1731,17 @@ static int test_aead_vec(const char *driver, int enc, struct aead_request *req, struct cipher_test_sglists *tsgls) { + char vec_name[16]; unsigned int i; int err; if (enc && vec->novrfy) return 0; + sprintf(vec_name, "%u", vec_num); + for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) { - err = test_aead_vec_cfg(driver, enc, vec, vec_num, + err = test_aead_vec_cfg(driver, enc, vec, vec_name, &default_cipher_testvec_configs[i], req, tsgls); if (err) @@ -1342,7 +1756,7 @@ static int test_aead_vec(const char *driver, int enc, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_aead_vec_cfg(driver, enc, vec, vec_num, + err = test_aead_vec_cfg(driver, enc, vec, vec_name, &cfg, req, tsgls); if (err) return err; @@ -1352,6 +1766,226 @@ static int test_aead_vec(const char *driver, int enc, return 0; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +/* + * Generate an AEAD test vector from the given implementation. + * Assumes the buffers in 'vec' were already allocated. + */ +static void generate_random_aead_testvec(struct aead_request *req, + struct aead_testvec *vec, + unsigned int maxkeysize, + unsigned int maxdatasize, + char *name, size_t max_namelen) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const unsigned int ivsize = crypto_aead_ivsize(tfm); + unsigned int maxauthsize = crypto_aead_alg(tfm)->maxauthsize; + unsigned int authsize; + unsigned int total_len; + int i; + struct scatterlist src[2], dst; + u8 iv[MAX_IVLEN]; + DECLARE_CRYPTO_WAIT(wait); + + /* Key: length in [0, maxkeysize], but usually choose maxkeysize */ + vec->klen = maxkeysize; + if (prandom_u32() % 4 == 0) + vec->klen = prandom_u32() % (maxkeysize + 1); + generate_random_bytes((u8 *)vec->key, vec->klen); + vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen); + + /* IV */ + generate_random_bytes((u8 *)vec->iv, ivsize); + + /* Tag length: in [0, maxauthsize], but usually choose maxauthsize */ + authsize = maxauthsize; + if (prandom_u32() % 4 == 0) + authsize = prandom_u32() % (maxauthsize + 1); + if (WARN_ON(authsize > maxdatasize)) + authsize = maxdatasize; + maxdatasize -= authsize; + vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize); + + /* Plaintext and associated data */ + total_len = generate_random_length(maxdatasize); + if (prandom_u32() % 4 == 0) + vec->alen = 0; + else + vec->alen = generate_random_length(total_len); + vec->plen = total_len - vec->alen; + generate_random_bytes((u8 *)vec->assoc, vec->alen); + generate_random_bytes((u8 *)vec->ptext, vec->plen); + + vec->clen = vec->plen + authsize; + + /* + * If the key or authentication tag size couldn't be set, no need to + * continue to encrypt. + */ + if (vec->setkey_error || vec->setauthsize_error) + goto done; + + /* Ciphertext */ + sg_init_table(src, 2); + i = 0; + if (vec->alen) + sg_set_buf(&src[i++], vec->assoc, vec->alen); + if (vec->plen) + sg_set_buf(&src[i++], vec->ptext, vec->plen); + sg_init_one(&dst, vec->ctext, vec->alen + vec->clen); + memcpy(iv, vec->iv, ivsize); + aead_request_set_callback(req, 0, crypto_req_done, &wait); + aead_request_set_crypt(req, src, &dst, vec->plen, iv); + aead_request_set_ad(req, vec->alen); + vec->crypt_error = crypto_wait_req(crypto_aead_encrypt(req), &wait); + if (vec->crypt_error == 0) + memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen); +done: + snprintf(name, max_namelen, + "\"random: alen=%u plen=%u authsize=%u klen=%u\"", + vec->alen, vec->plen, authsize, vec->klen); +} + +/* + * Test the AEAD algorithm represented by @req against the corresponding generic + * implementation, if one is available. + */ +static int test_aead_vs_generic_impl(const char *driver, + const struct alg_test_desc *test_desc, + struct aead_request *req, + struct cipher_test_sglists *tsgls) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + const unsigned int ivsize = crypto_aead_ivsize(tfm); + const unsigned int maxauthsize = crypto_aead_alg(tfm)->maxauthsize; + const unsigned int blocksize = crypto_aead_blocksize(tfm); + const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; + const char *algname = crypto_aead_alg(tfm)->base.cra_name; + const char *generic_driver = test_desc->generic_driver; + char _generic_driver[CRYPTO_MAX_ALG_NAME]; + struct crypto_aead *generic_tfm = NULL; + struct aead_request *generic_req = NULL; + unsigned int maxkeysize; + unsigned int i; + struct aead_testvec vec = { 0 }; + char vec_name[64]; + struct testvec_config cfg; + char cfgname[TESTVEC_CONFIG_NAMELEN]; + int err; + + if (noextratests) + return 0; + + if (!generic_driver) { /* Use default naming convention? */ + err = build_generic_driver_name(algname, _generic_driver); + if (err) + return err; + generic_driver = _generic_driver; + } + + if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */ + return 0; + + generic_tfm = crypto_alloc_aead(generic_driver, 0, 0); + if (IS_ERR(generic_tfm)) { + err = PTR_ERR(generic_tfm); + if (err == -ENOENT) { + pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n", + driver, generic_driver); + return 0; + } + pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n", + generic_driver, algname, err); + return err; + } + + generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL); + if (!generic_req) { + err = -ENOMEM; + goto out; + } + + /* Check the algorithm properties for consistency. */ + + if (maxauthsize != crypto_aead_alg(generic_tfm)->maxauthsize) { + pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n", + driver, maxauthsize, + crypto_aead_alg(generic_tfm)->maxauthsize); + err = -EINVAL; + goto out; + } + + if (ivsize != crypto_aead_ivsize(generic_tfm)) { + pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n", + driver, ivsize, crypto_aead_ivsize(generic_tfm)); + err = -EINVAL; + goto out; + } + + if (blocksize != crypto_aead_blocksize(generic_tfm)) { + pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n", + driver, blocksize, crypto_aead_blocksize(generic_tfm)); + err = -EINVAL; + goto out; + } + + /* + * Now generate test vectors using the generic implementation, and test + * the other implementation against them. + */ + + maxkeysize = 0; + for (i = 0; i < test_desc->suite.aead.count; i++) + maxkeysize = max_t(unsigned int, maxkeysize, + test_desc->suite.aead.vecs[i].klen); + + vec.key = kmalloc(maxkeysize, GFP_KERNEL); + vec.iv = kmalloc(ivsize, GFP_KERNEL); + vec.assoc = kmalloc(maxdatasize, GFP_KERNEL); + vec.ptext = kmalloc(maxdatasize, GFP_KERNEL); + vec.ctext = kmalloc(maxdatasize, GFP_KERNEL); + if (!vec.key || !vec.iv || !vec.assoc || !vec.ptext || !vec.ctext) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < fuzz_iterations * 8; i++) { + generate_random_aead_testvec(generic_req, &vec, + maxkeysize, maxdatasize, + vec_name, sizeof(vec_name)); + generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + + err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, &cfg, + req, tsgls); + if (err) + goto out; + err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, &cfg, + req, tsgls); + if (err) + goto out; + cond_resched(); + } + err = 0; +out: + kfree(vec.key); + kfree(vec.iv); + kfree(vec.assoc); + kfree(vec.ptext); + kfree(vec.ctext); + crypto_free_aead(generic_tfm); + aead_request_free(generic_req); + return err; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int test_aead_vs_generic_impl(const char *driver, + const struct alg_test_desc *test_desc, + struct aead_request *req, + struct cipher_test_sglists *tsgls) +{ + return 0; +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + static int test_aead(const char *driver, int enc, const struct aead_test_suite *suite, struct aead_request *req, @@ -1411,6 +2045,10 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver, goto out; err = test_aead(driver, DECRYPT, suite, req, tsgls); + if (err) + goto out; + + err = test_aead_vs_generic_impl(driver, desc, req, tsgls); out: free_cipher_test_sglists(tsgls); aead_request_free(req); @@ -1462,13 +2100,20 @@ static int test_cipher(struct crypto_cipher *tfm, int enc, ret = crypto_cipher_setkey(tfm, template[i].key, template[i].klen); - if (template[i].fail == !ret) { - printk(KERN_ERR "alg: cipher: setkey failed " - "on test %d for %s: flags=%x\n", j, - algo, crypto_cipher_get_flags(tfm)); + if (ret) { + if (ret == template[i].setkey_error) + continue; + pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n", + algo, j, template[i].setkey_error, ret, + crypto_cipher_get_flags(tfm)); goto out; - } else if (ret) - continue; + } + if (template[i].setkey_error) { + pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n", + algo, j, template[i].setkey_error); + ret = -EINVAL; + goto out; + } for (k = 0; k < template[i].len; k += crypto_cipher_blocksize(tfm)) { @@ -1500,7 +2145,7 @@ out_nobuf: static int test_skcipher_vec_cfg(const char *driver, int enc, const struct cipher_testvec *vec, - unsigned int vec_num, + const char *vec_name, const struct testvec_config *cfg, struct skcipher_request *req, struct cipher_test_sglists *tsgls) @@ -1526,15 +2171,16 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); err = crypto_skcipher_setkey(tfm, vec->key, vec->klen); if (err) { - if (vec->fail) /* expectedly failed to set key? */ + if (err == vec->setkey_error) return 0; - pr_err("alg: skcipher: %s setkey failed with err %d on test vector %u; flags=%#x\n", - driver, err, vec_num, crypto_skcipher_get_flags(tfm)); + pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, + crypto_skcipher_get_flags(tfm)); return err; } - if (vec->fail) { - pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %u\n", - driver, vec_num); + if (vec->setkey_error) { + pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); return -EINVAL; } @@ -1550,8 +2196,8 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, memset(iv, 0, ivsize); } else { if (vec->generates_iv) { - pr_err("alg: skcipher: %s has ivsize=0 but test vector %u generates IV!\n", - driver, vec_num); + pr_err("alg: skcipher: %s has ivsize=0 but test vector %s generates IV!\n", + driver, vec_name); return -EINVAL; } iv = NULL; @@ -1563,8 +2209,8 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, err = build_cipher_test_sglists(tsgls, cfg, alignmask, vec->len, vec->len, &input, 1); if (err) { - pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } @@ -1573,13 +2219,12 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, skcipher_request_set_callback(req, req_flags, crypto_req_done, &wait); skcipher_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr, vec->len, iv); - err = crypto_wait_req(enc ? crypto_skcipher_encrypt(req) : - crypto_skcipher_decrypt(req), &wait); - if (err) { - pr_err("alg: skcipher: %s %s failed with err %d on test vector %u, cfg=\"%s\"\n", - driver, op, err, vec_num, cfg->name); - return err; - } + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = crypto_wait_req(err, &wait); /* Check that the algorithm didn't overwrite things it shouldn't have */ if (req->cryptlen != vec->len || @@ -1590,8 +2235,8 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, req->base.complete != crypto_req_done || req->base.flags != req_flags || req->base.data != &wait) { - pr_err("alg: skcipher: %s %s corrupted request struct on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); if (req->cryptlen != vec->len) pr_err("alg: skcipher: changed 'req->cryptlen'\n"); if (req->iv != iv) @@ -1611,14 +2256,28 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, return -EINVAL; } if (is_test_sglist_corrupted(&tsgls->src)) { - pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return -EINVAL; } if (tsgls->dst.sgl_ptr != tsgls->src.sgl && is_test_sglist_corrupted(&tsgls->dst)) { - pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); + return -EINVAL; + } + + /* Check for success or failure */ + if (err) { + if (err == vec->crypt_error) + return 0; + pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, vec->crypt_error, err, cfg->name); + return err; + } + if (vec->crypt_error) { + pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, op, vec_name, vec->crypt_error, cfg->name); return -EINVAL; } @@ -1626,20 +2285,20 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext, vec->len, 0, true); if (err == -EOVERFLOW) { - pr_err("alg: skcipher: %s %s overran dst buffer on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } if (err) { - pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); return err; } /* If applicable, check that the algorithm generated the correct IV */ if (vec->iv_out && memcmp(iv, vec->iv_out, ivsize) != 0) { - pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %u, cfg=\"%s\"\n", - driver, op, vec_num, cfg->name); + pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n", + driver, op, vec_name, cfg->name); hexdump(iv, ivsize); return -EINVAL; } @@ -1653,14 +2312,17 @@ static int test_skcipher_vec(const char *driver, int enc, struct skcipher_request *req, struct cipher_test_sglists *tsgls) { + char vec_name[16]; unsigned int i; int err; if (fips_enabled && vec->fips_skip) return 0; + sprintf(vec_name, "%u", vec_num); + for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) { - err = test_skcipher_vec_cfg(driver, enc, vec, vec_num, + err = test_skcipher_vec_cfg(driver, enc, vec, vec_name, &default_cipher_testvec_configs[i], req, tsgls); if (err) @@ -1675,7 +2337,7 @@ static int test_skcipher_vec(const char *driver, int enc, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_skcipher_vec_cfg(driver, enc, vec, vec_num, + err = test_skcipher_vec_cfg(driver, enc, vec, vec_name, &cfg, req, tsgls); if (err) return err; @@ -1685,6 +2347,186 @@ static int test_skcipher_vec(const char *driver, int enc, return 0; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +/* + * Generate a symmetric cipher test vector from the given implementation. + * Assumes the buffers in 'vec' were already allocated. + */ +static void generate_random_cipher_testvec(struct skcipher_request *req, + struct cipher_testvec *vec, + unsigned int maxdatasize, + char *name, size_t max_namelen) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const unsigned int maxkeysize = tfm->keysize; + const unsigned int ivsize = crypto_skcipher_ivsize(tfm); + struct scatterlist src, dst; + u8 iv[MAX_IVLEN]; + DECLARE_CRYPTO_WAIT(wait); + + /* Key: length in [0, maxkeysize], but usually choose maxkeysize */ + vec->klen = maxkeysize; + if (prandom_u32() % 4 == 0) + vec->klen = prandom_u32() % (maxkeysize + 1); + generate_random_bytes((u8 *)vec->key, vec->klen); + vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen); + + /* IV */ + generate_random_bytes((u8 *)vec->iv, ivsize); + + /* Plaintext */ + vec->len = generate_random_length(maxdatasize); + generate_random_bytes((u8 *)vec->ptext, vec->len); + + /* If the key couldn't be set, no need to continue to encrypt. */ + if (vec->setkey_error) + goto done; + + /* Ciphertext */ + sg_init_one(&src, vec->ptext, vec->len); + sg_init_one(&dst, vec->ctext, vec->len); + memcpy(iv, vec->iv, ivsize); + skcipher_request_set_callback(req, 0, crypto_req_done, &wait); + skcipher_request_set_crypt(req, &src, &dst, vec->len, iv); + vec->crypt_error = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); +done: + snprintf(name, max_namelen, "\"random: len=%u klen=%u\"", + vec->len, vec->klen); +} + +/* + * Test the skcipher algorithm represented by @req against the corresponding + * generic implementation, if one is available. + */ +static int test_skcipher_vs_generic_impl(const char *driver, + const char *generic_driver, + struct skcipher_request *req, + struct cipher_test_sglists *tsgls) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const unsigned int ivsize = crypto_skcipher_ivsize(tfm); + const unsigned int blocksize = crypto_skcipher_blocksize(tfm); + const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; + const char *algname = crypto_skcipher_alg(tfm)->base.cra_name; + char _generic_driver[CRYPTO_MAX_ALG_NAME]; + struct crypto_skcipher *generic_tfm = NULL; + struct skcipher_request *generic_req = NULL; + unsigned int i; + struct cipher_testvec vec = { 0 }; + char vec_name[64]; + struct testvec_config cfg; + char cfgname[TESTVEC_CONFIG_NAMELEN]; + int err; + + if (noextratests) + return 0; + + /* Keywrap isn't supported here yet as it handles its IV differently. */ + if (strncmp(algname, "kw(", 3) == 0) + return 0; + + if (!generic_driver) { /* Use default naming convention? */ + err = build_generic_driver_name(algname, _generic_driver); + if (err) + return err; + generic_driver = _generic_driver; + } + + if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */ + return 0; + + generic_tfm = crypto_alloc_skcipher(generic_driver, 0, 0); + if (IS_ERR(generic_tfm)) { + err = PTR_ERR(generic_tfm); + if (err == -ENOENT) { + pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n", + driver, generic_driver); + return 0; + } + pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n", + generic_driver, algname, err); + return err; + } + + generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL); + if (!generic_req) { + err = -ENOMEM; + goto out; + } + + /* Check the algorithm properties for consistency. */ + + if (tfm->keysize != generic_tfm->keysize) { + pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n", + driver, tfm->keysize, generic_tfm->keysize); + err = -EINVAL; + goto out; + } + + if (ivsize != crypto_skcipher_ivsize(generic_tfm)) { + pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n", + driver, ivsize, crypto_skcipher_ivsize(generic_tfm)); + err = -EINVAL; + goto out; + } + + if (blocksize != crypto_skcipher_blocksize(generic_tfm)) { + pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n", + driver, blocksize, + crypto_skcipher_blocksize(generic_tfm)); + err = -EINVAL; + goto out; + } + + /* + * Now generate test vectors using the generic implementation, and test + * the other implementation against them. + */ + + vec.key = kmalloc(tfm->keysize, GFP_KERNEL); + vec.iv = kmalloc(ivsize, GFP_KERNEL); + vec.ptext = kmalloc(maxdatasize, GFP_KERNEL); + vec.ctext = kmalloc(maxdatasize, GFP_KERNEL); + if (!vec.key || !vec.iv || !vec.ptext || !vec.ctext) { + err = -ENOMEM; + goto out; + } + + for (i = 0; i < fuzz_iterations * 8; i++) { + generate_random_cipher_testvec(generic_req, &vec, maxdatasize, + vec_name, sizeof(vec_name)); + generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + + err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name, + &cfg, req, tsgls); + if (err) + goto out; + err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name, + &cfg, req, tsgls); + if (err) + goto out; + cond_resched(); + } + err = 0; +out: + kfree(vec.key); + kfree(vec.iv); + kfree(vec.ptext); + kfree(vec.ctext); + crypto_free_skcipher(generic_tfm); + skcipher_request_free(generic_req); + return err; +} +#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int test_skcipher_vs_generic_impl(const char *driver, + const char *generic_driver, + struct skcipher_request *req, + struct cipher_test_sglists *tsgls) +{ + return 0; +} +#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ + static int test_skcipher(const char *driver, int enc, const struct cipher_test_suite *suite, struct skcipher_request *req, @@ -1744,6 +2586,11 @@ static int alg_test_skcipher(const struct alg_test_desc *desc, goto out; err = test_skcipher(driver, DECRYPT, suite, req, tsgls); + if (err) + goto out; + + err = test_skcipher_vs_generic_impl(driver, desc->generic_driver, req, + tsgls); out: free_cipher_test_sglists(tsgls); skcipher_request_free(req); @@ -2179,7 +3026,6 @@ static int alg_test_crc32c(const struct alg_test_desc *desc, u32 *ctx = (u32 *)shash_desc_ctx(shash); shash->tfm = tfm; - shash->flags = 0; *ctx = 420553207; err = crypto_shash_final(shash, (u8 *)&val); @@ -2493,6 +3339,12 @@ static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver, return err; } +static u8 *test_pack_u32(u8 *dst, u32 val) +{ + memcpy(dst, &val, sizeof(val)); + return dst + sizeof(val); +} + static int test_akcipher_one(struct crypto_akcipher *tfm, const struct akcipher_testvec *vecs) { @@ -2503,10 +3355,11 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, struct crypto_wait wait; unsigned int out_len_max, out_len = 0; int err = -ENOMEM; - struct scatterlist src, dst, src_tab[2]; + struct scatterlist src, dst, src_tab[3]; const char *m, *c; unsigned int m_size, c_size; const char *op; + u8 *key, *ptr; if (testmgr_alloc_buf(xbuf)) return err; @@ -2517,22 +3370,29 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, crypto_init_wait(&wait); + key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len, + GFP_KERNEL); + if (!key) + goto free_xbuf; + memcpy(key, vecs->key, vecs->key_len); + ptr = key + vecs->key_len; + ptr = test_pack_u32(ptr, vecs->algo); + ptr = test_pack_u32(ptr, vecs->param_len); + memcpy(ptr, vecs->params, vecs->param_len); + if (vecs->public_key_vec) - err = crypto_akcipher_set_pub_key(tfm, vecs->key, - vecs->key_len); + err = crypto_akcipher_set_pub_key(tfm, key, vecs->key_len); else - err = crypto_akcipher_set_priv_key(tfm, vecs->key, - vecs->key_len); + err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len); if (err) goto free_req; - err = -ENOMEM; - out_len_max = crypto_akcipher_maxsize(tfm); - /* * First run test which do not require a private key, such as * encrypt or verify. */ + err = -ENOMEM; + out_len_max = crypto_akcipher_maxsize(tfm); outbuf_enc = kzalloc(out_len_max, GFP_KERNEL); if (!outbuf_enc) goto free_req; @@ -2558,12 +3418,20 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, goto free_all; memcpy(xbuf[0], m, m_size); - sg_init_table(src_tab, 2); + sg_init_table(src_tab, 3); sg_set_buf(&src_tab[0], xbuf[0], 8); sg_set_buf(&src_tab[1], xbuf[0] + 8, m_size - 8); - sg_init_one(&dst, outbuf_enc, out_len_max); - akcipher_request_set_crypt(req, src_tab, &dst, m_size, - out_len_max); + if (vecs->siggen_sigver_test) { + if (WARN_ON(c_size > PAGE_SIZE)) + goto free_all; + memcpy(xbuf[1], c, c_size); + sg_set_buf(&src_tab[2], xbuf[1], c_size); + akcipher_request_set_crypt(req, src_tab, NULL, m_size, c_size); + } else { + sg_init_one(&dst, outbuf_enc, out_len_max); + akcipher_request_set_crypt(req, src_tab, &dst, m_size, + out_len_max); + } akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &wait); @@ -2576,18 +3444,21 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, pr_err("alg: akcipher: %s test failed. err %d\n", op, err); goto free_all; } - if (req->dst_len != c_size) { - pr_err("alg: akcipher: %s test failed. Invalid output len\n", - op); - err = -EINVAL; - goto free_all; - } - /* verify that encrypted message is equal to expected */ - if (memcmp(c, outbuf_enc, c_size)) { - pr_err("alg: akcipher: %s test failed. Invalid output\n", op); - hexdump(outbuf_enc, c_size); - err = -EINVAL; - goto free_all; + if (!vecs->siggen_sigver_test) { + if (req->dst_len != c_size) { + pr_err("alg: akcipher: %s test failed. Invalid output len\n", + op); + err = -EINVAL; + goto free_all; + } + /* verify that encrypted message is equal to expected */ + if (memcmp(c, outbuf_enc, c_size) != 0) { + pr_err("alg: akcipher: %s test failed. Invalid output\n", + op); + hexdump(outbuf_enc, c_size); + err = -EINVAL; + goto free_all; + } } /* @@ -2642,6 +3513,7 @@ free_all: kfree(outbuf_enc); free_req: akcipher_request_free(req); + kfree(key); free_xbuf: testmgr_free_buf(xbuf); return err; @@ -2699,12 +3571,14 @@ static int alg_test_null(const struct alg_test_desc *desc, static const struct alg_test_desc alg_test_descs[] = { { .alg = "adiantum(xchacha12,aes)", + .generic_driver = "adiantum(xchacha12-generic,aes-generic,nhpoly1305-generic)", .test = alg_test_skcipher, .suite = { .cipher = __VECS(adiantum_xchacha12_aes_tv_template) }, }, { .alg = "adiantum(xchacha20,aes)", + .generic_driver = "adiantum(xchacha20-generic,aes-generic,nhpoly1305-generic)", .test = alg_test_skcipher, .suite = { .cipher = __VECS(adiantum_xchacha20_aes_tv_template) @@ -2921,6 +3795,12 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + /* Same as cbc(sm4) except the key is stored in + * hardware secure memory which we reference by index + */ + .alg = "cbc(psm4)", + .test = alg_test_null, + }, { .alg = "cbc(serpent)", .test = alg_test_skcipher, .suite = { @@ -2947,6 +3827,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "ccm(aes)", + .generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3055,6 +3936,13 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + + /* Same as ctr(sm4) except the key is stored in + * hardware secure memory which we reference by index + */ + .alg = "ctr(psm4)", + .test = alg_test_null, + }, { .alg = "ctr(serpent)", .test = alg_test_skcipher, .suite = { @@ -3080,6 +3968,13 @@ static const struct alg_test_desc alg_test_descs[] = { .cipher = __VECS(cts_mode_tv_template) } }, { + /* Same as cts(cbc((aes)) except the key is stored in + * hardware secure memory which we reference by index + */ + .alg = "cts(cbc(paes))", + .test = alg_test_null, + .fips_allowed = 1, + }, { .alg = "deflate", .test = alg_test_comp, .fips_allowed = 1, @@ -3358,7 +4253,14 @@ static const struct alg_test_desc alg_test_descs[] = { .kpp = __VECS(ecdh_tv_template) } }, { + .alg = "ecrdsa", + .test = alg_test_akcipher, + .suite = { + .akcipher = __VECS(ecrdsa_tv_template) + } + }, { .alg = "gcm(aes)", + .generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3477,30 +4379,35 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "lrw(aes)", + .generic_driver = "lrw(ecb(aes-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(aes_lrw_tv_template) } }, { .alg = "lrw(camellia)", + .generic_driver = "lrw(ecb(camellia-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(camellia_lrw_tv_template) } }, { .alg = "lrw(cast6)", + .generic_driver = "lrw(ecb(cast6-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(cast6_lrw_tv_template) } }, { .alg = "lrw(serpent)", + .generic_driver = "lrw(ecb(serpent-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(serpent_lrw_tv_template) } }, { .alg = "lrw(twofish)", + .generic_driver = "lrw(ecb(twofish-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(tf_lrw_tv_template) @@ -3625,6 +4532,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4106(gcm(aes))", + .generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3632,6 +4540,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4309(ccm(aes))", + .generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -3639,6 +4548,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4543(gcm(aes))", + .generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(aes_gcm_rfc4543_tv_template) @@ -3835,6 +4745,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, }, { .alg = "xts(aes)", + .generic_driver = "xts(ecb(aes-generic))", .test = alg_test_skcipher, .fips_allowed = 1, .suite = { @@ -3842,12 +4753,14 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "xts(camellia)", + .generic_driver = "xts(ecb(camellia-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(camellia_xts_tv_template) } }, { .alg = "xts(cast6)", + .generic_driver = "xts(ecb(cast6-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(cast6_xts_tv_template) @@ -3861,12 +4774,14 @@ static const struct alg_test_desc alg_test_descs[] = { .fips_allowed = 1, }, { .alg = "xts(serpent)", + .generic_driver = "xts(ecb(serpent-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(serpent_xts_tv_template) } }, { .alg = "xts(twofish)", + .generic_driver = "xts(ecb(twofish-generic))", .test = alg_test_skcipher, .suite = { .cipher = __VECS(tf_xts_tv_template) @@ -4020,8 +4935,9 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) type, mask); test_done: - if (fips_enabled && rc) - panic("%s: %s alg self test failed in fips mode!\n", driver, alg); + if (rc && (fips_enabled || panic_on_fail)) + panic("alg: self-tests for %s (%s) failed in %s mode!\n", + driver, alg, fips_enabled ? "fips" : "panic_on_fail"); if (fips_enabled && !rc) pr_info("alg: self-tests for %s (%s) passed\n", driver, alg); diff --git a/crypto/testmgr.h b/crypto/testmgr.h index d18a37629f05..b6daae1f6a1d 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -25,6 +25,8 @@ #ifndef _CRYPTO_TESTMGR_H #define _CRYPTO_TESTMGR_H +#include <linux/oid_registry.h> + #define MAX_IVLEN 32 /* @@ -34,6 +36,8 @@ * @digest: Pointer to expected digest * @psize: Length of source data in bytes * @ksize: Length of @key in bytes (0 if no key) + * @setkey_error: Expected error from setkey() + * @digest_error: Expected error from digest() */ struct hash_testvec { const char *key; @@ -41,6 +45,8 @@ struct hash_testvec { const char *digest; unsigned short psize; unsigned short ksize; + int setkey_error; + int digest_error; }; /* @@ -52,12 +58,13 @@ struct hash_testvec { * @ptext: Pointer to plaintext * @ctext: Pointer to ciphertext * @len: Length of @ptext and @ctext in bytes - * @fail: If set to one, the test need to fail * @wk: Does the test need CRYPTO_TFM_REQ_FORBID_WEAK_KEYS? * ( e.g. test needs to fail due to a weak key ) * @fips_skip: Skip the test vector in FIPS mode * @generates_iv: Encryption should ignore the given IV, and output @iv_out. * Decryption takes @iv_out. Needed for AES Keywrap ("kw(aes)"). + * @setkey_error: Expected error from setkey() + * @crypt_error: Expected error from encrypt() and decrypt() */ struct cipher_testvec { const char *key; @@ -65,12 +72,13 @@ struct cipher_testvec { const char *iv_out; const char *ptext; const char *ctext; - bool fail; unsigned char wk; /* weak key flag */ - unsigned char klen; + unsigned short klen; unsigned short len; bool fips_skip; bool generates_iv; + int setkey_error; + int crypt_error; }; /* @@ -82,7 +90,6 @@ struct cipher_testvec { * @ctext: Pointer to the full authenticated ciphertext. For AEADs that * produce a separate "ciphertext" and "authentication tag", these * two parts are concatenated: ciphertext || tag. - * @fail: setkey() failure expected? * @novrfy: Decryption verification failure expected? * @wk: Does the test need CRYPTO_TFM_REQ_FORBID_WEAK_KEYS? * (e.g. setkey() needs to fail due to a weak key) @@ -90,6 +97,9 @@ struct cipher_testvec { * @plen: Length of @ptext in bytes * @alen: Length of @assoc in bytes * @clen: Length of @ctext in bytes + * @setkey_error: Expected error from setkey() + * @setauthsize_error: Expected error from setauthsize() + * @crypt_error: Expected error from encrypt() and decrypt() */ struct aead_testvec { const char *key; @@ -97,13 +107,15 @@ struct aead_testvec { const char *ptext; const char *assoc; const char *ctext; - bool fail; unsigned char novrfy; unsigned char wk; unsigned char klen; unsigned short plen; unsigned short clen; unsigned short alen; + int setkey_error; + int setauthsize_error; + int crypt_error; }; struct cprng_testvec { @@ -135,13 +147,16 @@ struct drbg_testvec { struct akcipher_testvec { const unsigned char *key; + const unsigned char *params; const unsigned char *m; const unsigned char *c; unsigned int key_len; + unsigned int param_len; unsigned int m_size; unsigned int c_size; bool public_key_vec; bool siggen_sigver_test; + enum OID algo; }; struct kpp_testvec { @@ -551,6 +566,160 @@ static const struct akcipher_testvec rsa_tv_template[] = { }; /* + * EC-RDSA test vectors are generated by gost-engine. + */ +static const struct akcipher_testvec ecrdsa_tv_template[] = { + { + .key = + "\x04\x40\xd5\xa7\x77\xf9\x26\x2f\x8c\xbd\xcc\xe3\x1f\x01\x94\x05" + "\x3d\x2f\xec\xb5\x00\x34\xf5\x51\x6d\x3b\x90\x4b\x23\x28\x6f\x1d" + "\xc8\x36\x61\x60\x36\xec\xbb\xb4\x0b\x95\x4e\x54\x4f\x15\x21\x05" + "\xd8\x52\x66\x44\x31\x7e\x5d\xc5\xd1\x26\x00\x5f\x60\xd8\xf0\xc7" + "\x27\xfc", + .key_len = 66, + .params = /* OID_gostCPSignA */ + "\x30\x13\x06\x07\x2a\x85\x03\x02\x02\x23\x01\x06\x08\x2a\x85\x03" + "\x07\x01\x01\x02\x02", + .param_len = 21, + .c = + "\x41\x32\x09\x73\xa4\xc1\x38\xd6\x63\x7d\x8b\xf7\x50\x3f\xda\x9f" + "\x68\x48\xc1\x50\xe3\x42\x3a\x9b\x2b\x28\x12\x2a\xa7\xc2\x75\x31" + "\x65\x77\x8c\x3c\x9e\x0d\x56\xb2\xf9\xdc\x04\x33\x3e\xb0\x9e\xf9" + "\x74\x4e\x59\xb3\x83\xf2\x91\x27\xda\x5e\xc7\x33\xc0\xc1\x8f\x41", + .c_size = 64, + .algo = OID_gost2012PKey256, + .m = + "\x75\x1b\x9b\x40\x25\xb9\x96\xd2\x9b\x00\x41\xb3\x58\xbf\x23\x14" + "\x79\xd2\x76\x64\xa3\xbd\x66\x10\x79\x05\x5a\x06\x42\xec\xb9\xc9", + .m_size = 32, + .public_key_vec = true, + .siggen_sigver_test = true, + }, + { + .key = + "\x04\x40\x66\x6f\xd6\xb7\x06\xd0\xf5\xa5\x6f\x69\x5c\xa5\x13\x45" + "\x14\xdd\xcb\x12\x9c\x1b\xf5\x28\x64\x7a\x49\x48\x29\x14\x66\x42" + "\xb8\x1b\x5c\xf9\x56\x6d\x08\x3b\xce\xbb\x62\x2f\xc2\x3c\xc5\x49" + "\x93\x27\x70\x20\xcc\x79\xeb\xdc\x76\x8e\x48\x6e\x04\x96\xc3\x29" + "\xa0\x73", + .key_len = 66, + .params = /* OID_gostCPSignB */ + "\x30\x13\x06\x07\x2a\x85\x03\x02\x02\x23\x02\x06\x08\x2a\x85\x03" + "\x07\x01\x01\x02\x02", + .param_len = 21, + .c = + "\x45\x6d\x4a\x03\x1d\x5c\x0b\x17\x79\xe7\x19\xdb\xbf\x81\x9f\x82" + "\xae\x06\xda\xf5\x47\x00\x05\x80\xc3\x16\x06\x9a\x8e\x7c\xb2\x8e" + "\x7f\x74\xaa\xec\x6b\x7b\x7f\x8b\xc6\x0b\x10\x42\x4e\x91\x2c\xdf" + "\x7b\x8b\x15\xf4\x9e\x59\x0f\xc7\xa4\x68\x2e\xce\x89\xdf\x84\xe9", + .c_size = 64, + .algo = OID_gost2012PKey256, + .m = + "\xd0\x54\x00\x27\x6a\xeb\xce\x6c\xf5\xf6\xfb\x57\x18\x18\x21\x13" + "\x11\x23\x4a\x70\x43\x52\x7a\x68\x11\x65\x45\x37\xbb\x25\xb7\x40", + .m_size = 32, + .public_key_vec = true, + .siggen_sigver_test = true, + }, + { + .key = + "\x04\x40\x05\x91\xa9\x7d\xcb\x87\xdc\x98\xa1\xbf\xff\xdd\x20\x61" + "\xaa\x58\x3b\x2d\x8e\x9c\x41\x9d\x4f\xc6\x23\x17\xf9\xca\x60\x65" + "\xbc\x97\x97\xf6\x6b\x24\xe8\xac\xb1\xa7\x61\x29\x3c\x71\xdc\xad" + "\xcb\x20\xbe\x96\xe8\xf4\x44\x2e\x49\xd5\x2c\xb9\xc9\x3b\x9c\xaa" + "\xba\x15", + .key_len = 66, + .params = /* OID_gostCPSignC */ + "\x30\x13\x06\x07\x2a\x85\x03\x02\x02\x23\x03\x06\x08\x2a\x85\x03" + "\x07\x01\x01\x02\x02", + .param_len = 21, + .c = + "\x3b\x2e\x2e\x74\x74\x47\xda\xea\x93\x90\x6a\xe2\xf5\xf5\xe6\x46" + "\x11\xfc\xab\xdc\x52\xbc\x58\xdb\x45\x44\x12\x4a\xf7\xd0\xab\xc9" + "\x73\xba\x64\xab\x0d\xac\x4e\x72\x10\xa8\x04\xf6\x1e\xe0\x48\x6a" + "\xcd\xe8\xe3\x78\x73\x77\x82\x24\x8d\xf1\xd3\xeb\x4c\x25\x7e\xc0", + .c_size = 64, + .algo = OID_gost2012PKey256, + .m = + "\x52\x33\xf4\x3f\x7b\x5d\xcf\x20\xee\xe4\x5c\xab\x0b\x3f\x14\xd6" + "\x9f\x16\xc6\x1c\xb1\x3f\x84\x41\x69\xec\x34\xfd\xf1\xf9\xa3\x39", + .m_size = 32, + .public_key_vec = true, + .siggen_sigver_test = true, + }, + { + .key = + "\x04\x81\x80\x85\x46\x8f\x16\xf8\x7a\x7e\x4a\xc3\x81\x9e\xf1\x6e" + "\x94\x1e\x5d\x02\x87\xea\xfa\xa0\x0a\x17\x70\x49\x64\xad\x95\x68" + "\x60\x0a\xf0\x57\x29\x41\x79\x30\x3c\x61\x69\xf2\xa6\x94\x87\x17" + "\x54\xfa\x97\x2c\xe6\x1e\x0a\xbb\x55\x10\x57\xbe\xf7\xc1\x77\x2b" + "\x11\x74\x0a\x50\x37\x14\x10\x2a\x45\xfc\x7a\xae\x1c\x4c\xce\x08" + "\x05\xb7\xa4\x50\xc8\x3d\x39\x3d\xdc\x5c\x8f\x96\x6c\xe7\xfc\x21" + "\xc3\x2d\x1e\x9f\x11\xb3\xec\x22\x18\x8a\x8c\x08\x6b\x8b\xed\xf5" + "\xc5\x47\x3c\x7e\x73\x59\x44\x1e\x77\x83\x84\x52\x9e\x3b\x7d\xff" + "\x9d\x86\x1a", + .key_len = 131, + .params = /* OID_gostTC26Sign512A */ + "\x30\x0b\x06\x09\x2a\x85\x03\x07\x01\x02\x01\x02\x01", + .param_len = 13, + .c = + "\x92\x81\x74\x5f\x95\x48\x38\x87\xd9\x8f\x5e\xc8\x8a\xbb\x01\x4e" + "\xb0\x75\x3c\x2f\xc7\x5a\x08\x4c\x68\xab\x75\x01\x32\x75\x75\xb5" + "\x37\xe0\x74\x6d\x94\x84\x31\x2a\x6b\xf4\xf7\xb7\xa7\x39\x7b\x46" + "\x07\xf0\x98\xbd\x33\x18\xa1\x72\xb2\x6d\x54\xe3\xde\x91\xc2\x2e" + "\x4f\x6a\xf8\xb7\xec\xa8\x83\xc9\x8f\xd9\xce\x7c\x45\x06\x02\xf4" + "\x4f\x21\xb5\x24\x3d\xb4\xb5\xd8\x58\x42\xbe\x2d\x29\xae\x93\xc0" + "\x13\x41\x96\x35\x08\x69\xe8\x36\xc7\xd1\x83\x81\xd7\xca\xfb\xc0" + "\xd2\xb7\x78\x32\x3e\x30\x1a\x1e\xce\xdc\x34\x35\xc6\xad\x68\x24", + .c_size = 128, + .algo = OID_gost2012PKey512, + .m = + "\x1f\x70\xb5\xe9\x55\x12\xd6\x88\xcc\x55\xb9\x0c\x7f\xc4\x94\xf2" + "\x04\x77\x41\x12\x02\xd6\xf1\x1f\x83\x56\xe9\xd6\x5a\x6a\x72\xb9" + "\x6e\x8e\x24\x2a\x84\xf1\xba\x67\xe8\xbf\xff\xc1\xd3\xde\xfb\xc6" + "\xa8\xf6\x80\x01\xb9\x27\xac\xd8\x45\x96\x66\xa1\xee\x48\x08\x3f", + .m_size = 64, + .public_key_vec = true, + .siggen_sigver_test = true, + }, + { + .key = + "\x04\x81\x80\x28\xf3\x2b\x92\x04\x32\xea\x66\x20\xde\xa0\x2f\x74" + "\xbf\x2d\xf7\xb5\x30\x76\xb1\xc8\xee\x38\x9f\xea\xe5\xad\xc6\xa3" + "\x28\x1e\x51\x3d\x67\xa3\x41\xcc\x6b\x81\xe2\xe2\x9e\x82\xf3\x78" + "\x56\xd7\x2e\xb2\xb5\xbe\xb4\x50\x21\x05\xe5\x29\x82\xef\x15\x1b" + "\xc0\xd7\x30\xd6\x2f\x96\xe8\xff\x99\x4c\x25\xcf\x9a\xfc\x54\x30" + "\xce\xdf\x59\xe9\xc6\x45\xce\xe4\x22\xe8\x01\xd5\xcd\x2f\xaa\x78" + "\x99\xc6\x04\x1e\x6f\x4c\x25\x6a\x76\xad\xff\x48\xf3\xb3\xb4\xd6" + "\x14\x5c\x2c\x0e\xea\xa2\x4b\xb9\x7e\x89\x77\x02\x3a\x29\xc8\x16" + "\x8e\x78\x48", + .key_len = 131, + .params = /* OID_gostTC26Sign512B */ + "\x30\x0b\x06\x09\x2a\x85\x03\x07\x01\x02\x01\x02\x02", + .param_len = 13, + .c = + "\x0a\xed\xb6\x27\xea\xa7\xa6\x7e\x2f\xc1\x02\x21\x74\xce\x27\xd2" + "\xee\x8a\x92\x4d\xa9\x43\x2d\xa4\x5b\xdc\x23\x02\xfc\x3a\xf3\xb2" + "\x10\x93\x0b\x40\x1b\x75\x95\x3e\x39\x41\x37\xb9\xab\x51\x09\xeb" + "\xf1\xb9\x49\x58\xec\x58\xc7\xf9\x2e\xb9\xc9\x40\xf2\x00\x39\x7e" + "\x3f\xde\x72\xe3\x85\x67\x06\xbe\xd8\xb8\xc1\x81\x1e\xe3\x0a\xfe" + "\xce\xd3\x77\x92\x56\x8c\x58\xf9\x37\x60\x2d\xe6\x8b\x66\xa3\xdd" + "\xd2\xf0\xf8\xda\x1b\x20\xbc\x9c\xec\x29\x5d\xd1\x8f\xcc\x37\xd1" + "\x3b\x8d\xb7\xc1\xe0\xb8\x3b\xef\x14\x1b\x87\xbc\xc1\x03\x9a\x93", + .c_size = 128, + .algo = OID_gost2012PKey512, + .m = + "\x11\x24\x21\x27\xf2\x42\x9f\xce\x5a\xf9\x01\x70\xe0\x07\x2b\x57" + "\xfb\x7d\x77\x5e\x74\x66\xe6\xa5\x40\x4c\x1a\x85\x18\xff\xd0\x63" + "\xe0\x39\xd3\xd6\xe5\x17\xf8\xc3\x4b\xc6\x1c\x33\x1a\xca\xa6\x66" + "\x6d\xf4\xd2\x45\xc2\x83\xa0\x42\x95\x05\x9d\x89\x8e\x0a\xca\xcc", + .m_size = 64, + .public_key_vec = true, + .siggen_sigver_test = true, + }, +}; + +/* * PKCS#1 RSA test vectors. Obtained from CAVS testing. */ static const struct akcipher_testvec pkcs1pad_rsa_tv_template[] = { @@ -7084,7 +7253,7 @@ static const struct cipher_testvec des_tv_template[] = { "\xb4\x99\x26\xf7\x1f\xe1\xd4\x90", .len = 24, }, { /* Weak key */ - .fail = true, + .setkey_error = -EINVAL, .wk = 1, .key = "\x01\x01\x01\x01\x01\x01\x01\x01", .klen = 8, diff --git a/crypto/tgr192.c b/crypto/tgr192.c index f8e1d9f9938f..40020f8adc46 100644 --- a/crypto/tgr192.c +++ b/crypto/tgr192.c @@ -677,7 +677,7 @@ MODULE_ALIAS_CRYPTO("tgr192"); MODULE_ALIAS_CRYPTO("tgr160"); MODULE_ALIAS_CRYPTO("tgr128"); -module_init(tgr192_mod_init); +subsys_initcall(tgr192_mod_init); module_exit(tgr192_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c index 07e62433fbfb..dbac6e233285 100644 --- a/crypto/twofish_generic.c +++ b/crypto/twofish_generic.c @@ -205,7 +205,7 @@ static void __exit twofish_mod_fini(void) crypto_unregister_alg(&alg); } -module_init(twofish_mod_init); +subsys_initcall(twofish_mod_init); module_exit(twofish_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/vmac.c b/crypto/vmac.c index 5f436dfdfc61..f50a85060b39 100644 --- a/crypto/vmac.c +++ b/crypto/vmac.c @@ -690,7 +690,7 @@ static void __exit vmac_module_exit(void) crypto_unregister_template(&vmac64_tmpl); } -module_init(vmac_module_init); +subsys_initcall(vmac_module_init); module_exit(vmac_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/wp512.c b/crypto/wp512.c index 149e577fb772..1b8e502d999f 100644 --- a/crypto/wp512.c +++ b/crypto/wp512.c @@ -1168,7 +1168,7 @@ MODULE_ALIAS_CRYPTO("wp512"); MODULE_ALIAS_CRYPTO("wp384"); MODULE_ALIAS_CRYPTO("wp256"); -module_init(wp512_mod_init); +subsys_initcall(wp512_mod_init); module_exit(wp512_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/xcbc.c b/crypto/xcbc.c index c055f57fab11..94ca694ef091 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -282,7 +282,7 @@ static void __exit crypto_xcbc_module_exit(void) crypto_unregister_template(&crypto_xcbc_tmpl); } -module_init(crypto_xcbc_module_init); +subsys_initcall(crypto_xcbc_module_init); module_exit(crypto_xcbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/xts.c b/crypto/xts.c index 2f948328cabb..33cf726df4ac 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -363,7 +363,7 @@ static void __exit crypto_module_exit(void) crypto_unregister_template(&crypto_tmpl); } -module_init(crypto_module_init); +subsys_initcall(crypto_module_init); module_exit(crypto_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/zstd.c b/crypto/zstd.c index 9a76b3ed8b8b..2c04055e407f 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -257,7 +257,7 @@ static void __exit zstd_mod_fini(void) crypto_unregister_scomp(&scomp); } -module_init(zstd_mod_init); +subsys_initcall(zstd_mod_init); module_exit(zstd_mod_fini); MODULE_LICENSE("GPL"); diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index c7ad88d91a09..843a9b9b3d74 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -5443,7 +5443,6 @@ static int drbd_do_auth(struct drbd_connection *connection) rcu_read_unlock(); desc->tfm = connection->cram_hmac_tfm; - desc->flags = 0; rv = crypto_shash_setkey(connection->cram_hmac_tfm, (u8 *)secret, key_len); if (rv) { diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c index 268ef0c5d4ab..6781bcf3ec26 100644 --- a/drivers/block/drbd/drbd_worker.c +++ b/drivers/block/drbd/drbd_worker.c @@ -304,7 +304,6 @@ void drbd_csum_ee(struct crypto_shash *tfm, struct drbd_peer_request *peer_req, void *src; desc->tfm = tfm; - desc->flags = 0; crypto_shash_init(desc); @@ -332,7 +331,6 @@ void drbd_csum_bio(struct crypto_shash *tfm, struct bio *bio, void *digest) struct bvec_iter iter; desc->tfm = tfm; - desc->flags = 0; crypto_shash_init(desc); diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index b65ff6962899..e9b6ac61fb7f 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -443,6 +443,7 @@ static int omap_rng_probe(struct platform_device *pdev) priv->rng.read = omap_rng_do_read; priv->rng.init = omap_rng_init; priv->rng.cleanup = omap_rng_cleanup; + priv->rng.quality = 900; priv->rng.priv = (unsigned long)priv; platform_set_drvdata(pdev, priv); diff --git a/drivers/char/hw_random/stm32-rng.c b/drivers/char/hw_random/stm32-rng.c index 042860d97b15..0ef5b6a3f560 100644 --- a/drivers/char/hw_random/stm32-rng.c +++ b/drivers/char/hw_random/stm32-rng.c @@ -161,6 +161,7 @@ static int stm32_rng_probe(struct platform_device *ofdev) #endif priv->rng.read = stm32_rng_read, priv->rng.priv = (unsigned long) dev; + priv->rng.quality = 900; pm_runtime_set_autosuspend_delay(dev, 100); pm_runtime_use_autosuspend(dev); @@ -169,6 +170,13 @@ static int stm32_rng_probe(struct platform_device *ofdev) return devm_hwrng_register(dev, &priv->rng); } +static int stm32_rng_remove(struct platform_device *ofdev) +{ + pm_runtime_disable(&ofdev->dev); + + return 0; +} + #ifdef CONFIG_PM static int stm32_rng_runtime_suspend(struct device *dev) { @@ -210,6 +218,7 @@ static struct platform_driver stm32_rng_driver = { .of_match_table = stm32_rng_match, }, .probe = stm32_rng_probe, + .remove = stm32_rng_remove, }; module_platform_driver(stm32_rng_driver); diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 0be55fcc19ba..177b7713bd2d 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -404,15 +404,6 @@ config CRYPTO_DEV_SAHARA This option enables support for the SAHARA HW crypto accelerator found in some Freescale i.MX chips. -config CRYPTO_DEV_MXC_SCC - tristate "Support for Freescale Security Controller (SCC)" - depends on ARCH_MXC && OF - select CRYPTO_BLKCIPHER - select CRYPTO_DES - help - This option enables support for the Security Controller (SCC) - found in Freescale i.MX25 chips. - config CRYPTO_DEV_EXYNOS_RNG tristate "EXYNOS HW pseudo random number generator support" depends on ARCH_EXYNOS || COMPILE_TEST diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 8e7e225d2446..a23a7197fcd7 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -18,7 +18,6 @@ obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += marvell/ obj-$(CONFIG_CRYPTO_DEV_MEDIATEK) += mediatek/ obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o -obj-$(CONFIG_CRYPTO_DEV_MXC_SCC) += mxc-scc.o obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o n2_crypto-y := n2_core.o n2_asm.o obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 4092c2aad8e2..307f5cfa9ba4 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -141,9 +141,10 @@ static int crypto4xx_setkey_aes(struct crypto_skcipher *cipher, /* Setup SA */ sa = ctx->sa_in; - set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, (cm == CRYPTO_MODE_CBC ? - SA_SAVE_IV : SA_NOT_SAVE_IV), - SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, (cm == CRYPTO_MODE_ECB ? + SA_NOT_SAVE_IV : SA_SAVE_IV), + SA_NOT_LOAD_HASH, (cm == CRYPTO_MODE_ECB ? + SA_LOAD_IV_FROM_SA : SA_LOAD_IV_FROM_STATE), SA_NO_HEADER_PROC, SA_HASH_ALG_NULL, SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, SA_OPCODE_DECRYPT, @@ -162,6 +163,11 @@ static int crypto4xx_setkey_aes(struct crypto_skcipher *cipher, memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); sa = ctx->sa_out; sa->sa_command_0.bf.dir = DIR_OUTBOUND; + /* + * SA_OPCODE_ENCRYPT is the same value as SA_OPCODE_DECRYPT. + * it's the DIR_(IN|OUT)BOUND that matters + */ + sa->sa_command_0.bf.opcode = SA_OPCODE_ENCRYPT; return 0; } @@ -258,10 +264,10 @@ crypto4xx_ctr_crypt(struct skcipher_request *req, bool encrypt) * overlow. */ if (counter + nblks < counter) { - struct skcipher_request *subreq = skcipher_request_ctx(req); + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->sw_cipher.cipher); int ret; - skcipher_request_set_tfm(subreq, ctx->sw_cipher.cipher); + skcipher_request_set_sync_tfm(subreq, ctx->sw_cipher.cipher); skcipher_request_set_callback(subreq, req->base.flags, NULL, NULL); skcipher_request_set_crypt(subreq, req->src, req->dst, @@ -283,14 +289,14 @@ static int crypto4xx_sk_setup_fallback(struct crypto4xx_ctx *ctx, { int rc; - crypto_skcipher_clear_flags(ctx->sw_cipher.cipher, + crypto_sync_skcipher_clear_flags(ctx->sw_cipher.cipher, CRYPTO_TFM_REQ_MASK); - crypto_skcipher_set_flags(ctx->sw_cipher.cipher, + crypto_sync_skcipher_set_flags(ctx->sw_cipher.cipher, crypto_skcipher_get_flags(cipher) & CRYPTO_TFM_REQ_MASK); - rc = crypto_skcipher_setkey(ctx->sw_cipher.cipher, key, keylen); + rc = crypto_sync_skcipher_setkey(ctx->sw_cipher.cipher, key, keylen); crypto_skcipher_clear_flags(cipher, CRYPTO_TFM_RES_MASK); crypto_skcipher_set_flags(cipher, - crypto_skcipher_get_flags(ctx->sw_cipher.cipher) & + crypto_sync_skcipher_get_flags(ctx->sw_cipher.cipher) & CRYPTO_TFM_RES_MASK); return rc; diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 06574a884715..3934c2523762 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -539,7 +539,7 @@ static void crypto4xx_cipher_done(struct crypto4xx_device *dev, req = skcipher_request_cast(pd_uinfo->async_req); - if (pd_uinfo->using_sd) { + if (pd_uinfo->sa_va->sa_command_0.bf.scatter) { crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, req->cryptlen, req->dst); } else { @@ -593,7 +593,7 @@ static void crypto4xx_aead_done(struct crypto4xx_device *dev, u32 icv[AES_BLOCK_SIZE]; int err = 0; - if (pd_uinfo->using_sd) { + if (pd_uinfo->sa_va->sa_command_0.bf.scatter) { crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, pd->pd_ctl_len.bf.pkt_len, dst); @@ -714,7 +714,23 @@ int crypto4xx_build_pd(struct crypto_async_request *req, size_t offset_to_sr_ptr; u32 gd_idx = 0; int tmp; - bool is_busy; + bool is_busy, force_sd; + + /* + * There's a very subtile/disguised "bug" in the hardware that + * gets indirectly mentioned in 18.1.3.5 Encryption/Decryption + * of the hardware spec: + * *drum roll* the AES/(T)DES OFB and CFB modes are listed as + * operation modes for >>> "Block ciphers" <<<. + * + * To workaround this issue and stop the hardware from causing + * "overran dst buffer" on crypttexts that are not a multiple + * of 16 (AES_BLOCK_SIZE), we force the driver to use the + * scatter buffers. + */ + force_sd = (req_sa->sa_command_1.bf.crypto_mode9_8 == CRYPTO_MODE_CFB + || req_sa->sa_command_1.bf.crypto_mode9_8 == CRYPTO_MODE_OFB) + && (datalen % AES_BLOCK_SIZE); /* figure how many gd are needed */ tmp = sg_nents_for_len(src, assoclen + datalen); @@ -732,7 +748,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, } /* figure how many sd are needed */ - if (sg_is_last(dst)) { + if (sg_is_last(dst) && force_sd == false) { num_sd = 0; } else { if (datalen > PPC4XX_SD_BUFFER_SIZE) { @@ -807,9 +823,10 @@ int crypto4xx_build_pd(struct crypto_async_request *req, pd->sa_len = sa_len; pd_uinfo = &dev->pdr_uinfo[pd_entry]; - pd_uinfo->async_req = req; pd_uinfo->num_gd = num_gd; pd_uinfo->num_sd = num_sd; + pd_uinfo->dest_va = dst; + pd_uinfo->async_req = req; if (iv_len) memcpy(pd_uinfo->sr_va->save_iv, iv, iv_len); @@ -828,7 +845,6 @@ int crypto4xx_build_pd(struct crypto_async_request *req, /* get first gd we are going to use */ gd_idx = fst_gd; pd_uinfo->first_gd = fst_gd; - pd_uinfo->num_gd = num_gd; gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx); pd->src = gd_dma; /* enable gather */ @@ -865,17 +881,13 @@ int crypto4xx_build_pd(struct crypto_async_request *req, * Indicate gather array is not used */ pd_uinfo->first_gd = 0xffffffff; - pd_uinfo->num_gd = 0; } - if (sg_is_last(dst)) { + if (!num_sd) { /* * we know application give us dst a whole piece of memory * no need to use scatter ring. */ - pd_uinfo->using_sd = 0; pd_uinfo->first_sd = 0xffffffff; - pd_uinfo->num_sd = 0; - pd_uinfo->dest_va = dst; sa->sa_command_0.bf.scatter = 0; pd->dest = (u32)dma_map_page(dev->core_dev->device, sg_page(dst), dst->offset, @@ -888,10 +900,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, u32 sd_idx = fst_sd; nbytes = datalen; sa->sa_command_0.bf.scatter = 1; - pd_uinfo->using_sd = 1; - pd_uinfo->dest_va = dst; pd_uinfo->first_sd = fst_sd; - pd_uinfo->num_sd = num_sd; sd = crypto4xx_get_sdp(dev, &sd_dma, sd_idx); pd->dest = sd_dma; /* setup scatter descriptor */ @@ -954,15 +963,10 @@ static int crypto4xx_sk_init(struct crypto_skcipher *sk) if (alg->base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) { ctx->sw_cipher.cipher = - crypto_alloc_skcipher(alg->base.cra_name, 0, - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_ASYNC); + crypto_alloc_sync_skcipher(alg->base.cra_name, 0, + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->sw_cipher.cipher)) return PTR_ERR(ctx->sw_cipher.cipher); - - crypto_skcipher_set_reqsize(sk, - sizeof(struct skcipher_request) + 32 + - crypto_skcipher_reqsize(ctx->sw_cipher.cipher)); } amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.cipher); @@ -981,7 +985,7 @@ static void crypto4xx_sk_exit(struct crypto_skcipher *sk) crypto4xx_common_exit(ctx); if (ctx->sw_cipher.cipher) - crypto_free_skcipher(ctx->sw_cipher.cipher); + crypto_free_sync_skcipher(ctx->sw_cipher.cipher); } static int crypto4xx_aead_init(struct crypto_aead *tfm) diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 18df695ca6b1..c624f8cd3d2e 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -64,7 +64,6 @@ union shadow_sa_buf { struct pd_uinfo { struct crypto4xx_device *dev; u32 state; - u32 using_sd; u32 first_gd; /* first gather discriptor used by this packet */ u32 num_gd; /* number of gather discriptor @@ -131,7 +130,7 @@ struct crypto4xx_ctx { __le32 iv_nonce; u32 sa_len; union { - struct crypto_skcipher *cipher; + struct crypto_sync_skcipher *cipher; struct crypto_aead *aead; } sw_cipher; }; diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 65bf1a299562..fa76620281e8 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -800,20 +800,14 @@ static int atmel_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int keylen) { struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm); - const char *alg_name; - - alg_name = crypto_tfm_alg_name(crypto_ablkcipher_tfm(tfm)); + u32 flags; + int err; - /* - * HW bug in cfb 3-keys mode. - */ - if (!ctx->dd->caps.has_cfb_3keys && strstr(alg_name, "cfb") - && (keylen != 2*DES_KEY_SIZE)) { - crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } else if ((keylen != 2*DES_KEY_SIZE) && (keylen != 3*DES_KEY_SIZE)) { - crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; + flags = crypto_ablkcipher_get_flags(tfm); + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(tfm, flags); + return err; } memcpy(ctx->key, key, keylen); @@ -1060,7 +1054,7 @@ static struct crypto_alg tdes_algs[] = { .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, .cra_u.ablkcipher = { - .min_keysize = 2 * DES_KEY_SIZE, + .min_keysize = 3 * DES_KEY_SIZE, .max_keysize = 3 * DES_KEY_SIZE, .setkey = atmel_tdes_setkey, .encrypt = atmel_tdes_ecb_encrypt, @@ -1079,7 +1073,7 @@ static struct crypto_alg tdes_algs[] = { .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, .cra_u.ablkcipher = { - .min_keysize = 2*DES_KEY_SIZE, + .min_keysize = 3*DES_KEY_SIZE, .max_keysize = 3*DES_KEY_SIZE, .ivsize = DES_BLOCK_SIZE, .setkey = atmel_tdes_setkey, @@ -1088,86 +1082,6 @@ static struct crypto_alg tdes_algs[] = { } }, { - .cra_name = "cfb(des3_ede)", - .cra_driver_name = "atmel-cfb-tdes", - .cra_priority = 100, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0x7, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = atmel_tdes_cra_init, - .cra_u.ablkcipher = { - .min_keysize = 2*DES_KEY_SIZE, - .max_keysize = 2*DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = atmel_tdes_setkey, - .encrypt = atmel_tdes_cfb_encrypt, - .decrypt = atmel_tdes_cfb_decrypt, - } -}, -{ - .cra_name = "cfb8(des3_ede)", - .cra_driver_name = "atmel-cfb8-tdes", - .cra_priority = 100, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = CFB8_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = atmel_tdes_cra_init, - .cra_u.ablkcipher = { - .min_keysize = 2*DES_KEY_SIZE, - .max_keysize = 2*DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = atmel_tdes_setkey, - .encrypt = atmel_tdes_cfb8_encrypt, - .decrypt = atmel_tdes_cfb8_decrypt, - } -}, -{ - .cra_name = "cfb16(des3_ede)", - .cra_driver_name = "atmel-cfb16-tdes", - .cra_priority = 100, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = CFB16_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0x1, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = atmel_tdes_cra_init, - .cra_u.ablkcipher = { - .min_keysize = 2*DES_KEY_SIZE, - .max_keysize = 2*DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = atmel_tdes_setkey, - .encrypt = atmel_tdes_cfb16_encrypt, - .decrypt = atmel_tdes_cfb16_decrypt, - } -}, -{ - .cra_name = "cfb32(des3_ede)", - .cra_driver_name = "atmel-cfb32-tdes", - .cra_priority = 100, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = CFB32_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct atmel_tdes_ctx), - .cra_alignmask = 0x3, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = atmel_tdes_cra_init, - .cra_u.ablkcipher = { - .min_keysize = 2*DES_KEY_SIZE, - .max_keysize = 2*DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = atmel_tdes_setkey, - .encrypt = atmel_tdes_cfb32_encrypt, - .decrypt = atmel_tdes_cfb32_decrypt, - } -}, -{ .cra_name = "ofb(des3_ede)", .cra_driver_name = "atmel-ofb-tdes", .cra_priority = 100, @@ -1179,7 +1093,7 @@ static struct crypto_alg tdes_algs[] = { .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, .cra_u.ablkcipher = { - .min_keysize = 2*DES_KEY_SIZE, + .min_keysize = 3*DES_KEY_SIZE, .max_keysize = 3*DES_KEY_SIZE, .ivsize = DES_BLOCK_SIZE, .setkey = atmel_tdes_setkey, diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 57e5dca3253f..d2fb72811442 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -2247,8 +2247,6 @@ artpec6_crypto_hash_set_key(struct crypto_ahash *tfm, SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash); hdesc->tfm = tfm_ctx->child_hash; - hdesc->flags = crypto_ahash_get_flags(tfm) & - CRYPTO_TFM_REQ_MAY_SLEEP; tfm_ctx->hmac_key_length = blocksize; ret = crypto_shash_digest(hdesc, key, keylen, diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 28f592f7e1b7..25f8d3913ceb 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -1840,13 +1840,14 @@ static int threedes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, struct iproc_ctx_s *ctx = crypto_ablkcipher_ctx(cipher); if (keylen == (DES_KEY_SIZE * 3)) { - const u32 *K = (const u32 *)key; - u32 flags = CRYPTO_TFM_RES_BAD_KEY_SCHED; + u32 flags; + int ret; - if (!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5]))) { + flags = crypto_ablkcipher_get_flags(cipher); + ret = __des3_verify_key(&flags, key); + if (unlikely(ret)) { crypto_ablkcipher_set_flags(cipher, flags); - return -EINVAL; + return ret; } ctx->cipher_type = CIPHER_TYPE_3DES; @@ -2139,7 +2140,6 @@ static int ahash_init(struct ahash_request *req) goto err_hash; } ctx->shash->tfm = hash; - ctx->shash->flags = 0; /* Set the key using data we already have from setkey */ if (ctx->authkeylen > 0) { @@ -2885,13 +2885,13 @@ static int aead_authenc_setkey(struct crypto_aead *cipher, break; case CIPHER_ALG_3DES: if (ctx->enckeylen == (DES_KEY_SIZE * 3)) { - const u32 *K = (const u32 *)keys.enckey; - u32 flags = CRYPTO_TFM_RES_BAD_KEY_SCHED; + u32 flags; - if (!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5]))) { + flags = crypto_aead_get_flags(cipher); + ret = __des3_verify_key(&flags, keys.enckey); + if (unlikely(ret)) { crypto_aead_set_flags(cipher, flags); - return -EINVAL; + return ret; } ctx->cipher_type = CIPHER_TYPE_3DES; diff --git a/drivers/crypto/bcm/spu.c b/drivers/crypto/bcm/spu.c index dbb5c03dde49..2baf6d7f2c1d 100644 --- a/drivers/crypto/bcm/spu.c +++ b/drivers/crypto/bcm/spu.c @@ -22,9 +22,6 @@ #include "spum.h" #include "cipher.h" -/* This array is based on the hash algo type supported in spu.h */ -char *tag_to_hash_idx[] = { "none", "md5", "sha1", "sha224", "sha256" }; - char *hash_alg_name[] = { "None", "md5", "sha1", "sha224", "sha256", "aes", "sha384", "sha512", "sha3_224", "sha3_256", "sha3_384", "sha3_512" }; diff --git a/drivers/crypto/bcm/util.c b/drivers/crypto/bcm/util.c index d8cda5fb75ad..91ec56399d84 100644 --- a/drivers/crypto/bcm/util.c +++ b/drivers/crypto/bcm/util.c @@ -242,7 +242,6 @@ int do_shash(unsigned char *name, unsigned char *result, goto do_shash_err; } sdesc->shash.tfm = hash; - sdesc->shash.flags = 0x0; if (key_len > 0) { rc = crypto_shash_setkey(hash, key, key_len); diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 579578498deb..3e23d4b2cce2 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -638,6 +638,39 @@ badkey: return -EINVAL; } +static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_authenc_keys keys; + u32 flags; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + goto badkey; + + err = -EINVAL; + if (keys.enckeylen != DES3_EDE_KEY_SIZE) + goto badkey; + + flags = crypto_aead_get_flags(aead); + err = __des3_verify_key(&flags, keys.enckey); + if (unlikely(err)) { + crypto_aead_set_flags(aead, flags); + goto out; + } + + err = aead_setkey(aead, key, keylen); + +out: + memzero_explicit(&keys, sizeof(keys)); + return err; + +badkey: + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); + goto out; +} + static int gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { @@ -2457,7 +2490,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2479,7 +2512,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2502,7 +2535,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2525,7 +2558,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2548,7 +2581,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2571,7 +2604,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2594,7 +2627,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2617,7 +2650,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2640,7 +2673,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2663,7 +2696,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2686,7 +2719,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2709,7 +2742,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -3460,7 +3493,7 @@ static int __init caam_algapi_init(void) u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst; u32 arc4_inst; unsigned int md_limit = SHA512_DIGEST_SIZE; - bool registered = false; + bool registered = false, gcm_support; dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); if (!dev_node) { @@ -3493,7 +3526,7 @@ static int __init caam_algapi_init(void) * First, detect presence and attributes of DES, AES, and MD blocks. */ if (priv->era < 10) { - u32 cha_vid, cha_inst; + u32 cha_vid, cha_inst, aes_rn; cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); aes_vid = cha_vid & CHA_ID_LS_AES_MASK; @@ -3508,6 +3541,10 @@ static int __init caam_algapi_init(void) CHA_ID_LS_ARC4_SHIFT; ccha_inst = 0; ptha_inst = 0; + + aes_rn = rd_reg32(&priv->ctrl->perfmon.cha_rev_ls) & + CHA_ID_LS_AES_MASK; + gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8); } else { u32 aesa, mdha; @@ -3523,6 +3560,8 @@ static int __init caam_algapi_init(void) ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK; ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK; arc4_inst = rd_reg32(&priv->ctrl->vreg.afha) & CHA_VER_NUM_MASK; + + gcm_support = aesa & CHA_VER_MISC_AES_GCM; } /* If MD is present, limit digest size based on LP256 */ @@ -3595,11 +3634,9 @@ static int __init caam_algapi_init(void) if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst) continue; - /* - * Check support for AES algorithms not available - * on LP devices. - */ - if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM) + /* Skip GCM algorithms if not supported by device */ + if (c1_alg_sel == OP_ALG_ALGSEL_AES && + alg_aai == OP_ALG_AAI_GCM && !gcm_support) continue; /* diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index c61921d32489..70af211d2d01 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -292,6 +292,39 @@ badkey: return -EINVAL; } +static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_authenc_keys keys; + u32 flags; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + goto badkey; + + err = -EINVAL; + if (keys.enckeylen != DES3_EDE_KEY_SIZE) + goto badkey; + + flags = crypto_aead_get_flags(aead); + err = __des3_verify_key(&flags, keys.enckey); + if (unlikely(err)) { + crypto_aead_set_flags(aead, flags); + goto out; + } + + err = aead_setkey(aead, key, keylen); + +out: + memzero_explicit(&keys, sizeof(keys)); + return err; + +badkey: + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); + goto out; +} + static int gcm_set_sh_desc(struct crypto_aead *aead) { struct caam_ctx *ctx = crypto_aead_ctx(aead); @@ -667,6 +700,13 @@ badkey: return -EINVAL; } +static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + return unlikely(des3_verify_key(skcipher, key)) ?: + skcipher_setkey(skcipher, key, keylen); +} + static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, unsigned int keylen) { @@ -1382,7 +1422,7 @@ static struct caam_skcipher_alg driver_algs[] = { .cra_driver_name = "cbc-3des-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = skcipher_setkey, + .setkey = des3_skcipher_setkey, .encrypt = skcipher_encrypt, .decrypt = skcipher_decrypt, .min_keysize = DES3_EDE_KEY_SIZE, @@ -1798,7 +1838,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1820,7 +1860,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1843,7 +1883,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1866,7 +1906,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1889,7 +1929,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1912,7 +1952,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1935,7 +1975,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1958,7 +1998,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1981,7 +2021,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2004,7 +2044,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2027,7 +2067,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2050,7 +2090,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index c2c1abc68f81..33a4df6b81de 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -323,6 +323,39 @@ badkey: return -EINVAL; } +static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_authenc_keys keys; + u32 flags; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + goto badkey; + + err = -EINVAL; + if (keys.enckeylen != DES3_EDE_KEY_SIZE) + goto badkey; + + flags = crypto_aead_get_flags(aead); + err = __des3_verify_key(&flags, keys.enckey); + if (unlikely(err)) { + crypto_aead_set_flags(aead, flags); + goto out; + } + + err = aead_setkey(aead, key, keylen); + +out: + memzero_explicit(&keys, sizeof(keys)); + return err; + +badkey: + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); + goto out; +} + static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, bool encrypt) { @@ -938,6 +971,13 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, return 0; } +static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, unsigned int keylen) +{ + return unlikely(des3_verify_key(skcipher, key)) ?: + skcipher_setkey(skcipher, key, keylen); +} + static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, unsigned int keylen) { @@ -1484,7 +1524,7 @@ static struct caam_skcipher_alg driver_algs[] = { .cra_driver_name = "cbc-3des-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = skcipher_setkey, + .setkey = des3_skcipher_setkey, .encrypt = skcipher_encrypt, .decrypt = skcipher_decrypt, .min_keysize = DES3_EDE_KEY_SIZE, @@ -1916,7 +1956,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1938,7 +1978,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1961,7 +2001,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -1984,7 +2024,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2007,7 +2047,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2030,7 +2070,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2053,7 +2093,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2076,7 +2116,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2099,7 +2139,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2122,7 +2162,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2145,7 +2185,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2168,7 +2208,7 @@ static struct caam_aead_alg driver_aeads[] = { "cbc-des3_ede-caam-qi2", .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .setkey = aead_setkey, + .setkey = des3_aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, @@ -2854,6 +2894,7 @@ struct caam_hash_state { struct caam_request caam_req; dma_addr_t buf_dma; dma_addr_t ctx_dma; + int ctx_dma_len; u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; int buflen_0; u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; @@ -2927,6 +2968,7 @@ static inline int ctx_map_to_qm_sg(struct device *dev, struct caam_hash_state *state, int ctx_len, struct dpaa2_sg_entry *qm_sg, u32 flag) { + state->ctx_dma_len = ctx_len; state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); if (dma_mapping_error(dev, state->ctx_dma)) { dev_err(dev, "unable to map ctx\n"); @@ -3018,13 +3060,13 @@ static void split_key_sh_done(void *cbk_ctx, u32 err) } /* Digest hash size if it is too large */ -static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, - u32 *keylen, u8 *key_out, u32 digestsize) +static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, + u32 digestsize) { struct caam_request *req_ctx; u32 *desc; struct split_key_sh_result result; - dma_addr_t src_dma, dst_dma; + dma_addr_t key_dma; struct caam_flc *flc; dma_addr_t flc_dma; int ret = -ENOMEM; @@ -3041,17 +3083,10 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, if (!flc) goto err_flc; - src_dma = dma_map_single(ctx->dev, (void *)key_in, *keylen, - DMA_TO_DEVICE); - if (dma_mapping_error(ctx->dev, src_dma)) { - dev_err(ctx->dev, "unable to map key input memory\n"); - goto err_src_dma; - } - dst_dma = dma_map_single(ctx->dev, (void *)key_out, digestsize, - DMA_FROM_DEVICE); - if (dma_mapping_error(ctx->dev, dst_dma)) { - dev_err(ctx->dev, "unable to map key output memory\n"); - goto err_dst_dma; + key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); + if (dma_mapping_error(ctx->dev, key_dma)) { + dev_err(ctx->dev, "unable to map key memory\n"); + goto err_key_dma; } desc = flc->sh_desc; @@ -3076,14 +3111,14 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, dpaa2_fl_set_final(in_fle, true); dpaa2_fl_set_format(in_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(in_fle, src_dma); + dpaa2_fl_set_addr(in_fle, key_dma); dpaa2_fl_set_len(in_fle, *keylen); dpaa2_fl_set_format(out_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(out_fle, dst_dma); + dpaa2_fl_set_addr(out_fle, key_dma); dpaa2_fl_set_len(out_fle, digestsize); print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1); + DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); @@ -3103,17 +3138,15 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, wait_for_completion(&result.completion); ret = result.err; print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_in, + DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1); } dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), DMA_TO_DEVICE); err_flc_dma: - dma_unmap_single(ctx->dev, dst_dma, digestsize, DMA_FROM_DEVICE); -err_dst_dma: - dma_unmap_single(ctx->dev, src_dma, *keylen, DMA_TO_DEVICE); -err_src_dma: + dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); +err_key_dma: kfree(flc); err_flc: kfree(req_ctx); @@ -3135,12 +3168,10 @@ static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); if (keylen > blocksize) { - hashed_key = kmalloc_array(digestsize, sizeof(*hashed_key), - GFP_KERNEL | GFP_DMA); + hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); if (!hashed_key) return -ENOMEM; - ret = hash_digest_key(ctx, key, &keylen, hashed_key, - digestsize); + ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); if (ret) goto bad_free_key; key = hashed_key; @@ -3165,14 +3196,12 @@ bad_free_key: } static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, - struct ahash_request *req, int dst_len) + struct ahash_request *req) { struct caam_hash_state *state = ahash_request_ctx(req); if (edesc->src_nents) dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); - if (edesc->dst_dma) - dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE); if (edesc->qm_sg_bytes) dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, @@ -3187,18 +3216,15 @@ static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, static inline void ahash_unmap_ctx(struct device *dev, struct ahash_edesc *edesc, - struct ahash_request *req, int dst_len, - u32 flag) + struct ahash_request *req, u32 flag) { - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); if (state->ctx_dma) { - dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag); + dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); state->ctx_dma = 0; } - ahash_unmap(dev, edesc, req, dst_len); + ahash_unmap(dev, edesc, req); } static void ahash_done(void *cbk_ctx, u32 status) @@ -3219,16 +3245,13 @@ static void ahash_done(void *cbk_ctx, u32 status) ecode = -EIO; } - ahash_unmap(ctx->dev, edesc, req, digestsize); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); + memcpy(req->result, state->caam_ctx, digestsize); qi_cache_free(edesc); print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, ctx->ctx_len, 1); - if (req->result) - print_hex_dump_debug("result@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); req->base.complete(&req->base, ecode); } @@ -3250,7 +3273,7 @@ static void ahash_done_bi(void *cbk_ctx, u32 status) ecode = -EIO; } - ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); switch_buf(state); qi_cache_free(edesc); @@ -3283,16 +3306,13 @@ static void ahash_done_ctx_src(void *cbk_ctx, u32 status) ecode = -EIO; } - ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_TO_DEVICE); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); + memcpy(req->result, state->caam_ctx, digestsize); qi_cache_free(edesc); print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, ctx->ctx_len, 1); - if (req->result) - print_hex_dump_debug("result@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); req->base.complete(&req->base, ecode); } @@ -3314,7 +3334,7 @@ static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) ecode = -EIO; } - ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); switch_buf(state); qi_cache_free(edesc); @@ -3452,7 +3472,7 @@ static int ahash_update_ctx(struct ahash_request *req) return ret; unmap_ctx: - ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); qi_cache_free(edesc); return ret; } @@ -3484,7 +3504,7 @@ static int ahash_final_ctx(struct ahash_request *req) sg_table = &edesc->sgt[0]; ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, - DMA_TO_DEVICE); + DMA_BIDIRECTIONAL); if (ret) goto unmap_ctx; @@ -3503,22 +3523,13 @@ static int ahash_final_ctx(struct ahash_request *req) } edesc->qm_sg_bytes = qm_sg_bytes; - edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, - DMA_FROM_DEVICE); - if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { - dev_err(ctx->dev, "unable to map dst\n"); - edesc->dst_dma = 0; - ret = -ENOMEM; - goto unmap_ctx; - } - memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); dpaa2_fl_set_final(in_fle, true); dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); dpaa2_fl_set_format(out_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(out_fle, edesc->dst_dma); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); dpaa2_fl_set_len(out_fle, digestsize); req_ctx->flc = &ctx->flc[FINALIZE]; @@ -3533,7 +3544,7 @@ static int ahash_final_ctx(struct ahash_request *req) return ret; unmap_ctx: - ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); qi_cache_free(edesc); return ret; } @@ -3586,7 +3597,7 @@ static int ahash_finup_ctx(struct ahash_request *req) sg_table = &edesc->sgt[0]; ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, - DMA_TO_DEVICE); + DMA_BIDIRECTIONAL); if (ret) goto unmap_ctx; @@ -3605,22 +3616,13 @@ static int ahash_finup_ctx(struct ahash_request *req) } edesc->qm_sg_bytes = qm_sg_bytes; - edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, - DMA_FROM_DEVICE); - if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { - dev_err(ctx->dev, "unable to map dst\n"); - edesc->dst_dma = 0; - ret = -ENOMEM; - goto unmap_ctx; - } - memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); dpaa2_fl_set_final(in_fle, true); dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); dpaa2_fl_set_format(out_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(out_fle, edesc->dst_dma); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); dpaa2_fl_set_len(out_fle, digestsize); req_ctx->flc = &ctx->flc[FINALIZE]; @@ -3635,7 +3637,7 @@ static int ahash_finup_ctx(struct ahash_request *req) return ret; unmap_ctx: - ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); qi_cache_free(edesc); return ret; } @@ -3704,18 +3706,19 @@ static int ahash_digest(struct ahash_request *req) dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); } - edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, + state->ctx_dma_len = digestsize; + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, DMA_FROM_DEVICE); - if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { - dev_err(ctx->dev, "unable to map dst\n"); - edesc->dst_dma = 0; + if (dma_mapping_error(ctx->dev, state->ctx_dma)) { + dev_err(ctx->dev, "unable to map ctx\n"); + state->ctx_dma = 0; goto unmap; } dpaa2_fl_set_final(in_fle, true); dpaa2_fl_set_len(in_fle, req->nbytes); dpaa2_fl_set_format(out_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(out_fle, edesc->dst_dma); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); dpaa2_fl_set_len(out_fle, digestsize); req_ctx->flc = &ctx->flc[DIGEST]; @@ -3729,7 +3732,7 @@ static int ahash_digest(struct ahash_request *req) return ret; unmap: - ahash_unmap(ctx->dev, edesc, req, digestsize); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); qi_cache_free(edesc); return ret; } @@ -3755,27 +3758,39 @@ static int ahash_final_no_ctx(struct ahash_request *req) if (!edesc) return ret; - state->buf_dma = dma_map_single(ctx->dev, buf, buflen, DMA_TO_DEVICE); - if (dma_mapping_error(ctx->dev, state->buf_dma)) { - dev_err(ctx->dev, "unable to map src\n"); - goto unmap; + if (buflen) { + state->buf_dma = dma_map_single(ctx->dev, buf, buflen, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, state->buf_dma)) { + dev_err(ctx->dev, "unable to map src\n"); + goto unmap; + } } - edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, + state->ctx_dma_len = digestsize; + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, DMA_FROM_DEVICE); - if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { - dev_err(ctx->dev, "unable to map dst\n"); - edesc->dst_dma = 0; + if (dma_mapping_error(ctx->dev, state->ctx_dma)) { + dev_err(ctx->dev, "unable to map ctx\n"); + state->ctx_dma = 0; goto unmap; } memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); dpaa2_fl_set_final(in_fle, true); - dpaa2_fl_set_format(in_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(in_fle, state->buf_dma); - dpaa2_fl_set_len(in_fle, buflen); + /* + * crypto engine requires the input entry to be present when + * "frame list" FD is used. + * Since engine does not support FMT=2'b11 (unused entry type), leaving + * in_fle zeroized (except for "Final" flag) is the best option. + */ + if (buflen) { + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, state->buf_dma); + dpaa2_fl_set_len(in_fle, buflen); + } dpaa2_fl_set_format(out_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(out_fle, edesc->dst_dma); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); dpaa2_fl_set_len(out_fle, digestsize); req_ctx->flc = &ctx->flc[DIGEST]; @@ -3790,7 +3805,7 @@ static int ahash_final_no_ctx(struct ahash_request *req) return ret; unmap: - ahash_unmap(ctx->dev, edesc, req, digestsize); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); qi_cache_free(edesc); return ret; } @@ -3870,6 +3885,7 @@ static int ahash_update_no_ctx(struct ahash_request *req) } edesc->qm_sg_bytes = qm_sg_bytes; + state->ctx_dma_len = ctx->ctx_len; state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, ctx->ctx_len, DMA_FROM_DEVICE); if (dma_mapping_error(ctx->dev, state->ctx_dma)) { @@ -3918,7 +3934,7 @@ static int ahash_update_no_ctx(struct ahash_request *req) return ret; unmap_ctx: - ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); qi_cache_free(edesc); return ret; } @@ -3983,11 +3999,12 @@ static int ahash_finup_no_ctx(struct ahash_request *req) } edesc->qm_sg_bytes = qm_sg_bytes; - edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, + state->ctx_dma_len = digestsize; + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, DMA_FROM_DEVICE); - if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { - dev_err(ctx->dev, "unable to map dst\n"); - edesc->dst_dma = 0; + if (dma_mapping_error(ctx->dev, state->ctx_dma)) { + dev_err(ctx->dev, "unable to map ctx\n"); + state->ctx_dma = 0; ret = -ENOMEM; goto unmap; } @@ -3998,7 +4015,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req) dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); dpaa2_fl_set_len(in_fle, buflen + req->nbytes); dpaa2_fl_set_format(out_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(out_fle, edesc->dst_dma); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); dpaa2_fl_set_len(out_fle, digestsize); req_ctx->flc = &ctx->flc[DIGEST]; @@ -4013,7 +4030,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req) return ret; unmap: - ahash_unmap(ctx->dev, edesc, req, digestsize); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); qi_cache_free(edesc); return -ENOMEM; } @@ -4100,6 +4117,7 @@ static int ahash_update_first(struct ahash_request *req) scatterwalk_map_and_copy(next_buf, req->src, to_hash, *next_buflen, 0); + state->ctx_dma_len = ctx->ctx_len; state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, ctx->ctx_len, DMA_FROM_DEVICE); if (dma_mapping_error(ctx->dev, state->ctx_dma)) { @@ -4143,7 +4161,7 @@ static int ahash_update_first(struct ahash_request *req) return ret; unmap_ctx: - ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); + ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); qi_cache_free(edesc); return ret; } @@ -4162,6 +4180,7 @@ static int ahash_init(struct ahash_request *req) state->final = ahash_final_no_ctx; state->ctx_dma = 0; + state->ctx_dma_len = 0; state->current_buf = 0; state->buf_dma = 0; state->buflen_0 = 0; diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h index 20890780fb82..be5085451053 100644 --- a/drivers/crypto/caam/caamalg_qi2.h +++ b/drivers/crypto/caam/caamalg_qi2.h @@ -162,14 +162,12 @@ struct skcipher_edesc { /* * ahash_edesc - s/w-extended ahash descriptor - * @dst_dma: I/O virtual address of req->result * @qm_sg_dma: I/O virtual address of h/w link table * @src_nents: number of segments in input scatterlist * @qm_sg_bytes: length of dma mapped qm_sg space * @sgt: pointer to h/w link table */ struct ahash_edesc { - dma_addr_t dst_dma; dma_addr_t qm_sg_dma; int src_nents; int qm_sg_bytes; diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index 58285642306e..fe24485274e1 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -994,8 +994,6 @@ static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm) static struct akcipher_alg caam_rsa = { .encrypt = caam_rsa_enc, .decrypt = caam_rsa_dec, - .sign = caam_rsa_dec, - .verify = caam_rsa_enc, .set_pub_key = caam_rsa_set_pub_key, .set_priv_key = caam_rsa_set_priv_key, .max_size = caam_rsa_max_size, diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 858bdc9ab4a3..e2ba3d202da5 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -468,6 +468,24 @@ static int caam_get_era(struct caam_ctrl __iomem *ctrl) return caam_get_era_from_hw(ctrl); } +/* + * ERRATA: imx6 devices (imx6D, imx6Q, imx6DL, imx6S, imx6DP and imx6DQ) + * have an issue wherein AXI bus transactions may not occur in the correct + * order. This isn't a problem running single descriptors, but can be if + * running multiple concurrent descriptors. Reworking the driver to throttle + * to single requests is impractical, thus the workaround is to limit the AXI + * pipeline to a depth of 1 (from it's default of 4) to preclude this situation + * from occurring. + */ +static void handle_imx6_err005766(u32 *mcr) +{ + if (of_machine_is_compatible("fsl,imx6q") || + of_machine_is_compatible("fsl,imx6dl") || + of_machine_is_compatible("fsl,imx6qp")) + clrsetbits_32(mcr, MCFGR_AXIPIPE_MASK, + 1 << MCFGR_AXIPIPE_SHIFT); +} + static const struct of_device_id caam_match[] = { { .compatible = "fsl,sec-v4.0", @@ -640,6 +658,8 @@ static int caam_probe(struct platform_device *pdev) (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); + handle_imx6_err005766(&ctrl->mcr); + /* * Read the Compile Time paramters and SCFGR to determine * if Virtualization is enabled for this platform diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c index 21a70fd32f5d..a4129a35a330 100644 --- a/drivers/crypto/caam/error.c +++ b/drivers/crypto/caam/error.c @@ -138,7 +138,7 @@ static const struct { { 0x46, "Annotation length exceeds offset (reuse mode)"}, { 0x48, "Annotation output enabled but ASA limited by ASAR (reuse mode)"}, { 0x49, "Data offset correction exceeds input frame data length (reuse mode)"}, - { 0x4B, "Annotation output enabled but ASA cannote be expanded (frame list)"}, + { 0x4B, "Annotation output enabled but ASA cannot be expanded (frame list)"}, { 0x51, "Unsupported IF reuse mode"}, { 0x52, "Unsupported FL use mode"}, { 0x53, "Unsupported RJD use mode"}, diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 5869ad58d497..3392615dc91b 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -49,13 +49,11 @@ struct caam_drv_private_jr { atomic_t tfm_count ____cacheline_aligned; /* Job ring info */ - int ringsize; /* Size of rings (assume input = output) */ struct caam_jrentry_info *entinfo; /* Alloc'ed 1 per ring entry */ spinlock_t inplock ____cacheline_aligned; /* Input ring index lock */ - int inp_ring_write_index; /* Input index "tail" */ + u32 inpring_avail; /* Number of free entries in input ring */ int head; /* entinfo (s/w ring) head index */ dma_addr_t *inpring; /* Base of input ring, alloc DMA-safe */ - spinlock_t outlock ____cacheline_aligned; /* Output ring index lock */ int out_ring_read_index; /* Output index "tail" */ int tail; /* entinfo (s/w ring) tail index */ struct jr_outentry *outring; /* Base of output ring, DMA-safe */ diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index d50085a03597..044a69b526f7 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -170,13 +170,13 @@ static void caam_jr_dequeue(unsigned long devarg) void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg); u32 *userdesc, userstatus; void *userarg; + u32 outring_used = 0; - while (rd_reg32(&jrp->rregs->outring_used)) { + while (outring_used || + (outring_used = rd_reg32(&jrp->rregs->outring_used))) { head = READ_ONCE(jrp->head); - spin_lock(&jrp->outlock); - sw_idx = tail = jrp->tail; hw_idx = jrp->out_ring_read_index; @@ -199,7 +199,7 @@ static void caam_jr_dequeue(unsigned long devarg) /* mark completed, avoid matching on a recycled desc addr */ jrp->entinfo[sw_idx].desc_addr_dma = 0; - /* Stash callback params for use outside of lock */ + /* Stash callback params */ usercall = jrp->entinfo[sw_idx].callbk; userarg = jrp->entinfo[sw_idx].cbkarg; userdesc = jrp->entinfo[sw_idx].desc_addr_virt; @@ -213,7 +213,7 @@ static void caam_jr_dequeue(unsigned long devarg) mb(); /* set done */ - wr_reg32(&jrp->rregs->outring_rmvd, 1); + wr_reg32_relaxed(&jrp->rregs->outring_rmvd, 1); jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) & (JOBR_DEPTH - 1); @@ -232,10 +232,9 @@ static void caam_jr_dequeue(unsigned long devarg) jrp->tail = tail; } - spin_unlock(&jrp->outlock); - /* Finally, execute user's callback */ usercall(dev, userdesc, userstatus, userarg); + outring_used--; } /* reenable / unmask IRQs */ @@ -345,7 +344,7 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, head = jrp->head; tail = READ_ONCE(jrp->tail); - if (!rd_reg32(&jrp->rregs->inpring_avail) || + if (!jrp->inpring_avail || CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) { spin_unlock_bh(&jrp->inplock); dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE); @@ -359,7 +358,7 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, head_entry->cbkarg = areq; head_entry->desc_addr_dma = desc_dma; - jrp->inpring[jrp->inp_ring_write_index] = cpu_to_caam_dma(desc_dma); + jrp->inpring[head] = cpu_to_caam_dma(desc_dma); /* * Guarantee that the descriptor's DMA address has been written to @@ -368,18 +367,22 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, */ smp_wmb(); - jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) & - (JOBR_DEPTH - 1); jrp->head = (head + 1) & (JOBR_DEPTH - 1); /* * Ensure that all job information has been written before - * notifying CAAM that a new job was added to the input ring. + * notifying CAAM that a new job was added to the input ring + * using a memory barrier. The wr_reg32() uses api iowrite32() + * to do the register write. iowrite32() issues a memory barrier + * before the write operation. */ - wmb(); wr_reg32(&jrp->rregs->inpring_jobadd, 1); + jrp->inpring_avail--; + if (!jrp->inpring_avail) + jrp->inpring_avail = rd_reg32(&jrp->rregs->inpring_avail); + spin_unlock_bh(&jrp->inplock); return 0; @@ -431,7 +434,6 @@ static int caam_jr_init(struct device *dev) jrp->entinfo[i].desc_addr_dma = !0; /* Setup rings */ - jrp->inp_ring_write_index = 0; jrp->out_ring_read_index = 0; jrp->head = 0; jrp->tail = 0; @@ -441,10 +443,9 @@ static int caam_jr_init(struct device *dev) wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH); wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH); - jrp->ringsize = JOBR_DEPTH; + jrp->inpring_avail = JOBR_DEPTH; spin_lock_init(&jrp->inplock); - spin_lock_init(&jrp->outlock); /* Select interrupt coalescing parameters */ clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC | diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 7cb8b1755e57..9f08f84cca59 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -318,7 +318,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) /* Create a new req FQ in parked state */ new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq, drv_ctx->context_a, 0); - if (IS_ERR_OR_NULL(new_fq)) { + if (IS_ERR(new_fq)) { dev_err(qidev, "FQ allocation for shdesc update failed\n"); return PTR_ERR(new_fq); } @@ -431,7 +431,7 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, /* Attach request FQ */ drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc, QMAN_INITFQ_FLAG_SCHED); - if (IS_ERR_OR_NULL(drv_ctx->req_fq)) { + if (IS_ERR(drv_ctx->req_fq)) { dev_err(qidev, "create_caam_req_fq failed\n"); dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL); kfree(drv_ctx); diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index 3cd0822ea819..c1fa1ec701d9 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -96,6 +96,14 @@ cpu_to_caam(16) cpu_to_caam(32) cpu_to_caam(64) +static inline void wr_reg32_relaxed(void __iomem *reg, u32 data) +{ + if (caam_little_end) + writel_relaxed(data, reg); + else + writel_relaxed(cpu_to_be32(data), reg); +} + static inline void wr_reg32(void __iomem *reg, u32 data) { if (caam_little_end) @@ -253,6 +261,9 @@ struct version_regs { #define CHA_VER_VID_SHIFT 24 #define CHA_VER_VID_MASK (0xffull << CHA_VER_VID_SHIFT) +/* CHA Miscellaneous Information - AESA_MISC specific */ +#define CHA_VER_MISC_AES_GCM BIT(1 + CHA_VER_MISC_SHIFT) + /* * caam_perfmon - Performance Monitor/Secure Memory Status/ * CAAM Global Status/Component Version IDs diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c index 600336d169a9..9810ad8ac519 100644 --- a/drivers/crypto/cavium/cpt/cptvf_algs.c +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -10,7 +10,6 @@ #include <crypto/aes.h> #include <crypto/algapi.h> #include <crypto/authenc.h> -#include <crypto/cryptd.h> #include <crypto/crypto_wq.h> #include <crypto/des.h> #include <crypto/xts.h> @@ -327,27 +326,36 @@ static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key, u32 keylen) { + u32 flags = crypto_ablkcipher_get_flags(cipher); + int err; + + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(cipher, flags); + return err; + } + return cvm_setkey(cipher, key, keylen, DES3_CBC); } static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key, u32 keylen) { + u32 flags = crypto_ablkcipher_get_flags(cipher); + int err; + + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(cipher, flags); + return err; + } + return cvm_setkey(cipher, key, keylen, DES3_ECB); } static int cvm_enc_dec_init(struct crypto_tfm *tfm) { - struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm); - - memset(ctx, 0, sizeof(*ctx)); - tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx) + - sizeof(struct ablkcipher_request); - /* Additional memory for ablkcipher_request is - * allocated since the cryptd daemon uses - * this memory for request_ctx information - */ - + tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx); return 0; } diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index 2ca431ed1db8..88a0166f5477 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -641,7 +641,7 @@ static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val) cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u); } -void cptvf_device_init(struct cpt_vf *cptvf) +static void cptvf_device_init(struct cpt_vf *cptvf) { u64 base_addr = 0; diff --git a/drivers/crypto/cavium/cpt/cptvf_mbox.c b/drivers/crypto/cavium/cpt/cptvf_mbox.c index d5ec3b8a9e61..4f438eceb506 100644 --- a/drivers/crypto/cavium/cpt/cptvf_mbox.c +++ b/drivers/crypto/cavium/cpt/cptvf_mbox.c @@ -17,23 +17,6 @@ static void cptvf_send_msg_to_pf(struct cpt_vf *cptvf, struct cpt_mbox *mbx) mbx->data); } -/* ACKs PF's mailbox message - */ -void cptvf_mbox_send_ack(struct cpt_vf *cptvf, struct cpt_mbox *mbx) -{ - mbx->msg = CPT_MBOX_MSG_TYPE_ACK; - cptvf_send_msg_to_pf(cptvf, mbx); -} - -/* NACKs PF's mailbox message that VF is not able to - * complete the action - */ -void cptvf_mbox_send_nack(struct cpt_vf *cptvf, struct cpt_mbox *mbx) -{ - mbx->msg = CPT_MBOX_MSG_TYPE_NACK; - cptvf_send_msg_to_pf(cptvf, mbx); -} - /* Interrupt handler to handle mailbox messages from VFs */ void cptvf_handle_mbox_intr(struct cpt_vf *cptvf) { diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c index ca549c5dc08e..f16f61504241 100644 --- a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c @@ -223,7 +223,7 @@ scatter_gather_clean: return ret; } -int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd, +static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd, u32 qno) { struct pci_dev *pdev = cptvf->pdev; @@ -270,7 +270,7 @@ int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd, return ret; } -void do_request_cleanup(struct cpt_vf *cptvf, +static void do_request_cleanup(struct cpt_vf *cptvf, struct cpt_info_buffer *info) { int i; @@ -316,7 +316,7 @@ void do_request_cleanup(struct cpt_vf *cptvf, kzfree(info); } -void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info) +static void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info) { struct pci_dev *pdev = cptvf->pdev; diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c index 4f43eacd2557..e4841eb2a09f 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_aead.c +++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c @@ -18,26 +18,6 @@ #define GCM_AES_SALT_SIZE 4 -/** - * struct nitrox_crypt_params - Params to set nitrox crypto request. - * @cryptlen: Encryption/Decryption data length - * @authlen: Assoc data length + Cryptlen - * @srclen: Input buffer length - * @dstlen: Output buffer length - * @iv: IV data - * @ivsize: IV data length - * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT) - */ -struct nitrox_crypt_params { - unsigned int cryptlen; - unsigned int authlen; - unsigned int srclen; - unsigned int dstlen; - u8 *iv; - int ivsize; - u8 ctrl_arg; -}; - union gph_p3 { struct { #ifdef __BIG_ENDIAN_BITFIELD @@ -94,36 +74,40 @@ static int nitrox_aead_setauthsize(struct crypto_aead *aead, return 0; } -static int alloc_src_sglist(struct aead_request *areq, char *iv, int ivsize, +static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq, + struct scatterlist *src, char *iv, int ivsize, int buflen) { - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - int nents = sg_nents_for_len(areq->src, buflen) + 1; + int nents = sg_nents_for_len(src, buflen); int ret; if (nents < 0) return nents; + /* IV entry */ + nents += 1; /* Allocate buffer to hold IV and input scatterlist array */ ret = alloc_src_req_buf(nkreq, nents, ivsize); if (ret) return ret; nitrox_creq_copy_iv(nkreq->src, iv, ivsize); - nitrox_creq_set_src_sg(nkreq, nents, ivsize, areq->src, buflen); + nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen); return 0; } -static int alloc_dst_sglist(struct aead_request *areq, int ivsize, int buflen) +static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq, + struct scatterlist *dst, int ivsize, int buflen) { - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - int nents = sg_nents_for_len(areq->dst, buflen) + 3; + int nents = sg_nents_for_len(dst, buflen); int ret; if (nents < 0) return nents; + /* IV, ORH, COMPLETION entries */ + nents += 3; /* Allocate buffer to hold ORH, COMPLETION and output scatterlist * array */ @@ -133,61 +117,54 @@ static int alloc_dst_sglist(struct aead_request *areq, int ivsize, int buflen) nitrox_creq_set_orh(nkreq); nitrox_creq_set_comp(nkreq); - nitrox_creq_set_dst_sg(nkreq, nents, ivsize, areq->dst, buflen); + nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen); return 0; } -static void free_src_sglist(struct aead_request *areq) +static void free_src_sglist(struct nitrox_kcrypt_request *nkreq) { - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - kfree(nkreq->src); } -static void free_dst_sglist(struct aead_request *areq) +static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq) { - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - kfree(nkreq->dst); } -static int nitrox_set_creq(struct aead_request *areq, - struct nitrox_crypt_params *params) +static int nitrox_set_creq(struct nitrox_aead_rctx *rctx) { - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - struct se_crypto_request *creq = &nkreq->creq; - struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct se_crypto_request *creq = &rctx->nkreq.creq; union gph_p3 param3; - struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); int ret; - creq->flags = areq->base.flags; - creq->gfp = (areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC; + creq->flags = rctx->flags; + creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : + GFP_ATOMIC; creq->ctrl.value = 0; creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; - creq->ctrl.s.arg = params->ctrl_arg; + creq->ctrl.s.arg = rctx->ctrl_arg; - creq->gph.param0 = cpu_to_be16(params->cryptlen); - creq->gph.param1 = cpu_to_be16(params->authlen); - creq->gph.param2 = cpu_to_be16(params->ivsize + areq->assoclen); + creq->gph.param0 = cpu_to_be16(rctx->cryptlen); + creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen); + creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen); param3.iv_offset = 0; - param3.auth_offset = params->ivsize; + param3.auth_offset = rctx->ivsize; creq->gph.param3 = cpu_to_be16(param3.param); - creq->ctx_handle = nctx->u.ctx_handle; + creq->ctx_handle = rctx->ctx_handle; creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); - ret = alloc_src_sglist(areq, params->iv, params->ivsize, - params->srclen); + ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize, + rctx->srclen); if (ret) return ret; - ret = alloc_dst_sglist(areq, params->ivsize, params->dstlen); + ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize, + rctx->dstlen); if (ret) { - free_src_sglist(areq); + free_src_sglist(&rctx->nkreq); return ret; } @@ -197,9 +174,10 @@ static int nitrox_set_creq(struct aead_request *areq, static void nitrox_aead_callback(void *arg, int err) { struct aead_request *areq = arg; + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); - free_src_sglist(areq); - free_dst_sglist(areq); + free_src_sglist(&rctx->nkreq); + free_dst_sglist(&rctx->nkreq); if (err) { pr_err_ratelimited("request failed status 0x%0x\n", err); err = -EINVAL; @@ -212,23 +190,25 @@ static int nitrox_aes_gcm_enc(struct aead_request *areq) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - struct se_crypto_request *creq = &nkreq->creq; + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); + struct se_crypto_request *creq = &rctx->nkreq.creq; struct flexi_crypto_context *fctx = nctx->u.fctx; - struct nitrox_crypt_params params; int ret; memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); - memset(¶ms, 0, sizeof(params)); - params.cryptlen = areq->cryptlen; - params.authlen = areq->assoclen + params.cryptlen; - params.srclen = params.authlen; - params.dstlen = params.srclen + aead->authsize; - params.iv = &areq->iv[GCM_AES_SALT_SIZE]; - params.ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; - params.ctrl_arg = ENCRYPT; - ret = nitrox_set_creq(areq, ¶ms); + rctx->cryptlen = areq->cryptlen; + rctx->assoclen = areq->assoclen; + rctx->srclen = areq->assoclen + areq->cryptlen; + rctx->dstlen = rctx->srclen + aead->authsize; + rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; + rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; + rctx->flags = areq->base.flags; + rctx->ctx_handle = nctx->u.ctx_handle; + rctx->src = areq->src; + rctx->dst = areq->dst; + rctx->ctrl_arg = ENCRYPT; + ret = nitrox_set_creq(rctx); if (ret) return ret; @@ -241,23 +221,25 @@ static int nitrox_aes_gcm_dec(struct aead_request *areq) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - struct se_crypto_request *creq = &nkreq->creq; + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); + struct se_crypto_request *creq = &rctx->nkreq.creq; struct flexi_crypto_context *fctx = nctx->u.fctx; - struct nitrox_crypt_params params; int ret; memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); - memset(¶ms, 0, sizeof(params)); - params.cryptlen = areq->cryptlen - aead->authsize; - params.authlen = areq->assoclen + params.cryptlen; - params.srclen = areq->cryptlen + areq->assoclen; - params.dstlen = params.srclen - aead->authsize; - params.iv = &areq->iv[GCM_AES_SALT_SIZE]; - params.ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; - params.ctrl_arg = DECRYPT; - ret = nitrox_set_creq(areq, ¶ms); + rctx->cryptlen = areq->cryptlen - aead->authsize; + rctx->assoclen = areq->assoclen; + rctx->srclen = areq->cryptlen + areq->assoclen; + rctx->dstlen = rctx->srclen - aead->authsize; + rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; + rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; + rctx->flags = areq->base.flags; + rctx->ctx_handle = nctx->u.ctx_handle; + rctx->src = areq->src; + rctx->dst = areq->dst; + rctx->ctrl_arg = DECRYPT; + ret = nitrox_set_creq(rctx); if (ret) return ret; @@ -290,7 +272,7 @@ static int nitrox_aead_init(struct crypto_aead *aead) return 0; } -static int nitrox_aes_gcm_init(struct crypto_aead *aead) +static int nitrox_gcm_common_init(struct crypto_aead *aead) { int ret; struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); @@ -308,8 +290,20 @@ static int nitrox_aes_gcm_init(struct crypto_aead *aead) flags->w0.auth_input_type = 1; flags->f = be64_to_cpu(flags->f); - crypto_aead_set_reqsize(aead, sizeof(struct aead_request) + - sizeof(struct nitrox_kcrypt_request)); + return 0; +} + +static int nitrox_aes_gcm_init(struct crypto_aead *aead) +{ + int ret; + + ret = nitrox_gcm_common_init(aead); + if (ret) + return ret; + + crypto_aead_set_reqsize(aead, + sizeof(struct aead_request) + + sizeof(struct nitrox_aead_rctx)); return 0; } @@ -332,6 +326,166 @@ static void nitrox_aead_exit(struct crypto_aead *aead) nctx->ndev = NULL; } +static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct flexi_crypto_context *fctx = nctx->u.fctx; + int ret; + + if (keylen < GCM_AES_SALT_SIZE) + return -EINVAL; + + keylen -= GCM_AES_SALT_SIZE; + ret = nitrox_aes_gcm_setkey(aead, key, keylen); + if (ret) + return ret; + + memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE); + return 0; +} + +static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return nitrox_aead_setauthsize(aead, authsize); +} + +static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq) +{ + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + struct scatterlist *sg; + + if (areq->assoclen != 16 && areq->assoclen != 20) + return -EINVAL; + + scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0); + sg_init_table(rctx->src, 3); + sg_set_buf(rctx->src, rctx->assoc, assoclen); + sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen); + if (sg != rctx->src + 1) + sg_chain(rctx->src, 2, sg); + + if (areq->src != areq->dst) { + sg_init_table(rctx->dst, 3); + sg_set_buf(rctx->dst, rctx->assoc, assoclen); + sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen); + if (sg != rctx->dst + 1) + sg_chain(rctx->dst, 2, sg); + } + + aead_rctx->src = rctx->src; + aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst; + + return 0; +} + +static void nitrox_rfc4106_callback(void *arg, int err) +{ + struct aead_request *areq = arg; + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq; + + free_src_sglist(nkreq); + free_dst_sglist(nkreq); + if (err) { + pr_err_ratelimited("request failed status 0x%0x\n", err); + err = -EINVAL; + } + + areq->base.complete(&areq->base, err); +} + +static int nitrox_rfc4106_enc(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + struct se_crypto_request *creq = &aead_rctx->nkreq.creq; + int ret; + + aead_rctx->cryptlen = areq->cryptlen; + aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen; + aead_rctx->dstlen = aead_rctx->srclen + aead->authsize; + aead_rctx->iv = areq->iv; + aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; + aead_rctx->flags = areq->base.flags; + aead_rctx->ctx_handle = nctx->u.ctx_handle; + aead_rctx->ctrl_arg = ENCRYPT; + + ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); + if (ret) + return ret; + + ret = nitrox_set_creq(aead_rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, + nitrox_rfc4106_callback, areq); +} + +static int nitrox_rfc4106_dec(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + struct se_crypto_request *creq = &aead_rctx->nkreq.creq; + int ret; + + aead_rctx->cryptlen = areq->cryptlen - aead->authsize; + aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + aead_rctx->srclen = + areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen; + aead_rctx->dstlen = aead_rctx->srclen - aead->authsize; + aead_rctx->iv = areq->iv; + aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; + aead_rctx->flags = areq->base.flags; + aead_rctx->ctx_handle = nctx->u.ctx_handle; + aead_rctx->ctrl_arg = DECRYPT; + + ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); + if (ret) + return ret; + + ret = nitrox_set_creq(aead_rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, + nitrox_rfc4106_callback, areq); +} + +static int nitrox_rfc4106_init(struct crypto_aead *aead) +{ + int ret; + + ret = nitrox_gcm_common_init(aead); + if (ret) + return ret; + + crypto_aead_set_reqsize(aead, sizeof(struct aead_request) + + sizeof(struct nitrox_rfc4106_rctx)); + + return 0; +} + static struct aead_alg nitrox_aeads[] = { { .base = { .cra_name = "gcm(aes)", @@ -351,6 +505,25 @@ static struct aead_alg nitrox_aeads[] = { { .exit = nitrox_aead_exit, .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, +}, { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "n5_rfc4106", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .setkey = nitrox_rfc4106_setkey, + .setauthsize = nitrox_rfc4106_setauthsize, + .encrypt = nitrox_rfc4106_enc, + .decrypt = nitrox_rfc4106_dec, + .init = nitrox_rfc4106_init, + .exit = nitrox_aead_exit, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, } }; int nitrox_register_aeads(void) diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c index c08d9f33a3b1..3f0df60267a9 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_hal.c +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -437,6 +437,45 @@ void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode) nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, vfcfg.value); } +static const char *get_core_option(u8 se_cores, u8 ae_cores) +{ + const char *option = ""; + + if (ae_cores == AE_MAX_CORES) { + switch (se_cores) { + case SE_MAX_CORES: + option = "60"; + break; + case 40: + option = "60s"; + break; + } + } else if (ae_cores == (AE_MAX_CORES / 2)) { + option = "30"; + } else { + option = "60i"; + } + + return option; +} + +static const char *get_feature_option(u8 zip_cores, int core_freq) +{ + if (zip_cores == 0) + return ""; + else if (zip_cores < ZIP_MAX_CORES) + return "-C15"; + + if (core_freq >= 850) + return "-C45"; + else if (core_freq >= 750) + return "-C35"; + else if (core_freq >= 550) + return "-C25"; + + return ""; +} + void nitrox_get_hwinfo(struct nitrox_device *ndev) { union emu_fuse_map emu_fuse; @@ -469,24 +508,14 @@ void nitrox_get_hwinfo(struct nitrox_device *ndev) ndev->hw.zip_cores = ZIP_MAX_CORES - dead_cores; } - /* determine the partname CNN55<cores>-<freq><pincount>-<rev>*/ - if (ndev->hw.ae_cores == AE_MAX_CORES) { - switch (ndev->hw.se_cores) { - case SE_MAX_CORES: - i = snprintf(name, sizeof(name), "CNN5560"); - break; - case 40: - i = snprintf(name, sizeof(name), "CNN5560s"); - break; - } - } else if (ndev->hw.ae_cores == (AE_MAX_CORES / 2)) { - i = snprintf(name, sizeof(name), "CNN5530"); - } else { - i = snprintf(name, sizeof(name), "CNN5560i"); - } - - snprintf(name + i, sizeof(name) - i, "-%3dBG676-1.%u", - ndev->hw.freq, ndev->hw.revision_id); + /* determine the partname + * CNN55<core option>-<freq><pincount>-<feature option>-<rev> + */ + snprintf(name, sizeof(name), "CNN55%s-%3dBG676%s-1.%u", + get_core_option(ndev->hw.se_cores, ndev->hw.ae_cores), + ndev->hw.freq, + get_feature_option(ndev->hw.zip_cores, ndev->hw.freq), + ndev->hw.revision_id); /* copy partname */ strncpy(ndev->hw.partname, name, sizeof(ndev->hw.partname)); diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h index 76c0f0be7233..efdbd0fc3e3b 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_req.h +++ b/drivers/crypto/cavium/nitrox/nitrox_req.h @@ -212,6 +212,50 @@ struct nitrox_kcrypt_request { }; /** + * struct nitrox_aead_rctx - AEAD request context + * @nkreq: Base request context + * @cryptlen: Encryption/Decryption data length + * @assoclen: AAD length + * @srclen: Input buffer length + * @dstlen: Output buffer length + * @iv: IV data + * @ivsize: IV data length + * @flags: AEAD req flags + * @ctx_handle: Device context handle + * @src: Source sglist + * @dst: Destination sglist + * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT) + */ +struct nitrox_aead_rctx { + struct nitrox_kcrypt_request nkreq; + unsigned int cryptlen; + unsigned int assoclen; + unsigned int srclen; + unsigned int dstlen; + u8 *iv; + int ivsize; + u32 flags; + u64 ctx_handle; + struct scatterlist *src; + struct scatterlist *dst; + u8 ctrl_arg; +}; + +/** + * struct nitrox_rfc4106_rctx - rfc4106 cipher request context + * @base: AEAD request context + * @src: Source sglist + * @dst: Destination sglist + * @assoc: AAD + */ +struct nitrox_rfc4106_rctx { + struct nitrox_aead_rctx base; + struct scatterlist src[3]; + struct scatterlist dst[3]; + u8 assoc[20]; +}; + +/** * struct pkt_instr_hdr - Packet Instruction Header * @g: Gather used * When [G] is set and [GSZ] != 0, the instruction is @@ -512,7 +556,7 @@ static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg, struct scatterlist *sg = to_sg; unsigned int sglen; - for (; buflen; buflen -= sglen) { + for (; buflen && from_sg; buflen -= sglen) { sglen = from_sg->length; if (sglen > buflen) sglen = buflen; diff --git a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c index d4935d6cefdd..7e4a5e69085e 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c +++ b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c @@ -257,12 +257,8 @@ static int nitrox_aes_decrypt(struct skcipher_request *skreq) static int nitrox_3des_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen) { - if (keylen != DES3_EDE_KEY_SIZE) { - crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } - - return nitrox_skcipher_setkey(cipher, 0, key, keylen); + return unlikely(des3_verify_key(cipher, key)) ?: + nitrox_skcipher_setkey(cipher, 0, key, keylen); } static int nitrox_3des_encrypt(struct skcipher_request *skreq) diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c index b92b6e7e100f..4985bc812b0e 100644 --- a/drivers/crypto/cavium/zip/zip_crypto.c +++ b/drivers/crypto/cavium/zip/zip_crypto.c @@ -69,7 +69,7 @@ static void zip_static_init_zip_ops(struct zip_operation *zip_ops, zip_ops->csum = 1; /* Adler checksum desired */ } -int zip_ctx_init(struct zip_kernel_ctx *zip_ctx, int lzs_flag) +static int zip_ctx_init(struct zip_kernel_ctx *zip_ctx, int lzs_flag) { struct zip_operation *comp_ctx = &zip_ctx->zip_comp; struct zip_operation *decomp_ctx = &zip_ctx->zip_decomp; @@ -107,7 +107,7 @@ err_comp_input: return -ENOMEM; } -void zip_ctx_exit(struct zip_kernel_ctx *zip_ctx) +static void zip_ctx_exit(struct zip_kernel_ctx *zip_ctx) { struct zip_operation *comp_ctx = &zip_ctx->zip_comp; struct zip_operation *dec_ctx = &zip_ctx->zip_decomp; @@ -119,7 +119,7 @@ void zip_ctx_exit(struct zip_kernel_ctx *zip_ctx) zip_data_buf_free(dec_ctx->output, MAX_OUTPUT_BUFFER_SIZE); } -int zip_compress(const u8 *src, unsigned int slen, +static int zip_compress(const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, struct zip_kernel_ctx *zip_ctx) { @@ -155,7 +155,7 @@ int zip_compress(const u8 *src, unsigned int slen, return ret; } -int zip_decompress(const u8 *src, unsigned int slen, +static int zip_decompress(const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, struct zip_kernel_ctx *zip_ctx) { diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c index c2ff551d215b..91482ffcac59 100644 --- a/drivers/crypto/ccp/ccp-crypto-des3.c +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -43,24 +43,11 @@ static int ccp_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, struct ccp_crypto_ablkcipher_alg *alg = ccp_crypto_ablkcipher_alg(crypto_ablkcipher_tfm(tfm)); u32 *flags = &tfm->base.crt_flags; + int err; - - /* From des_generic.c: - * - * RFC2451: - * If the first two or last two independent 64-bit keys are - * equal (k1 == k2 or k2 == k3), then the DES3 operation is simply the - * same as DES. Implementers MUST reject keys that exhibit this - * property. - */ - const u32 *K = (const u32 *)key; - - if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } + err = __des3_verify_key(flags, key); + if (unlikely(err)) + return err; /* It's not clear that there is any support for a keysize of 112. * If needed, the caller should make K1 == K3 diff --git a/drivers/crypto/ccp/ccp-crypto-rsa.c b/drivers/crypto/ccp/ccp-crypto-rsa.c index 05850dfd7940..a2570c0c8cdc 100644 --- a/drivers/crypto/ccp/ccp-crypto-rsa.c +++ b/drivers/crypto/ccp/ccp-crypto-rsa.c @@ -37,10 +37,9 @@ static inline int ccp_copy_and_save_keypart(u8 **kpbuf, unsigned int *kplen, if (buf[nskip]) break; *kplen = sz - nskip; - *kpbuf = kzalloc(*kplen, GFP_KERNEL); + *kpbuf = kmemdup(buf + nskip, *kplen, GFP_KERNEL); if (!*kpbuf) return -ENOMEM; - memcpy(*kpbuf, buf + nskip, *kplen); return 0; } @@ -214,8 +213,6 @@ static void ccp_rsa_exit_tfm(struct crypto_akcipher *tfm) static struct akcipher_alg ccp_rsa_defaults = { .encrypt = ccp_rsa_encrypt, .decrypt = ccp_rsa_decrypt, - .sign = ccp_rsa_decrypt, - .verify = ccp_rsa_encrypt, .set_pub_key = ccp_rsa_setpubkey, .set_priv_key = ccp_rsa_setprivkey, .max_size = ccp_rsa_maxsize, @@ -248,7 +245,8 @@ static struct ccp_rsa_def rsa_algs[] = { } }; -int ccp_register_rsa_alg(struct list_head *head, const struct ccp_rsa_def *def) +static int ccp_register_rsa_alg(struct list_head *head, + const struct ccp_rsa_def *def) { struct ccp_crypto_akcipher_alg *ccp_alg; struct akcipher_alg *alg; diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c index 10a61cd54fce..3e10573f589e 100644 --- a/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -293,8 +293,6 @@ static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key, if (key_len > block_size) { /* Must hash the input key */ sdesc->tfm = shash; - sdesc->flags = crypto_ahash_get_flags(tfm) & - CRYPTO_TFM_REQ_MAY_SLEEP; ret = crypto_shash_digest(sdesc, key, key_len, ctx->u.sha.key); diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c index fadf859a14b8..656838433f2f 100644 --- a/drivers/crypto/ccp/psp-dev.c +++ b/drivers/crypto/ccp/psp-dev.c @@ -583,6 +583,69 @@ e_free: return ret; } +static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp) +{ + struct sev_user_data_get_id2 input; + struct sev_data_get_id *data; + void *id_blob = NULL; + int ret; + + /* SEV GET_ID is available from SEV API v0.16 and up */ + if (!SEV_VERSION_GREATER_OR_EQUAL(0, 16)) + return -ENOTSUPP; + + if (copy_from_user(&input, (void __user *)argp->data, sizeof(input))) + return -EFAULT; + + /* Check if we have write access to the userspace buffer */ + if (input.address && + input.length && + !access_ok(input.address, input.length)) + return -EFAULT; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + if (input.address && input.length) { + id_blob = kmalloc(input.length, GFP_KERNEL); + if (!id_blob) { + kfree(data); + return -ENOMEM; + } + + data->address = __psp_pa(id_blob); + data->len = input.length; + } + + ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, data, &argp->error); + + /* + * Firmware will return the length of the ID value (either the minimum + * required length or the actual length written), return it to the user. + */ + input.length = data->len; + + if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) { + ret = -EFAULT; + goto e_free; + } + + if (id_blob) { + if (copy_to_user((void __user *)input.address, + id_blob, data->len)) { + ret = -EFAULT; + goto e_free; + } + } + +e_free: + kfree(id_blob); + kfree(data); + + return ret; +} + static int sev_ioctl_do_get_id(struct sev_issue_cmd *argp) { struct sev_data_get_id *data; @@ -761,8 +824,12 @@ static long sev_ioctl(struct file *file, unsigned int ioctl, unsigned long arg) ret = sev_ioctl_do_pdh_export(&input); break; case SEV_GET_ID: + pr_warn_once("SEV_GET_ID command is deprecated, use SEV_GET_ID2\n"); ret = sev_ioctl_do_get_id(&input); break; + case SEV_GET_ID2: + ret = sev_ioctl_do_get_id2(&input); + break; default: ret = -EINVAL; goto out; @@ -997,7 +1064,7 @@ void psp_pci_init(void) rc = sev_platform_init(&error); if (rc) { dev_err(sp->dev, "SEV: failed to INIT error %#x\n", error); - goto err; + return; } dev_info(sp->dev, "SEV API:%d.%d build:%d\n", psp_master->api_major, diff --git a/drivers/crypto/ccree/Makefile b/drivers/crypto/ccree/Makefile index bdc27970f95f..145e50bdbf16 100644 --- a/drivers/crypto/ccree/Makefile +++ b/drivers/crypto/ccree/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2012-2019 ARM Limited (or its affiliates). obj-$(CONFIG_CRYPTO_DEV_CCREE) := ccree.o ccree-y := cc_driver.o cc_buffer_mgr.o cc_request_mgr.o cc_cipher.o cc_hash.o cc_aead.o cc_ivgen.o cc_sram_mgr.o diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c index a3527c00b29a..7aa4cbe19a86 100644 --- a/drivers/crypto/ccree/cc_aead.c +++ b/drivers/crypto/ccree/cc_aead.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #include <linux/kernel.h> #include <linux/module.h> @@ -23,12 +23,8 @@ #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE) #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE) -#define AES_CCM_RFC4309_NONCE_SIZE 3 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE -/* Value of each ICV_CMP byte (of 8) in case of success */ -#define ICV_VERIF_OK 0x01 - struct cc_aead_handle { cc_sram_addr_t sram_workspace_addr; struct list_head aead_list; @@ -220,6 +216,10 @@ static void cc_aead_complete(struct device *dev, void *cc_req, int err) struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req); struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); + /* BACKLOG notification */ + if (err == -EINPROGRESS) + goto done; + cc_unmap_aead_request(dev, areq); /* Restore ordinary iv pointer */ @@ -424,7 +424,7 @@ static int validate_keys_sizes(struct cc_aead_ctx *ctx) /* This function prepers the user key so it can pass to the hmac processing * (copy to intenral buffer or hash in case of key longer than block */ -static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *key, +static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey, unsigned int keylen) { dma_addr_t key_dma_addr = 0; @@ -437,6 +437,7 @@ static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *key, unsigned int hashmode; unsigned int idx = 0; int rc = 0; + u8 *key = NULL; struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; dma_addr_t padded_authkey_dma_addr = ctx->auth_state.hmac.padded_authkey_dma_addr; @@ -455,11 +456,17 @@ static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *key, } if (keylen != 0) { + + key = kmemdup(authkey, keylen, GFP_KERNEL); + if (!key) + return -ENOMEM; + key_dma_addr = dma_map_single(dev, (void *)key, keylen, DMA_TO_DEVICE); if (dma_mapping_error(dev, key_dma_addr)) { dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", key, keylen); + kzfree(key); return -ENOMEM; } if (keylen > blocksize) { @@ -542,6 +549,8 @@ static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *key, if (key_dma_addr) dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE); + kzfree(key); + return rc; } @@ -650,6 +659,39 @@ setkey_error: return rc; } +static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct crypto_authenc_keys keys; + u32 flags; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + goto badkey; + + err = -EINVAL; + if (keys.enckeylen != DES3_EDE_KEY_SIZE) + goto badkey; + + flags = crypto_aead_get_flags(aead); + err = __des3_verify_key(&flags, keys.enckey); + if (unlikely(err)) { + crypto_aead_set_flags(aead, flags); + goto out; + } + + err = cc_aead_setkey(aead, key, keylen); + +out: + memzero_explicit(&keys, sizeof(keys)); + return err; + +badkey: + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); + goto out; +} + static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { @@ -731,7 +773,7 @@ static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode, dev_dbg(dev, "ASSOC buffer type DLLI\n"); hw_desc_init(&desc[idx]); set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src), - areq->assoclen, NS_BIT); + areq_ctx->assoclen, NS_BIT); set_flow_mode(&desc[idx], flow_mode); if (ctx->auth_mode == DRV_HASH_XCBC_MAC && areq_ctx->cryptlen > 0) @@ -1080,9 +1122,11 @@ static void cc_proc_header_desc(struct aead_request *req, struct cc_hw_desc desc[], unsigned int *seq_size) { + struct aead_req_ctx *areq_ctx = aead_request_ctx(req); unsigned int idx = *seq_size; + /* Hash associated data */ - if (req->assoclen > 0) + if (areq_ctx->assoclen > 0) cc_set_assoc_desc(req, DIN_HASH, desc, &idx); /* Hash IV */ @@ -1159,9 +1203,9 @@ static void cc_mlli_to_sram(struct aead_request *req, struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); struct device *dev = drvdata_to_dev(ctx->drvdata); - if (req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || + if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || req_ctx->data_buff_type == CC_DMA_BUF_MLLI || - !req_ctx->is_single_pass) { + !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) { dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n", (unsigned int)ctx->drvdata->mlli_sram_addr, req_ctx->mlli_params.mlli_len); @@ -1310,7 +1354,7 @@ static int validate_data_size(struct cc_aead_ctx *ctx, { struct aead_req_ctx *areq_ctx = aead_request_ctx(req); struct device *dev = drvdata_to_dev(ctx->drvdata); - unsigned int assoclen = req->assoclen; + unsigned int assoclen = areq_ctx->assoclen; unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ? (req->cryptlen - ctx->authsize) : req->cryptlen; @@ -1469,7 +1513,7 @@ static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[], idx++; /* process assoc data */ - if (req->assoclen > 0) { + if (req_ctx->assoclen > 0) { cc_set_assoc_desc(req, DIN_HASH, desc, &idx); } else { hw_desc_init(&desc[idx]); @@ -1561,7 +1605,7 @@ static int config_ccm_adata(struct aead_request *req) * NIST Special Publication 800-38C */ *b0 |= (8 * ((m - 2) / 2)); - if (req->assoclen > 0) + if (req_ctx->assoclen > 0) *b0 |= 64; /* Enable bit 6 if Adata exists. */ rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */ @@ -1572,7 +1616,7 @@ static int config_ccm_adata(struct aead_request *req) /* END of "taken from crypto/ccm.c" */ /* l(a) - size of associated data. */ - req_ctx->ccm_hdr_size = format_ccm_a0(a0, req->assoclen); + req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen); memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1); req->iv[15] = 1; @@ -1604,7 +1648,7 @@ static void cc_proc_rfc4309_ccm(struct aead_request *req) memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv, CCM_BLOCK_IV_SIZE); req->iv = areq_ctx->ctr_iv; - req->assoclen -= CCM_BLOCK_IV_SIZE; + areq_ctx->assoclen -= CCM_BLOCK_IV_SIZE; } static void cc_set_ghash_desc(struct aead_request *req, @@ -1812,7 +1856,7 @@ static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[], // for gcm and rfc4106. cc_set_ghash_desc(req, desc, seq_size); /* process(ghash) assoc data */ - if (req->assoclen > 0) + if (req_ctx->assoclen > 0) cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); cc_set_gctr_desc(req, desc, seq_size); /* process(gctr+ghash) */ @@ -1836,8 +1880,8 @@ static int config_gcm_context(struct aead_request *req) (req->cryptlen - ctx->authsize); __be32 counter = cpu_to_be32(2); - dev_dbg(dev, "%s() cryptlen = %d, req->assoclen = %d ctx->authsize = %d\n", - __func__, cryptlen, req->assoclen, ctx->authsize); + dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n", + __func__, cryptlen, req_ctx->assoclen, ctx->authsize); memset(req_ctx->hkey, 0, AES_BLOCK_SIZE); @@ -1853,7 +1897,7 @@ static int config_gcm_context(struct aead_request *req) if (!req_ctx->plaintext_authenticate_only) { __be64 temp64; - temp64 = cpu_to_be64(req->assoclen * 8); + temp64 = cpu_to_be64(req_ctx->assoclen * 8); memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); temp64 = cpu_to_be64(cryptlen * 8); memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); @@ -1863,8 +1907,8 @@ static int config_gcm_context(struct aead_request *req) */ __be64 temp64; - temp64 = cpu_to_be64((req->assoclen + GCM_BLOCK_RFC4_IV_SIZE + - cryptlen) * 8); + temp64 = cpu_to_be64((req_ctx->assoclen + + GCM_BLOCK_RFC4_IV_SIZE + cryptlen) * 8); memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); temp64 = 0; memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); @@ -1884,7 +1928,7 @@ static void cc_proc_rfc4_gcm(struct aead_request *req) memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv, GCM_BLOCK_RFC4_IV_SIZE); req->iv = areq_ctx->ctr_iv; - req->assoclen -= GCM_BLOCK_RFC4_IV_SIZE; + areq_ctx->assoclen -= GCM_BLOCK_RFC4_IV_SIZE; } static int cc_proc_aead(struct aead_request *req, @@ -1909,7 +1953,7 @@ static int cc_proc_aead(struct aead_request *req, /* Check data length according to mode */ if (validate_data_size(ctx, direct, req)) { dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n", - req->cryptlen, req->assoclen); + req->cryptlen, areq_ctx->assoclen); crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN); return -EINVAL; } @@ -2058,8 +2102,11 @@ static int cc_aead_encrypt(struct aead_request *req) struct aead_req_ctx *areq_ctx = aead_request_ctx(req); int rc; + memset(areq_ctx, 0, sizeof(*areq_ctx)); + /* No generated IV required */ areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; areq_ctx->backup_giv = NULL; areq_ctx->is_gcm4543 = false; @@ -2087,8 +2134,11 @@ static int cc_rfc4309_ccm_encrypt(struct aead_request *req) goto out; } + memset(areq_ctx, 0, sizeof(*areq_ctx)); + /* No generated IV required */ areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; areq_ctx->backup_giv = NULL; areq_ctx->is_gcm4543 = true; @@ -2106,8 +2156,11 @@ static int cc_aead_decrypt(struct aead_request *req) struct aead_req_ctx *areq_ctx = aead_request_ctx(req); int rc; + memset(areq_ctx, 0, sizeof(*areq_ctx)); + /* No generated IV required */ areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; areq_ctx->backup_giv = NULL; areq_ctx->is_gcm4543 = false; @@ -2133,8 +2186,11 @@ static int cc_rfc4309_ccm_decrypt(struct aead_request *req) goto out; } + memset(areq_ctx, 0, sizeof(*areq_ctx)); + /* No generated IV required */ areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; areq_ctx->backup_giv = NULL; areq_ctx->is_gcm4543 = true; @@ -2250,8 +2306,11 @@ static int cc_rfc4106_gcm_encrypt(struct aead_request *req) goto out; } + memset(areq_ctx, 0, sizeof(*areq_ctx)); + /* No generated IV required */ areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; areq_ctx->backup_giv = NULL; areq_ctx->plaintext_authenticate_only = false; @@ -2273,11 +2332,14 @@ static int cc_rfc4543_gcm_encrypt(struct aead_request *req) struct aead_req_ctx *areq_ctx = aead_request_ctx(req); int rc; + memset(areq_ctx, 0, sizeof(*areq_ctx)); + //plaintext is not encryped with rfc4543 areq_ctx->plaintext_authenticate_only = true; /* No generated IV required */ areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; areq_ctx->backup_giv = NULL; cc_proc_rfc4_gcm(req); @@ -2305,8 +2367,11 @@ static int cc_rfc4106_gcm_decrypt(struct aead_request *req) goto out; } + memset(areq_ctx, 0, sizeof(*areq_ctx)); + /* No generated IV required */ areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; areq_ctx->backup_giv = NULL; areq_ctx->plaintext_authenticate_only = false; @@ -2328,11 +2393,14 @@ static int cc_rfc4543_gcm_decrypt(struct aead_request *req) struct aead_req_ctx *areq_ctx = aead_request_ctx(req); int rc; + memset(areq_ctx, 0, sizeof(*areq_ctx)); + //plaintext is not decryped with rfc4543 areq_ctx->plaintext_authenticate_only = true; /* No generated IV required */ areq_ctx->backup_iv = req->iv; + areq_ctx->assoclen = req->assoclen; areq_ctx->backup_giv = NULL; cc_proc_rfc4_gcm(req); @@ -2372,7 +2440,7 @@ static struct cc_alg_template aead_algs[] = { .driver_name = "authenc-hmac-sha1-cbc-des3-ccree", .blocksize = DES3_EDE_BLOCK_SIZE, .template_aead = { - .setkey = cc_aead_setkey, + .setkey = cc_des3_aead_setkey, .setauthsize = cc_aead_setauthsize, .encrypt = cc_aead_encrypt, .decrypt = cc_aead_decrypt, @@ -2412,7 +2480,7 @@ static struct cc_alg_template aead_algs[] = { .driver_name = "authenc-hmac-sha256-cbc-des3-ccree", .blocksize = DES3_EDE_BLOCK_SIZE, .template_aead = { - .setkey = cc_aead_setkey, + .setkey = cc_des3_aead_setkey, .setauthsize = cc_aead_setauthsize, .encrypt = cc_aead_encrypt, .decrypt = cc_aead_decrypt, diff --git a/drivers/crypto/ccree/cc_aead.h b/drivers/crypto/ccree/cc_aead.h index 5edf3b351fa4..e51724b96c56 100644 --- a/drivers/crypto/ccree/cc_aead.h +++ b/drivers/crypto/ccree/cc_aead.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ /* \file cc_aead.h * ARM CryptoCell AEAD Crypto API @@ -67,6 +67,7 @@ struct aead_req_ctx { u8 backup_mac[MAX_MAC_SIZE]; u8 *backup_iv; /*store iv for generated IV flow*/ u8 *backup_giv; /*store iv for rfc3686(ctr) flow*/ + u32 assoclen; /* internal assoclen */ dma_addr_t mac_buf_dma_addr; /* internal ICV DMA buffer */ /* buffer for internal ccm configurations */ dma_addr_t ccm_iv0_dma_addr; diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c index 0ee1c52da0a4..c81ad33f9115 100644 --- a/drivers/crypto/ccree/cc_buffer_mgr.c +++ b/drivers/crypto/ccree/cc_buffer_mgr.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #include <crypto/internal/aead.h> #include <crypto/authenc.h> @@ -65,7 +65,7 @@ static void cc_copy_mac(struct device *dev, struct aead_request *req, { struct aead_req_ctx *areq_ctx = aead_request_ctx(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req); - u32 skip = req->assoclen + req->cryptlen; + u32 skip = areq_ctx->assoclen + req->cryptlen; if (areq_ctx->is_gcm4543) skip += crypto_aead_ivsize(tfm); @@ -83,24 +83,17 @@ static void cc_copy_mac(struct device *dev, struct aead_request *req, */ static unsigned int cc_get_sgl_nents(struct device *dev, struct scatterlist *sg_list, - unsigned int nbytes, u32 *lbytes, - bool *is_chained) + unsigned int nbytes, u32 *lbytes) { unsigned int nents = 0; while (nbytes && sg_list) { - if (sg_list->length) { - nents++; - /* get the number of bytes in the last entry */ - *lbytes = nbytes; - nbytes -= (sg_list->length > nbytes) ? - nbytes : sg_list->length; - sg_list = sg_next(sg_list); - } else { - sg_list = (struct scatterlist *)sg_page(sg_list); - if (is_chained) - *is_chained = true; - } + nents++; + /* get the number of bytes in the last entry */ + *lbytes = nbytes; + nbytes -= (sg_list->length > nbytes) ? + nbytes : sg_list->length; + sg_list = sg_next(sg_list); } dev_dbg(dev, "nents %d last bytes %d\n", nents, *lbytes); return nents; @@ -140,9 +133,9 @@ void cc_zero_sgl(struct scatterlist *sgl, u32 data_len) void cc_copy_sg_portion(struct device *dev, u8 *dest, struct scatterlist *sg, u32 to_skip, u32 end, enum cc_sg_cpy_direct direct) { - u32 nents, lbytes; + u32 nents; - nents = cc_get_sgl_nents(dev, sg, end, &lbytes, NULL); + nents = sg_nents_for_len(sg, end); sg_copy_buffer(sg, nents, (void *)dest, (end - to_skip + 1), to_skip, (direct == CC_SG_TO_BUF)); } @@ -314,40 +307,10 @@ static void cc_add_sg_entry(struct device *dev, struct buffer_array *sgl_data, sgl_data->num_of_buffers++; } -static int cc_dma_map_sg(struct device *dev, struct scatterlist *sg, u32 nents, - enum dma_data_direction direction) -{ - u32 i, j; - struct scatterlist *l_sg = sg; - - for (i = 0; i < nents; i++) { - if (!l_sg) - break; - if (dma_map_sg(dev, l_sg, 1, direction) != 1) { - dev_err(dev, "dma_map_page() sg buffer failed\n"); - goto err; - } - l_sg = sg_next(l_sg); - } - return nents; - -err: - /* Restore mapped parts */ - for (j = 0; j < i; j++) { - if (!sg) - break; - dma_unmap_sg(dev, sg, 1, direction); - sg = sg_next(sg); - } - return 0; -} - static int cc_map_sg(struct device *dev, struct scatterlist *sg, unsigned int nbytes, int direction, u32 *nents, u32 max_sg_nents, u32 *lbytes, u32 *mapped_nents) { - bool is_chained = false; - if (sg_is_last(sg)) { /* One entry only case -set to DLLI */ if (dma_map_sg(dev, sg, 1, direction) != 1) { @@ -361,35 +324,21 @@ static int cc_map_sg(struct device *dev, struct scatterlist *sg, *nents = 1; *mapped_nents = 1; } else { /*sg_is_last*/ - *nents = cc_get_sgl_nents(dev, sg, nbytes, lbytes, - &is_chained); + *nents = cc_get_sgl_nents(dev, sg, nbytes, lbytes); if (*nents > max_sg_nents) { *nents = 0; dev_err(dev, "Too many fragments. current %d max %d\n", *nents, max_sg_nents); return -ENOMEM; } - if (!is_chained) { - /* In case of mmu the number of mapped nents might - * be changed from the original sgl nents - */ - *mapped_nents = dma_map_sg(dev, sg, *nents, direction); - if (*mapped_nents == 0) { - *nents = 0; - dev_err(dev, "dma_map_sg() sg buffer failed\n"); - return -ENOMEM; - } - } else { - /*In this case the driver maps entry by entry so it - * must have the same nents before and after map - */ - *mapped_nents = cc_dma_map_sg(dev, sg, *nents, - direction); - if (*mapped_nents != *nents) { - *nents = *mapped_nents; - dev_err(dev, "dma_map_sg() sg buffer failed\n"); - return -ENOMEM; - } + /* In case of mmu the number of mapped nents might + * be changed from the original sgl nents + */ + *mapped_nents = dma_map_sg(dev, sg, *nents, direction); + if (*mapped_nents == 0) { + *nents = 0; + dev_err(dev, "dma_map_sg() sg buffer failed\n"); + return -ENOMEM; } } @@ -457,7 +406,7 @@ void cc_unmap_cipher_request(struct device *dev, void *ctx, dev_dbg(dev, "Unmapped iv: iv_dma_addr=%pad iv_size=%u\n", &req_ctx->gen_ctx.iv_dma_addr, ivsize); dma_unmap_single(dev, req_ctx->gen_ctx.iv_dma_addr, - ivsize, DMA_TO_DEVICE); + ivsize, DMA_BIDIRECTIONAL); } /* Release pool */ if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI && @@ -499,7 +448,7 @@ int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx, dump_byte_array("iv", (u8 *)info, ivsize); req_ctx->gen_ctx.iv_dma_addr = dma_map_single(dev, (void *)info, - ivsize, DMA_TO_DEVICE); + ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, req_ctx->gen_ctx.iv_dma_addr)) { dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n", ivsize, info); @@ -568,11 +517,7 @@ void cc_unmap_aead_request(struct device *dev, struct aead_request *req) { struct aead_req_ctx *areq_ctx = aead_request_ctx(req); unsigned int hw_iv_size = areq_ctx->hw_iv_size; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct cc_drvdata *drvdata = dev_get_drvdata(dev); - u32 dummy; - bool chained; - u32 size_to_unmap = 0; if (areq_ctx->mac_buf_dma_addr) { dma_unmap_single(dev, areq_ctx->mac_buf_dma_addr, @@ -612,6 +557,7 @@ void cc_unmap_aead_request(struct device *dev, struct aead_request *req) if (areq_ctx->gen_ctx.iv_dma_addr) { dma_unmap_single(dev, areq_ctx->gen_ctx.iv_dma_addr, hw_iv_size, DMA_BIDIRECTIONAL); + kzfree(areq_ctx->gen_ctx.iv); } /* Release pool */ @@ -628,23 +574,13 @@ void cc_unmap_aead_request(struct device *dev, struct aead_request *req) dev_dbg(dev, "Unmapping src sgl: req->src=%pK areq_ctx->src.nents=%u areq_ctx->assoc.nents=%u assoclen:%u cryptlen=%u\n", sg_virt(req->src), areq_ctx->src.nents, areq_ctx->assoc.nents, - req->assoclen, req->cryptlen); - size_to_unmap = req->assoclen + req->cryptlen; - if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_ENCRYPT) - size_to_unmap += areq_ctx->req_authsize; - if (areq_ctx->is_gcm4543) - size_to_unmap += crypto_aead_ivsize(tfm); + areq_ctx->assoclen, req->cryptlen); - dma_unmap_sg(dev, req->src, - cc_get_sgl_nents(dev, req->src, size_to_unmap, - &dummy, &chained), - DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL); if (req->src != req->dst) { dev_dbg(dev, "Unmapping dst sgl: req->dst=%pK\n", sg_virt(req->dst)); - dma_unmap_sg(dev, req->dst, - cc_get_sgl_nents(dev, req->dst, size_to_unmap, - &dummy, &chained), + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL); } if (drvdata->coherent && @@ -658,55 +594,10 @@ void cc_unmap_aead_request(struct device *dev, struct aead_request *req) } } -static int cc_get_aead_icv_nents(struct device *dev, struct scatterlist *sgl, - unsigned int sgl_nents, unsigned int authsize, - u32 last_entry_data_size, - bool *is_icv_fragmented) +static bool cc_is_icv_frag(unsigned int sgl_nents, unsigned int authsize, + u32 last_entry_data_size) { - unsigned int icv_max_size = 0; - unsigned int icv_required_size = authsize > last_entry_data_size ? - (authsize - last_entry_data_size) : - authsize; - unsigned int nents; - unsigned int i; - - if (sgl_nents < MAX_ICV_NENTS_SUPPORTED) { - *is_icv_fragmented = false; - return 0; - } - - for (i = 0 ; i < (sgl_nents - MAX_ICV_NENTS_SUPPORTED) ; i++) { - if (!sgl) - break; - sgl = sg_next(sgl); - } - - if (sgl) - icv_max_size = sgl->length; - - if (last_entry_data_size > authsize) { - /* ICV attached to data in last entry (not fragmented!) */ - nents = 0; - *is_icv_fragmented = false; - } else if (last_entry_data_size == authsize) { - /* ICV placed in whole last entry (not fragmented!) */ - nents = 1; - *is_icv_fragmented = false; - } else if (icv_max_size > icv_required_size) { - nents = 1; - *is_icv_fragmented = true; - } else if (icv_max_size == icv_required_size) { - nents = 2; - *is_icv_fragmented = true; - } else { - dev_err(dev, "Unsupported num. of ICV fragments (> %d)\n", - MAX_ICV_NENTS_SUPPORTED); - nents = -1; /*unsupported*/ - } - dev_dbg(dev, "is_frag=%s icv_nents=%u\n", - (*is_icv_fragmented ? "true" : "false"), nents); - - return nents; + return ((sgl_nents > 1) && (last_entry_data_size < authsize)); } static int cc_aead_chain_iv(struct cc_drvdata *drvdata, @@ -717,19 +608,27 @@ static int cc_aead_chain_iv(struct cc_drvdata *drvdata, struct aead_req_ctx *areq_ctx = aead_request_ctx(req); unsigned int hw_iv_size = areq_ctx->hw_iv_size; struct device *dev = drvdata_to_dev(drvdata); + gfp_t flags = cc_gfp_flags(&req->base); int rc = 0; if (!req->iv) { areq_ctx->gen_ctx.iv_dma_addr = 0; + areq_ctx->gen_ctx.iv = NULL; goto chain_iv_exit; } - areq_ctx->gen_ctx.iv_dma_addr = dma_map_single(dev, req->iv, - hw_iv_size, - DMA_BIDIRECTIONAL); + areq_ctx->gen_ctx.iv = kmemdup(req->iv, hw_iv_size, flags); + if (!areq_ctx->gen_ctx.iv) + return -ENOMEM; + + areq_ctx->gen_ctx.iv_dma_addr = + dma_map_single(dev, areq_ctx->gen_ctx.iv, hw_iv_size, + DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, areq_ctx->gen_ctx.iv_dma_addr)) { dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n", hw_iv_size, req->iv); + kzfree(areq_ctx->gen_ctx.iv); + areq_ctx->gen_ctx.iv = NULL; rc = -ENOMEM; goto chain_iv_exit; } @@ -760,11 +659,9 @@ static int cc_aead_chain_assoc(struct cc_drvdata *drvdata, { struct aead_req_ctx *areq_ctx = aead_request_ctx(req); int rc = 0; - u32 mapped_nents = 0; - struct scatterlist *current_sg = req->src; + int mapped_nents = 0; struct crypto_aead *tfm = crypto_aead_reqtfm(req); - unsigned int sg_index = 0; - u32 size_of_assoc = req->assoclen; + unsigned int size_of_assoc = areq_ctx->assoclen; struct device *dev = drvdata_to_dev(drvdata); if (areq_ctx->is_gcm4543) @@ -775,7 +672,7 @@ static int cc_aead_chain_assoc(struct cc_drvdata *drvdata, goto chain_assoc_exit; } - if (req->assoclen == 0) { + if (areq_ctx->assoclen == 0) { areq_ctx->assoc_buff_type = CC_DMA_BUF_NULL; areq_ctx->assoc.nents = 0; areq_ctx->assoc.mlli_nents = 0; @@ -785,26 +682,10 @@ static int cc_aead_chain_assoc(struct cc_drvdata *drvdata, goto chain_assoc_exit; } - //iterate over the sgl to see how many entries are for associated data - //it is assumed that if we reach here , the sgl is already mapped - sg_index = current_sg->length; - //the first entry in the scatter list contains all the associated data - if (sg_index > size_of_assoc) { - mapped_nents++; - } else { - while (sg_index <= size_of_assoc) { - current_sg = sg_next(current_sg); - /* if have reached the end of the sgl, then this is - * unexpected - */ - if (!current_sg) { - dev_err(dev, "reached end of sg list. unexpected\n"); - return -EINVAL; - } - sg_index += current_sg->length; - mapped_nents++; - } - } + mapped_nents = sg_nents_for_len(req->src, size_of_assoc); + if (mapped_nents < 0) + return mapped_nents; + if (mapped_nents > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) { dev_err(dev, "Too many fragments. current %d max %d\n", mapped_nents, LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES); @@ -835,7 +716,7 @@ static int cc_aead_chain_assoc(struct cc_drvdata *drvdata, cc_dma_buf_type(areq_ctx->assoc_buff_type), areq_ctx->assoc.nents); cc_add_sg_entry(dev, sg_data, areq_ctx->assoc.nents, req->src, - req->assoclen, 0, is_last, + areq_ctx->assoclen, 0, is_last, &areq_ctx->assoc.mlli_nents); areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI; } @@ -850,39 +731,32 @@ static void cc_prepare_aead_data_dlli(struct aead_request *req, struct aead_req_ctx *areq_ctx = aead_request_ctx(req); enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type; unsigned int authsize = areq_ctx->req_authsize; + struct scatterlist *sg; + ssize_t offset; areq_ctx->is_icv_fragmented = false; - if (req->src == req->dst) { - /*INPLACE*/ - areq_ctx->icv_dma_addr = sg_dma_address(areq_ctx->src_sgl) + - (*src_last_bytes - authsize); - areq_ctx->icv_virt_addr = sg_virt(areq_ctx->src_sgl) + - (*src_last_bytes - authsize); - } else if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) { - /*NON-INPLACE and DECRYPT*/ - areq_ctx->icv_dma_addr = sg_dma_address(areq_ctx->src_sgl) + - (*src_last_bytes - authsize); - areq_ctx->icv_virt_addr = sg_virt(areq_ctx->src_sgl) + - (*src_last_bytes - authsize); + + if ((req->src == req->dst) || direct == DRV_CRYPTO_DIRECTION_DECRYPT) { + sg = areq_ctx->src_sgl; + offset = *src_last_bytes - authsize; } else { - /*NON-INPLACE and ENCRYPT*/ - areq_ctx->icv_dma_addr = sg_dma_address(areq_ctx->dst_sgl) + - (*dst_last_bytes - authsize); - areq_ctx->icv_virt_addr = sg_virt(areq_ctx->dst_sgl) + - (*dst_last_bytes - authsize); + sg = areq_ctx->dst_sgl; + offset = *dst_last_bytes - authsize; } + + areq_ctx->icv_dma_addr = sg_dma_address(sg) + offset; + areq_ctx->icv_virt_addr = sg_virt(sg) + offset; } -static int cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata, - struct aead_request *req, - struct buffer_array *sg_data, - u32 *src_last_bytes, u32 *dst_last_bytes, - bool is_last_table) +static void cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata, + struct aead_request *req, + struct buffer_array *sg_data, + u32 *src_last_bytes, u32 *dst_last_bytes, + bool is_last_table) { struct aead_req_ctx *areq_ctx = aead_request_ctx(req); enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type; unsigned int authsize = areq_ctx->req_authsize; - int rc = 0, icv_nents; struct device *dev = drvdata_to_dev(drvdata); struct scatterlist *sg; @@ -893,14 +767,9 @@ static int cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata, areq_ctx->src_offset, is_last_table, &areq_ctx->src.mlli_nents); - icv_nents = cc_get_aead_icv_nents(dev, areq_ctx->src_sgl, - areq_ctx->src.nents, - authsize, *src_last_bytes, - &areq_ctx->is_icv_fragmented); - if (icv_nents < 0) { - rc = -ENOTSUPP; - goto prepare_data_mlli_exit; - } + areq_ctx->is_icv_fragmented = + cc_is_icv_frag(areq_ctx->src.nents, authsize, + *src_last_bytes); if (areq_ctx->is_icv_fragmented) { /* Backup happens only when ICV is fragmented, ICV @@ -942,16 +811,11 @@ static int cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata, areq_ctx->dst_offset, is_last_table, &areq_ctx->dst.mlli_nents); - icv_nents = cc_get_aead_icv_nents(dev, areq_ctx->src_sgl, - areq_ctx->src.nents, - authsize, *src_last_bytes, - &areq_ctx->is_icv_fragmented); - if (icv_nents < 0) { - rc = -ENOTSUPP; - goto prepare_data_mlli_exit; - } - + areq_ctx->is_icv_fragmented = + cc_is_icv_frag(areq_ctx->src.nents, authsize, + *src_last_bytes); /* Backup happens only when ICV is fragmented, ICV + * verification is made by CPU compare in order to simplify * MAC verification upon request completion */ @@ -979,14 +843,9 @@ static int cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata, areq_ctx->src_offset, is_last_table, &areq_ctx->src.mlli_nents); - icv_nents = cc_get_aead_icv_nents(dev, areq_ctx->dst_sgl, - areq_ctx->dst.nents, - authsize, *dst_last_bytes, - &areq_ctx->is_icv_fragmented); - if (icv_nents < 0) { - rc = -ENOTSUPP; - goto prepare_data_mlli_exit; - } + areq_ctx->is_icv_fragmented = + cc_is_icv_frag(areq_ctx->dst.nents, authsize, + *dst_last_bytes); if (!areq_ctx->is_icv_fragmented) { sg = &areq_ctx->dst_sgl[areq_ctx->dst.nents - 1]; @@ -1000,9 +859,6 @@ static int cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata, areq_ctx->icv_virt_addr = areq_ctx->mac_buf; } } - -prepare_data_mlli_exit: - return rc; } static int cc_aead_chain_data(struct cc_drvdata *drvdata, @@ -1019,12 +875,12 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata, u32 src_mapped_nents = 0, dst_mapped_nents = 0; u32 offset = 0; /* non-inplace mode */ - unsigned int size_for_map = req->assoclen + req->cryptlen; + unsigned int size_for_map = areq_ctx->assoclen + req->cryptlen; struct crypto_aead *tfm = crypto_aead_reqtfm(req); u32 sg_index = 0; - bool chained = false; bool is_gcm4543 = areq_ctx->is_gcm4543; - u32 size_to_skip = req->assoclen; + u32 size_to_skip = areq_ctx->assoclen; + struct scatterlist *sgl; if (is_gcm4543) size_to_skip += crypto_aead_ivsize(tfm); @@ -1043,19 +899,17 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata, size_for_map += (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? authsize : 0; src_mapped_nents = cc_get_sgl_nents(dev, req->src, size_for_map, - &src_last_bytes, &chained); + &src_last_bytes); sg_index = areq_ctx->src_sgl->length; //check where the data starts while (sg_index <= size_to_skip) { + src_mapped_nents--; offset -= areq_ctx->src_sgl->length; - areq_ctx->src_sgl = sg_next(areq_ctx->src_sgl); - //if have reached the end of the sgl, then this is unexpected - if (!areq_ctx->src_sgl) { - dev_err(dev, "reached end of sg list. unexpected\n"); - return -EINVAL; - } + sgl = sg_next(areq_ctx->src_sgl); + if (!sgl) + break; + areq_ctx->src_sgl = sgl; sg_index += areq_ctx->src_sgl->length; - src_mapped_nents--; } if (src_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) { dev_err(dev, "Too many fragments. current %d max %d\n", @@ -1068,7 +922,7 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata, areq_ctx->src_offset = offset; if (req->src != req->dst) { - size_for_map = req->assoclen + req->cryptlen; + size_for_map = areq_ctx->assoclen + req->cryptlen; size_for_map += (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? authsize : 0; if (is_gcm4543) @@ -1083,21 +937,19 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata, } dst_mapped_nents = cc_get_sgl_nents(dev, req->dst, size_for_map, - &dst_last_bytes, &chained); + &dst_last_bytes); sg_index = areq_ctx->dst_sgl->length; offset = size_to_skip; //check where the data starts while (sg_index <= size_to_skip) { + dst_mapped_nents--; offset -= areq_ctx->dst_sgl->length; - areq_ctx->dst_sgl = sg_next(areq_ctx->dst_sgl); - //if have reached the end of the sgl, then this is unexpected - if (!areq_ctx->dst_sgl) { - dev_err(dev, "reached end of sg list. unexpected\n"); - return -EINVAL; - } + sgl = sg_next(areq_ctx->dst_sgl); + if (!sgl) + break; + areq_ctx->dst_sgl = sgl; sg_index += areq_ctx->dst_sgl->length; - dst_mapped_nents--; } if (dst_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) { dev_err(dev, "Too many fragments. current %d max %d\n", @@ -1110,9 +962,9 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata, dst_mapped_nents > 1 || do_chain) { areq_ctx->data_buff_type = CC_DMA_BUF_MLLI; - rc = cc_prepare_aead_data_mlli(drvdata, req, sg_data, - &src_last_bytes, - &dst_last_bytes, is_last_table); + cc_prepare_aead_data_mlli(drvdata, req, sg_data, + &src_last_bytes, &dst_last_bytes, + is_last_table); } else { areq_ctx->data_buff_type = CC_DMA_BUF_DLLI; cc_prepare_aead_data_dlli(req, &src_last_bytes, @@ -1234,7 +1086,7 @@ int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) areq_ctx->ccm_iv0_dma_addr = dma_addr; rc = cc_set_aead_conf_buf(dev, areq_ctx, areq_ctx->ccm_config, - &sg_data, req->assoclen); + &sg_data, areq_ctx->assoclen); if (rc) goto aead_map_failure; } @@ -1285,7 +1137,7 @@ int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) areq_ctx->gcm_iv_inc2_dma_addr = dma_addr; } - size_to_map = req->cryptlen + req->assoclen; + size_to_map = req->cryptlen + areq_ctx->assoclen; if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_ENCRYPT) size_to_map += authsize; @@ -1483,8 +1335,7 @@ int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, if (total_in_len < block_size) { dev_dbg(dev, " less than one block: curr_buff=%pK *curr_buff_cnt=0x%X copy_to=%pK\n", curr_buff, *curr_buff_cnt, &curr_buff[*curr_buff_cnt]); - areq_ctx->in_nents = - cc_get_sgl_nents(dev, src, nbytes, &dummy, NULL); + areq_ctx->in_nents = sg_nents_for_len(src, nbytes); sg_copy_to_buffer(src, areq_ctx->in_nents, &curr_buff[*curr_buff_cnt], nbytes); *curr_buff_cnt += nbytes; diff --git a/drivers/crypto/ccree/cc_buffer_mgr.h b/drivers/crypto/ccree/cc_buffer_mgr.h index 3ec4b4db5247..a726016bdbc1 100644 --- a/drivers/crypto/ccree/cc_buffer_mgr.h +++ b/drivers/crypto/ccree/cc_buffer_mgr.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ /* \file cc_buffer_mgr.h * Buffer Manager diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index d9c17078517b..5b58226ea24d 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #include <linux/kernel.h> #include <linux/module.h> @@ -34,6 +34,18 @@ struct cc_hw_key_info { enum cc_hw_crypto_key key2_slot; }; +struct cc_cpp_key_info { + u8 slot; + enum cc_cpp_alg alg; +}; + +enum cc_key_type { + CC_UNPROTECTED_KEY, /* User key */ + CC_HW_PROTECTED_KEY, /* HW (FDE) key */ + CC_POLICY_PROTECTED_KEY, /* CPP key */ + CC_INVALID_PROTECTED_KEY /* Invalid key */ +}; + struct cc_cipher_ctx { struct cc_drvdata *drvdata; int keylen; @@ -41,19 +53,22 @@ struct cc_cipher_ctx { int cipher_mode; int flow_mode; unsigned int flags; - bool hw_key; + enum cc_key_type key_type; struct cc_user_key_info user; - struct cc_hw_key_info hw; + union { + struct cc_hw_key_info hw; + struct cc_cpp_key_info cpp; + }; struct crypto_shash *shash_tfm; }; static void cc_cipher_complete(struct device *dev, void *cc_req, int err); -static inline bool cc_is_hw_key(struct crypto_tfm *tfm) +static inline enum cc_key_type cc_key_type(struct crypto_tfm *tfm) { struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); - return ctx_p->hw_key; + return ctx_p->key_type; } static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) @@ -232,7 +247,7 @@ struct tdes_keys { u8 key3[DES_KEY_SIZE]; }; -static enum cc_hw_crypto_key cc_slot_to_hw_key(int slot_num) +static enum cc_hw_crypto_key cc_slot_to_hw_key(u8 slot_num) { switch (slot_num) { case 0: @@ -247,6 +262,22 @@ static enum cc_hw_crypto_key cc_slot_to_hw_key(int slot_num) return END_OF_KEYS; } +static u8 cc_slot_to_cpp_key(u8 slot_num) +{ + return (slot_num - CC_FIRST_CPP_KEY_SLOT); +} + +static inline enum cc_key_type cc_slot_to_key_type(u8 slot_num) +{ + if (slot_num >= CC_FIRST_HW_KEY_SLOT && slot_num <= CC_LAST_HW_KEY_SLOT) + return CC_HW_PROTECTED_KEY; + else if (slot_num >= CC_FIRST_CPP_KEY_SLOT && + slot_num <= CC_LAST_CPP_KEY_SLOT) + return CC_POLICY_PROTECTED_KEY; + else + return CC_INVALID_PROTECTED_KEY; +} + static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, unsigned int keylen) { @@ -261,18 +292,13 @@ static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, /* STAT_PHASE_0: Init and sanity checks */ - /* This check the size of the hardware key token */ + /* This check the size of the protected key token */ if (keylen != sizeof(hki)) { - dev_err(dev, "Unsupported HW key size %d.\n", keylen); + dev_err(dev, "Unsupported protected key size %d.\n", keylen); crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } - if (ctx_p->flow_mode != S_DIN_to_AES) { - dev_err(dev, "HW key not supported for non-AES flows\n"); - return -EINVAL; - } - memcpy(&hki, key, keylen); /* The real key len for crypto op is the size of the HW key @@ -286,31 +312,70 @@ static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, return -EINVAL; } - ctx_p->hw.key1_slot = cc_slot_to_hw_key(hki.hw_key1); - if (ctx_p->hw.key1_slot == END_OF_KEYS) { - dev_err(dev, "Unsupported hw key1 number (%d)\n", hki.hw_key1); - return -EINVAL; - } + ctx_p->keylen = keylen; - if (ctx_p->cipher_mode == DRV_CIPHER_XTS || - ctx_p->cipher_mode == DRV_CIPHER_ESSIV || - ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) { - if (hki.hw_key1 == hki.hw_key2) { - dev_err(dev, "Illegal hw key numbers (%d,%d)\n", - hki.hw_key1, hki.hw_key2); + switch (cc_slot_to_key_type(hki.hw_key1)) { + case CC_HW_PROTECTED_KEY: + if (ctx_p->flow_mode == S_DIN_to_SM4) { + dev_err(dev, "Only AES HW protected keys are supported\n"); return -EINVAL; } - ctx_p->hw.key2_slot = cc_slot_to_hw_key(hki.hw_key2); - if (ctx_p->hw.key2_slot == END_OF_KEYS) { - dev_err(dev, "Unsupported hw key2 number (%d)\n", - hki.hw_key2); + + ctx_p->hw.key1_slot = cc_slot_to_hw_key(hki.hw_key1); + if (ctx_p->hw.key1_slot == END_OF_KEYS) { + dev_err(dev, "Unsupported hw key1 number (%d)\n", + hki.hw_key1); return -EINVAL; } - } - ctx_p->keylen = keylen; - ctx_p->hw_key = true; - dev_dbg(dev, "cc_is_hw_key ret 0"); + if (ctx_p->cipher_mode == DRV_CIPHER_XTS || + ctx_p->cipher_mode == DRV_CIPHER_ESSIV || + ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) { + if (hki.hw_key1 == hki.hw_key2) { + dev_err(dev, "Illegal hw key numbers (%d,%d)\n", + hki.hw_key1, hki.hw_key2); + return -EINVAL; + } + + ctx_p->hw.key2_slot = cc_slot_to_hw_key(hki.hw_key2); + if (ctx_p->hw.key2_slot == END_OF_KEYS) { + dev_err(dev, "Unsupported hw key2 number (%d)\n", + hki.hw_key2); + return -EINVAL; + } + } + + ctx_p->key_type = CC_HW_PROTECTED_KEY; + dev_dbg(dev, "HW protected key %d/%d set\n.", + ctx_p->hw.key1_slot, ctx_p->hw.key2_slot); + break; + + case CC_POLICY_PROTECTED_KEY: + if (ctx_p->drvdata->hw_rev < CC_HW_REV_713) { + dev_err(dev, "CPP keys not supported in this hardware revision.\n"); + return -EINVAL; + } + + if (ctx_p->cipher_mode != DRV_CIPHER_CBC && + ctx_p->cipher_mode != DRV_CIPHER_CTR) { + dev_err(dev, "CPP keys only supported in CBC or CTR modes.\n"); + return -EINVAL; + } + + ctx_p->cpp.slot = cc_slot_to_cpp_key(hki.hw_key1); + if (ctx_p->flow_mode == S_DIN_to_AES) + ctx_p->cpp.alg = CC_CPP_AES; + else /* Must be SM4 since due to sethkey registration */ + ctx_p->cpp.alg = CC_CPP_SM4; + ctx_p->key_type = CC_POLICY_PROTECTED_KEY; + dev_dbg(dev, "policy protected key alg: %d slot: %d.\n", + ctx_p->cpp.alg, ctx_p->cpp.slot); + break; + + default: + dev_err(dev, "Unsupported protected key (%d)\n", hki.hw_key1); + return -EINVAL; + } return 0; } @@ -321,7 +386,6 @@ static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); struct device *dev = drvdata_to_dev(ctx_p->drvdata); - u32 tmp[DES3_EDE_EXPKEY_WORDS]; struct cc_crypto_alg *cc_alg = container_of(tfm->__crt_alg, struct cc_crypto_alg, skcipher_alg.base); @@ -339,7 +403,7 @@ static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, return -EINVAL; } - ctx_p->hw_key = false; + ctx_p->key_type = CC_UNPROTECTED_KEY; /* * Verify DES weak keys @@ -347,6 +411,7 @@ static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, * HW does the expansion on its own. */ if (ctx_p->flow_mode == S_DIN_to_DES) { + u32 tmp[DES3_EDE_EXPKEY_WORDS]; if (keylen == DES3_EDE_KEY_SIZE && __des3_ede_setkey(tmp, &tfm->crt_flags, key, DES3_EDE_KEY_SIZE)) { @@ -399,7 +464,77 @@ static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, return 0; } -static void cc_setup_cipher_desc(struct crypto_tfm *tfm, +static int cc_out_setup_mode(struct cc_cipher_ctx *ctx_p) +{ + switch (ctx_p->flow_mode) { + case S_DIN_to_AES: + return S_AES_to_DOUT; + case S_DIN_to_DES: + return S_DES_to_DOUT; + case S_DIN_to_SM4: + return S_SM4_to_DOUT; + default: + return ctx_p->flow_mode; + } +} + +static void cc_setup_readiv_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + unsigned int ivsize, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + int cipher_mode = ctx_p->cipher_mode; + int flow_mode = cc_out_setup_mode(ctx_p); + int direction = req_ctx->gen_ctx.op_type; + dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; + + if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) + return; + + switch (cipher_mode) { + case DRV_CIPHER_ECB: + break; + case DRV_CIPHER_CBC: + case DRV_CIPHER_CBC_CTS: + case DRV_CIPHER_CTR: + case DRV_CIPHER_OFB: + /* Read next IV */ + hw_desc_init(&desc[*seq_size]); + set_dout_dlli(&desc[*seq_size], iv_dma_addr, ivsize, NS_BIT, 1); + set_cipher_config0(&desc[*seq_size], direction); + set_flow_mode(&desc[*seq_size], flow_mode); + set_cipher_mode(&desc[*seq_size], cipher_mode); + if (cipher_mode == DRV_CIPHER_CTR || + cipher_mode == DRV_CIPHER_OFB) { + set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); + } else { + set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE0); + } + set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); + (*seq_size)++; + break; + case DRV_CIPHER_XTS: + case DRV_CIPHER_ESSIV: + case DRV_CIPHER_BITLOCKER: + /* IV */ + hw_desc_init(&desc[*seq_size]); + set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); + set_cipher_mode(&desc[*seq_size], cipher_mode); + set_cipher_config0(&desc[*seq_size], direction); + set_flow_mode(&desc[*seq_size], flow_mode); + set_dout_dlli(&desc[*seq_size], iv_dma_addr, CC_AES_BLOCK_SIZE, + NS_BIT, 1); + set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); + (*seq_size)++; + break; + default: + dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); + } +} + +static void cc_setup_state_desc(struct crypto_tfm *tfm, struct cipher_req_ctx *req_ctx, unsigned int ivsize, unsigned int nbytes, struct cc_hw_desc desc[], @@ -423,11 +558,13 @@ static void cc_setup_cipher_desc(struct crypto_tfm *tfm, du_size = cc_alg->data_unit; switch (cipher_mode) { + case DRV_CIPHER_ECB: + break; case DRV_CIPHER_CBC: case DRV_CIPHER_CBC_CTS: case DRV_CIPHER_CTR: case DRV_CIPHER_OFB: - /* Load cipher state */ + /* Load IV */ hw_desc_init(&desc[*seq_size]); set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, ivsize, NS_BIT); @@ -441,57 +578,15 @@ static void cc_setup_cipher_desc(struct crypto_tfm *tfm, set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE0); } (*seq_size)++; - /*FALLTHROUGH*/ - case DRV_CIPHER_ECB: - /* Load key */ - hw_desc_init(&desc[*seq_size]); - set_cipher_mode(&desc[*seq_size], cipher_mode); - set_cipher_config0(&desc[*seq_size], direction); - if (flow_mode == S_DIN_to_AES) { - if (cc_is_hw_key(tfm)) { - set_hw_crypto_key(&desc[*seq_size], - ctx_p->hw.key1_slot); - } else { - set_din_type(&desc[*seq_size], DMA_DLLI, - key_dma_addr, ((key_len == 24) ? - AES_MAX_KEY_SIZE : - key_len), NS_BIT); - } - set_key_size_aes(&desc[*seq_size], key_len); - } else { - /*des*/ - set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr, - key_len, NS_BIT); - set_key_size_des(&desc[*seq_size], key_len); - } - set_flow_mode(&desc[*seq_size], flow_mode); - set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); - (*seq_size)++; break; case DRV_CIPHER_XTS: case DRV_CIPHER_ESSIV: case DRV_CIPHER_BITLOCKER: - /* Load AES key */ - hw_desc_init(&desc[*seq_size]); - set_cipher_mode(&desc[*seq_size], cipher_mode); - set_cipher_config0(&desc[*seq_size], direction); - if (cc_is_hw_key(tfm)) { - set_hw_crypto_key(&desc[*seq_size], - ctx_p->hw.key1_slot); - } else { - set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr, - (key_len / 2), NS_BIT); - } - set_key_size_aes(&desc[*seq_size], (key_len / 2)); - set_flow_mode(&desc[*seq_size], flow_mode); - set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); - (*seq_size)++; - /* load XEX key */ hw_desc_init(&desc[*seq_size]); set_cipher_mode(&desc[*seq_size], cipher_mode); set_cipher_config0(&desc[*seq_size], direction); - if (cc_is_hw_key(tfm)) { + if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { set_hw_crypto_key(&desc[*seq_size], ctx_p->hw.key2_slot); } else { @@ -505,7 +600,7 @@ static void cc_setup_cipher_desc(struct crypto_tfm *tfm, set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY); (*seq_size)++; - /* Set state */ + /* Load IV */ hw_desc_init(&desc[*seq_size]); set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); set_cipher_mode(&desc[*seq_size], cipher_mode); @@ -521,48 +616,113 @@ static void cc_setup_cipher_desc(struct crypto_tfm *tfm, } } -static void cc_setup_cipher_data(struct crypto_tfm *tfm, - struct cipher_req_ctx *req_ctx, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes, - void *areq, struct cc_hw_desc desc[], - unsigned int *seq_size) +static int cc_out_flow_mode(struct cc_cipher_ctx *ctx_p) { - struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); - struct device *dev = drvdata_to_dev(ctx_p->drvdata); - unsigned int flow_mode = ctx_p->flow_mode; - switch (ctx_p->flow_mode) { case S_DIN_to_AES: - flow_mode = DIN_AES_DOUT; - break; + return DIN_AES_DOUT; case S_DIN_to_DES: - flow_mode = DIN_DES_DOUT; - break; + return DIN_DES_DOUT; case S_DIN_to_SM4: - flow_mode = DIN_SM4_DOUT; - break; + return DIN_SM4_DOUT; default: - dev_err(dev, "invalid flow mode, flow_mode = %d\n", flow_mode); - return; + return ctx_p->flow_mode; } - /* Process */ - if (req_ctx->dma_buf_type == CC_DMA_BUF_DLLI) { - dev_dbg(dev, " data params addr %pad length 0x%X\n", - &sg_dma_address(src), nbytes); - dev_dbg(dev, " data params addr %pad length 0x%X\n", - &sg_dma_address(dst), nbytes); +} + +static void cc_setup_key_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + unsigned int nbytes, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + int cipher_mode = ctx_p->cipher_mode; + int flow_mode = ctx_p->flow_mode; + int direction = req_ctx->gen_ctx.op_type; + dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; + unsigned int key_len = ctx_p->keylen; + unsigned int din_size; + + switch (cipher_mode) { + case DRV_CIPHER_CBC: + case DRV_CIPHER_CBC_CTS: + case DRV_CIPHER_CTR: + case DRV_CIPHER_OFB: + case DRV_CIPHER_ECB: + /* Load key */ hw_desc_init(&desc[*seq_size]); - set_din_type(&desc[*seq_size], DMA_DLLI, sg_dma_address(src), - nbytes, NS_BIT); - set_dout_dlli(&desc[*seq_size], sg_dma_address(dst), - nbytes, NS_BIT, (!areq ? 0 : 1)); - if (areq) - set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); + set_cipher_mode(&desc[*seq_size], cipher_mode); + set_cipher_config0(&desc[*seq_size], direction); + if (cc_key_type(tfm) == CC_POLICY_PROTECTED_KEY) { + /* We use the AES key size coding for all CPP algs */ + set_key_size_aes(&desc[*seq_size], key_len); + set_cpp_crypto_key(&desc[*seq_size], ctx_p->cpp.slot); + flow_mode = cc_out_flow_mode(ctx_p); + } else { + if (flow_mode == S_DIN_to_AES) { + if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { + set_hw_crypto_key(&desc[*seq_size], + ctx_p->hw.key1_slot); + } else { + /* CC_POLICY_UNPROTECTED_KEY + * Invalid keys are filtered out in + * sethkey() + */ + din_size = (key_len == 24) ? + AES_MAX_KEY_SIZE : key_len; + + set_din_type(&desc[*seq_size], DMA_DLLI, + key_dma_addr, din_size, + NS_BIT); + } + set_key_size_aes(&desc[*seq_size], key_len); + } else { + /*des*/ + set_din_type(&desc[*seq_size], DMA_DLLI, + key_dma_addr, key_len, NS_BIT); + set_key_size_des(&desc[*seq_size], key_len); + } + set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); + } set_flow_mode(&desc[*seq_size], flow_mode); (*seq_size)++; - } else { + break; + case DRV_CIPHER_XTS: + case DRV_CIPHER_ESSIV: + case DRV_CIPHER_BITLOCKER: + /* Load AES key */ + hw_desc_init(&desc[*seq_size]); + set_cipher_mode(&desc[*seq_size], cipher_mode); + set_cipher_config0(&desc[*seq_size], direction); + if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { + set_hw_crypto_key(&desc[*seq_size], + ctx_p->hw.key1_slot); + } else { + set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr, + (key_len / 2), NS_BIT); + } + set_key_size_aes(&desc[*seq_size], (key_len / 2)); + set_flow_mode(&desc[*seq_size], flow_mode); + set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); + (*seq_size)++; + break; + default: + dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); + } +} + +static void cc_setup_mlli_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes, void *areq, + struct cc_hw_desc desc[], unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + + if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { /* bypass */ dev_dbg(dev, " bypass params addr %pad length 0x%X addr 0x%08X\n", &req_ctx->mlli_params.mlli_dma_addr, @@ -577,7 +737,38 @@ static void cc_setup_cipher_data(struct crypto_tfm *tfm, req_ctx->mlli_params.mlli_len); set_flow_mode(&desc[*seq_size], BYPASS); (*seq_size)++; + } +} + +static void cc_setup_flow_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes, struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + unsigned int flow_mode = cc_out_flow_mode(ctx_p); + bool last_desc = (ctx_p->key_type == CC_POLICY_PROTECTED_KEY || + ctx_p->cipher_mode == DRV_CIPHER_ECB); + /* Process */ + if (req_ctx->dma_buf_type == CC_DMA_BUF_DLLI) { + dev_dbg(dev, " data params addr %pad length 0x%X\n", + &sg_dma_address(src), nbytes); + dev_dbg(dev, " data params addr %pad length 0x%X\n", + &sg_dma_address(dst), nbytes); + hw_desc_init(&desc[*seq_size]); + set_din_type(&desc[*seq_size], DMA_DLLI, sg_dma_address(src), + nbytes, NS_BIT); + set_dout_dlli(&desc[*seq_size], sg_dma_address(dst), + nbytes, NS_BIT, (!last_desc ? 0 : 1)); + if (last_desc) + set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); + + set_flow_mode(&desc[*seq_size], flow_mode); + (*seq_size)++; + } else { hw_desc_init(&desc[*seq_size]); set_din_type(&desc[*seq_size], DMA_MLLI, ctx_p->drvdata->mlli_sram_addr, @@ -589,7 +780,7 @@ static void cc_setup_cipher_data(struct crypto_tfm *tfm, set_dout_mlli(&desc[*seq_size], ctx_p->drvdata->mlli_sram_addr, req_ctx->in_mlli_nents, NS_BIT, - (!areq ? 0 : 1)); + (!last_desc ? 0 : 1)); } else { dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", (unsigned int)ctx_p->drvdata->mlli_sram_addr, @@ -600,9 +791,9 @@ static void cc_setup_cipher_data(struct crypto_tfm *tfm, (LLI_ENTRY_BYTE_SIZE * req_ctx->in_mlli_nents)), req_ctx->out_mlli_nents, NS_BIT, - (!areq ? 0 : 1)); + (!last_desc ? 0 : 1)); } - if (areq) + if (last_desc) set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); set_flow_mode(&desc[*seq_size], flow_mode); @@ -610,38 +801,6 @@ static void cc_setup_cipher_data(struct crypto_tfm *tfm, } } -/* - * Update a CTR-AES 128 bit counter - */ -static void cc_update_ctr(u8 *ctr, unsigned int increment) -{ - if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || - IS_ALIGNED((unsigned long)ctr, 8)) { - - __be64 *high_be = (__be64 *)ctr; - __be64 *low_be = high_be + 1; - u64 orig_low = __be64_to_cpu(*low_be); - u64 new_low = orig_low + (u64)increment; - - *low_be = __cpu_to_be64(new_low); - - if (new_low < orig_low) - *high_be = __cpu_to_be64(__be64_to_cpu(*high_be) + 1); - } else { - u8 *pos = (ctr + AES_BLOCK_SIZE); - u8 val; - unsigned int size; - - for (; increment; increment--) - for (size = AES_BLOCK_SIZE; size; size--) { - val = *--pos + 1; - *pos = val; - if (val) - break; - } - } -} - static void cc_cipher_complete(struct device *dev, void *cc_req, int err) { struct skcipher_request *req = (struct skcipher_request *)cc_req; @@ -649,44 +808,15 @@ static void cc_cipher_complete(struct device *dev, void *cc_req, int err) struct scatterlist *src = req->src; struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); - struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm); - struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); - unsigned int len; - cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); - - switch (ctx_p->cipher_mode) { - case DRV_CIPHER_CBC: - /* - * The crypto API expects us to set the req->iv to the last - * ciphertext block. For encrypt, simply copy from the result. - * For decrypt, we must copy from a saved buffer since this - * could be an in-place decryption operation and the src is - * lost by this point. - */ - if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { - memcpy(req->iv, req_ctx->backup_info, ivsize); - kzfree(req_ctx->backup_info); - } else if (!err) { - len = req->cryptlen - ivsize; - scatterwalk_map_and_copy(req->iv, req->dst, len, - ivsize, 0); - } - break; - - case DRV_CIPHER_CTR: - /* Compute the counter of the last block */ - len = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / AES_BLOCK_SIZE; - cc_update_ctr((u8 *)req->iv, len); - break; - - default: - break; + if (err != -EINPROGRESS) { + /* Not a BACKLOG notification */ + cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); + memcpy(req->iv, req_ctx->iv, ivsize); + kzfree(req_ctx->iv); } - kzfree(req_ctx->iv); - skcipher_request_complete(req, err); } @@ -741,6 +871,13 @@ static int cc_cipher_process(struct skcipher_request *req, cc_req.user_cb = (void *)cc_cipher_complete; cc_req.user_arg = (void *)req; + /* Setup CPP operation details */ + if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) { + cc_req.cpp.is_cpp = true; + cc_req.cpp.alg = ctx_p->cpp.alg; + cc_req.cpp.slot = ctx_p->cpp.slot; + } + /* Setup request context */ req_ctx->gen_ctx.op_type = direction; @@ -755,11 +892,16 @@ static int cc_cipher_process(struct skcipher_request *req, /* STAT_PHASE_2: Create sequence */ - /* Setup processing */ - cc_setup_cipher_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); + /* Setup IV and XEX key used */ + cc_setup_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); + /* Setup MLLI line, if needed */ + cc_setup_mlli_desc(tfm, req_ctx, dst, src, nbytes, req, desc, &seq_len); + /* Setup key */ + cc_setup_key_desc(tfm, req_ctx, nbytes, desc, &seq_len); /* Data processing */ - cc_setup_cipher_data(tfm, req_ctx, dst, src, nbytes, req, desc, - &seq_len); + cc_setup_flow_desc(tfm, req_ctx, dst, src, nbytes, desc, &seq_len); + /* Read next IV */ + cc_setup_readiv_desc(tfm, req_ctx, ivsize, desc, &seq_len); /* STAT_PHASE_3: Lock HW and push sequence */ @@ -774,7 +916,6 @@ static int cc_cipher_process(struct skcipher_request *req, exit_process: if (rc != -EINPROGRESS && rc != -EBUSY) { - kzfree(req_ctx->backup_info); kzfree(req_ctx->iv); } @@ -792,31 +933,10 @@ static int cc_cipher_encrypt(struct skcipher_request *req) static int cc_cipher_decrypt(struct skcipher_request *req) { - struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); - struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm); - struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); - unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); - gfp_t flags = cc_gfp_flags(&req->base); - unsigned int len; memset(req_ctx, 0, sizeof(*req_ctx)); - if ((ctx_p->cipher_mode == DRV_CIPHER_CBC) && - (req->cryptlen >= ivsize)) { - - /* Allocate and save the last IV sized bytes of the source, - * which will be lost in case of in-place decryption. - */ - req_ctx->backup_info = kzalloc(ivsize, flags); - if (!req_ctx->backup_info) - return -ENOMEM; - - len = req->cryptlen - ivsize; - scatterwalk_map_and_copy(req_ctx->backup_info, req->src, len, - ivsize, 0); - } - return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_DECRYPT); } @@ -838,6 +958,7 @@ static const struct cc_alg_template skcipher_algs[] = { .flow_mode = S_DIN_to_AES, .min_hw_rev = CC_HW_REV_630, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "xts512(paes)", @@ -856,6 +977,7 @@ static const struct cc_alg_template skcipher_algs[] = { .data_unit = 512, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "xts4096(paes)", @@ -874,6 +996,7 @@ static const struct cc_alg_template skcipher_algs[] = { .data_unit = 4096, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "essiv(paes)", @@ -891,6 +1014,7 @@ static const struct cc_alg_template skcipher_algs[] = { .flow_mode = S_DIN_to_AES, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "essiv512(paes)", @@ -909,6 +1033,7 @@ static const struct cc_alg_template skcipher_algs[] = { .data_unit = 512, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "essiv4096(paes)", @@ -927,6 +1052,7 @@ static const struct cc_alg_template skcipher_algs[] = { .data_unit = 4096, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "bitlocker(paes)", @@ -944,6 +1070,7 @@ static const struct cc_alg_template skcipher_algs[] = { .flow_mode = S_DIN_to_AES, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "bitlocker512(paes)", @@ -962,6 +1089,7 @@ static const struct cc_alg_template skcipher_algs[] = { .data_unit = 512, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "bitlocker4096(paes)", @@ -980,6 +1108,7 @@ static const struct cc_alg_template skcipher_algs[] = { .data_unit = 4096, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "ecb(paes)", @@ -997,6 +1126,7 @@ static const struct cc_alg_template skcipher_algs[] = { .flow_mode = S_DIN_to_AES, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "cbc(paes)", @@ -1014,6 +1144,7 @@ static const struct cc_alg_template skcipher_algs[] = { .flow_mode = S_DIN_to_AES, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "ofb(paes)", @@ -1031,6 +1162,7 @@ static const struct cc_alg_template skcipher_algs[] = { .flow_mode = S_DIN_to_AES, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "cts(cbc(paes))", @@ -1048,6 +1180,7 @@ static const struct cc_alg_template skcipher_algs[] = { .flow_mode = S_DIN_to_AES, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "ctr(paes)", @@ -1065,6 +1198,7 @@ static const struct cc_alg_template skcipher_algs[] = { .flow_mode = S_DIN_to_AES, .min_hw_rev = CC_HW_REV_712, .std_body = CC_STD_NIST, + .sec_func = true, }, { .name = "xts(aes)", @@ -1429,6 +1563,42 @@ static const struct cc_alg_template skcipher_algs[] = { .min_hw_rev = CC_HW_REV_713, .std_body = CC_STD_OSCCA, }, + { + .name = "cbc(psm4)", + .driver_name = "cbc-psm4-ccree", + .blocksize = SM4_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_sethkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = CC_HW_KEY_SIZE, + .max_keysize = CC_HW_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CBC, + .flow_mode = S_DIN_to_SM4, + .min_hw_rev = CC_HW_REV_713, + .std_body = CC_STD_OSCCA, + .sec_func = true, + }, + { + .name = "ctr(psm4)", + .driver_name = "ctr-psm4-ccree", + .blocksize = SM4_BLOCK_SIZE, + .template_skcipher = { + .setkey = cc_cipher_sethkey, + .encrypt = cc_cipher_encrypt, + .decrypt = cc_cipher_decrypt, + .min_keysize = CC_HW_KEY_SIZE, + .max_keysize = CC_HW_KEY_SIZE, + .ivsize = SM4_BLOCK_SIZE, + }, + .cipher_mode = DRV_CIPHER_CTR, + .flow_mode = S_DIN_to_SM4, + .min_hw_rev = CC_HW_REV_713, + .std_body = CC_STD_OSCCA, + .sec_func = true, + }, }; static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, @@ -1504,7 +1674,8 @@ int cc_cipher_alloc(struct cc_drvdata *drvdata) ARRAY_SIZE(skcipher_algs)); for (alg = 0; alg < ARRAY_SIZE(skcipher_algs); alg++) { if ((skcipher_algs[alg].min_hw_rev > drvdata->hw_rev) || - !(drvdata->std_bodies & skcipher_algs[alg].std_body)) + !(drvdata->std_bodies & skcipher_algs[alg].std_body) || + (drvdata->sec_disabled && skcipher_algs[alg].sec_func)) continue; dev_dbg(dev, "creating %s\n", skcipher_algs[alg].driver_name); diff --git a/drivers/crypto/ccree/cc_cipher.h b/drivers/crypto/ccree/cc_cipher.h index 4dbc0a1e6d5c..da3a38707fae 100644 --- a/drivers/crypto/ccree/cc_cipher.h +++ b/drivers/crypto/ccree/cc_cipher.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ /* \file cc_cipher.h * ARM CryptoCell Cipher Crypto API @@ -20,7 +20,6 @@ struct cipher_req_ctx { u32 in_mlli_nents; u32 out_nents; u32 out_mlli_nents; - u8 *backup_info; /*store iv for generated IV flow*/ u8 *iv; struct mlli_params mlli_params; }; diff --git a/drivers/crypto/ccree/cc_crypto_ctx.h b/drivers/crypto/ccree/cc_crypto_ctx.h index c8dac273c563..ccf960a0d989 100644 --- a/drivers/crypto/ccree/cc_crypto_ctx.h +++ b/drivers/crypto/ccree/cc_crypto_ctx.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #ifndef _CC_CRYPTO_CTX_H_ #define _CC_CRYPTO_CTX_H_ @@ -55,6 +55,14 @@ #define CC_DRV_ALG_MAX_BLOCK_SIZE CC_HASH_BLOCK_SIZE_MAX +#define CC_CPP_NUM_SLOTS 8 +#define CC_CPP_NUM_ALGS 2 + +enum cc_cpp_alg { + CC_CPP_SM4 = 1, + CC_CPP_AES = 0 +}; + enum drv_engine_type { DRV_ENGINE_NULL = 0, DRV_ENGINE_AES = 1, diff --git a/drivers/crypto/ccree/cc_debugfs.c b/drivers/crypto/ccree/cc_debugfs.c index 5fa05a7bcf36..566999738698 100644 --- a/drivers/crypto/ccree/cc_debugfs.c +++ b/drivers/crypto/ccree/cc_debugfs.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited or its affiliates. */ #include <linux/kernel.h> #include <linux/debugfs.h> @@ -25,9 +25,24 @@ struct cc_debugfs_ctx { */ static struct dentry *cc_debugfs_dir; -static struct debugfs_reg32 debug_regs[] = { +static struct debugfs_reg32 ver_sig_regs[] = { { .name = "SIGNATURE" }, /* Must be 0th */ { .name = "VERSION" }, /* Must be 1st */ +}; + +static struct debugfs_reg32 pid_cid_regs[] = { + CC_DEBUG_REG(PERIPHERAL_ID_0), + CC_DEBUG_REG(PERIPHERAL_ID_1), + CC_DEBUG_REG(PERIPHERAL_ID_2), + CC_DEBUG_REG(PERIPHERAL_ID_3), + CC_DEBUG_REG(PERIPHERAL_ID_4), + CC_DEBUG_REG(COMPONENT_ID_0), + CC_DEBUG_REG(COMPONENT_ID_1), + CC_DEBUG_REG(COMPONENT_ID_2), + CC_DEBUG_REG(COMPONENT_ID_3), +}; + +static struct debugfs_reg32 debug_regs[] = { CC_DEBUG_REG(HOST_IRR), CC_DEBUG_REG(HOST_POWER_DOWN_EN), CC_DEBUG_REG(AXIM_MON_ERR), @@ -53,10 +68,7 @@ int cc_debugfs_init(struct cc_drvdata *drvdata) { struct device *dev = drvdata_to_dev(drvdata); struct cc_debugfs_ctx *ctx; - struct debugfs_regset32 *regset; - - debug_regs[0].offset = drvdata->sig_offset; - debug_regs[1].offset = drvdata->ver_offset; + struct debugfs_regset32 *regset, *verset; ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); if (!ctx) @@ -75,8 +87,26 @@ int cc_debugfs_init(struct cc_drvdata *drvdata) debugfs_create_regset32("regs", 0400, ctx->dir, regset); debugfs_create_bool("coherent", 0400, ctx->dir, &drvdata->coherent); - drvdata->debugfs = ctx; + verset = devm_kzalloc(dev, sizeof(*verset), GFP_KERNEL); + /* Failing here is not important enough to fail the module load */ + if (!verset) + goto out; + + if (drvdata->hw_rev <= CC_HW_REV_712) { + ver_sig_regs[0].offset = drvdata->sig_offset; + ver_sig_regs[1].offset = drvdata->ver_offset; + verset->regs = ver_sig_regs; + verset->nregs = ARRAY_SIZE(ver_sig_regs); + } else { + verset->regs = pid_cid_regs; + verset->nregs = ARRAY_SIZE(pid_cid_regs); + } + verset->base = drvdata->cc_base; + debugfs_create_regset32("version", 0400, ctx->dir, verset); + +out: + drvdata->debugfs = ctx; return 0; } diff --git a/drivers/crypto/ccree/cc_debugfs.h b/drivers/crypto/ccree/cc_debugfs.h index 01cbd9a95659..664ff402e0e9 100644 --- a/drivers/crypto/ccree/cc_debugfs.h +++ b/drivers/crypto/ccree/cc_debugfs.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #ifndef __CC_DEBUGFS_H__ #define __CC_DEBUGFS_H__ diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 3bcc6c76e090..86ac7b443355 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited or its affiliates. */ #include <linux/kernel.h> #include <linux/module.h> @@ -30,27 +30,47 @@ bool cc_dump_desc; module_param_named(dump_desc, cc_dump_desc, bool, 0600); MODULE_PARM_DESC(cc_dump_desc, "Dump descriptors to kernel log as debugging aid"); - bool cc_dump_bytes; module_param_named(dump_bytes, cc_dump_bytes, bool, 0600); MODULE_PARM_DESC(cc_dump_bytes, "Dump buffers to kernel log as debugging aid"); +static bool cc_sec_disable; +module_param_named(sec_disable, cc_sec_disable, bool, 0600); +MODULE_PARM_DESC(cc_sec_disable, "Disable security functions"); + struct cc_hw_data { char *name; enum cc_hw_rev rev; u32 sig; + u32 cidr_0123; + u32 pidr_0124; int std_bodies; }; +#define CC_NUM_IDRS 4 + +/* Note: PIDR3 holds CMOD/Rev so ignored for HW identification purposes */ +static const u32 pidr_0124_offsets[CC_NUM_IDRS] = { + CC_REG(PERIPHERAL_ID_0), CC_REG(PERIPHERAL_ID_1), + CC_REG(PERIPHERAL_ID_2), CC_REG(PERIPHERAL_ID_4) +}; + +static const u32 cidr_0123_offsets[CC_NUM_IDRS] = { + CC_REG(COMPONENT_ID_0), CC_REG(COMPONENT_ID_1), + CC_REG(COMPONENT_ID_2), CC_REG(COMPONENT_ID_3) +}; + /* Hardware revisions defs. */ /* The 703 is a OSCCA only variant of the 713 */ static const struct cc_hw_data cc703_hw = { - .name = "703", .rev = CC_HW_REV_713, .std_bodies = CC_STD_OSCCA + .name = "703", .rev = CC_HW_REV_713, .cidr_0123 = 0xB105F00DU, + .pidr_0124 = 0x040BB0D0U, .std_bodies = CC_STD_OSCCA }; static const struct cc_hw_data cc713_hw = { - .name = "713", .rev = CC_HW_REV_713, .std_bodies = CC_STD_ALL + .name = "713", .rev = CC_HW_REV_713, .cidr_0123 = 0xB105F00DU, + .pidr_0124 = 0x040BB0D0U, .std_bodies = CC_STD_ALL }; static const struct cc_hw_data cc712_hw = { @@ -78,6 +98,20 @@ static const struct of_device_id arm_ccree_dev_of_match[] = { }; MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match); +static u32 cc_read_idr(struct cc_drvdata *drvdata, const u32 *idr_offsets) +{ + int i; + union { + u8 regs[CC_NUM_IDRS]; + __le32 val; + } idr; + + for (i = 0; i < CC_NUM_IDRS; ++i) + idr.regs[i] = cc_ioread(drvdata, idr_offsets[i]); + + return le32_to_cpu(idr.val); +} + void __dump_byte_array(const char *name, const u8 *buf, size_t len) { char prefix[64]; @@ -114,12 +148,12 @@ static irqreturn_t cc_isr(int irq, void *dev_id) drvdata->irq = irr; /* Completion interrupt - most probable */ - if (irr & CC_COMP_IRQ_MASK) { - /* Mask AXI completion interrupt - will be unmasked in - * Deferred service handler + if (irr & drvdata->comp_mask) { + /* Mask all completion interrupts - will be unmasked in + * deferred service handler */ - cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | CC_COMP_IRQ_MASK); - irr &= ~CC_COMP_IRQ_MASK; + cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | drvdata->comp_mask); + irr &= ~drvdata->comp_mask; complete_request(drvdata); } #ifdef CONFIG_CRYPTO_FIPS @@ -159,11 +193,14 @@ int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe) unsigned int val, cache_params; struct device *dev = drvdata_to_dev(drvdata); - /* Unmask all AXI interrupt sources AXI_CFG1 register */ - val = cc_ioread(drvdata, CC_REG(AXIM_CFG)); - cc_iowrite(drvdata, CC_REG(AXIM_CFG), val & ~CC_AXI_IRQ_MASK); - dev_dbg(dev, "AXIM_CFG=0x%08X\n", - cc_ioread(drvdata, CC_REG(AXIM_CFG))); + /* Unmask all AXI interrupt sources AXI_CFG1 register */ + /* AXI interrupt config are obsoleted startign at cc7x3 */ + if (drvdata->hw_rev <= CC_HW_REV_712) { + val = cc_ioread(drvdata, CC_REG(AXIM_CFG)); + cc_iowrite(drvdata, CC_REG(AXIM_CFG), val & ~CC_AXI_IRQ_MASK); + dev_dbg(dev, "AXIM_CFG=0x%08X\n", + cc_ioread(drvdata, CC_REG(AXIM_CFG))); + } /* Clear all pending interrupts */ val = cc_ioread(drvdata, CC_REG(HOST_IRR)); @@ -171,7 +208,7 @@ int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe) cc_iowrite(drvdata, CC_REG(HOST_ICR), val); /* Unmask relevant interrupt cause */ - val = CC_COMP_IRQ_MASK | CC_AXI_ERR_IRQ_MASK; + val = drvdata->comp_mask | CC_AXI_ERR_IRQ_MASK; if (drvdata->hw_rev >= CC_HW_REV_712) val |= CC_GPR0_IRQ_MASK; @@ -201,7 +238,7 @@ static int init_cc_resources(struct platform_device *plat_dev) struct cc_drvdata *new_drvdata; struct device *dev = &plat_dev->dev; struct device_node *np = dev->of_node; - u32 signature_val; + u32 val, hw_rev_pidr, sig_cidr; u64 dma_mask; const struct cc_hw_data *hw_rev; const struct of_device_id *dev_id; @@ -231,6 +268,8 @@ static int init_cc_resources(struct platform_device *plat_dev) new_drvdata->ver_offset = CC_REG(HOST_VERSION_630); } + new_drvdata->comp_mask = CC_COMP_IRQ_MASK; + platform_set_drvdata(plat_dev, new_drvdata); new_drvdata->plat_dev = plat_dev; @@ -311,22 +350,57 @@ static int init_cc_resources(struct platform_device *plat_dev) return rc; } + new_drvdata->sec_disabled = cc_sec_disable; + if (hw_rev->rev <= CC_HW_REV_712) { /* Verify correct mapping */ - signature_val = cc_ioread(new_drvdata, new_drvdata->sig_offset); - if (signature_val != hw_rev->sig) { + val = cc_ioread(new_drvdata, new_drvdata->sig_offset); + if (val != hw_rev->sig) { dev_err(dev, "Invalid CC signature: SIGNATURE=0x%08X != expected=0x%08X\n", - signature_val, hw_rev->sig); + val, hw_rev->sig); + rc = -EINVAL; + goto post_clk_err; + } + sig_cidr = val; + hw_rev_pidr = cc_ioread(new_drvdata, new_drvdata->ver_offset); + } else { + /* Verify correct mapping */ + val = cc_read_idr(new_drvdata, pidr_0124_offsets); + if (val != hw_rev->pidr_0124) { + dev_err(dev, "Invalid CC PIDR: PIDR0124=0x%08X != expected=0x%08X\n", + val, hw_rev->pidr_0124); + rc = -EINVAL; + goto post_clk_err; + } + hw_rev_pidr = val; + + val = cc_read_idr(new_drvdata, cidr_0123_offsets); + if (val != hw_rev->cidr_0123) { + dev_err(dev, "Invalid CC CIDR: CIDR0123=0x%08X != expected=0x%08X\n", + val, hw_rev->cidr_0123); rc = -EINVAL; goto post_clk_err; } - dev_dbg(dev, "CC SIGNATURE=0x%08X\n", signature_val); + sig_cidr = val; + + /* Check security disable state */ + val = cc_ioread(new_drvdata, CC_REG(SECURITY_DISABLED)); + val &= CC_SECURITY_DISABLED_MASK; + new_drvdata->sec_disabled |= !!val; + + if (!new_drvdata->sec_disabled) { + new_drvdata->comp_mask |= CC_CPP_SM4_ABORT_MASK; + if (new_drvdata->std_bodies & CC_STD_NIST) + new_drvdata->comp_mask |= CC_CPP_AES_ABORT_MASK; + } } + if (new_drvdata->sec_disabled) + dev_info(dev, "Security Disabled mode is in effect. Security functions disabled.\n"); + /* Display HW versions */ - dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X, Driver version %s\n", - hw_rev->name, cc_ioread(new_drvdata, new_drvdata->ver_offset), - DRV_MODULE_VERSION); + dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X/0x%8X, Driver version %s\n", + hw_rev->name, hw_rev_pidr, sig_cidr, DRV_MODULE_VERSION); rc = init_cc_regs(new_drvdata, true); if (rc) { diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index 33dbf3e6d15d..b76181335c08 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ /* \file cc_driver.h * ARM CryptoCell Linux Crypto Driver @@ -65,10 +65,32 @@ enum cc_std_body { #define CC_COMP_IRQ_MASK BIT(CC_HOST_IRR_AXIM_COMP_INT_BIT_SHIFT) +#define CC_SECURITY_DISABLED_MASK BIT(CC_SECURITY_DISABLED_VALUE_BIT_SHIFT) + #define AXIM_MON_COMP_VALUE GENMASK(CC_AXIM_MON_COMP_VALUE_BIT_SIZE + \ CC_AXIM_MON_COMP_VALUE_BIT_SHIFT, \ CC_AXIM_MON_COMP_VALUE_BIT_SHIFT) +#define CC_CPP_AES_ABORT_MASK ( \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_0_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_1_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_2_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_3_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_4_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_5_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_6_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_AES_7_MASK_BIT_SHIFT)) + +#define CC_CPP_SM4_ABORT_MASK ( \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_0_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_1_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_2_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_3_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_4_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_5_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_6_MASK_BIT_SHIFT) | \ + BIT(CC_HOST_IMR_REE_OP_ABORTED_SM_7_MASK_BIT_SHIFT)) + /* Register name mangling macro */ #define CC_REG(reg_name) CC_ ## reg_name ## _REG_OFFSET @@ -81,7 +103,6 @@ enum cc_std_body { #define MAX_REQUEST_QUEUE_SIZE 4096 #define MAX_MLLI_BUFF_SIZE 2080 -#define MAX_ICV_NENTS_SUPPORTED 2 /* Definitions for HW descriptors DIN/DOUT fields */ #define NS_BIT 1 @@ -90,6 +111,12 @@ enum cc_std_body { * field in the HW descriptor. The DMA engine +8 that value. */ +struct cc_cpp_req { + bool is_cpp; + enum cc_cpp_alg alg; + u8 slot; +}; + #define CC_MAX_IVGEN_DMA_ADDRESSES 3 struct cc_crypto_req { void (*user_cb)(struct device *dev, void *req, int err); @@ -104,6 +131,7 @@ struct cc_crypto_req { /* The generated IV size required, 8/16 B allowed. */ unsigned int ivgen_size; struct completion seq_compl; /* request completion */ + struct cc_cpp_req cpp; }; /** @@ -136,6 +164,8 @@ struct cc_drvdata { u32 sig_offset; u32 ver_offset; int std_bodies; + bool sec_disabled; + u32 comp_mask; }; struct cc_crypto_alg { @@ -162,12 +192,14 @@ struct cc_alg_template { int auth_mode; u32 min_hw_rev; enum cc_std_body std_body; + bool sec_func; unsigned int data_unit; struct cc_drvdata *drvdata; }; struct async_gen_req_ctx { dma_addr_t iv_dma_addr; + u8 *iv; enum drv_crypto_direction op_type; }; diff --git a/drivers/crypto/ccree/cc_fips.c b/drivers/crypto/ccree/cc_fips.c index b4d0a6d983e0..5ad3ffb7acaa 100644 --- a/drivers/crypto/ccree/cc_fips.c +++ b/drivers/crypto/ccree/cc_fips.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #include <linux/kernel.h> #include <linux/fips.h> @@ -49,8 +49,6 @@ void cc_fips_fini(struct cc_drvdata *drvdata) /* Kill tasklet */ tasklet_kill(&fips_h->tasklet); - - kfree(fips_h); drvdata->fips_handle = NULL; } @@ -72,20 +70,28 @@ static inline void tee_fips_error(struct device *dev) dev_err(dev, "TEE reported error!\n"); } +/* + * This function check if cryptocell tee fips error occurred + * and in such case triggers system error + */ +void cc_tee_handle_fips_error(struct cc_drvdata *p_drvdata) +{ + struct device *dev = drvdata_to_dev(p_drvdata); + + if (!cc_get_tee_fips_status(p_drvdata)) + tee_fips_error(dev); +} + /* Deferred service handler, run as interrupt-fired tasklet */ static void fips_dsr(unsigned long devarg) { struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg; - struct device *dev = drvdata_to_dev(drvdata); - u32 irq, state, val; + u32 irq, val; irq = (drvdata->irq & (CC_GPR0_IRQ_MASK)); if (irq) { - state = cc_ioread(drvdata, CC_REG(GPR_HOST)); - - if (state != (CC_FIPS_SYNC_TEE_STATUS | CC_FIPS_SYNC_MODULE_OK)) - tee_fips_error(dev); + cc_tee_handle_fips_error(drvdata); } /* after verifing that there is nothing to do, @@ -104,7 +110,7 @@ int cc_fips_init(struct cc_drvdata *p_drvdata) if (p_drvdata->hw_rev < CC_HW_REV_712) return 0; - fips_h = kzalloc(sizeof(*fips_h), GFP_KERNEL); + fips_h = devm_kzalloc(dev, sizeof(*fips_h), GFP_KERNEL); if (!fips_h) return -ENOMEM; @@ -113,8 +119,7 @@ int cc_fips_init(struct cc_drvdata *p_drvdata) dev_dbg(dev, "Initializing fips tasklet\n"); tasklet_init(&fips_h->tasklet, fips_dsr, (unsigned long)p_drvdata); - if (!cc_get_tee_fips_status(p_drvdata)) - tee_fips_error(dev); + cc_tee_handle_fips_error(p_drvdata); return 0; } diff --git a/drivers/crypto/ccree/cc_fips.h b/drivers/crypto/ccree/cc_fips.h index 645e096a7a82..fc33eeb4d566 100644 --- a/drivers/crypto/ccree/cc_fips.h +++ b/drivers/crypto/ccree/cc_fips.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #ifndef __CC_FIPS_H__ #define __CC_FIPS_H__ @@ -18,6 +18,7 @@ int cc_fips_init(struct cc_drvdata *p_drvdata); void cc_fips_fini(struct cc_drvdata *drvdata); void fips_handler(struct cc_drvdata *drvdata); void cc_set_ree_fips_status(struct cc_drvdata *drvdata, bool ok); +void cc_tee_handle_fips_error(struct cc_drvdata *p_drvdata); #else /* CONFIG_CRYPTO_FIPS */ @@ -30,6 +31,7 @@ static inline void cc_fips_fini(struct cc_drvdata *drvdata) {} static inline void cc_set_ree_fips_status(struct cc_drvdata *drvdata, bool ok) {} static inline void fips_handler(struct cc_drvdata *drvdata) {} +static inline void cc_tee_handle_fips_error(struct cc_drvdata *p_drvdata) {} #endif /* CONFIG_CRYPTO_FIPS */ diff --git a/drivers/crypto/ccree/cc_hash.c b/drivers/crypto/ccree/cc_hash.c index 2c4ddc8fb76b..a6abe4e3bb0e 100644 --- a/drivers/crypto/ccree/cc_hash.c +++ b/drivers/crypto/ccree/cc_hash.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #include <linux/kernel.h> #include <linux/module.h> @@ -69,6 +69,7 @@ struct cc_hash_alg { struct hash_key_req_ctx { u32 keylen; dma_addr_t key_dma_addr; + u8 *key; }; /* hash per-session context */ @@ -280,9 +281,13 @@ static void cc_update_complete(struct device *dev, void *cc_req, int err) dev_dbg(dev, "req=%pK\n", req); - cc_unmap_hash_request(dev, state, req->src, false); - cc_unmap_req(dev, state, ctx); - req->base.complete(&req->base, err); + if (err != -EINPROGRESS) { + /* Not a BACKLOG notification */ + cc_unmap_hash_request(dev, state, req->src, false); + cc_unmap_req(dev, state, ctx); + } + + ahash_request_complete(req, err); } static void cc_digest_complete(struct device *dev, void *cc_req, int err) @@ -295,10 +300,14 @@ static void cc_digest_complete(struct device *dev, void *cc_req, int err) dev_dbg(dev, "req=%pK\n", req); - cc_unmap_hash_request(dev, state, req->src, false); - cc_unmap_result(dev, state, digestsize, req->result); - cc_unmap_req(dev, state, ctx); - req->base.complete(&req->base, err); + if (err != -EINPROGRESS) { + /* Not a BACKLOG notification */ + cc_unmap_hash_request(dev, state, req->src, false); + cc_unmap_result(dev, state, digestsize, req->result); + cc_unmap_req(dev, state, ctx); + } + + ahash_request_complete(req, err); } static void cc_hash_complete(struct device *dev, void *cc_req, int err) @@ -311,10 +320,14 @@ static void cc_hash_complete(struct device *dev, void *cc_req, int err) dev_dbg(dev, "req=%pK\n", req); - cc_unmap_hash_request(dev, state, req->src, false); - cc_unmap_result(dev, state, digestsize, req->result); - cc_unmap_req(dev, state, ctx); - req->base.complete(&req->base, err); + if (err != -EINPROGRESS) { + /* Not a BACKLOG notification */ + cc_unmap_hash_request(dev, state, req->src, false); + cc_unmap_result(dev, state, digestsize, req->result); + cc_unmap_req(dev, state, ctx); + } + + ahash_request_complete(req, err); } static int cc_fin_result(struct cc_hw_desc *desc, struct ahash_request *req, @@ -730,13 +743,20 @@ static int cc_hash_setkey(struct crypto_ahash *ahash, const u8 *key, ctx->key_params.keylen = keylen; ctx->key_params.key_dma_addr = 0; ctx->is_hmac = true; + ctx->key_params.key = NULL; if (keylen) { + ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL); + if (!ctx->key_params.key) + return -ENOMEM; + ctx->key_params.key_dma_addr = - dma_map_single(dev, (void *)key, keylen, DMA_TO_DEVICE); + dma_map_single(dev, (void *)ctx->key_params.key, keylen, + DMA_TO_DEVICE); if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) { dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", - key, keylen); + ctx->key_params.key, keylen); + kzfree(ctx->key_params.key); return -ENOMEM; } dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n", @@ -887,6 +907,9 @@ out: dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n", &ctx->key_params.key_dma_addr, ctx->key_params.keylen); } + + kzfree(ctx->key_params.key); + return rc; } @@ -913,11 +936,16 @@ static int cc_xcbc_setkey(struct crypto_ahash *ahash, ctx->key_params.keylen = keylen; + ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL); + if (!ctx->key_params.key) + return -ENOMEM; + ctx->key_params.key_dma_addr = - dma_map_single(dev, (void *)key, keylen, DMA_TO_DEVICE); + dma_map_single(dev, ctx->key_params.key, keylen, DMA_TO_DEVICE); if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) { dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", key, keylen); + kzfree(ctx->key_params.key); return -ENOMEM; } dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n", @@ -969,6 +997,8 @@ static int cc_xcbc_setkey(struct crypto_ahash *ahash, dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n", &ctx->key_params.key_dma_addr, ctx->key_params.keylen); + kzfree(ctx->key_params.key); + return rc; } @@ -1621,7 +1651,7 @@ static struct cc_hash_template driver_hash[] = { .setkey = cc_hash_setkey, .halg = { .digestsize = SHA224_DIGEST_SIZE, - .statesize = CC_STATE_SIZE(SHA224_DIGEST_SIZE), + .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE), }, }, .hash_mode = DRV_HASH_SHA224, @@ -1648,7 +1678,7 @@ static struct cc_hash_template driver_hash[] = { .setkey = cc_hash_setkey, .halg = { .digestsize = SHA384_DIGEST_SIZE, - .statesize = CC_STATE_SIZE(SHA384_DIGEST_SIZE), + .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE), }, }, .hash_mode = DRV_HASH_SHA384, diff --git a/drivers/crypto/ccree/cc_hash.h b/drivers/crypto/ccree/cc_hash.h index 2e5bf8b0bbb6..0d6dc61484d7 100644 --- a/drivers/crypto/ccree/cc_hash.h +++ b/drivers/crypto/ccree/cc_hash.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ /* \file cc_hash.h * ARM CryptoCell Hash Crypto API diff --git a/drivers/crypto/ccree/cc_host_regs.h b/drivers/crypto/ccree/cc_host_regs.h index 616b2e1c41ba..d0764147573f 100644 --- a/drivers/crypto/ccree/cc_host_regs.h +++ b/drivers/crypto/ccree/cc_host_regs.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited or its affiliates. */ #ifndef __CC_HOST_H__ #define __CC_HOST_H__ @@ -7,33 +7,102 @@ // -------------------------------------- // BLOCK: HOST_P // -------------------------------------- + + +/* IRR */ #define CC_HOST_IRR_REG_OFFSET 0xA00UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SIZE 0x1UL #define CC_HOST_IRR_DSCRPTR_COMPLETION_LOW_INT_BIT_SHIFT 0x2UL #define CC_HOST_IRR_DSCRPTR_COMPLETION_LOW_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT 0x3UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT 0x4UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT 0x5UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT 0x6UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT 0x7UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SIZE 0x1UL #define CC_HOST_IRR_AXI_ERR_INT_BIT_SHIFT 0x8UL #define CC_HOST_IRR_AXI_ERR_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT 0x9UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT 0xAUL +#define CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SIZE 0x1UL #define CC_HOST_IRR_GPR0_BIT_SHIFT 0xBUL #define CC_HOST_IRR_GPR0_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT 0xCUL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT 0xDUL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT 0xEUL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT 0xFUL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT 0x10UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT 0x11UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT 0x12UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SIZE 0x1UL #define CC_HOST_IRR_DSCRPTR_WATERMARK_INT_BIT_SHIFT 0x13UL #define CC_HOST_IRR_DSCRPTR_WATERMARK_INT_BIT_SIZE 0x1UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT 0x14UL +#define CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SIZE 0x1UL #define CC_HOST_IRR_AXIM_COMP_INT_BIT_SHIFT 0x17UL #define CC_HOST_IRR_AXIM_COMP_INT_BIT_SIZE 0x1UL #define CC_HOST_SEP_SRAM_THRESHOLD_REG_OFFSET 0xA10UL #define CC_HOST_SEP_SRAM_THRESHOLD_VALUE_BIT_SHIFT 0x0UL #define CC_HOST_SEP_SRAM_THRESHOLD_VALUE_BIT_SIZE 0xCUL -#define CC_HOST_IMR_REG_OFFSET 0xA04UL -#define CC_HOST_IMR_NOT_USED_MASK_BIT_SHIFT 0x1UL -#define CC_HOST_IMR_NOT_USED_MASK_BIT_SIZE 0x1UL + +/* IMR */ +#define CC_HOST_IMR_REG_OFFSET 0x0A04UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_0_MASK_BIT_SHIFT 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_0_MASK_BIT_SIZE 0x1UL #define CC_HOST_IMR_DSCRPTR_COMPLETION_MASK_BIT_SHIFT 0x2UL #define CC_HOST_IMR_DSCRPTR_COMPLETION_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_1_MASK_BIT_SHIFT 0x3UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_1_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_2_MASK_BIT_SHIFT 0x4UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_2_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_3_MASK_BIT_SHIFT 0x5UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_3_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_4_MASK_BIT_SHIFT 0x6UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_4_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_5_MASK_BIT_SHIFT 0x7UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_5_MASK_BIT_SIZE 0x1UL #define CC_HOST_IMR_AXI_ERR_MASK_BIT_SHIFT 0x8UL #define CC_HOST_IMR_AXI_ERR_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_6_MASK_BIT_SHIFT 0x9UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_6_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_7_MASK_BIT_SHIFT 0xAUL +#define CC_HOST_IMR_REE_OP_ABORTED_AES_7_MASK_BIT_SIZE 0x1UL #define CC_HOST_IMR_GPR0_BIT_SHIFT 0xBUL #define CC_HOST_IMR_GPR0_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_0_MASK_BIT_SHIFT 0xCUL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_0_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_1_MASK_BIT_SHIFT 0xDUL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_1_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_2_MASK_BIT_SHIFT 0xEUL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_2_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_3_MASK_BIT_SHIFT 0xFUL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_3_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_4_MASK_BIT_SHIFT 0x10UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_4_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_5_MASK_BIT_SHIFT 0x11UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_5_MASK_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_6_MASK_BIT_SHIFT 0x12UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_6_MASK_BIT_SIZE 0x1UL #define CC_HOST_IMR_DSCRPTR_WATERMARK_MASK0_BIT_SHIFT 0x13UL #define CC_HOST_IMR_DSCRPTR_WATERMARK_MASK0_BIT_SIZE 0x1UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_7_MASK_BIT_SHIFT 0x14UL +#define CC_HOST_IMR_REE_OP_ABORTED_SM_7_MASK_BIT_SIZE 0x1UL #define CC_HOST_IMR_AXIM_COMP_INT_MASK_BIT_SHIFT 0x17UL #define CC_HOST_IMR_AXIM_COMP_INT_MASK_BIT_SIZE 0x1UL + +/* ICR */ #define CC_HOST_ICR_REG_OFFSET 0xA08UL #define CC_HOST_ICR_DSCRPTR_COMPLETION_BIT_SHIFT 0x2UL #define CC_HOST_ICR_DSCRPTR_COMPLETION_BIT_SIZE 0x1UL @@ -45,6 +114,9 @@ #define CC_HOST_ICR_DSCRPTR_WATERMARK_QUEUE0_CLEAR_BIT_SIZE 0x1UL #define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SHIFT 0x17UL #define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SIZE 0x1UL +#define CC_SECURITY_DISABLED_REG_OFFSET 0x0A1CUL +#define CC_SECURITY_DISABLED_VALUE_BIT_SHIFT 0x0UL +#define CC_SECURITY_DISABLED_VALUE_BIT_SIZE 0x1UL #define CC_HOST_SIGNATURE_712_REG_OFFSET 0xA24UL #define CC_HOST_SIGNATURE_630_REG_OFFSET 0xAC8UL #define CC_HOST_SIGNATURE_VALUE_BIT_SHIFT 0x0UL @@ -132,6 +204,49 @@ #define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SHIFT 0x0UL #define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SIZE 0x1UL // -------------------------------------- +// BLOCK: ID_REGISTERS +// -------------------------------------- +#define CC_PERIPHERAL_ID_4_REG_OFFSET 0x0FD0UL +#define CC_PERIPHERAL_ID_4_VALUE_BIT_SHIFT 0x0UL +#define CC_PERIPHERAL_ID_4_VALUE_BIT_SIZE 0x4UL +#define CC_PIDRESERVED0_REG_OFFSET 0x0FD4UL +#define CC_PIDRESERVED1_REG_OFFSET 0x0FD8UL +#define CC_PIDRESERVED2_REG_OFFSET 0x0FDCUL +#define CC_PERIPHERAL_ID_0_REG_OFFSET 0x0FE0UL +#define CC_PERIPHERAL_ID_0_VALUE_BIT_SHIFT 0x0UL +#define CC_PERIPHERAL_ID_0_VALUE_BIT_SIZE 0x8UL +#define CC_PERIPHERAL_ID_1_REG_OFFSET 0x0FE4UL +#define CC_PERIPHERAL_ID_1_PART_1_BIT_SHIFT 0x0UL +#define CC_PERIPHERAL_ID_1_PART_1_BIT_SIZE 0x4UL +#define CC_PERIPHERAL_ID_1_DES_0_JEP106_BIT_SHIFT 0x4UL +#define CC_PERIPHERAL_ID_1_DES_0_JEP106_BIT_SIZE 0x4UL +#define CC_PERIPHERAL_ID_2_REG_OFFSET 0x0FE8UL +#define CC_PERIPHERAL_ID_2_DES_1_JEP106_BIT_SHIFT 0x0UL +#define CC_PERIPHERAL_ID_2_DES_1_JEP106_BIT_SIZE 0x3UL +#define CC_PERIPHERAL_ID_2_JEDEC_BIT_SHIFT 0x3UL +#define CC_PERIPHERAL_ID_2_JEDEC_BIT_SIZE 0x1UL +#define CC_PERIPHERAL_ID_2_REVISION_BIT_SHIFT 0x4UL +#define CC_PERIPHERAL_ID_2_REVISION_BIT_SIZE 0x4UL +#define CC_PERIPHERAL_ID_3_REG_OFFSET 0x0FECUL +#define CC_PERIPHERAL_ID_3_CMOD_BIT_SHIFT 0x0UL +#define CC_PERIPHERAL_ID_3_CMOD_BIT_SIZE 0x4UL +#define CC_PERIPHERAL_ID_3_REVAND_BIT_SHIFT 0x4UL +#define CC_PERIPHERAL_ID_3_REVAND_BIT_SIZE 0x4UL +#define CC_COMPONENT_ID_0_REG_OFFSET 0x0FF0UL +#define CC_COMPONENT_ID_0_VALUE_BIT_SHIFT 0x0UL +#define CC_COMPONENT_ID_0_VALUE_BIT_SIZE 0x8UL +#define CC_COMPONENT_ID_1_REG_OFFSET 0x0FF4UL +#define CC_COMPONENT_ID_1_PRMBL_1_BIT_SHIFT 0x0UL +#define CC_COMPONENT_ID_1_PRMBL_1_BIT_SIZE 0x4UL +#define CC_COMPONENT_ID_1_CLASS_BIT_SHIFT 0x4UL +#define CC_COMPONENT_ID_1_CLASS_BIT_SIZE 0x4UL +#define CC_COMPONENT_ID_2_REG_OFFSET 0x0FF8UL +#define CC_COMPONENT_ID_2_VALUE_BIT_SHIFT 0x0UL +#define CC_COMPONENT_ID_2_VALUE_BIT_SIZE 0x8UL +#define CC_COMPONENT_ID_3_REG_OFFSET 0x0FFCUL +#define CC_COMPONENT_ID_3_VALUE_BIT_SHIFT 0x0UL +#define CC_COMPONENT_ID_3_VALUE_BIT_SIZE 0x8UL +// -------------------------------------- // BLOCK: HOST_SRAM // -------------------------------------- #define CC_SRAM_DATA_REG_OFFSET 0xF00UL diff --git a/drivers/crypto/ccree/cc_hw_queue_defs.h b/drivers/crypto/ccree/cc_hw_queue_defs.h index 7a9b90db7db7..9f4db9956e91 100644 --- a/drivers/crypto/ccree/cc_hw_queue_defs.h +++ b/drivers/crypto/ccree/cc_hw_queue_defs.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #ifndef __CC_HW_QUEUE_DEFS_H__ #define __CC_HW_QUEUE_DEFS_H__ @@ -28,11 +28,13 @@ GENMASK(CC_REG_HIGH(word, name), CC_REG_LOW(word, name)) #define WORD0_VALUE CC_GENMASK(0, VALUE) +#define WORD0_CPP_CIPHER_MODE CC_GENMASK(0, CPP_CIPHER_MODE) #define WORD1_DIN_CONST_VALUE CC_GENMASK(1, DIN_CONST_VALUE) #define WORD1_DIN_DMA_MODE CC_GENMASK(1, DIN_DMA_MODE) #define WORD1_DIN_SIZE CC_GENMASK(1, DIN_SIZE) #define WORD1_NOT_LAST CC_GENMASK(1, NOT_LAST) #define WORD1_NS_BIT CC_GENMASK(1, NS_BIT) +#define WORD1_LOCK_QUEUE CC_GENMASK(1, LOCK_QUEUE) #define WORD2_VALUE CC_GENMASK(2, VALUE) #define WORD3_DOUT_DMA_MODE CC_GENMASK(3, DOUT_DMA_MODE) #define WORD3_DOUT_LAST_IND CC_GENMASK(3, DOUT_LAST_IND) @@ -176,6 +178,15 @@ enum cc_hw_crypto_key { END_OF_KEYS = S32_MAX, }; +#define CC_NUM_HW_KEY_SLOTS 4 +#define CC_FIRST_HW_KEY_SLOT 0 +#define CC_LAST_HW_KEY_SLOT (CC_FIRST_HW_KEY_SLOT + CC_NUM_HW_KEY_SLOTS - 1) + +#define CC_NUM_CPP_KEY_SLOTS 8 +#define CC_FIRST_CPP_KEY_SLOT 16 +#define CC_LAST_CPP_KEY_SLOT (CC_FIRST_CPP_KEY_SLOT + \ + CC_NUM_CPP_KEY_SLOTS - 1) + enum cc_hw_aes_key_size { AES_128_KEY = 0, AES_192_KEY = 1, @@ -189,6 +200,9 @@ enum cc_hash_cipher_pad { HASH_CIPHER_DO_PADDING_RESERVE32 = S32_MAX, }; +#define CC_CPP_DIN_ADDR 0xFF00FF00UL +#define CC_CPP_DIN_SIZE 0xFF00FFUL + /*****************************/ /* Descriptor packing macros */ /*****************************/ @@ -249,6 +263,25 @@ static inline void set_din_no_dma(struct cc_hw_desc *pdesc, u32 addr, u32 size) } /* + * Setup the special CPP descriptor + * + * @pdesc: pointer HW descriptor struct + * @alg: cipher used (AES / SM4) + * @mode: mode used (CTR or CBC) + * @slot: slot number + * @ksize: key size + */ +static inline void set_cpp_crypto_key(struct cc_hw_desc *pdesc, u8 slot) +{ + pdesc->word[0] |= CC_CPP_DIN_ADDR; + + pdesc->word[1] |= FIELD_PREP(WORD1_DIN_SIZE, CC_CPP_DIN_SIZE); + pdesc->word[1] |= FIELD_PREP(WORD1_LOCK_QUEUE, 1); + + pdesc->word[4] |= FIELD_PREP(WORD4_SETUP_OPERATION, slot); +} + +/* * Set the DIN field of a HW descriptors to SRAM mode. * Note: No need to check SRAM alignment since host requests do not use SRAM and * adaptor will enforce alignment check. diff --git a/drivers/crypto/ccree/cc_ivgen.c b/drivers/crypto/ccree/cc_ivgen.c index 769458323394..99dc69383e20 100644 --- a/drivers/crypto/ccree/cc_ivgen.c +++ b/drivers/crypto/ccree/cc_ivgen.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #include <crypto/ctr.h> #include "cc_driver.h" @@ -154,9 +154,6 @@ void cc_ivgen_fini(struct cc_drvdata *drvdata) } ivgen_ctx->pool = NULL_SRAM_ADDR; - - /* release "this" context */ - kfree(ivgen_ctx); } /*! @@ -174,10 +171,12 @@ int cc_ivgen_init(struct cc_drvdata *drvdata) int rc; /* Allocate "this" context */ - ivgen_ctx = kzalloc(sizeof(*ivgen_ctx), GFP_KERNEL); + ivgen_ctx = devm_kzalloc(device, sizeof(*ivgen_ctx), GFP_KERNEL); if (!ivgen_ctx) return -ENOMEM; + drvdata->ivgen_handle = ivgen_ctx; + /* Allocate pool's header for initial enc. key/IV */ ivgen_ctx->pool_meta = dma_alloc_coherent(device, CC_IVPOOL_META_SIZE, &ivgen_ctx->pool_meta_dma, @@ -196,8 +195,6 @@ int cc_ivgen_init(struct cc_drvdata *drvdata) goto out; } - drvdata->ivgen_handle = ivgen_ctx; - return cc_init_iv_sram(drvdata); out: diff --git a/drivers/crypto/ccree/cc_ivgen.h b/drivers/crypto/ccree/cc_ivgen.h index b6ac16903dda..a9f5e8bba4f1 100644 --- a/drivers/crypto/ccree/cc_ivgen.h +++ b/drivers/crypto/ccree/cc_ivgen.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #ifndef __CC_IVGEN_H__ #define __CC_IVGEN_H__ diff --git a/drivers/crypto/ccree/cc_kernel_regs.h b/drivers/crypto/ccree/cc_kernel_regs.h index 8d7262a35156..582bae450596 100644 --- a/drivers/crypto/ccree/cc_kernel_regs.h +++ b/drivers/crypto/ccree/cc_kernel_regs.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #ifndef __CC_CRYS_KERNEL_H__ #define __CC_CRYS_KERNEL_H__ diff --git a/drivers/crypto/ccree/cc_lli_defs.h b/drivers/crypto/ccree/cc_lli_defs.h index 64b15ac9f1d3..f891ab813f41 100644 --- a/drivers/crypto/ccree/cc_lli_defs.h +++ b/drivers/crypto/ccree/cc_lli_defs.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #ifndef _CC_LLI_DEFS_H_ #define _CC_LLI_DEFS_H_ @@ -14,7 +14,7 @@ #define CC_MAX_MLLI_ENTRY_SIZE 0xFFFF #define LLI_MAX_NUM_OF_DATA_ENTRIES 128 -#define LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES 4 +#define LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES 8 #define MLLI_TABLE_MIN_ALIGNMENT 4 /* 32 bit alignment */ #define MAX_NUM_OF_BUFFERS_IN_MLLI 4 #define MAX_NUM_OF_TOTAL_MLLI_ENTRIES \ diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index 6ff7e75ad90e..2dad9c9543c6 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #include <linux/kernel.h> #include <linux/interrupt.h> @@ -11,6 +11,7 @@ #include "cc_ivgen.h" #include "cc_hash.h" #include "cc_pm.h" +#include "cc_fips.h" #define POWER_DOWN_ENABLE 0x01 #define POWER_DOWN_DISABLE 0x00 @@ -25,13 +26,13 @@ int cc_pm_suspend(struct device *dev) int rc; dev_dbg(dev, "set HOST_POWER_DOWN_EN\n"); - cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_ENABLE); rc = cc_suspend_req_queue(drvdata); if (rc) { dev_err(dev, "cc_suspend_req_queue (%x)\n", rc); return rc; } fini_cc_regs(drvdata); + cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_ENABLE); cc_clk_off(drvdata); return 0; } @@ -42,19 +43,21 @@ int cc_pm_resume(struct device *dev) struct cc_drvdata *drvdata = dev_get_drvdata(dev); dev_dbg(dev, "unset HOST_POWER_DOWN_EN\n"); - cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE); - + /* Enables the device source clk */ rc = cc_clk_on(drvdata); if (rc) { dev_err(dev, "failed getting clock back on. We're toast.\n"); return rc; } + cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE); rc = init_cc_regs(drvdata, false); if (rc) { dev_err(dev, "init_cc_regs (%x)\n", rc); return rc; } + /* check if tee fips error occurred during power down */ + cc_tee_handle_fips_error(drvdata); rc = cc_resume_req_queue(drvdata); if (rc) { diff --git a/drivers/crypto/ccree/cc_pm.h b/drivers/crypto/ccree/cc_pm.h index 907a6db4d6c0..6190cdba5dad 100644 --- a/drivers/crypto/ccree/cc_pm.h +++ b/drivers/crypto/ccree/cc_pm.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ /* \file cc_pm.h */ diff --git a/drivers/crypto/ccree/cc_request_mgr.c b/drivers/crypto/ccree/cc_request_mgr.c index 83a8aaae61c7..0bc6ccb0b899 100644 --- a/drivers/crypto/ccree/cc_request_mgr.c +++ b/drivers/crypto/ccree/cc_request_mgr.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #include <linux/kernel.h> +#include <linux/nospec.h> #include "cc_driver.h" #include "cc_buffer_mgr.h" #include "cc_request_mgr.h" @@ -52,11 +53,38 @@ struct cc_bl_item { bool notif; }; +static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = { + { BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) }, + { BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT), + BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) } +}; + static void comp_handler(unsigned long devarg); #ifdef COMP_IN_WQ static void comp_work_handler(struct work_struct *work); #endif +static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot) +{ + alg = array_index_nospec(alg, CC_CPP_NUM_ALGS); + slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS); + + return cc_cpp_int_masks[alg][slot]; +} + void cc_req_mgr_fini(struct cc_drvdata *drvdata) { struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; @@ -336,10 +364,12 @@ static void cc_enqueue_backlog(struct cc_drvdata *drvdata, struct cc_bl_item *bli) { struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; + struct device *dev = drvdata_to_dev(drvdata); spin_lock_bh(&mgr->bl_lock); list_add_tail(&bli->list, &mgr->backlog); ++mgr->bl_len; + dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len); spin_unlock_bh(&mgr->bl_lock); tasklet_schedule(&mgr->comptask); } @@ -349,7 +379,7 @@ static void cc_proc_backlog(struct cc_drvdata *drvdata) struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; struct cc_bl_item *bli; struct cc_crypto_req *creq; - struct crypto_async_request *req; + void *req; bool ivgen; unsigned int total_len; struct device *dev = drvdata_to_dev(drvdata); @@ -359,17 +389,20 @@ static void cc_proc_backlog(struct cc_drvdata *drvdata) while (mgr->bl_len) { bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list); + dev_dbg(dev, "---bl len: %d\n", mgr->bl_len); + spin_unlock(&mgr->bl_lock); + creq = &bli->creq; - req = (struct crypto_async_request *)creq->user_arg; + req = creq->user_arg; /* * Notify the request we're moving out of the backlog * but only if we haven't done so already. */ if (!bli->notif) { - req->complete(req, -EINPROGRESS); + creq->user_cb(dev, req, -EINPROGRESS); bli->notif = true; } @@ -579,6 +612,8 @@ static void proc_completions(struct cc_drvdata *drvdata) drvdata->request_mgr_handle; unsigned int *tail = &request_mgr_handle->req_queue_tail; unsigned int *head = &request_mgr_handle->req_queue_head; + int rc; + u32 mask; while (request_mgr_handle->axi_completed) { request_mgr_handle->axi_completed--; @@ -596,8 +631,22 @@ static void proc_completions(struct cc_drvdata *drvdata) cc_req = &request_mgr_handle->req_queue[*tail]; + if (cc_req->cpp.is_cpp) { + + dev_dbg(dev, "CPP request completion slot: %d alg:%d\n", + cc_req->cpp.slot, cc_req->cpp.alg); + mask = cc_cpp_int_mask(cc_req->cpp.alg, + cc_req->cpp.slot); + rc = (drvdata->irq & mask ? -EPERM : 0); + dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask, + drvdata->irq, rc); + } else { + dev_dbg(dev, "None CPP request completion\n"); + rc = 0; + } + if (cc_req->user_cb) - cc_req->user_cb(dev, cc_req->user_arg, 0); + cc_req->user_cb(dev, cc_req->user_arg, rc); *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1); dev_dbg(dev, "Dequeue request tail=%u\n", *tail); dev_dbg(dev, "Request completed. axi_completed=%d\n", @@ -618,47 +667,50 @@ static void comp_handler(unsigned long devarg) struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg; struct cc_req_mgr_handle *request_mgr_handle = drvdata->request_mgr_handle; - + struct device *dev = drvdata_to_dev(drvdata); u32 irq; - irq = (drvdata->irq & CC_COMP_IRQ_MASK); + dev_dbg(dev, "Completion handler called!\n"); + irq = (drvdata->irq & drvdata->comp_mask); - if (irq & CC_COMP_IRQ_MASK) { - /* To avoid the interrupt from firing as we unmask it, - * we clear it now - */ - cc_iowrite(drvdata, CC_REG(HOST_ICR), CC_COMP_IRQ_MASK); + /* To avoid the interrupt from firing as we unmask it, + * we clear it now + */ + cc_iowrite(drvdata, CC_REG(HOST_ICR), irq); - /* Avoid race with above clear: Test completion counter - * once more - */ - request_mgr_handle->axi_completed += - cc_axi_comp_count(drvdata); - - while (request_mgr_handle->axi_completed) { - do { - proc_completions(drvdata); - /* At this point (after proc_completions()), - * request_mgr_handle->axi_completed is 0. - */ - request_mgr_handle->axi_completed = - cc_axi_comp_count(drvdata); - } while (request_mgr_handle->axi_completed > 0); + /* Avoid race with above clear: Test completion counter once more */ - cc_iowrite(drvdata, CC_REG(HOST_ICR), - CC_COMP_IRQ_MASK); + request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata); + + dev_dbg(dev, "AXI completion after updated: %d\n", + request_mgr_handle->axi_completed); + + while (request_mgr_handle->axi_completed) { + do { + drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR)); + irq = (drvdata->irq & drvdata->comp_mask); + proc_completions(drvdata); + /* At this point (after proc_completions()), + * request_mgr_handle->axi_completed is 0. + */ request_mgr_handle->axi_completed += - cc_axi_comp_count(drvdata); - } + cc_axi_comp_count(drvdata); + } while (request_mgr_handle->axi_completed > 0); + + cc_iowrite(drvdata, CC_REG(HOST_ICR), irq); + + request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata); } + /* after verifing that there is nothing to do, * unmask AXI completion interrupt */ cc_iowrite(drvdata, CC_REG(HOST_IMR), - cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~irq); + cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask); cc_proc_backlog(drvdata); + dev_dbg(dev, "Comp. handler done.\n"); } /* diff --git a/drivers/crypto/ccree/cc_request_mgr.h b/drivers/crypto/ccree/cc_request_mgr.h index 573cb97af085..f46cf766fe4d 100644 --- a/drivers/crypto/ccree/cc_request_mgr.h +++ b/drivers/crypto/ccree/cc_request_mgr.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ /* \file cc_request_mgr.h * Request Manager diff --git a/drivers/crypto/ccree/cc_sram_mgr.c b/drivers/crypto/ccree/cc_sram_mgr.c index c8c276f6dee9..62c885e6e791 100644 --- a/drivers/crypto/ccree/cc_sram_mgr.c +++ b/drivers/crypto/ccree/cc_sram_mgr.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #include "cc_driver.h" #include "cc_sram_mgr.h" @@ -19,8 +19,7 @@ struct cc_sram_ctx { */ void cc_sram_mgr_fini(struct cc_drvdata *drvdata) { - /* Free "this" context */ - kfree(drvdata->sram_mgr_handle); + /* Nothing needed */ } /** @@ -48,7 +47,7 @@ int cc_sram_mgr_init(struct cc_drvdata *drvdata) } /* Allocate "this" context */ - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; diff --git a/drivers/crypto/ccree/cc_sram_mgr.h b/drivers/crypto/ccree/cc_sram_mgr.h index d48649fb3323..1d14de9ee8c3 100644 --- a/drivers/crypto/ccree/cc_sram_mgr.h +++ b/drivers/crypto/ccree/cc_sram_mgr.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ +/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ #ifndef __CC_SRAM_MGR_H__ #define __CC_SRAM_MGR_H__ diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 8d8cf80b9294..8a76fce22943 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -2130,7 +2130,6 @@ static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, * ipad in hmacctx->ipad and opad in hmacctx->opad location */ shash->tfm = hmacctx->base_hash; - shash->flags = crypto_shash_get_flags(hmacctx->base_hash); if (keylen > bs) { err = crypto_shash_digest(shash, key, keylen, hmacctx->ipad); @@ -3517,7 +3516,6 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, SHASH_DESC_ON_STACK(shash, base_hash); shash->tfm = base_hash; - shash->flags = crypto_shash_get_flags(base_hash); bs = crypto_shash_blocksize(base_hash); align = KEYCTX_ALIGN_PAD(max_authsize); o_ptr = actx->h_iopad + param.result_size + align; diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index dad212cabe63..d656be0a142b 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -1976,6 +1976,29 @@ static int hifn_setkey(struct crypto_ablkcipher *cipher, const u8 *key, return 0; } +static int hifn_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + unsigned int len) +{ + struct hifn_context *ctx = crypto_ablkcipher_ctx(cipher); + struct hifn_device *dev = ctx->dev; + u32 flags; + int err; + + flags = crypto_ablkcipher_get_flags(cipher); + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(cipher, flags); + return err; + } + + dev->flags &= ~HIFN_FLAG_OLD_KEY; + + memcpy(ctx->key, key, len); + ctx->keysize = len; + + return 0; +} + static int hifn_handle_req(struct ablkcipher_request *req) { struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); @@ -2240,7 +2263,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { .ablkcipher = { .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, - .setkey = hifn_setkey, + .setkey = hifn_des3_setkey, .encrypt = hifn_encrypt_3des_cfb, .decrypt = hifn_decrypt_3des_cfb, }, @@ -2250,7 +2273,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { .ablkcipher = { .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, - .setkey = hifn_setkey, + .setkey = hifn_des3_setkey, .encrypt = hifn_encrypt_3des_ofb, .decrypt = hifn_decrypt_3des_ofb, }, @@ -2261,7 +2284,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { .ivsize = HIFN_IV_LENGTH, .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, - .setkey = hifn_setkey, + .setkey = hifn_des3_setkey, .encrypt = hifn_encrypt_3des_cbc, .decrypt = hifn_decrypt_3des_cbc, }, @@ -2271,7 +2294,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { .ablkcipher = { .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, - .setkey = hifn_setkey, + .setkey = hifn_des3_setkey, .encrypt = hifn_encrypt_3des_ecb, .decrypt = hifn_decrypt_3des_ecb, }, diff --git a/drivers/crypto/hisilicon/sec/sec_algs.c b/drivers/crypto/hisilicon/sec/sec_algs.c index adc0cd8ae97b..02768af0dccd 100644 --- a/drivers/crypto/hisilicon/sec/sec_algs.c +++ b/drivers/crypto/hisilicon/sec/sec_algs.c @@ -365,20 +365,16 @@ static int sec_alg_skcipher_setkey_des_cbc(struct crypto_skcipher *tfm, static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { - if (keylen != DES_KEY_SIZE * 3) - return -EINVAL; - - return sec_alg_skcipher_setkey(tfm, key, keylen, + return unlikely(des3_verify_key(tfm, key)) ?: + sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_3DES_ECB_192_3KEY); } static int sec_alg_skcipher_setkey_3des_cbc(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { - if (keylen != DES3_EDE_KEY_SIZE) - return -EINVAL; - - return sec_alg_skcipher_setkey(tfm, key, keylen, + return unlikely(des3_verify_key(tfm, key)) ?: + sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_3DES_CBC_192_3KEY); } diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 7ef30a98cb24..de4be10b172f 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -1039,13 +1039,12 @@ static int safexcel_cbc_des3_ede_decrypt(struct skcipher_request *req) static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, const u8 *key, unsigned int len) { - struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); - struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); + int err; - if (len != DES3_EDE_KEY_SIZE) { - crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } + err = des3_verify_key(ctfm, key); + if (unlikely(err)) + return err; /* if context exits and key changed, need to invalidate it */ if (ctx->base.ctxr_dma) { diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 5c4659b04d70..9bbde2f26cac 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -758,14 +758,6 @@ static int setup_cipher(struct crypto_tfm *tfm, int encrypt, return -EINVAL; } cipher_cfg |= keylen_cfg; - } else if (cipher_cfg & MOD_3DES) { - const u32 *K = (const u32 *)key; - if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5])))) - { - *flags |= CRYPTO_TFM_RES_BAD_KEY_SCHED; - return -EINVAL; - } } else { u32 tmp[DES_EXPKEY_WORDS]; if (des_ekey(tmp, key) == 0) { @@ -859,6 +851,19 @@ out: return ret; } +static int ablk_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len) +{ + u32 flags = crypto_ablkcipher_get_flags(tfm); + int err; + + err = __des3_verify_key(&flags, key); + if (unlikely(err)) + crypto_ablkcipher_set_flags(tfm, flags); + + return ablk_setkey(tfm, key, key_len); +} + static int ablk_rfc3686_setkey(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int key_len) { @@ -1175,6 +1180,43 @@ badkey: return -EINVAL; } +static int des3_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct ixp_ctx *ctx = crypto_aead_ctx(tfm); + u32 flags = CRYPTO_TFM_RES_BAD_KEY_LEN; + struct crypto_authenc_keys keys; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + goto badkey; + + err = -EINVAL; + if (keys.authkeylen > sizeof(ctx->authkey)) + goto badkey; + + if (keys.enckeylen != DES3_EDE_KEY_SIZE) + goto badkey; + + flags = crypto_aead_get_flags(tfm); + err = __des3_verify_key(&flags, keys.enckey); + if (unlikely(err)) + goto badkey; + + memcpy(ctx->authkey, keys.authkey, keys.authkeylen); + memcpy(ctx->enckey, keys.enckey, keys.enckeylen); + ctx->authkey_len = keys.authkeylen; + ctx->enckey_len = keys.enckeylen; + + memzero_explicit(&keys, sizeof(keys)); + return aead_setup(tfm, crypto_aead_authsize(tfm)); +badkey: + crypto_aead_set_flags(tfm, flags); + memzero_explicit(&keys, sizeof(keys)); + return err; +} + static int aead_encrypt(struct aead_request *req) { return aead_perform(req, 1, req->assoclen, req->cryptlen, req->iv); @@ -1220,6 +1262,7 @@ static struct ixp_alg ixp4xx_algos[] = { .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = ablk_des3_setkey, } } }, @@ -1232,6 +1275,7 @@ static struct ixp_alg ixp4xx_algos[] = { .cra_u = { .ablkcipher = { .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = ablk_des3_setkey, } } }, @@ -1313,6 +1357,7 @@ static struct ixp_aead_alg ixp4xx_aeads[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, + .setkey = des3_aead_setkey, }, .hash = &hash_alg_md5, .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192, @@ -1337,6 +1382,7 @@ static struct ixp_aead_alg ixp4xx_aeads[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, + .setkey = des3_aead_setkey, }, .hash = &hash_alg_sha1, .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192, @@ -1443,7 +1489,7 @@ static int __init ixp_module_init(void) /* authenc */ cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC; - cra->setkey = aead_setkey; + cra->setkey = cra->setkey ?: aead_setkey; cra->setauthsize = aead_setauthsize; cra->encrypt = aead_encrypt; cra->decrypt = aead_decrypt; diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c index fb279b3a1ca1..2fd936b19c6d 100644 --- a/drivers/crypto/marvell/cipher.c +++ b/drivers/crypto/marvell/cipher.c @@ -299,13 +299,12 @@ static int mv_cesa_des_setkey(struct crypto_skcipher *cipher, const u8 *key, static int mv_cesa_des3_ede_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int len) { - struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); - struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(tfm); + struct mv_cesa_des_ctx *ctx = crypto_skcipher_ctx(cipher); + int err; - if (len != DES3_EDE_KEY_SIZE) { - crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } + err = des3_verify_key(cipher, key); + if (unlikely(err)) + return err; memcpy(ctx->key, key, DES3_EDE_KEY_SIZE); diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 99ff54cc8a15..fd456dd703bf 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -135,11 +135,10 @@ static int mv_cesa_ahash_pad_len(struct mv_cesa_ahash_req *creq) static int mv_cesa_ahash_pad_req(struct mv_cesa_ahash_req *creq, u8 *buf) { - unsigned int index, padlen; + unsigned int padlen; buf[0] = 0x80; /* Pad out to 56 mod 64 */ - index = creq->len & CESA_HASH_BLOCK_SIZE_MSK; padlen = mv_cesa_ahash_pad_len(creq); memset(buf + 1, 0, padlen - 1); diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index 5f4f845adbb8..a0806ba40c68 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -365,7 +365,6 @@ static int mtk_sha_finish_hmac(struct ahash_request *req) SHASH_DESC_ON_STACK(shash, bctx->shash); shash->tfm = bctx->shash; - shash->flags = 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */ return crypto_shash_init(shash) ?: crypto_shash_update(shash, bctx->opad, ctx->bs) ?: @@ -810,8 +809,6 @@ static int mtk_sha_setkey(struct crypto_ahash *tfm, const u8 *key, SHASH_DESC_ON_STACK(shash, bctx->shash); shash->tfm = bctx->shash; - shash->flags = crypto_shash_get_flags(bctx->shash) & - CRYPTO_TFM_REQ_MAY_SLEEP; if (keylen > bs) { err = crypto_shash_digest(shash, key, keylen, bctx->ipad); diff --git a/drivers/crypto/mxc-scc.c b/drivers/crypto/mxc-scc.c deleted file mode 100644 index 519086730791..000000000000 --- a/drivers/crypto/mxc-scc.c +++ /dev/null @@ -1,767 +0,0 @@ -/* - * Copyright (C) 2016 Pengutronix, Steffen Trumtrar <kernel@pengutronix.de> - * - * The driver is based on information gathered from - * drivers/mxc/security/mxc_scc.c which can be found in - * the Freescale linux-2.6-imx.git in the imx_2.6.35_maintain branch. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ -#include <linux/clk.h> -#include <linux/crypto.h> -#include <linux/interrupt.h> -#include <linux/io.h> -#include <linux/irq.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/mutex.h> -#include <linux/of.h> -#include <linux/of_device.h> -#include <linux/platform_device.h> - -#include <crypto/algapi.h> -#include <crypto/des.h> - -/* Secure Memory (SCM) registers */ -#define SCC_SCM_RED_START 0x0000 -#define SCC_SCM_BLACK_START 0x0004 -#define SCC_SCM_LENGTH 0x0008 -#define SCC_SCM_CTRL 0x000C -#define SCC_SCM_STATUS 0x0010 -#define SCC_SCM_ERROR_STATUS 0x0014 -#define SCC_SCM_INTR_CTRL 0x0018 -#define SCC_SCM_CFG 0x001C -#define SCC_SCM_INIT_VECTOR_0 0x0020 -#define SCC_SCM_INIT_VECTOR_1 0x0024 -#define SCC_SCM_RED_MEMORY 0x0400 -#define SCC_SCM_BLACK_MEMORY 0x0800 - -/* Security Monitor (SMN) Registers */ -#define SCC_SMN_STATUS 0x1000 -#define SCC_SMN_COMMAND 0x1004 -#define SCC_SMN_SEQ_START 0x1008 -#define SCC_SMN_SEQ_END 0x100C -#define SCC_SMN_SEQ_CHECK 0x1010 -#define SCC_SMN_BIT_COUNT 0x1014 -#define SCC_SMN_BITBANK_INC_SIZE 0x1018 -#define SCC_SMN_BITBANK_DECREMENT 0x101C -#define SCC_SMN_COMPARE_SIZE 0x1020 -#define SCC_SMN_PLAINTEXT_CHECK 0x1024 -#define SCC_SMN_CIPHERTEXT_CHECK 0x1028 -#define SCC_SMN_TIMER_IV 0x102C -#define SCC_SMN_TIMER_CONTROL 0x1030 -#define SCC_SMN_DEBUG_DETECT_STAT 0x1034 -#define SCC_SMN_TIMER 0x1038 - -#define SCC_SCM_CTRL_START_CIPHER BIT(2) -#define SCC_SCM_CTRL_CBC_MODE BIT(1) -#define SCC_SCM_CTRL_DECRYPT_MODE BIT(0) - -#define SCC_SCM_STATUS_LEN_ERR BIT(12) -#define SCC_SCM_STATUS_SMN_UNBLOCKED BIT(11) -#define SCC_SCM_STATUS_CIPHERING_DONE BIT(10) -#define SCC_SCM_STATUS_ZEROIZING_DONE BIT(9) -#define SCC_SCM_STATUS_INTR_STATUS BIT(8) -#define SCC_SCM_STATUS_SEC_KEY BIT(7) -#define SCC_SCM_STATUS_INTERNAL_ERR BIT(6) -#define SCC_SCM_STATUS_BAD_SEC_KEY BIT(5) -#define SCC_SCM_STATUS_ZEROIZE_FAIL BIT(4) -#define SCC_SCM_STATUS_SMN_BLOCKED BIT(3) -#define SCC_SCM_STATUS_CIPHERING BIT(2) -#define SCC_SCM_STATUS_ZEROIZING BIT(1) -#define SCC_SCM_STATUS_BUSY BIT(0) - -#define SCC_SMN_STATUS_STATE_MASK 0x0000001F -#define SCC_SMN_STATE_START 0x0 -/* The SMN is zeroizing its RAM during reset */ -#define SCC_SMN_STATE_ZEROIZE_RAM 0x5 -/* SMN has passed internal checks */ -#define SCC_SMN_STATE_HEALTH_CHECK 0x6 -/* Fatal Security Violation. SMN is locked, SCM is inoperative. */ -#define SCC_SMN_STATE_FAIL 0x9 -/* SCC is in secure state. SCM is using secret key. */ -#define SCC_SMN_STATE_SECURE 0xA -/* SCC is not secure. SCM is using default key. */ -#define SCC_SMN_STATE_NON_SECURE 0xC - -#define SCC_SCM_INTR_CTRL_ZEROIZE_MEM BIT(2) -#define SCC_SCM_INTR_CTRL_CLR_INTR BIT(1) -#define SCC_SCM_INTR_CTRL_MASK_INTR BIT(0) - -/* Size, in blocks, of Red memory. */ -#define SCC_SCM_CFG_BLACK_SIZE_MASK 0x07fe0000 -#define SCC_SCM_CFG_BLACK_SIZE_SHIFT 17 -/* Size, in blocks, of Black memory. */ -#define SCC_SCM_CFG_RED_SIZE_MASK 0x0001ff80 -#define SCC_SCM_CFG_RED_SIZE_SHIFT 7 -/* Number of bytes per block. */ -#define SCC_SCM_CFG_BLOCK_SIZE_MASK 0x0000007f - -#define SCC_SMN_COMMAND_TAMPER_LOCK BIT(4) -#define SCC_SMN_COMMAND_CLR_INTR BIT(3) -#define SCC_SMN_COMMAND_CLR_BIT_BANK BIT(2) -#define SCC_SMN_COMMAND_EN_INTR BIT(1) -#define SCC_SMN_COMMAND_SET_SOFTWARE_ALARM BIT(0) - -#define SCC_KEY_SLOTS 20 -#define SCC_MAX_KEY_SIZE 32 -#define SCC_KEY_SLOT_SIZE 32 - -#define SCC_CRC_CCITT_START 0xFFFF - -/* - * Offset into each RAM of the base of the area which is not - * used for Stored Keys. - */ -#define SCC_NON_RESERVED_OFFSET (SCC_KEY_SLOTS * SCC_KEY_SLOT_SIZE) - -/* Fixed padding for appending to plaintext to fill out a block */ -static char scc_block_padding[8] = { 0x80, 0, 0, 0, 0, 0, 0, 0 }; - -enum mxc_scc_state { - SCC_STATE_OK, - SCC_STATE_UNIMPLEMENTED, - SCC_STATE_FAILED -}; - -struct mxc_scc { - struct device *dev; - void __iomem *base; - struct clk *clk; - bool hw_busy; - spinlock_t lock; - struct crypto_queue queue; - struct crypto_async_request *req; - int block_size_bytes; - int black_ram_size_blocks; - int memory_size_bytes; - int bytes_remaining; - - void __iomem *red_memory; - void __iomem *black_memory; -}; - -struct mxc_scc_ctx { - struct mxc_scc *scc; - struct scatterlist *sg_src; - size_t src_nents; - struct scatterlist *sg_dst; - size_t dst_nents; - unsigned int offset; - unsigned int size; - unsigned int ctrl; -}; - -struct mxc_scc_crypto_tmpl { - struct mxc_scc *scc; - struct crypto_alg alg; -}; - -static int mxc_scc_get_data(struct mxc_scc_ctx *ctx, - struct crypto_async_request *req) -{ - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mxc_scc *scc = ctx->scc; - size_t len; - void __iomem *from; - - if (ctx->ctrl & SCC_SCM_CTRL_DECRYPT_MODE) - from = scc->red_memory; - else - from = scc->black_memory; - - dev_dbg(scc->dev, "pcopy: from 0x%p %zu bytes\n", from, - ctx->dst_nents * 8); - len = sg_pcopy_from_buffer(ablkreq->dst, ctx->dst_nents, - from, ctx->size, ctx->offset); - if (!len) { - dev_err(scc->dev, "pcopy err from 0x%p (len=%zu)\n", from, len); - return -EINVAL; - } - -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "red memory@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, - scc->red_memory, ctx->size, 1); - print_hex_dump(KERN_ERR, - "black memory@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, - scc->black_memory, ctx->size, 1); -#endif - - ctx->offset += len; - - if (ctx->offset < ablkreq->nbytes) - return -EINPROGRESS; - - return 0; -} - -static int mxc_scc_ablkcipher_req_init(struct ablkcipher_request *req, - struct mxc_scc_ctx *ctx) -{ - struct mxc_scc *scc = ctx->scc; - int nents; - - nents = sg_nents_for_len(req->src, req->nbytes); - if (nents < 0) { - dev_err(scc->dev, "Invalid number of src SC"); - return nents; - } - ctx->src_nents = nents; - - nents = sg_nents_for_len(req->dst, req->nbytes); - if (nents < 0) { - dev_err(scc->dev, "Invalid number of dst SC"); - return nents; - } - ctx->dst_nents = nents; - - ctx->size = 0; - ctx->offset = 0; - - return 0; -} - -static int mxc_scc_ablkcipher_req_complete(struct crypto_async_request *req, - struct mxc_scc_ctx *ctx, - int result) -{ - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mxc_scc *scc = ctx->scc; - - scc->req = NULL; - scc->bytes_remaining = scc->memory_size_bytes; - - if (ctx->ctrl & SCC_SCM_CTRL_CBC_MODE) - memcpy(ablkreq->info, scc->base + SCC_SCM_INIT_VECTOR_0, - scc->block_size_bytes); - - req->complete(req, result); - scc->hw_busy = false; - - return 0; -} - -static int mxc_scc_put_data(struct mxc_scc_ctx *ctx, - struct ablkcipher_request *req) -{ - u8 padding_buffer[sizeof(u16) + sizeof(scc_block_padding)]; - size_t len = min_t(size_t, req->nbytes - ctx->offset, - ctx->scc->bytes_remaining); - unsigned int padding_byte_count = 0; - struct mxc_scc *scc = ctx->scc; - void __iomem *to; - - if (ctx->ctrl & SCC_SCM_CTRL_DECRYPT_MODE) - to = scc->black_memory; - else - to = scc->red_memory; - - if (ctx->ctrl & SCC_SCM_CTRL_CBC_MODE && req->info) - memcpy(scc->base + SCC_SCM_INIT_VECTOR_0, req->info, - scc->block_size_bytes); - - len = sg_pcopy_to_buffer(req->src, ctx->src_nents, - to, len, ctx->offset); - if (!len) { - dev_err(scc->dev, "pcopy err to 0x%p (len=%zu)\n", to, len); - return -EINVAL; - } - - ctx->size = len; - -#ifdef DEBUG - dev_dbg(scc->dev, "copied %d bytes to 0x%p\n", len, to); - print_hex_dump(KERN_ERR, - "init vector0@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, - scc->base + SCC_SCM_INIT_VECTOR_0, scc->block_size_bytes, - 1); - print_hex_dump(KERN_ERR, - "red memory@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, - scc->red_memory, ctx->size, 1); - print_hex_dump(KERN_ERR, - "black memory@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, - scc->black_memory, ctx->size, 1); -#endif - - scc->bytes_remaining -= len; - - padding_byte_count = len % scc->block_size_bytes; - - if (padding_byte_count) { - memcpy(padding_buffer, scc_block_padding, padding_byte_count); - memcpy(to + len, padding_buffer, padding_byte_count); - ctx->size += padding_byte_count; - } - -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "data to encrypt@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, - to, ctx->size, 1); -#endif - - return 0; -} - -static void mxc_scc_ablkcipher_next(struct mxc_scc_ctx *ctx, - struct crypto_async_request *req) -{ - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mxc_scc *scc = ctx->scc; - int err; - - dev_dbg(scc->dev, "dispatch request (nbytes=%d, src=%p, dst=%p)\n", - ablkreq->nbytes, ablkreq->src, ablkreq->dst); - - writel(0, scc->base + SCC_SCM_ERROR_STATUS); - - err = mxc_scc_put_data(ctx, ablkreq); - if (err) { - mxc_scc_ablkcipher_req_complete(req, ctx, err); - return; - } - - dev_dbg(scc->dev, "Start encryption (0x%x/0x%x)\n", - readl(scc->base + SCC_SCM_RED_START), - readl(scc->base + SCC_SCM_BLACK_START)); - - /* clear interrupt control registers */ - writel(SCC_SCM_INTR_CTRL_CLR_INTR, - scc->base + SCC_SCM_INTR_CTRL); - - writel((ctx->size / ctx->scc->block_size_bytes) - 1, - scc->base + SCC_SCM_LENGTH); - - dev_dbg(scc->dev, "Process %d block(s) in 0x%p\n", - ctx->size / ctx->scc->block_size_bytes, - (ctx->ctrl & SCC_SCM_CTRL_DECRYPT_MODE) ? scc->black_memory : - scc->red_memory); - - writel(ctx->ctrl, scc->base + SCC_SCM_CTRL); -} - -static irqreturn_t mxc_scc_int(int irq, void *priv) -{ - struct crypto_async_request *req; - struct mxc_scc_ctx *ctx; - struct mxc_scc *scc = priv; - int status; - int ret; - - status = readl(scc->base + SCC_SCM_STATUS); - - /* clear interrupt control registers */ - writel(SCC_SCM_INTR_CTRL_CLR_INTR, scc->base + SCC_SCM_INTR_CTRL); - - if (status & SCC_SCM_STATUS_BUSY) - return IRQ_NONE; - - req = scc->req; - if (req) { - ctx = crypto_tfm_ctx(req->tfm); - ret = mxc_scc_get_data(ctx, req); - if (ret != -EINPROGRESS) - mxc_scc_ablkcipher_req_complete(req, ctx, ret); - else - mxc_scc_ablkcipher_next(ctx, req); - } - - return IRQ_HANDLED; -} - -static int mxc_scc_cra_init(struct crypto_tfm *tfm) -{ - struct mxc_scc_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_alg *alg = tfm->__crt_alg; - struct mxc_scc_crypto_tmpl *algt; - - algt = container_of(alg, struct mxc_scc_crypto_tmpl, alg); - - ctx->scc = algt->scc; - return 0; -} - -static void mxc_scc_dequeue_req_unlocked(struct mxc_scc_ctx *ctx) -{ - struct crypto_async_request *req, *backlog; - - if (ctx->scc->hw_busy) - return; - - spin_lock_bh(&ctx->scc->lock); - backlog = crypto_get_backlog(&ctx->scc->queue); - req = crypto_dequeue_request(&ctx->scc->queue); - ctx->scc->req = req; - ctx->scc->hw_busy = true; - spin_unlock_bh(&ctx->scc->lock); - - if (!req) - return; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - mxc_scc_ablkcipher_next(ctx, req); -} - -static int mxc_scc_queue_req(struct mxc_scc_ctx *ctx, - struct crypto_async_request *req) -{ - int ret; - - spin_lock_bh(&ctx->scc->lock); - ret = crypto_enqueue_request(&ctx->scc->queue, req); - spin_unlock_bh(&ctx->scc->lock); - - if (ret != -EINPROGRESS) - return ret; - - mxc_scc_dequeue_req_unlocked(ctx); - - return -EINPROGRESS; -} - -static int mxc_scc_des3_op(struct mxc_scc_ctx *ctx, - struct ablkcipher_request *req) -{ - int err; - - err = mxc_scc_ablkcipher_req_init(req, ctx); - if (err) - return err; - - return mxc_scc_queue_req(ctx, &req->base); -} - -static int mxc_scc_ecb_des_encrypt(struct ablkcipher_request *req) -{ - struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req); - struct mxc_scc_ctx *ctx = crypto_ablkcipher_ctx(cipher); - - ctx->ctrl = SCC_SCM_CTRL_START_CIPHER; - - return mxc_scc_des3_op(ctx, req); -} - -static int mxc_scc_ecb_des_decrypt(struct ablkcipher_request *req) -{ - struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req); - struct mxc_scc_ctx *ctx = crypto_ablkcipher_ctx(cipher); - - ctx->ctrl = SCC_SCM_CTRL_START_CIPHER; - ctx->ctrl |= SCC_SCM_CTRL_DECRYPT_MODE; - - return mxc_scc_des3_op(ctx, req); -} - -static int mxc_scc_cbc_des_encrypt(struct ablkcipher_request *req) -{ - struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req); - struct mxc_scc_ctx *ctx = crypto_ablkcipher_ctx(cipher); - - ctx->ctrl = SCC_SCM_CTRL_START_CIPHER; - ctx->ctrl |= SCC_SCM_CTRL_CBC_MODE; - - return mxc_scc_des3_op(ctx, req); -} - -static int mxc_scc_cbc_des_decrypt(struct ablkcipher_request *req) -{ - struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req); - struct mxc_scc_ctx *ctx = crypto_ablkcipher_ctx(cipher); - - ctx->ctrl = SCC_SCM_CTRL_START_CIPHER; - ctx->ctrl |= SCC_SCM_CTRL_CBC_MODE; - ctx->ctrl |= SCC_SCM_CTRL_DECRYPT_MODE; - - return mxc_scc_des3_op(ctx, req); -} - -static void mxc_scc_hw_init(struct mxc_scc *scc) -{ - int offset; - - offset = SCC_NON_RESERVED_OFFSET / scc->block_size_bytes; - - /* Fill the RED_START register */ - writel(offset, scc->base + SCC_SCM_RED_START); - - /* Fill the BLACK_START register */ - writel(offset, scc->base + SCC_SCM_BLACK_START); - - scc->red_memory = scc->base + SCC_SCM_RED_MEMORY + - SCC_NON_RESERVED_OFFSET; - - scc->black_memory = scc->base + SCC_SCM_BLACK_MEMORY + - SCC_NON_RESERVED_OFFSET; - - scc->bytes_remaining = scc->memory_size_bytes; -} - -static int mxc_scc_get_config(struct mxc_scc *scc) -{ - int config; - - config = readl(scc->base + SCC_SCM_CFG); - - scc->block_size_bytes = config & SCC_SCM_CFG_BLOCK_SIZE_MASK; - - scc->black_ram_size_blocks = config & SCC_SCM_CFG_BLACK_SIZE_MASK; - - scc->memory_size_bytes = (scc->block_size_bytes * - scc->black_ram_size_blocks) - - SCC_NON_RESERVED_OFFSET; - - return 0; -} - -static enum mxc_scc_state mxc_scc_get_state(struct mxc_scc *scc) -{ - enum mxc_scc_state state; - int status; - - status = readl(scc->base + SCC_SMN_STATUS) & - SCC_SMN_STATUS_STATE_MASK; - - /* If in Health Check, try to bringup to secure state */ - if (status & SCC_SMN_STATE_HEALTH_CHECK) { - /* - * Write a simple algorithm to the Algorithm Sequence - * Checker (ASC) - */ - writel(0xaaaa, scc->base + SCC_SMN_SEQ_START); - writel(0x5555, scc->base + SCC_SMN_SEQ_END); - writel(0x5555, scc->base + SCC_SMN_SEQ_CHECK); - - status = readl(scc->base + SCC_SMN_STATUS) & - SCC_SMN_STATUS_STATE_MASK; - } - - switch (status) { - case SCC_SMN_STATE_NON_SECURE: - case SCC_SMN_STATE_SECURE: - state = SCC_STATE_OK; - break; - case SCC_SMN_STATE_FAIL: - state = SCC_STATE_FAILED; - break; - default: - state = SCC_STATE_UNIMPLEMENTED; - break; - } - - return state; -} - -static struct mxc_scc_crypto_tmpl scc_ecb_des = { - .alg = { - .cra_name = "ecb(des3_ede)", - .cra_driver_name = "ecb-des3-scc", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mxc_scc_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mxc_scc_cra_init, - .cra_u.ablkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .encrypt = mxc_scc_ecb_des_encrypt, - .decrypt = mxc_scc_ecb_des_decrypt, - } - } -}; - -static struct mxc_scc_crypto_tmpl scc_cbc_des = { - .alg = { - .cra_name = "cbc(des3_ede)", - .cra_driver_name = "cbc-des3-scc", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mxc_scc_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mxc_scc_cra_init, - .cra_u.ablkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .encrypt = mxc_scc_cbc_des_encrypt, - .decrypt = mxc_scc_cbc_des_decrypt, - } - } -}; - -static struct mxc_scc_crypto_tmpl *scc_crypto_algs[] = { - &scc_ecb_des, - &scc_cbc_des, -}; - -static int mxc_scc_crypto_register(struct mxc_scc *scc) -{ - int i; - int err = 0; - - for (i = 0; i < ARRAY_SIZE(scc_crypto_algs); i++) { - scc_crypto_algs[i]->scc = scc; - err = crypto_register_alg(&scc_crypto_algs[i]->alg); - if (err) - goto err_out; - } - - return 0; - -err_out: - while (--i >= 0) - crypto_unregister_alg(&scc_crypto_algs[i]->alg); - - return err; -} - -static void mxc_scc_crypto_unregister(void) -{ - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(scc_crypto_algs); i++) - crypto_unregister_alg(&scc_crypto_algs[i]->alg); -} - -static int mxc_scc_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct resource *res; - struct mxc_scc *scc; - enum mxc_scc_state state; - int irq; - int ret; - int i; - - scc = devm_kzalloc(dev, sizeof(*scc), GFP_KERNEL); - if (!scc) - return -ENOMEM; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - scc->base = devm_ioremap_resource(dev, res); - if (IS_ERR(scc->base)) - return PTR_ERR(scc->base); - - scc->clk = devm_clk_get(&pdev->dev, "ipg"); - if (IS_ERR(scc->clk)) { - dev_err(dev, "Could not get ipg clock\n"); - return PTR_ERR(scc->clk); - } - - ret = clk_prepare_enable(scc->clk); - if (ret) - return ret; - - /* clear error status register */ - writel(0x0, scc->base + SCC_SCM_ERROR_STATUS); - - /* clear interrupt control registers */ - writel(SCC_SCM_INTR_CTRL_CLR_INTR | - SCC_SCM_INTR_CTRL_MASK_INTR, - scc->base + SCC_SCM_INTR_CTRL); - - writel(SCC_SMN_COMMAND_CLR_INTR | - SCC_SMN_COMMAND_EN_INTR, - scc->base + SCC_SMN_COMMAND); - - scc->dev = dev; - platform_set_drvdata(pdev, scc); - - ret = mxc_scc_get_config(scc); - if (ret) - goto err_out; - - state = mxc_scc_get_state(scc); - - if (state != SCC_STATE_OK) { - dev_err(dev, "SCC in unusable state %d\n", state); - ret = -EINVAL; - goto err_out; - } - - mxc_scc_hw_init(scc); - - spin_lock_init(&scc->lock); - /* FIXME: calculate queue from RAM slots */ - crypto_init_queue(&scc->queue, 50); - - for (i = 0; i < 2; i++) { - irq = platform_get_irq(pdev, i); - if (irq < 0) { - dev_err(dev, "failed to get irq resource: %d\n", irq); - ret = irq; - goto err_out; - } - - ret = devm_request_threaded_irq(dev, irq, NULL, mxc_scc_int, - IRQF_ONESHOT, dev_name(dev), scc); - if (ret) - goto err_out; - } - - ret = mxc_scc_crypto_register(scc); - if (ret) { - dev_err(dev, "could not register algorithms"); - goto err_out; - } - - dev_info(dev, "registered successfully.\n"); - - return 0; - -err_out: - clk_disable_unprepare(scc->clk); - - return ret; -} - -static int mxc_scc_remove(struct platform_device *pdev) -{ - struct mxc_scc *scc = platform_get_drvdata(pdev); - - mxc_scc_crypto_unregister(); - - clk_disable_unprepare(scc->clk); - - return 0; -} - -static const struct of_device_id mxc_scc_dt_ids[] = { - { .compatible = "fsl,imx25-scc", .data = NULL, }, - { /* sentinel */ } -}; -MODULE_DEVICE_TABLE(of, mxc_scc_dt_ids); - -static struct platform_driver mxc_scc_driver = { - .probe = mxc_scc_probe, - .remove = mxc_scc_remove, - .driver = { - .name = "mxc-scc", - .of_match_table = mxc_scc_dt_ids, - }, -}; - -module_platform_driver(mxc_scc_driver); -MODULE_AUTHOR("Steffen Trumtrar <kernel@pengutronix.de>"); -MODULE_DESCRIPTION("Freescale i.MX25 SCC Crypto driver"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index a2105cf33abb..b4429891e368 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -471,7 +471,7 @@ static int mxs_dcp_aes_enqueue(struct ablkcipher_request *req, int enc, int ecb) wake_up_process(sdcp->thread[actx->chan]); - return -EINPROGRESS; + return ret; } static int mxs_dcp_aes_ecb_decrypt(struct ablkcipher_request *req) @@ -700,11 +700,7 @@ static int dcp_chan_thread_sha(void *data) struct crypto_async_request *backlog; struct crypto_async_request *arq; - - struct dcp_sha_req_ctx *rctx; - - struct ahash_request *req; - int ret, fini; + int ret; while (!kthread_should_stop()) { set_current_state(TASK_INTERRUPTIBLE); @@ -725,11 +721,7 @@ static int dcp_chan_thread_sha(void *data) backlog->complete(backlog, -EINPROGRESS); if (arq) { - req = ahash_request_cast(arq); - rctx = ahash_request_ctx(req); - ret = dcp_sha_req_to_buf(arq); - fini = rctx->fini; arq->complete(arq, ret); } } @@ -797,7 +789,7 @@ static int dcp_sha_update_fx(struct ahash_request *req, int fini) wake_up_process(sdcp->thread[actx->chan]); mutex_unlock(&actx->mutex); - return -EINPROGRESS; + return ret; } static int dcp_sha_update(struct ahash_request *req) diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 9450c41211b2..0d5d3d8eb680 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -469,8 +469,6 @@ static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key, return err; shash->tfm = child_shash; - shash->flags = crypto_ahash_get_flags(tfm) & - CRYPTO_TFM_REQ_MAY_SLEEP; bs = crypto_shash_blocksize(child_shash); ds = crypto_shash_digestsize(child_shash); @@ -788,13 +786,18 @@ static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm); struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm); + u32 flags; + int err; + + flags = crypto_ablkcipher_get_flags(cipher); + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(cipher, flags); + return err; + } ctx->enc_type = n2alg->enc_type; - if (keylen != (3 * DES_KEY_SIZE)) { - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } ctx->key_len = keylen; memcpy(ctx->key.des3, key, keylen); return 0; diff --git a/drivers/crypto/nx/nx-842-pseries.c b/drivers/crypto/nx/nx-842-pseries.c index 66869976cfa2..57932848361b 100644 --- a/drivers/crypto/nx/nx-842-pseries.c +++ b/drivers/crypto/nx/nx-842-pseries.c @@ -296,7 +296,7 @@ static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen, struct nx842_workmem *workmem; struct nx842_scatterlist slin, slout; struct nx_csbcpb *csbcpb; - int ret = 0, max_sync_size; + int ret = 0; unsigned long inbuf, outbuf; struct vio_pfo_op op = { .done = NULL, @@ -319,7 +319,6 @@ static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen, rcu_read_unlock(); return -ENODEV; } - max_sync_size = local_devdata->max_sync_size; dev = local_devdata->dev; /* Init scatterlist */ @@ -427,7 +426,7 @@ static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen, struct nx842_workmem *workmem; struct nx842_scatterlist slin, slout; struct nx_csbcpb *csbcpb; - int ret = 0, max_sync_size; + int ret = 0; unsigned long inbuf, outbuf; struct vio_pfo_op op = { .done = NULL, @@ -451,7 +450,6 @@ static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen, rcu_read_unlock(); return -ENODEV; } - max_sync_size = local_devdata->max_sync_size; dev = local_devdata->dev; workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c index d94e25df503b..f06565df2a12 100644 --- a/drivers/crypto/nx/nx-842.c +++ b/drivers/crypto/nx/nx-842.c @@ -353,7 +353,7 @@ static int decompress(struct nx842_crypto_ctx *ctx, unsigned int adj_slen = slen; u8 *src = p->in, *dst = p->out; u16 padding = be16_to_cpu(g->padding); - int ret, spadding = 0, dpadding = 0; + int ret, spadding = 0; ktime_t timeout; if (!slen || !required_len) @@ -413,7 +413,6 @@ usesw: spadding = 0; dst = p->out; dlen = p->oremain; - dpadding = 0; if (dlen < required_len) { /* have ignore bytes */ dst = ctx->dbounce; dlen = BOUNCE_BUFFER_SIZE; diff --git a/drivers/crypto/nx/nx-aes-xcbc.c b/drivers/crypto/nx/nx-aes-xcbc.c index ad3358e74f5c..8f5820b78a83 100644 --- a/drivers/crypto/nx/nx-aes-xcbc.c +++ b/drivers/crypto/nx/nx-aes-xcbc.c @@ -105,8 +105,7 @@ static int nx_xcbc_empty(struct shash_desc *desc, u8 *out) nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); - rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, - desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP); + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); if (rc) goto out; atomic_inc(&(nx_ctx->stats->aes_ops)); @@ -134,8 +133,7 @@ static int nx_xcbc_empty(struct shash_desc *desc, u8 *out) nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); - rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, - desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP); + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); if (rc) goto out; atomic_inc(&(nx_ctx->stats->aes_ops)); @@ -279,8 +277,7 @@ static int nx_xcbc_update(struct shash_desc *desc, goto out; } - rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, - desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP); + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); if (rc) goto out; @@ -361,8 +358,7 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out) goto out; } - rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, - desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP); + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); if (rc) goto out; diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c index a6764af83c6d..e06f0431dee5 100644 --- a/drivers/crypto/nx/nx-sha256.c +++ b/drivers/crypto/nx/nx-sha256.c @@ -162,8 +162,7 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data, goto out; } - rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, - desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP); + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); if (rc) goto out; @@ -243,8 +242,7 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out) goto out; } - rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, - desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP); + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); if (rc) goto out; diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c index 92956bc6e45e..0293b17903d0 100644 --- a/drivers/crypto/nx/nx-sha512.c +++ b/drivers/crypto/nx/nx-sha512.c @@ -166,8 +166,7 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data, goto out; } - rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, - desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP); + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); if (rc) goto out; @@ -249,8 +248,7 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out) goto out; } - rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, - desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP); + rc = nx_hcall_sync(nx_ctx, &nx_ctx->op, 0); if (rc) goto out; diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 1ba2633e90d6..3d82d18ff810 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -656,9 +656,6 @@ static int omap_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(cipher); struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); - if (keylen != DES_KEY_SIZE && keylen != (3*DES_KEY_SIZE)) - return -EINVAL; - pr_debug("enter, keylen: %d\n", keylen); /* Do we need to test against weak key? */ @@ -678,6 +675,28 @@ static int omap_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, return 0; } +static int omap_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(cipher); + u32 flags; + int err; + + pr_debug("enter, keylen: %d\n", keylen); + + flags = crypto_ablkcipher_get_flags(cipher); + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(cipher, flags); + return err; + } + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + static int omap_des_ecb_encrypt(struct ablkcipher_request *req) { return omap_des_crypt(req, FLAGS_ENCRYPT); @@ -788,7 +807,7 @@ static struct crypto_alg algs_ecb_cbc[] = { .cra_u.ablkcipher = { .min_keysize = 3*DES_KEY_SIZE, .max_keysize = 3*DES_KEY_SIZE, - .setkey = omap_des_setkey, + .setkey = omap_des3_setkey, .encrypt = omap_des_ecb_encrypt, .decrypt = omap_des_ecb_decrypt, } @@ -811,7 +830,7 @@ static struct crypto_alg algs_ecb_cbc[] = { .min_keysize = 3*DES_KEY_SIZE, .max_keysize = 3*DES_KEY_SIZE, .ivsize = DES_BLOCK_SIZE, - .setkey = omap_des_setkey, + .setkey = omap_des3_setkey, .encrypt = omap_des_cbc_encrypt, .decrypt = omap_des_cbc_decrypt, } diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 0641185bd82f..51b20abac464 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -1055,7 +1055,6 @@ static int omap_sham_finish_hmac(struct ahash_request *req) SHASH_DESC_ON_STACK(shash, bctx->shash); shash->tfm = bctx->shash; - shash->flags = 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */ return crypto_shash_init(shash) ?: crypto_shash_update(shash, bctx->opad, bs) ?: @@ -1226,7 +1225,6 @@ static int omap_sham_shash_digest(struct crypto_shash *tfm, u32 flags, SHASH_DESC_ON_STACK(shash, tfm); shash->tfm = tfm; - shash->flags = flags & CRYPTO_TFM_REQ_MAY_SLEEP; return crypto_shash_digest(shash, data, len, out); } diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index 21e5cae0a1e0..e641481a3cd9 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -39,7 +39,6 @@ static int padlock_sha_init(struct shash_desc *desc) struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); dctx->fallback.tfm = ctx->fallback; - dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; return crypto_shash_init(&dctx->fallback); } @@ -48,7 +47,6 @@ static int padlock_sha_update(struct shash_desc *desc, { struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; return crypto_shash_update(&dctx->fallback, data, length); } @@ -65,7 +63,6 @@ static int padlock_sha_import(struct shash_desc *desc, const void *in) struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); dctx->fallback.tfm = ctx->fallback; - dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; return crypto_shash_import(&dctx->fallback, in); } @@ -91,7 +88,6 @@ static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in, unsigned int leftover; int err; - dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; err = crypto_shash_export(&dctx->fallback, &state); if (err) goto out; @@ -153,7 +149,6 @@ static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, unsigned int leftover; int err; - dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; err = crypto_shash_export(&dctx->fallback, &state); if (err) goto out; diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index 1b3acdeffede..05b89e703903 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -753,11 +753,6 @@ static int spacc_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm); u32 tmp[DES_EXPKEY_WORDS]; - if (len > DES3_EDE_KEY_SIZE) { - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } - if (unlikely(!des_ekey(tmp, key)) && (crypto_ablkcipher_get_flags(cipher) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { @@ -772,6 +767,30 @@ static int spacc_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, } /* + * Set the 3DES key for a block cipher transform. This also performs weak key + * checking if the transform has requested it. + */ +static int spacc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + unsigned int len) +{ + struct spacc_ablk_ctx *ctx = crypto_ablkcipher_ctx(cipher); + u32 flags; + int err; + + flags = crypto_ablkcipher_get_flags(cipher); + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(cipher, flags); + return err; + } + + memcpy(ctx->key, key, len); + ctx->key_len = len; + + return 0; +} + +/* * Set the key for an AES block cipher. Some key lengths are not supported in * hardware so this must also check whether a fallback is needed. */ @@ -1196,7 +1215,7 @@ static const struct dev_pm_ops spacc_pm_ops = { static inline struct spacc_engine *spacc_dev_to_engine(struct device *dev) { - return dev ? platform_get_drvdata(to_platform_device(dev)) : NULL; + return dev ? dev_get_drvdata(dev) : NULL; } static ssize_t spacc_stat_irq_thresh_show(struct device *dev, @@ -1353,7 +1372,7 @@ static struct spacc_alg ipsec_engine_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_ablkcipher = { - .setkey = spacc_des_setkey, + .setkey = spacc_des3_setkey, .encrypt = spacc_ablk_encrypt, .decrypt = spacc_ablk_decrypt, .min_keysize = DES3_EDE_KEY_SIZE, @@ -1380,7 +1399,7 @@ static struct spacc_alg ipsec_engine_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_ablkcipher = { - .setkey = spacc_des_setkey, + .setkey = spacc_des3_setkey, .encrypt = spacc_ablk_encrypt, .decrypt = spacc_ablk_decrypt, .min_keysize = DES3_EDE_KEY_SIZE, diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 975c75198f56..c8d401646902 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -164,7 +164,6 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, memset(ctx->ipad, 0, block_size); memset(ctx->opad, 0, block_size); shash->tfm = ctx->hash_tfm; - shash->flags = 0x0; if (auth_keylen > block_size) { int ret = crypto_shash_digest(shash, auth_key, diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index c9f324730d71..692a7aaee749 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -1300,8 +1300,6 @@ static void qat_rsa_exit_tfm(struct crypto_akcipher *tfm) static struct akcipher_alg rsa = { .encrypt = qat_rsa_enc, .decrypt = qat_rsa_dec, - .sign = qat_rsa_dec, - .verify = qat_rsa_enc, .set_pub_key = qat_rsa_setpubkey, .set_priv_key = qat_rsa_setprivkey, .max_size = qat_rsa_max_size, diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c index 154b6baa124e..8d3493855a70 100644 --- a/drivers/crypto/qce/ablkcipher.c +++ b/drivers/crypto/qce/ablkcipher.c @@ -198,6 +198,25 @@ weakkey: return -EINVAL; } +static int qce_des3_setkey(struct crypto_ablkcipher *ablk, const u8 *key, + unsigned int keylen) +{ + struct qce_cipher_ctx *ctx = crypto_ablkcipher_ctx(ablk); + u32 flags; + int err; + + flags = crypto_ablkcipher_get_flags(ablk); + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(ablk, flags); + return err; + } + + ctx->enc_keylen = keylen; + memcpy(ctx->enc_key, key, keylen); + return 0; +} + static int qce_ablkcipher_crypt(struct ablkcipher_request *req, int encrypt) { struct crypto_tfm *tfm = @@ -363,7 +382,8 @@ static int qce_ablkcipher_register_one(const struct qce_ablkcipher_def *def, alg->cra_ablkcipher.ivsize = def->ivsize; alg->cra_ablkcipher.min_keysize = def->min_keysize; alg->cra_ablkcipher.max_keysize = def->max_keysize; - alg->cra_ablkcipher.setkey = qce_ablkcipher_setkey; + alg->cra_ablkcipher.setkey = IS_3DES(def->flags) ? + qce_des3_setkey : qce_ablkcipher_setkey; alg->cra_ablkcipher.encrypt = qce_ablkcipher_encrypt; alg->cra_ablkcipher.decrypt = qce_ablkcipher_decrypt; diff --git a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c index 02dac6ae7e53..313759521a0f 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c @@ -46,24 +46,36 @@ static int rk_aes_setkey(struct crypto_ablkcipher *cipher, return 0; } -static int rk_tdes_setkey(struct crypto_ablkcipher *cipher, - const u8 *key, unsigned int keylen) +static int rk_des_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) { struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm); u32 tmp[DES_EXPKEY_WORDS]; - if (keylen != DES_KEY_SIZE && keylen != DES3_EDE_KEY_SIZE) { - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + if (!des_ekey(tmp, key) && + (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { + tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } - if (keylen == DES_KEY_SIZE) { - if (!des_ekey(tmp, key) && - (tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { - tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } + ctx->keylen = keylen; + memcpy_toio(ctx->dev->reg + RK_CRYPTO_TDES_KEY1_0, key, keylen); + return 0; +} + +static int rk_tdes_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(cipher); + u32 flags; + int err; + + flags = crypto_ablkcipher_get_flags(cipher); + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(cipher, flags); + return err; } ctx->keylen = keylen; @@ -250,9 +262,14 @@ static int rk_set_data_start(struct rk_crypto_info *dev) u8 *src_last_blk = page_address(sg_page(dev->sg_src)) + dev->sg_src->offset + dev->sg_src->length - ivsize; - /* store the iv that need to be updated in chain mode */ - if (ctx->mode & RK_CRYPTO_DEC) + /* Store the iv that need to be updated in chain mode. + * And update the IV buffer to contain the next IV for decryption mode. + */ + if (ctx->mode & RK_CRYPTO_DEC) { memcpy(ctx->iv, src_last_blk, ivsize); + sg_pcopy_to_buffer(dev->first, dev->src_nents, req->info, + ivsize, dev->total - ivsize); + } err = dev->load_data(dev, dev->sg_src, dev->sg_dst); if (!err) @@ -288,13 +305,19 @@ static void rk_iv_copyback(struct rk_crypto_info *dev) struct ablkcipher_request *req = ablkcipher_request_cast(dev->async_req); struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); u32 ivsize = crypto_ablkcipher_ivsize(tfm); - if (ivsize == DES_BLOCK_SIZE) - memcpy_fromio(req->info, dev->reg + RK_CRYPTO_TDES_IV_0, - ivsize); - else if (ivsize == AES_BLOCK_SIZE) - memcpy_fromio(req->info, dev->reg + RK_CRYPTO_AES_IV_0, ivsize); + /* Update the IV buffer to contain the next IV for encryption mode. */ + if (!(ctx->mode & RK_CRYPTO_DEC)) { + if (dev->aligned) { + memcpy(req->info, sg_virt(dev->sg_dst) + + dev->sg_dst->length - ivsize, ivsize); + } else { + memcpy(req->info, dev->addr_vir + + dev->count - ivsize, ivsize); + } + } } static void rk_update_iv(struct rk_crypto_info *dev) @@ -457,7 +480,7 @@ struct rk_crypto_tmp rk_ecb_des_alg = { .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, - .setkey = rk_tdes_setkey, + .setkey = rk_des_setkey, .encrypt = rk_des_ecb_encrypt, .decrypt = rk_des_ecb_decrypt, } @@ -483,7 +506,7 @@ struct rk_crypto_tmp rk_cbc_des_alg = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, .ivsize = DES_BLOCK_SIZE, - .setkey = rk_tdes_setkey, + .setkey = rk_des_setkey, .encrypt = rk_des_cbc_encrypt, .decrypt = rk_des_cbc_decrypt, } diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 1afdcb81d8ed..9ef25230c199 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -1534,7 +1534,6 @@ static int s5p_hash_shash_digest(struct crypto_shash *tfm, u32 flags, SHASH_DESC_ON_STACK(shash, tfm); shash->tfm = tfm; - shash->flags = flags & ~CRYPTO_TFM_REQ_MAY_SLEEP; return crypto_shash_digest(shash, data, len, out); } diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 8c32a3059b4a..fd11162a915e 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -354,7 +354,7 @@ static void sahara_decode_status(struct sahara_dev *dev, unsigned int status) { u8 state; - if (!IS_ENABLED(DEBUG)) + if (!__is_defined(DEBUG)) return; state = SAHARA_STATUS_GET_STATE(status); @@ -406,7 +406,7 @@ static void sahara_dump_descriptors(struct sahara_dev *dev) { int i; - if (!IS_ENABLED(DEBUG)) + if (!__is_defined(DEBUG)) return; for (i = 0; i < SAHARA_MAX_HW_DESC; i++) { @@ -427,7 +427,7 @@ static void sahara_dump_links(struct sahara_dev *dev) { int i; - if (!IS_ENABLED(DEBUG)) + if (!__is_defined(DEBUG)) return; for (i = 0; i < SAHARA_MAX_HW_LINK; i++) { diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig index 63aa78c0b12b..4491e2197d9f 100644 --- a/drivers/crypto/stm32/Kconfig +++ b/drivers/crypto/stm32/Kconfig @@ -24,6 +24,7 @@ config CRYPTO_DEV_STM32_CRYP depends on ARCH_STM32 select CRYPTO_HASH select CRYPTO_ENGINE + select CRYPTO_DES help This enables support for the CRYP (AES/DES/TDES) hw accelerator which can be found on STMicroelectronics STM32 SOC. diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 23b0b7bd64c7..cddcc97875b2 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -137,7 +137,6 @@ struct stm32_cryp { struct crypto_engine *engine; - struct mutex lock; /* protects req / areq */ struct ablkcipher_request *req; struct aead_request *areq; @@ -394,6 +393,23 @@ static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, u32 *iv) } } +static void stm32_cryp_get_iv(struct stm32_cryp *cryp) +{ + struct ablkcipher_request *req = cryp->req; + u32 *tmp = req->info; + + if (!tmp) + return; + + *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR)); + *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR)); + + if (is_aes(cryp)) { + *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR)); + *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR)); + } +} + static void stm32_cryp_hw_write_key(struct stm32_cryp *c) { unsigned int i; @@ -623,6 +639,9 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) /* Phase 4 : output tag */ err = stm32_cryp_read_auth_tag(cryp); + if (!err && (!(is_gcm(cryp) || is_ccm(cryp)))) + stm32_cryp_get_iv(cryp); + if (cryp->sgs_copied) { void *buf_in, *buf_out; int pages, len; @@ -645,18 +664,13 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) pm_runtime_mark_last_busy(cryp->dev); pm_runtime_put_autosuspend(cryp->dev); - if (is_gcm(cryp) || is_ccm(cryp)) { + if (is_gcm(cryp) || is_ccm(cryp)) crypto_finalize_aead_request(cryp->engine, cryp->areq, err); - cryp->areq = NULL; - } else { + else crypto_finalize_ablkcipher_request(cryp->engine, cryp->req, err); - cryp->req = NULL; - } memset(cryp->ctx->key, 0, cryp->ctx->keylen); - - mutex_unlock(&cryp->lock); } static int stm32_cryp_cpu_start(struct stm32_cryp *cryp) @@ -753,19 +767,35 @@ static int stm32_cryp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, static int stm32_cryp_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int keylen) { + u32 tmp[DES_EXPKEY_WORDS]; + if (keylen != DES_KEY_SIZE) return -EINVAL; - else - return stm32_cryp_setkey(tfm, key, keylen); + + if ((crypto_ablkcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) && + unlikely(!des_ekey(tmp, key))) { + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); + return -EINVAL; + } + + return stm32_cryp_setkey(tfm, key, keylen); } static int stm32_cryp_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int keylen) { - if (keylen != (3 * DES_KEY_SIZE)) - return -EINVAL; - else - return stm32_cryp_setkey(tfm, key, keylen); + u32 flags; + int err; + + flags = crypto_ablkcipher_get_flags(tfm); + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(tfm, flags); + return err; + } + + return stm32_cryp_setkey(tfm, key, keylen); } static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, @@ -917,8 +947,6 @@ static int stm32_cryp_prepare_req(struct ablkcipher_request *req, if (!cryp) return -ENODEV; - mutex_lock(&cryp->lock); - rctx = req ? ablkcipher_request_ctx(req) : aead_request_ctx(areq); rctx->mode &= FLG_MODE_MASK; @@ -930,6 +958,7 @@ static int stm32_cryp_prepare_req(struct ablkcipher_request *req, if (req) { cryp->req = req; + cryp->areq = NULL; cryp->total_in = req->nbytes; cryp->total_out = cryp->total_in; } else { @@ -955,6 +984,7 @@ static int stm32_cryp_prepare_req(struct ablkcipher_request *req, * <---------- total_out -----------------> */ cryp->areq = areq; + cryp->req = NULL; cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); cryp->total_in = areq->assoclen + areq->cryptlen; if (is_encrypt(cryp)) @@ -976,19 +1006,19 @@ static int stm32_cryp_prepare_req(struct ablkcipher_request *req, if (cryp->in_sg_len < 0) { dev_err(cryp->dev, "Cannot get in_sg_len\n"); ret = cryp->in_sg_len; - goto out; + return ret; } cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out); if (cryp->out_sg_len < 0) { dev_err(cryp->dev, "Cannot get out_sg_len\n"); ret = cryp->out_sg_len; - goto out; + return ret; } ret = stm32_cryp_copy_sgs(cryp); if (ret) - goto out; + return ret; scatterwalk_start(&cryp->in_walk, cryp->in_sg); scatterwalk_start(&cryp->out_walk, cryp->out_sg); @@ -1000,10 +1030,6 @@ static int stm32_cryp_prepare_req(struct ablkcipher_request *req, } ret = stm32_cryp_hw_init(cryp); -out: - if (ret) - mutex_unlock(&cryp->lock); - return ret; } @@ -1943,8 +1969,6 @@ static int stm32_cryp_probe(struct platform_device *pdev) cryp->dev = dev; - mutex_init(&cryp->lock); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); cryp->regs = devm_ioremap_resource(dev, res); if (IS_ERR(cryp->regs)) diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index 4a6cc8a3045d..bfc49e67124b 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -181,8 +181,6 @@ struct stm32_hash_dev { u32 dma_mode; u32 dma_maxburst; - spinlock_t lock; /* lock to protect queue */ - struct ahash_request *req; struct crypto_engine *engine; @@ -977,7 +975,7 @@ static int stm32_hash_export(struct ahash_request *req, void *out) pm_runtime_get_sync(hdev->dev); - while (!(stm32_hash_read(hdev, HASH_SR) & HASH_SR_DATA_INPUT_READY)) + while ((stm32_hash_read(hdev, HASH_SR) & HASH_SR_BUSY)) cpu_relax(); rctx->hw_context = kmalloc_array(3 + HASH_CSR_REGISTER_NUMBER, diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c index 54fd714d53ca..b060a0810934 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -41,11 +41,6 @@ static int sun4i_ss_opti_poll(struct skcipher_request *areq) if (!areq->cryptlen) return 0; - if (!areq->iv) { - dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n"); - return -EINVAL; - } - if (!areq->src || !areq->dst) { dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n"); return -EINVAL; @@ -134,6 +129,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) struct scatterlist *out_sg = areq->dst; unsigned int ivsize = crypto_skcipher_ivsize(tfm); struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun4i_ss_alg_template *algt; u32 mode = ctx->mode; /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ u32 rx_cnt = SS_RX_DEFAULT; @@ -153,20 +150,20 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) unsigned int obo = 0; /* offset in bufo*/ unsigned int obl = 0; /* length of data in bufo */ unsigned long flags; + bool need_fallback; if (!areq->cryptlen) return 0; - if (!areq->iv) { - dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n"); - return -EINVAL; - } - if (!areq->src || !areq->dst) { dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n"); return -EINVAL; } + algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + if (areq->cryptlen % algt->alg.crypto.base.cra_blocksize) + need_fallback = true; + /* * if we have only SGs with size multiple of 4, * we can use the SS optimized function @@ -182,9 +179,24 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) out_sg = sg_next(out_sg); } - if (no_chunk == 1) + if (no_chunk == 1 && !need_fallback) return sun4i_ss_opti_poll(areq); + if (need_fallback) { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm); + skcipher_request_set_sync_tfm(subreq, op->fallback_tfm); + skcipher_request_set_callback(subreq, areq->base.flags, NULL, + NULL); + skcipher_request_set_crypt(subreq, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (ctx->mode & SS_DECRYPTION) + err = crypto_skcipher_decrypt(subreq); + else + err = crypto_skcipher_encrypt(subreq); + skcipher_request_zero(subreq); + return err; + } + spin_lock_irqsave(&ss->slock, flags); for (i = 0; i < op->keylen; i += 4) @@ -458,6 +470,7 @@ int sun4i_ss_cipher_init(struct crypto_tfm *tfm) { struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm); struct sun4i_ss_alg_template *algt; + const char *name = crypto_tfm_alg_name(tfm); memset(op, 0, sizeof(struct sun4i_tfm_ctx)); @@ -468,9 +481,22 @@ int sun4i_ss_cipher_init(struct crypto_tfm *tfm) crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), sizeof(struct sun4i_cipher_req_ctx)); + op->fallback_tfm = crypto_alloc_sync_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(op->ss->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(op->fallback_tfm)); + return PTR_ERR(op->fallback_tfm); + } + return 0; } +void sun4i_ss_cipher_exit(struct crypto_tfm *tfm) +{ + struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm); + crypto_free_sync_skcipher(op->fallback_tfm); +} + /* check and set the AES key, prepare the mode to be used */ int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -495,7 +521,11 @@ int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, } op->keylen = keylen; memcpy(op->key, key, keylen); - return 0; + + crypto_sync_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(op->fallback_tfm, key, keylen); } /* check and set the DES key, prepare the mode to be used */ @@ -525,7 +555,11 @@ int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key, op->keylen = keylen; memcpy(op->key, key, keylen); - return 0; + + crypto_sync_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(op->fallback_tfm, key, keylen); } /* check and set the 3DES key, prepare the mode to be used */ @@ -533,14 +567,18 @@ int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); - struct sun4i_ss_ctx *ss = op->ss; + int err; + + err = des3_verify_key(tfm, key); + if (unlikely(err)) + return err; - if (unlikely(keylen != 3 * DES_KEY_SIZE)) { - dev_err(ss->dev, "Invalid keylen %u\n", keylen); - crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } op->keylen = keylen; memcpy(op->key, key, keylen); - return 0; + + crypto_sync_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(op->fallback_tfm, key, keylen); + } diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c index 89adf9e0fed2..05b3d3c32f6d 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c @@ -92,11 +92,12 @@ static struct sun4i_ss_alg_template ss_algs[] = { .cra_driver_name = "cbc-aes-sun4i-ss", .cra_priority = 300, .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, } } }, @@ -107,17 +108,17 @@ static struct sun4i_ss_alg_template ss_algs[] = { .decrypt = sun4i_ss_ecb_aes_decrypt, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, .base = { .cra_name = "ecb(aes)", .cra_driver_name = "ecb-aes-sun4i-ss", .cra_priority = 300, .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, } } }, @@ -134,11 +135,12 @@ static struct sun4i_ss_alg_template ss_algs[] = { .cra_driver_name = "cbc-des-sun4i-ss", .cra_priority = 300, .cra_blocksize = DES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, .cra_ctxsize = sizeof(struct sun4i_req_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, } } }, @@ -154,11 +156,12 @@ static struct sun4i_ss_alg_template ss_algs[] = { .cra_driver_name = "ecb-des-sun4i-ss", .cra_priority = 300, .cra_blocksize = DES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, .cra_ctxsize = sizeof(struct sun4i_req_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, } } }, @@ -175,11 +178,12 @@ static struct sun4i_ss_alg_template ss_algs[] = { .cra_driver_name = "cbc-des3-sun4i-ss", .cra_priority = 300, .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, .cra_ctxsize = sizeof(struct sun4i_req_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, } } }, @@ -190,16 +194,17 @@ static struct sun4i_ss_alg_template ss_algs[] = { .decrypt = sun4i_ss_ecb_des3_decrypt, .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, .base = { .cra_name = "ecb(des3_ede)", .cra_driver_name = "ecb-des3-sun4i-ss", .cra_priority = 300, .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_NEED_FALLBACK, .cra_ctxsize = sizeof(struct sun4i_req_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 3, .cra_init = sun4i_ss_cipher_init, + .cra_exit = sun4i_ss_cipher_exit, } } }, diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index a4b5ff2b72f8..f6936bb3b7be 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -240,7 +240,10 @@ static int sun4i_hash(struct ahash_request *areq) } } else { /* Since we have the flag final, we can go up to modulo 4 */ - end = ((areq->nbytes + op->len) / 4) * 4 - op->len; + if (areq->nbytes < 4) + end = 0; + else + end = ((areq->nbytes + op->len) / 4) * 4 - op->len; } /* TODO if SGlen % 4 and !op->len then DMA */ diff --git a/drivers/crypto/sunxi-ss/sun4i-ss.h b/drivers/crypto/sunxi-ss/sun4i-ss.h index f3ac90692ac6..8c4ec9e93565 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss.h +++ b/drivers/crypto/sunxi-ss/sun4i-ss.h @@ -161,6 +161,7 @@ struct sun4i_tfm_ctx { u32 keylen; u32 keymode; struct sun4i_ss_ctx *ss; + struct crypto_sync_skcipher *fallback_tfm; }; struct sun4i_cipher_req_ctx { @@ -203,6 +204,7 @@ int sun4i_ss_ecb_des3_encrypt(struct skcipher_request *areq); int sun4i_ss_ecb_des3_decrypt(struct skcipher_request *areq); int sun4i_ss_cipher_init(struct crypto_tfm *tfm); +void sun4i_ss_cipher_exit(struct crypto_tfm *tfm); int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen); int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key, diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index de78b54bcfb1..1d429fc073d1 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -913,6 +913,54 @@ badkey: return -EINVAL; } +static int aead_des3_setkey(struct crypto_aead *authenc, + const u8 *key, unsigned int keylen) +{ + struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + struct device *dev = ctx->dev; + struct crypto_authenc_keys keys; + u32 flags; + int err; + + err = crypto_authenc_extractkeys(&keys, key, keylen); + if (unlikely(err)) + goto badkey; + + err = -EINVAL; + if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) + goto badkey; + + if (keys.enckeylen != DES3_EDE_KEY_SIZE) + goto badkey; + + flags = crypto_aead_get_flags(authenc); + err = __des3_verify_key(&flags, keys.enckey); + if (unlikely(err)) { + crypto_aead_set_flags(authenc, flags); + goto out; + } + + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); + + ctx->keylen = keys.authkeylen + keys.enckeylen; + ctx->enckeylen = keys.enckeylen; + ctx->authkeylen = keys.authkeylen; + ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, + DMA_TO_DEVICE); + +out: + memzero_explicit(&keys, sizeof(keys)); + return err; + +badkey: + crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); + goto out; +} + /* * talitos_edesc - s/w-extended descriptor * @src_nents: number of segments in input scatterlist @@ -1527,12 +1575,22 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, { struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); struct device *dev = ctx->dev; - u32 tmp[DES_EXPKEY_WORDS]; - if (keylen > TALITOS_MAX_KEY_SIZE) { - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + + memcpy(&ctx->key, key, keylen); + ctx->keylen = keylen; + + ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); + + return 0; +} + +static int ablkcipher_des_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + u32 tmp[DES_EXPKEY_WORDS]; if (unlikely(crypto_ablkcipher_get_flags(cipher) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) && @@ -1541,15 +1599,23 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, return -EINVAL; } - if (ctx->keylen) - dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + return ablkcipher_setkey(cipher, key, keylen); +} - memcpy(&ctx->key, key, keylen); - ctx->keylen = keylen; +static int ablkcipher_des3_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + u32 flags; + int err; - ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); + flags = crypto_ablkcipher_get_flags(cipher); + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(cipher, flags); + return err; + } - return 0; + return ablkcipher_setkey(cipher, key, keylen); } static void common_nonsnoop_unmap(struct device *dev, @@ -2313,6 +2379,7 @@ static struct talitos_alg_template driver_algs[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, + .setkey = aead_des3_setkey, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2336,6 +2403,7 @@ static struct talitos_alg_template driver_algs[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, + .setkey = aead_des3_setkey, }, .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | DESC_HDR_SEL0_DEU | @@ -2399,6 +2467,7 @@ static struct talitos_alg_template driver_algs[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, + .setkey = aead_des3_setkey, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2422,6 +2491,7 @@ static struct talitos_alg_template driver_algs[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, + .setkey = aead_des3_setkey, }, .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | DESC_HDR_SEL0_DEU | @@ -2485,6 +2555,7 @@ static struct talitos_alg_template driver_algs[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, + .setkey = aead_des3_setkey, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2508,6 +2579,7 @@ static struct talitos_alg_template driver_algs[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, + .setkey = aead_des3_setkey, }, .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | DESC_HDR_SEL0_DEU | @@ -2550,6 +2622,7 @@ static struct talitos_alg_template driver_algs[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, + .setkey = aead_des3_setkey, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2592,6 +2665,7 @@ static struct talitos_alg_template driver_algs[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, + .setkey = aead_des3_setkey, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2654,6 +2728,7 @@ static struct talitos_alg_template driver_algs[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, + .setkey = aead_des3_setkey, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2676,6 +2751,7 @@ static struct talitos_alg_template driver_algs[] = { }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, + .setkey = aead_des3_setkey, }, .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | DESC_HDR_SEL0_DEU | @@ -2748,6 +2824,7 @@ static struct talitos_alg_template driver_algs[] = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, .ivsize = DES_BLOCK_SIZE, + .setkey = ablkcipher_des_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2764,6 +2841,7 @@ static struct talitos_alg_template driver_algs[] = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, .ivsize = DES_BLOCK_SIZE, + .setkey = ablkcipher_des_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2781,6 +2859,7 @@ static struct talitos_alg_template driver_algs[] = { .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = ablkcipher_des3_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2798,6 +2877,7 @@ static struct talitos_alg_template driver_algs[] = { .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = ablkcipher_des3_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -3144,7 +3224,8 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, alg->cra_init = talitos_cra_init; alg->cra_exit = talitos_cra_exit; alg->cra_type = &crypto_ablkcipher_type; - alg->cra_ablkcipher.setkey = ablkcipher_setkey; + alg->cra_ablkcipher.setkey = alg->cra_ablkcipher.setkey ?: + ablkcipher_setkey; alg->cra_ablkcipher.encrypt = ablkcipher_encrypt; alg->cra_ablkcipher.decrypt = ablkcipher_decrypt; break; @@ -3152,7 +3233,8 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, alg = &t_alg->algt.alg.aead.base; alg->cra_exit = talitos_cra_exit; t_alg->algt.alg.aead.init = talitos_cra_init_aead; - t_alg->algt.alg.aead.setkey = aead_setkey; + t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?: + aead_setkey; t_alg->algt.alg.aead.encrypt = aead_encrypt; t_alg->algt.alg.aead.decrypt = aead_decrypt; if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && diff --git a/drivers/crypto/ux500/cryp/Makefile b/drivers/crypto/ux500/cryp/Makefile index b497ae3dde07..dc8fff29c4b3 100644 --- a/drivers/crypto/ux500/cryp/Makefile +++ b/drivers/crypto/ux500/cryp/Makefile @@ -3,11 +3,7 @@ # * Author: shujuan.chen@stericsson.com for ST-Ericsson. # * License terms: GNU General Public License (GPL) version 2 */ -ifdef CONFIG_CRYPTO_DEV_UX500_DEBUG -CFLAGS_cryp_core.o := -DDEBUG -CFLAGS_cryp.o := -DDEBUG -CFLAGS_cryp_irq.o := -DDEBUG -endif +ccflags-$(CONFIG_CRYPTO_DEV_UX500_DEBUG) += -DDEBUG obj-$(CONFIG_CRYPTO_DEV_UX500_CRYP) += ux500_cryp.o ux500_cryp-objs := cryp.o cryp_irq.o cryp_core.o diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index 3235611928f2..7a93cba0877f 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -1019,37 +1019,16 @@ static int des3_ablkcipher_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher); - u32 *flags = &cipher->base.crt_flags; - const u32 *K = (const u32 *)key; - u32 tmp[DES3_EDE_EXPKEY_WORDS]; - int i, ret; + u32 flags; + int err; pr_debug(DEV_DBG_NAME " [%s]", __func__); - if (keylen != DES3_EDE_KEY_SIZE) { - *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; - pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_BAD_KEY_LEN", - __func__); - return -EINVAL; - } - /* Checking key interdependency for weak key detection. */ - if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_WEAK_KEY", - __func__); - return -EINVAL; - } - for (i = 0; i < 3; i++) { - ret = des_ekey(tmp, key + i*DES_KEY_SIZE); - if (unlikely(ret == 0) && - (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_WEAK_KEY", - __func__); - return -EINVAL; - } + flags = crypto_ablkcipher_get_flags(cipher); + err = __des3_verify_key(&flags, key); + if (unlikely(err)) { + crypto_ablkcipher_set_flags(cipher, flags); + return err; } memcpy(ctx->key, key, keylen); @@ -1219,57 +1198,6 @@ static struct cryp_algo_template cryp_algs[] = { { .algomode = CRYP_ALGO_DES_ECB, .crypto = { - .cra_name = "des", - .cra_driver_name = "des-ux500", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_ASYNC, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct cryp_ctx), - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_init = cryp_cra_init, - .cra_module = THIS_MODULE, - .cra_u = { - .ablkcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = des_ablkcipher_setkey, - .encrypt = cryp_blk_encrypt, - .decrypt = cryp_blk_decrypt - } - } - } - - }, - { - .algomode = CRYP_ALGO_TDES_ECB, - .crypto = { - .cra_name = "des3_ede", - .cra_driver_name = "des3_ede-ux500", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_ASYNC, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct cryp_ctx), - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_init = cryp_cra_init, - .cra_module = THIS_MODULE, - .cra_u = { - .ablkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .setkey = des_ablkcipher_setkey, - .encrypt = cryp_blk_encrypt, - .decrypt = cryp_blk_decrypt - } - } - } - }, - { - .algomode = CRYP_ALGO_DES_ECB, - .crypto = { .cra_name = "ecb(des)", .cra_driver_name = "ecb-des-ux500", .cra_priority = 300, diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c index d7316f7a3a69..603a62081994 100644 --- a/drivers/crypto/vmx/aes.c +++ b/drivers/crypto/vmx/aes.c @@ -23,9 +23,10 @@ #include <linux/err.h> #include <linux/crypto.h> #include <linux/delay.h> -#include <linux/hardirq.h> +#include <asm/simd.h> #include <asm/switch_to.h> #include <crypto/aes.h> +#include <crypto/internal/simd.h> #include "aesp8-ppc.h" @@ -78,20 +79,21 @@ static int p8_aes_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_disable(); enable_kernel_vsx(); ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); - ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key); + ret |= aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key); disable_kernel_vsx(); pagefault_enable(); preempt_enable(); - ret += crypto_cipher_setkey(ctx->fallback, key, keylen); - return ret; + ret |= crypto_cipher_setkey(ctx->fallback, key, keylen); + + return ret ? -EINVAL : 0; } static void p8_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); - if (in_interrupt()) { + if (!crypto_simd_usable()) { crypto_cipher_encrypt_one(ctx->fallback, dst, src); } else { preempt_disable(); @@ -108,7 +110,7 @@ static void p8_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); - if (in_interrupt()) { + if (!crypto_simd_usable()) { crypto_cipher_decrypt_one(ctx->fallback, dst, src); } else { preempt_disable(); diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c index c5c5ff82b52e..a1a9a6f0d42c 100644 --- a/drivers/crypto/vmx/aes_cbc.c +++ b/drivers/crypto/vmx/aes_cbc.c @@ -23,9 +23,10 @@ #include <linux/err.h> #include <linux/crypto.h> #include <linux/delay.h> -#include <linux/hardirq.h> +#include <asm/simd.h> #include <asm/switch_to.h> #include <crypto/aes.h> +#include <crypto/internal/simd.h> #include <crypto/scatterwalk.h> #include <crypto/skcipher.h> @@ -81,13 +82,14 @@ static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_disable(); enable_kernel_vsx(); ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); - ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key); + ret |= aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key); disable_kernel_vsx(); pagefault_enable(); preempt_enable(); - ret += crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); - return ret; + ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); + + return ret ? -EINVAL : 0; } static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc, @@ -99,7 +101,7 @@ static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc, struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); - if (in_interrupt()) { + if (!crypto_simd_usable()) { SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); skcipher_request_set_sync_tfm(req, ctx->fallback); skcipher_request_set_callback(req, desc->flags, NULL, NULL); @@ -138,7 +140,7 @@ static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc, struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); - if (in_interrupt()) { + if (!crypto_simd_usable()) { SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); skcipher_request_set_sync_tfm(req, ctx->fallback); skcipher_request_set_callback(req, desc->flags, NULL, NULL); diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c index 8a2fe092cb8e..192a53512f5e 100644 --- a/drivers/crypto/vmx/aes_ctr.c +++ b/drivers/crypto/vmx/aes_ctr.c @@ -23,9 +23,10 @@ #include <linux/err.h> #include <linux/crypto.h> #include <linux/delay.h> -#include <linux/hardirq.h> +#include <asm/simd.h> #include <asm/switch_to.h> #include <crypto/aes.h> +#include <crypto/internal/simd.h> #include <crypto/scatterwalk.h> #include <crypto/skcipher.h> @@ -83,8 +84,9 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_enable(); preempt_enable(); - ret += crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); - return ret; + ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); + + return ret ? -EINVAL : 0; } static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx, @@ -118,7 +120,7 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc, struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); - if (in_interrupt()) { + if (!crypto_simd_usable()) { SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); skcipher_request_set_sync_tfm(req, ctx->fallback); skcipher_request_set_callback(req, desc->flags, NULL, NULL); diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c index ecd64e5cc5bb..00d412d811ae 100644 --- a/drivers/crypto/vmx/aes_xts.c +++ b/drivers/crypto/vmx/aes_xts.c @@ -23,9 +23,10 @@ #include <linux/err.h> #include <linux/crypto.h> #include <linux/delay.h> -#include <linux/hardirq.h> +#include <asm/simd.h> #include <asm/switch_to.h> #include <crypto/aes.h> +#include <crypto/internal/simd.h> #include <crypto/scatterwalk.h> #include <crypto/xts.h> #include <crypto/skcipher.h> @@ -86,14 +87,15 @@ static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_disable(); enable_kernel_vsx(); ret = aes_p8_set_encrypt_key(key + keylen/2, (keylen/2) * 8, &ctx->tweak_key); - ret += aes_p8_set_encrypt_key(key, (keylen/2) * 8, &ctx->enc_key); - ret += aes_p8_set_decrypt_key(key, (keylen/2) * 8, &ctx->dec_key); + ret |= aes_p8_set_encrypt_key(key, (keylen/2) * 8, &ctx->enc_key); + ret |= aes_p8_set_decrypt_key(key, (keylen/2) * 8, &ctx->dec_key); disable_kernel_vsx(); pagefault_enable(); preempt_enable(); - ret += crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); - return ret; + ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); + + return ret ? -EINVAL : 0; } static int p8_aes_xts_crypt(struct blkcipher_desc *desc, @@ -108,7 +110,7 @@ static int p8_aes_xts_crypt(struct blkcipher_desc *desc, struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); - if (in_interrupt()) { + if (!crypto_simd_usable()) { SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); skcipher_request_set_sync_tfm(req, ctx->fallback); skcipher_request_set_callback(req, desc->flags, NULL, NULL); diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl index d6a9f63d65ba..de78282b8f44 100644 --- a/drivers/crypto/vmx/aesp8-ppc.pl +++ b/drivers/crypto/vmx/aesp8-ppc.pl @@ -1854,7 +1854,7 @@ Lctr32_enc8x_three: stvx_u $out1,$x10,$out stvx_u $out2,$x20,$out addi $out,$out,0x30 - b Lcbc_dec8x_done + b Lctr32_enc8x_done .align 5 Lctr32_enc8x_two: @@ -1866,7 +1866,7 @@ Lctr32_enc8x_two: stvx_u $out0,$x00,$out stvx_u $out1,$x10,$out addi $out,$out,0x20 - b Lcbc_dec8x_done + b Lctr32_enc8x_done .align 5 Lctr32_enc8x_one: diff --git a/drivers/crypto/vmx/ghash.c b/drivers/crypto/vmx/ghash.c index dd8b8716467a..b5a6883bb09e 100644 --- a/drivers/crypto/vmx/ghash.c +++ b/drivers/crypto/vmx/ghash.c @@ -23,16 +23,15 @@ #include <linux/err.h> #include <linux/crypto.h> #include <linux/delay.h> -#include <linux/hardirq.h> +#include <asm/simd.h> #include <asm/switch_to.h> #include <crypto/aes.h> #include <crypto/ghash.h> #include <crypto/scatterwalk.h> #include <crypto/internal/hash.h> +#include <crypto/internal/simd.h> #include <crypto/b128ops.h> -#define IN_INTERRUPT in_interrupt() - void gcm_init_p8(u128 htable[16], const u64 Xi[2]); void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); void gcm_ghash_p8(u64 Xi[2], const u128 htable[16], @@ -102,7 +101,6 @@ static int p8_ghash_init(struct shash_desc *desc) dctx->bytes = 0; memset(dctx->shash, 0, GHASH_DIGEST_SIZE); dctx->fallback_desc.tfm = ctx->fallback; - dctx->fallback_desc.flags = desc->flags; return crypto_shash_init(&dctx->fallback_desc); } @@ -131,7 +129,7 @@ static int p8_ghash_update(struct shash_desc *desc, struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - if (IN_INTERRUPT) { + if (!crypto_simd_usable()) { return crypto_shash_update(&dctx->fallback_desc, src, srclen); } else { @@ -182,7 +180,7 @@ static int p8_ghash_final(struct shash_desc *desc, u8 *out) struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - if (IN_INTERRUPT) { + if (!crypto_simd_usable()) { return crypto_shash_final(&dctx->fallback_desc, out); } else { if (dctx->bytes) { diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c index 31a98dc6f849..a9f519830615 100644 --- a/drivers/crypto/vmx/vmx.c +++ b/drivers/crypto/vmx/vmx.c @@ -41,7 +41,7 @@ static struct crypto_alg *algs[] = { NULL, }; -int __init p8_init(void) +static int __init p8_init(void) { int ret = 0; struct crypto_alg **alg_it; @@ -67,7 +67,7 @@ int __init p8_init(void) return ret; } -void __exit p8_exit(void) +static void __exit p8_exit(void) { struct crypto_alg **alg_it; diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h index 2e2dff478833..ecf6e659c0da 100644 --- a/drivers/infiniband/sw/rxe/rxe.h +++ b/drivers/infiniband/sw/rxe/rxe.h @@ -80,7 +80,6 @@ static inline u32 rxe_crc32(struct rxe_dev *rxe, SHASH_DESC_ON_STACK(shash, rxe->tfm); shash->tfm = rxe->tfm; - shash->flags = 0; *(u32 *)shash_desc_ctx(shash) = crc; err = crypto_shash_update(shash, next, len); if (unlikely(err)) { diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index dd6565798778..9faed1c92b52 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -332,7 +332,6 @@ static int crypt_iv_essiv_init(struct crypt_config *cc) int err; desc->tfm = essiv->hash_tfm; - desc->flags = 0; err = crypto_shash_digest(desc, cc->key, cc->key_size, essiv->salt); shash_desc_zero(desc); @@ -606,7 +605,6 @@ static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv, int i, r; desc->tfm = lmk->hash_tfm; - desc->flags = 0; r = crypto_shash_init(desc); if (r) @@ -768,7 +766,6 @@ static int crypt_iv_tcw_whitening(struct crypt_config *cc, /* calculate crc32 for every 32bit part and xor it */ desc->tfm = tcw->crc32_tfm; - desc->flags = 0; for (i = 0; i < 4; i++) { r = crypto_shash_init(desc); if (r) diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 7c678f50aaa3..95ae4bf34203 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -532,7 +532,6 @@ static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result unsigned j, size; desc->tfm = ic->journal_mac; - desc->flags = 0; r = crypto_shash_init(desc); if (unlikely(r)) { @@ -1276,7 +1275,6 @@ static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector unsigned digest_size; req->tfm = ic->internal_hash; - req->flags = 0; r = crypto_shash_init(req); if (unlikely(r < 0)) { diff --git a/drivers/net/ppp/ppp_mppe.c b/drivers/net/ppp/ppp_mppe.c index 7ccdc62c6052..ff61dd8748de 100644 --- a/drivers/net/ppp/ppp_mppe.c +++ b/drivers/net/ppp/ppp_mppe.c @@ -222,7 +222,6 @@ static void *mppe_alloc(unsigned char *options, int optlen) goto out_free; } state->sha1->tfm = shash; - state->sha1->flags = 0; digestsize = crypto_shash_digestsize(shash); if (digestsize < MPPE_MAX_KEY_LEN) diff --git a/drivers/net/wireless/intersil/orinoco/mic.c b/drivers/net/wireless/intersil/orinoco/mic.c index 67b0c05afbdb..a324bc4b7938 100644 --- a/drivers/net/wireless/intersil/orinoco/mic.c +++ b/drivers/net/wireless/intersil/orinoco/mic.c @@ -65,7 +65,6 @@ int orinoco_mic(struct crypto_shash *tfm_michael, u8 *key, hdr[ETH_ALEN * 2 + 3] = 0; desc->tfm = tfm_michael; - desc->flags = 0; err = crypto_shash_setkey(tfm_michael, key, MIC_KEYLEN); if (err) diff --git a/drivers/nfc/s3fwrn5/firmware.c b/drivers/nfc/s3fwrn5/firmware.c index b7828fb252f2..b681073ae8ba 100644 --- a/drivers/nfc/s3fwrn5/firmware.c +++ b/drivers/nfc/s3fwrn5/firmware.c @@ -449,7 +449,6 @@ int s3fwrn5_fw_download(struct s3fwrn5_fw_info *fw_info) SHASH_DESC_ON_STACK(desc, tfm); desc->tfm = tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; ret = crypto_shash_digest(desc, fw->image, image_size, hash_data); diff --git a/drivers/staging/ks7010/ks_hostif.c b/drivers/staging/ks7010/ks_hostif.c index 06ebea0be118..122d4c0af363 100644 --- a/drivers/staging/ks7010/ks_hostif.c +++ b/drivers/staging/ks7010/ks_hostif.c @@ -219,7 +219,6 @@ michael_mic(u8 *key, u8 *data, unsigned int len, u8 priority, u8 *result) } desc->tfm = tfm; - desc->flags = 0; ret = crypto_shash_init(desc); if (ret < 0) diff --git a/drivers/staging/rtl8192e/rtllib_crypt_tkip.c b/drivers/staging/rtl8192e/rtllib_crypt_tkip.c index 55da8c9dfe50..a084e1501f9d 100644 --- a/drivers/staging/rtl8192e/rtllib_crypt_tkip.c +++ b/drivers/staging/rtl8192e/rtllib_crypt_tkip.c @@ -507,7 +507,6 @@ static int michael_mic(struct crypto_shash *tfm_michael, u8 *key, u8 *hdr, int err; desc->tfm = tfm_michael; - desc->flags = 0; if (crypto_shash_setkey(tfm_michael, key, 8)) return -1; diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c index 829fa4bd253c..d67bb57994c4 100644 --- a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c +++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c @@ -503,7 +503,6 @@ static int michael_mic(struct crypto_shash *tfm_michael, u8 *key, u8 *hdr, int err; desc->tfm = tfm_michael; - desc->flags = 0; if (crypto_shash_setkey(tfm_michael, key, 8)) return -1; diff --git a/drivers/target/iscsi/iscsi_target_auth.c b/drivers/target/iscsi/iscsi_target_auth.c index 4e680d753941..ca7d7e8aecc0 100644 --- a/drivers/target/iscsi/iscsi_target_auth.c +++ b/drivers/target/iscsi/iscsi_target_auth.c @@ -252,7 +252,6 @@ static int chap_server_compute_md5( } desc->tfm = tfm; - desc->flags = 0; ret = crypto_shash_init(desc); if (ret < 0) { diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 7416bdbd8576..b7980c856898 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -678,7 +678,6 @@ int tb_domain_challenge_switch_key(struct tb *tb, struct tb_switch *sw) } shash->tfm = tfm; - shash->flags = CRYPTO_TFM_REQ_MAY_SLEEP; memset(hmac, 0, sizeof(hmac)); ret = crypto_shash_digest(shash, challenge, sizeof(hmac), hmac); diff --git a/fs/cifs/misc.c b/fs/cifs/misc.c index 0dc6f08020ac..b1a696a73f7c 100644 --- a/fs/cifs/misc.c +++ b/fs/cifs/misc.c @@ -959,7 +959,6 @@ cifs_alloc_hash(const char *name, } (*sdesc)->shash.tfm = *shash; - (*sdesc)->shash.flags = 0x0; return 0; } diff --git a/fs/crypto/keyinfo.c b/fs/crypto/keyinfo.c index 322ce9686bdb..2cb4956f8511 100644 --- a/fs/crypto/keyinfo.c +++ b/fs/crypto/keyinfo.c @@ -402,7 +402,6 @@ static int derive_essiv_salt(const u8 *key, int keysize, u8 *salt) { SHASH_DESC_ON_STACK(desc, tfm); desc->tfm = tfm; - desc->flags = 0; return crypto_shash_digest(desc, key, keysize, salt); } diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c index f664da55234e..491cf5baa8c2 100644 --- a/fs/ecryptfs/crypto.c +++ b/fs/ecryptfs/crypto.c @@ -68,7 +68,6 @@ static int ecryptfs_hash_digest(struct crypto_shash *tfm, int err; desc->tfm = tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; err = crypto_shash_digest(desc, src, len, dst); shash_desc_zero(desc); return err; diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c index e74fe84d0886..90fbac5d485b 100644 --- a/fs/ecryptfs/keystore.c +++ b/fs/ecryptfs/keystore.c @@ -769,7 +769,6 @@ ecryptfs_write_tag_70_packet(char *dest, size_t *remaining_bytes, } s->hash_desc->tfm = s->hash_tfm; - s->hash_desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; rc = crypto_shash_digest(s->hash_desc, (u8 *)s->auth_tok->token.password.session_key_encryption_key, diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 82ffdacdc7fa..0833b5fc0668 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -2024,7 +2024,6 @@ static inline u32 ext4_chksum(struct ext4_sb_info *sbi, u32 crc, BUG_ON(crypto_shash_descsize(sbi->s_chksum_driver)!=sizeof(desc.ctx)); desc.shash.tfm = sbi->s_chksum_driver; - desc.shash.flags = 0; *(u32 *)desc.ctx = crc; BUG_ON(crypto_shash_update(&desc.shash, address, length)); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 87f75ebd2fd6..21b0ab6bd15a 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1422,7 +1422,6 @@ static inline u32 __f2fs_crc32(struct f2fs_sb_info *sbi, u32 crc, BUG_ON(crypto_shash_descsize(sbi->s_chksum_driver) != sizeof(desc.ctx)); desc.shash.tfm = sbi->s_chksum_driver; - desc.shash.flags = 0; *(u32 *)desc.ctx = crc; err = crypto_shash_update(&desc.shash, address, length); diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index 5188f9f70c78..8c8563441208 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -126,7 +126,6 @@ nfs4_make_rec_clidname(char *dname, const struct xdr_netobj *clname) SHASH_DESC_ON_STACK(desc, tfm); desc->tfm = tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; status = crypto_shash_digest(desc, clname->data, clname->len, cksum.data); diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c index 5bf5fd08879e..b758004085c4 100644 --- a/fs/ubifs/auth.c +++ b/fs/ubifs/auth.c @@ -33,7 +33,6 @@ int __ubifs_node_calc_hash(const struct ubifs_info *c, const void *node, int err; shash->tfm = c->hash_tfm; - shash->flags = CRYPTO_TFM_REQ_MAY_SLEEP; err = crypto_shash_digest(shash, node, le32_to_cpu(ch->len), hash); if (err < 0) @@ -56,7 +55,6 @@ static int ubifs_hash_calc_hmac(const struct ubifs_info *c, const u8 *hash, int err; shash->tfm = c->hmac_tfm; - shash->flags = CRYPTO_TFM_REQ_MAY_SLEEP; err = crypto_shash_digest(shash, hash, c->hash_len, hmac); if (err < 0) @@ -88,7 +86,6 @@ int ubifs_prepare_auth_node(struct ubifs_info *c, void *node, return -ENOMEM; hash_desc->tfm = c->hash_tfm; - hash_desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; ubifs_shash_copy_state(c, inhash, hash_desc); err = crypto_shash_final(hash_desc, hash); @@ -123,7 +120,6 @@ static struct shash_desc *ubifs_get_desc(const struct ubifs_info *c, return ERR_PTR(-ENOMEM); desc->tfm = tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; err = crypto_shash_init(desc); if (err) { @@ -364,7 +360,6 @@ static int ubifs_node_calc_hmac(const struct ubifs_info *c, const void *node, ubifs_assert(c, ofs_hmac + hmac_len < len); shash->tfm = c->hmac_tfm; - shash->flags = CRYPTO_TFM_REQ_MAY_SLEEP; err = crypto_shash_init(shash); if (err) @@ -483,7 +478,6 @@ int ubifs_hmac_wkm(struct ubifs_info *c, u8 *hmac) return 0; shash->tfm = c->hmac_tfm; - shash->flags = CRYPTO_TFM_REQ_MAY_SLEEP; err = crypto_shash_init(shash); if (err) diff --git a/fs/ubifs/replay.c b/fs/ubifs/replay.c index 0a0e65c07c6d..5c8a81a019a4 100644 --- a/fs/ubifs/replay.c +++ b/fs/ubifs/replay.c @@ -576,7 +576,6 @@ static int authenticate_sleb_hash(struct ubifs_info *c, struct shash_desc *log_h SHASH_DESC_ON_STACK(hash_desc, c->hash_tfm); hash_desc->tfm = c->hash_tfm; - hash_desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; ubifs_shash_copy_state(c, log_hash, hash_desc); return crypto_shash_final(hash_desc, hash); @@ -587,7 +586,6 @@ static int authenticate_sleb_hmac(struct ubifs_info *c, u8 *hash, u8 *hmac) SHASH_DESC_ON_STACK(hmac_desc, c->hmac_tfm); hmac_desc->tfm = c->hmac_tfm; - hmac_desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; return crypto_shash_digest(hmac_desc, hash, c->hash_len, hmac); } diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 852eaa9cd4db..0fdb542c70cd 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -28,10 +28,10 @@ struct crypto_aes_ctx { u32 key_length; }; -extern const u32 crypto_ft_tab[4][256]; -extern const u32 crypto_fl_tab[4][256]; -extern const u32 crypto_it_tab[4][256]; -extern const u32 crypto_il_tab[4][256]; +extern const u32 crypto_ft_tab[4][256] ____cacheline_aligned; +extern const u32 crypto_fl_tab[4][256] ____cacheline_aligned; +extern const u32 crypto_it_tab[4][256] ____cacheline_aligned; +extern const u32 crypto_il_tab[4][256] ____cacheline_aligned; int crypto_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len); diff --git a/include/crypto/akcipher.h b/include/crypto/akcipher.h index 2d690494568c..8884046659a0 100644 --- a/include/crypto/akcipher.h +++ b/include/crypto/akcipher.h @@ -19,14 +19,20 @@ * * @base: Common attributes for async crypto requests * @src: Source data - * @dst: Destination data + * For verify op this is signature + digest, in that case + * total size of @src is @src_len + @dst_len. + * @dst: Destination data (Should be NULL for verify op) * @src_len: Size of the input buffer - * @dst_len: Size of the output buffer. It needs to be at least - * as big as the expected result depending on the operation + * For verify op it's size of signature part of @src, this part + * is supposed to be operated by cipher. + * @dst_len: Size of @dst buffer (for all ops except verify). + * It needs to be at least as big as the expected result + * depending on the operation. * After operation it will be updated with the actual size of the * result. * In case of error where the dst sgl size was insufficient, * it will be updated to the size required for the operation. + * For verify op this is size of digest part in @src. * @__ctx: Start of private context data */ struct akcipher_request { @@ -55,10 +61,9 @@ struct crypto_akcipher { * algorithm. In case of error, where the dst_len was insufficient, * the req->dst_len will be updated to the size required for the * operation - * @verify: Function performs a sign operation as defined by public key - * algorithm. In case of error, where the dst_len was insufficient, - * the req->dst_len will be updated to the size required for the - * operation + * @verify: Function performs a complete verify operation as defined by + * public key algorithm, returning verification status. Requires + * digest value as input parameter. * @encrypt: Function performs an encrypt operation as defined by public key * algorithm. In case of error, where the dst_len was insufficient, * the req->dst_len will be updated to the size required for the @@ -69,10 +74,10 @@ struct crypto_akcipher { * operation * @set_pub_key: Function invokes the algorithm specific set public key * function, which knows how to decode and interpret - * the BER encoded public key + * the BER encoded public key and parameters * @set_priv_key: Function invokes the algorithm specific set private key * function, which knows how to decode and interpret - * the BER encoded private key + * the BER encoded private key and parameters * @max_size: Function returns dest buffer size required for a given key. * @init: Initialize the cryptographic transformation object. * This function is used to initialize the cryptographic @@ -238,9 +243,10 @@ static inline void akcipher_request_set_callback(struct akcipher_request *req, * * @req: public key request * @src: ptr to input scatter list - * @dst: ptr to output scatter list + * @dst: ptr to output scatter list or NULL for verify op * @src_len: size of the src input scatter list to be processed - * @dst_len: size of the dst output scatter list + * @dst_len: size of the dst output scatter list or size of signature + * portion in @src for verify op */ static inline void akcipher_request_set_crypt(struct akcipher_request *req, struct scatterlist *src, @@ -343,14 +349,18 @@ static inline int crypto_akcipher_sign(struct akcipher_request *req) } /** - * crypto_akcipher_verify() - Invoke public key verify operation + * crypto_akcipher_verify() - Invoke public key signature verification * - * Function invokes the specific public key verify operation for a given - * public key algorithm + * Function invokes the specific public key signature verification operation + * for a given public key algorithm. * * @req: asymmetric key request * - * Return: zero on success; error code in case of error + * Note: req->dst should be NULL, req->src should point to SG of size + * (req->src_size + req->dst_size), containing signature (of req->src_size + * length) with appended digest (of req->dst_size length). + * + * Return: zero on verification success; error code in case of error. */ static inline int crypto_akcipher_verify(struct akcipher_request *req) { @@ -369,11 +379,12 @@ static inline int crypto_akcipher_verify(struct akcipher_request *req) * crypto_akcipher_set_pub_key() - Invoke set public key operation * * Function invokes the algorithm specific set key function, which knows - * how to decode and interpret the encoded key + * how to decode and interpret the encoded key and parameters * * @tfm: tfm handle - * @key: BER encoded public key - * @keylen: length of the key + * @key: BER encoded public key, algo OID, paramlen, BER encoded + * parameters + * @keylen: length of the key (not including other data) * * Return: zero on success; error code in case of error */ @@ -390,11 +401,12 @@ static inline int crypto_akcipher_set_pub_key(struct crypto_akcipher *tfm, * crypto_akcipher_set_priv_key() - Invoke set private key operation * * Function invokes the algorithm specific set key function, which knows - * how to decode and interpret the encoded key + * how to decode and interpret the encoded key and parameters * * @tfm: tfm handle - * @key: BER encoded private key - * @keylen: length of the key + * @key: BER encoded private key, algo OID, paramlen, BER encoded + * parameters + * @keylen: length of the key (not including other data) * * Return: zero on success; error code in case of error */ diff --git a/include/crypto/cryptd.h b/include/crypto/cryptd.h index 1e64f354c2b8..23169f4d87e6 100644 --- a/include/crypto/cryptd.h +++ b/include/crypto/cryptd.h @@ -18,27 +18,11 @@ #include <crypto/hash.h> #include <crypto/skcipher.h> -struct cryptd_ablkcipher { - struct crypto_ablkcipher base; -}; - -static inline struct cryptd_ablkcipher *__cryptd_ablkcipher_cast( - struct crypto_ablkcipher *tfm) -{ - return (struct cryptd_ablkcipher *)tfm; -} - -/* alg_name should be algorithm to be cryptd-ed */ -struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, - u32 type, u32 mask); -struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm); -bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm); -void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm); - struct cryptd_skcipher { struct crypto_skcipher base; }; +/* alg_name should be algorithm to be cryptd-ed */ struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name, u32 type, u32 mask); struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm); diff --git a/include/crypto/des.h b/include/crypto/des.h index d4094d58ac54..72c7c8e5a5a7 100644 --- a/include/crypto/des.h +++ b/include/crypto/des.h @@ -6,6 +6,11 @@ #ifndef __CRYPTO_DES_H #define __CRYPTO_DES_H +#include <crypto/skcipher.h> +#include <linux/compiler.h> +#include <linux/fips.h> +#include <linux/string.h> + #define DES_KEY_SIZE 8 #define DES_EXPKEY_WORDS 32 #define DES_BLOCK_SIZE 8 @@ -14,6 +19,44 @@ #define DES3_EDE_EXPKEY_WORDS (3 * DES_EXPKEY_WORDS) #define DES3_EDE_BLOCK_SIZE DES_BLOCK_SIZE +static inline int __des3_verify_key(u32 *flags, const u8 *key) +{ + int err = -EINVAL; + u32 K[6]; + + memcpy(K, key, DES3_EDE_KEY_SIZE); + + if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || + !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && + (fips_enabled || + (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS))) + goto bad; + + if (unlikely(!((K[0] ^ K[4]) | (K[1] ^ K[5]))) && fips_enabled) + goto bad; + + err = 0; + +out: + memzero_explicit(K, DES3_EDE_KEY_SIZE); + + return err; + +bad: + *flags |= CRYPTO_TFM_RES_WEAK_KEY; + goto out; +} + +static inline int des3_verify_key(struct crypto_skcipher *tfm, const u8 *key) +{ + u32 flags; + int err; + + flags = crypto_skcipher_get_flags(tfm); + err = __des3_verify_key(&flags, key); + crypto_skcipher_set_flags(tfm, flags); + return err; +} extern unsigned long des_ekey(u32 *pe, const u8 *k); diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 3b31c1b349ae..d21bea2c4382 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -146,8 +146,6 @@ struct ahash_alg { struct shash_desc { struct crypto_shash *tfm; - u32 flags; - void *__ctx[] CRYPTO_MINALIGN_ATTR; }; @@ -819,6 +817,7 @@ static inline void *shash_desc_ctx(struct shash_desc *desc) * cipher handle must point to a keyed message digest cipher in order for this * function to succeed. * + * Context: Any context. * Return: 0 if the setting of the key was successful; < 0 if an error occurred */ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, @@ -835,6 +834,7 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, * crypto_shash_update and crypto_shash_final. The parameters have the same * meaning as discussed for those separate three functions. * + * Context: Any context. * Return: 0 if the message digest creation was successful; < 0 if an error * occurred */ @@ -850,6 +850,7 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data, * caller-allocated output buffer out which must have sufficient size (e.g. by * calling crypto_shash_descsize). * + * Context: Any context. * Return: 0 if the export creation was successful; < 0 if an error occurred */ static inline int crypto_shash_export(struct shash_desc *desc, void *out) @@ -866,6 +867,7 @@ static inline int crypto_shash_export(struct shash_desc *desc, void *out) * the input buffer. That buffer should have been generated with the * crypto_ahash_export function. * + * Context: Any context. * Return: 0 if the import was successful; < 0 if an error occurred */ static inline int crypto_shash_import(struct shash_desc *desc, const void *in) @@ -886,6 +888,7 @@ static inline int crypto_shash_import(struct shash_desc *desc, const void *in) * operational state handle. Any potentially existing state created by * previous operations is discarded. * + * Context: Any context. * Return: 0 if the message digest initialization was successful; < 0 if an * error occurred */ @@ -907,6 +910,7 @@ static inline int crypto_shash_init(struct shash_desc *desc) * * Updates the message digest state of the operational state handle. * + * Context: Any context. * Return: 0 if the message digest update was successful; < 0 if an error * occurred */ @@ -923,6 +927,7 @@ int crypto_shash_update(struct shash_desc *desc, const u8 *data, * into the output buffer. The caller must ensure that the output buffer is * large enough by using crypto_shash_digestsize. * + * Context: Any context. * Return: 0 if the message digest creation was successful; < 0 if an error * occurred */ @@ -939,6 +944,7 @@ int crypto_shash_final(struct shash_desc *desc, u8 *out); * crypto_shash_update and crypto_shash_final. The parameters have the same * meaning as discussed for those separate functions. * + * Context: Any context. * Return: 0 if the message digest creation was successful; < 0 if an error * occurred */ diff --git a/include/crypto/internal/simd.h b/include/crypto/internal/simd.h index f18344518e32..d2316242a988 100644 --- a/include/crypto/internal/simd.h +++ b/include/crypto/internal/simd.h @@ -6,6 +6,11 @@ #ifndef _CRYPTO_INTERNAL_SIMD_H #define _CRYPTO_INTERNAL_SIMD_H +#include <linux/percpu.h> +#include <linux/types.h> + +/* skcipher support */ + struct simd_skcipher_alg; struct skcipher_alg; @@ -22,4 +27,43 @@ int simd_register_skciphers_compat(struct skcipher_alg *algs, int count, void simd_unregister_skciphers(struct skcipher_alg *algs, int count, struct simd_skcipher_alg **simd_algs); +/* AEAD support */ + +struct simd_aead_alg; +struct aead_alg; + +struct simd_aead_alg *simd_aead_create_compat(const char *algname, + const char *drvname, + const char *basename); +struct simd_aead_alg *simd_aead_create(const char *algname, + const char *basename); +void simd_aead_free(struct simd_aead_alg *alg); + +int simd_register_aeads_compat(struct aead_alg *algs, int count, + struct simd_aead_alg **simd_algs); + +void simd_unregister_aeads(struct aead_alg *algs, int count, + struct simd_aead_alg **simd_algs); + +/* + * crypto_simd_usable() - is it allowed at this time to use SIMD instructions or + * access the SIMD register file? + * + * This delegates to may_use_simd(), except that this also returns false if SIMD + * in crypto code has been temporarily disabled on this CPU by the crypto + * self-tests, in order to test the no-SIMD fallback code. This override is + * currently limited to configurations where the extra self-tests are enabled, + * because it might be a bit too invasive to be part of the regular self-tests. + * + * This is a macro so that <asm/simd.h>, which some architectures don't have, + * doesn't have to be included directly here. + */ +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +DECLARE_PER_CPU(bool, crypto_simd_disabled_for_test); +#define crypto_simd_usable() \ + (may_use_simd() && !this_cpu_read(crypto_simd_disabled_for_test)) +#else +#define crypto_simd_usable() may_use_simd() +#endif + #endif /* _CRYPTO_INTERNAL_SIMD_H */ diff --git a/include/crypto/morus1280_glue.h b/include/crypto/morus1280_glue.h index ad2aa743dd99..5cefddb1991f 100644 --- a/include/crypto/morus1280_glue.h +++ b/include/crypto/morus1280_glue.h @@ -47,16 +47,7 @@ int crypto_morus1280_glue_setauthsize(struct crypto_aead *tfm, int crypto_morus1280_glue_encrypt(struct aead_request *req); int crypto_morus1280_glue_decrypt(struct aead_request *req); -int cryptd_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int keylen); -int cryptd_morus1280_glue_setauthsize(struct crypto_aead *aead, - unsigned int authsize); -int cryptd_morus1280_glue_encrypt(struct aead_request *req); -int cryptd_morus1280_glue_decrypt(struct aead_request *req); -int cryptd_morus1280_glue_init_tfm(struct crypto_aead *aead); -void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead); - -#define MORUS1280_DECLARE_ALGS(id, driver_name, priority) \ +#define MORUS1280_DECLARE_ALG(id, driver_name, priority) \ static const struct morus1280_glue_ops crypto_morus1280_##id##_ops = {\ .init = crypto_morus1280_##id##_init, \ .ad = crypto_morus1280_##id##_ad, \ @@ -77,55 +68,29 @@ void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead); { \ } \ \ - static struct aead_alg crypto_morus1280_##id##_algs[] = {\ - { \ - .setkey = crypto_morus1280_glue_setkey, \ - .setauthsize = crypto_morus1280_glue_setauthsize, \ - .encrypt = crypto_morus1280_glue_encrypt, \ - .decrypt = crypto_morus1280_glue_decrypt, \ - .init = crypto_morus1280_##id##_init_tfm, \ - .exit = crypto_morus1280_##id##_exit_tfm, \ - \ - .ivsize = MORUS_NONCE_SIZE, \ - .maxauthsize = MORUS_MAX_AUTH_SIZE, \ - .chunksize = MORUS1280_BLOCK_SIZE, \ - \ - .base = { \ - .cra_flags = CRYPTO_ALG_INTERNAL, \ - .cra_blocksize = 1, \ - .cra_ctxsize = sizeof(struct morus1280_ctx), \ - .cra_alignmask = 0, \ - \ - .cra_name = "__morus1280", \ - .cra_driver_name = "__"driver_name, \ - \ - .cra_module = THIS_MODULE, \ - } \ - }, { \ - .setkey = cryptd_morus1280_glue_setkey, \ - .setauthsize = cryptd_morus1280_glue_setauthsize, \ - .encrypt = cryptd_morus1280_glue_encrypt, \ - .decrypt = cryptd_morus1280_glue_decrypt, \ - .init = cryptd_morus1280_glue_init_tfm, \ - .exit = cryptd_morus1280_glue_exit_tfm, \ + static struct aead_alg crypto_morus1280_##id##_alg = { \ + .setkey = crypto_morus1280_glue_setkey, \ + .setauthsize = crypto_morus1280_glue_setauthsize, \ + .encrypt = crypto_morus1280_glue_encrypt, \ + .decrypt = crypto_morus1280_glue_decrypt, \ + .init = crypto_morus1280_##id##_init_tfm, \ + .exit = crypto_morus1280_##id##_exit_tfm, \ + \ + .ivsize = MORUS_NONCE_SIZE, \ + .maxauthsize = MORUS_MAX_AUTH_SIZE, \ + .chunksize = MORUS1280_BLOCK_SIZE, \ + \ + .base = { \ + .cra_flags = CRYPTO_ALG_INTERNAL, \ + .cra_blocksize = 1, \ + .cra_ctxsize = sizeof(struct morus1280_ctx), \ + .cra_alignmask = 0, \ + .cra_priority = priority, \ \ - .ivsize = MORUS_NONCE_SIZE, \ - .maxauthsize = MORUS_MAX_AUTH_SIZE, \ - .chunksize = MORUS1280_BLOCK_SIZE, \ + .cra_name = "__morus1280", \ + .cra_driver_name = "__"driver_name, \ \ - .base = { \ - .cra_flags = CRYPTO_ALG_ASYNC, \ - .cra_blocksize = 1, \ - .cra_ctxsize = sizeof(struct crypto_aead *), \ - .cra_alignmask = 0, \ - \ - .cra_priority = priority, \ - \ - .cra_name = "morus1280", \ - .cra_driver_name = driver_name, \ - \ - .cra_module = THIS_MODULE, \ - } \ + .cra_module = THIS_MODULE, \ } \ } diff --git a/include/crypto/morus640_glue.h b/include/crypto/morus640_glue.h index df8e1103ff94..0ee6266cb26c 100644 --- a/include/crypto/morus640_glue.h +++ b/include/crypto/morus640_glue.h @@ -47,16 +47,7 @@ int crypto_morus640_glue_setauthsize(struct crypto_aead *tfm, int crypto_morus640_glue_encrypt(struct aead_request *req); int crypto_morus640_glue_decrypt(struct aead_request *req); -int cryptd_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int keylen); -int cryptd_morus640_glue_setauthsize(struct crypto_aead *aead, - unsigned int authsize); -int cryptd_morus640_glue_encrypt(struct aead_request *req); -int cryptd_morus640_glue_decrypt(struct aead_request *req); -int cryptd_morus640_glue_init_tfm(struct crypto_aead *aead); -void cryptd_morus640_glue_exit_tfm(struct crypto_aead *aead); - -#define MORUS640_DECLARE_ALGS(id, driver_name, priority) \ +#define MORUS640_DECLARE_ALG(id, driver_name, priority) \ static const struct morus640_glue_ops crypto_morus640_##id##_ops = {\ .init = crypto_morus640_##id##_init, \ .ad = crypto_morus640_##id##_ad, \ @@ -77,55 +68,29 @@ void cryptd_morus640_glue_exit_tfm(struct crypto_aead *aead); { \ } \ \ - static struct aead_alg crypto_morus640_##id##_algs[] = {\ - { \ - .setkey = crypto_morus640_glue_setkey, \ - .setauthsize = crypto_morus640_glue_setauthsize, \ - .encrypt = crypto_morus640_glue_encrypt, \ - .decrypt = crypto_morus640_glue_decrypt, \ - .init = crypto_morus640_##id##_init_tfm, \ - .exit = crypto_morus640_##id##_exit_tfm, \ - \ - .ivsize = MORUS_NONCE_SIZE, \ - .maxauthsize = MORUS_MAX_AUTH_SIZE, \ - .chunksize = MORUS640_BLOCK_SIZE, \ - \ - .base = { \ - .cra_flags = CRYPTO_ALG_INTERNAL, \ - .cra_blocksize = 1, \ - .cra_ctxsize = sizeof(struct morus640_ctx), \ - .cra_alignmask = 0, \ - \ - .cra_name = "__morus640", \ - .cra_driver_name = "__"driver_name, \ - \ - .cra_module = THIS_MODULE, \ - } \ - }, { \ - .setkey = cryptd_morus640_glue_setkey, \ - .setauthsize = cryptd_morus640_glue_setauthsize, \ - .encrypt = cryptd_morus640_glue_encrypt, \ - .decrypt = cryptd_morus640_glue_decrypt, \ - .init = cryptd_morus640_glue_init_tfm, \ - .exit = cryptd_morus640_glue_exit_tfm, \ + static struct aead_alg crypto_morus640_##id##_alg = {\ + .setkey = crypto_morus640_glue_setkey, \ + .setauthsize = crypto_morus640_glue_setauthsize, \ + .encrypt = crypto_morus640_glue_encrypt, \ + .decrypt = crypto_morus640_glue_decrypt, \ + .init = crypto_morus640_##id##_init_tfm, \ + .exit = crypto_morus640_##id##_exit_tfm, \ + \ + .ivsize = MORUS_NONCE_SIZE, \ + .maxauthsize = MORUS_MAX_AUTH_SIZE, \ + .chunksize = MORUS640_BLOCK_SIZE, \ + \ + .base = { \ + .cra_flags = CRYPTO_ALG_INTERNAL, \ + .cra_blocksize = 1, \ + .cra_ctxsize = sizeof(struct morus640_ctx), \ + .cra_alignmask = 0, \ + .cra_priority = priority, \ \ - .ivsize = MORUS_NONCE_SIZE, \ - .maxauthsize = MORUS_MAX_AUTH_SIZE, \ - .chunksize = MORUS640_BLOCK_SIZE, \ + .cra_name = "__morus640", \ + .cra_driver_name = "__"driver_name, \ \ - .base = { \ - .cra_flags = CRYPTO_ALG_ASYNC, \ - .cra_blocksize = 1, \ - .cra_ctxsize = sizeof(struct crypto_aead *), \ - .cra_alignmask = 0, \ - \ - .cra_priority = priority, \ - \ - .cra_name = "morus640", \ - .cra_driver_name = driver_name, \ - \ - .cra_module = THIS_MODULE, \ - } \ + .cra_module = THIS_MODULE, \ } \ } diff --git a/include/crypto/public_key.h b/include/crypto/public_key.h index be626eac9113..712fe1214b5f 100644 --- a/include/crypto/public_key.h +++ b/include/crypto/public_key.h @@ -15,6 +15,7 @@ #define _LINUX_PUBLIC_KEY_H #include <linux/keyctl.h> +#include <linux/oid_registry.h> /* * Cryptographic data for the public-key subtype of the asymmetric key type. @@ -25,6 +26,9 @@ struct public_key { void *key; u32 keylen; + enum OID algo; + void *params; + u32 paramlen; bool key_is_private; const char *id_type; const char *pkey_algo; diff --git a/include/crypto/streebog.h b/include/crypto/streebog.h index 856e32af8657..cae1b4a01971 100644 --- a/include/crypto/streebog.h +++ b/include/crypto/streebog.h @@ -23,7 +23,10 @@ struct streebog_uint512 { }; struct streebog_state { - u8 buffer[STREEBOG_BLOCK_SIZE]; + union { + u8 buffer[STREEBOG_BLOCK_SIZE]; + struct streebog_uint512 m; + }; struct streebog_uint512 hash; struct streebog_uint512 h; struct streebog_uint512 N; diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index 0f919d5fe84f..c2ffff5f9ae2 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -1606,7 +1606,6 @@ static inline u32 jbd2_chksum(journal_t *journal, u32 crc, JBD_MAX_CHECKSUM_SIZE); desc.shash.tfm = journal->j_chksum_driver; - desc.shash.flags = 0; *(u32 *)desc.ctx = crc; err = crypto_shash_update(&desc.shash, address, length); diff --git a/include/linux/oid_registry.h b/include/linux/oid_registry.h index d2fa9ca42e9a..7f30446348c4 100644 --- a/include/linux/oid_registry.h +++ b/include/linux/oid_registry.h @@ -93,6 +93,24 @@ enum OID { OID_authorityKeyIdentifier, /* 2.5.29.35 */ OID_extKeyUsage, /* 2.5.29.37 */ + /* EC-RDSA */ + OID_gostCPSignA, /* 1.2.643.2.2.35.1 */ + OID_gostCPSignB, /* 1.2.643.2.2.35.2 */ + OID_gostCPSignC, /* 1.2.643.2.2.35.3 */ + OID_gost2012PKey256, /* 1.2.643.7.1.1.1.1 */ + OID_gost2012PKey512, /* 1.2.643.7.1.1.1.2 */ + OID_gost2012Digest256, /* 1.2.643.7.1.1.2.2 */ + OID_gost2012Digest512, /* 1.2.643.7.1.1.2.3 */ + OID_gost2012Signature256, /* 1.2.643.7.1.1.3.2 */ + OID_gost2012Signature512, /* 1.2.643.7.1.1.3.3 */ + OID_gostTC26Sign256A, /* 1.2.643.7.1.2.1.1.1 */ + OID_gostTC26Sign256B, /* 1.2.643.7.1.2.1.1.2 */ + OID_gostTC26Sign256C, /* 1.2.643.7.1.2.1.1.3 */ + OID_gostTC26Sign256D, /* 1.2.643.7.1.2.1.1.4 */ + OID_gostTC26Sign512A, /* 1.2.643.7.1.2.1.2.1 */ + OID_gostTC26Sign512B, /* 1.2.643.7.1.2.1.2.2 */ + OID_gostTC26Sign512C, /* 1.2.643.7.1.2.1.2.3 */ + OID__NR }; diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index 827c601841c4..6f89fc8d4b8e 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -5,8 +5,7 @@ * * Author: Brijesh Singh <brijesh.singh@amd.com> * - * SEV spec 0.14 is available at: - * http://support.amd.com/TechDocs/55766_SEV-KM API_Specification.pdf + * SEV API spec is available at https://developer.amd.com/sev * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/include/uapi/linux/psp-sev.h b/include/uapi/linux/psp-sev.h index ac8c60bcc83b..43521d500c2b 100644 --- a/include/uapi/linux/psp-sev.h +++ b/include/uapi/linux/psp-sev.h @@ -6,8 +6,7 @@ * * Author: Brijesh Singh <brijesh.singh@amd.com> * - * SEV spec 0.14 is available at: - * http://support.amd.com/TechDocs/55766_SEV-KM%20API_Specification.pdf + * SEV API specification is available at: https://developer.amd.com/sev/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -30,7 +29,8 @@ enum { SEV_PDH_GEN, SEV_PDH_CERT_EXPORT, SEV_PEK_CERT_IMPORT, - SEV_GET_ID, + SEV_GET_ID, /* This command is deprecated, use SEV_GET_ID2 */ + SEV_GET_ID2, SEV_MAX, }; @@ -125,7 +125,7 @@ struct sev_user_data_pdh_cert_export { } __packed; /** - * struct sev_user_data_get_id - GET_ID command parameters + * struct sev_user_data_get_id - GET_ID command parameters (deprecated) * * @socket1: Buffer to pass unique ID of first socket * @socket2: Buffer to pass unique ID of second socket @@ -136,6 +136,16 @@ struct sev_user_data_get_id { } __packed; /** + * struct sev_user_data_get_id2 - GET_ID command parameters + * @address: Buffer to store unique ID + * @length: length of the unique ID + */ +struct sev_user_data_get_id2 { + __u64 address; /* In */ + __u32 length; /* In/Out */ +} __packed; + +/** * struct sev_issue_cmd - SEV ioctl parameters * * @cmd: SEV commands to execute diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index f1d0e00a3971..f7fb8f6a688f 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -688,7 +688,6 @@ static int kexec_calculate_store_digests(struct kimage *image) goto out_free_desc; desc->tfm = tfm; - desc->flags = 0; ret = crypto_shash_init(desc); if (ret < 0) diff --git a/lib/crc-t10dif.c b/lib/crc-t10dif.c index 4d0d47c1ffbd..e89ebfdbb0fc 100644 --- a/lib/crc-t10dif.c +++ b/lib/crc-t10dif.c @@ -69,7 +69,6 @@ __u16 crc_t10dif_update(__u16 crc, const unsigned char *buffer, size_t len) rcu_read_lock(); desc.shash.tfm = rcu_dereference(crct10dif_tfm); - desc.shash.flags = 0; *(__u16 *)desc.ctx = crc; err = crypto_shash_update(&desc.shash, buffer, len); diff --git a/lib/digsig.c b/lib/digsig.c index 6ba6fcd92dd1..3b0a579bdcdf 100644 --- a/lib/digsig.c +++ b/lib/digsig.c @@ -240,7 +240,6 @@ int digsig_verify(struct key *keyring, const char *sig, int siglen, goto err; desc->tfm = shash; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; crypto_shash_init(desc); crypto_shash_update(desc, data, datalen); diff --git a/lib/libcrc32c.c b/lib/libcrc32c.c index f0a2934605bf..4e9829c4d64c 100644 --- a/lib/libcrc32c.c +++ b/lib/libcrc32c.c @@ -47,7 +47,6 @@ u32 crc32c(u32 crc, const void *address, unsigned int length) int err; shash->tfm = tfm; - shash->flags = 0; *ctx = crc; err = crypto_shash_update(shash, address, length); diff --git a/net/bluetooth/amp.c b/net/bluetooth/amp.c index 78bec8df8525..aaa39409eeb7 100644 --- a/net/bluetooth/amp.c +++ b/net/bluetooth/amp.c @@ -161,7 +161,6 @@ static int hmac_sha256(u8 *key, u8 ksize, char *plaintext, u8 psize, u8 *output) } shash->tfm = tfm; - shash->flags = CRYPTO_TFM_REQ_MAY_SLEEP; ret = crypto_shash_digest(shash, plaintext, psize, output); diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index 621146d04c03..e68c715f8d37 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -183,7 +183,6 @@ static int aes_cmac(struct crypto_shash *tfm, const u8 k[16], const u8 *m, } desc->tfm = tfm; - desc->flags = 0; /* Swap key and message from LSB to MSB */ swap_buf(k, tmp, 16); diff --git a/net/sctp/auth.c b/net/sctp/auth.c index 39d72e58b8e5..31569f4809f6 100644 --- a/net/sctp/auth.c +++ b/net/sctp/auth.c @@ -760,7 +760,6 @@ void sctp_auth_calculate_hmac(const struct sctp_association *asoc, SHASH_DESC_ON_STACK(desc, tfm); desc->tfm = tfm; - desc->flags = 0; crypto_shash_digest(desc, (u8 *)auth, end - (unsigned char *)auth, digest); shash_desc_zero(desc); diff --git a/net/sctp/sm_make_chunk.c b/net/sctp/sm_make_chunk.c index d05c57664e36..72e74503f9fc 100644 --- a/net/sctp/sm_make_chunk.c +++ b/net/sctp/sm_make_chunk.c @@ -1684,7 +1684,6 @@ static struct sctp_cookie_param *sctp_pack_cookie( /* Sign the message. */ desc->tfm = sctp_sk(ep->base.sk)->hmac; - desc->flags = 0; err = crypto_shash_setkey(desc->tfm, ep->secret_key, sizeof(ep->secret_key)) ?: @@ -1755,7 +1754,6 @@ struct sctp_association *sctp_unpack_cookie( int err; desc->tfm = sctp_sk(ep->base.sk)->hmac; - desc->flags = 0; err = crypto_shash_setkey(desc->tfm, ep->secret_key, sizeof(ep->secret_key)) ?: diff --git a/net/sunrpc/auth_gss/gss_krb5_crypto.c b/net/sunrpc/auth_gss/gss_krb5_crypto.c index 4f43383971ba..6f2d30d7b766 100644 --- a/net/sunrpc/auth_gss/gss_krb5_crypto.c +++ b/net/sunrpc/auth_gss/gss_krb5_crypto.c @@ -977,7 +977,6 @@ krb5_rc4_setup_seq_key(struct krb5_ctx *kctx, } desc->tfm = hmac; - desc->flags = 0; /* Compute intermediate Kseq from session key */ err = crypto_shash_setkey(hmac, kctx->Ksess, kctx->gk5e->keylength); @@ -1045,7 +1044,6 @@ krb5_rc4_setup_enc_key(struct krb5_ctx *kctx, } desc->tfm = hmac; - desc->flags = 0; /* Compute intermediate Kcrypt from session key */ for (i = 0; i < kctx->gk5e->keylength; i++) diff --git a/net/sunrpc/auth_gss/gss_krb5_mech.c b/net/sunrpc/auth_gss/gss_krb5_mech.c index 56cc85c5bc06..6e5d6d240215 100644 --- a/net/sunrpc/auth_gss/gss_krb5_mech.c +++ b/net/sunrpc/auth_gss/gss_krb5_mech.c @@ -438,7 +438,6 @@ context_derive_keys_rc4(struct krb5_ctx *ctx) } desc->tfm = hmac; - desc->flags = 0; err = crypto_shash_digest(desc, sigkeyconstant, slen, ctx->cksum); kzfree(desc); diff --git a/net/wireless/lib80211_crypt_tkip.c b/net/wireless/lib80211_crypt_tkip.c index 35f06563207d..11eaa5956f00 100644 --- a/net/wireless/lib80211_crypt_tkip.c +++ b/net/wireless/lib80211_crypt_tkip.c @@ -501,7 +501,6 @@ static int michael_mic(struct crypto_shash *tfm_michael, u8 *key, u8 *hdr, } desc->tfm = tfm_michael; - desc->flags = 0; if (crypto_shash_setkey(tfm_michael, key, 8)) return -1; diff --git a/security/apparmor/crypto.c b/security/apparmor/crypto.c index af03d98c7552..baba63bc66b1 100644 --- a/security/apparmor/crypto.c +++ b/security/apparmor/crypto.c @@ -43,7 +43,6 @@ char *aa_calc_hash(void *data, size_t len) goto fail; desc->tfm = apparmor_tfm; - desc->flags = 0; error = crypto_shash_init(desc); if (error) @@ -81,7 +80,6 @@ int aa_calc_profile_hash(struct aa_profile *profile, u32 version, void *start, goto fail; desc->tfm = apparmor_tfm; - desc->flags = 0; error = crypto_shash_init(desc); if (error) diff --git a/security/integrity/digsig_asymmetric.c b/security/integrity/digsig_asymmetric.c index d775e03fbbcc..99080871eb9f 100644 --- a/security/integrity/digsig_asymmetric.c +++ b/security/integrity/digsig_asymmetric.c @@ -104,9 +104,16 @@ int asymmetric_verify(struct key *keyring, const char *sig, memset(&pks, 0, sizeof(pks)); - pks.pkey_algo = "rsa"; pks.hash_algo = hash_algo_name[hdr->hash_algo]; - pks.encoding = "pkcs1"; + if (hdr->hash_algo == HASH_ALGO_STREEBOG_256 || + hdr->hash_algo == HASH_ALGO_STREEBOG_512) { + /* EC-RDSA and Streebog should go together. */ + pks.pkey_algo = "ecrdsa"; + pks.encoding = "raw"; + } else { + pks.pkey_algo = "rsa"; + pks.encoding = "pkcs1"; + } pks.digest = (u8 *)data; pks.digest_size = datalen; pks.s = hdr->sig; diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index c37d08118af5..e11564eb645b 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -124,7 +124,6 @@ out: return ERR_PTR(-ENOMEM); desc->tfm = *tfm; - desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; rc = crypto_shash_init(desc); if (rc) { diff --git a/security/integrity/ima/ima_crypto.c b/security/integrity/ima/ima_crypto.c index 16a4f45863b1..a32878e10ebc 100644 --- a/security/integrity/ima/ima_crypto.c +++ b/security/integrity/ima/ima_crypto.c @@ -333,7 +333,6 @@ static int ima_calc_file_hash_tfm(struct file *file, SHASH_DESC_ON_STACK(shash, tfm); shash->tfm = tfm; - shash->flags = 0; hash->length = crypto_shash_digestsize(tfm); @@ -469,7 +468,6 @@ static int ima_calc_field_array_hash_tfm(struct ima_field_data *field_data, int rc, i; shash->tfm = tfm; - shash->flags = 0; hash->length = crypto_shash_digestsize(tfm); @@ -591,7 +589,6 @@ static int calc_buffer_shash_tfm(const void *buf, loff_t size, int rc; shash->tfm = tfm; - shash->flags = 0; hash->length = crypto_shash_digestsize(tfm); @@ -664,7 +661,6 @@ static int __init ima_calc_boot_aggregate_tfm(char *digest, SHASH_DESC_ON_STACK(shash, tfm); shash->tfm = tfm; - shash->flags = 0; rc = crypto_shash_init(shash); if (rc != 0) diff --git a/security/keys/dh.c b/security/keys/dh.c index 711e89d8c415..23f95dec771b 100644 --- a/security/keys/dh.c +++ b/security/keys/dh.c @@ -112,7 +112,6 @@ static int kdf_alloc(struct kdf_sdesc **sdesc_ret, char *hashname) if (!sdesc) goto out_free_tfm; sdesc->shash.tfm = tfm; - sdesc->shash.flags = 0x0; *sdesc_ret = sdesc; diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c index 347108f660a1..1b1456b21a93 100644 --- a/security/keys/encrypted-keys/encrypted.c +++ b/security/keys/encrypted-keys/encrypted.c @@ -333,7 +333,6 @@ static int calc_hash(struct crypto_shash *tfm, u8 *digest, int err; desc->tfm = tfm; - desc->flags = 0; err = crypto_shash_digest(desc, buf, buflen, digest); shash_desc_zero(desc); diff --git a/security/keys/trusted.c b/security/keys/trusted.c index efdbf17f3915..a75b2f0f1230 100644 --- a/security/keys/trusted.c +++ b/security/keys/trusted.c @@ -55,7 +55,6 @@ static struct sdesc *init_sdesc(struct crypto_shash *alg) if (!sdesc) return ERR_PTR(-ENOMEM); sdesc->shash.tfm = alg; - sdesc->shash.flags = 0x0; return sdesc; } |