Commit e151a8d2 authored by Eric Biggers's avatar Eric Biggers Committed by Herbert Xu

crypto: x86/morus1280 - convert to use AEAD SIMD helpers

Convert the x86 implementations of MORUS-1280 to use the AEAD SIMD
helpers, rather than hand-rolling the same functionality.  This
simplifies the code and also fixes the bug where the user-provided
aead_request is modified.
Signed-off-by: default avatarEric Biggers <ebiggers@google.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 47730958
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
*/ */
#include <crypto/internal/aead.h> #include <crypto/internal/aead.h>
#include <crypto/internal/simd.h>
#include <crypto/morus1280_glue.h> #include <crypto/morus1280_glue.h>
#include <linux/module.h> #include <linux/module.h>
#include <asm/fpu/api.h> #include <asm/fpu/api.h>
...@@ -35,7 +36,9 @@ asmlinkage void crypto_morus1280_avx2_dec_tail(void *state, const void *src, ...@@ -35,7 +36,9 @@ asmlinkage void crypto_morus1280_avx2_dec_tail(void *state, const void *src,
asmlinkage void crypto_morus1280_avx2_final(void *state, void *tag_xor, asmlinkage void crypto_morus1280_avx2_final(void *state, void *tag_xor,
u64 assoclen, u64 cryptlen); u64 assoclen, u64 cryptlen);
MORUS1280_DECLARE_ALGS(avx2, "morus1280-avx2", 400); MORUS1280_DECLARE_ALG(avx2, "morus1280-avx2", 400);
static struct simd_aead_alg *simd_alg;
static int __init crypto_morus1280_avx2_module_init(void) static int __init crypto_morus1280_avx2_module_init(void)
{ {
...@@ -44,14 +47,13 @@ static int __init crypto_morus1280_avx2_module_init(void) ...@@ -44,14 +47,13 @@ static int __init crypto_morus1280_avx2_module_init(void)
!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
return -ENODEV; return -ENODEV;
return crypto_register_aeads(crypto_morus1280_avx2_algs, return simd_register_aeads_compat(&crypto_morus1280_avx2_alg, 1,
ARRAY_SIZE(crypto_morus1280_avx2_algs)); &simd_alg);
} }
static void __exit crypto_morus1280_avx2_module_exit(void) static void __exit crypto_morus1280_avx2_module_exit(void)
{ {
crypto_unregister_aeads(crypto_morus1280_avx2_algs, simd_unregister_aeads(&crypto_morus1280_avx2_alg, 1, &simd_alg);
ARRAY_SIZE(crypto_morus1280_avx2_algs));
} }
module_init(crypto_morus1280_avx2_module_init); module_init(crypto_morus1280_avx2_module_init);
......
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
*/ */
#include <crypto/internal/aead.h> #include <crypto/internal/aead.h>
#include <crypto/internal/simd.h>
#include <crypto/morus1280_glue.h> #include <crypto/morus1280_glue.h>
#include <linux/module.h> #include <linux/module.h>
#include <asm/fpu/api.h> #include <asm/fpu/api.h>
...@@ -35,7 +36,9 @@ asmlinkage void crypto_morus1280_sse2_dec_tail(void *state, const void *src, ...@@ -35,7 +36,9 @@ asmlinkage void crypto_morus1280_sse2_dec_tail(void *state, const void *src,
asmlinkage void crypto_morus1280_sse2_final(void *state, void *tag_xor, asmlinkage void crypto_morus1280_sse2_final(void *state, void *tag_xor,
u64 assoclen, u64 cryptlen); u64 assoclen, u64 cryptlen);
MORUS1280_DECLARE_ALGS(sse2, "morus1280-sse2", 350); MORUS1280_DECLARE_ALG(sse2, "morus1280-sse2", 350);
static struct simd_aead_alg *simd_alg;
static int __init crypto_morus1280_sse2_module_init(void) static int __init crypto_morus1280_sse2_module_init(void)
{ {
...@@ -43,14 +46,13 @@ static int __init crypto_morus1280_sse2_module_init(void) ...@@ -43,14 +46,13 @@ static int __init crypto_morus1280_sse2_module_init(void)
!cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
return -ENODEV; return -ENODEV;
return crypto_register_aeads(crypto_morus1280_sse2_algs, return simd_register_aeads_compat(&crypto_morus1280_sse2_alg, 1,
ARRAY_SIZE(crypto_morus1280_sse2_algs)); &simd_alg);
} }
static void __exit crypto_morus1280_sse2_module_exit(void) static void __exit crypto_morus1280_sse2_module_exit(void)
{ {
crypto_unregister_aeads(crypto_morus1280_sse2_algs, simd_unregister_aeads(&crypto_morus1280_sse2_alg, 1, &simd_alg);
ARRAY_SIZE(crypto_morus1280_sse2_algs));
} }
module_init(crypto_morus1280_sse2_module_init); module_init(crypto_morus1280_sse2_module_init);
......
...@@ -11,7 +11,6 @@ ...@@ -11,7 +11,6 @@
* any later version. * any later version.
*/ */
#include <crypto/cryptd.h>
#include <crypto/internal/aead.h> #include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h> #include <crypto/internal/skcipher.h>
#include <crypto/morus1280_glue.h> #include <crypto/morus1280_glue.h>
...@@ -205,90 +204,6 @@ void crypto_morus1280_glue_init_ops(struct crypto_aead *aead, ...@@ -205,90 +204,6 @@ void crypto_morus1280_glue_init_ops(struct crypto_aead *aead,
} }
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_init_ops); EXPORT_SYMBOL_GPL(crypto_morus1280_glue_init_ops);
int cryptd_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
struct cryptd_aead *cryptd_tfm = *ctx;
return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_setkey);
int cryptd_morus1280_glue_setauthsize(struct crypto_aead *aead,
unsigned int authsize)
{
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
struct cryptd_aead *cryptd_tfm = *ctx;
return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_setauthsize);
int cryptd_morus1280_glue_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
struct cryptd_aead *cryptd_tfm = *ctx;
aead = &cryptd_tfm->base;
if (irq_fpu_usable() && (!in_atomic() ||
!cryptd_aead_queued(cryptd_tfm)))
aead = cryptd_aead_child(cryptd_tfm);
aead_request_set_tfm(req, aead);
return crypto_aead_encrypt(req);
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_encrypt);
int cryptd_morus1280_glue_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
struct cryptd_aead *cryptd_tfm = *ctx;
aead = &cryptd_tfm->base;
if (irq_fpu_usable() && (!in_atomic() ||
!cryptd_aead_queued(cryptd_tfm)))
aead = cryptd_aead_child(cryptd_tfm);
aead_request_set_tfm(req, aead);
return crypto_aead_decrypt(req);
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_decrypt);
int cryptd_morus1280_glue_init_tfm(struct crypto_aead *aead)
{
struct cryptd_aead *cryptd_tfm;
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
const char *name = crypto_aead_alg(aead)->base.cra_driver_name;
char internal_name[CRYPTO_MAX_ALG_NAME];
if (snprintf(internal_name, CRYPTO_MAX_ALG_NAME, "__%s", name)
>= CRYPTO_MAX_ALG_NAME)
return -ENAMETOOLONG;
cryptd_tfm = cryptd_alloc_aead(internal_name, CRYPTO_ALG_INTERNAL,
CRYPTO_ALG_INTERNAL);
if (IS_ERR(cryptd_tfm))
return PTR_ERR(cryptd_tfm);
*ctx = cryptd_tfm;
crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
return 0;
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_init_tfm);
void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead)
{
struct cryptd_aead **ctx = crypto_aead_ctx(aead);
cryptd_free_aead(*ctx);
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_exit_tfm);
MODULE_LICENSE("GPL"); MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>"); MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-1280 AEAD mode -- glue for x86 optimizations"); MODULE_DESCRIPTION("MORUS-1280 AEAD mode -- glue for x86 optimizations");
...@@ -363,7 +363,7 @@ config CRYPTO_MORUS1280_GLUE ...@@ -363,7 +363,7 @@ config CRYPTO_MORUS1280_GLUE
tristate tristate
depends on X86 depends on X86
select CRYPTO_AEAD select CRYPTO_AEAD
select CRYPTO_CRYPTD select CRYPTO_SIMD
help help
Common glue for SIMD optimizations of the MORUS-1280 dedicated AEAD Common glue for SIMD optimizations of the MORUS-1280 dedicated AEAD
algorithm. algorithm.
......
...@@ -47,16 +47,7 @@ int crypto_morus1280_glue_setauthsize(struct crypto_aead *tfm, ...@@ -47,16 +47,7 @@ int crypto_morus1280_glue_setauthsize(struct crypto_aead *tfm,
int crypto_morus1280_glue_encrypt(struct aead_request *req); int crypto_morus1280_glue_encrypt(struct aead_request *req);
int crypto_morus1280_glue_decrypt(struct aead_request *req); int crypto_morus1280_glue_decrypt(struct aead_request *req);
int cryptd_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key, #define MORUS1280_DECLARE_ALG(id, driver_name, priority) \
unsigned int keylen);
int cryptd_morus1280_glue_setauthsize(struct crypto_aead *aead,
unsigned int authsize);
int cryptd_morus1280_glue_encrypt(struct aead_request *req);
int cryptd_morus1280_glue_decrypt(struct aead_request *req);
int cryptd_morus1280_glue_init_tfm(struct crypto_aead *aead);
void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead);
#define MORUS1280_DECLARE_ALGS(id, driver_name, priority) \
static const struct morus1280_glue_ops crypto_morus1280_##id##_ops = {\ static const struct morus1280_glue_ops crypto_morus1280_##id##_ops = {\
.init = crypto_morus1280_##id##_init, \ .init = crypto_morus1280_##id##_init, \
.ad = crypto_morus1280_##id##_ad, \ .ad = crypto_morus1280_##id##_ad, \
...@@ -77,55 +68,29 @@ void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead); ...@@ -77,55 +68,29 @@ void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead);
{ \ { \
} \ } \
\ \
static struct aead_alg crypto_morus1280_##id##_algs[] = {\ static struct aead_alg crypto_morus1280_##id##_alg = { \
{ \ .setkey = crypto_morus1280_glue_setkey, \
.setkey = crypto_morus1280_glue_setkey, \ .setauthsize = crypto_morus1280_glue_setauthsize, \
.setauthsize = crypto_morus1280_glue_setauthsize, \ .encrypt = crypto_morus1280_glue_encrypt, \
.encrypt = crypto_morus1280_glue_encrypt, \ .decrypt = crypto_morus1280_glue_decrypt, \
.decrypt = crypto_morus1280_glue_decrypt, \ .init = crypto_morus1280_##id##_init_tfm, \
.init = crypto_morus1280_##id##_init_tfm, \ .exit = crypto_morus1280_##id##_exit_tfm, \
.exit = crypto_morus1280_##id##_exit_tfm, \ \
\ .ivsize = MORUS_NONCE_SIZE, \
.ivsize = MORUS_NONCE_SIZE, \ .maxauthsize = MORUS_MAX_AUTH_SIZE, \
.maxauthsize = MORUS_MAX_AUTH_SIZE, \ .chunksize = MORUS1280_BLOCK_SIZE, \
.chunksize = MORUS1280_BLOCK_SIZE, \ \
\ .base = { \
.base = { \ .cra_flags = CRYPTO_ALG_INTERNAL, \
.cra_flags = CRYPTO_ALG_INTERNAL, \ .cra_blocksize = 1, \
.cra_blocksize = 1, \ .cra_ctxsize = sizeof(struct morus1280_ctx), \
.cra_ctxsize = sizeof(struct morus1280_ctx), \ .cra_alignmask = 0, \
.cra_alignmask = 0, \ .cra_priority = priority, \
\
.cra_name = "__morus1280", \
.cra_driver_name = "__"driver_name, \
\
.cra_module = THIS_MODULE, \
} \
}, { \
.setkey = cryptd_morus1280_glue_setkey, \
.setauthsize = cryptd_morus1280_glue_setauthsize, \
.encrypt = cryptd_morus1280_glue_encrypt, \
.decrypt = cryptd_morus1280_glue_decrypt, \
.init = cryptd_morus1280_glue_init_tfm, \
.exit = cryptd_morus1280_glue_exit_tfm, \
\ \
.ivsize = MORUS_NONCE_SIZE, \ .cra_name = "__morus1280", \
.maxauthsize = MORUS_MAX_AUTH_SIZE, \ .cra_driver_name = "__"driver_name, \
.chunksize = MORUS1280_BLOCK_SIZE, \
\ \
.base = { \ .cra_module = THIS_MODULE, \
.cra_flags = CRYPTO_ALG_ASYNC, \
.cra_blocksize = 1, \
.cra_ctxsize = sizeof(struct crypto_aead *), \
.cra_alignmask = 0, \
\
.cra_priority = priority, \
\
.cra_name = "morus1280", \
.cra_driver_name = driver_name, \
\
.cra_module = THIS_MODULE, \
} \
} \ } \
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment