s390/sha3: Support sha3 performance enhancements

On newer machines the SHA3 performance of CPACF instructions KIMD and
KLMD can be enhanced by using additional modifier bits. This allows the
application to omit initializing the ICV, but also affects the internal
processing of the instructions. Performance is mostly gained when
processing short messages.

The new CPACF feature is backwards compatible with older machines, i.e.
the new modifier bits are ignored on older machines. However, to save the
ICV initialization, the application must detect the MSA level and omit
the ICV initialization only if this feature is supported.
Reviewed-by: Holger Dengler <dengler@linux.ibm.com>
Signed-off-by: Joerg Schmidbauer <jschmidb@de.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

arch/s390/crypto/sha.h

@@ -25,6 +25,7 @@ struct s390_sha_ctx {
 	u32 state[CPACF_MAX_PARMBLOCK_SIZE / sizeof(u32)];
 	u8 buf[SHA_MAX_BLOCK_SIZE];
 	int func;		/* KIMD function to use */
+	int first_message_part;
 };
 
 struct shash_desc;

arch/s390/crypto/sha3_256_s390.c

@@ -21,9 +21,11 @@ static int sha3_256_init(struct shash_desc *desc)
 {
 	struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
 
-	memset(sctx->state, 0, sizeof(sctx->state));
+	if (!test_facility(86)) /* msa 12 */
+		memset(sctx->state, 0, sizeof(sctx->state));
 	sctx->count = 0;
 	sctx->func = CPACF_KIMD_SHA3_256;
+	sctx->first_message_part = 1;
 
 	return 0;
 }
@@ -88,9 +90,11 @@ static int sha3_224_init(struct shash_desc *desc)
 {
 	struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
 
-	memset(sctx->state, 0, sizeof(sctx->state));
+	if (!test_facility(86)) /* msa 12 */
+		memset(sctx->state, 0, sizeof(sctx->state));
 	sctx->count = 0;
 	sctx->func = CPACF_KIMD_SHA3_224;
+	sctx->first_message_part = 1;
 
 	return 0;
 }

arch/s390/crypto/sha3_512_s390.c

@@ -20,9 +20,11 @@ static int sha3_512_init(struct shash_desc *desc)
 {
 	struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
 
-	memset(sctx->state, 0, sizeof(sctx->state));
+	if (!test_facility(86)) /* msa 12 */
+		memset(sctx->state, 0, sizeof(sctx->state));
 	sctx->count = 0;
 	sctx->func = CPACF_KIMD_SHA3_512;
+	sctx->first_message_part = 1;
 
 	return 0;
 }
@@ -97,9 +99,11 @@ static int sha3_384_init(struct shash_desc *desc)
 {
 	struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
 
-	memset(sctx->state, 0, sizeof(sctx->state));
+	if (!test_facility(86)) /* msa 12 */
+		memset(sctx->state, 0, sizeof(sctx->state));
 	sctx->count = 0;
 	sctx->func = CPACF_KIMD_SHA3_384;
+	sctx->first_message_part = 1;
 
 	return 0;
 }

arch/s390/crypto/sha_common.c

@@ -18,6 +18,7 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len)
 	struct s390_sha_ctx *ctx = shash_desc_ctx(desc);
 	unsigned int bsize = crypto_shash_blocksize(desc->tfm);
 	unsigned int index, n;
+	int fc;
 
 	/* how much is already in the buffer? */
 	index = ctx->count % bsize;
@@ -26,10 +27,15 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len)
 	if ((index + len) < bsize)
 		goto store;
 
+	fc = ctx->func;
+	if (ctx->first_message_part)
+		fc |= test_facility(86) ? CPACF_KIMD_NIP : 0;
+
 	/* process one stored block */
 	if (index) {
 		memcpy(ctx->buf + index, data, bsize - index);
-		cpacf_kimd(ctx->func, ctx->state, ctx->buf, bsize);
+		cpacf_kimd(fc, ctx->state, ctx->buf, bsize);
+		ctx->first_message_part = 0;
 		data += bsize - index;
 		len -= bsize - index;
 		index = 0;
@@ -38,7 +44,8 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len)
 	/* process as many blocks as possible */
 	if (len >= bsize) {
 		n = (len / bsize) * bsize;
-		cpacf_kimd(ctx->func, ctx->state, data, n);
+		cpacf_kimd(fc, ctx->state, data, n);
+		ctx->first_message_part = 0;
 		data += n;
 		len -= n;
 	}
@@ -75,7 +82,7 @@ int s390_sha_final(struct shash_desc *desc, u8 *out)
 	unsigned int bsize = crypto_shash_blocksize(desc->tfm);
 	u64 bits;
 	unsigned int n;
-	int mbl_offset;
+	int mbl_offset, fc;
 
 	n = ctx->count % bsize;
 	bits = ctx->count * 8;
@@ -109,7 +116,11 @@ int s390_sha_final(struct shash_desc *desc, u8 *out)
 		return -EINVAL;
 	}
 
-	cpacf_klmd(ctx->func, ctx->state, ctx->buf, n);
+	fc = ctx->func;
+	fc |= test_facility(86) ? CPACF_KLMD_DUFOP : 0;
+	if (ctx->first_message_part)
+		fc |= CPACF_KLMD_NIP;
+	cpacf_klmd(fc, ctx->state, ctx->buf, n);
 
 	/* copy digest to out */
 	memcpy(out, ctx->state, crypto_shash_digestsize(desc->tfm));

arch/s390/include/asm/cpacf.h

@@ -171,6 +171,14 @@
 #define CPACF_KMA_LAAD	0x200	/* Last-AAD */
 #define CPACF_KMA_HS	0x400	/* Hash-subkey Supplied */
 
+/*
+ * Flags for the KIMD/KLMD (COMPUTE INTERMEDIATE/LAST MESSAGE DIGEST)
+ * instructions
+ */
+#define CPACF_KIMD_NIP		0x8000
+#define CPACF_KLMD_DUFOP	0x4000
+#define CPACF_KLMD_NIP		0x8000
+
 typedef struct { unsigned char bytes[16]; } cpacf_mask_t;
 
 /*
@@ -397,7 +405,7 @@ static inline void cpacf_kimd(unsigned long func, void *param,
 	asm volatile(
 		"	lgr	0,%[fc]\n"
 		"	lgr	1,%[pba]\n"
-		"0:	.insn	rre,%[opc] << 16,0,%[src]\n"
+		"0:	.insn	rrf,%[opc] << 16,0,%[src],8,0\n"
 		"	brc	1,0b\n" /* handle partial completion */
 		: [src] "+&d" (s.pair)
 		: [fc] "d" (func), [pba] "d" ((unsigned long)(param)),
@@ -422,7 +430,7 @@ static inline void cpacf_klmd(unsigned long func, void *param,
 	asm volatile(
 		"	lgr	0,%[fc]\n"
 		"	lgr	1,%[pba]\n"
-		"0:	.insn	rre,%[opc] << 16,0,%[src]\n"
+		"0:	.insn	rrf,%[opc] << 16,0,%[src],8,0\n"
 		"	brc	1,0b\n" /* handle partial completion */
 		: [src] "+&d" (s.pair)
 		: [fc] "d" (func), [pba] "d" ((unsigned long)param),