[PATCH 08/10] twofish: reduce CTR bulk counter carry handling to 16 bits

* cipher/twofish.c (twofish_setkey): Assign ctr16be_enc bulk op.
(_gcry_twofish_ctr_enc): Use cipher_block_add_be16.
* cipher/twofish-amd64.S (RT0bl, RT1bl): New.
(_gcry_twofish_amd64_ctr_enc): Increment low 16 counter bits in place,
drop full-width carry and byteswap.
* cipher/twofish-avx2-amd64.S (inc_le128): Remove.
(_gcry_twofish_avx2_ctr_enc): Add to low 16 counter bits only, drop
full-width carry handling.
--

Twofish CTR built IVs with full 128-bit increment and byteswap
round-trip. Under ctr16be_enc contract generic ctr code splits work at
low 16-bit overflow, so bulk function adds only to low 16 counter bits.
Scalar amd64 path now increments low counter bytes in place, dropping
byteswap. AVX2 path uses 16-bit stepping and drops full-width carry.

Moves twofish off transitional ctr_enc alias.

Signed-off-by: Jussi Kivilinna <[email protected]>
---
 cipher/twofish-amd64.S      | 53 +++++++++++++--------------
 cipher/twofish-avx2-amd64.S | 71 +++++++------------------------------
 cipher/twofish.c            |  4 +--
 3 files changed, 39 insertions(+), 89 deletions(-)

diff --git a/cipher/twofish-amd64.S b/cipher/twofish-amd64.S
index 913b252d..beafd10f 100644
--- a/cipher/twofish-amd64.S
+++ b/cipher/twofish-amd64.S
@@ -359,6 +359,9 @@ ELF(.size _gcry_twofish_amd64_encrypt_block,.-_gcry_twofish_amd64_encrypt_block;
 #define RT0d %edx
 #define RT1d %esi
 
+#define RT0bl %dl
+#define RT1bl %sil
+
 #define do16bit_ror(rot, op1, op2, T0, T1, tmp1, tmp2, ab, dst) \
 	movzbl ab ## bl,		tmp2 ## d; \
 	movzbl ab ## bh,		tmp1 ## d; \
@@ -650,35 +653,27 @@ _gcry_twofish_amd64_ctr_enc:
 	movq %rdx, (7 * 8)(%rsp);
 	movq %rcx, RX0;
 
-	/* load IV and byteswap */
-	movq 8(RX0), RT0;
-	movq 0(RX0), RT1;
-	movq RT0, RCD0;
-	movq RT1, RAB0;
-	bswapq RT0;
-	bswapq RT1;
-
-	/* construct IVs */
-	movq RT0, RCD1;
-	movq RT1, RAB1;
-	movq RT0, RCD2;
-	movq RT1, RAB2;
-	addq $1, RCD1;
-	adcq $0, RAB1;
-	bswapq RCD1;
-	bswapq RAB1;
-	addq $2, RCD2;
-	adcq $0, RAB2;
-	bswapq RCD2;
-	bswapq RAB2;
-	addq $3, RT0;
-	adcq $0, RT1;
-	bswapq RT0;
-	bswapq RT1;
-
-	/* store new IV */
-	movq RT0, 8(RX0);
-	movq RT1, 0(RX0);
+	/* Construct and update IV */
+	movzbl 15(RX0), RT0d;
+	movzbl 14(RX0), RT1d;
+	movq 8(RX0), RCD0;
+	movq 0(RX0), RAB0;
+	movq RAB0, RAB1;
+	movq RAB0, RAB2;
+	addb $1, RT0bl;
+	adcb $0, RT1bl;
+	movb RT0bl, 15(RX0);
+	movb RT1bl, 14(RX0);
+	movq 8(RX0), RCD1;
+	addb $1, RT0bl;
+	adcb $0, RT1bl;
+	movb RT0bl, 15(RX0);
+	movb RT1bl, 14(RX0);
+	movq 8(RX0), RCD2;
+	addb $1, RT0bl;
+	adcb $0, RT1bl;
+	movb RT0bl, 15(RX0);
+	movb RT1bl, 14(RX0);
 
 	call __twofish_enc_blk3;
 
diff --git a/cipher/twofish-avx2-amd64.S b/cipher/twofish-avx2-amd64.S
index 67c1e52e..72a39d17 100644
--- a/cipher/twofish-avx2-amd64.S
+++ b/cipher/twofish-avx2-amd64.S
@@ -553,12 +553,6 @@ _gcry_twofish_avx2_blk16:
 	CFI_ENDPROC();
 ELF(.size _gcry_twofish_avx2_blk16,.-_gcry_twofish_avx2_blk16;)
 
-#define inc_le128(x, minus_one, tmp) \
-	vpcmpeqq minus_one, x, tmp; \
-	vpsubq minus_one, x, x; \
-	vpslldq $8, tmp, tmp; \
-	vpsubq tmp, x, x;
-
 .align 16
 .globl _gcry_twofish_avx2_ctr_enc
 ELF(.type   _gcry_twofish_avx2_ctr_enc,@function;)
@@ -571,79 +565,40 @@ _gcry_twofish_avx2_ctr_enc:
 	 */
 	CFI_STARTPROC();
 
-	movq 8(%rcx), %rax;
-	bswapq %rax;
-
 	vzeroupper;
 
 	vbroadcasti128 .Lbswap128_mask rRIP, RTMP3;
 	vpcmpeqd RNOT, RNOT, RNOT;
-	vpsrldq $8, RNOT, RNOT;   /* ab: -1:0 ; cd: -1:0 */
-	vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
+	vpsrldq $14, RNOT, RNOT;   /* ab: -1:0 ; cd: -1:0 */
+	vpaddw RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
 
 	/* load IV and byteswap */
 	vmovdqu (%rcx), RTMP4x;
 	vpshufb RTMP3x, RTMP4x, RTMP4x;
 	vmovdqa RTMP4x, RTMP0x;
-	inc_le128(RTMP4x, RNOTx, RTMP1x);
+	vpsubw RNOTx, RTMP4x, RTMP4x;
 	vinserti128 $1, RTMP4x, RTMP0, RTMP0;
 	vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */
 
-	/* check need for handling 64-bit overflow and carry */
-	cmpq $(0xffffffffffffffff - 16), %rax;
-	ja .Lhandle_ctr_carry;
+	/* Update IV */
+	addb $16, 15(%rcx);
+	adcb $0, 14(%rcx);
 
 	/* construct IVs */
-	vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
+	vpsubw RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
 	vpshufb RTMP3, RTMP0, RB0;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
+	vpsubw RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
 	vpshufb RTMP3, RTMP0, RC0;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
+	vpsubw RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
 	vpshufb RTMP3, RTMP0, RD0;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
+	vpsubw RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
 	vpshufb RTMP3, RTMP0, RA1;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
+	vpsubw RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
 	vpshufb RTMP3, RTMP0, RB1;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
+	vpsubw RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
 	vpshufb RTMP3, RTMP0, RC1;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
+	vpsubw RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
 	vpshufb RTMP3, RTMP0, RD1;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +16 */
-	vpshufb RTMP3x, RTMP0x, RTMP0x;
-
-	jmp .Lctr_carry_done;
-
-.Lhandle_ctr_carry:
-	/* construct IVs */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RB0; /* +3 ; +2 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RC0; /* +5 ; +4 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RD0; /* +7 ; +6 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RA1; /* +9 ; +8 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RC1; /* +13 ; +12 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RD1; /* +15 ; +14 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vextracti128 $1, RTMP0, RTMP0x;
-	vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */
-
-.align 4
-.Lctr_carry_done:
-	/* store new IV */
-	vmovdqu RTMP0x, (%rcx);
 
 	call __twofish_enc_blk16;
 
diff --git a/cipher/twofish.c b/cipher/twofish.c
index 11a6e251..4341a7fb 100644
--- a/cipher/twofish.c
+++ b/cipher/twofish.c
@@ -774,7 +774,7 @@ twofish_setkey (void *context, const byte *key, unsigned int keylen,
   memset (bulk_ops, 0, sizeof(*bulk_ops));
   bulk_ops->cbc_dec = _gcry_twofish_cbc_dec;
   bulk_ops->cfb_dec = _gcry_twofish_cfb_dec;
-  bulk_ops->ctr_enc = _gcry_twofish_ctr_enc;
+  bulk_ops->ctr16be_enc = _gcry_twofish_ctr_enc;
   bulk_ops->ocb_crypt = _gcry_twofish_ocb_crypt;
   bulk_ops->ocb_auth = _gcry_twofish_ocb_auth;
   bulk_ops->xts_crypt = _gcry_twofish_xts_crypt;
@@ -1197,7 +1197,7 @@ _gcry_twofish_ctr_enc(void *context, unsigned char *ctr, void *outbuf_arg,
       outbuf += TWOFISH_BLOCKSIZE;
       inbuf  += TWOFISH_BLOCKSIZE;
       /* Increment the counter.  */
-      cipher_block_add(ctr, 1, TWOFISH_BLOCKSIZE);
+      cipher_block_add_be16(ctr, 1, TWOFISH_BLOCKSIZE);
     }
 
   wipememory(tmpbuf, sizeof(tmpbuf));
-- 
2.53.0