[PATCH 02/10] aria: reduce CTR bulk counter carry handling to 16 bits

* cipher/aria.c (aria_setkey): Assign ctr16be_enc bulk op.
* cipher/aria-aesni-avx-amd64.S (inc_le128): Remove.
(__aria_aesni_avx_ctr_gen_keystream_16way): Add to low 16 counter bits
only, drop full-width overflow path.
* cipher/aria-aesni-avx2-amd64.S: Likewise.
* cipher/aria-gfni-avx512-amd64.S: Likewise.
--

ARIA CTR keystream generators did full 128-bit little-endian counter
increment with separate 64-bit overflow path. Under ctr16be_enc
contract generic ctr code splits work at low 16-bit overflow, so bulk
function adds only to low 16 counter bits. Drop inc_le128 and overflow
path, use 16-bit stepping.

Moves ARIA off transitional ctr_enc alias.

Signed-off-by: Jussi Kivilinna <[email protected]>
---
 cipher/aria-aesni-avx-amd64.S   |  86 ++++++----------
 cipher/aria-aesni-avx2-amd64.S  | 170 ++++++--------------------------
 cipher/aria-gfni-avx512-amd64.S | 107 +++++---------------
 cipher/aria.c                   |   2 +-
 4 files changed, 86 insertions(+), 279 deletions(-)

diff --git a/cipher/aria-aesni-avx-amd64.S b/cipher/aria-aesni-avx-amd64.S
index 29597488..b177ca5e 100644
--- a/cipher/aria-aesni-avx-amd64.S
+++ b/cipher/aria-aesni-avx-amd64.S
@@ -66,12 +66,6 @@
 	  ((l0) << (7 * 8)) )
 
 /* asm macros */
-#define inc_le128(x, minus_one, tmp)			\
-	vpcmpeqq minus_one, x, tmp;			\
-	vpsubq minus_one, x, x;				\
-	vpslldq $8, tmp, tmp;				\
-	vpsubq tmp, x, x;
-
 #define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0)	\
 	vpand x, mask4bit, tmp0;			\
 	vpandn x, mask4bit, x;				\
@@ -1097,82 +1091,52 @@ __aria_aesni_avx_ctr_gen_keystream_16way:
 	vpshufb %xmm1, %xmm8, %xmm3; /* be => le */
 
 	vpcmpeqd %xmm0, %xmm0, %xmm0;
-	vpsrldq $8, %xmm0, %xmm0; /* low: -1, high: 0 */
+	vpsrldq $14, %xmm0, %xmm0; /* low: -1, high: 0 */
 
 	/* construct IVs */
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
 	vpshufb %xmm1, %xmm3, %xmm9;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
-	vpshufb %xmm1, %xmm3, %xmm10;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
+	vpshufb %xmm1, %xmm3, %xmm2;
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
 	vpshufb %xmm1, %xmm3, %xmm11;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
-	vpshufb %xmm1, %xmm3, %xmm12;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
-	vpshufb %xmm1, %xmm3, %xmm13;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
-	vpshufb %xmm1, %xmm3, %xmm14;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
-	vpshufb %xmm1, %xmm3, %xmm15;
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
+	vpshufb %xmm1, %xmm3, %xmm4;
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
+	vpshufb %xmm1, %xmm3, %xmm5;
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
+	vpshufb %xmm1, %xmm3, %xmm6;
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
+	vpshufb %xmm1, %xmm3, %xmm7;
 	vmovdqu %xmm8, (0 * 16)(%rcx);
 	vmovdqu %xmm9, (1 * 16)(%rcx);
-	vmovdqu %xmm10, (2 * 16)(%rcx);
 	vmovdqu %xmm11, (3 * 16)(%rcx);
-	vmovdqu %xmm12, (4 * 16)(%rcx);
-	vmovdqu %xmm13, (5 * 16)(%rcx);
-	vmovdqu %xmm14, (6 * 16)(%rcx);
-	vmovdqu %xmm15, (7 * 16)(%rcx);
 
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
 	vpshufb %xmm1, %xmm3, %xmm8;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
 	vpshufb %xmm1, %xmm3, %xmm9;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
 	vpshufb %xmm1, %xmm3, %xmm10;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
 	vpshufb %xmm1, %xmm3, %xmm11;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
 	vpshufb %xmm1, %xmm3, %xmm12;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
 	vpshufb %xmm1, %xmm3, %xmm13;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
 	vpshufb %xmm1, %xmm3, %xmm14;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+	vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
 	vpshufb %xmm1, %xmm3, %xmm15;
-	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
-	vpshufb %xmm1, %xmm3, %xmm4;
-	vmovdqu %xmm4, (%r8);
 
 	vmovdqu (0 * 16)(%rcx), %xmm0;
 	vmovdqu (1 * 16)(%rcx), %xmm1;
-	vmovdqu (2 * 16)(%rcx), %xmm2;
 	vmovdqu (3 * 16)(%rcx), %xmm3;
-	vmovdqu (4 * 16)(%rcx), %xmm4;
-	vmovdqu (5 * 16)(%rcx), %xmm5;
-	vmovdqu (6 * 16)(%rcx), %xmm6;
-	vmovdqu (7 * 16)(%rcx), %xmm7;
 
-	ret_spec_stop;
+	jmp .Lctr_gen_keystream_done;
 
 .align 8
-.Lctr_byteadd_full_ctr_carry:
-	addb $16, 15(%r8);
-	pushq %rcx;
-	CFI_ADJUST_CFA_OFFSET(8);
-	movl $14, %ecx;
-	1:
-	  adcb $0, (%r8, %rcx);
-	  jnc 2f;
-	  loop 1b;
-	2:
-	popq %rcx;
-	CFI_ADJUST_CFA_OFFSET(-8);
-	jmp .Lctr_byteadd_xmm;
-.align 8
 .Lctr_byteadd:
-	je .Lctr_byteadd_full_ctr_carry;
-	addb $16, 15(%r8);
-.Lctr_byteadd_xmm:
 	vmovdqa %xmm8, %xmm0;
 	vpaddb .Lbige_addb_1 rRIP, %xmm8, %xmm1;
 	vpaddb .Lbige_addb_2 rRIP, %xmm8, %xmm2;
@@ -1190,6 +1154,12 @@ __aria_aesni_avx_ctr_gen_keystream_16way:
 	vpaddb .Lbige_addb_14 rRIP, %xmm0, %xmm14;
 	vpaddb .Lbige_addb_15 rRIP, %xmm0, %xmm15;
 
+.align 16
+.Lctr_gen_keystream_done:
+	/* Update IV */
+	addb $16, 15(%r8);
+	adcb $0, 14(%r8);
+
 	ret_spec_stop;
 	CFI_ENDPROC();
 ELF(.size __aria_aesni_avx_ctr_gen_keystream_16way,.-__aria_aesni_avx_ctr_gen_keystream_16way;)
diff --git a/cipher/aria-aesni-avx2-amd64.S b/cipher/aria-aesni-avx2-amd64.S
index 08bf5a0e..99eca542 100644
--- a/cipher/aria-aesni-avx2-amd64.S
+++ b/cipher/aria-aesni-avx2-amd64.S
@@ -86,12 +86,6 @@
 	  ((l0) << (7 * 8)) )
 
 /* asm macros */
-#define inc_le128(x, minus_one, tmp)			\
-	vpcmpeqq minus_one, x, tmp;			\
-	vpsubq minus_one, x, x;				\
-	vpslldq $8, tmp, tmp;				\
-	vpsubq tmp, x, x;
-
 #define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0)	\
 	vpand x, mask4bit, tmp0;			\
 	vpandn x, mask4bit, x;				\
@@ -1246,171 +1240,63 @@ __aria_aesni_avx2_ctr_gen_keystream_32way:
 	cmpb $(0x100 - 32), 15(%r8);
 	jbe .Lctr_byteadd;
 
-	movq 8(%r8), %r11;
-	bswapq %r11;
-
 	vbroadcasti128 .Lbswap128_mask rRIP, %ymm6;
 	vpcmpeqd %ymm0, %ymm0, %ymm0;
-	vpsrldq $8, %ymm0, %ymm0;   /* ab: -1:0 ; cd: -1:0 */
-	vpaddq %ymm0, %ymm0, %ymm5; /* ab: -2:0 ; cd: -2:0 */
+	vpsrldq $14, %ymm0, %ymm0;   /* ab: -1:0 ; cd: -1:0 */
+	vpaddw %ymm0, %ymm0, %ymm5; /* ab: -2:0 ; cd: -2:0 */
 
-	/* load IV and byteswap */
+	/* load IV and byteswap. */
 	vmovdqu (%r8), %xmm7;
 	vpshufb %xmm6, %xmm7, %xmm7;
 	vmovdqa %xmm7, %xmm3;
-	inc_le128(%xmm7, %xmm0, %xmm4);
+	vpsubw %xmm0, %xmm7, %xmm7;
 	vinserti128 $1, %xmm7, %ymm3, %ymm3;
-	vpshufb %ymm6, %ymm3, %ymm8; /* +1 ; +0 */
-
-	/* check need for handling 64-bit overflow and carry */
-	cmpq $(0xffffffffffffffff - 32), %r11;
-	ja .Lhandle_ctr_carry;
+	vpshufb %ymm6, %ymm3, %ymm0; /* +1 ; +0 */
 
 	/* construct IVs */
-	vpsubq %ymm5, %ymm3, %ymm3; /* +3 ; +2 */
-	vpshufb %ymm6, %ymm3, %ymm9;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +5 ; +4 */
-	vpshufb %ymm6, %ymm3, %ymm10;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +7 ; +6 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +3 ; +2 */
+	vpshufb %ymm6, %ymm3, %ymm1;
+	vpsubw %ymm5, %ymm3, %ymm3; /* +5 ; +4 */
+	vpshufb %ymm6, %ymm3, %ymm2;
+	vpsubw %ymm5, %ymm3, %ymm3; /* +7 ; +6 */
 	vpshufb %ymm6, %ymm3, %ymm11;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +9 ; +8 */
-	vpshufb %ymm6, %ymm3, %ymm12;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +11 ; +10 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +9 ; +8 */
+	vpshufb %ymm6, %ymm3, %ymm4;
+	vpsubw %ymm5, %ymm3, %ymm3; /* +11 ; +10 */
 	vpshufb %ymm6, %ymm3, %ymm13;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +13 ; +12 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +13 ; +12 */
 	vpshufb %ymm6, %ymm3, %ymm14;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +15 ; +14 */
-	vpshufb %ymm6, %ymm3, %ymm15;
-	vmovdqu %ymm8, (0 * 32)(%rcx);
-	vmovdqu %ymm9, (1 * 32)(%rcx);
-	vmovdqu %ymm10, (2 * 32)(%rcx);
+	vpsubw %ymm5, %ymm3, %ymm3; /* +15 ; +14 */
+	vpshufb %ymm6, %ymm3, %ymm7;
 	vmovdqu %ymm11, (3 * 32)(%rcx);
-	vmovdqu %ymm12, (4 * 32)(%rcx);
 	vmovdqu %ymm13, (5 * 32)(%rcx);
 	vmovdqu %ymm14, (6 * 32)(%rcx);
-	vmovdqu %ymm15, (7 * 32)(%rcx);
 
-	vpsubq %ymm5, %ymm3, %ymm3; /* +17 ; +16 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +17 ; +16 */
 	vpshufb %ymm6, %ymm3, %ymm8;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +19 ; +18 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +19 ; +18 */
 	vpshufb %ymm6, %ymm3, %ymm9;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +21 ; +20 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +21 ; +20 */
 	vpshufb %ymm6, %ymm3, %ymm10;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +23 ; +22 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +23 ; +22 */
 	vpshufb %ymm6, %ymm3, %ymm11;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +25 ; +24 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +25 ; +24 */
 	vpshufb %ymm6, %ymm3, %ymm12;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +27 ; +26 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +27 ; +26 */
 	vpshufb %ymm6, %ymm3, %ymm13;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +29 ; +28 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +29 ; +28 */
 	vpshufb %ymm6, %ymm3, %ymm14;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +31 ; +30 */
+	vpsubw %ymm5, %ymm3, %ymm3; /* +31 ; +30 */
 	vpshufb %ymm6, %ymm3, %ymm15;
-	vpsubq %ymm5, %ymm3, %ymm3; /* +32 */
-	vpshufb %xmm6, %xmm3, %xmm3;
-	vmovdqu %xmm3, (%r8);
-	vmovdqu (0 * 32)(%rcx), %ymm0;
-	vmovdqu (1 * 32)(%rcx), %ymm1;
-	vmovdqu (2 * 32)(%rcx), %ymm2;
-	vmovdqu (3 * 32)(%rcx), %ymm3;
-	vmovdqu (4 * 32)(%rcx), %ymm4;
-	vmovdqu (5 * 32)(%rcx), %ymm5;
-	vmovdqu (6 * 32)(%rcx), %ymm6;
-	vmovdqu (7 * 32)(%rcx), %ymm7;
-	jmp .Lctr_carry_done;
-
-	.Lhandle_ctr_carry:
-	/* construct IVs */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm9; /* +3 ; +2 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm10; /* +5 ; +4 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm11; /* +7 ; +6 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm12; /* +9 ; +8 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm13; /* +11 ; +10 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm14; /* +13 ; +12 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm15; /* +15 ; +14 */
-	vmovdqu %ymm8, (0 * 32)(%rcx);
-	vmovdqu %ymm9, (1 * 32)(%rcx);
-	vmovdqu %ymm10, (2 * 32)(%rcx);
-	vmovdqu %ymm11, (3 * 32)(%rcx);
-	vmovdqu %ymm12, (4 * 32)(%rcx);
-	vmovdqu %ymm13, (5 * 32)(%rcx);
-	vmovdqu %ymm14, (6 * 32)(%rcx);
-	vmovdqu %ymm15, (7 * 32)(%rcx);
-
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm8; /* +17 ; +16 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm9; /* +19 ; +18 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm10; /* +21 ; +20 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm11; /* +23 ; +22 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm12; /* +25 ; +24 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm13; /* +27 ; +26 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm14; /* +29 ; +28 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vpshufb %ymm6, %ymm3, %ymm15; /* +31 ; +30 */
-	inc_le128(%ymm3, %ymm0, %ymm4);
-	vextracti128 $1, %ymm3, %xmm3;
-	vpshufb %xmm6, %xmm3, %xmm3; /* +32 */
-	vmovdqu %xmm3, (%r8);
-	vmovdqu (0 * 32)(%rcx), %ymm0;
-	vmovdqu (1 * 32)(%rcx), %ymm1;
-	vmovdqu (2 * 32)(%rcx), %ymm2;
 	vmovdqu (3 * 32)(%rcx), %ymm3;
-	vmovdqu (4 * 32)(%rcx), %ymm4;
 	vmovdqu (5 * 32)(%rcx), %ymm5;
 	vmovdqu (6 * 32)(%rcx), %ymm6;
-	vmovdqu (7 * 32)(%rcx), %ymm7;
 
-.Lctr_carry_done:
-	ret_spec_stop;
+	jmp .Lctr_gen_keystream_done;
 
 .align 8
-.Lctr_byteadd_full_ctr_carry:
-	addb $32, 15(%r8);
-	pushq %rcx;
-	CFI_ADJUST_CFA_OFFSET(8);
-	movl $14, %ecx;
-	1:
-	  adcb $0, (%r8, %rcx);
-	  jnc 2f;
-	  loop 1b;
-	2:
-	popq %rcx;
-	CFI_ADJUST_CFA_OFFSET(-8);
-	jmp .Lctr_byteadd_ymm;
-.align 8
 .Lctr_byteadd:
 	vbroadcasti128 (%r8), %ymm8;
-	je .Lctr_byteadd_full_ctr_carry;
-	addb $32, 15(%r8);
-.Lctr_byteadd_ymm:
 	vpaddb .Lbige_addb_16_16 rRIP, %ymm8, %ymm15;
 	vpaddb .Lbige_addb_0_1 rRIP, %ymm8, %ymm0;
 	vpaddb .Lbige_addb_2_3 rRIP, %ymm8, %ymm1;
@@ -1429,6 +1315,12 @@ __aria_aesni_avx2_ctr_gen_keystream_32way:
 	vpaddb .Lbige_addb_12_13 rRIP, %ymm15, %ymm14;
 	vpaddb .Lbige_addb_14_15 rRIP, %ymm15, %ymm15;
 
+.align 16
+.Lctr_gen_keystream_done:
+	/* Update IV */
+	addb $32, 15(%r8);
+	adcb $0, 14(%r8);
+
 	ret_spec_stop;
 	CFI_ENDPROC();
 ELF(.size __aria_aesni_avx2_ctr_gen_keystream_32way,
diff --git a/cipher/aria-gfni-avx512-amd64.S b/cipher/aria-gfni-avx512-amd64.S
index a497b395..c25caf86 100644
--- a/cipher/aria-gfni-avx512-amd64.S
+++ b/cipher/aria-gfni-avx512-amd64.S
@@ -75,16 +75,9 @@
 	clear_vec4(%ymm19, %ymm23, %ymm27, %ymm31)
 
 #define clear_regs() \
-	kxorq %k1, %k1, %k1; \
 	vzeroall; \
 	clear_zmm16_zmm31()
 
-#define add_le128(out, in, lo_counter, hi_counter1)	\
-	vpaddq lo_counter, in, out;			\
-	vpcmpuq $1, lo_counter, out, %k1;		\
-	kaddb %k1, %k1, %k1;				\
-	vpaddq hi_counter1, out, out{%k1};
-
 #define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0)	\
 	vpandq x, mask4bit, tmp0;			\
 	vpandqn x, mask4bit, x;				\
@@ -583,8 +576,6 @@ SECTION_RODATA
 	.quad 8, 0
 .Lcounter16161616_lo:
 	.quad 16, 0
-.Lcounter1111_hi:
-	.quad 0, 1
 
 /* For CTR-mode IV byteswap */
 .Lbswap128_mask:
@@ -815,62 +806,30 @@ __aria_gfni_avx512_ctr_gen_keystream_64way:
 	vbroadcasti64x2 .Lcounter4444_lo rRIP, %zmm22;
 	vbroadcasti64x2 .Lcounter8888_lo rRIP, %zmm23;
 	vbroadcasti64x2 .Lcounter16161616_lo rRIP, %zmm24;
-	vbroadcasti64x2 .Lcounter1111_hi rRIP, %zmm25;
 
-	/* load IV and byteswap */
-	movq 8(%r8), %r11;
-	movq (%r8), %r10;
-	bswapq %r11;
-	bswapq %r10;
+	/* load IV and byteswap. */
 	vbroadcasti64x2 (%r8), %zmm20;
 	vpshufb %zmm19, %zmm20, %zmm20;
 
-	/* check need for handling 64-bit overflow and carry */
-	cmpq $(0xffffffffffffffff - 64), %r11;
-	ja .Lload_ctr_carry;
-
-	/* construct IVs */
-	vpaddq %zmm21, %zmm20, %zmm0; /* +0:+1:+2:+3 */
-	vpaddq %zmm22, %zmm0, %zmm1; /* +4:+5:+6:+7 */
-	vpaddq %zmm23, %zmm0, %zmm2; /* +8:+9:+10:+11 */
-	vpaddq %zmm23, %zmm1, %zmm3; /* +12:+13:+14:+15 */
-	vpaddq %zmm24, %zmm0, %zmm4; /* +16... */
-	vpaddq %zmm24, %zmm1, %zmm5; /* +20... */
-	vpaddq %zmm24, %zmm2, %zmm6; /* +24... */
-	vpaddq %zmm24, %zmm3, %zmm7; /* +28... */
-	vpaddq %zmm24, %zmm4, %zmm8; /* +32... */
-	vpaddq %zmm24, %zmm5, %zmm9; /* +36... */
-	vpaddq %zmm24, %zmm6, %zmm10; /* +40... */
-	vpaddq %zmm24, %zmm7, %zmm11; /* +44... */
-	vpaddq %zmm24, %zmm8, %zmm12; /* +48... */
-	vpaddq %zmm24, %zmm9, %zmm13; /* +52... */
-	vpaddq %zmm24, %zmm10, %zmm14; /* +56... */
-	vpaddq %zmm24, %zmm11, %zmm15; /* +60... */
-	jmp .Lload_ctr_done;
-
-.Lload_ctr_carry:
 	/* construct IVs */
-	add_le128(%zmm0, %zmm20, %zmm21, %zmm25); /* +0:+1:+2:+3 */
-	add_le128(%zmm1, %zmm0, %zmm22, %zmm25); /* +4:+5:+6:+7 */
-	add_le128(%zmm2, %zmm0, %zmm23, %zmm25); /* +8:+9:+10:+11 */
-	add_le128(%zmm3, %zmm1, %zmm23, %zmm25); /* +12:+13:+14:+15 */
-	add_le128(%zmm4, %zmm0, %zmm24, %zmm25); /* +16... */
-	add_le128(%zmm5, %zmm1, %zmm24, %zmm25); /* +20... */
-	add_le128(%zmm6, %zmm2, %zmm24, %zmm25); /* +24... */
-	add_le128(%zmm7, %zmm3, %zmm24, %zmm25); /* +28... */
-	add_le128(%zmm8, %zmm4, %zmm24, %zmm25); /* +32... */
-	add_le128(%zmm9, %zmm5, %zmm24, %zmm25); /* +36... */
-	add_le128(%zmm10, %zmm6, %zmm24, %zmm25); /* +40... */
-	add_le128(%zmm11, %zmm7, %zmm24, %zmm25); /* +44... */
-	add_le128(%zmm12, %zmm8, %zmm24, %zmm25); /* +48... */
-	add_le128(%zmm13, %zmm9, %zmm24, %zmm25); /* +52... */
-	add_le128(%zmm14, %zmm10, %zmm24, %zmm25); /* +56... */
-	add_le128(%zmm15, %zmm11, %zmm24, %zmm25); /* +60... */
-
-.Lload_ctr_done:
-	/* Byte-swap IVs and update counter. */
-	addq $64, %r11;
-	adcq $0, %r10;
+	vpaddw %zmm21, %zmm20, %zmm0; /* +0:+1:+2:+3 */
+	vpaddw %zmm22, %zmm0, %zmm1; /* +4:+5:+6:+7 */
+	vpaddw %zmm23, %zmm0, %zmm2; /* +8:+9:+10:+11 */
+	vpaddw %zmm23, %zmm1, %zmm3; /* +12:+13:+14:+15 */
+	vpaddw %zmm24, %zmm0, %zmm4; /* +16... */
+	vpaddw %zmm24, %zmm1, %zmm5; /* +20... */
+	vpaddw %zmm24, %zmm2, %zmm6; /* +24... */
+	vpaddw %zmm24, %zmm3, %zmm7; /* +28... */
+	vpaddw %zmm24, %zmm4, %zmm8; /* +32... */
+	vpaddw %zmm24, %zmm5, %zmm9; /* +36... */
+	vpaddw %zmm24, %zmm6, %zmm10; /* +40... */
+	vpaddw %zmm24, %zmm7, %zmm11; /* +44... */
+	vpaddw %zmm24, %zmm8, %zmm12; /* +48... */
+	vpaddw %zmm24, %zmm9, %zmm13; /* +52... */
+	vpaddw %zmm24, %zmm10, %zmm14; /* +56... */
+	vpaddw %zmm24, %zmm11, %zmm15; /* +60... */
+
+	/* Byte-swap IVs. */
 	vpshufb %zmm19, %zmm15, %zmm15;
 	vpshufb %zmm19, %zmm14, %zmm14;
 	vpshufb %zmm19, %zmm13, %zmm13;
@@ -879,8 +838,6 @@ __aria_gfni_avx512_ctr_gen_keystream_64way:
 	vpshufb %zmm19, %zmm10, %zmm10;
 	vpshufb %zmm19, %zmm9, %zmm9;
 	vpshufb %zmm19, %zmm8, %zmm8;
-	bswapq %r11;
-	bswapq %r10;
 	vpshufb %zmm19, %zmm7, %zmm7;
 	vpshufb %zmm19, %zmm6, %zmm6;
 	vpshufb %zmm19, %zmm5, %zmm5;
@@ -889,30 +846,12 @@ __aria_gfni_avx512_ctr_gen_keystream_64way:
 	vpshufb %zmm19, %zmm2, %zmm2;
 	vpshufb %zmm19, %zmm1, %zmm1;
 	vpshufb %zmm19, %zmm0, %zmm0;
-	movq %r11, 8(%r8);
-	movq %r10, (%r8);
 
-	ret_spec_stop;
+	jmp .Lctr_gen_keystream_done;
 
 .align 16
-.Lctr_byteadd_full_ctr_carry:
-	movq 8(%r8), %r11;
-	movq (%r8), %r10;
-	bswapq %r11;
-	bswapq %r10;
-	addq $64, %r11;
-	adcq $0, %r10;
-	bswapq %r11;
-	bswapq %r10;
-	movq %r11, 8(%r8);
-	movq %r10, (%r8);
-	jmp .Lctr_byteadd_zmm;
-.align 16
 .Lctr_byteadd:
 	vbroadcasti64x2 (%r8), %zmm3;
-	je .Lctr_byteadd_full_ctr_carry;
-	addb $64, 15(%r8);
-.Lctr_byteadd_zmm:
 	vbroadcasti64x2 .Lbige_addb_16 rRIP, %zmm16;
 	vmovdqa64 .Lbige_addb_0_1 rRIP, %zmm17;
 	vmovdqa64 .Lbige_addb_4_5 rRIP, %zmm18;
@@ -938,6 +877,12 @@ __aria_gfni_avx512_ctr_gen_keystream_64way:
 	vpaddb %zmm19, %zmm15, %zmm14;
 	vpaddb %zmm20, %zmm15, %zmm15;
 
+.align 16
+.Lctr_gen_keystream_done:
+	/* Update IV */
+	addb $64, 15(%r8);
+	adcb $0, 14(%r8);
+
 	ret_spec_stop;
 	CFI_ENDPROC();
 ELF(.size __aria_gfni_avx512_ctr_gen_keystream_64way,
diff --git a/cipher/aria.c b/cipher/aria.c
index bb67ed03..227affb4 100644
--- a/cipher/aria.c
+++ b/cipher/aria.c
@@ -1656,7 +1656,7 @@ aria_setkey(void *c, const byte *key, unsigned keylen,
   bulk_ops->cbc_dec = _gcry_aria_cbc_dec;
   bulk_ops->cfb_enc = _gcry_aria_cfb_enc;
   bulk_ops->cfb_dec = _gcry_aria_cfb_dec;
-  bulk_ops->ctr_enc = _gcry_aria_ctr_enc;
+  bulk_ops->ctr16be_enc = _gcry_aria_ctr_enc;
   bulk_ops->ctr32le_enc = _gcry_aria_ctr32le_enc;
   bulk_ops->ecb_crypt = _gcry_aria_ecb_crypt;
   bulk_ops->xts_crypt = _gcry_aria_xts_crypt;
-- 
2.53.0