[PATCH 01/10] cipher: reduce CTR bulk counter carry handling to 16 bits
Jussi Kivilinna <[email protected]> Fri, 24 Jul 2026 21:50:08 +0300
| Newsgroups | gmane.comp.encryption.gpg.libgcrypt.devel |
|---|---|
| Message-ID | <[email protected]> |
* cipher/cipher-internal.h (cipher_bulk_ops_t): Rename 'ctr_enc' member to 'ctr16be_enc'. (ctr_enc): New transitional alias macro. (cipher_block_add): Widen 'add' to u64. (cipher_block_add_be16): New. * cipher/cipher-ctr.c (_gcry_cipher_ctr_encrypt_ctx): Split bulk work at low 16-bit counter overflow and do full-width carry addition here. * cipher/bulkhelp.h (bulk_ctr_enc_128): Use cipher_block_add_be16. * cipher/rijndael.c (do_setkey): Assign ctr16be_enc bulk op. (_gcry_aes_ctr_enc): Use cipher_block_add_be16. * cipher/rijndael-aesni.c (do_aesni_ctr, do_aesni_ctr_4) (do_aesni_ctr_8): Drop full-width counter carry handling. * cipher/rijndael-ssse3-amd64.c (_gcry_aes_ssse3_ctr_enc): Likewise. * cipher/rijndael-vp-simd128.h (FUNC_CTR_ENC): Likewise. * cipher/rijndael-armv8-aarch32-ce.S (_gcry_aes_ctr_enc_armv8_ce): Likewise. * cipher/rijndael-armv8-aarch64-ce.S (_gcry_aes_ctr_enc_armv8_ce): Likewise. * cipher/rijndael-ppc-functions.h (CTR_ENC_FUNC): Likewise. * cipher/rijndael-riscv-zvkned.c (_gcry_aes_riscv_zvkned_ctr_enc): Likewise. * cipher/rijndael-vaes-avx2-amd64.S (_gcry_vaes_avx2_ctr_enc_amd64): Likewise. * cipher/rijndael-vaes-avx2-i386.S (_gcry_vaes_avx2_ctr_enc_i386): Likewise. * cipher/rijndael-vaes-avx512-amd64.S (_gcry_vaes_avx512_ctr_enc_amd64): Likewise. * tests/basic.c (cipher_cbc_bulk_test, cipher_cfb_bulk_test) (cipher_ctr_bulk_test): Add verbose output. (cipher_ctr16_overflow_test, check_ctr16_overflow): New. (check_cipher_modes): Call check_ctr16_overflow. -- Every SIMD CTR implementation carried code to propagate big-endian counter carry across bytes, up to full 128 bits. Move carry out of assembly: bulk functions now add only to low 16 counter bits, and generic ctr code splits work at each 16-bit overflow to do full-width carry addition in C. Low 16-bit overflow never happens inside bulk call, so per-implementation carry handling drops to low-halfword increment. ctr_enc bulk op renamed to ctr16be_enc. Temporary ctr_enc alias keeps not-yet-converted ciphers building. Rijndael converted as example. New test drives single CTR call over 0x10000+ blocks across several start counters, checking keystream and counter state against ECB reference. Signed-off-by: Jussi Kivilinna <[email protected]> --- cipher/bulkhelp.h | 4 +- cipher/cipher-ctr.c | 50 ++++++++- cipher/cipher-internal.h | 26 ++++- cipher/rijndael-aesni.c | 123 ++++----------------- cipher/rijndael-armv8-aarch32-ce.S | 104 +++++------------- cipher/rijndael-armv8-aarch64-ce.S | 82 +++++++------- cipher/rijndael-ppc-common.h | 10 ++ cipher/rijndael-ppc-functions.h | 33 +++--- cipher/rijndael-riscv-zvkned.c | 103 +++++++----------- cipher/rijndael-ssse3-amd64.c | 13 +-- cipher/rijndael-vaes-avx2-amd64.S | 130 ++++++++-------------- cipher/rijndael-vaes-avx2-i386.S | 161 +++++++++------------------- cipher/rijndael-vaes-avx512-amd64.S | 89 +++++---------- cipher/rijndael-vp-simd128.h | 56 ++-------- cipher/rijndael.c | 24 ++--- tests/basic.c | 154 ++++++++++++++++++++++++++ 16 files changed, 524 insertions(+), 638 deletions(-) diff --git a/cipher/bulkhelp.h b/cipher/bulkhelp.h index 833262e2..1834a9fb 100644 --- a/cipher/bulkhelp.h +++ b/cipher/bulkhelp.h @@ -155,9 +155,9 @@ bulk_ctr_enc_128 (void *priv, bulk_crypt_fn_t crypt_fn, byte *outbuf, for (i = 1; i < curr_blks; i++) { cipher_block_cpy (&tmpbuf[i * 16], ctr, 16); - cipher_block_add (&tmpbuf[i * 16], i, 16); + cipher_block_add_be16 (&tmpbuf[i * 16], i, 16); } - cipher_block_add (ctr, curr_blks, 16); + cipher_block_add_be16 (ctr, curr_blks, 16); nburn = crypt_fn (priv, tmpbuf, tmpbuf, curr_blks); burn_depth = nburn > burn_depth ? nburn : burn_depth; diff --git a/cipher/cipher-ctr.c b/cipher/cipher-ctr.c index 98363334..1ddaa7e6 100644 --- a/cipher/cipher-ctr.c +++ b/cipher/cipher-ctr.c @@ -64,12 +64,52 @@ _gcry_cipher_ctr_encrypt_ctx (gcry_cipher_hd_t c, /* Use a bulk method if available. */ nblocks = inbuflen >> blocksize_shift; - if (nblocks && c->bulk.ctr_enc) + if (nblocks && c->bulk.ctr16be_enc) { - c->bulk.ctr_enc (algo_context, c->u_ctr.ctr, outbuf, inbuf, nblocks); - inbuf += nblocks << blocksize_shift; - outbuf += nblocks << blocksize_shift; - inbuflen -= nblocks << blocksize_shift; + byte ctr_copy[MAX_BLOCKSIZE]; + + do + { + /* Bulk CTR function only handles 16-bit big-endian addition of + * counter as optimization. Let bulk function to process only up to + * next 16-bit overflow. */ + u32 ctr_low32 = buf_get_be32(&c->u_ctr.ctr[blocksize - sizeof(u32)]); + unsigned int ctr_low16 = ctr_low32 & 0xffffU; + size_t blks_to_overflow = (size_t)0x10000U - ctr_low16; + size_t curr_blks = nblocks; + int ctr_overflows = 0; + + if (nblocks >= blks_to_overflow) + { + curr_blks = blks_to_overflow; + + ctr_overflows = 1; + cipher_block_cpy(ctr_copy, c->u_ctr.ctr, blocksize); + } + + c->bulk.ctr16be_enc (algo_context, c->u_ctr.ctr, outbuf, inbuf, + curr_blks); + + if (ctr_overflows) + { + /* Lower 16-bits of CTR should now be zero. */ + ctr_low32 = buf_get_be32(&c->u_ctr.ctr[blocksize - sizeof(u32)]); + ctr_low16 = ctr_low32 & 0xffffU; + gcry_assert(ctr_low16 == 0); + + /* Handle full blocksize addition. */ + cipher_block_cpy(c->u_ctr.ctr, ctr_copy, blocksize); + cipher_block_add(c->u_ctr.ctr, curr_blks, blocksize); + + wipememory(ctr_copy, sizeof(ctr_copy)); + } + + inbuf += curr_blks << blocksize_shift; + outbuf += curr_blks << blocksize_shift; + inbuflen -= curr_blks << blocksize_shift; + nblocks -= curr_blks; + } + while (nblocks); } /* If we don't have a bulk method use the standard method. We also diff --git a/cipher/cipher-internal.h b/cipher/cipher-internal.h index bc023c76..c99bd189 100644 --- a/cipher/cipher-internal.h +++ b/cipher/cipher-internal.h @@ -196,8 +196,8 @@ typedef struct cipher_bulk_ops const void *inbuf_arg, size_t nblocks); void (*ofb_enc)(void *context, unsigned char *iv, void *outbuf_arg, const void *inbuf_arg, size_t nblocks); - void (*ctr_enc)(void *context, unsigned char *iv, void *outbuf_arg, - const void *inbuf_arg, size_t nblocks); + void (*ctr16be_enc)(void *context, unsigned char *iv, void *outbuf_arg, + const void *inbuf_arg, size_t nblocks); void (*ctr32le_enc)(void *context, unsigned char *iv, void *outbuf_arg, const void *inbuf_arg, size_t nblocks); size_t (*ocb_crypt)(gcry_cipher_hd_t c, void *outbuf_arg, @@ -209,6 +209,11 @@ typedef struct cipher_bulk_ops const void *inbuf_arg, size_t nblocks, int encrypt); } cipher_bulk_ops_t; +/* Temporary alias for transition period from full 128-bit big-endian + * counter addition in bulk processing function to 16-bit big-endian + * addition. */ +#define ctr_enc ctr16be_enc + /* A VIA processor with the Padlock engine as well as the Intel AES_NI instructions require an alignment of most data on a 16 byte @@ -820,9 +825,9 @@ cipher_bytecounter_add (u32 ctr[2], size_t add) } -/* Optimized function for adding value to cipher block. */ +/* Optimized function for block-size big-endian addition. */ static inline void -cipher_block_add(void *_dstsrc, unsigned int add, size_t blocksize) +cipher_block_add(void *_dstsrc, u64 add, size_t blocksize) { byte *dstsrc = _dstsrc; u64 s[2]; @@ -843,6 +848,19 @@ cipher_block_add(void *_dstsrc, unsigned int add, size_t blocksize) } +/* Optimized function for 16-bit big-endian addition to cipher block. */ +static inline void +cipher_block_add_be16(void *_dstsrc, unsigned int add, size_t blocksize) +{ + byte *dstsrc = _dstsrc; + byte add_lo = add & 0xff; + byte add_hi = add >> 8; + + dstsrc[blocksize - 1] += add_lo; + dstsrc[blocksize - 2] += add_hi + (dstsrc[blocksize - 1] < add_lo); +} + + /* Optimized function for cipher block copying */ static inline void cipher_block_cpy(void *_dst, const void *_src, size_t blocksize) diff --git a/cipher/rijndael-aesni.c b/cipher/rijndael-aesni.c index dea78fd0..0093aaec 100644 --- a/cipher/rijndael-aesni.c +++ b/cipher/rijndael-aesni.c @@ -1073,26 +1073,10 @@ do_aesni_ctr (const RIJNDAEL_context *ctx, #define aesenc_xmm1_xmm0 ".byte 0x66, 0x0f, 0x38, 0xdc, 0xc1\n\t" #define aesenclast_xmm1_xmm0 ".byte 0x66, 0x0f, 0x38, 0xdd, 0xc1\n\t" - asm volatile ("movdqa %%xmm5, %%xmm0\n\t" /* xmm0 := CTR (xmm5) */ - "pcmpeqd %%xmm1, %%xmm1\n\t" - "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */ - - "pshufb %%xmm6, %%xmm5\n\t" - "psubq %%xmm1, %%xmm5\n\t" /* xmm5++ (big endian) */ - - /* detect if 64-bit carry handling is needed */ - "cmpl $0xffffffff, 8(%[ctr])\n\t" - "jne .Lno_carry%=\n\t" - "cmpl $0xffffffff, 12(%[ctr])\n\t" - "jne .Lno_carry%=\n\t" - - "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */ - "psubq %%xmm1, %%xmm5\n\t" /* add carry to upper 64bits */ + asm volatile ("movdqa (%[ctr]), %%xmm0\n\t" /* xmm0 := CTR (mem) */ - ".Lno_carry%=:\n\t" - - "pshufb %%xmm6, %%xmm5\n\t" - "movdqa %%xmm5, (%[ctr])\n\t" /* Update CTR (mem). */ + "addb $1, 15(%[ctr])\n\t" /* Update CTR (mem). */ + "adcb $0, 14(%[ctr])\n\t" "pxor (%[key]), %%xmm0\n\t" /* xmm1 ^= key[0] */ "movdqa 0x10(%[key]), %%xmm1\n\t" @@ -1194,45 +1178,21 @@ do_aesni_ctr_4 (const RIJNDAEL_context *ctx, "jmp .Ldone_ctr%=\n\t" ".Ladd32bit%=:\n\t" - "movdqa %%xmm5, (%[ctr])\n\t" /* Restore CTR. */ + "addb $1, 14(%[ctr])\n\t" /* Update CTR. */ "movdqa %%xmm5, %%xmm0\n\t" /* xmm0, xmm2 := CTR (xmm5) */ "movdqa %%xmm0, %%xmm2\n\t" "pcmpeqd %%xmm1, %%xmm1\n\t" - "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */ + "psrldq $14, %%xmm1\n\t" /* xmm1 = -1 */ "pshufb %%xmm6, %%xmm2\n\t" /* xmm2 := le(xmm2) */ - "psubq %%xmm1, %%xmm2\n\t" /* xmm2++ */ + "psubw %%xmm1, %%xmm2\n\t" /* xmm2++ */ "movdqa %%xmm2, %%xmm3\n\t" /* xmm3 := xmm2 */ - "psubq %%xmm1, %%xmm3\n\t" /* xmm3++ */ + "psubw %%xmm1, %%xmm3\n\t" /* xmm3++ */ "movdqa %%xmm3, %%xmm4\n\t" /* xmm4 := xmm3 */ - "psubq %%xmm1, %%xmm4\n\t" /* xmm4++ */ + "psubw %%xmm1, %%xmm4\n\t" /* xmm4++ */ "movdqa %%xmm4, %%xmm5\n\t" /* xmm5 := xmm4 */ - "psubq %%xmm1, %%xmm5\n\t" /* xmm5++ */ - - /* detect if 64-bit carry handling is needed */ - "cmpl $0xffffffff, 8(%[ctr])\n\t" - "jne .Lno_carry%=\n\t" - "movl 12(%[ctr]), %%esi\n\t" - "bswapl %%esi\n\t" - "cmpl $0xfffffffc, %%esi\n\t" - "jb .Lno_carry%=\n\t" /* no carry */ - - "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */ - "je .Lcarry_xmm5%=\n\t" /* esi == 0xfffffffc */ - "cmpl $0xfffffffe, %%esi\n\t" - "jb .Lcarry_xmm4%=\n\t" /* esi == 0xfffffffd */ - "je .Lcarry_xmm3%=\n\t" /* esi == 0xfffffffe */ - /* esi == 0xffffffff */ - - "psubq %%xmm1, %%xmm2\n\t" - ".Lcarry_xmm3%=:\n\t" - "psubq %%xmm1, %%xmm3\n\t" - ".Lcarry_xmm4%=:\n\t" - "psubq %%xmm1, %%xmm4\n\t" - ".Lcarry_xmm5%=:\n\t" - "psubq %%xmm1, %%xmm5\n\t" - - ".Lno_carry%=:\n\t" + "psubw %%xmm1, %%xmm5\n\t" /* xmm5++ */ + "movdqa (%[key]), %%xmm1\n\t" /* xmm1 := key[0] */ "pshufb %%xmm6, %%xmm2\n\t" /* xmm2 := be(xmm2) */ @@ -1240,8 +1200,6 @@ do_aesni_ctr_4 (const RIJNDAEL_context *ctx, "pshufb %%xmm6, %%xmm4\n\t" /* xmm4 := be(xmm4) */ "pshufb %%xmm6, %%xmm5\n\t" /* xmm5 := be(xmm5) */ - "movdqa %%xmm5, (%[ctr])\n\t" /* Update CTR (mem). */ - ".Ldone_ctr%=:\n\t" : : [ctr] "r" (ctr), @@ -1447,67 +1405,29 @@ do_aesni_ctr_8 (const RIJNDAEL_context *ctx, "jmp .Ldone_ctr%=\n\t" ".Ladd32bit%=:\n\t" - "movdqa %%xmm5, (%[ctr])\n\t" /* Restore CTR. */ + "addb $1, 14(%[ctr])\n\t" /* Update CTR. */ "movdqa %%xmm5, %%xmm0\n\t" /* xmm0, xmm2 := CTR (xmm5) */ "movdqa %%xmm0, %%xmm2\n\t" "pcmpeqd %%xmm1, %%xmm1\n\t" - "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */ + "psrldq $14, %%xmm1\n\t" /* xmm1 = -1 */ "pshufb %%xmm6, %%xmm2\n\t" /* xmm2 := le(xmm2) */ - "psubq %%xmm1, %%xmm2\n\t" /* xmm2++ */ + "psubw %%xmm1, %%xmm2\n\t" /* xmm2++ */ "movdqa %%xmm2, %%xmm3\n\t" /* xmm3 := xmm2 */ - "psubq %%xmm1, %%xmm3\n\t" /* xmm3++ */ + "psubw %%xmm1, %%xmm3\n\t" /* xmm3++ */ "movdqa %%xmm3, %%xmm4\n\t" /* xmm4 := xmm3 */ - "psubq %%xmm1, %%xmm4\n\t" /* xmm4++ */ + "psubw %%xmm1, %%xmm4\n\t" /* xmm4++ */ "movdqa %%xmm4, %%xmm8\n\t" /* xmm8 := xmm4 */ - "psubq %%xmm1, %%xmm8\n\t" /* xmm8++ */ + "psubw %%xmm1, %%xmm8\n\t" /* xmm8++ */ "movdqa %%xmm8, %%xmm9\n\t" /* xmm9 := xmm8 */ - "psubq %%xmm1, %%xmm9\n\t" /* xmm9++ */ + "psubw %%xmm1, %%xmm9\n\t" /* xmm9++ */ "movdqa %%xmm9, %%xmm10\n\t" /* xmm10 := xmm9 */ - "psubq %%xmm1, %%xmm10\n\t" /* xmm10++ */ + "psubw %%xmm1, %%xmm10\n\t" /* xmm10++ */ "movdqa %%xmm10, %%xmm11\n\t" /* xmm11 := xmm10 */ - "psubq %%xmm1, %%xmm11\n\t" /* xmm11++ */ + "psubw %%xmm1, %%xmm11\n\t" /* xmm11++ */ "movdqa %%xmm11, %%xmm5\n\t" /* xmm5 := xmm11 */ - "psubq %%xmm1, %%xmm5\n\t" /* xmm5++ */ - - /* detect if 64-bit carry handling is needed */ - "cmpl $0xffffffff, 8(%[ctr])\n\t" - "jne .Lno_carry%=\n\t" - "movl 12(%[ctr]), %%esi\n\t" - "bswapl %%esi\n\t" - "cmpl $0xfffffff8, %%esi\n\t" - "jb .Lno_carry%=\n\t" /* no carry */ - - "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */ - "je .Lcarry_xmm5%=\n\t" /* esi == 0xfffffff8 */ - "cmpl $0xfffffffa, %%esi\n\t" - "jb .Lcarry_xmm11%=\n\t" /* esi == 0xfffffff9 */ - "je .Lcarry_xmm10%=\n\t" /* esi == 0xfffffffa */ - "cmpl $0xfffffffc, %%esi\n\t" - "jb .Lcarry_xmm9%=\n\t" /* esi == 0xfffffffb */ - "je .Lcarry_xmm8%=\n\t" /* esi == 0xfffffffc */ - "cmpl $0xfffffffe, %%esi\n\t" - "jb .Lcarry_xmm4%=\n\t" /* esi == 0xfffffffd */ - "je .Lcarry_xmm3%=\n\t" /* esi == 0xfffffffe */ - /* esi == 0xffffffff */ - - "psubq %%xmm1, %%xmm2\n\t" - ".Lcarry_xmm3%=:\n\t" - "psubq %%xmm1, %%xmm3\n\t" - ".Lcarry_xmm4%=:\n\t" - "psubq %%xmm1, %%xmm4\n\t" - ".Lcarry_xmm8%=:\n\t" - "psubq %%xmm1, %%xmm8\n\t" - ".Lcarry_xmm9%=:\n\t" - "psubq %%xmm1, %%xmm9\n\t" - ".Lcarry_xmm10%=:\n\t" - "psubq %%xmm1, %%xmm10\n\t" - ".Lcarry_xmm11%=:\n\t" - "psubq %%xmm1, %%xmm11\n\t" - ".Lcarry_xmm5%=:\n\t" - "psubq %%xmm1, %%xmm5\n\t" - - ".Lno_carry%=:\n\t" + "psubw %%xmm1, %%xmm5\n\t" /* xmm5++ */ + "movdqa (%[key]), %%xmm1\n\t" /* xmm1 := key[0] */ "movdqa 16(%[key]), %%xmm7\n\t" /* xmm7 := key[1] */ @@ -1536,7 +1456,6 @@ do_aesni_ctr_8 (const RIJNDAEL_context *ctx, "aesenc %%xmm7, %%xmm11\n\t" "pshufb %%xmm6, %%xmm5\n\t" /* xmm5 := be(xmm5) */ - "movdqa %%xmm5, (%[ctr])\n\t" /* Update CTR (mem). */ ".align 16\n\t" ".Ldone_ctr%=:\n\t" diff --git a/cipher/rijndael-armv8-aarch32-ce.S b/cipher/rijndael-armv8-aarch32-ce.S index 7d85aff9..7a0623cf 100644 --- a/cipher/rijndael-armv8-aarch32-ce.S +++ b/cipher/rijndael-armv8-aarch32-ce.S @@ -1007,19 +1007,18 @@ _gcry_aes_ctr_enc_armv8_ce: */ vpush {q4-q7} - push {r4-r12,lr} /* 4*16 + 4*10 = 104b */ - ldr r4, [sp, #(104+0)] - ldr r5, [sp, #(104+4)] + push {r4-r7,lr} /* 4*16 + 4*5 = 84b */ + ldr r4, [sp, #(84+0)] + ldr r5, [sp, #(84+4)] cmp r4, #0 beq .Lctr_enc_skip cmp r5, #12 - ldm r3, {r7-r10} + ldr r7, [r3, #12] vld1.8 {q0}, [r3] /* load IV */ rev r7, r7 - rev r8, r8 - rev r9, r9 - rev r10, r10 + + mov r5, #1 aes_preload_keys(r0, r6); @@ -1032,58 +1031,22 @@ _gcry_aes_ctr_enc_armv8_ce: blo .Lctr_enc_loop_##bits; \ \ .Lctr_enc_loop4_##bits: \ - cmp r10, #0xfffffffc; \ + vmov.i8 q2, #0; \ sub r4, r4, #4; \ - blo .Lctr_enc_loop4_##bits##_nocarry; \ - cmp r9, #0xffffffff; \ - bne .Lctr_enc_loop4_##bits##_nocarry; \ - \ - adds r10, #1; \ - vmov q1, q0; \ - blcs .Lctr_overflow_one; \ - rev r11, r10; \ - vmov.32 d1[1], r11; \ - \ - adds r10, #1; \ - vmov q2, q0; \ - blcs .Lctr_overflow_one; \ - rev r11, r10; \ - vmov.32 d1[1], r11; \ - \ - adds r10, #1; \ - vmov q3, q0; \ - blcs .Lctr_overflow_one; \ - rev r11, r10; \ - vmov.32 d1[1], r11; \ - \ - adds r10, #1; \ - vmov q4, q0; \ - blcs .Lctr_overflow_one; \ - rev r11, r10; \ - vmov.32 d1[1], r11; \ - \ - b .Lctr_enc_loop4_##bits##_store_ctr; \ - \ - .Lctr_enc_loop4_##bits##_nocarry: \ - \ - veor q2, q2; \ - vrev64.8 q1, q0; \ - vceq.u32 d5, d5; \ - vadd.u64 q3, q2, q2; \ - vadd.u64 q4, q3, q2; \ - vadd.u64 q0, q3, q3; \ - vsub.u64 q2, q1, q2; \ - vsub.u64 q3, q1, q3; \ - vsub.u64 q4, q1, q4; \ - vsub.u64 q0, q1, q0; \ - vrev64.8 q1, q1; \ - vrev64.8 q2, q2; \ - vrev64.8 q3, q3; \ - vrev64.8 q0, q0; \ - vrev64.8 q4, q4; \ - add r10, #4; \ - \ - .Lctr_enc_loop4_##bits##_store_ctr: \ + vmov.16 d5[3], r5; \ + vrev16.8 q1, q0; \ + \ + vadd.u16 q0, q2, q2; \ + vadd.u16 q2, q1, q2; \ + vadd.u16 q3, q1, q0; \ + vrev16.8 q1, q1; \ + vadd.u16 q4, q2, q0; \ + vrev16.8 q2, q2; \ + vadd.u16 q0, q3, q0; \ + vrev16.8 q3, q3; \ + vrev16.8 q4, q4; \ + vrev16.8 q0, q0; \ + add r7, #4; \ \ vst1.8 {q0}, [r3]; \ cmp r4, #4; \ @@ -1109,13 +1072,12 @@ _gcry_aes_ctr_enc_armv8_ce: \ .Lctr_enc_loop_##bits: \ \ - adds r10, #1; \ + add r7, #1; \ vmov q1, q0; \ - blcs .Lctr_overflow_one; \ - rev r11, r10; \ + rev r5, r7; \ subs r4, r4, #1; \ vld1.8 {q2}, [r2]!; /* load ciphertext */ \ - vmov.32 d1[1], r11; \ + vmov.32 d1[1], r5; \ \ do_aes_one##bits(e, mc, q1, q1, ##__VA_ARGS__); \ \ @@ -1148,21 +1110,9 @@ _gcry_aes_ctr_enc_armv8_ce: CLEAR_REG(q15) .Lctr_enc_skip: - pop {r4-r12,lr} + pop {r4-r7,lr} vpop {q4-q7} bx lr - -.Lctr_overflow_one: - adcs r9, #0 - adcs r8, #0 - adc r7, #0 - rev r11, r9 - rev r12, r8 - vmov.32 d1[0], r11 - rev r11, r7 - vmov.32 d0[1], r12 - vmov.32 d0[0], r11 - bx lr .size _gcry_aes_ctr_enc_armv8_ce,.-_gcry_aes_ctr_enc_armv8_ce; @@ -1208,7 +1158,7 @@ _gcry_aes_ctr32le_enc_armv8_ce: blo .Lctr32le_enc_loop_##bits; \ \ .Lctr32le_enc_loop4_##bits: \ - veor q2, q2; \ + vmov.i8 q2, #0; \ sub r4, r4, #4; \ vmov.i64 d4, #0xffffffff; /* q2 <= -1:0:0:0 */ \ vmov q1, q0; \ @@ -1244,7 +1194,7 @@ _gcry_aes_ctr32le_enc_armv8_ce: \ .Lctr32le_enc_loop_##bits: \ \ - veor q2, q2; \ + vmov.i8 q2, #0; \ vmov q1, q0; \ vmov.i64 d4, #0xffffffff; /* q2 <= -1:0:0:0 */ \ subs r4, r4, #1; \ diff --git a/cipher/rijndael-armv8-aarch64-ce.S b/cipher/rijndael-armv8-aarch64-ce.S index 64f67fbe..5152fab7 100644 --- a/cipher/rijndael-armv8-aarch64-ce.S +++ b/cipher/rijndael-armv8-aarch64-ce.S @@ -734,14 +734,11 @@ _gcry_aes_ctr_enc_armv8_ce: movi v16.16b, #0 mov v16.S[3], w6 /* 1 */ - /* load IV */ - ldp x9, x10, [x3] + /* load IV, x9 gets counter low byte at bits 56:63 for carry detection */ + ldr x9, [x3, #8] ld1 {v0.16b}, [x3] - rev x9, x9 - rev x10, x10 - mov x12, #(4 << 56) - lsl x11, x10, #56 + mov x10, #(4 << 56) aes_preload_keys(x0, w5); @@ -751,25 +748,25 @@ _gcry_aes_ctr_enc_armv8_ce: #define CTR_ENC(bits) \ .Lctr_enc_entry_##bits: \ cmp x4, #4; \ - b.lo .Lctr_enc_loop_##bits; \ + b.lo .Lctr_enc_loop_entry_##bits; \ \ st1 {v8.16b-v11.16b}, [sp]; /* store callee saved registers */ \ \ - adds x11, x11, x12; \ - add v9.4s, v16.4s, v16.4s; /* 2 */ \ - add v10.4s, v16.4s, v9.4s; /* 3 */ \ - add v11.4s, v9.4s, v9.4s; /* 4 */ \ + adds x9, x9, x10; \ + add v9.16b, v16.16b, v16.16b; /* 2 */ \ + add v10.16b, v16.16b, v9.16b; /* 3 */ \ + add v11.16b, v9.16b, v9.16b; /* 4 */ \ mov x7, #1; \ sub x4, x4, #4; \ ld1 {v5.16b-v8.16b}, [x2], #64; /* preload ciphertext */ \ b.cs .Lctr_enc_carry4_##bits; \ \ + /* 8-bit addition */ \ mov v1.16b, v0.16b; \ - add x10, x10, #4; \ add v2.16b, v0.16b, v16.16b; \ - add v3.8h, v0.8h, v9.8h; \ - add v4.4s, v0.4s, v10.4s; \ - add v0.2d, v0.2d, v11.2d; \ + add v3.16b, v0.16b, v9.16b; \ + add v4.16b, v0.16b, v10.16b; \ + add v0.16b, v0.16b, v11.16b; \ \ .Lctr_enc_entry4_##bits##_carry_done: \ mov x7, #0; \ @@ -786,16 +783,16 @@ _gcry_aes_ctr_enc_armv8_ce: eor v8.16b, v8.16b, vklast.16b; \ do_aes_4_part2_##bits(e, mc, v12, v13, v14, v15, v1, v2, v3, v4, v5, v6, v7, v8); \ ld1 {v5.16b-v8.16b}, [x2], #64; /* preload ciphertext */ \ - adds x11, x11, x12; \ + adds x9, x9, x10; \ sub x4, x4, #4; \ b.cs .Lctr_enc_carry4_##bits; \ \ + /* 8-bit addition */ \ mov v1.16b, v0.16b; \ - add x10, x10, #4; \ add v2.16b, v0.16b, v16.16b; \ - add v3.8h, v0.8h, v9.8h; \ - add v4.4s, v0.4s, v10.4s; \ - add v0.2d, v0.2d, v11.2d; \ + add v3.16b, v0.16b, v9.16b; \ + add v4.16b, v0.16b, v10.16b; \ + add v0.16b, v0.16b, v11.16b; \ \ .Lctr_enc_loop4_##bits##_carry_done: \ cmp x4, #4; \ @@ -815,7 +812,6 @@ _gcry_aes_ctr_enc_armv8_ce: \ st1 {v5.16b-v8.16b}, [x1], #64; /* store plaintext */ \ \ - CLEAR_REG(v3); \ CLEAR_REG(v4); \ ld1 {v8.16b-v11.16b}, [sp]; /* restore callee saved registers */ \ CLEAR_REG(v5); \ @@ -823,16 +819,17 @@ _gcry_aes_ctr_enc_armv8_ce: CLEAR_REG(v7); \ cbz x4, .Lctr_enc_done; \ \ + .Lctr_enc_loop_entry_##bits: \ + rev16 v3.16b, v16.16b; \ + \ .Lctr_enc_loop_##bits: \ \ - adds x10, x10, #1; \ mov v1.16b, v0.16b; \ - adc x9, x9, xzr; \ - dup v0.2d, x10; \ + rev16 v0.16b, v0.16b; /* be->le */ \ + add v0.8h, v0.8h, v3.8h; \ sub x4, x4, #1; \ - ins v0.D[0], x9; \ ld1 {v2.16b}, [x2], #16; /* load ciphertext */ \ - rev64 v0.16b, v0.16b; \ + rev16 v0.16b, v0.16b; \ \ do_aes_one_part1(e, mc, v1, vk0); \ eor v2.16b, v2.16b, vklast.16b; \ @@ -846,27 +843,19 @@ _gcry_aes_ctr_enc_armv8_ce: \ .Lctr_enc_carry4_##bits: \ \ - adds x13, x10, #1; \ + /* 16-bit addition */ \ + rev16 v4.16b, v0.16b; /* be->le */ \ + rev16 v3.16b, v16.16b; \ mov v1.16b, v0.16b; \ - adc x14, x9, xzr; \ - dup v2.2d, x13; \ - adds x13, x10, #2; \ - ins v2.D[0], x14; \ - adc x14, x9, xzr; \ - rev64 v2.16b, v2.16b; \ - dup v3.2d, x13; \ - adds x13, x10, #3; \ - ins v3.D[0], x14; \ - adc x14, x9, xzr; \ - rev64 v3.16b, v3.16b; \ - dup v4.2d, x13; \ - adds x10, x10, #4; \ - ins v4.D[0], x14; \ - adc x9, x9, xzr; \ - rev64 v4.16b, v4.16b; \ - dup v0.2d, x10; \ - ins v0.D[0], x9; \ - rev64 v0.16b, v0.16b; \ + rev16 v0.16b, v9.16b; \ + add v2.8h, v4.8h, v3.8h; /* ctr+1 */ \ + add v3.8h, v4.8h, v0.8h; /* ctr+2 */ \ + add v4.8h, v0.8h, v2.8h; /* ctr+1+2 */ \ + add v0.8h, v0.8h, v3.8h; /* ctr+2+2 */ \ + rev16 v2.16b, v2.16b; /* le->be */ \ + rev16 v3.16b, v3.16b; /* le->be */ \ + rev16 v4.16b, v4.16b; /* le->be */ \ + rev16 v0.16b, v0.16b; /* le->be */ \ \ cbz x7, .Lctr_enc_loop4_##bits##_carry_done; \ b .Lctr_enc_entry4_##bits##_carry_done; @@ -885,6 +874,7 @@ _gcry_aes_ctr_enc_armv8_ce: CLEAR_REG(v0) CLEAR_REG(v1) CLEAR_REG(v2) + CLEAR_REG(v3) CLEAR_REG(v16) add sp, sp, #128; diff --git a/cipher/rijndael-ppc-common.h b/cipher/rijndael-ppc-common.h index 611b5871..d7a35c26 100644 --- a/cipher/rijndael-ppc-common.h +++ b/cipher/rijndael-ppc-common.h @@ -232,6 +232,16 @@ asm_add_uint64(block a, block b) return res; } +static ASM_FUNC_ATTR_INLINE block +asm_add_uint16(block a, block b) +{ + block res; + __asm__ volatile ("vadduhm %0,%1,%2\n\t" + : "=v" (res) + : "v" (a), "v" (b)); + return res; +} + static ASM_FUNC_ATTR_INLINE block asm_sra_int64(block a, block b) { diff --git a/cipher/rijndael-ppc-functions.h b/cipher/rijndael-ppc-functions.h index eb39717d..31b75aee 100644 --- a/cipher/rijndael-ppc-functions.h +++ b/cipher/rijndael-ppc-functions.h @@ -888,25 +888,24 @@ CTR_ENC_FUNC (void *context, unsigned char *ctr_arg, void *outbuf_arg, { block in0, in1, in2, in3, in4, in5, in6, in7; block b0, b1, b2, b3, b4, b5, b6, b7; - block two, three, four; + block two, four; block rkey; - two = asm_add_uint128 (one, one); - three = asm_add_uint128 (two, one); - four = asm_add_uint128 (two, two); + two = asm_add_uint16 (one, one); + four = asm_add_uint16 (two, two); for (; nblocks >= 8; nblocks -= 8) { - b1 = asm_add_uint128 (ctr, one); - b2 = asm_add_uint128 (ctr, two); - b3 = asm_add_uint128 (ctr, three); - b4 = asm_add_uint128 (ctr, four); - b5 = asm_add_uint128 (b1, four); - b6 = asm_add_uint128 (b2, four); - b7 = asm_add_uint128 (b3, four); + b1 = asm_add_uint16 (ctr, one); + b2 = asm_add_uint16 (ctr, two); + b3 = asm_add_uint16 (b1, two); + b4 = asm_add_uint16 (b2, two); + b5 = asm_add_uint16 (b1, four); + b6 = asm_add_uint16 (b2, four); + b7 = asm_add_uint16 (b3, four); b0 = asm_xor (rkey0, ctr); rkey = ALIGNED_LOAD (rk, 1); - ctr = asm_add_uint128 (b4, four); + ctr = asm_add_uint16 (b4, four); b1 = asm_xor (rkey0, b1); b2 = asm_xor (rkey0, b2); b3 = asm_xor (rkey0, b3); @@ -1012,11 +1011,11 @@ CTR_ENC_FUNC (void *context, unsigned char *ctr_arg, void *outbuf_arg, if (nblocks >= 4) { - b1 = asm_add_uint128 (ctr, one); - b2 = asm_add_uint128 (ctr, two); - b3 = asm_add_uint128 (ctr, three); + b1 = asm_add_uint16 (ctr, one); + b2 = asm_add_uint16 (ctr, two); + b3 = asm_add_uint16 (b1, two); b0 = asm_xor (rkey0, ctr); - ctr = asm_add_uint128 (ctr, four); + ctr = asm_add_uint16 (b2, two); b1 = asm_xor (rkey0, b1); b2 = asm_xor (rkey0, b2); b3 = asm_xor (rkey0, b3); @@ -1080,7 +1079,7 @@ CTR_ENC_FUNC (void *context, unsigned char *ctr_arg, void *outbuf_arg, for (; nblocks; nblocks--) { b = ctr; - ctr = asm_add_uint128 (ctr, one); + ctr = asm_add_uint16 (ctr, one); rkeylast = rkeylast_orig ^ VEC_LOAD_BE (in, 0, bige_const); AES_ENCRYPT (b, rounds); diff --git a/cipher/rijndael-riscv-zvkned.c b/cipher/rijndael-riscv-zvkned.c index 9472a99e..ed3887ae 100644 --- a/cipher/rijndael-riscv-zvkned.c +++ b/cipher/rijndael-riscv-zvkned.c @@ -738,8 +738,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3 }, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4 } }; - static const u64 carry_add[2] = { 1, 1 }; - static const u64 nocarry_add[2] = { 1, 0 }; + static const u32 le_add[4] = { 1, 0, 0, 0 }; RIJNDAEL_context *ctx = context; unsigned char *outbuf = outbuf_arg; const unsigned char *inbuf = inbuf_arg; @@ -749,12 +748,12 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, size_t vl_bytes = vl * 4; u64 ctrlow; vuint32m1_t ctr; - vuint8m1_t add1; + vuint32m1_t add1; ROUND_KEY_VARIABLES; PRELOAD_ROUND_KEYS (rk, rounds, vl); - add1 = __riscv_vle8_v_u8m1(add_u8_array[0], vl_bytes); + add1 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[0], vl_bytes)); ctr = unaligned_load_u32m1(ctr_arg, vl); ctrlow = __riscv_vmv_x_s_u64m1_u64(cast_u32m1_u64m1(bswap128_u32m1(ctr, vl))); @@ -762,9 +761,12 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, if (nblocks >= 4) { - vuint8m1_t add2 = __riscv_vle8_v_u8m1(add_u8_array[1], vl_bytes); - vuint8m1_t add3 = __riscv_vle8_v_u8m1(add_u8_array[2], vl_bytes); - vuint8m1_t add4 = __riscv_vle8_v_u8m1(add_u8_array[3], vl_bytes); + vuint32m1_t add2 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[1], + vl_bytes)); + vuint32m1_t add3 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[2], + vl_bytes)); + vuint32m1_t add4 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[3], + vl_bytes)); memory_barrier_with_vec(add2); memory_barrier_with_vec(add3); @@ -778,67 +780,43 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, /* detect if 8-bit carry handling is needed */ if (UNLIKELY(((ctrlow += 4) & 0xff) <= 3)) { - static const u64 *adders[5][4] = - { - { nocarry_add, nocarry_add, nocarry_add, carry_add }, - { nocarry_add, nocarry_add, carry_add, nocarry_add }, - { nocarry_add, carry_add, nocarry_add, nocarry_add }, - { carry_add, nocarry_add, nocarry_add, nocarry_add }, - { nocarry_add, nocarry_add, nocarry_add, nocarry_add } - }; - unsigned int idx = ctrlow <= 3 ? ctrlow : 4; - vuint64m1_t ctr_u64; - vuint32m1_t ctr_u32_1; - vuint32m1_t ctr_u32_2; - vuint32m1_t ctr_u32_3; - vuint32m1_t ctr_u32_4; - vuint64m1_t add_u64; + vuint32m1_t add_1 = __riscv_vle32_v_u32m1(le_add, vl); + vuint32m1_t add_2 = __riscv_vadd_vv_u32m1(add_1, add_1, vl); + vuint32m1_t ctr_le; + vuint32m1_t ctr_1; + vuint32m1_t ctr_2; + vuint32m1_t ctr_3; + vuint32m1_t ctr_4; /* Byte swap counter */ - ctr_u64 = cast_u32m1_u64m1(bswap128_u32m1(ctr, vl)); + ctr_le = bswap128_u32m1(ctr, vl); /* Addition with carry handling */ - add_u64 = __riscv_vle64_v_u64m1(adders[idx][0], vl / 2); - ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2); - ctr_u32_1 = cast_u64m1_u32m1(ctr_u64); - - add_u64 = __riscv_vle64_v_u64m1(adders[idx][1], vl / 2); - ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2); - ctr_u32_2 = cast_u64m1_u32m1(ctr_u64); - - add_u64 = __riscv_vle64_v_u64m1(adders[idx][2], vl / 2); - ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2); - ctr_u32_3 = cast_u64m1_u32m1(ctr_u64); - - add_u64 = __riscv_vle64_v_u64m1(adders[idx][3], vl / 2); - ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2); - ctr_u32_4 = cast_u64m1_u32m1(ctr_u64); + ctr_1 = __riscv_vadd_vv_u32m1(ctr_le, add_1, vl); + ctr_2 = __riscv_vadd_vv_u32m1(ctr_le, add_2, vl); + ctr_3 = __riscv_vadd_vv_u32m1(ctr_1, add_2, vl); + ctr_4 = __riscv_vadd_vv_u32m1(ctr_2, add_2, vl); /* Byte swap counters */ - ctr_u32_1 = bswap128_u32m1(ctr_u32_1, vl); - ctr_u32_2 = bswap128_u32m1(ctr_u32_2, vl); - ctr_u32_3 = bswap128_u32m1(ctr_u32_3, vl); - ctr_u32_4 = bswap128_u32m1(ctr_u32_4, vl); - - ctr4blks = merge_4x_u32m1_to_u32m4(ctr, ctr_u32_1, ctr_u32_2, - ctr_u32_3); - ctr = ctr_u32_4; + ctr_1 = bswap128_u32m1(ctr_1, vl); + ctr_2 = bswap128_u32m1(ctr_2, vl); + ctr_3 = bswap128_u32m1(ctr_3, vl); + ctr_4 = bswap128_u32m1(ctr_4, vl); + + ctr4blks = merge_4x_u32m1_to_u32m4(ctr, ctr_1, ctr_2, ctr_3); + ctr = ctr_4; } else { /* Fast path addition without carry handling */ - vuint8m1_t ctr_u8 = cast_u32m1_u8m1(ctr); - vuint8m1_t ctr1 = __riscv_vadd_vv_u8m1(ctr_u8, add1, vl_bytes); - vuint8m1_t ctr2 = __riscv_vadd_vv_u8m1(ctr_u8, add2, vl_bytes); - vuint8m1_t ctr3 = __riscv_vadd_vv_u8m1(ctr_u8, add3, vl_bytes); - - ctr = cast_u8m1_u32m1(__riscv_vadd_vv_u8m1(ctr_u8, add4, - vl_bytes)); - - ctr4blks = merge_4x_u32m1_to_u32m4(cast_u8m1_u32m1(ctr_u8), - cast_u8m1_u32m1(ctr1), - cast_u8m1_u32m1(ctr2), - cast_u8m1_u32m1(ctr3)); + vuint32m1_t ctr0 = ctr; + vuint32m1_t ctr1 = __riscv_vadd_vv_u32m1(ctr, add1, vl); + vuint32m1_t ctr2 = __riscv_vadd_vv_u32m1(ctr, add2, vl); + vuint32m1_t ctr3 = __riscv_vadd_vv_u32m1(ctr, add3, vl); + + ctr = __riscv_vadd_vv_u32m1(ctr, add4, vl); + + ctr4blks = merge_4x_u32m1_to_u32m4(ctr0, ctr1, ctr2, ctr3); } data4blks = __riscv_vle8_v_u8m4(inbuf, vl_bytes * 4); @@ -862,15 +840,13 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, /* detect if 8-bit carry handling is needed */ if (UNLIKELY((++ctrlow & 0xff) == 0)) { - const u64 *add_arr = UNLIKELY(ctrlow == 0) ? carry_add : nocarry_add; - vuint64m1_t add_val = __riscv_vle64_v_u64m1(add_arr, vl / 2); + vuint32m1_t add_val = __riscv_vle32_v_u32m1(le_add, vl); /* Byte swap counter */ ctr = bswap128_u32m1(ctr, vl); /* Addition with carry handling */ - ctr = cast_u64m1_u32m1(__riscv_vadd_vv_u64m1(cast_u32m1_u64m1(ctr), - add_val, vl / 2)); + ctr = __riscv_vadd_vv_u32m1(ctr, add_val, vl); /* Byte swap counter */ ctr = bswap128_u32m1(ctr, vl); @@ -878,8 +854,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, else { /* Fast path addition without carry handling */ - ctr = cast_u8m1_u32m1(__riscv_vadd_vv_u8m1(cast_u32m1_u8m1(ctr), - add1, vl_bytes)); + ctr = __riscv_vadd_vv_u32m1(ctr, add1, vl); } AES_CRYPT(e, m1, rounds, block, vl); diff --git a/cipher/rijndael-ssse3-amd64.c b/cipher/rijndael-ssse3-amd64.c index 0f0abf62..d39173a3 100644 --- a/cipher/rijndael-ssse3-amd64.c +++ b/cipher/rijndael-ssse3-amd64.c @@ -375,19 +375,10 @@ _gcry_aes_ssse3_ctr_enc (RIJNDAEL_context *ctx, unsigned char *ctr, { asm volatile ("movdqa %%xmm7, %%xmm0\n\t" /* xmm0 := CTR (xmm7) */ "pcmpeqd %%xmm1, %%xmm1\n\t" - "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */ + "psrldq $14, %%xmm1\n\t" /* xmm1 = -1 */ "pshufb %%xmm6, %%xmm7\n\t" - "psubq %%xmm1, %%xmm7\n\t" /* xmm7++ (big endian) */ - - /* detect if 64-bit carry handling is needed */ - "incq %q[ctrlow]\n\t" - "jnz .Lno_carry%=\n\t" - - "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */ - "psubq %%xmm1, %%xmm7\n\t" /* add carry to upper 64bits */ - - ".Lno_carry%=:\n\t" + "psubw %%xmm1, %%xmm7\n\t" /* xmm7++ (big endian) */ "pshufb %%xmm6, %%xmm7\n\t" : [ctrlow] "+r" (ctrlow) diff --git a/cipher/rijndael-vaes-avx2-amd64.S b/cipher/rijndael-vaes-avx2-amd64.S index 07e6f1ca..65fd79fe 100644 --- a/cipher/rijndael-vaes-avx2-amd64.S +++ b/cipher/rijndael-vaes-avx2-amd64.S @@ -714,40 +714,14 @@ _gcry_vaes_avx2_ctr_enc_amd64: */ CFI_STARTPROC(); - movq 8(%rsi), %r10; - movq 0(%rsi), %r11; - bswapq %r10; - bswapq %r11; + movzbl 15(%rsi), %r10d; + movzbl 14(%rsi), %r11d; vpcmpeqd %ymm15, %ymm15, %ymm15; - vpsrldq $8, %ymm15, %ymm15; // 0:-1 - vpaddq %ymm15, %ymm15, %ymm14; // 0:-2 + vpsrldq $14, %ymm15, %ymm15; // 0:-1 + vpaddw %ymm15, %ymm15, %ymm14; // 0:-2 vbroadcasti128 .Lbswap128_mask rRIP, %ymm13; -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ - vpcmpeqq minus_one, x, tmp1; \ - vpcmpeqq minus_two, x, tmp2; \ - vpor tmp1, tmp2, tmp2; \ - vpsubq minus_two, x, x; \ - vpslldq $8, tmp2, tmp2; \ - vpsubq tmp2, x, x; - -#define handle_ctr_128bit_add(nblks) \ - addq $(nblks), %r10; \ - adcq $0, %r11; \ - bswapq %r10; \ - bswapq %r11; \ - movq %r10, 8(%rsi); \ - movq %r11, 0(%rsi); \ - bswapq %r10; \ - bswapq %r11; - /* Process 16 blocks per loop. */ .align 8 .Lctr_enc_blk16: @@ -760,11 +734,9 @@ _gcry_vaes_avx2_ctr_enc_amd64: vbroadcasti128 (0 * 16)(%rdi), %ymm8; /* detect if carry handling is needed */ - addb $16, 15(%rsi); + addb $16, %r10b; jc .Lctr_enc_blk16_handle_carry; - leaq 16(%r10), %r10; - .Lctr_enc_blk16_byte_bige_add: /* Increment counters. */ vpaddb .Lbige_addb_0 rRIP, %ymm7, %ymm0; @@ -777,6 +749,7 @@ _gcry_vaes_avx2_ctr_enc_amd64: vpaddb .Lbige_addb_14 rRIP, %ymm7, %ymm7; .Lctr_enc_blk16_rounds: + movb %r10b, 15(%rsi); /* AES rounds */ XOR8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7); vbroadcasti128 (1 * 16)(%rdi), %ymm8; @@ -841,34 +814,30 @@ _gcry_vaes_avx2_ctr_enc_amd64: jmp .Lctr_enc_blk16; - .align 8 - .Lctr_enc_blk16_handle_only_ctr_carry: - handle_ctr_128bit_add(16); - jmp .Lctr_enc_blk16_byte_bige_add; - .align 8 .Lctr_enc_blk16_handle_carry: - jz .Lctr_enc_blk16_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + leal 1(%r11d), %r11d; + movb %r11b, 14(%rsi); + jz .Lctr_enc_blk16_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ vpshufb %xmm13, %xmm7, %xmm1; /* be => le */ - vmovdqa %xmm1, %xmm0; - inc_le128(%xmm1, %xmm15, %xmm5); - vinserti128 $1, %xmm1, %ymm0, %ymm7; /* ctr: +1:+0 */ + vpsubw %xmm15, %xmm1, %xmm0; + vinserti128 $1, %xmm0, %ymm1, %ymm7; /* ctr: +1:+0 */ vpshufb %ymm13, %ymm7, %ymm0; - handle_ctr_128bit_add(16); - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +3:+2 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +3:+2 */ vpshufb %ymm13, %ymm7, %ymm1; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +5:+4 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +5:+4 */ vpshufb %ymm13, %ymm7, %ymm2; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +7:+6 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +7:+6 */ vpshufb %ymm13, %ymm7, %ymm3; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +9:+8 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +9:+8 */ vpshufb %ymm13, %ymm7, %ymm4; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +11:+10 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +11:+10 */ vpshufb %ymm13, %ymm7, %ymm5; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +13:+12 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +13:+12 */ vpshufb %ymm13, %ymm7, %ymm6; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +15:+14 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +15:+14 */ vpshufb %ymm13, %ymm7, %ymm7; jmp .Lctr_enc_blk16_rounds; @@ -885,11 +854,9 @@ _gcry_vaes_avx2_ctr_enc_amd64: vbroadcasti128 (0 * 16)(%rdi), %ymm4; /* detect if carry handling is needed */ - addb $8, 15(%rsi); + addb $8, %r10b; jc .Lctr_enc_blk8_handle_carry; - leaq 8(%r10), %r10; - .Lctr_enc_blk8_byte_bige_add: /* Increment counters. */ vpaddb .Lbige_addb_0 rRIP, %ymm3, %ymm0; @@ -898,6 +865,7 @@ _gcry_vaes_avx2_ctr_enc_amd64: vpaddb .Lbige_addb_6 rRIP, %ymm3, %ymm3; .Lctr_enc_blk8_rounds: + movb %r10b, 15(%rsi); /* AES rounds */ XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (1 * 16)(%rdi), %ymm4; @@ -950,26 +918,22 @@ _gcry_vaes_avx2_ctr_enc_amd64: jmp .Lctr_enc_blk4; - .align 8 - .Lctr_enc_blk8_handle_only_ctr_carry: - handle_ctr_128bit_add(8); - jmp .Lctr_enc_blk8_byte_bige_add; - .align 8 .Lctr_enc_blk8_handle_carry: - jz .Lctr_enc_blk8_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + leal 1(%r11d), %r11d; + movb %r11b, 14(%rsi); + jz .Lctr_enc_blk8_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ vpshufb %xmm13, %xmm3, %xmm1; /* be => le */ - vmovdqa %xmm1, %xmm0; - inc_le128(%xmm1, %xmm15, %xmm5); - vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */ + vpsubw %xmm15, %xmm1, %xmm0; + vinserti128 $1, %xmm0, %ymm1, %ymm3; /* ctr: +1:+0 */ vpshufb %ymm13, %ymm3, %ymm0; - handle_ctr_128bit_add(8); - add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +3:+2 */ + vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +3:+2 */ vpshufb %ymm13, %ymm3, %ymm1; - add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +5:+4 */ + vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +5:+4 */ vpshufb %ymm13, %ymm3, %ymm2; - add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +7:+6 */ + vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +7:+6 */ vpshufb %ymm13, %ymm3, %ymm3; jmp .Lctr_enc_blk8_rounds; @@ -986,17 +950,16 @@ _gcry_vaes_avx2_ctr_enc_amd64: vbroadcasti128 (0 * 16)(%rdi), %ymm4; /* detect if carry handling is needed */ - addb $4, 15(%rsi); + addb $4, %r10b; jc .Lctr_enc_blk4_handle_carry; - leaq 4(%r10), %r10; - .Lctr_enc_blk4_byte_bige_add: /* Increment counters. */ vpaddb .Lbige_addb_0 rRIP, %ymm3, %ymm0; vpaddb .Lbige_addb_2 rRIP, %ymm3, %ymm1; .Lctr_enc_blk4_rounds: + movb %r10b, 15(%rsi); /* AES rounds */ XOR2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (1 * 16)(%rdi), %ymm4; @@ -1043,22 +1006,18 @@ _gcry_vaes_avx2_ctr_enc_amd64: jmp .Lctr_enc_blk1; - .align 8 - .Lctr_enc_blk4_handle_only_ctr_carry: - handle_ctr_128bit_add(4); - jmp .Lctr_enc_blk4_byte_bige_add; - .align 8 .Lctr_enc_blk4_handle_carry: - jz .Lctr_enc_blk4_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + leal 1(%r11d), %r11d; + movb %r11b, 14(%rsi); + jz .Lctr_enc_blk4_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ vpshufb %xmm13, %xmm3, %xmm1; /* be => le */ - vmovdqa %xmm1, %xmm0; - inc_le128(%xmm1, %xmm15, %xmm5); - vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */ + vpsubw %xmm15, %xmm1, %xmm0; + vinserti128 $1, %xmm0, %ymm1, %ymm3; /* ctr: +1:+0 */ vpshufb %ymm13, %ymm3, %ymm0; - handle_ctr_128bit_add(4); - add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +3:+2 */ + vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +3:+2 */ vpshufb %ymm13, %ymm3, %ymm1; jmp .Lctr_enc_blk4_rounds; @@ -1073,13 +1032,16 @@ _gcry_vaes_avx2_ctr_enc_amd64: /* Load and increament counter. */ vmovdqu (%rsi), %xmm0; - handle_ctr_128bit_add(1); + addb $1, %r10b; + adcb $0, %r11b; /* AES rounds. */ vpxor (0 * 16)(%rdi), %xmm0, %xmm0; vaesenc (1 * 16)(%rdi), %xmm0, %xmm0; vaesenc (2 * 16)(%rdi), %xmm0, %xmm0; vaesenc (3 * 16)(%rdi), %xmm0, %xmm0; + movb %r10b, 15(%rsi); + movb %r11b, 14(%rsi); vaesenc (4 * 16)(%rdi), %xmm0, %xmm0; vaesenc (5 * 16)(%rdi), %xmm0, %xmm0; vaesenc (6 * 16)(%rdi), %xmm0, %xmm0; diff --git a/cipher/rijndael-vaes-avx2-i386.S b/cipher/rijndael-vaes-avx2-i386.S index 245e8443..d1bf8bb5 100644 --- a/cipher/rijndael-vaes-avx2-i386.S +++ b/cipher/rijndael-vaes-avx2-i386.S @@ -556,15 +556,15 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): movl %esp, %ebp; CFI_DEF_CFA_REGISTER(%ebp); - subl $(3 * 32 + 3 * 4), %esp; + subl $(1 * 32 + 3 * 4), %esp; andl $-32, %esp; - movl %edi, (3 * 32 + 0 * 4)(%esp); - CFI_REG_ON_STACK(edi, 3 * 32 + 0 * 4); - movl %esi, (3 * 32 + 1 * 4)(%esp); - CFI_REG_ON_STACK(esi, 3 * 32 + 1 * 4); - movl %ebx, (3 * 32 + 2 * 4)(%esp); - CFI_REG_ON_STACK(ebx, 3 * 32 + 2 * 4); + movl %edi, (1 * 32 + 0 * 4)(%esp); + CFI_REG_ON_STACK(edi, 1 * 32 + 0 * 4); + movl %esi, (1 * 32 + 1 * 4)(%esp); + CFI_REG_ON_STACK(esi, 1 * 32 + 1 * 4); + movl %ebx, (1 * 32 + 2 * 4)(%esp); + CFI_REG_ON_STACK(ebx, 1 * 32 + 2 * 4); movl %eax, %ebx; movl 4+4(%ebp), %edi; @@ -574,50 +574,8 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): #define prepare_ctr_const(minus_one, minus_two) \ vpcmpeqd minus_one, minus_one, minus_one; \ - vpsrldq $8, minus_one, minus_one; /* 0:-1 */ \ - vpaddq minus_one, minus_one, minus_two; /* 0:-2 */ - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ - vpcmpeqq minus_one, x, tmp1; \ - vpcmpeqq minus_two, x, tmp2; \ - vpor tmp1, tmp2, tmp2; \ - vpsubq minus_two, x, x; \ - vpslldq $8, tmp2, tmp2; \ - vpsubq tmp2, x, x; - -#define handle_ctr_128bit_add(nblks) \ - movl 12(%esi), %eax; \ - bswapl %eax; \ - addl $nblks, %eax; \ - bswapl %eax; \ - movl %eax, 12(%esi); \ - jnc 1f; \ - \ - movl 8(%esi), %eax; \ - bswapl %eax; \ - adcl $0, %eax; \ - bswapl %eax; \ - movl %eax, 8(%esi); \ - \ - movl 4(%esi), %eax; \ - bswapl %eax; \ - adcl $0, %eax; \ - bswapl %eax; \ - movl %eax, 4(%esi); \ - \ - movl 0(%esi), %eax; \ - bswapl %eax; \ - adcl $0, %eax; \ - bswapl %eax; \ - movl %eax, 0(%esi); \ - .align 8; \ - 1:; + vpsrldq $14, minus_one, minus_one; /* 0:-1 */ \ + vpaddw minus_one, minus_one, minus_two; /* 0:-2 */ cmpl $12, 4+20(%ebp); jae .Lctr_enc_blk12_loop; @@ -631,10 +589,8 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): vbroadcasti128 (%esi), %ymm6; /* detect if carry handling is needed */ - movl 12(%esi), %eax; - addl $(12 << 24), %eax; + addb $12, 15(%esi); jc .Lctr_enc_blk12_handle_carry; - movl %eax, 12(%esi); .Lctr_enc_blk12_byte_bige_add: /* Increment counters. */ @@ -707,39 +663,32 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): jae .Lctr_enc_blk12_loop; jmp .Lctr_enc_blk4; - .align 8 - .Lctr_enc_blk12_handle_only_ctr_carry: - handle_ctr_128bit_add(12); - jmp .Lctr_enc_blk12_byte_bige_add; - .align 8 .Lctr_enc_blk12_handle_carry: - jz .Lctr_enc_blk12_handle_only_ctr_carry; - /* Increment counters (handle carry). */ - prepare_ctr_const(%ymm4, %ymm7); - vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm2; - vpshufb %xmm2, %xmm6, %xmm1; /* be => le */ - vmovdqa %xmm1, %xmm0; - inc_le128(%xmm1, %xmm4, %xmm5); - vinserti128 $1, %xmm1, %ymm0, %ymm6; /* ctr: +1:+0 */ - handle_ctr_128bit_add(12); - vpshufb %ymm2, %ymm6, %ymm0; + movzbl 14(%esi), %eax; + leal 1(%eax), %eax; + movb %al, 14(%esi); + jz .Lctr_enc_blk12_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ + prepare_ctr_const(%ymm2, %ymm7); + vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm4; + vpshufb %xmm4, %xmm6, %xmm1; /* be => le */ + vpsubw %xmm2, %xmm1, %xmm0; + vinserti128 $1, %xmm0, %ymm1, %ymm6; /* ctr: +1:+0 */ + vpshufb %ymm4, %ymm6, %ymm0; vmovdqa %ymm0, (0 * 32)(%esp); - add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +3:+2 */ - vpshufb %ymm2, %ymm6, %ymm0; - vmovdqa %ymm0, (1 * 32)(%esp); - add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +5:+4 */ - vpshufb %ymm2, %ymm6, %ymm0; - vmovdqa %ymm0, (2 * 32)(%esp); - add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +7:+6 */ - vpshufb %ymm2, %ymm6, %ymm3; - add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +9:+8 */ - vpshufb %ymm2, %ymm6, %ymm5; - add2_le128(%ymm6, %ymm4, %ymm7, %ymm2, %ymm1); /* ctr: +11:+10 */ + vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +3:+2 */ + vpshufb %ymm4, %ymm6, %ymm1; + vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +5:+4 */ + vpshufb %ymm4, %ymm6, %ymm2; + vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +7:+6 */ + vpshufb %ymm4, %ymm6, %ymm3; + vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +9:+8 */ + vpshufb %ymm4, %ymm6, %ymm5; + vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +11:+10 */ vmovdqa (0 * 32)(%esp), %ymm0; - vmovdqa (1 * 32)(%esp), %ymm1; - vmovdqa (2 * 32)(%esp), %ymm2; - vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm6, %ymm6; + vpshufb %ymm4, %ymm6, %ymm6; jmp .Lctr_enc_blk12_rounds; @@ -754,10 +703,8 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): vbroadcasti128 (%esi), %ymm3; /* detect if carry handling is needed */ - movl 12(%esi), %eax; - addl $(4 << 24), %eax; + addb $4, 15(%esi); jc .Lctr_enc_blk4_handle_carry; - movl %eax, 12(%esi); .Lctr_enc_blk4_byte_bige_add: /* Increment counters. */ @@ -812,24 +759,22 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): jmp .Lctr_enc_blk1; - .align 8 - .Lctr_enc_blk4_handle_only_ctr_carry: - handle_ctr_128bit_add(4); - jmp .Lctr_enc_blk4_byte_bige_add; - .align 8 .Lctr_enc_blk4_handle_carry: - jz .Lctr_enc_blk4_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + movzbl 14(%esi), %eax; + leal 1(%eax), %eax; + movb %al, 14(%esi); + jz .Lctr_enc_blk4_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ prepare_ctr_const(%ymm4, %ymm7); - vpshufb CADDR(.Lbswap128_mask, %ebx), %xmm3, %xmm1; /* be => le */ - vmovdqa %xmm1, %xmm0; - inc_le128(%xmm1, %xmm4, %xmm5); - vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */ - vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm3, %ymm0; - handle_ctr_128bit_add(4); - add2_le128(%ymm3, %ymm4, %ymm7, %ymm5, %ymm6); /* ctr: +3:+2 */ - vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm3, %ymm1; + vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm2; + vpshufb %xmm2, %xmm3, %xmm1; /* be => le */ + vpsubw %xmm4, %xmm1, %xmm0; + vinserti128 $1, %xmm0, %ymm1, %ymm3; /* ctr: +1:+0 */ + vpshufb %ymm2, %ymm3, %ymm0; + vpsubw %ymm7, %ymm3, %ymm3; /* ctr: +3:+2 */ + vpshufb %ymm2, %ymm3, %ymm1; jmp .Lctr_enc_blk4_rounds; @@ -841,15 +786,16 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): subl $1, 4+20(%ebp); - /* Load and increament counter. */ + /* Load counter. */ vmovdqu (%esi), %xmm0; - handle_ctr_128bit_add(1); /* AES rounds. */ vpxor (0 * 16)(%edi), %xmm0, %xmm0; vaesenc (1 * 16)(%edi), %xmm0, %xmm0; vaesenc (2 * 16)(%edi), %xmm0, %xmm0; vaesenc (3 * 16)(%edi), %xmm0, %xmm0; + addb $1, 15(%esi); /* Increament counter. */ + adcb $0, 14(%esi); vaesenc (4 * 16)(%edi), %xmm0, %xmm0; vaesenc (5 * 16)(%edi), %xmm0, %xmm0; vaesenc (6 * 16)(%edi), %xmm0, %xmm0; @@ -880,18 +826,17 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): .align 8 .Ldone_ctr_enc: vpxor %ymm0, %ymm0, %ymm0; - movl (3 * 32 + 0 * 4)(%esp), %edi; + movl (1 * 32 + 0 * 4)(%esp), %edi; CFI_RESTORE(edi); - movl (3 * 32 + 1 * 4)(%esp), %esi; + movl (1 * 32 + 1 * 4)(%esp), %esi; CFI_RESTORE(esi); - movl (3 * 32 + 2 * 4)(%esp), %ebx; + movl (1 * 32 + 2 * 4)(%esp), %ebx; CFI_RESTORE(ebx); vmovdqa %ymm0, (0 * 32)(%esp); - vmovdqa %ymm0, (1 * 32)(%esp); - vmovdqa %ymm0, (2 * 32)(%esp); leave; CFI_LEAVE(); vzeroall; + xorl %eax, %eax; ret_spec_stop CFI_ENDPROC(); ELF(.size SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386), diff --git a/cipher/rijndael-vaes-avx512-amd64.S b/cipher/rijndael-vaes-avx512-amd64.S index f20998b0..96cbdd0a 100644 --- a/cipher/rijndael-vaes-avx512-amd64.S +++ b/cipher/rijndael-vaes-avx512-amd64.S @@ -519,16 +519,14 @@ _gcry_vaes_avx512_ctr_enc_amd64: spec_stop_avx512; - movq 8(%rsi), %r10; - movq 0(%rsi), %r11; - bswapq %r10; - bswapq %r11; - vmovdqa32 .Lbige_addb_0 rRIP, %zmm20; vmovdqa32 .Lbige_addb_4 rRIP, %zmm21; vmovdqa32 .Lbige_addb_8 rRIP, %zmm22; vmovdqa32 .Lbige_addb_12 rRIP, %zmm23; + movzbl 15(%rsi), %r10d; + movzbl 14(%rsi), %r11d; + /* Load first and last key. */ leal (, %r9d, 4), %eax; vbroadcasti32x4 (%rdi), %zmm30; @@ -542,22 +540,6 @@ _gcry_vaes_avx512_ctr_enc_amd64: vmovdqa32 .Lbige_addb_24 rRIP, %zmm26; vmovdqa32 .Lbige_addb_28 rRIP, %zmm27; -#define add_le128(out, in, lo_counter, hi_counter1) \ - vpaddq lo_counter, in, out; \ - vpcmpuq $1, lo_counter, out, %k1; \ - kaddb %k1, %k1, %k1; \ - vpaddq hi_counter1, out, out{%k1}; - -#define handle_ctr_128bit_add(nblks) \ - addq $(nblks), %r10; \ - adcq $0, %r11; \ - bswapq %r10; \ - bswapq %r11; \ - movq %r10, 8(%rsi); \ - movq %r11, 0(%rsi); \ - bswapq %r10; \ - bswapq %r11; - /* Process 32 blocks per loop. */ .align 16 .Lctr_enc_blk32: @@ -567,11 +549,9 @@ _gcry_vaes_avx512_ctr_enc_amd64: vbroadcasti32x4 (1 * 16)(%rdi), %zmm8; /* detect if carry handling is needed */ - addb $32, 15(%rsi); + addb $32, %r10b; jc .Lctr_enc_blk32_handle_carry; - leaq 32(%r10), %r10; - .Lctr_enc_blk32_byte_bige_add: /* Increment counters. */ vpaddb %zmm20, %zmm7, %zmm0; @@ -584,6 +564,7 @@ _gcry_vaes_avx512_ctr_enc_amd64: vpaddb %zmm27, %zmm7, %zmm7; .Lctr_enc_blk32_rounds: + movb %r10b, 15(%rsi); /* AES rounds */ XOR8(%zmm30, %zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7); VAESENC8(%zmm8, %zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7); @@ -656,35 +637,31 @@ _gcry_vaes_avx512_ctr_enc_amd64: jmp .Lctr_enc_blk16; - .align 16 - .Lctr_enc_blk32_handle_only_ctr_carry: - handle_ctr_128bit_add(32); - jmp .Lctr_enc_blk32_byte_bige_add; - .align 16 .Lctr_enc_blk32_handle_carry: - jz .Lctr_enc_blk32_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + leal 1(%r11d), %r11d; + movb %r11b, 14(%rsi); + jz .Lctr_enc_blk32_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ vbroadcasti32x4 .Lbswap128_mask rRIP, %zmm15; vpmovzxbq .Lcounter0_1_2_3_lo_bq rRIP, %zmm10; - vpmovzxbq .Lcounter1_1_1_1_hi_bq rRIP, %zmm13; vpshufb %zmm15, %zmm7, %zmm7; /* be => le */ vpmovzxbq .Lcounter4_4_4_4_lo_bq rRIP, %zmm11; vpmovzxbq .Lcounter8_8_8_8_lo_bq rRIP, %zmm12; - handle_ctr_128bit_add(32); - add_le128(%zmm0, %zmm7, %zmm10, %zmm13); /* +0:+1:+2:+3 */ - add_le128(%zmm1, %zmm0, %zmm11, %zmm13); /* +4:+5:+6:+7 */ - add_le128(%zmm2, %zmm0, %zmm12, %zmm13); /* +8:... */ + vpaddw %zmm10, %zmm7, %zmm0; /* +0:+1:+2:+3 */ + vpaddw %zmm11, %zmm0, %zmm1; /* +4:+5:+6:+7 */ + vpaddw %zmm12, %zmm0, %zmm2; /* +8:... */ vpshufb %zmm15, %zmm0, %zmm0; /* le => be */ - add_le128(%zmm3, %zmm1, %zmm12, %zmm13); /* +12:... */ + vpaddw %zmm12, %zmm1, %zmm3; /* +12:... */ vpshufb %zmm15, %zmm1, %zmm1; /* le => be */ - add_le128(%zmm4, %zmm2, %zmm12, %zmm13); /* +16:... */ + vpaddw %zmm12, %zmm2, %zmm4; /* +16:... */ vpshufb %zmm15, %zmm2, %zmm2; /* le => be */ - add_le128(%zmm5, %zmm3, %zmm12, %zmm13); /* +20:... */ + vpaddw %zmm12, %zmm3, %zmm5; /* +20:... */ vpshufb %zmm15, %zmm3, %zmm3; /* le => be */ - add_le128(%zmm6, %zmm4, %zmm12, %zmm13); /* +24:... */ + vpaddw %zmm12, %zmm4, %zmm6; /* +24:... */ vpshufb %zmm15, %zmm4, %zmm4; /* le => be */ - add_le128(%zmm7, %zmm5, %zmm12, %zmm13); /* +28:... */ + vpaddw %zmm12, %zmm5, %zmm7; /* +28:... */ vpshufb %zmm15, %zmm5, %zmm5; /* le => be */ vpshufb %zmm15, %zmm6, %zmm6; /* le => be */ vpshufb %zmm15, %zmm7, %zmm7; /* le => be */ @@ -703,11 +680,9 @@ _gcry_vaes_avx512_ctr_enc_amd64: vbroadcasti32x4 (1 * 16)(%rdi), %zmm4; /* detect if carry handling is needed */ - addb $16, 15(%rsi); + addb $16, %r10b; jc .Lctr_enc_blk16_handle_carry; - leaq 16(%r10), %r10; - .Lctr_enc_blk16_byte_bige_add: /* Increment counters. */ vpaddb %zmm20, %zmm3, %zmm0; @@ -716,6 +691,7 @@ _gcry_vaes_avx512_ctr_enc_amd64: vpaddb %zmm23, %zmm3, %zmm3; .Lctr_enc_blk16_rounds: + movb %r10b, 15(%rsi); /* AES rounds */ XOR4(%zmm30, %zmm0, %zmm1, %zmm2, %zmm3); VAESENC4(%zmm4, %zmm0, %zmm1, %zmm2, %zmm3); @@ -767,27 +743,23 @@ _gcry_vaes_avx512_ctr_enc_amd64: jmp .Lctr_enc_tail; - .align 16 - .Lctr_enc_blk16_handle_only_ctr_carry: - handle_ctr_128bit_add(16); - jmp .Lctr_enc_blk16_byte_bige_add; - .align 16 .Lctr_enc_blk16_handle_carry: - jz .Lctr_enc_blk16_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + leal 1(%r11d), %r11d; + movb %r11b, 14(%rsi); + jz .Lctr_enc_blk16_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ vbroadcasti32x4 .Lbswap128_mask rRIP, %zmm15; vpmovzxbq .Lcounter0_1_2_3_lo_bq rRIP, %zmm10; - vpmovzxbq .Lcounter1_1_1_1_hi_bq rRIP, %zmm13; vpshufb %zmm15, %zmm3, %zmm3; /* be => le */ vpmovzxbq .Lcounter4_4_4_4_lo_bq rRIP, %zmm11; vpmovzxbq .Lcounter8_8_8_8_lo_bq rRIP, %zmm12; - handle_ctr_128bit_add(16); - add_le128(%zmm0, %zmm3, %zmm10, %zmm13); /* +0:+1:+2:+3 */ - add_le128(%zmm1, %zmm0, %zmm11, %zmm13); /* +4:+5:+6:+7 */ - add_le128(%zmm2, %zmm0, %zmm12, %zmm13); /* +8:... */ + vpaddw %zmm10, %zmm3, %zmm0; /* +0:+1:+2:+3 */ + vpaddw %zmm11, %zmm0, %zmm1; /* +4:+5:+6:+7 */ + vpaddw %zmm12, %zmm0, %zmm2; /* +8:... */ vpshufb %zmm15, %zmm0, %zmm0; /* le => be */ - add_le128(%zmm3, %zmm1, %zmm12, %zmm13); /* +12:... */ + vpaddw %zmm12, %zmm1, %zmm3; /* +12:... */ vpshufb %zmm15, %zmm1, %zmm1; /* le => be */ vpshufb %zmm15, %zmm2, %zmm2; /* le => be */ vpshufb %zmm15, %zmm3, %zmm3; /* le => be */ @@ -806,7 +778,6 @@ _gcry_vaes_avx512_ctr_enc_amd64: vpxord %ymm23, %ymm23, %ymm23; vpxord %ymm30, %ymm30, %ymm30; vpxord %ymm31, %ymm31, %ymm31; - kxorq %k1, %k1, %k1; vzeroall; .align 16 @@ -2462,8 +2433,6 @@ _gcry_vaes_avx512_consts: .byte 16, 0, 16, 0, 16, 0, 16, 0 .Lcounter32_32_32_32_lo_bq: .byte 32, 0, 32, 0, 32, 0, 32, 0 -.Lcounter1_1_1_1_hi_bq: - .byte 0, 1, 0, 1, 0, 1, 0, 1 ELF(.size _gcry_vaes_avx512_consts,.-_gcry_vaes_avx512_consts) diff --git a/cipher/rijndael-vp-simd128.h b/cipher/rijndael-vp-simd128.h index 805faa6f..84dab316 100644 --- a/cipher/rijndael-vp-simd128.h +++ b/cipher/rijndael-vp-simd128.h @@ -1732,8 +1732,7 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr, M128I_BYTE(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); static const __m128i_const bigendian_add = M128I_BYTE(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); - static const __m128i_const carry_add = M128I_U64(1, 1); - static const __m128i_const nocarry_add = M128I_U64(1, 0); + static const __m128i_const littleendian_add = M128I_U64(1, 0); u64 ctrlow = buf_get_be64(ctr + 8); struct vp_aes_config_s config; @@ -1758,32 +1757,20 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr, /* detect if 8-bit carry handling is needed */ if (UNLIKELY(((ctrlow += 4) & 0xff) <= 3)) { - static const __m128i_const *adders[5][4] = - { - { &nocarry_add, &nocarry_add, &nocarry_add, &carry_add }, - { &nocarry_add, &nocarry_add, &carry_add, &nocarry_add }, - { &nocarry_add, &carry_add, &nocarry_add, &nocarry_add }, - { &carry_add, &nocarry_add, &nocarry_add, &nocarry_add }, - { &nocarry_add, &nocarry_add, &nocarry_add, &nocarry_add } - }; - unsigned int idx = ctrlow <= 3 ? ctrlow : 4; - pshufb128(xmm6, xmm7); - - paddq128_amemld(adders[idx][0], xmm7); + paddd128_amemld(&littleendian_add, xmm7); movdqa128(xmm7, xmm2); pshufb128(xmm6, xmm2); insert256_hi128(xmm2, ymm0); - paddq128_amemld(adders[idx][1], xmm7); + paddd128_amemld(&littleendian_add, xmm7); movdqa128(xmm7, xmm2); pshufb128(xmm6, xmm2); movdqa128_256(xmm2, ymm1); - paddq128_amemld(adders[idx][2], xmm7); + paddd128_amemld(&littleendian_add, xmm7); movdqa128(xmm7, xmm2); pshufb128(xmm6, xmm2); insert256_hi128(xmm2, ymm1); - paddq128_amemld(adders[idx][3], xmm7); - + paddd128_amemld(&littleendian_add, xmm7); pshufb128(xmm6, xmm7); } else @@ -1822,30 +1809,10 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr, if (UNLIKELY(((ctrlow += 2) & 0xff) <= 1)) { pshufb128(xmm6, xmm7); - - /* detect if 64-bit carry handling is needed */ - if (UNLIKELY(ctrlow == 1)) - { - paddq128_amemld(&carry_add, xmm7); - movdqa128(xmm7, xmm1); - pshufb128(xmm6, xmm1); - paddq128_amemld(&nocarry_add, xmm7); - } - else if (UNLIKELY(ctrlow == 0)) - { - paddq128_amemld(&nocarry_add, xmm7); - movdqa128(xmm7, xmm1); - pshufb128(xmm6, xmm1); - paddq128_amemld(&carry_add, xmm7); - } - else - { - paddq128_amemld(&nocarry_add, xmm7); - movdqa128(xmm7, xmm1); - pshufb128(xmm6, xmm1); - paddq128_amemld(&nocarry_add, xmm7); - } - + paddd128_amemld(&littleendian_add, xmm7); + movdqa128(xmm7, xmm1); + paddq128_amemld(&littleendian_add, xmm7); + pshufb128(xmm6, xmm1); pshufb128(xmm6, xmm7); } else @@ -1877,10 +1844,7 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr, if (UNLIKELY((++ctrlow & 0xff) == 0)) { pshufb128(xmm6, xmm7); - - /* detect if 64-bit carry handling is needed */ - paddq128_amemld(UNLIKELY(ctrlow == 0) ? &carry_add : &nocarry_add, xmm7); - + paddd128_amemld(&littleendian_add, xmm7); pshufb128(xmm6, xmm7); } else diff --git a/cipher/rijndael.c b/cipher/rijndael.c index 645c0e2f..cf7f4c23 100644 --- a/cipher/rijndael.c +++ b/cipher/rijndael.c @@ -691,7 +691,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_ctr_enc; bulk_ops->ocb_crypt = _gcry_aes_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_ocb_auth; bulk_ops->xts_crypt = _gcry_aes_xts_crypt; @@ -719,7 +719,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_aesni_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_aesni_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_aesni_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_aesni_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_aesni_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_aesni_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_aesni_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_aesni_ocb_auth; @@ -737,7 +737,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, /* Setup VAES bulk encryption routines. */ bulk_ops->cfb_dec = _gcry_aes_vaes_cfb_dec; bulk_ops->cbc_dec = _gcry_aes_vaes_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_vaes_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_vaes_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_vaes_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_vaes_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_vaes_ocb_auth; @@ -752,7 +752,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, /* Setup VAES bulk encryption routines. */ bulk_ops->cfb_dec = _gcry_aes_vaes_cfb_dec; bulk_ops->cbc_dec = _gcry_aes_vaes_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_vaes_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_vaes_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_vaes_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_vaes_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_vaes_ocb_auth; @@ -788,7 +788,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_ssse3_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_ssse3_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_ssse3_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_ssse3_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_ssse3_ctr_enc; bulk_ops->ocb_crypt = _gcry_aes_ssse3_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_ssse3_ocb_auth; } @@ -808,7 +808,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_armv8_ce_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_armv8_ce_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_armv8_ce_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_armv8_ce_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_armv8_ce_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_armv8_ce_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_armv8_ce_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_armv8_ce_ocb_auth; @@ -831,7 +831,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_vp_aarch64_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_vp_aarch64_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_vp_aarch64_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_vp_aarch64_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_vp_aarch64_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_vp_aarch64_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_vp_aarch64_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_vp_aarch64_ocb_auth; @@ -858,7 +858,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_riscv_zvkned_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_riscv_zvkned_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_riscv_zvkned_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_riscv_zvkned_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_riscv_zvkned_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_riscv_zvkned_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_riscv_zvkned_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_riscv_zvkned_ocb_auth; @@ -884,7 +884,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_vp_riscv_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_vp_riscv_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_vp_riscv_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_vp_riscv_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_vp_riscv_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_vp_riscv_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_vp_riscv_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_vp_riscv_ocb_auth; @@ -908,7 +908,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_ppc9le_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_ppc9le_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_ppc9le_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_ppc9le_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_ppc9le_ctr_enc; bulk_ops->ocb_crypt = _gcry_aes_ppc9le_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_ppc9le_ocb_auth; bulk_ops->xts_crypt = _gcry_aes_ppc9le_xts_crypt; @@ -939,7 +939,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_ppc8_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_ppc8_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_ppc8_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_ppc8_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_ppc8_ctr_enc; bulk_ops->ocb_crypt = _gcry_aes_ppc8_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_ppc8_ocb_auth; bulk_ops->xts_crypt = _gcry_aes_ppc8_xts_crypt; @@ -1348,7 +1348,7 @@ _gcry_aes_ctr_enc (void *context, unsigned char *ctr, outbuf += BLOCKSIZE; inbuf += BLOCKSIZE; /* Increment the counter. */ - cipher_block_add(ctr, 1, BLOCKSIZE); + cipher_block_add_be16(ctr, 1, BLOCKSIZE); } wipememory(&tmp, sizeof(tmp)); diff --git a/tests/basic.c b/tests/basic.c index 1f4273ea..9ec1b596 100644 --- a/tests/basic.c +++ b/tests/basic.c @@ -12666,6 +12666,10 @@ cipher_cbc_bulk_test (int cipher_algo) return -1; } + if (verbose) + fprintf (stderr, " checking CBC bulk encryption for %s [%i]\n", + cipher, cipher_algo); + memsize = (blocksize * 2) + (blocksize * nblocks * 3) + 16 + (blocksize + 1); mem = xcalloc (1, memsize); @@ -12906,6 +12910,10 @@ cipher_cfb_bulk_test (int cipher_algo) return -1; } + if (verbose) + fprintf (stderr, " checking CFB bulk encryption for %s [%i]\n", + cipher, cipher_algo); + memsize = (blocksize * 2) + (blocksize * nblocks * 3) + 16 + (blocksize + 1); mem = xcalloc (1, memsize); @@ -13135,6 +13143,10 @@ cipher_ctr_bulk_test (int cipher_algo) return -1; } + if (verbose) + fprintf (stderr, " checking CTR bulk encryption for %s [%i]\n", + cipher, cipher_algo); + memsize = (blocksize * 2) + (blocksize * nblocks * 4) + 16 + (blocksize + 1); mem = xcalloc (1, memsize); @@ -13440,6 +13452,132 @@ cipher_ctr_bulk_test (int cipher_algo) } +/* Regression test for the 16-bit-overflow split in bulk CTR encryption: a + single call spanning >= 0x10000 blocks must not reuse keystream nor drop the + counter carry. Reference keystream is built from ECB with a full-width + byte-wise counter, correct independent of the split logic. */ +static int +cipher_ctr16_overflow_test (int cipher_algo) +{ + static const unsigned int low16_start[] = + { 0, 1, 0x7fff, 0x8000, 0xfffe, 0xffff }; + const size_t nblocks = 2 * 0x10000 + 5; + int blocksize; + const char *cipher; + gcry_cipher_hd_t hd_ecb = NULL; + gcry_cipher_hd_t hd_ctr = NULL; + unsigned char *plaintext = NULL; + unsigned char *reftext = NULL; + unsigned char *enctext = NULL; + unsigned char iv[16]; + unsigned char getctr[17]; + size_t buflen, i, j, tc; + unsigned int keylen; + static const unsigned char key[32] = { + 0x06,0x9A,0x00,0x7F,0xC7,0x6A,0x45,0x9F, + 0x98,0xBA,0xF9,0x17,0xFE,0xDF,0x95,0x21, + 0x06,0x9A,0x00,0x7F,0xC7,0x6A,0x45,0x9F, + 0x98,0xBA,0xF9,0x17,0xFE,0xDF,0x95,0x21 + }; + + if (gcry_cipher_test_algo (cipher_algo)) + return 0; + blocksize = gcry_cipher_get_algo_blklen (cipher_algo); + if (blocksize < 8 || blocksize > 16) + return 0; + cipher = gcry_cipher_algo_name (cipher_algo); + keylen = gcry_cipher_get_algo_keylen (cipher_algo); + if (keylen > sizeof(key)) + return 0; + + if (verbose) + fprintf (stderr, " checking CTR 16-bit overflow for %s [%i]\n", + cipher, cipher_algo); + + buflen = nblocks * (size_t)blocksize; + plaintext = xmalloc (buflen); + reftext = xmalloc (buflen); + enctext = xmalloc (buflen); + + if (gcry_cipher_open (&hd_ecb, cipher_algo, GCRY_CIPHER_MODE_ECB, 0) + || gcry_cipher_open (&hd_ctr, cipher_algo, GCRY_CIPHER_MODE_CTR, 0) + || gcry_cipher_setkey (hd_ecb, key, keylen) + || gcry_cipher_setkey (hd_ctr, key, keylen)) + { + fail ("%s-CTR-%d ctr16 test failed (setup)", cipher, blocksize * 8); + goto leave; + } + + for (i = 0; i < buflen; i++) + plaintext[i] = (unsigned char)i; + + for (tc = 0; tc < DIM (low16_start); tc++) + { + unsigned int low16 = low16_start[tc]; + + /* Fixed non-zero upper bytes exercise carry past the low 16 bits. */ + memset (iv, 0x5a, blocksize); + iv[blocksize - 3] = 0x40; + iv[blocksize - 2] = (low16 >> 8) & 0xff; + iv[blocksize - 1] = low16 & 0xff; + + if (gcry_cipher_setctr (hd_ctr, iv, blocksize)) + { + fail ("%s-CTR-%d ctr16 test failed (setctr, low16=0x%04x)", + cipher, blocksize * 8, low16); + goto leave; + } + + /* iv ends holding start + nblocks, the reference final counter. */ + for (i = 0; i < buflen; i += blocksize) + { + memcpy (&reftext[i], iv, blocksize); + for (j = blocksize; j > 0; j--) + if (++iv[j - 1]) + break; + } + if (gcry_cipher_encrypt (hd_ecb, reftext, buflen, NULL, 0)) + { + fail ("%s-CTR-%d ctr16 test failed (ecb, low16=0x%04x)", + cipher, blocksize * 8, low16); + goto leave; + } + for (i = 0; i < buflen; i++) + reftext[i] ^= plaintext[i]; + + if (gcry_cipher_encrypt (hd_ctr, enctext, buflen, plaintext, buflen)) + { + fail ("%s-CTR-%d ctr16 test failed (ctr, low16=0x%04x)", + cipher, blocksize * 8, low16); + goto leave; + } + if (memcmp (enctext, reftext, buflen)) + { + fail ("%s-CTR-%d ctr16 test failed (keystream, low16=0x%04x)", + cipher, blocksize * 8, low16); + goto leave; + } + + if (gcry_cipher_ctl (hd_ctr, PRIV_CIPHERCTL_GET_COUNTER, getctr, + blocksize + 1) + || getctr[0] != blocksize + || memcmp (getctr + 1, iv, blocksize)) + { + fail ("%s-CTR-%d ctr16 test failed (counter, low16=0x%04x)", + cipher, blocksize * 8, low16); + goto leave; + } + } + +leave: + gcry_cipher_close (hd_ecb); + gcry_cipher_close (hd_ctr); + xfree (plaintext); + xfree (reftext); + xfree (enctext); + return 0; +} + static void check_ciphers (void) @@ -13603,6 +13741,20 @@ check_ciphers (void) } +static void +check_ctr16_overflow (void) +{ + /* One 128-bit and one 64-bit block cipher cover both blocksize code paths + in _gcry_cipher_ctr_encrypt_ctx. */ + if (verbose) + fprintf (stderr, " Starting CTR 16-bit overflow checks.\n"); + cipher_ctr16_overflow_test (GCRY_CIPHER_AES); + cipher_ctr16_overflow_test (GCRY_CIPHER_BLOWFISH); + if (verbose) + fprintf (stderr, " Completed CTR 16-bit overflow checks.\n"); +} + + static void check_cipher_modes(void) { @@ -13613,6 +13765,8 @@ check_cipher_modes(void) check_aes128_cbc_cts_cipher (); check_cbc_mac_cipher (); check_ctr_cipher (); + check_ctr16_overflow (); + check_cfb_cipher (); check_ofb_cipher (); check_ccm_cipher (); -- 2.53.0