RE: [PATCH] aarch64: Use SVE SPLICE for scalable recurrences

> -----Original Message-----
> From: [email protected] <[email protected]>
> Sent: 20 August 2026 13:34
> To: [email protected]
> Cc: Tamar Christina <[email protected]>; Kyrylo Tkachov
> <[email protected]>
> Subject: [PATCH] aarch64: Use SVE SPLICE for scalable recurrences
> 
> From: Kyrylo Tkachov <[email protected]>
> 
> The vectorizer represents a first-order recurrence as a scalable
> permutation that takes the final D elements from one vector and the
> remaining elements from the next vector.  The AArch64 permutation
> expander did not recognize this variable-length shape.
> 
> Recognize the exact two-input permutation and build a predicate for D
> elements.  Reverse that predicate and use the SVE SPLICE pattern.  Reject
> fixed-length vectors, rotations, and distances that cannot be represented
> at the minimum vector length.
> 
> The compile test checks seven direct SPLICE sequences across byte,
> halfword, word, and doubleword modes.  A 67-element runtime test covers
> distances 1 and 2 with two seed values, and partial final vectors.

Nice!

> 
> On an example testcase:
> 
>         typedef __UINT32_TYPE__ uint32_t;
> 
>         void
>         recur (const uint32_t *__restrict a, uint32_t *__restrict b,
> 	const uint32_t *__restrict init)
>         {
>           uint32_t prev = *init;
>           for (int i = 0; i < 67; ++i)
>             {
>               b[i] = a[i] - prev;
>               prev = a[i];
>             }
>         }
> 
> With -O3 -march=armv8.2-a+sve
> We emitted before:
> 
>         ldr     w4, [x2]
>         mov     w3, 0
>         .p2align 5,,15
>         .L2:
>         mov     w2, w4
>         ldr     w4, [x0, x3]
>         sub     w2, w4, w2
>         str     w2, [x1, x3]
>         add     x3, x3, 4
>         cmp     x3, 268
>         bne     .L2
> 
> And after this patch:
>         mov     w3, 0
>         cntw    x5
>         ptrue   p5.b, all
>         mov     w4, 67
>         ptrue   p6.b, vl1
>         mov     p7.b, p5.b
>         ld1rw   z31.s, p5/z, [x2]
>         rev     p6.s, p6.s
>         .p2align 5,,15
>         .L2:
>         mov     z30.d, z31.d
>         ld1w    z31.s, p7/z, [x0, x3, lsl 2]
>         splice  z30.s, p6, z30.s, z31.s
>         sub     z30.s, z31.s, z30.s
>         st1w    z30.s, p7, [x1, x3, lsl 2]
>         add     x3, x3, x5
>         whilelo p7.s, w3, w4
>         b.any   .L2
> 
> This also vectorizes some loops in OpenEXR 4.0.0.
> 
> Upstream PXR24, RLE, ZIP, and ZIPS tests pass, and the baseline and
> patched encoders produced identical files.  A paired exrmetrics run on an
> NVIDIA Vera system measured the following write-time improvements.
> 
>                                 Flowers.exr   multipart.0001.exr
>         PXR24, float                   2.28%                 4.58%
>         PXR24, original                3.22%                 8.07%
>         RLE                           13.52%                23.27%
>         ZIP                            3.67%                 9.59%
>         ZIPS                           3.15%                 5.62%
> 
> Bootstrapped and tested on aarch64-none-linux-gnu.
> Ok for trunk?
> Thanks,
> Kyrill
> 
> gcc/ChangeLog:
> 
> 	* config/aarch64/aarch64.cc (aarch64_evpc_splice): New function.
> 	(aarch64_expand_vec_perm_const_1): Use it.
> 
> gcc/testsuite/ChangeLog:
> 
> 	* gcc.target/aarch64/sve/perm_splice_1.c: New test.
> 	* gcc.target/aarch64/sve/perm_splice_1-run.c: Likewise.
> 
> Signed-off-by: Kyrylo Tkachov <[email protected]>
> ---
>  gcc/config/aarch64/aarch64.cc                 | 42 +++++++++
>  .../aarch64/sve/perm_splice_1-run.c           | 94 +++++++++++++++++++
>  .../gcc.target/aarch64/sve/perm_splice_1.c    | 63 +++++++++++++
>  3 files changed, 199 insertions(+)
>  create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/perm_splice_1-
> run.c
>  create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/perm_splice_1.c
> 
> diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
> index fdffb13ad22..2e440ce916d 100644
> --- a/gcc/config/aarch64/aarch64.cc
> +++ b/gcc/config/aarch64/aarch64.cc
> @@ -28261,6 +28261,46 @@ aarch64_evpc_ext (struct expand_vec_perm_d
> *d)
>    return true;
>  }
> 
> +/* Return true if D describes a scalable-vector permutation that takes the
> +   last DIST elements from the first input and the remaining elements from
> +   the second input.  */
> +
> +static bool
> +aarch64_evpc_splice (struct expand_vec_perm_d *d)
> +{
> +  poly_int64 nelt = d->perm.length ();
> +  HOST_WIDE_INT dist;
> +
> +  if (d->vec_flags != VEC_SVE_DATA
> +      || d->one_vector_p
> +      || nelt.is_constant ()
> +      || !(nelt - d->perm[0]).is_constant (&dist)
> +      || !IN_RANGE (dist, 1, INT_MAX)
> +      || !d->perm.series_p (0, 1, nelt - dist, 1))
> +    return false;
> +
> +  machine_mode pred_mode = aarch64_sve_pred_mode (d->vmode);
> +  if (aarch64_svpattern_for_vl (pred_mode, dist)
> +      == AARCH64_NUM_SVPATTERNS)
> +    return false;
> +
> +  if (d->testing_p)
> +    return true;
> +
> +  rtx_vector_builder builder (pred_mode, dist, 2);
> +  for (HOST_WIDE_INT i = 0; i < dist; ++i)
> +    builder.quick_push (CONST1_RTX (BImode));
> +  for (HOST_WIDE_INT i = 0; i < dist; ++i)
> +    builder.quick_push (CONST0_RTX (BImode));
> +
> +  rtx head = force_reg (pred_mode, builder.build ());

Since you already have the pattern above, why not save the result of the
aarch64_svpattern_for...

and then replace this builder code with

  rtx head = aarch64_ptrue_reg (pred_mode, pattern);

which is slightly cleaner?

LGTM with that change.

Thanks,
Tamar

> +  rtx pred = gen_reg_rtx (pred_mode);
> +  emit_insn (gen_aarch64_sve_rev (pred_mode, pred, head));
> +  emit_insn (gen_aarch64_sve_splice (d->vmode, d->target, pred,
> +				     d->op0, d->op1));
> +  return true;
> +}
> +
>  /* Recognize patterns for the REV{64,32,16} insns, which reverse elements
>     within each 64-bit, 32-bit or 16-bit granule.  */
> 
> @@ -28786,6 +28826,8 @@ aarch64_expand_vec_perm_const_1 (struct
> expand_vec_perm_d *d)
>  	    return true;
>  	  else if (aarch64_evpc_rev_global (d))
>  	    return true;
> +	  else if (aarch64_evpc_splice (d))
> +	    return true;
>  	  else if (aarch64_evpc_ext (d))
>  	    return true;
>  	  else if (aarch64_evpc_dup (d))
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_splice_1-run.c
> b/gcc/testsuite/gcc.target/aarch64/sve/perm_splice_1-run.c
> new file mode 100644
> index 00000000000..b2c13120c53
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_splice_1-run.c
> @@ -0,0 +1,94 @@
> +/* { dg-do run } */
> +/* { dg-require-effective-target aarch64_sve_hw } */
> +/* { dg-options "-O3 -march=armv8.2-a+sve -mautovec-preference=sve-
> only" } */
> +/* { dg-additional-options "-msve-vector-bits=scalable" } */
> +/* { dg-additional-options "-fvect-cost-model=unlimited" } */
> +
> +typedef __UINT8_TYPE__ uint8_t;
> +typedef __UINT16_TYPE__ uint16_t;
> +typedef __UINT32_TYPE__ uint32_t;
> +typedef __UINT64_TYPE__ uint64_t;
> +
> +#define N 67
> +
> +#define DEFINE_RECUR(TYPE, SUFFIX) \
> +  __attribute__((noipa)) \
> +  static void \
> +  recur1_##SUFFIX (TYPE *__restrict a, TYPE *__restrict b, \
> +		   const TYPE *__restrict init) \
> +  { \
> +    TYPE prev = init[0]; \
> +    for (int i = 0; i < N; ++i) \
> +      { \
> +	b[i] = a[i] - prev; \
> +	prev = a[i]; \
> +      } \
> +  } \
> +  \
> +  __attribute__((noipa)) \
> +  static void \
> +  recur2_##SUFFIX (TYPE *__restrict a, TYPE *__restrict b, \
> +		   const TYPE *__restrict init) \
> +  { \
> +    TYPE prev0 = init[0]; \
> +    TYPE prev1 = init[0]; \
> +    for (int i = 0; i + 1 < N; i += 2) \
> +      { \
> +	b[i] = a[i] - prev0; \
> +	prev0 = a[i]; \
> +	b[i + 1] = a[i + 1] - prev1; \
> +	prev1 = a[i + 1]; \
> +      } \
> +    b[N - 1] = 0; \
> +  }
> +
> +DEFINE_RECUR (uint8_t, u8)
> +DEFINE_RECUR (uint16_t, u16)
> +DEFINE_RECUR (uint32_t, u32)
> +DEFINE_RECUR (uint64_t, u64)
> +
> +#define CHECK_RECUR(TYPE, SUFFIX) \
> +  do \
> +    { \
> +      TYPE a[N]; \
> +      TYPE b[N]; \
> +      const TYPE init[2] = { (TYPE) 19, (TYPE) 43 }; \
> +      \
> +      for (int i = 0; i < N; ++i) \
> +	a[i] = (TYPE) ((unsigned int) i * (unsigned int) i \
> +			  + 5U * (unsigned int) i + 11U); \
> +      \
> +      for (int seed = 0; seed < 2; ++seed) \
> +	{ \
> +	  recur1_##SUFFIX (a, b, &init[seed]); \
> +	  if (b[0] != (TYPE) (a[0] - init[seed])) \
> +	    __builtin_abort (); \
> +	  _Pragma ("GCC novector") \
> +	  for (int i = 1; i < N; ++i) \
> +	    if (b[i] != (TYPE) (a[i] - a[i - 1])) \
> +	      __builtin_abort (); \
> +	  \
> +	  recur2_##SUFFIX (a, b, &init[seed]); \
> +	  if (b[0] != (TYPE) (a[0] - init[seed]) \
> +	      || b[1] != (TYPE) (a[1] - init[seed])) \
> +	    __builtin_abort (); \
> +	  _Pragma ("GCC novector") \
> +	  for (int i = 2; i < N - 1; ++i) \
> +	    if (b[i] != (TYPE) (a[i] - a[i - 2])) \
> +	      __builtin_abort (); \
> +	  if (b[N - 1] != 0) \
> +	    __builtin_abort (); \
> +	} \
> +    } \
> +  while (0)
> +
> +int
> +main (void)
> +{
> +  CHECK_RECUR (uint8_t, u8);
> +  CHECK_RECUR (uint16_t, u16);
> +  CHECK_RECUR (uint32_t, u32);
> +  CHECK_RECUR (uint64_t, u64);
> +
> +  return 0;
> +}
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/perm_splice_1.c
> b/gcc/testsuite/gcc.target/aarch64/sve/perm_splice_1.c
> new file mode 100644
> index 00000000000..c2a9879008b
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/perm_splice_1.c
> @@ -0,0 +1,63 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O3 -march=armv8.2-a+sve -mautovec-preference=sve-
> only" } */
> +/* { dg-additional-options "-msve-vector-bits=scalable" } */
> +/* { dg-additional-options "-fvect-cost-model=unlimited -fdump-tree-vect-
> details" } */
> +
> +typedef __UINT8_TYPE__ uint8_t;
> +typedef __UINT16_TYPE__ uint16_t;
> +typedef __UINT32_TYPE__ uint32_t;
> +typedef __UINT64_TYPE__ uint64_t;
> +
> +#define DEF_RECUR1(TYPE, SUFFIX)                                     \
> +  __attribute__((noipa))                                             \
> +  void                                                               \
> +  recur1_##SUFFIX (TYPE *__restrict a, TYPE *__restrict b,           \
> +		   const TYPE *__restrict init)                         \
> +  {                                                                  \
> +    TYPE prev = *init;                                               \
> +    for (int i = 0; i < 67; ++i)                                    \
> +      {                                                              \
> +	b[i] = a[i] - prev;                                           \
> +	prev = a[i];                                                  \
> +      }                                                              \
> +  }
> +
> +#define DEF_RECUR2(TYPE, SUFFIX)                                     \
> +  __attribute__((noipa))                                             \
> +  void                                                               \
> +  recur2_##SUFFIX (TYPE *__restrict a, TYPE *__restrict b,           \
> +		   const TYPE *__restrict init)                         \
> +  {                                                                  \
> +    TYPE prev0 = *init;                                              \
> +    TYPE prev1 = *init;                                              \
> +    for (int i = 0; i < 66; i += 2)                                 \
> +      {                                                              \
> +	b[i] = a[i] - prev0;                                          \
> +	prev0 = a[i];                                                 \
> +	b[i + 1] = a[i + 1] - prev1;                                 \
> +	prev1 = a[i + 1];                                             \
> +      }                                                              \
> +  }
> +
> +DEF_RECUR1 (uint8_t, u8)
> +DEF_RECUR1 (uint16_t, u16)
> +DEF_RECUR1 (uint32_t, u32)
> +DEF_RECUR1 (uint64_t, u64)
> +
> +DEF_RECUR2 (uint8_t, u8)
> +DEF_RECUR2 (uint16_t, u16)
> +DEF_RECUR2 (uint32_t, u32)
> +
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.b, vl1} 4 } } */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.b, vl2} 1 } } */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.h, vl2} 1 } } */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s, vl2} 1 } } */
> +/* { dg-final { scan-assembler-times {\trev\tp[0-9]+\.b, p[0-9]+\.b} 2 } } */
> +/* { dg-final { scan-assembler-times {\trev\tp[0-9]+\.h, p[0-9]+\.h} 2 } } */
> +/* { dg-final { scan-assembler-times {\trev\tp[0-9]+\.s, p[0-9]+\.s} 2 } } */
> +/* { dg-final { scan-assembler-times {\trev\tp[0-9]+\.d, p[0-9]+\.d} 1 } } */
> +/* { dg-final { scan-assembler-times {\tsplice\tz[0-9]+\.b, p[0-9]+, z[0-9]+\.b,
> z[0-9]+\.b} 2 } } */
> +/* { dg-final { scan-assembler-times {\tsplice\tz[0-9]+\.h, p[0-9]+, z[0-9]+\.h,
> z[0-9]+\.h} 2 } } */
> +/* { dg-final { scan-assembler-times {\tsplice\tz[0-9]+\.s, p[0-9]+, z[0-9]+\.s,
> z[0-9]+\.s} 2 } } */
> +/* { dg-final { scan-assembler-times {\tsplice\tz[0-9]+\.d, p[0-9]+, z[0-9]+\.d,
> z[0-9]+\.d} 1 } } */
> +/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 7 "vect"
> } } */
> --
> 2.50.1 (Apple Git-155)