[gcc r17-2754] aarch64: Add Support for F16F32DOT
Richard Ball via Gcc-cvs <[email protected]>
| Newsgroups | gmane.comp.gcc.cvs |
|---|---|
| Message-ID | <[email protected]> |
https://gcc.gnu.org/g:4811e4c9836e52a88d8d51423a8fb6fec834b7c5 commit r17-2754-g4811e4c9836e52a88d8d51423a8fb6fec834b7c5 Author: Richard Ball <[email protected]> Date: Tue Jul 28 15:34:22 2026 +0100 aarch64: Add Support for F16F32DOT This patch adds the following advsimd intrinsics: vdot_f32_f16 vdotq_f32_f16 vdot_lane_f32_f16 vdot_laneq_f32_f16 vdotq_lane_f32_f16 vdotq_laneq_f32_f16 gcc/ChangeLog: * config/aarch64/aarch64-c.cc (aarch64_update_cpp_builtins): New target flag. * config/aarch64/aarch64-simd-builtins.def: New functions. * config/aarch64/aarch64-simd.md (sdot_prod<VDQSF:vsi2qi><vczle><vczbe>): New pattern. (sdot_lane<VF:isquadop><VDQSF:vsi2qi><vczle><vczbe>): Likewise. * config/aarch64/aarch64.h (TARGET_F16F32DOT): New target flag. * config/aarch64/arm_neon.h (target): Add f16f32dot section. (__attribute__): New builtin function calls. (vdot_f32_f16): Likewise. (vdotq_f32_f16): Likewise. (vdot_lane_f32_f16): Likewise. (vdot_laneq_f32_f16): Likewise. (vdotq_lane_f32_f16): Likewise. (vdotq_laneq_f32_f16): Likewise. * config/aarch64/iterators.md (q): New options for isquadop. * doc/invoke.texi: Add f16f32dot. gcc/testsuite/ChangeLog: * lib/target-supports.exp: New check_effective_target. * gcc.target/aarch64/advsimd-intrinsics/fdot-1.c: New test. * gcc.target/aarch64/advsimd-intrinsics/fdot-2.c: New test. * gcc.target/aarch64/advsimd-intrinsics/fdot-3.c: New test. Diff: --- gcc/config/aarch64/aarch64-c.cc | 1 + gcc/config/aarch64/aarch64-simd-builtins.def | 6 ++ gcc/config/aarch64/aarch64-simd.md | 30 ++++++++++ gcc/config/aarch64/aarch64.h | 2 + gcc/config/aarch64/arm_neon.h | 50 ++++++++++++++++ gcc/config/aarch64/iterators.md | 12 +++- gcc/doc/invoke.texi | 2 + .../gcc.target/aarch64/advsimd-intrinsics/fdot-1.c | 68 +++++++++++++++++++++ .../gcc.target/aarch64/advsimd-intrinsics/fdot-2.c | 70 ++++++++++++++++++++++ .../gcc.target/aarch64/advsimd-intrinsics/fdot-3.c | 32 ++++++++++ gcc/testsuite/lib/target-supports.exp | 41 +++++++++++++ 11 files changed, 311 insertions(+), 3 deletions(-) diff --git a/gcc/config/aarch64/aarch64-c.cc b/gcc/config/aarch64/aarch64-c.cc index c4ddcc6d717d..eb47a57ede44 100644 --- a/gcc/config/aarch64/aarch64-c.cc +++ b/gcc/config/aarch64/aarch64-c.cc @@ -320,6 +320,7 @@ aarch64_update_cpp_builtins (cpp_reader *pfile) aarch64_def_or_undef (AARCH64_HAVE_ISA (SSVE_FEXPA), "__ARM_FEATURE_SSVE_FEXPA", pfile); builtin_define ("__ARM_PREFETCH_RANGE"); + aarch64_def_or_undef (TARGET_F16F32DOT, "__ARM_FEATURE_F16F32DOT", pfile); // Function multi-versioning defines aarch64_def_or_undef (targetm.has_ifunc_p (), diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def b/gcc/config/aarch64/aarch64-simd-builtins.def index 02938d48b4eb..9c61cc05a050 100644 --- a/gcc/config/aarch64/aarch64-simd-builtins.def +++ b/gcc/config/aarch64/aarch64-simd-builtins.def @@ -921,6 +921,12 @@ VAR2 (QUADOP_LANE_PAIR, bfdot_lane, 0, QUIET, v2sf, v4sf) VAR2 (QUADOP_LANE_PAIR, bfdot_laneq, 0, QUIET, v2sf, v4sf) + /* Implemented by sdot_prod<mode>. */ + BUILTIN_VF (TERNOP, sdot_prod, 10, DEFAULT) + /* Implemented by sdot_lane{q}<dot_mode>. */ + BUILTIN_VF (QUADOP_LANE_PAIR, sdot_lane, 10, DEFAULT) + BUILTIN_VF (QUADOP_LANE_PAIR, sdot_laneq, 10, DEFAULT) + /* Implemented by aarch64_bfmmlaqv4sf */ VAR1 (TERNOP, bfmmlaq, 0, QUIET, v4sf) diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index ce51e24da36a..433f16052bf2 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -782,6 +782,36 @@ [(set_attr "type" "neon_dot<VS:q>")] ) +(define_insn "sdot_prod<VDQSF:vsi2qi><vczle><vczbe>" + [(set (match_operand:VDQSF 0 "register_operand" "=w") + (plus:VDQSF + (unspec:VDQSF + [(match_operand:<VSI2QI> 2 "register_operand" "w") + (match_operand:<VSI2QI> 3 "register_operand" "w")] + UNSPEC_FDOT) + (match_operand:VDQSF 1 "register_operand" "0")))] + "TARGET_F16F32DOT" + "fdot\t%0.<Vtype>, %2.<Vbfdottype>, %3.<Vbfdottype>" + [(set_attr "type" "neon_dot<q>")] +) + +(define_insn "sdot_lane<VF:isquadop><VDQSF:vsi2qi><vczle><vczbe>" + [(set (match_operand:VDQSF 0 "register_operand" "=w") + (plus:VDQSF + (unspec:VDQSF + [(match_operand:<VDQSF:VSI2QI> 2 "register_operand" "w") + (match_operand:VF 3 "register_operand" "w") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_FDOT) + (match_operand:VDQSF 1 "register_operand" "0")))] + "TARGET_F16F32DOT" +{ + operands[4] = aarch64_endian_lane_rtx (<VF:MODE>mode, INTVAL (operands[4])); + return "fdot\t%0.<VDQSF:Vtype>, %2.<VDQSF:Vbfdottype>, %3.2h[%4]"; +} + [(set_attr "type" "neon_dot<VDQSF:q>")] +) + (define_expand "copysign<mode>3" [(match_operand:VHSDF 0 "register_operand") (match_operand:VHSDF 1 "register_operand") diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h index 752f74386c6e..50577b136aca 100644 --- a/gcc/config/aarch64/aarch64.h +++ b/gcc/config/aarch64/aarch64.h @@ -435,6 +435,8 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE ATTRIBUTE_UNUSED #define TARGET_F8F16MM (AARCH64_HAVE_ISA (F8F16MM)) /* SVE_F16F32MM instructions, enabled through +sve-f16f32mm. */ #define TARGET_SVE_F16F32MM (AARCH64_HAVE_ISA (SVE_F16F32MM)) +/* F16F32DOT instructions enabled through +f16f32dot. */ +#define TARGET_F16F32DOT (AARCH64_HAVE_ISA (F16F32DOT)) /* Make sure this is always defined so we don't have to check for ifdefs but rather use normal ifs. */ diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h index 3e401dbe6a0f..101a6e521068 100644 --- a/gcc/config/aarch64/arm_neon.h +++ b/gcc/config/aarch64/arm_neon.h @@ -22404,6 +22404,56 @@ vst4q_lane_bf16 (bfloat16_t *__ptr, bfloat16x8x4_t __val, const int __lane) #pragma GCC pop_options +#pragma GCC push_options +#pragma GCC target ("+nothing+f16f32dot") + +__extension__ extern __inline float32x2_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vdot_f32_f16 (float32x2_t __r, float16x4_t __a, float16x4_t __b) +{ + return __builtin_aarch64_sdot_prodv4hf (__r, __a, __b); +} + +__extension__ extern __inline float32x4_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vdotq_f32_f16 (float32x4_t __r, float16x8_t __a, float16x8_t __b) +{ + return __builtin_aarch64_sdot_prodv8hf (__r, __a, __b); +} + +__extension__ extern __inline float32x2_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vdot_lane_f32_f16 (float32x2_t __r, float16x4_t __a, float16x4_t __b, + const int __index) +{ + return __builtin_aarch64_sdot_lanev4hf (__r, __a, __b, __index); +} + +__extension__ extern __inline float32x2_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vdot_laneq_f32_f16 (float32x2_t __r, float16x4_t __a, float16x8_t __b, + const int __index) +{ + return __builtin_aarch64_sdot_laneqv4hf (__r, __a, __b, __index); +} + +__extension__ extern __inline float32x4_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vdotq_lane_f32_f16 (float32x4_t __r, float16x8_t __a, float16x4_t __b, + const int __index) +{ + return __builtin_aarch64_sdot_lanev8hf (__r, __a, __b, __index); +} + +__extension__ extern __inline float32x4_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vdotq_laneq_f32_f16 (float32x4_t __r, float16x8_t __a, float16x8_t __b, + const int __index) +{ + return __builtin_aarch64_sdot_laneqv8hf (__r, __a, __b, __index); +} + +#pragma GCC pop_options /* AdvSIMD 8-bit Integer Matrix Multiply (I8MM) intrinsics. */ #pragma GCC push_options diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md index a8b976e4b71e..5dfb27449703 100644 --- a/gcc/config/aarch64/iterators.md +++ b/gcc/config/aarch64/iterators.md @@ -172,6 +172,9 @@ ;; BFmode vector modes. (define_mode_iterator VBF [V4BF V8BF]) +;; Fmode vector modes. +(define_mode_iterator VF [V4HF V8HF]) + ;; This mode iterator allows :P to be used for patterns that operate on ;; addresses in different modes. In LP64, only DI will match, while in ;; ILP32, either can match. @@ -2668,9 +2671,11 @@ (V4HF "v") (V8HF "v")]) (define_mode_attr vsi2qi [(V2SI "v8qi") (V4SI "v16qi") - (VNx4SI "vnx16qi") (VNx2DI "vnx8hi")]) + (VNx4SI "vnx16qi") (VNx2DI "vnx8hi") + (V2SF "v4hf") (V4SF "v8hf")]) (define_mode_attr VSI2QI [(V2SI "V8QI") (V4SI "V16QI") - (VNx4SI "VNx16QI") (VNx2DI "VNx8HI")]) + (VNx4SI "VNx16QI") (VNx2DI "VNx8HI") + (V2SF "V4HF") (V4SF "V8HF")]) ;; Register suffix for DOTPROD input types from the return type. @@ -2720,7 +2725,8 @@ (define_mode_attr f16quad [(V2SF "") (V4SF "q")]) -(define_mode_attr isquadop [(V8QI "") (V16QI "q") (V4BF "") (V8BF "q")]) +(define_mode_attr isquadop [(V8QI "") (V16QI "q") (V4BF "") (V8BF "q") + (V4HF "") (V8HF "q")]) (define_code_attr f16mac [(plus "a") (minus "s")]) diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi index 322d0a726b9c..3d4b3d2bba03 100644 --- a/gcc/doc/invoke.texi +++ b/gcc/doc/invoke.texi @@ -22109,6 +22109,8 @@ extension. @item ssve-fp8dot2 Enable the fp8 (8-bit floating point) to half-precision 2-way dot product extension in streaming mode. +@item f16f32dot +Enable the Half-precision to single-precision dot product extension. @item faminmax Enable the Floating Point Absolute Maximum/Minimum extension. @item lut diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c new file mode 100644 index 000000000000..4abbdf57ef69 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-1.c @@ -0,0 +1,68 @@ +/* { dg-do assemble { target { aarch64*-*-* } } } */ +/* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */ +/* { dg-add-options aarch64_f16f32dot_neon } */ +/* { dg-additional-options "-save-temps" } */ +/* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */ +/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */ + +#include <arm_neon.h> + +/* +**ufoo: +** fdot v0.2s, (v1.4h, v2.4h|v2.4h, v1.4h) +** ret +*/ +float32x2_t ufoo(float32x2_t r, float16x4_t x, float16x4_t y) +{ + return vdot_f32_f16 (r, x, y); +} + +/* +**ufooq: +** fdot v0.4s, (v1.8h, v2.8h|v2.8h, v1.8h) +** ret +*/ +float32x4_t ufooq(float32x4_t r, float16x8_t x, float16x8_t y) +{ + return vdotq_f32_f16 (r, x, y); +} + +/* +**ufoo_lane: +** fdot v0.2s, v1.4h, v2.2h\[0\] +** ret +*/ +float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y) +{ + return vdot_lane_f32_f16 (r, x, y, 0); +} + +/* +**ufoo_laneq: +** fdot v0.2s, v1.4h, v2.2h\[2\] +** ret +*/ +float32x2_t ufoo_laneq(float32x2_t r, float16x4_t x, float16x8_t y) +{ + return vdot_laneq_f32_f16 (r, x, y, 2); +} + +/* +**ufooq_lane: +** fdot v0.4s, v1.8h, v2.2h\[1\] +** ret +*/ +float32x4_t ufooq_lane(float32x4_t r, float16x8_t x, float16x4_t y) +{ + return vdotq_lane_f32_f16 (r, x, y, 1); +} + +/* +**ufooq_laneq: +** fdot v0.4s, v1.8h, v2.2h\[2\] +** ret +*/ +float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y) +{ + return vdotq_laneq_f32_f16 (r, x, y, 2); +} \ No newline at end of file diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c new file mode 100644 index 000000000000..b42bfa09c77c --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-2.c @@ -0,0 +1,70 @@ +/* { dg-do assemble { target { aarch64*-*-* } } } */ +/* { dg-require-effective-target stdint_types_mbig_endian } */ +/* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */ +/* { dg-add-options aarch64_f16f32dot_neon } */ +/* { dg-additional-options "-mbig-endian --save-temps" } */ +/* { dg-final { check-function-bodies "**" "" {-O[^0]} } } */ +/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */ + + +#include <arm_neon.h> + +/* +**ufoo: +** fdot v0.2s, (v1.4h, v2.4h|v2.4h, v1.4h) +** ret +*/ +float32x2_t ufoo(float32x2_t r, float16x4_t x, float16x4_t y) +{ + return vdot_f32_f16 (r, x, y); +} + +/* +**ufooq: +** fdot v0.4s, (v1.8h, v2.8h|v2.8h, v1.8h) +** ret +*/ +float32x4_t ufooq(float32x4_t r, float16x8_t x, float16x8_t y) +{ + return vdotq_f32_f16 (r, x, y); +} + +/* +**ufoo_lane: +** fdot v0.2s, v1.4h, v2.2h\[0\] +** ret +*/ +float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y) +{ + return vdot_lane_f32_f16 (r, x, y, 0); +} + +/* +**ufoo_laneq: +** fdot v0.2s, v1.4h, v2.2h\[2\] +** ret +*/ +float32x2_t ufoo_laneq(float32x2_t r, float16x4_t x, float16x8_t y) +{ + return vdot_laneq_f32_f16 (r, x, y, 2); +} + +/* +**ufooq_lane: +** fdot v0.4s, v1.8h, v2.2h\[1\] +** ret +*/ +float32x4_t ufooq_lane(float32x4_t r, float16x8_t x, float16x4_t y) +{ + return vdotq_lane_f32_f16 (r, x, y, 1); +} + +/* +**ufooq_laneq: +** fdot v0.4s, v1.8h, v2.2h\[2\] +** ret +*/ +float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y) +{ + return vdotq_laneq_f32_f16 (r, x, y, 2); +} \ No newline at end of file diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c new file mode 100644 index 000000000000..314322a08345 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/fdot-3.c @@ -0,0 +1,32 @@ +/* { dg-do assemble { target { aarch64*-*-* } } } */ +/* { dg-skip-if "" { *-*-* } { "-fno-fat-lto-objects" } } */ +/* { dg-require-effective-target aarch64_f16f32dot_neon_ok } */ +/* { dg-add-options aarch64_f16f32dot_neon } */ +/* { dg-additional-options "--save-temps" } */ + +#include <arm_neon.h> + +float32x2_t ufoo_lane(float32x2_t r, float16x4_t x, float16x4_t y) +{ + return vdot_lane_f32_f16 (r, x, y, 2); +} + +float32x2_t ufoo_laneq(float32x2_t r, float16x4_t x, float16x8_t y) +{ + return vdot_laneq_f32_f16 (r, x, y, 4); +} + +float32x4_t ufooq_lane(float32x4_t r, float16x8_t x, float16x4_t y) +{ + return vdotq_lane_f32_f16 (r, x, y, 3); +} + +float32x4_t ufooq_laneq(float32x4_t r, float16x8_t x, float16x8_t y) +{ + return vdotq_laneq_f32_f16 (r, x, y, 5); +} + +/* { dg-error {lane 2 out of range 0 - 1} "" { target *-*-* } 0 } */ +/* { dg-error {lane 4 out of range 0 - 3} "" { target *-*-* } 0 } */ +/* { dg-error {lane 3 out of range 0 - 1} "" { target *-*-* } 0 } */ +/* { dg-error {lane 5 out of range 0 - 3} "" { target *-*-* } 0 } */ \ No newline at end of file diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp index c22033c47b8d..42cbb5ce3df0 100644 --- a/gcc/testsuite/lib/target-supports.exp +++ b/gcc/testsuite/lib/target-supports.exp @@ -7413,6 +7413,47 @@ proc add_options_for_arm_fp16fml_neon { flags } { return "$flags $et_arm_fp16fml_neon_flags" } +# Return 1 if the target supports F16F32DOT +# instructions, 0 otherwise. This test is valid +# for AARCH64. +# Record the command line options needed. + +proc check_effective_target_aarch64_f16f32dot_neon_ok_nocache { } { + global et_aarch64_f16f32dot_neon_flags + set et_aarch64_f16f32dot_neon_flags "" + + if { ![istarget aarch64*-*-*] } { + return 0; + } + + foreach flags {"" "" } { + if { [check_no_compiler_messages_nocache aarch64_f16f32dot_neon_ok object { + #include <arm_neon.h> + #if !defined (__ARM_FEATURE_F16F32DOT) + #error "__ARM_FEATURE_F16F32DOT not defined" + #endif + } "$flags -march=armv8.2-a+fp16+f16f32dot"] } { + set et_aarch64_f16f32dot_neon_flags "$flags -march=armv8.2-a+fp16+f16f32dot" + return 1 + } + } + + return 0; +} + +proc check_effective_target_aarch64_f16f32dot_neon_ok { } { + return [check_cached_effective_target aarch64_f16f32dot_neon_ok \ + check_effective_target_aarch64_f16f32dot_neon_ok_nocache] +} + +proc add_options_for_aarch64_f16f32dot_neon { flags } { + if { ! [check_effective_target_aarch64_f16f32dot_neon_ok] } { + return "$flags" + } + global et_aarch64_f16f32dot_neon_flags + return "$flags $et_aarch64_f16f32dot_neon_flags" +} + # Return 1 if the target supports BFloat16 SIMD instructions, 0 otherwise. # The test is valid for ARM and for AArch64.