[gcc r17-3504] tree: make TYPE_OVERFLOW_SANITIZED accept vector types

https://gcc.gnu.org/g:00a0e7d2375850a0256c392005e10c7d28aeac97

commit r17-3504-g00a0e7d2375850a0256c392005e10c7d28aeac97
Author: Kyrylo Tkachov <[email protected]>
Date:   Thu Aug 20 08:47:01 2026 +0200

    tree: make TYPE_OVERFLOW_SANITIZED accept vector types
    
    UBSan instruments signed integral vector arithmetic.  However,
    TYPE_OVERFLOW_SANITIZED only accepts scalar integral types.  Folding can use
    the false result to remove a vector operation before UBSan instruments it.
    For example:
    
      typedef int v4si __attribute__ ((vector_size (16)));
    
      v4si f (v4si x) { return -(-x); }
    
    Both negations can overflow.  The scalar-only predicate lets fold-const reduce
    the function to x and remove both diagnostics.
    
    aarch64 -O2 -fsanitize=signed-integer-overflow
            -fsanitize-trap=signed-integer-overflow before:
    
      f:
              ret
    
    After:
    
      f:
              fmov    w0, s0
              negs    w0, w0
              bvs     .L27
              ...
              neg     v31.4s, v0.4s
              fmov    w0, s31
              negs    w0, w0
              bvs     .L27
              ...
              ret
      .L27:
              brk     #1000
    
    Use ANY_INTEGRAL_TYPE_P so the predicate also accepts integral vector types.
    The test checks that both vector negations remain for UBSan instrumentation.
    
    Bootstrapped and tested on aarch64-none-linux-gnu.
    Tested on x86_64-pc-linux-gnu.
    
    gcc/ChangeLog:
    
            PR sanitizer/88109
            * tree.h (TYPE_OVERFLOW_SANITIZED): Use ANY_INTEGRAL_TYPE_P.
    
    gcc/testsuite/ChangeLog:
    
            PR sanitizer/88109
            * c-c++-common/ubsan/overflow-vec-3.c: New test.
    
    Signed-off-by: Kyrylo Tkachov <[email protected]>

Diff:
---
 gcc/testsuite/c-c++-common/ubsan/overflow-vec-3.c | 12 ++++++++++++
 gcc/tree.h                                        |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/gcc/testsuite/c-c++-common/ubsan/overflow-vec-3.c b/gcc/testsuite/c-c++-common/ubsan/overflow-vec-3.c
new file mode 100644
index 000000000000..56b00e81a761
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/ubsan/overflow-vec-3.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wno-psabi -fsanitize=signed-integer-overflow -fdump-tree-ubsan" } */
+
+typedef int v4si __attribute__ ((vector_size (4 * sizeof (int))));
+
+v4si
+f (v4si x)
+{
+  return -(-x);
+}
+
+/* { dg-final { scan-tree-dump-times "\\.UBSAN_CHECK_SUB" 2 "ubsan" } } */
diff --git a/gcc/tree.h b/gcc/tree.h
index bc6a304631dd..3fb643ea0fb4 100644
--- a/gcc/tree.h
+++ b/gcc/tree.h
@@ -1005,7 +1005,7 @@ extern void omp_clause_range_check_failed (const_tree, const char *, int,
 
 /* True if an overflow is to be preserved for sanitization.  */
 #define TYPE_OVERFLOW_SANITIZED(TYPE)			\
-  (INTEGRAL_TYPE_P (TYPE)				\
+  (ANY_INTEGRAL_TYPE_P (TYPE)				\
    && !TYPE_OVERFLOW_WRAPS (TYPE)			\
    && (flag_sanitize & SANITIZE_SI_OVERFLOW))