[gcc r17-2359] i386: Fix Win64 __int128 return handling [PR78799]

https://gcc.gnu.org/g:e63a873f1df7493b8b657acafb451bf40166044e

commit r17-2359-ge63a873f1df7493b8b657acafb451bf40166044e
Author: Oleg Tolmatcev <[email protected]>
Date:   Sun Jun 14 00:22:07 2026 +0200

    i386: Fix Win64 __int128 return handling [PR78799]
    
    GCC could return __int128 values in SSE (%xmm0) on Windows x64 instead
    of following the MS x64 ABI. Root cause: return classification allowed
    128-bit integer-like scalars to be treated as direct register returns.
    This patch updates the Windows x64 return-classification and codegen to
    treat int128 as an indirect return (caller-provided slot passed as first
    argument, pointer returned in RAX).
    
    gcc:
            PR target/78799
            * config/i386/i386.cc (function_value_ms_64): Do not treat
            integral 16-byte values as SSE returns.
            (ix86_return_in_memory): Likewise avoid treating integral 16-byte
            values as XMM returns.
    
    gcc/testsuite:
            * gcc.target/i386/pr78799.c: New test.
    
    Signed-off-by: Oleg Tolmatcev <[email protected]>
    Signed-off-by: Jonathan Yong <[email protected]>

Diff:
---
 gcc/config/i386/i386.cc                 |  6 ++----
 gcc/testsuite/gcc.target/i386/pr78799.c | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index 23f65ab401d1..31a057e81cd9 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -4348,10 +4348,9 @@ function_value_ms_64 (machine_mode orig_mode, machine_mode mode,
 	    break;
 	  if (valtype != NULL_TREE
 	      && !VECTOR_INTEGER_TYPE_P (valtype)
-	      && !INTEGRAL_TYPE_P (valtype)
 	      && !VECTOR_FLOAT_TYPE_P (valtype))
 	    break;
-	  if ((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
+	  if (VECTOR_MODE_P (mode)
 	      && !COMPLEX_MODE_P (mode))
 	    regno = FIRST_SSE_REG;
 	  break;
@@ -4458,9 +4457,8 @@ ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
 	  /* __m128 is returned in xmm0.  256/512-bit vector values are
 	     returned in ymm0/zmm0 when AVX/AVX512 is enabled.  */
 	  if ((!type || VECTOR_INTEGER_TYPE_P (type)
-	       || INTEGRAL_TYPE_P (type)
 	       || VECTOR_FLOAT_TYPE_P (type))
-	      && (SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
+	      && VECTOR_MODE_P (mode)
 	      && !COMPLEX_MODE_P (mode)
 	      && ((GET_MODE_SIZE (mode) == 16 || size == 16)
 		  || (TARGET_AVX && (GET_MODE_SIZE (mode) == 32 || size == 32))
diff --git a/gcc/testsuite/gcc.target/i386/pr78799.c b/gcc/testsuite/gcc.target/i386/pr78799.c
new file mode 100644
index 000000000000..fcd6fe3c9d8a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr78799.c
@@ -0,0 +1,15 @@
+/* { dg-do compile { target { x86_64-*-mingw* } } } */
+/* { dg-options "-O2" } */
+
+/* PR78799: verify Win64 __int128 return uses an indirect sret-style path:
+   RCX points to the return slot, RDX points to the argument object.  */
+__attribute__((ms_abi, noinline, noclone))
+__int128
+ret_i128 (__int128 a)
+{
+  return a;
+}
+
+/* { dg-final { scan-assembler "movdqa\t\\(%rdx\\), %xmm0" } } */
+/* { dg-final { scan-assembler "movq\t%rcx, %rax" } } */
+/* { dg-final { scan-assembler "movaps\t%xmm0, \\(%rcx\\)" } } */