[binutils-gdb] gdb: prefer lhs type's address spaces/classes in check_typedef

https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;h=981c2441c22487d30fac7303ad761218dbeeb3fa

commit 981c2441c22487d30fac7303ad761218dbeeb3fa
Author: Tankut Baris Aktemur <[email protected]>
Date:   Wed Aug 5 08:02:57 2026 +0200

    gdb: prefer lhs type's address spaces/classes in check_typedef
    
    In commit 92fdad7 "gdb: convert type instance flags to bitfields",
    `operator|=` of type_instance_flags required the address space and
    address class values of left-hand-side to be zero.  This introduced
    the following bug (thanks to Keith Seitz for reporting it at
    https://inbox.sourceware.org/gdb-patches/[email protected]/T/#u):
    
      typedef int myint;
    
      (gdb) ptype (@code myint) 3
      ../../src/gdb/gdbtypes.h:127: internal-error: operator|=: Assertion
      `harvard_aspace == 0' failed.
      A problem internal to GDB has been detected,
      further debugging may prove unreliable.
      ----- Backtrace -----
      0x5bb1d1 gdb_internal_backtrace_1
              ../../src/gdb/bt-utils.c:122
      0x5bb210 _Z22gdb_internal_backtracev
              ../../src/gdb/bt-utils.c:173
      0xdfbbaa internal_vproblem
              ../../src/gdb/utils.c:434
      0xdfbf45 _Z15internal_verrorPKciS0_P13__va_list_tag
              ../../src/gdb/utils.c:514
      0x162763f _Z18internal_error_locPKciS0_z
              ../../src/gdbsupport/errors.cc:57
      0x87e8e6 _ZN19type_instance_flagsoRERKS_
              ../../src/gdb/gdbtypes.h:127
      0x875d71 _Z13check_typedefP4type
              ../../src/gdb/gdbtypes.c:3072
    
    The |= operator is used in check_typedef as follows:
    
          /* Preserve the instance flags as we traverse down the typedef chain.
    
             Handling address spaces/classes is nasty, what do we do if there's a
             conflict?
             E.g., what if an outer typedef marks the type as class_1 and an inner
             typedef marks the type as class_2?
             This is the wrong place to do such error checking.  We leave it to
             the code that created the typedef in the first place to flag the
             error.  We just pick the outer address space (akin to letting the
             outer cast in a chain of casting win), instead of assuming
             "it can't happen".  */
          {
            type_instance_flags new_instance_flags = type->instance_flags ();
    
            /* Treat code vs data spaces and address classes separately.  */
            if (instance_flags.harvard_aspace != HARVARD_ASPACE_NONE)
              new_instance_flags.harvard_aspace = HARVARD_ASPACE_NONE;
            if (instance_flags.address_class != 0)
              new_instance_flags.address_class = 0;
    
            instance_flags |= new_instance_flags;
          }
    
    So, the assertion in operator|= was wrong.  The outer type, which is
    the left-hand-side in this case, should preserve its values if they
    are non-zero.  The right-hand-side values are used, if lhs values are
    zero.  Fix the bug accordingly.
    
    Furthermore, rename operator|= to "merge".  Type instance flags are no
    longer stored as a bitmask value, but rather as a struct.  Having an
    operator like |= gives the wrong impression that we are doing a
    bitmask OR.  Using a method makes the intention clearer.
    
    Include a regression test.
    
    Reviewed-By: Keith Seitz <[email protected]>
    Approved-By: Tom Tromey <[email protected]>

Diff:
---
 gdb/gdbtypes.c                            | 12 +-----------
 gdb/gdbtypes.h                            | 15 ++++++++-------
 gdb/testsuite/gdb.cp/typedef-operator.exp |  3 +++
 3 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/gdb/gdbtypes.c b/gdb/gdbtypes.c
index 409601abe03..f5c0177bff2 100644
--- a/gdb/gdbtypes.c
+++ b/gdb/gdbtypes.c
@@ -3036,17 +3036,7 @@ check_typedef (struct type *type)
 	 error.  We just pick the outer address space (akin to letting the
 	 outer cast in a chain of casting win), instead of assuming
 	 "it can't happen".  */
-      {
-	type_instance_flags new_instance_flags = type->instance_flags ();
-
-	/* Treat code vs data spaces and address classes separately.  */
-	if (instance_flags.harvard_aspace != HARVARD_ASPACE_NONE)
-	  new_instance_flags.harvard_aspace = HARVARD_ASPACE_NONE;
-	if (instance_flags.address_class != 0)
-	  new_instance_flags.address_class = 0;
-
-	instance_flags |= new_instance_flags;
-      }
+      instance_flags.merge (type->instance_flags ());
     }
 
   /* If this is a struct/class/union with no fields, then check
diff --git a/gdb/gdbtypes.h b/gdb/gdbtypes.h
index eda09641248..dd2d24fa8e2 100644
--- a/gdb/gdbtypes.h
+++ b/gdb/gdbtypes.h
@@ -119,21 +119,22 @@ struct type_instance_flags
     return !(*this == other);
   }
 
-  type_instance_flags &operator|= (const type_instance_flags &other)
+  /* Merge OTHER flags to THIS.  Address space and address class
+     values of THIS are preserved, if they are non-zero.  Otherwise,
+     OTHER's values are used.  */
+  void merge (const type_instance_flags &other)
   {
     is_const = is_const || other.is_const;
     is_volatile = is_volatile || other.is_volatile;
 
-    gdb_assert (harvard_aspace == 0);
-    harvard_aspace = other.harvard_aspace;
-
-    gdb_assert (address_class == 0);
-    address_class = other.address_class;
+    if (harvard_aspace == HARVARD_ASPACE_NONE)
+      harvard_aspace = other.harvard_aspace;
+    if (address_class == 0)
+      address_class = other.address_class;
 
     is_nottext = is_nottext || other.is_nottext;
     is_restrict = is_restrict || other.is_restrict;
     is_atomic = is_atomic || other.is_atomic;
-    return *this;
   }
 
   /* Constant type.  If this is set, the corresponding type has a
diff --git a/gdb/testsuite/gdb.cp/typedef-operator.exp b/gdb/testsuite/gdb.cp/typedef-operator.exp
index 9513b3725dc..e03502769a4 100644
--- a/gdb/testsuite/gdb.cp/typedef-operator.exp
+++ b/gdb/testsuite/gdb.cp/typedef-operator.exp
@@ -32,3 +32,6 @@ if {![runto_main]} {
 }
 
 gdb_test "p *v" " = 42" "test typedef"
+
+# Test that address spaces are preserved.
+gdb_test "ptype (@data D)u" "type = @data class C .*"