Re: Basic support for AVX512 mask registers

From c0ae9664acfde21e234b9a375ef7eab9bed5ee0b Mon Sep 17 00:00:00 2001
From: arthur <[email protected]>
Date: Fri, 21 Aug 2026 17:18:05 +0200
Subject: [PATCH] add infrastructure for EVEX compressed displacement

* Add dstate-disp-n to disassem-state
* Add reg/mem arg types for tuple sizes 1,2,4,8,16,32,64.
* Extend avx512-inst-printer-list to accept :disp-n.
* Scale EVEX disp8 by the instruction-specific tuple size in decode-mod-r/m
* Annotate full-vector EVEX moves and emit compressed displacement for them.
* Add disassembler tests for the functionality in this patch

This covers the full-vector move subset. For other EVEX memory
instructions more work is needed.
---
 src/cold/exports.lisp                 |   1 +
 src/compiler/target-dstate.lisp       |  16 +--
 src/compiler/x86-64/avx2-insts.lisp   | 102 ++++++++++++-------
 src/compiler/x86-64/avx512-insts.lisp |  33 ++++--
 src/compiler/x86-64/target-insts.lisp |  24 +++--
 tests/simd-pack-512.pure.lisp         | 138 ++++++++++++++++++++++++--
 6 files changed, 248 insertions(+), 66 deletions(-)

diff --git a/src/cold/exports.lisp b/src/cold/exports.lisp
index f47fd4766..0c2af38ba 100644
--- a/src/cold/exports.lisp
+++ b/src/cold/exports.lisp
@@ -3302,6 +3302,7 @@ structure representations")
            "DSTATE-SETPROP"
            "DSTATE-SEGMENT-SAP"
            "DSTATE-OPERANDS"
+           "DSTATE-DISP-N"
            "FIND-INST"
            "GET-CODE-SEGMENTS" "GET-FUN-SEGMENTS"
            "GET-INST-SPACE" "HANDLE-BREAK-ARGS"
diff --git a/src/compiler/target-dstate.lisp b/src/compiler/target-dstate.lisp
index 7b6e34db3..c42eb280a 100644
--- a/src/compiler/target-dstate.lisp
+++ b/src/compiler/target-dstate.lisp
@@ -76,7 +76,7 @@
   ;; what to align to in most cases
   (alignment sb-vm:n-word-bytes :type alignment :read-only t)
   (byte-order sb-c:*backend-byte-order*
-              :type (member :big-endian :little-endian))
+   :type (member :big-endian :little-endian))
   ;; current instruction as found in instruction space
   (inst)
   (operands (make-array 10) :read-only t) ; enough for anybody
@@ -85,7 +85,7 @@
   ;; non-prefix instruction is processed
   (inst-properties 0 :type fixnum)
   (filtered-values (make-array max-filtered-value-index)
-                   :type filtered-value-vector)
+   :type filtered-value-vector)
   ;; to avoid consing decoded values, a prefilter can keep a chain
   ;; of objects in these slots. The objects returned here
   ;; are reusable for the next instruction.
@@ -96,9 +96,9 @@
   (argument-column 0 :type column)
   ;; to make output look nicer
   (output-state :beginning
-                :type (member :beginning
-                              :block-boundary
-                              nil))
+   :type (member :beginning
+                 :block-boundary
+                 nil))
   (previous-chunk) ; as an integer, however much was fetched
   (known-register-contents)
 
@@ -118,7 +118,11 @@
   (notes nil) ; a single note or a list of notes
 
   ;; currently active source variables
-  (current-valid-locations nil :type (or null (vector bit))))
+  (current-valid-locations nil :type (or null (vector bit)))
+
+  ;; used for evex compressed-displacement scale
+  ;; N is the tuple size; a mod=01 disp8 means disp8*N, 0 if unknown/disabled.
+  (disp-n 0 :type (unsigned-byte 7)))
 
 (declaim (freeze-type disassem-state))
 (defmethod print-object ((dstate disassem-state) stream)
diff --git a/src/compiler/x86-64/avx2-insts.lisp b/src/compiler/x86-64/avx2-insts.lisp
index 555d9a4e8..70ae8d968 100644
--- a/src/compiler/x86-64/avx2-insts.lisp
+++ b/src/compiler/x86-64/avx2-insts.lisp
@@ -27,6 +27,20 @@
   :prefilter #'prefilter-xmmreg/mem
   :printer #'print-ymmreg/mem)
 
+(macrolet ((define-disp-arg-type (name n)
+             `(define-arg-type ,name
+                :prefilter (lambda (dstate mod r/m)
+                             (setf (dstate-disp-n dstate) ,n)
+                             (decode-mod-r/m dstate mod r/m 'fpr))
+                :printer #'print-ymmreg/mem)))
+  (define-disp-arg-type evex-ymmreg/mem-disp1  1)
+  (define-disp-arg-type evex-ymmreg/mem-disp2  2)
+  (define-disp-arg-type evex-ymmreg/mem-disp4  4)
+  (define-disp-arg-type evex-ymmreg/mem-disp8  8)
+  (define-disp-arg-type evex-ymmreg/mem-disp16 16)
+  (define-disp-arg-type evex-ymmreg/mem-disp32 32)
+  (define-disp-arg-type evex-ymmreg/mem-disp64 64))
+
 (define-arg-type vm
   :prefilter #'prefilter-xmmreg/mem
   :printer #'print-ymmreg/mem)
@@ -697,42 +711,60 @@ REG is the source (encoded in ModR/M.r/m).
 
 
 (eval-when (#-sb-xc :compile-toplevel :load-toplevel :execute)
+  ;; if both disp-n and a *-mem-size are supplied, this version uses the disp-n
+  ;; arg type and loses the explicit size indicator. For the common full-vector
+  ;; load/store cases that is fine, but needs more work to fully support sized +
+  ;; compressed displacement
   (defun avx512-inst-printer-list (inst-format-stem prefix opcode
-                                   &key more-fields printer
-                                        (opcode-prefix #x0F)
-                                        reg-mem-size
-                                        xmmreg-mem-size
-                                        w
-                                        ll
-                                        nds)
-    (let ((fields `((pp ,(vex-encode-pp prefix))
-                    (mm ,(evex-encode-mm opcode-prefix))
-                    (op ,opcode)
-                    ,@(and w `((w ,w)))
-                    ,@(and ll `((ll ,ll)))
-                    ,@(cond (xmmreg-mem-size
-                             `((reg/mem nil :type ',(case xmmreg-mem-size
-                                                      (:qword 'sized-xmmreg/mem-default-qword)
-                                                      (:dword 'sized-dword-xmmreg/mem)
-                                                      (:word 'sized-word-xmmreg/mem)
-                                                      (:byte 'sized-byte-xmmreg/mem)
-                                                      (:sized 'sized-xmmreg/mem)))))
-                            (reg-mem-size
-                             `((reg/mem nil :type ',(case reg-mem-size
-                                                      (:qword 'sized-reg/mem-default-qword)
-                                                      (:dword 'sized-dword-reg/mem)
-                                                      (:word 'sized-word-reg/mem)
-                                                      (:byte 'sized-byte-reg/mem)
-                                                      (:sized 'sized-reg/mem))))))
-                    ,@more-fields))
-          (inst-format (symbolicate "EVEX-" inst-format-stem)))
-      (list `(:printer ,inst-format ,fields
-                       ,@(cond (printer
-                                `(',printer))
-                               ((eq nds 'to-mem)
-                                `('(:name :tab reg/mem ", " vvvv ", " reg)))
-                               (nds
-                                `('(:name :tab reg ", " vvvv ", " reg/mem))))))))
+                                 &key more-fields printer
+                                      (opcode-prefix #x0F)
+                                      reg-mem-size
+                                      xmmreg-mem-size
+                                      w
+                                      ll
+                                      nds
+                                      disp-n)
+  (let ((fields `((pp ,(vex-encode-pp prefix))
+                  (mm ,(evex-encode-mm opcode-prefix))
+                  (op ,opcode)
+                  ,@(and w `((w ,w)))
+                  ,@(and ll `((ll ,ll)))
+                  ,@(cond
+                      (disp-n
+                       `((reg/mem nil :type
+                          ',(ecase disp-n
+                              (1  'evex-ymmreg/mem-disp1)
+                              (2  'evex-ymmreg/mem-disp2)
+                              (4  'evex-ymmreg/mem-disp4)
+                              (8  'evex-ymmreg/mem-disp8)
+                              (16 'evex-ymmreg/mem-disp16)
+                              (32 'evex-ymmreg/mem-disp32)
+                              (64 'evex-ymmreg/mem-disp64)))))
+                      (xmmreg-mem-size
+                       `((reg/mem nil :type
+                          ',(case xmmreg-mem-size
+                              (:qword 'sized-xmmreg/mem-default-qword)
+                              (:dword 'sized-dword-xmmreg/mem)
+                              (:word  'sized-word-xmmreg/mem)
+                              (:byte  'sized-byte-xmmreg/mem)
+                              (:sized 'sized-xmmreg/mem)))))
+                      (reg-mem-size
+                       `((reg/mem nil :type
+                          ',(case reg-mem-size
+                              (:qword 'sized-reg/mem-default-qword)
+                              (:dword 'sized-dword-reg/mem)
+                              (:word  'sized-word-reg/mem)
+                              (:byte  'sized-byte-reg/mem)
+                              (:sized 'sized-reg/mem))))))
+                  ,@more-fields))
+        (inst-format (symbolicate "EVEX-" inst-format-stem)))
+    (list `(:printer ,inst-format ,fields
+                     ,@(cond (printer
+                              `(',printer))
+                             ((eq nds 'to-mem)
+                              `('(:name :tab reg/mem ", " vvvv ", " reg)))
+                             (nds
+                              `('(:name :tab reg ", " vvvv ", " reg/mem))))))))
 
   (defun avx2-inst-printer-list (inst-format-stem prefix opcode
                                  &key more-fields printer
diff --git a/src/compiler/x86-64/avx512-insts.lisp b/src/compiler/x86-64/avx512-insts.lisp
index a5965dd26..825be70b3 100644
--- a/src/compiler/x86-64/avx512-insts.lisp
+++ b/src/compiler/x86-64/avx512-insts.lisp
@@ -23,22 +23,41 @@
 
 ;;;; AVX-512 (EVEX-only) instruction definitions
 
+;;; Return the compressed-displacement scale N for a full-vector
+;;; load/store instruction.  The vector width determines N:
+;;;   128-bit -> 16
+;;;   256-bit -> 32
+;;;   512-bit -> 64
+(defun full-vector-disp-n (reg)
+  (cond ((zmm-register-p reg) 64)
+        ((ymm-register-p reg) 32)
+        ((xmm-register-p reg) 16)
+        (t 0)))
+
 ;;; EVEX-only aligned/unaligned moves
 (macrolet ((def (name prefix opcode-from opcode-to w)
              `(define-instruction ,name (segment dst src)
-                ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode-from
-                                            :w w)
-                ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode-to
-                                            :printer '(:name :tab reg/mem ", " reg)
-                                            :w w)
+                ,@(loop for (ll n) in '((#b00 16) (#b01 32) (#b10 64))
+                        append
+                        (avx512-inst-printer-list
+                         'ymm-ymm/mem prefix opcode-from
+                         :w w :ll ll :disp-n n))
+                ,@(loop for (ll n) in '((#b00 16) (#b01 32) (#b10 64))
+                        append
+                        (avx512-inst-printer-list
+                         'ymm-ymm/mem prefix opcode-to
+                         :w w :ll ll :disp-n n
+                         :printer '(:name :tab reg/mem ", " reg)))
                 (:emitter
                  (cond ((xmm-register-p dst)
                         (emit-avx512-inst segment src dst ,prefix ,opcode-from
-                                         :opcode-prefix #x0F :w ,w))
+                                         :opcode-prefix #x0F :w ,w
+                                         :disp-n (full-vector-disp-n dst)))
                        (t
                         (aver (xmm-register-p src))
                         (emit-avx512-inst segment dst src ,prefix ,opcode-to
-                                         :opcode-prefix #x0F :w ,w)))))))
+                                         :opcode-prefix #x0F :w ,w
+                                         :disp-n (full-vector-disp-n src))))))))
   (def vmovdqa32 #x66 #x6f #x7f 0)
   (def vmovdqa64 #x66 #x6f #x7f 1)
   (def vmovdqu8  #xf2 #x6f #x7f 0)
diff --git a/src/compiler/x86-64/target-insts.lisp b/src/compiler/x86-64/target-insts.lisp
index d2caed762..dd695bcd7 100644
--- a/src/compiler/x86-64/target-insts.lisp
+++ b/src/compiler/x86-64/target-insts.lisp
@@ -347,16 +347,20 @@
            (case mod
              (#b01
               (let ((disp8 (read-signed-suffix 8 dstate)))
-                ;; EVEX compressed displacement: the CPU multiplies disp8
-                ;; by N (tuple size). Scale for display so the shown offset
-                ;; matches the effective address. +evex-l1+ is only set by
-                ;; the EVEX L'L prefilter (never VEX), so it reliably
-                ;; identifies EVEX 512-bit where N=64 for Full tuple type.
-                ;; This is approximate (narrow-load instructions have
-                ;; smaller N) but covers the common case correctly.
-                (if (dstate-getprop dstate +evex-l1+)
-                    (* disp8 64)
-                    disp8)))
+                ;; EVEX compressed displacement: with mod=01, the effective
+                ;; displacement is disp8 * N, where N is the instruction's tuple
+                ;; size.  N is not encoded in the instruction; it is supplied by
+                ;; the printer's specialized reg/mem arg type, which stores it in
+                ;; SB-DISASSEM::DSTATE-DISP-N slot, before DECODE-MOD-R/M runs.
+                ;;
+                ;; If DSTATE-DISP-N is zero, the instruction was not annotated with
+                ;; a compressed-displacement tuple size.  In that case do not scale
+                ;; disp8; leave it as-is rather than guessing a scale (which
+                ;; produces a silently wrong EA).
+                (let ((disp-n (dstate-disp-n dstate)))
+                  (if (plusp disp-n)
+                      (* disp8 disp-n)
+                      disp8))))
              (#b10 (read-signed-suffix 32 dstate))))
          (extend (bit-name reg)
            (logior (if (dstate-getprop dstate bit-name) 8 0) reg)))
diff --git a/tests/simd-pack-512.pure.lisp b/tests/simd-pack-512.pure.lisp
index 932fac407..00ff3e9ad 100644
--- a/tests/simd-pack-512.pure.lisp
+++ b/tests/simd-pack-512.pure.lisp
@@ -288,12 +288,19 @@
 ;; evex patch
 (cl:in-package "SB-VM")
 
-(sb-c::defknown %test-evex-high-regs ()
-    (unsigned-byte 64)
-  (sb-c::flushable sb-c::movable))
-
-(defun %test-evex-high-regs ()
-  (error "%test-evex-high-regs stub"))
+(macrolet ((def (name)
+             `(progn
+                (sb-c::defknown ,name ()
+                    (unsigned-byte 64)
+                  (sb-c::flushable sb-c::movable))
+                (defun ,name ()
+                  (error ,(format nil "~A stub" name))))))
+  (def %test-evex-high-regs)
+  (def %test-evex-disp8)
+  (def %test-evex-disp-vector-lengths)
+  (def %test-evex-disp-negative)
+  (def %test-evex-disp-nonmultiple)
+  (def %test-evex-disp-large))
 
 (define-vop (%test-evex-high-regs)
   (:translate %test-evex-high-regs)
@@ -307,12 +314,71 @@
     (inst vaddps z16 z17 z18)
     (inst xor :dword res res)))
 
+(define-vop (%test-evex-disp8)
+  (:translate %test-evex-disp8)
+  (:policy :fast-safe)
+  (:temporary (:sc single-avx512-reg :offset 0) zmm)
+  (:temporary (:sc unsigned-reg :offset rsp-offset) rsp)
+  (:results (res :scs (unsigned-reg)))
+  (:result-types unsigned-num)
+  (:generator 1
+    (inst xor :dword res res)
+    (inst vmovdqu64 zmm (ea 64 rsp))))
+
+(define-vop (%test-evex-disp-vector-lengths)
+  (:translate %test-evex-disp-vector-lengths)
+  (:policy :fast-safe)
+  (:temporary (:sc single-sse-reg :offset 0) xmm)
+  (:temporary (:sc single-avx2-reg :offset 1) ymm)
+  (:temporary (:sc single-avx512-reg :offset 2) zmm)
+  (:temporary (:sc unsigned-reg :offset rsp-offset) rsp)
+  (:results (res :scs (unsigned-reg)))
+  (:result-types unsigned-num)
+  (:generator 1
+    (inst xor :dword res res)
+    (inst vmovdqu64 xmm (ea 16 rsp))
+    (inst vmovdqu64 ymm (ea 32 rsp))
+    (inst vmovdqu64 zmm (ea 64 rsp))))
+
+(define-vop (%test-evex-disp-negative)
+  (:translate %test-evex-disp-negative)
+  (:policy :fast-safe)
+  (:temporary (:sc single-avx512-reg :offset 0) zmm)
+  (:temporary (:sc unsigned-reg :offset rsp-offset) rsp)
+  (:results (res :scs (unsigned-reg)))
+  (:result-types unsigned-num)
+  (:generator 1
+    (inst xor :dword res res)
+    (inst vmovdqu64 zmm (ea -64 rsp))))
+
+(define-vop (%test-evex-disp-nonmultiple)
+  (:translate %test-evex-disp-nonmultiple)
+  (:policy :fast-safe)
+  (:temporary (:sc single-avx512-reg :offset 0) zmm)
+  (:temporary (:sc unsigned-reg :offset rsp-offset) rsp)
+  (:results (res :scs (unsigned-reg)))
+  (:result-types unsigned-num)
+  (:generator 1
+    (inst xor :dword res res)
+    (inst vmovdqu64 zmm (ea 65 rsp))))
+
+(define-vop (%test-evex-disp-large)
+  (:translate %test-evex-disp-large)
+  (:policy :fast-safe)
+  (:temporary (:sc single-avx512-reg :offset 0) zmm)
+  (:temporary (:sc unsigned-reg :offset rsp-offset) rsp)
+  (:results (res :scs (unsigned-reg)))
+  (:result-types unsigned-num)
+  (:generator 1
+    (inst xor :dword res res)
+    (inst vmovdqu64 zmm (ea 8192 rsp))))
+
 (cl:in-package :test-util)
 
 (with-test (:name :evex-high-register-disassembly)
   (let* ((fun (compile nil
                        '(lambda ()
-                          (sb-vm::%test-evex-high-regs))))
+                         (sb-vm::%test-evex-high-regs))))
          (text (with-output-to-string (s)
                  (disassemble fun :stream s))))
     ;; These names can only appear if the disassembler correctly
@@ -321,4 +387,60 @@
     (assert (search "ZMM17" text))
     (assert (search "ZMM18" text))
     ;; Ideally we see a decoded instruction, not raw EVEX bytes.
-    (assert (search "VADDPS" text))))
+    (assert (search "VADDPS" text))
+    ;; While development, the decoder was a bit too broad
+    (assert (not (search "VADDPS-MASKED" text)))))
+
+#| tests for evex compressed displacement patch:
+
+   EVEX vector lengths
+   signed negative compressed displacement
+   non-compressible displacement fallback to disp32
+   compressible displacement too large for disp8
+|#
+
+(with-test (:name :evex-compressed-displacement-vector-lengths)
+  (let* ((fun (compile nil
+                       '(lambda ()
+                         (sb-vm::%test-evex-disp-vector-lengths))))
+         (text (with-output-to-string (s)
+                 (disassemble fun :stream s))))
+    (assert (search "VMOVDQU64 XMM0, [RSP+16]" text))
+    (assert (search "VMOVDQU64 YMM1, [RSP+32]" text))
+    (assert (search "VMOVDQU64 ZMM2, [RSP+64]" text))))
+
+(with-test (:name :evex-compressed-displacement)
+  (let* ((fun (compile nil
+                       '(lambda ()
+                         (sb-vm::%test-evex-disp8))))
+         (text (with-output-to-string (s)
+                 (disassemble fun :stream s))))
+    ;; The disassembler must scale EVEX disp8 by 64.
+    (assert (search "VMOVDQU64 ZMM0, [RSP+64]" text))
+    ;; A failure mode is showing the unscaled compressed byte instead:
+    (assert (not (search "[RSP+1]" text)))))
+
+(with-test (:name :evex-compressed-displacement-negative)
+  (let* ((fun (compile nil
+                       '(lambda ()
+                         (sb-vm::%test-evex-disp-negative))))
+         (text (with-output-to-string (s)
+                 (disassemble fun :stream s))))
+    (assert (search "VMOVDQU64 ZMM0, [RSP-64]" text))))
+
+(with-test (:name :evex-compressed-displacement-nonmultiple)
+  (let* ((fun (compile nil
+                       '(lambda ()
+                         (sb-vm::%test-evex-disp-nonmultiple))))
+         (text (with-output-to-string (s)
+                 (disassemble fun :stream s))))
+    (assert (search "VMOVDQU64 ZMM0, [RSP+65]" text))))
+
+(with-test (:name :evex-compressed-displacement-large)
+  (let* ((fun (compile nil
+                       '(lambda ()
+                         (sb-vm::%test-evex-disp-large))))
+         (text (with-output-to-string (s)
+                 (disassemble fun :stream s))))
+    (assert (search "VMOVDQU64 ZMM0, [RSP+8192]" text))))
+
-- 
2.55.0