master: add infrastructure for EVEX compressed displacement
stassats via Sbcl-commits <[email protected]>
| Newsgroups | gmane.lisp.steel-bank.cvs |
|---|---|
| Message-ID | <[email protected]> |
The branch "master" has been updated in SBCL:
via f7fe8b09dfc812cefa73e8962d194e89bdd48ac9 (commit)
from e2ff58b3023bc2a11fb7010c5d5ff739b90d1898 (commit)
- Log -----------------------------------------------------------------
commit f7fe8b09dfc812cefa73e8962d194e89bdd48ac9
Author: arthur <[email protected]>
Date: Fri Aug 21 17:18:05 2026 +0200
add infrastructure for EVEX compressed displacement
* Add dstate-disp-n to disassem-state
* Add reg/mem arg types for tuple sizes 1,2,4,8,16,32,64.
* Extend avx512-inst-printer-list to accept :disp-n.
* Scale EVEX disp8 by the instruction-specific tuple size in decode-mod-r/m
* Annotate full-vector EVEX moves and emit compressed displacement for them.
* Add disassembler tests for the functionality in this patch
This covers the full-vector move subset. For other EVEX memory
instructions more work is needed.
---
src/cold/exports.lisp | 1 +
src/compiler/target-dstate.lisp | 7 +-
src/compiler/x86-64/avx2-insts.lisp | 102 ++++++++++++++++---------
src/compiler/x86-64/avx512-insts.lisp | 33 ++++++--
src/compiler/x86-64/target-insts.lisp | 24 +++---
tests/simd-pack-512.pure.lisp | 137 ++++++++++++++++++++++++++++++++--
6 files changed, 243 insertions(+), 61 deletions(-)
diff --git a/src/cold/exports.lisp b/src/cold/exports.lisp
index f47fd4766..0c2af38ba 100644
--- a/src/cold/exports.lisp
+++ b/src/cold/exports.lisp
@@ -3302,6 +3302,7 @@ structure representations")
"DSTATE-SETPROP"
"DSTATE-SEGMENT-SAP"
"DSTATE-OPERANDS"
+ "DSTATE-DISP-N"
"FIND-INST"
"GET-CODE-SEGMENTS" "GET-FUN-SEGMENTS"
"GET-INST-SPACE" "HANDLE-BREAK-ARGS"
diff --git a/src/compiler/target-dstate.lisp b/src/compiler/target-dstate.lisp
index 7b6e34db3..75cbc2a32 100644
--- a/src/compiler/target-dstate.lisp
+++ b/src/compiler/target-dstate.lisp
@@ -118,7 +118,12 @@
(notes nil) ; a single note or a list of notes
;; currently active source variables
- (current-valid-locations nil :type (or null (vector bit))))
+ (current-valid-locations nil :type (or null (vector bit)))
+
+ ;; used for evex compressed-displacement scale
+ ;; N is the tuple size; a mod=01 disp8 means disp8*N, 0 if unknown/disabled.
+ #+x86-64
+ (disp-n 0 :type (unsigned-byte 7)))
(declaim (freeze-type disassem-state))
(defmethod print-object ((dstate disassem-state) stream)
diff --git a/src/compiler/x86-64/avx2-insts.lisp b/src/compiler/x86-64/avx2-insts.lisp
index 555d9a4e8..70ae8d968 100644
--- a/src/compiler/x86-64/avx2-insts.lisp
+++ b/src/compiler/x86-64/avx2-insts.lisp
@@ -27,6 +27,20 @@
:prefilter #'prefilter-xmmreg/mem
:printer #'print-ymmreg/mem)
+(macrolet ((define-disp-arg-type (name n)
+ `(define-arg-type ,name
+ :prefilter (lambda (dstate mod r/m)
+ (setf (dstate-disp-n dstate) ,n)
+ (decode-mod-r/m dstate mod r/m 'fpr))
+ :printer #'print-ymmreg/mem)))
+ (define-disp-arg-type evex-ymmreg/mem-disp1 1)
+ (define-disp-arg-type evex-ymmreg/mem-disp2 2)
+ (define-disp-arg-type evex-ymmreg/mem-disp4 4)
+ (define-disp-arg-type evex-ymmreg/mem-disp8 8)
+ (define-disp-arg-type evex-ymmreg/mem-disp16 16)
+ (define-disp-arg-type evex-ymmreg/mem-disp32 32)
+ (define-disp-arg-type evex-ymmreg/mem-disp64 64))
+
(define-arg-type vm
:prefilter #'prefilter-xmmreg/mem
:printer #'print-ymmreg/mem)
@@ -697,42 +711,60 @@ REG is the source (encoded in ModR/M.r/m).
(eval-when (#-sb-xc :compile-toplevel :load-toplevel :execute)
+ ;; if both disp-n and a *-mem-size are supplied, this version uses the disp-n
+ ;; arg type and loses the explicit size indicator. For the common full-vector
+ ;; load/store cases that is fine, but needs more work to fully support sized +
+ ;; compressed displacement
(defun avx512-inst-printer-list (inst-format-stem prefix opcode
- &key more-fields printer
- (opcode-prefix #x0F)
- reg-mem-size
- xmmreg-mem-size
- w
- ll
- nds)
- (let ((fields `((pp ,(vex-encode-pp prefix))
- (mm ,(evex-encode-mm opcode-prefix))
- (op ,opcode)
- ,@(and w `((w ,w)))
- ,@(and ll `((ll ,ll)))
- ,@(cond (xmmreg-mem-size
- `((reg/mem nil :type ',(case xmmreg-mem-size
- (:qword 'sized-xmmreg/mem-default-qword)
- (:dword 'sized-dword-xmmreg/mem)
- (:word 'sized-word-xmmreg/mem)
- (:byte 'sized-byte-xmmreg/mem)
- (:sized 'sized-xmmreg/mem)))))
- (reg-mem-size
- `((reg/mem nil :type ',(case reg-mem-size
- (:qword 'sized-reg/mem-default-qword)
- (:dword 'sized-dword-reg/mem)
- (:word 'sized-word-reg/mem)
- (:byte 'sized-byte-reg/mem)
- (:sized 'sized-reg/mem))))))
- ,@more-fields))
- (inst-format (symbolicate "EVEX-" inst-format-stem)))
- (list `(:printer ,inst-format ,fields
- ,@(cond (printer
- `(',printer))
- ((eq nds 'to-mem)
- `('(:name :tab reg/mem ", " vvvv ", " reg)))
- (nds
- `('(:name :tab reg ", " vvvv ", " reg/mem))))))))
+ &key more-fields printer
+ (opcode-prefix #x0F)
+ reg-mem-size
+ xmmreg-mem-size
+ w
+ ll
+ nds
+ disp-n)
+ (let ((fields `((pp ,(vex-encode-pp prefix))
+ (mm ,(evex-encode-mm opcode-prefix))
+ (op ,opcode)
+ ,@(and w `((w ,w)))
+ ,@(and ll `((ll ,ll)))
+ ,@(cond
+ (disp-n
+ `((reg/mem nil :type
+ ',(ecase disp-n
+ (1 'evex-ymmreg/mem-disp1)
+ (2 'evex-ymmreg/mem-disp2)
+ (4 'evex-ymmreg/mem-disp4)
+ (8 'evex-ymmreg/mem-disp8)
+ (16 'evex-ymmreg/mem-disp16)
+ (32 'evex-ymmreg/mem-disp32)
+ (64 'evex-ymmreg/mem-disp64)))))
+ (xmmreg-mem-size
+ `((reg/mem nil :type
+ ',(case xmmreg-mem-size
+ (:qword 'sized-xmmreg/mem-default-qword)
+ (:dword 'sized-dword-xmmreg/mem)
+ (:word 'sized-word-xmmreg/mem)
+ (:byte 'sized-byte-xmmreg/mem)
+ (:sized 'sized-xmmreg/mem)))))
+ (reg-mem-size
+ `((reg/mem nil :type
+ ',(case reg-mem-size
+ (:qword 'sized-reg/mem-default-qword)
+ (:dword 'sized-dword-reg/mem)
+ (:word 'sized-word-reg/mem)
+ (:byte 'sized-byte-reg/mem)
+ (:sized 'sized-reg/mem))))))
+ ,@more-fields))
+ (inst-format (symbolicate "EVEX-" inst-format-stem)))
+ (list `(:printer ,inst-format ,fields
+ ,@(cond (printer
+ `(',printer))
+ ((eq nds 'to-mem)
+ `('(:name :tab reg/mem ", " vvvv ", " reg)))
+ (nds
+ `('(:name :tab reg ", " vvvv ", " reg/mem))))))))
(defun avx2-inst-printer-list (inst-format-stem prefix opcode
&key more-fields printer
diff --git a/src/compiler/x86-64/avx512-insts.lisp b/src/compiler/x86-64/avx512-insts.lisp
index a5965dd26..825be70b3 100644
--- a/src/compiler/x86-64/avx512-insts.lisp
+++ b/src/compiler/x86-64/avx512-insts.lisp
@@ -23,22 +23,41 @@
;;;; AVX-512 (EVEX-only) instruction definitions
+;;; Return the compressed-displacement scale N for a full-vector
+;;; load/store instruction. The vector width determines N:
+;;; 128-bit -> 16
+;;; 256-bit -> 32
+;;; 512-bit -> 64
+(defun full-vector-disp-n (reg)
+ (cond ((zmm-register-p reg) 64)
+ ((ymm-register-p reg) 32)
+ ((xmm-register-p reg) 16)
+ (t 0)))
+
;;; EVEX-only aligned/unaligned moves
(macrolet ((def (name prefix opcode-from opcode-to w)
`(define-instruction ,name (segment dst src)
- ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode-from
- :w w)
- ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode-to
- :printer '(:name :tab reg/mem ", " reg)
- :w w)
+ ,@(loop for (ll n) in '((#b00 16) (#b01 32) (#b10 64))
+ append
+ (avx512-inst-printer-list
+ 'ymm-ymm/mem prefix opcode-from
+ :w w :ll ll :disp-n n))
+ ,@(loop for (ll n) in '((#b00 16) (#b01 32) (#b10 64))
+ append
+ (avx512-inst-printer-list
+ 'ymm-ymm/mem prefix opcode-to
+ :w w :ll ll :disp-n n
+ :printer '(:name :tab reg/mem ", " reg)))
(:emitter
(cond ((xmm-register-p dst)
(emit-avx512-inst segment src dst ,prefix ,opcode-from
- :opcode-prefix #x0F :w ,w))
+ :opcode-prefix #x0F :w ,w
+ :disp-n (full-vector-disp-n dst)))
(t
(aver (xmm-register-p src))
(emit-avx512-inst segment dst src ,prefix ,opcode-to
- :opcode-prefix #x0F :w ,w)))))))
+ :opcode-prefix #x0F :w ,w
+ :disp-n (full-vector-disp-n src))))))))
(def vmovdqa32 #x66 #x6f #x7f 0)
(def vmovdqa64 #x66 #x6f #x7f 1)
(def vmovdqu8 #xf2 #x6f #x7f 0)
diff --git a/src/compiler/x86-64/target-insts.lisp b/src/compiler/x86-64/target-insts.lisp
index d2caed762..dd695bcd7 100644
--- a/src/compiler/x86-64/target-insts.lisp
+++ b/src/compiler/x86-64/target-insts.lisp
@@ -347,16 +347,20 @@
(case mod
(#b01
(let ((disp8 (read-signed-suffix 8 dstate)))
- ;; EVEX compressed displacement: the CPU multiplies disp8
- ;; by N (tuple size). Scale for display so the shown offset
- ;; matches the effective address. +evex-l1+ is only set by
- ;; the EVEX L'L prefilter (never VEX), so it reliably
- ;; identifies EVEX 512-bit where N=64 for Full tuple type.
- ;; This is approximate (narrow-load instructions have
- ;; smaller N) but covers the common case correctly.
- (if (dstate-getprop dstate +evex-l1+)
- (* disp8 64)
- disp8)))
+ ;; EVEX compressed displacement: with mod=01, the effective
+ ;; displacement is disp8 * N, where N is the instruction's tuple
+ ;; size. N is not encoded in the instruction; it is supplied by
+ ;; the printer's specialized reg/mem arg type, which stores it in
+ ;; SB-DISASSEM::DSTATE-DISP-N slot, before DECODE-MOD-R/M runs.
+ ;;
+ ;; If DSTATE-DISP-N is zero, the instruction was not annotated with
+ ;; a compressed-displacement tuple size. In that case do not scale
+ ;; disp8; leave it as-is rather than guessing a scale (which
+ ;; produces a silently wrong EA).
+ (let ((disp-n (dstate-disp-n dstate)))
+ (if (plusp disp-n)
+ (* disp8 disp-n)
+ disp8))))
(#b10 (read-signed-suffix 32 dstate))))
(extend (bit-name reg)
(logior (if (dstate-getprop dstate bit-name) 8 0) reg)))
diff --git a/tests/simd-pack-512.pure.lisp b/tests/simd-pack-512.pure.lisp
index 932fac407..5a52e1748 100644
--- a/tests/simd-pack-512.pure.lisp
+++ b/tests/simd-pack-512.pure.lisp
@@ -288,12 +288,19 @@
;; evex patch
(cl:in-package "SB-VM")
-(sb-c::defknown %test-evex-high-regs ()
- (unsigned-byte 64)
- (sb-c::flushable sb-c::movable))
-
-(defun %test-evex-high-regs ()
- (error "%test-evex-high-regs stub"))
+(macrolet ((def (name)
+ `(progn
+ (sb-c::defknown ,name ()
+ (unsigned-byte 64)
+ (sb-c::flushable sb-c::movable))
+ (defun ,name ()
+ (error ,(format nil "~A stub" name))))))
+ (def %test-evex-high-regs)
+ (def %test-evex-disp8)
+ (def %test-evex-disp-vector-lengths)
+ (def %test-evex-disp-negative)
+ (def %test-evex-disp-nonmultiple)
+ (def %test-evex-disp-large))
(define-vop (%test-evex-high-regs)
(:translate %test-evex-high-regs)
@@ -307,12 +314,71 @@
(inst vaddps z16 z17 z18)
(inst xor :dword res res)))
+(define-vop (%test-evex-disp8)
+ (:translate %test-evex-disp8)
+ (:policy :fast-safe)
+ (:temporary (:sc single-avx512-reg :offset 0) zmm)
+ (:temporary (:sc unsigned-reg :offset rsp-offset) rsp)
+ (:results (res :scs (unsigned-reg)))
+ (:result-types unsigned-num)
+ (:generator 1
+ (inst xor :dword res res)
+ (inst vmovdqu64 zmm (ea 64 rsp))))
+
+(define-vop (%test-evex-disp-vector-lengths)
+ (:translate %test-evex-disp-vector-lengths)
+ (:policy :fast-safe)
+ (:temporary (:sc single-sse-reg :offset 0) xmm)
+ (:temporary (:sc single-avx2-reg :offset 1) ymm)
+ (:temporary (:sc single-avx512-reg :offset 2) zmm)
+ (:temporary (:sc unsigned-reg :offset rsp-offset) rsp)
+ (:results (res :scs (unsigned-reg)))
+ (:result-types unsigned-num)
+ (:generator 1
+ (inst xor :dword res res)
+ (inst vmovdqu64 xmm (ea 16 rsp))
+ (inst vmovdqu64 ymm (ea 32 rsp))
+ (inst vmovdqu64 zmm (ea 64 rsp))))
+
+(define-vop (%test-evex-disp-negative)
+ (:translate %test-evex-disp-negative)
+ (:policy :fast-safe)
+ (:temporary (:sc single-avx512-reg :offset 0) zmm)
+ (:temporary (:sc unsigned-reg :offset rsp-offset) rsp)
+ (:results (res :scs (unsigned-reg)))
+ (:result-types unsigned-num)
+ (:generator 1
+ (inst xor :dword res res)
+ (inst vmovdqu64 zmm (ea -64 rsp))))
+
+(define-vop (%test-evex-disp-nonmultiple)
+ (:translate %test-evex-disp-nonmultiple)
+ (:policy :fast-safe)
+ (:temporary (:sc single-avx512-reg :offset 0) zmm)
+ (:temporary (:sc unsigned-reg :offset rsp-offset) rsp)
+ (:results (res :scs (unsigned-reg)))
+ (:result-types unsigned-num)
+ (:generator 1
+ (inst xor :dword res res)
+ (inst vmovdqu64 zmm (ea 65 rsp))))
+
+(define-vop (%test-evex-disp-large)
+ (:translate %test-evex-disp-large)
+ (:policy :fast-safe)
+ (:temporary (:sc single-avx512-reg :offset 0) zmm)
+ (:temporary (:sc unsigned-reg :offset rsp-offset) rsp)
+ (:results (res :scs (unsigned-reg)))
+ (:result-types unsigned-num)
+ (:generator 1
+ (inst xor :dword res res)
+ (inst vmovdqu64 zmm (ea 8192 rsp))))
+
(cl:in-package :test-util)
(with-test (:name :evex-high-register-disassembly)
(let* ((fun (compile nil
'(lambda ()
- (sb-vm::%test-evex-high-regs))))
+ (sb-vm::%test-evex-high-regs))))
(text (with-output-to-string (s)
(disassemble fun :stream s))))
;; These names can only appear if the disassembler correctly
@@ -321,4 +387,59 @@
(assert (search "ZMM17" text))
(assert (search "ZMM18" text))
;; Ideally we see a decoded instruction, not raw EVEX bytes.
- (assert (search "VADDPS" text))))
+ (assert (search "VADDPS" text))
+ ;; While development, the decoder was a bit too broad
+ (assert (not (search "VADDPS-MASKED" text)))))
+
+#| tests for evex compressed displacement patch:
+
+ EVEX vector lengths
+ signed negative compressed displacement
+ non-compressible displacement fallback to disp32
+ compressible displacement too large for disp8
+|#
+
+(with-test (:name :evex-compressed-displacement-vector-lengths)
+ (let* ((fun (compile nil
+ '(lambda ()
+ (sb-vm::%test-evex-disp-vector-lengths))))
+ (text (with-output-to-string (s)
+ (disassemble fun :stream s))))
+ (assert (search "VMOVDQU64 XMM0, [RSP+16]" text))
+ (assert (search "VMOVDQU64 YMM1, [RSP+32]" text))
+ (assert (search "VMOVDQU64 ZMM2, [RSP+64]" text))))
+
+(with-test (:name :evex-compressed-displacement)
+ (let* ((fun (compile nil
+ '(lambda ()
+ (sb-vm::%test-evex-disp8))))
+ (text (with-output-to-string (s)
+ (disassemble fun :stream s))))
+ ;; The disassembler must scale EVEX disp8 by 64.
+ (assert (search "VMOVDQU64 ZMM0, [RSP+64]" text))
+ ;; A failure mode is showing the unscaled compressed byte instead:
+ (assert (not (search "[RSP+1]" text)))))
+
+(with-test (:name :evex-compressed-displacement-negative)
+ (let* ((fun (compile nil
+ '(lambda ()
+ (sb-vm::%test-evex-disp-negative))))
+ (text (with-output-to-string (s)
+ (disassemble fun :stream s))))
+ (assert (search "VMOVDQU64 ZMM0, [RSP-64]" text))))
+
+(with-test (:name :evex-compressed-displacement-nonmultiple)
+ (let* ((fun (compile nil
+ '(lambda ()
+ (sb-vm::%test-evex-disp-nonmultiple))))
+ (text (with-output-to-string (s)
+ (disassemble fun :stream s))))
+ (assert (search "VMOVDQU64 ZMM0, [RSP+65]" text))))
+
+(with-test (:name :evex-compressed-displacement-large)
+ (let* ((fun (compile nil
+ '(lambda ()
+ (sb-vm::%test-evex-disp-large))))
+ (text (with-output-to-string (s)
+ (disassemble fun :stream s))))
+ (assert (search "VMOVDQU64 ZMM0, [RSP+8192]" text))))
-----------------------------------------------------------------------
hooks/post-receive
--
SBCL