More evex encoder
arthur miller <[email protected]>
| Newsgroups | gmane.lisp.steel-bank.devel |
|---|---|
| Message-ID | <VI1PR09MB2496DE6AC1D97F4933F07A0096A02@VI1PR09MB2496.eurprd09.prod.outlook.com> |
I rebase my worktree constantly with master, but for some reason
git format-patch master, generated patches for two previous patches
too. I hope these works. All tests pass on my computer, and they
seem to pass on CI too ("CL-host" is still chewing).
If anyone is interested to chim in and help with avx512: more
work on compressed displacement remains, VSIB encoding, lots of
Instructions to add and of course sb-simd support.
_______________________________________________
Sbcl-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/sbcl-devel
0003-Fix-VEX-EVEX-W-handling-for-auto-promoted-instructio.patch
(text/x-patch, 37.5 KB)
From 22fd68c16aaaded85815933119571b7f19308d41 Mon Sep 17 00:00:00 2001 From: arthur <[email protected]> Date: Mon, 24 Aug 2026 04:23:07 +0200 Subject: [PATCH 3/5] Fix VEX/EVEX W handling for auto-promoted instructions Skip conflicting VEX instructions and extend broadcast auto-promotion Disp-n plumbing for auto-promoted EVEX printers Tests for W handling and auto-promoted compressed displacement --- src/compiler/x86-64/avx2-insts.lisp | 302 +++++++++++++++------------- tests/simd-pack-512.pure.lisp | 283 +++++++++++++++++++++++--- 2 files changed, 424 insertions(+), 161 deletions(-) diff --git a/src/compiler/x86-64/avx2-insts.lisp b/src/compiler/x86-64/avx2-insts.lisp index 70ae8d968..3a5eadf72 100644 --- a/src/compiler/x86-64/avx2-insts.lisp +++ b/src/compiler/x86-64/avx2-insts.lisp @@ -716,121 +716,145 @@ REG is the source (encoded in ModR/M.r/m). ;; load/store cases that is fine, but needs more work to fully support sized + ;; compressed displacement (defun avx512-inst-printer-list (inst-format-stem prefix opcode - &key more-fields printer - (opcode-prefix #x0F) - reg-mem-size - xmmreg-mem-size - w - ll - nds - disp-n) + &key more-fields printer + (opcode-prefix #x0F) + reg-mem-size + xmmreg-mem-size + w + ll + nds + disp-n) + (let* ((aa (assoc 'aaa more-fields)) + (aa (if aa (second aa) 0)) + ;; Remove the custom aaa from more-fields so we don't splice it twice. + (more-fields (remove 'aaa more-fields :key #'car)) + (fields `((pp ,(vex-encode-pp prefix)) + (mm ,(evex-encode-mm opcode-prefix)) + (op ,opcode) + (aaa ,aa) + ,@(and w `((w ,w))) + ,@(and ll `((ll ,ll))) + ,@(cond + (disp-n + `((reg/mem nil :type + ',(ecase disp-n + (1 'evex-ymmreg/mem-disp1) + (2 'evex-ymmreg/mem-disp2) + (4 'evex-ymmreg/mem-disp4) + (8 'evex-ymmreg/mem-disp8) + (16 'evex-ymmreg/mem-disp16) + (32 'evex-ymmreg/mem-disp32) + (64 'evex-ymmreg/mem-disp64))))) + (xmmreg-mem-size + `((reg/mem nil :type + ',(case xmmreg-mem-size + (:qword 'sized-xmmreg/mem-default-qword) + (:dword 'sized-dword-xmmreg/mem) + (:word 'sized-word-xmmreg/mem) + (:byte 'sized-byte-xmmreg/mem) + (:sized 'sized-xmmreg/mem))))) + (reg-mem-size + `((reg/mem nil :type + ',(case reg-mem-size + (:qword 'sized-reg/mem-default-qword) + (:dword 'sized-dword-reg/mem) + (:word 'sized-word-reg/mem) + (:byte 'sized-byte-reg/mem) + (:sized 'sized-reg/mem)))))) + ,@more-fields)) + (inst-format (symbolicate "EVEX-" inst-format-stem))) + (list `(:printer ,inst-format ,fields + ,@(cond (printer + `(',printer)) + ((eq nds 'to-mem) + `('(:name :tab reg/mem ", " vvvv ", " reg))) + (nds + `('(:name :tab reg ", " vvvv ", " reg/mem)))))))) + + (defun avx2-inst-printer-list (inst-format-stem prefix opcode + &key more-fields printer + (opcode-prefix #x0F) + reg-mem-size + xmmreg-mem-size + w + l + nds + disp-n) (let ((fields `((pp ,(vex-encode-pp prefix)) - (mm ,(evex-encode-mm opcode-prefix)) + (m-mmmm ,(vex-encode-m-mmmm opcode-prefix)) (op ,opcode) ,@(and w `((w ,w))) - ,@(and ll `((ll ,ll))) - ,@(cond - (disp-n - `((reg/mem nil :type - ',(ecase disp-n - (1 'evex-ymmreg/mem-disp1) - (2 'evex-ymmreg/mem-disp2) - (4 'evex-ymmreg/mem-disp4) - (8 'evex-ymmreg/mem-disp8) - (16 'evex-ymmreg/mem-disp16) - (32 'evex-ymmreg/mem-disp32) - (64 'evex-ymmreg/mem-disp64))))) - (xmmreg-mem-size - `((reg/mem nil :type - ',(case xmmreg-mem-size - (:qword 'sized-xmmreg/mem-default-qword) - (:dword 'sized-dword-xmmreg/mem) - (:word 'sized-word-xmmreg/mem) - (:byte 'sized-byte-xmmreg/mem) - (:sized 'sized-xmmreg/mem))))) - (reg-mem-size - `((reg/mem nil :type - ',(case reg-mem-size - (:qword 'sized-reg/mem-default-qword) - (:dword 'sized-dword-reg/mem) - (:word 'sized-word-reg/mem) - (:byte 'sized-byte-reg/mem) - (:sized 'sized-reg/mem)))))) + ,@(and l `((l ,l))) + ,@(cond (xmmreg-mem-size + `((reg/mem nil :type + ',(case xmmreg-mem-size + (:qword 'sized-xmmreg/mem-default-qword) + (:dword 'sized-dword-xmmreg/mem) + (:word 'sized-word-xmmreg/mem) + (:byte 'sized-byte-xmmreg/mem) + (:sized 'sized-xmmreg/mem))))) + (reg-mem-size + `((reg/mem nil :type + ',(case reg-mem-size + (:qword 'sized-reg/mem-default-qword) + (:dword 'sized-dword-reg/mem) + (:word 'sized-word-reg/mem) + (:byte 'sized-byte-reg/mem) + (:sized 'sized-reg/mem)))))) ,@more-fields)) - (inst-format (symbolicate "EVEX-" inst-format-stem))) - (list `(:printer ,inst-format ,fields - ,@(cond (printer - `(',printer)) - ((eq nds 'to-mem) - `('(:name :tab reg/mem ", " vvvv ", " reg))) - (nds - `('(:name :tab reg ", " vvvv ", " reg/mem)))))))) - - (defun avx2-inst-printer-list (inst-format-stem prefix opcode - &key more-fields printer - (opcode-prefix #x0F) - reg-mem-size - xmmreg-mem-size - w - l - nds) - (let ((fields `((pp ,(vex-encode-pp prefix)) - (m-mmmm ,(vex-encode-m-mmmm opcode-prefix)) - (op ,opcode) - ,@(and w `((w ,w))) - ,@(and l `((l ,l))) - ,@(cond (xmmreg-mem-size - `((reg/mem nil :type ',(case xmmreg-mem-size - (:qword 'sized-xmmreg/mem-default-qword) - (:dword 'sized-dword-xmmreg/mem) - (:word 'sized-word-xmmreg/mem) - (:byte 'sized-byte-xmmreg/mem) - (:sized 'sized-xmmreg/mem))))) - (reg-mem-size - `((reg/mem nil :type ',(case reg-mem-size - (:qword 'sized-reg/mem-default-qword) - (:dword 'sized-dword-reg/mem) - (:word 'sized-word-reg/mem) - (:byte 'sized-byte-reg/mem) - (:sized 'sized-reg/mem)))))) - ,@more-fields)) - (inst-formats (if (or (eql w 1) - (/= opcode-prefix #x0F)) - (list (symbolicate "VEX3-" inst-format-stem)) - (list (symbolicate "VEX2-" inst-format-stem) - (symbolicate "VEX3-" inst-format-stem))))) - (append - (mapcar (lambda (inst-format) - `(:printer ,inst-format ,fields - ,@(cond (printer - `(',printer)) - ((eq nds 'to-mem) - `('(:name :tab reg/mem ", " vvvv ", " reg))) - (nds - `('(:name :tab reg ", " vvvv ", " reg/mem)))))) - inst-formats) - ;; Generate EVEX printer entries so VEX instructions auto-promoted - ;; to EVEX can be disassembled. Map 0F is always safe (no EVEX-only - ;; instructions reuse those opcodes). Map 0F38 has many conflicts - ;; (broadcasts, vmaskmov vs vscalef, etc.), so we only include the - ;; FMA range (#x96-#xBF) which is safe. Map 0F3A is skipped entirely. - (when (or (= opcode-prefix #x0F) - (and (= opcode-prefix #x0F38) - (<= #x96 opcode #xbf))) + (inst-formats (if (or (eql w 1) + (/= opcode-prefix #x0F)) + (list (symbolicate "VEX3-" inst-format-stem)) + (list (symbolicate "VEX2-" inst-format-stem) + (symbolicate "VEX3-" inst-format-stem))))) + (append + (mapcar (lambda (inst-format) + `(:printer ,inst-format ,fields + ,@(cond (printer + `(',printer)) + ((eq nds 'to-mem) + `('(:name :tab reg/mem ", " vvvv ", " reg))) + (nds + `('(:name :tab reg ", " vvvv ", " reg/mem)))))) + inst-formats) + + ;; Auto-generated EVEX printers for safe 1:1 VEX->EVEX forms. + ;; Skip VEX instructions whose EVEX forms have different names + ;; selected by W: VPAND/VPANDD, VPANDN/VPANDND, etc. + ;; All of those are already emitted explicitly in avx512-insts.lisp + (let ((skip-list '((#x66 #xdb) ; VPAND + (#x66 #xdf) ; VPANDN + (#x66 #xeb) ; VPOR + (#x66 #xef) ; VPXOR + (#xf3 #xe6) ; VCVTDQ2PD/VCVTQQ2PD + (#x66 #x6f) ; VMOVDQA / VMOVDQA32 + (#x66 #x7f) ; VMOVDQA / VMOVDQA32 store + (#xf3 #x6f) ; VMOVDQU / VMOVDQU32 load + (#xf3 #x7f) ; VMOVDQU / VMOVDQU32 storee + (#x66 #x1a) ; VBROADCASTF128 vs VBROADCASTF32X4 + (#x66 #x5a)))) ; VBROADCASTI128 vs VBROADCASTI32X4 + (when (and (not (member (list prefix opcode) skip-list :test #'equal)) + (not (assoc 'reg/mem more-fields)) + (or (= opcode-prefix #x0F) + (and (= opcode-prefix #x0F38) + (or (<= #x96 opcode #xbf) + ;; +evex-0f38-broadcast-opcodes+ + (find opcode #(#x18 #x19 #x1A #x58 #x59 #x5A #x78 #x79)))))) (avx512-inst-printer-list inst-format-stem prefix opcode :more-fields more-fields :printer printer :opcode-prefix opcode-prefix - :reg-mem-size reg-mem-size - :xmmreg-mem-size xmmreg-mem-size - :w w - :nds nds)))))) + :w (or w 0) + :ll #b10 + :disp-n disp-n + :nds nds))))))) (macrolet ((def (name opcode /i) `(define-instruction ,name (segment dst src imm) ,@(avx2-inst-printer-list 'ymm-ymm-imm #x66 opcode - :more-fields `((/i ,/i))) + :more-fields `((/i ,/i)) + :w 0) (:emitter (emit-avx2-inst-imm segment dst src imm #x66 ,opcode ,/i))))) @@ -840,16 +864,15 @@ REG is the source (encoded in ModR/M.r/m). (macrolet ((def (name opcode vopcode /i &optional (evex-w 0)) `(define-instruction ,name (segment dst src src2/imm) - ,@(avx2-inst-printer-list 'ymm-ymm-imm #x66 opcode + ,@(avx2-inst-printer-list 'ymm-ymm-imm #x66 opcode :w evex-w :more-fields `((/i ,/i))) - ,@(avx2-inst-printer-list 'ymm-ymm/mem #x66 vopcode :nds t) + ,@(avx2-inst-printer-list 'ymm-ymm/mem #x66 vopcode :nds t :w evex-w) (:emitter (if (integerp src2/imm) (emit-avx2-inst-imm segment dst src src2/imm - #x66 ,opcode ,/i :evex-w ,evex-w) + #x66 ,opcode ,/i :evex-w ,evex-w :w ,evex-w) (emit-avx2-inst segment src2/imm dst #x66 ,vopcode - :evex-w ,evex-w - :vvvv src)))))) + :evex-w ,evex-w :w ,evex-w :vvvv src)))))) (def vpsllw #x71 #xf1 6) (def vpslld #x72 #xf2 6) (def vpsllq #x73 #xf3 6 1) ; evex-w=1 for EVEX qword @@ -864,11 +887,13 @@ REG is the source (encoded in ModR/M.r/m). (macrolet ((def (name prefix opcode &optional (opcode-prefix #x0F) (evex-w 0)) `(define-instruction ,name (segment dst src src2) ,@(avx2-inst-printer-list 'ymm-ymm/mem prefix opcode :nds t - :opcode-prefix opcode-prefix) + :opcode-prefix opcode-prefix + :w evex-w) (:emitter (emit-avx2-inst segment src2 dst ,prefix ,opcode :opcode-prefix ,opcode-prefix :evex-w ,evex-w + :w ,evex-w :vvvv src))))) ;; logical (def vandpd #x66 #x54 #x0F 1) ; evex-w=1 for double-precision @@ -1017,14 +1042,16 @@ REG is the source (encoded in ModR/M.r/m). `(define-instruction ,name (segment dst src) ,@(avx2-inst-printer-list 'ymm-ymm/mem prefix opcode :opcode-prefix opcode-prefix + :w evex-w :more-fields (and (eq l :from-thing) '((reg nil :type 'xmmreg)))) (:emitter (emit-avx2-inst segment src dst ,prefix ,opcode :opcode-prefix ,opcode-prefix :evex-w ,evex-w + :w ,evex-w ,@(and l - `(:l ,l))))))) + `(:l ,l))))))) ;; moves (def vmovshdup #xf3 #x16) (def vmovsldup #xf3 #x12) @@ -1071,11 +1098,12 @@ REG is the source (encoded in ModR/M.r/m). (def vpmovzxwq #x66 #x34 #x0f38) (def vpmovzxdq #x66 #x35 #x0f38)) -(macrolet ((def (name prefix) +(macrolet ((def (name prefix &key (evex-w 0)) `(define-instruction ,name (segment dst src pattern) ,@(avx2-inst-printer-list 'ymm-ymm/mem-imm prefix #x70 - :printer '(:name :tab reg ", " reg/mem ", " imm)) + :printer '(:name :tab reg ", " reg/mem ", " imm) + :w evex-w) (:emitter (emit-avx2-inst segment src dst ,prefix #x70 :remaining-bytes 1) @@ -1084,16 +1112,16 @@ REG is the source (encoded in ModR/M.r/m). (def vpshufhw #xf3) (def vpshuflw #xf2)) -(macrolet ((def (name prefix) +(macrolet ((def (name prefix &key (evex-w 0)) `(define-instruction ,name (segment dst src src2 pattern) ,@(avx2-inst-printer-list - 'ymm-ymm/mem-imm prefix #xc6) + 'ymm-ymm/mem-imm prefix #xc6 :w evex-w) (:emitter (emit-avx2-inst segment src2 dst ,prefix #xc6 :vvvv src :remaining-bytes 1) (emit-byte segment pattern))))) - (def vshufpd #x66) + (def vshufpd #x66 :evex-w 1) (def vshufps nil)) (macrolet @@ -1140,10 +1168,12 @@ REG is the source (encoded in ModR/M.r/m). ,@(avx2-inst-printer-list 'ymm-ymm/mem-imm prefix opcode :more-fields `((imm nil :type 'avx-condition-code)) :printer `("VCMP" imm ,name-suffix - :tab reg ", " vvvv ", " reg/mem)) + :tab reg ", " vvvv ", " reg/mem) + :w evex-w) (:emitter (emit-avx2-inst segment src2 dst ,prefix ,opcode :evex-w ,evex-w + :w ,evex-w :vvvv src :remaining-bytes 1) (emit-byte segment (or (position condition +avx-conditions+) @@ -1173,12 +1203,12 @@ REG is the source (encoded in ModR/M.r/m). (def vblendvpd #x66 #x4B)) (macrolet ((def (name prefix opcode-from opcode-to - &key force-to-mem - reg-reg-name - l - (opcode-prefix #x0F) - (evex-w 0) - nds) + &key force-to-mem + reg-reg-name + l + (opcode-prefix #x0F) + (evex-w 0) + nds) `(progn ,(when reg-reg-name `(define-instruction ,reg-reg-name (segment dst src ,@(if nds '(src2))) @@ -1199,12 +1229,14 @@ REG is the source (encoded in ModR/M.r/m). ,@(when opcode-from (avx2-inst-printer-list 'ymm-ymm/mem prefix opcode-from :opcode-prefix opcode-prefix - :nds nds)) + :nds nds + :w evex-w)) ,@(when opcode-to (avx2-inst-printer-list 'ymm-ymm/mem prefix opcode-to :printer '(:name :tab reg/mem ", " reg) - :opcode-prefix opcode-prefix)) + :opcode-prefix opcode-prefix + :w evex-w)) (:emitter ,@(when nds `((aver (register-p src)))) @@ -1221,6 +1253,7 @@ REG is the source (encoded in ModR/M.r/m). ,prefix ,opcode-from :opcode-prefix ,opcode-prefix :evex-w ,evex-w + :w ,evex-w ,@(and nds `(:vvvv src)) :l ,l)))) @@ -1233,6 +1266,7 @@ REG is the source (encoded in ModR/M.r/m). ,prefix ,opcode-to :opcode-prefix ,opcode-prefix :evex-w ,evex-w + :w ,evex-w :l ,l)))))))) ;; direction bit? (def vmovapd #x66 #x28 #x29 :evex-w 1) @@ -1243,10 +1277,10 @@ REG is the source (encoded in ModR/M.r/m). (def vmovups nil #x10 #x11) ;; streaming - (def vmovntdq #x66 nil #xe7 :force-to-mem t) + (def vmovntdq #x66 nil #xe7 :force-to-mem t) (def vmovntdqa #x66 #x2a nil :force-to-mem t :opcode-prefix #x0F38) - (def vmovntpd #x66 nil #x2b :force-to-mem t :evex-w 1) - (def vmovntps nil nil #x2b :force-to-mem t) + (def vmovntpd #x66 nil #x2b :force-to-mem t :evex-w 1) + (def vmovntps nil nil #x2b :force-to-mem t) ;; use vmovhps for vmovlhps and vmovlps for vmovhlps (def vmovhpd #x66 #x16 #x17 :force-to-mem t :l 0 :nds t) @@ -1262,7 +1296,6 @@ REG is the source (encoded in ModR/M.r/m). . #.(avx2-inst-printer-list 'ymm-ymm/mem #xf2 #xf0)) - (macrolet ((def (name prefix) `(define-instruction ,name (segment dst src &optional src2) ,@(avx2-inst-printer-list 'ymm-ymm/mem-dir prefix #b0001000) @@ -1436,19 +1469,19 @@ REG is the source (encoded in ModR/M.r/m). ,@(avx2-inst-printer-list 'ymm-ymm/mem #x66 opcode :opcode-prefix #x0f38 :xmmreg-mem-size mem-size - :w 0 :l l) + :w evex-w :l l) (:emitter (emit-avx2-inst segment src dst #x66 ,opcode :opcode-prefix #x0f38 :evex-w ,evex-w :l ,l))))) - (def vbroadcastss #x18 nil :dword) - (def vbroadcastsd #x19 1) + (def vbroadcastss #x18 nil :dword) + (def vbroadcastsd #x19 1) (def vbroadcastf128 #x1a 1) (def vbroadcasti128 #x5a 1) - (def vpbroadcastb #x78 nil :byte) - (def vpbroadcastw #x79 nil :word) - (def vpbroadcastd #x58 nil :dword) - (def vpbroadcastq #x59 nil :qword 1)) ; evex-w=1 for EVEX qword + (def vpbroadcastb #x78 nil :byte) + (def vpbroadcastw #x79 nil :word) + (def vpbroadcastd #x58 nil :dword) + (def vpbroadcastq #x59 nil :qword 1)) ; evex-w=1 for EVEX qword (macrolet ((def-insert (name prefix op) `(define-instruction ,name (segment dst src src2 imm) @@ -1896,4 +1929,3 @@ REG is the source (encoded in ModR/M.r/m). (def blsr 1) (def blsmsk 2) (def blsi 3)) - diff --git a/tests/simd-pack-512.pure.lisp b/tests/simd-pack-512.pure.lisp index 5a52e1748..097843ca9 100644 --- a/tests/simd-pack-512.pure.lisp +++ b/tests/simd-pack-512.pure.lisp @@ -300,7 +300,17 @@ (def %test-evex-disp-vector-lengths) (def %test-evex-disp-negative) (def %test-evex-disp-nonmultiple) - (def %test-evex-disp-large)) + (def %test-evex-disp-large) + (def %test-auto-promoted-vaddps) + (def %test-auto-promoted-vaddpd) + (def %test-auto-promoted-vpbroadcastq) + (def %test-auto-promoted-vmovdqu) + (def %test-evex-vpternlogd) + (def %test-evex-vpermt2d) + (def %test-evex-vblendmps) + (def %test-evex-vpcmpd) + (def %test-evex-vpmovqd) + (def %test-auto-promoted-vmovaps-disp8)) (define-vop (%test-evex-high-regs) (:translate %test-evex-high-regs) @@ -373,6 +383,125 @@ (inst xor :dword res res) (inst vmovdqu64 zmm (ea 8192 rsp)))) + +(define-vop (%test-auto-promoted-vaddps) + (:translate %test-auto-promoted-vaddps) + (:policy :fast-safe) + (:temporary (:sc single-avx512-reg :offset 16) z16) + (:temporary (:sc single-avx512-reg :offset 17) z17) + (:temporary (:sc single-avx512-reg :offset 18) z18) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst vaddps z16 z17 z18) + (inst xor :dword res res))) + +;; needed for testing w-bit with auto-promoted instructions +(define-vop (%test-auto-promoted-vaddpd) + (:translate %test-auto-promoted-vaddpd) + (:policy :fast-safe) + (:temporary (:sc double-avx512-reg :offset 16) z16) + (:temporary (:sc double-avx512-reg :offset 17) z17) + (:temporary (:sc double-avx512-reg :offset 18) z18) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst vaddpd z16 z17 z18) + (inst xor :dword res res))) + +(define-vop (%test-auto-promoted-vpbroadcastq) + (:translate %test-auto-promoted-vpbroadcastq) + (:policy :fast-safe) + (:temporary (:sc int-avx512-reg :offset 0) zmm) + (:temporary (:sc unsigned-reg :offset rsp-offset) rsp) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst xor :dword res res) + (inst vpbroadcastq zmm (ea 64 rsp)))) + +(define-vop (%test-auto-promoted-vmovdqu) + (:translate %test-auto-promoted-vmovdqu) + (:policy :fast-safe) + (:temporary (:sc single-avx512-reg :offset 0) zmm) + (:temporary (:sc unsigned-reg :offset rsp-offset) rsp) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst xor :dword res res) + (inst vmovdqu zmm (ea 64 rsp)))) + +(define-vop (%test-evex-vpternlogd) + (:translate %test-evex-vpternlogd) + (:policy :fast-safe) + (:temporary (:sc int-avx512-reg :offset 0) z0) + (:temporary (:sc int-avx512-reg :offset 1) z1) + (:temporary (:sc int-avx512-reg :offset 2) z2) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst vpternlogd z0 z1 z2 #xFF) + (inst xor :dword res res))) + +;; needed for evex emitters and printers +(define-vop (%test-evex-vpermt2d) + (:translate %test-evex-vpermt2d) + (:policy :fast-safe) + (:temporary (:sc int-avx512-reg :offset 0) z0) + (:temporary (:sc int-avx512-reg :offset 1) z1) + (:temporary (:sc int-avx512-reg :offset 2) z2) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst vpermt2d z0 z1 z2) + (inst xor :dword res res))) + +(define-vop (%test-evex-vblendmps) + (:translate %test-evex-vblendmps) + (:policy :fast-safe) + (:temporary (:sc single-avx512-reg :offset 16) z16) + (:temporary (:sc single-avx512-reg :offset 17) z17) + (:temporary (:sc single-avx512-reg :offset 18) z18) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst vblendmps z16 z17 z18) + (inst xor :dword res res))) + +(define-vop (%test-evex-vpcmpd) + (:translate %test-evex-vpcmpd) + (:policy :fast-safe) + (:temporary (:sc mask-reg :offset 1) k1) + (:temporary (:sc int-avx512-reg :offset 0) z0) + (:temporary (:sc int-avx512-reg :offset 1) z1) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst vpcmpd k1 z0 z1 #x1) + (inst xor :dword res res))) + +(define-vop (%test-evex-vpmovqd) + (:translate %test-evex-vpmovqd) + (:policy :fast-safe) + (:temporary (:sc int-avx512-reg :offset 0) z0) + (:temporary (:sc unsigned-reg :offset rsp-offset) rsp) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst vpmovqd (ea 32 rsp) z0) + (inst xor :dword res res))) + +(define-vop (%test-auto-promoted-vmovaps-disp8) + (:translate %test-auto-promoted-vmovaps-disp8) + (:policy :fast-safe) + (:temporary (:sc single-avx512-reg :offset 0) zmm) + (:temporary (:sc unsigned-reg :offset rsp-offset) rsp) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst vmovaps zmm (ea 64 rsp)) + (inst xor :dword res res))) + (cl:in-package :test-util) (with-test (:name :evex-high-register-disassembly) @@ -400,46 +529,148 @@ |# (with-test (:name :evex-compressed-displacement-vector-lengths) - (let* ((fun (compile nil - '(lambda () - (sb-vm::%test-evex-disp-vector-lengths)))) - (text (with-output-to-string (s) - (disassemble fun :stream s)))) + (let* ((func (compile nil '(lambda () (sb-vm::%test-evex-disp-vector-lengths)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) (assert (search "VMOVDQU64 XMM0, [RSP+16]" text)) (assert (search "VMOVDQU64 YMM1, [RSP+32]" text)) (assert (search "VMOVDQU64 ZMM2, [RSP+64]" text)))) (with-test (:name :evex-compressed-displacement) - (let* ((fun (compile nil - '(lambda () - (sb-vm::%test-evex-disp8)))) - (text (with-output-to-string (s) - (disassemble fun :stream s)))) + (let* ((func (compile nil '(lambda () (sb-vm::%test-evex-disp8)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) ;; The disassembler must scale EVEX disp8 by 64. (assert (search "VMOVDQU64 ZMM0, [RSP+64]" text)) ;; A failure mode is showing the unscaled compressed byte instead: (assert (not (search "[RSP+1]" text))))) (with-test (:name :evex-compressed-displacement-negative) - (let* ((fun (compile nil - '(lambda () - (sb-vm::%test-evex-disp-negative)))) - (text (with-output-to-string (s) - (disassemble fun :stream s)))) + (let* ((func (compile nil '(lambda () (sb-vm::%test-evex-disp-negative)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) (assert (search "VMOVDQU64 ZMM0, [RSP-64]" text)))) (with-test (:name :evex-compressed-displacement-nonmultiple) - (let* ((fun (compile nil - '(lambda () - (sb-vm::%test-evex-disp-nonmultiple)))) - (text (with-output-to-string (s) - (disassemble fun :stream s)))) + (let* ((func (compile nil '(lambda () (sb-vm::%test-evex-disp-nonmultiple)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) (assert (search "VMOVDQU64 ZMM0, [RSP+65]" text)))) (with-test (:name :evex-compressed-displacement-large) - (let* ((fun (compile nil - '(lambda () - (sb-vm::%test-evex-disp-large)))) - (text (with-output-to-string (s) - (disassemble fun :stream s)))) + (let* ((func (compile nil '(lambda () (sb-vm::%test-evex-disp-large)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) (assert (search "VMOVDQU64 ZMM0, [RSP+8192]" text)))) + +;; call avx2-inst-printer-list for vmovdqu and assert it is restricted to +;; L'L = 512 (ll = 2), uses disp-n = 64 => selects evex-ymmreg/mem-disp64. +(with-test (:name :auto-promoted-evex-disp8-printer) + (let* ((asm-pkg (find-package "SB-X86-64-ASM")) + (printer-fun (find-symbol "AVX2-INST-PRINTER-LIST" asm-pkg)) + (inst-format (find-symbol "YMM-YMM/MEM" asm-pkg)) + (disp64 (find-symbol "EVEX-YMMREG/MEM-DISP64" asm-pkg))) + (when (and printer-fun inst-format disp64) + (let* ((printer-forms + (funcall printer-fun + inst-format + #xf3 ; prefix + #x6f ; opcode + :opcode-prefix #x0f + :w 0)) + (evex-form + (find-if (lambda (form) + (and (eq (first form) :printer) + (let ((name (second form))) + (and (symbolp name) + (search "EVEX-" (symbol-name name)))))) + printer-forms))) + (assert evex-form) + (let ((fields (third evex-form))) + ;; Auto-promoted EVEX forms are ZMM-only, so L'L = #b10. + (assert (equal (second (assoc 'll fields)) 2)) + ;; The full-vector ZMM memory form uses compressed displacement N=64. + (assert (eq (third (assoc 'reg/mem fields)) disp64))))))) + +(with-test (:name :auto-promoted-evex-vaddps-disasm) + (let* ((func (compile nil '(lambda () (sb-vm::%test-auto-promoted-vaddps)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) + (assert (search "VADDPS" text)) + (assert (search "ZMM16" text)) + (assert (search "ZMM17" text)) + (assert (search "ZMM18" text)) + ;; Must not be confused with the W=1 double-precision form + (assert (not (search "VADDPD" text))))) + +(with-test (:name :auto-promoted-evex-vaddpd-disasm) + (let* ((func (compile nil '(lambda () (sb-vm::%test-auto-promoted-vaddpd)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) + (assert (search "VADDPD" text)) + (assert (search "ZMM16" text)) + (assert (search "ZMM17" text)) + (assert (search "ZMM18" text)) + ;; Must not be confused with the W=0 single-precision form + (assert (not (search "VADDPS" text))))) + +(with-test (:name :auto-promoted-evex-vpbroadcastq-disasm) + (let* ((func (compile nil '(lambda () (sb-vm::%test-auto-promoted-vpbroadcastq)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) + (assert (search "VPBROADCASTQ" text)) + (assert (search "ZMM0" text)) + ;; The EVEX form should use W=1 (qword broadcast) + ;; This is indirectly checked by correct mnemonic. + (assert (search "[RSP+64]" text)))) + +(with-test (:name :auto-promoted-vmovdqu-skip-conflict-disasm) + ;; VMOVDQU (VEX, prefix #xF3) auto-promotes to EVEX ZMM. + ;; Because the explicit VMOVDQU32 is skipped, the disassembler + ;; should still choose a valid printer, not signal a conflict. + (let* ((func (compile nil '(lambda () (sb-vm::%test-auto-promoted-vmovdqu)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) + ;; The actual mnemonic is VMOVDQU32 because that is the official + ;; AVX512 name, and the printer for the explicit EVEX instruction wins. + (assert (search "VMOVDQU32" text)) + (assert (search "ZMM0" text)) + (assert (search "[RSP+64]" text)))) + +(with-test (:name :evex-explicit-vpternlogd-disasm) + (let* ((func (compile nil '(lambda () (sb-vm::%test-evex-vpternlogd)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) + (assert (search "VPTERNLOGD" text)) + (assert (search "ZMM0" text)) + (assert (search "ZMM1" text)) + (assert (search "ZMM2" text)) + (assert (search "255" text)))) + +(with-test (:name :evex-explicit-vpermt2d-disasm) + (let* ((func (compile nil '(lambda () (sb-vm::%test-evex-vpermt2d)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) + (assert (search "VPERMT2D" text)) + (assert (search "ZMM0" text)) + (assert (search "ZMM1" text)) + (assert (search "ZMM2" text)))) + +(with-test (:name :evex-explicit-vblendmps-disasm) + (let* ((func (compile nil '(lambda () (sb-vm::%test-evex-vblendmps)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) + (assert (search "VBLENDMPS" text)) + (assert (search "ZMM16" text)) + (assert (search "ZMM17" text)) + (assert (search "ZMM18" text)))) + +(with-test (:name :evex-explicit-vpcmpd-disasm) + (let* ((func (compile nil '(lambda () (sb-vm::%test-evex-vpcmpd)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) + (assert (search "VPCMPD" text)) + (assert (search "K1" text)) + (assert (search "ZMM0" text)) + (assert (search "ZMM1" text)))) + +(with-test (:name :evex-explicit-vpmovqd-disasm) + (let* ((func (compile nil '(lambda () (sb-vm::%test-evex-vpmovqd)))) + (text (with-output-to-string (s) (disassemble func :stream s)))) + (assert (search "VPMOVQD" text)) + (assert (search "[RSP+32]" text)) + (assert (search "ZMM0" text)))) + +(with-test (:name :auto-promoted-evex-vmovaps-disp8-disasm) + (let* ((fun (compile nil '(lambda () (sb-vm::%test-auto-promoted-vmovaps-disp8)))) + (text (with-output-to-string (s) (disassemble fun :stream s)))) + (assert (search "VMOVAPS" text)) + (assert (search "ZMM0" text)) + (assert (search "[RSP+64]" text)))) -- 2.55.0
0004-Add-compressed-displacement-for-auto-promoted-full-v.patch
(text/x-patch, 4.3 KB)
From 09c32b57583f20c2cd90e4604104aad55d79db80 Mon Sep 17 00:00:00 2001 From: arthur <[email protected]> Date: Mon, 24 Aug 2026 04:52:23 +0200 Subject: [PATCH 4/5] Add compressed displacement for auto-promoted full-vector moves * Add disp-n to the direction-bit move printer macro * Set disp-n 64 for safe full-vector auto-promoted forms (vmovaps, vmovapd, vmovups, vmovupd, vmovnt*) * Another printer test --- src/compiler/x86-64/avx2-insts.lisp | 22 ++++++++++++---------- tests/simd-pack-512.pure.lisp | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/compiler/x86-64/avx2-insts.lisp b/src/compiler/x86-64/avx2-insts.lisp index 3a5eadf72..ad1e9c26d 100644 --- a/src/compiler/x86-64/avx2-insts.lisp +++ b/src/compiler/x86-64/avx2-insts.lisp @@ -1208,7 +1208,8 @@ REG is the source (encoded in ModR/M.r/m). l (opcode-prefix #x0F) (evex-w 0) - nds) + nds + disp-n) `(progn ,(when reg-reg-name `(define-instruction ,reg-reg-name (segment dst src ,@(if nds '(src2))) @@ -1236,7 +1237,8 @@ REG is the source (encoded in ModR/M.r/m). 'ymm-ymm/mem prefix opcode-to :printer '(:name :tab reg/mem ", " reg) :opcode-prefix opcode-prefix - :w evex-w)) + :w evex-w + :disp-n disp-n)) (:emitter ,@(when nds `((aver (register-p src)))) @@ -1269,18 +1271,18 @@ REG is the source (encoded in ModR/M.r/m). :w ,evex-w :l ,l)))))))) ;; direction bit? - (def vmovapd #x66 #x28 #x29 :evex-w 1) - (def vmovaps nil #x28 #x29) + (def vmovapd #x66 #x28 #x29 :evex-w 1 :disp-n 64) + (def vmovaps nil #x28 #x29 :disp-n 64) (def vmovdqa #x66 #x6f #x7f) (def vmovdqu #xf3 #x6f #x7f) - (def vmovupd #x66 #x10 #x11 :evex-w 1) - (def vmovups nil #x10 #x11) + (def vmovupd #x66 #x10 #x11 :evex-w 1 :disp-n 64) + (def vmovups nil #x10 #x11 :disp-n 64) ;; streaming - (def vmovntdq #x66 nil #xe7 :force-to-mem t) - (def vmovntdqa #x66 #x2a nil :force-to-mem t :opcode-prefix #x0F38) - (def vmovntpd #x66 nil #x2b :force-to-mem t :evex-w 1) - (def vmovntps nil nil #x2b :force-to-mem t) + (def vmovntdq #x66 nil #xe7 :force-to-mem t :disp-n 64) + (def vmovntdqa #x66 #x2a nil :force-to-mem t :opcode-prefix #x0F38 :disp-n 64) + (def vmovntpd #x66 nil #x2b :force-to-mem t :evex-w 1 :disp-n 64) + (def vmovntps nil nil #x2b :force-to-mem t :disp-n 64) ;; use vmovhps for vmovlhps and vmovlps for vmovhlps (def vmovhpd #x66 #x16 #x17 :force-to-mem t :l 0 :nds t) diff --git a/tests/simd-pack-512.pure.lisp b/tests/simd-pack-512.pure.lisp index 097843ca9..5f5be8d36 100644 --- a/tests/simd-pack-512.pure.lisp +++ b/tests/simd-pack-512.pure.lisp @@ -674,3 +674,28 @@ (assert (search "VMOVAPS" text)) (assert (search "ZMM0" text)) (assert (search "[RSP+64]" text)))) + +(with-test (:name :auto-promoted-vmovaps-disp8-printer) + (let* ((asm-pkg (find-package "SB-X86-64-ASM")) + (printer-fun (find-symbol "AVX2-INST-PRINTER-LIST" asm-pkg)) + (inst-format (find-symbol "YMM-YMM/MEM" asm-pkg)) + (disp64 (find-symbol "EVEX-YMMREG/MEM-DISP64" asm-pkg))) + (when (and printer-fun inst-format disp64) + (let* ((printer-forms + (funcall printer-fun + inst-format + nil ; prefix for vmovaps + #x28 ; opcode-from + :opcode-prefix #x0f + :w 0 + :disp-n 64)) + (evex-form + (find-if (lambda (form) + (and (eq (first form) :printer) + (let ((name (second form))) + (and (symbolp name) + (search "EVEX-" (symbol-name name)))))) + printer-forms))) + (assert evex-form) + (let ((fields (third evex-form))) + (assert (eq (third (assoc 'reg/mem fields)) disp64))))))) -- 2.55.0
0005-More-compressed-displacement-support-for-auto-promot.patch
(text/x-patch, 6 KB)
From 05578c120c578dcabc27c61f152f5ac610c1b27e Mon Sep 17 00:00:00 2001 From: arthur <[email protected]> Date: Mon, 24 Aug 2026 05:21:24 +0200 Subject: [PATCH 5/5] More compressed displacement support for auto-promoted full-vector moves Pass disp-n through auto-promoted EVEX emitter path Set disp-n based on vector width for full-vector move forms Add tests for vmovaps disp8 and disp32 fallback --- src/compiler/x86-64/avx2-insts.lisp | 16 +++++++++--- tests/simd-pack-512.pure.lisp | 38 ++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/compiler/x86-64/avx2-insts.lisp b/src/compiler/x86-64/avx2-insts.lisp index ad1e9c26d..8f762251f 100644 --- a/src/compiler/x86-64/avx2-insts.lisp +++ b/src/compiler/x86-64/avx2-insts.lisp @@ -641,7 +641,8 @@ produces silently wrong addresses." evex-w vvvv is4 - vm) + vm + (disp-n 0)) ;; Auto-detect ZMM operands and delegate to EVEX encoding (when (or (and (register-p reg) (is-zmm-id-p (reg-id reg))) (and (register-p thing) (is-zmm-id-p (reg-id thing))) @@ -659,7 +660,7 @@ produces silently wrong addresses." ;; Force disp32 for auto-promoted VEX instructions: ;; the correct N depends on tuple type which varies ;; per instruction. disp-n=0 disables disp8 entirely. - :disp-n 0))) + :disp-n disp-n))) (emit-vex segment vvvv thing reg prefix opcode-prefix l w) (emit-bytes segment opcode) (when is4 @@ -1231,7 +1232,8 @@ REG is the source (encoded in ModR/M.r/m). (avx2-inst-printer-list 'ymm-ymm/mem prefix opcode-from :opcode-prefix opcode-prefix :nds nds - :w evex-w)) + :w evex-w + :disp-n disp-n)) ,@(when opcode-to (avx2-inst-printer-list 'ymm-ymm/mem prefix opcode-to @@ -1256,6 +1258,10 @@ REG is the source (encoded in ModR/M.r/m). :opcode-prefix ,opcode-prefix :evex-w ,evex-w :w ,evex-w + :disp-n (cond ((zmm-register-p dst) 64) + ((ymm-register-p dst) 32) + ((xmm-register-p dst) 16) + (t 0)) ,@(and nds `(:vvvv src)) :l ,l)))) @@ -1269,6 +1275,10 @@ REG is the source (encoded in ModR/M.r/m). :opcode-prefix ,opcode-prefix :evex-w ,evex-w :w ,evex-w + :disp-n (cond ((zmm-register-p src) 64) + ((ymm-register-p src) 32) + ((xmm-register-p src) 16) + (t 0)) :l ,l)))))))) ;; direction bit? (def vmovapd #x66 #x28 #x29 :evex-w 1 :disp-n 64) diff --git a/tests/simd-pack-512.pure.lisp b/tests/simd-pack-512.pure.lisp index 5f5be8d36..2e3f1e6b3 100644 --- a/tests/simd-pack-512.pure.lisp +++ b/tests/simd-pack-512.pure.lisp @@ -310,7 +310,8 @@ (def %test-evex-vblendmps) (def %test-evex-vpcmpd) (def %test-evex-vpmovqd) - (def %test-auto-promoted-vmovaps-disp8)) + (def %test-auto-promoted-vmovaps-disp8) + (def %test-auto-promoted-vmovaps-disp-nonmultiple)) (define-vop (%test-evex-high-regs) (:translate %test-evex-high-regs) @@ -502,6 +503,17 @@ (inst vmovaps zmm (ea 64 rsp)) (inst xor :dword res res))) +(define-vop (%test-auto-promoted-vmovaps-disp-nonmultiple) + (:translate %test-auto-promoted-vmovaps-disp-nonmultiple) + (:policy :fast-safe) + (:temporary (:sc single-avx512-reg :offset 0) zmm) + (:temporary (:sc unsigned-reg :offset rsp-offset) rsp) + (:results (res :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 1 + (inst vmovaps zmm (ea 65 rsp)) + (inst xor :dword res res))) + (cl:in-package :test-util) (with-test (:name :evex-high-register-disassembly) @@ -699,3 +711,27 @@ (assert evex-form) (let ((fields (third evex-form))) (assert (eq (third (assoc 'reg/mem fields)) disp64))))))) + +(with-test (:name :auto-promoted-vmovaps-disp8-disasm) + (let* ((fun (compile nil + '(lambda () + (sb-vm::%test-auto-promoted-vmovaps-disp8)))) + (text (with-output-to-string (s) + (disassemble fun :stream s)))) + ;; Auto-promoted VMOVAPS should use compressed disp8*64 and print +64. + (assert (search "VMOVAPS" text)) + (assert (search "ZMM0" text)) + (assert (search "[RSP+64]" text)) + ;; The compressed byte itself must not leak through as +1. + (assert (not (search "[RSP+1]" text))))) + +(with-test (:name :auto-promoted-vmovaps-disp-nonmultiple-disasm) + (let* ((fun (compile nil + '(lambda () + (sb-vm::%test-auto-promoted-vmovaps-disp-nonmultiple)))) + (text (with-output-to-string (s) + (disassemble fun :stream s)))) + ;; Non-multiple displacement must fall back to disp32. + (assert (search "VMOVAPS" text)) + (assert (search "ZMM0" text)) + (assert (search "[RSP+65]" text)))) -- 2.55.0