master: sb-simd: arm64 support
stassats via Sbcl-commits <[email protected]> Tue, 30 Jun 2026 22:46:56 +0000
| Newsgroups | gmane.lisp.steel-bank.cvs |
|---|---|
| Message-ID | <[email protected]> |
The branch "master" has been updated in SBCL:
via cb716e24d642f69fa40dbdc09f2a7bfe50b29823 (commit)
from bb63ab41bff760762ec43564844f0a928116f6bb (commit)
- Log -----------------------------------------------------------------
commit cb716e24d642f69fa40dbdc09f2a7bfe50b29823
Author: Sylvia Harrington <[email protected]>
Date: Sun Jun 28 10:24:33 2026 +0100
sb-simd: arm64 support
---
contrib/sb-simd/code/cpu-identification.lisp | 3 +
contrib/sb-simd/code/define-custom-vops.lisp | 210 +++++-
contrib/sb-simd/code/define-fake-vops.lisp | 132 ++++
contrib/sb-simd/code/define-instruction-vops.lisp | 59 +-
contrib/sb-simd/code/define-modify-macros.lisp | 31 +
contrib/sb-simd/code/define-simd-casts.lisp | 1 +
contrib/sb-simd/code/define-vref-vops.lisp | 72 ++
contrib/sb-simd/code/instruction-sets/arm64.lisp | 13 +
contrib/sb-simd/code/instruction-sets/neon.lisp | 764 +++++++++++++++++++++
contrib/sb-simd/code/packages.lisp | 553 ++++++++++++++-
contrib/sb-simd/code/record.lisp | 8 +-
contrib/sb-simd/sb-simd.asd | 4 +-
contrib/sb-simd/test-suite/test-arefs.lisp | 13 +
.../test-suite/test-hairy-simd-functions.lisp | 2 +
.../test-suite/test-horizontal-functions.lisp | 42 ++
contrib/sb-simd/test-suite/test-packages.lisp | 1 +
.../test-suite/test-simple-simd-functions.lisp | 149 ++++
contrib/sb-simd/test-suite/test-suite.lisp | 10 +-
contrib/sb-simd/test-suite/utilities.lisp | 1 +
doc/manual/beyond-ansi.texinfo | 2 +-
doc/manual/compiler.texinfo | 2 +-
make-config.sh | 7 +-
tests/sb-simd.impure.lisp | 4 -
xperfecthash63.lisp-expr | 74 ++
24 files changed, 2138 insertions(+), 19 deletions(-)
diff --git a/contrib/sb-simd/code/cpu-identification.lisp b/contrib/sb-simd/code/cpu-identification.lisp
index 2981f29f5..140720c6b 100644
--- a/contrib/sb-simd/code/cpu-identification.lisp
+++ b/contrib/sb-simd/code/cpu-identification.lisp
@@ -70,3 +70,6 @@
(defun fma-supported-p ()
nil))
+
+(defun neon-supported-p ()
+ #+arm64 t)
diff --git a/contrib/sb-simd/code/define-custom-vops.lisp b/contrib/sb-simd/code/define-custom-vops.lisp
index 8ed639a34..d1f8c3963 100644
--- a/contrib/sb-simd/code/define-custom-vops.lisp
+++ b/contrib/sb-simd/code/define-custom-vops.lisp
@@ -166,4 +166,212 @@
(:results (dst))
(:generator
(inst vxorpd dst dst dst)
- (inst movsd dst src))))
+ (inst movsd dst src)))
+
+;; Neon
+
+(define-custom-vop sb-simd-neon::f32!-from-p128
+ (:args (src :scs (single-neon-reg) :to :save))
+ (:results (dst :scs (single-reg)))
+ (:generator
+ (inst movi dst 0 :4s)
+ (inst dup dst src nil 0)))
+
+(define-custom-vop sb-simd-neon::f32.4!-from-f32
+ (:args (src :scs (single-reg) :target dst))
+ (:results (dst :scs (single-neon-reg)))
+ (:generator
+ (unless (location= src dst)
+ (inst ins dst 0 src 0 :s))))
+
+(define-custom-vop sb-simd-neon:f32.4-dup
+ (:args (src :scs (single-neon-reg)))
+ (:info lane)
+ (:results (dst :scs (single-neon-reg)))
+ (:generator
+ (inst dup dst src :4s lane)))
+
+(define-custom-vop sb-simd-neon:f32.4-lane-extract
+ (:args (src :scs (single-neon-reg)))
+ (:info lane)
+ (:results (dst :scs (single-reg)))
+ (:generator
+ (unless (and (location= dst src)
+ (eql lane 0))
+ (inst dup dst src nil lane))))
+
+(define-custom-vop sb-simd-neon:f32.4-lane-insert
+ (:args (src1 :scs (single-neon-reg) :target dst)
+ (src2 :scs (single-reg) :to :save))
+ (:info lane)
+ (:results (dst :scs (single-neon-reg)))
+ (:generator
+ (unless (location= dst src1)
+ (inst mov dst src1 :16b))
+ (inst ins dst lane
+ (make-random-tn (sc-or-lose 'single-neon-reg) (tn-offset src2)) 0
+ :s)))
+
+(define-custom-vop sb-simd-neon:f32.4-ins
+ (:args (src1 :scs (single-neon-reg) :target dst)
+ (src2 :scs (single-neon-reg) :to :save))
+ (:info idx1 idx2)
+ (:results (dst :scs (single-neon-reg)))
+ (:generator
+ (unless (location= dst src1)
+ (inst mov dst src1 :16b))
+ (inst ins dst idx1 src2 idx2 :s)))
+
+(define-custom-vop sb-simd-neon::f64!-from-p128
+ (:args (src :scs (double-neon-reg) :to :save))
+ (:results (dst :scs (double-reg)))
+ (:generator
+ (inst movi dst 0 :2d)
+ (inst dup dst src nil 0)))
+
+(define-custom-vop sb-simd-neon::f64.2!-from-f64
+ (:args (src :scs (double-reg) :target dst))
+ (:results (dst :scs (double-neon-reg)))
+ (:generator
+ (unless (location= src dst)
+ (inst ins dst 0 src 0 :d))))
+
+(define-custom-vop sb-simd-neon:f64.2-dup
+ (:args (src :scs (double-neon-reg)))
+ (:info lane)
+ (:results (dst :scs (double-neon-reg)))
+ (:generator
+ (inst dup dst src :2d lane)))
+
+(define-custom-vop sb-simd-neon:f64.2-lane-extract
+ (:args (src :scs (double-neon-reg)))
+ (:info lane)
+ (:results (dst :scs (double-reg)))
+ (:generator
+ (inst dup dst src nil lane)))
+
+(define-custom-vop sb-simd-neon:f64.2-lane-insert
+ (:args (src1 :scs (double-neon-reg) :target dst)
+ (src2 :scs (double-reg) :to :save))
+ (:info lane)
+ (:results (dst :scs (double-neon-reg)))
+ (:generator
+ (unless (location= dst src1)
+ (inst mov dst src1 :16b))
+ (inst ins dst lane
+ (make-random-tn (sc-or-lose 'single-neon-reg) (tn-offset src2)) 0
+ :d)))
+
+(define-custom-vop sb-simd-neon:f64.2-ins
+ (:args (src1 :scs (double-neon-reg) :target dst)
+ (src2 :scs (double-neon-reg) :to :save))
+ (:info idx1 idx2)
+ (:results (dst :scs (double-neon-reg)))
+ (:generator
+ (unless (location= dst src1)
+ (inst mov dst src1 :16b))
+ (inst ins dst idx1 src2 idx2 :d)))
+
+(define-custom-vop sb-simd-neon:u8.16-shuffle
+ (:args (src1 :scs (single-neon-reg))
+ (control :scs (single-neon-reg)))
+ (:results (dst :scs (single-neon-reg)))
+ (:generator
+ (inst tbl dst (list src1) control :16b)))
+
+(macrolet
+ ((def (sign width count)
+ (multiple-value-bind (arrangement partial-arrangement smaller-arrangement smaller-partial-arrangement larger-arrangement element)
+ (ecase width
+ (8 (values :16b :8b nil nil :8h :b))
+ (16 (values :8h :4h :16b :8b :4s :h))
+ (32 (values :4s :2s :8h :4h :2d :s))
+ (64 (values :2d nil :4s :2s nil :d)))
+ (let ((smaller-width (/ width 2))
+ (smaller-count (* count 2))
+ (scalar-reg (if (eql sign :s) 'signed-reg 'unsigned-reg)))
+ (flet ((name (format-control &rest format-arguments)
+ (intern (apply #'format nil format-control format-arguments)
+ (find-package :sb-simd-neon))))
+ `(progn
+ (define-custom-vop ,(name "~a~d.~d!-FROM-~a~a" sign width count sign width)
+ (:args (src :scs (,scalar-reg) :target dst))
+ (:results (dst :scs (int-neon-reg)))
+ (:generator
+ (inst fmov (make-random-tn (sc-or-lose ',(if (eql width 64) 'double-reg 'single-reg)) (tn-offset dst)) src)))
+ (define-custom-vop ,(name "~a~d.~d-DUP" sign width count)
+ (:args (src :scs (int-neon-reg)))
+ (:info lane)
+ (:results (dst :scs (int-neon-reg)))
+ (:generator
+ (inst dup dst src ,arrangement lane)))
+ (define-custom-vop ,(name "~a~d.~d-LANE-EXTRACT" sign width count)
+ (:args (src :scs (int-neon-reg)))
+ (:info lane)
+ (:results (dst :scs (,scalar-reg)))
+ (:generator
+ (inst umov dst src lane ,element)))
+ (define-custom-vop ,(name "~a~d.~d-LANE-INSERT" sign width count)
+ (:args (src1 :scs (int-neon-reg) :target dst)
+ (src2 :scs (,scalar-reg) :to :save))
+ (:info lane)
+ (:results (dst :scs (int-neon-reg)))
+ (:generator
+ (unless (location= dst src1)
+ (inst mov dst src1 :16b))
+ (inst ins dst lane src2 nil ,element)))
+ (define-custom-vop ,(name "~a~d.~d-INS" sign width count)
+ (:args (src1 :scs (int-neon-reg) :target dst)
+ (src2 :scs (int-neon-reg) :to :save))
+ (:info idx1 idx2)
+ (:results (dst :scs (int-neon-reg)))
+ (:generator
+ (unless (location= dst src1)
+ (inst mov dst src1 :16b))
+ (inst ins dst idx1 src2 idx2 ,element)))
+ ;; Conversion from the smaller type.
+ ,@(when smaller-arrangement
+ `((define-custom-vop ,(name "~a~d.~d-FROM-~a~d.~d" sign width count sign smaller-width smaller-count)
+ (:args (src :scs (int-neon-reg)))
+ (:results (dst :scs (int-neon-reg)))
+ (:generator
+ (inst ,(if (eql sign :s) 'sshll 'ushll) dst ,arrangement src ,smaller-partial-arrangement)))
+ (define-custom-vop ,(name "~a~d.~d-FROM-~a~d.~d-HI" sign width count sign smaller-width smaller-count)
+ (:args (src :scs (int-neon-reg)))
+ (:results (dst :scs (int-neon-reg)))
+ (:generator
+ (inst ,(if (eql sign :s) 'sshll2 'ushll2) dst ,arrangement src ,smaller-arrangement)))))
+ (define-custom-vop ,(name "~a~d.~d-SHIFTR" sign width count)
+ (:args (src :scs (int-neon-reg)))
+ (:info shift)
+ (:results (dst :scs (int-neon-reg)))
+ (:generator
+ (inst ,(if (eql sign :s) 'sshr 'ushr) dst src shift ,arrangement)))
+ (define-custom-vop ,(name "~a~d.~d-SHIFTL" sign width count)
+ (:args (src :scs (int-neon-reg)))
+ (:info shift)
+ (:results (dst :scs (int-neon-reg)))
+ (:generator
+ (inst shl dst src shift ,arrangement)))
+ ,@(when larger-arrangement
+ `((define-custom-vop ,(name "~a~d.~d-SHIFTL-LONG" sign width count)
+ (:args (src :scs (int-neon-reg)))
+ (:info shift)
+ (:results (dst :scs (int-neon-reg)))
+ (:generator
+ (inst ,(if (eql sign :s) 'sshll 'ushll) dst ,larger-arrangement src ,partial-arrangement shift)))
+ (define-custom-vop ,(name "~a~d.~d-SHIFTL-LONG-HI" sign width count)
+ (:args (src :scs (int-neon-reg)))
+ (:info shift)
+ (:results (dst :scs (int-neon-reg)))
+ (:generator
+ (inst ,(if (eql sign :s) 'sshll2 'ushll2) dst ,larger-arrangement src ,arrangement shift)))))))))))
+ (def :u 8 16)
+ (def :u 16 8)
+ (def :u 32 4)
+ (def :u 64 2)
+ (def :s 8 16)
+ (def :s 16 8)
+ (def :s 32 4)
+ (def :s 64 2))
+)
diff --git a/contrib/sb-simd/code/define-fake-vops.lisp b/contrib/sb-simd/code/define-fake-vops.lisp
index b31f41f66..3121b753e 100644
--- a/contrib/sb-simd/code/define-fake-vops.lisp
+++ b/contrib/sb-simd/code/define-fake-vops.lisp
@@ -1739,3 +1739,135 @@
(define-fake-vop f64.4-reverse (a)
(%f64.4-permute4x64 a #b00011011))
+
+(in-package #:sb-simd-neon)
+
+(define-fake-vop f32.4-values (x)
+ (values (%f32.4-lane-extract x 0)
+ (%f32.4-lane-extract x 1)
+ (%f32.4-lane-extract x 2)
+ (%f32.4-lane-extract x 3)))
+
+(define-fake-vop make-f32.4 (x y z w)
+ (let* ((t1 (%f32.4!-from-f32 x))
+ (t2 (%f32.4-lane-insert t1 y 1))
+ (t3 (%f32.4-lane-insert t2 z 2)))
+ (%f32.4-lane-insert t3 w 3)))
+
+(define-fake-vop two-arg-f32.4/= (a b)
+ (%u32.4-not
+ (%two-arg-f32.4= a b)))
+
+(define-fake-vop two-arg-f32.4< (a b)
+ (%two-arg-f32.4> b a))
+
+(define-fake-vop two-arg-f32.4<= (a b)
+ (%two-arg-f32.4>= b a))
+
+(define-fake-vop f32.4-horizontal+ (x)
+ (let ((tmp (%f32.4-pair+ x x)))
+ ;; TODO: This could use the scalar faddp instead, avoiding the extraction.
+ (%f32.4-lane-extract (%f32.4-pair+ tmp tmp) 0)))
+
+(define-fake-vop f32.4-andc1 (a b)
+ (%f32.4-andc2 b a))
+
+(define-fake-vop f64.2-values (x)
+ (values (%f64.2-lane-extract x 0)
+ (%f64.2-lane-extract x 1)))
+
+(define-fake-vop make-f64.2 (x y)
+ (%f64.2-lane-insert (%f64.2!-from-f64 x) y 1))
+
+(define-fake-vop two-arg-f64.2/= (a b)
+ (%u64.2-not
+ (%two-arg-f64.2= a b)))
+
+(define-fake-vop two-arg-f64.2< (a b)
+ (%two-arg-f64.2> b a))
+
+(define-fake-vop two-arg-f64.2<= (a b)
+ (%two-arg-f64.2>= b a))
+
+(define-fake-vop f64.2-andc1 (a b)
+ (%f64.2-andc2 b a))
+
+(macrolet
+ ((def (sign width count)
+ (flet ((name (format-control &rest format-arguments)
+ (intern (apply #'format nil format-control format-arguments)
+ (find-package :sb-simd-neon))))
+ (let ((make-args (loop for i from 0 below count
+ collect (make-symbol (format nil "V~a" i))))
+ (from (name "%~a~d.~d!-FROM-~a~d" sign width count sign width))
+ (insert (name "%~a~d.~d-LANE-INSERT" sign width count))
+ (extract (name "%~a~d.~d-LANE-EXTRACT" sign width count)))
+ `(progn
+ (define-fake-vop ,(name "MAKE-~a~d.~d" sign width count) ,make-args
+ ,(loop with result = (list from (first make-args))
+ for arg in (rest make-args)
+ for i from 1
+ do (setf result (list insert result arg i))
+ finally (return result)))
+ (define-fake-vop ,(name "~a~d.~d-VALUES" sign width count) (x)
+ (values ,@(loop for i from 0 below count
+ collect `(,extract x ,i))))
+ (define-fake-vop ,(name "~a~d.~d-ANDC1" sign width count) (a b)
+ (,(name "%~a~d.~d-ANDC2" sign width count) b a))
+ (define-fake-vop ,(name "TWO-ARG-~a~d.~d/=" sign width count) (a b)
+ (,(name "%U~d.~d-NOT" width count)
+ (,(name "%TWO-ARG-~a~d.~d=" sign width count) a b)))
+ (define-fake-vop ,(name "TWO-ARG-~a~d.~d<" sign width count) (a b)
+ (,(name "%TWO-ARG-~a~d.~d>" sign width count) b a))
+ (define-fake-vop ,(name "TWO-ARG-~a~d.~d<=" sign width count) (a b)
+ (,(name "%TWO-ARG-~a~d.~d>=" sign width count) b a)))))))
+ (def :u 8 16)
+ (def :u 16 8)
+ (def :u 32 4)
+ (def :u 64 2)
+ (def :s 8 16)
+ (def :s 16 8)
+ (def :s 32 4)
+ (def :s 64 2))
+
+(define-fake-vop u64.2-pair-min (a b)
+ (%make-u64.2 (sb-simd::%two-arg-u64-min (%u64.2-lane-extract a 0) (%u64.2-lane-extract a 1))
+ (sb-simd::%two-arg-u64-min (%u64.2-lane-extract b 0) (%u64.2-lane-extract b 1))))
+
+(define-fake-vop u64.2-horizontal-min (a)
+ (sb-simd::%two-arg-u64-min (%u64.2-lane-extract a 0) (%u64.2-lane-extract a 1)))
+
+(define-fake-vop u64.2-pair-max (a b)
+ (%make-u64.2 (sb-simd::%two-arg-u64-max (%u64.2-lane-extract a 0) (%u64.2-lane-extract a 1))
+ (sb-simd::%two-arg-u64-max (%u64.2-lane-extract b 0) (%u64.2-lane-extract b 1))))
+
+(define-fake-vop u64.2-horizontal-max (a)
+ (sb-simd::%two-arg-u64-max (%u64.2-lane-extract a 0) (%u64.2-lane-extract a 1)))
+
+(define-fake-vop u64.2-pair+ (a b)
+ (%make-u64.2 (sb-simd::%two-arg-u64+ (%u64.2-lane-extract a 0) (%u64.2-lane-extract a 1))
+ (sb-simd::%two-arg-u64+ (%u64.2-lane-extract b 0) (%u64.2-lane-extract b 1))))
+
+(define-fake-vop u64.2-horizontal+ (a)
+ (sb-simd::%two-arg-u64+ (%u64.2-lane-extract a 0) (%u64.2-lane-extract a 1)))
+
+(define-fake-vop s64.2-pair-min (a b)
+ (%make-s64.2 (sb-simd::%two-arg-s64-min (%s64.2-lane-extract a 0) (%s64.2-lane-extract a 1))
+ (sb-simd::%two-arg-s64-min (%s64.2-lane-extract b 0) (%s64.2-lane-extract b 1))))
+
+(define-fake-vop s64.2-horizontal-min (a)
+ (sb-simd::%two-arg-s64-min (%s64.2-lane-extract a 0) (%s64.2-lane-extract a 1)))
+
+(define-fake-vop s64.2-pair-max (a b)
+ (%make-s64.2 (sb-simd::%two-arg-s64-max (%s64.2-lane-extract a 0) (%s64.2-lane-extract a 1))
+ (sb-simd::%two-arg-s64-max (%s64.2-lane-extract b 0) (%s64.2-lane-extract b 1))))
+
+(define-fake-vop s64.2-horizontal-max (a)
+ (sb-simd::%two-arg-s64-max (%s64.2-lane-extract a 0) (%s64.2-lane-extract a 1)))
+
+(define-fake-vop s64.2-pair+ (a b)
+ (%make-s64.2 (sb-simd::%two-arg-s64+ (%s64.2-lane-extract a 0) (%s64.2-lane-extract a 1))
+ (sb-simd::%two-arg-s64+ (%s64.2-lane-extract b 0) (%s64.2-lane-extract b 1))))
+
+(define-fake-vop s64.2-horizontal+ (a)
+ (sb-simd::%two-arg-s64+ (%s64.2-lane-extract a 0) (%s64.2-lane-extract a 1)))
diff --git a/contrib/sb-simd/code/define-instruction-vops.lisp b/contrib/sb-simd/code/define-instruction-vops.lisp
index f3944d0ec..b4530e70e 100644
--- a/contrib/sb-simd/code/define-instruction-vops.lisp
+++ b/contrib/sb-simd/code/define-instruction-vops.lisp
@@ -162,7 +162,64 @@
(t
(move tmp ,x)
(inst ,mnemonic ,@prefix tmp ,y ,z ,@rest ,@suffix)
- (move ,r tmp))))))))))))
+ (move ,r tmp))))))))
+ (:neon-rmw
+ (assert mnemonic)
+ (let ((x (first asyms))
+ (y (second asyms))
+ (rest (rest (rest asyms)))
+ (r (first rsyms)))
+ `(progn
+ ,defknown
+ (define-vop (,vop)
+ (:translate ,vop)
+ (:policy :fast-safe)
+ (:args (,@(first args) :target ,r) ,@(loop for arg in (rest args) collect (append arg (list :to :save))))
+ (:temporary (:sc ,(first (sb-simd-internals:value-record-scs (first argument-records)))) tmp)
+ (:info ,@info)
+ (:results ,@results)
+ (:arg-types ,@arg-types)
+ (:result-types ,@result-types)
+ (:generator
+ ,cost
+ (cond ((location= ,x ,r)
+ (inst ,mnemonic ,@prefix ,r ,y ,@rest ,@suffix))
+ ((or (not (tn-p ,y))
+ (not (location= ,y ,r)))
+ (inst mov ,r ,x :16b)
+ (inst ,mnemonic ,@prefix ,r ,y ,@rest ,@suffix))
+ (t
+ (inst mov tmp ,x :16b)
+ (inst ,mnemonic ,@prefix tmp ,y ,@rest ,@suffix)
+ (inst mov ,r tmp :16b))))))))
+ (:neon-int-result
+ (assert mnemonic)
+ (let* ((result-ty (sb-simd-internals:value-record-type (first result-records)))
+ (signedp (and (listp result-ty)
+ (eql (first result-ty) 'signed-byte)))
+ (width (ecase (sb-simd-internals:value-record-bits (first result-records))
+ (8 :b)
+ (16 :h)
+ (32 :s)
+ (64 :d))))
+ `(progn
+ ,defknown
+ (define-vop (,vop)
+ (:translate ,vop)
+ (:policy :fast-safe)
+ (:args ,@args)
+ (:temporary (:sc ,(first (sb-simd-internals:value-record-scs (first argument-records)))) tmp)
+ (:info ,@info)
+ (:results ,@results)
+ (:arg-types ,@arg-types)
+ (:result-types ,@result-types)
+ (:generator
+ ,cost
+ (inst ,mnemonic ,@prefix tmp ,@asyms ,@suffix)
+ ,(cond ((and signedp (not (eql width :d)))
+ `(inst smov ,(first rsyms) tmp 0 ,width))
+ (t
+ `(inst umov ,(first rsyms) tmp 0 ,width))))))))))))
(define-instruction-vops ()
`(progn
,@(loop for instruction-record
diff --git a/contrib/sb-simd/code/define-modify-macros.lisp b/contrib/sb-simd/code/define-modify-macros.lisp
index 141a28532..b3e229151 100644
--- a/contrib/sb-simd/code/define-modify-macros.lisp
+++ b/contrib/sb-simd/code/define-modify-macros.lisp
@@ -140,3 +140,34 @@
(define-modify-macro s64.4-incf (&optional (num 1)) two-arg-s64.4+)
(define-modify-macro s64.4-decf (&optional (num 1)) two-arg-s64.4-)
+(in-package #:sb-simd-neon)
+
+(define-modify-macro f32.4-incf (&optional (num 1f0)) two-arg-f32.4+)
+(define-modify-macro f32.4-decf (&optional (num 1f0)) two-arg-f32.4-)
+
+(define-modify-macro f64.2-incf (&optional (num 1d0)) two-arg-f64.2+)
+(define-modify-macro f64.2-decf (&optional (num 1d0)) two-arg-f64.2-)
+
+(define-modify-macro u8.16-incf (&optional (num 1)) two-arg-u8.16+)
+(define-modify-macro u8.16-decf (&optional (num 1)) two-arg-u8.16-)
+
+(define-modify-macro u16.8-incf (&optional (num 1)) two-arg-u16.8+)
+(define-modify-macro u16.8-decf (&optional (num 1)) two-arg-u16.8-)
+
+(define-modify-macro u32.4-incf (&optional (num 1)) two-arg-u32.4+)
+(define-modify-macro u32.4-decf (&optional (num 1)) two-arg-u32.4-)
+
+(define-modify-macro u64.2-incf (&optional (num 1)) two-arg-u64.2+)
+(define-modify-macro u64.2-decf (&optional (num 1)) two-arg-u64.2-)
+
+(define-modify-macro s8.16-incf (&optional (num 1)) two-arg-s8.16+)
+(define-modify-macro s8.16-decf (&optional (num 1)) two-arg-s8.16-)
+
+(define-modify-macro s16.8-incf (&optional (num 1)) two-arg-s16.8+)
+(define-modify-macro s16.8-decf (&optional (num 1)) two-arg-s16.8-)
+
+(define-modify-macro s32.4-incf (&optional (num 1)) two-arg-s32.4+)
+(define-modify-macro s32.4-decf (&optional (num 1)) two-arg-s32.4-)
+
+(define-modify-macro s64.2-incf (&optional (num 1)) two-arg-s64.2+)
+(define-modify-macro s64.2-decf (&optional (num 1)) two-arg-s64.2-)
diff --git a/contrib/sb-simd/code/define-simd-casts.lisp b/contrib/sb-simd/code/define-simd-casts.lisp
index acb1d9b49..e4f01a34e 100644
--- a/contrib/sb-simd/code/define-simd-casts.lisp
+++ b/contrib/sb-simd/code/define-simd-casts.lisp
@@ -16,6 +16,7 @@
(define-inline sb-simd-sse:p128 (x) (the sb-simd-sse:p128 x))
(define-inline sb-simd-avx:p128 (x) (the sb-simd-avx:p128 x))
(define-inline sb-simd-avx:p256 (x) (the sb-simd-avx:p256 x))
+(define-inline sb-simd-neon:p128 (x) (the sb-simd-neon:p128 x))
(macrolet
(;; We cannot call known functions directly in the definition of a
diff --git a/contrib/sb-simd/code/define-vref-vops.lisp b/contrib/sb-simd/code/define-vref-vops.lisp
index 3a93fd41b..3f143b722 100644
--- a/contrib/sb-simd/code/define-vref-vops.lisp
+++ b/contrib/sb-simd/code/define-vref-vops.lisp
@@ -96,3 +96,75 @@
#'sb-simd-internals:vref-record-p)
collect `(define-vref-vop ,(sb-simd-internals:vref-record-name vref-record))))))
(define-vref-vops))
+
+#+arm64
+(macrolet
+ ((define-vref-vop (vref-record-name)
+ (with-accessors ((name sb-simd-internals:vref-record-name)
+ (vop sb-simd-internals:vref-record-vop)
+ (vop-c sb-simd-internals:vref-record-vop-c)
+ (mnemonic sb-simd-internals:vref-record-mnemonic)
+ (value-record sb-simd-internals:vref-record-value-record)
+ (vector-record sb-simd-internals:vref-record-vector-record)
+ (store sb-simd-internals:store-record-p))
+ (sb-simd-internals:find-function-record vref-record-name)
+ (let* ((vector-type (sb-simd-internals:value-record-type vector-record))
+ (vector-primitive-type (sb-simd-internals:value-record-primitive-type vector-record))
+ (value-scs (sb-simd-internals:value-record-scs value-record))
+ (value-type (sb-simd-internals:value-record-type value-record))
+ (value-primitive-type (sb-simd-internals:value-record-primitive-type value-record))
+ (scalar-record
+ (etypecase value-record
+ (sb-simd-internals:simd-record (sb-simd-internals:simd-record-scalar-record value-record))
+ (sb-simd-internals:value-record value-record)))
+ (bits-per-element (sb-simd-internals:value-record-bits scalar-record))
+ (bytes-per-element (ceiling bits-per-element 8))
+ (shift (1- (integer-length bytes-per-element))))
+ `(progn
+ (defknown ,vop (,@(when store `(,value-type)) ,vector-type index (integer 0 0))
+ (values ,value-type &optional)
+ (always-translatable)
+ :overwrite-fndb-silently t)
+ (define-vop (,vop)
+ (:translate ,vop)
+ (:policy :fast-safe)
+ (:args ,@(when store `((value :scs (,@value-scs zero))))
+ (object :scs (descriptor-reg))
+ (index :scs (any-reg unsigned-reg signed-reg immediate)))
+ (:arg-types ,@(when store `(,value-primitive-type))
+ ,vector-primitive-type
+ tagged-num
+ (:constant (integer 0 0)))
+ (:info addend) ; always zero
+ ,@(unless store
+ `((:results (value :scs ,value-scs))
+ (:result-types ,value-primitive-type)))
+ (:generator 2
+ (let ((addend addend))
+ (declare (ignore addend))
+ (sc-case index
+ (immediate
+ (inst ,(if store 'str 'ldr)
+ value
+ (@ object (load-store-offset
+ (+ (ash (tn-value index) ,shift)
+ (- (ash vector-data-offset word-shift)
+ other-pointer-lowtag))))))
+ (t
+ (let ((shift ,shift))
+ (when (sc-is index any-reg)
+ (decf shift n-fixnum-tag-bits))
+ (inst add tmp-tn object (if (minusp shift)
+ (asr index (- shift))
+ (lsl index shift))))
+ (inst ,(if store 'str 'ldr)
+ value
+ (@ tmp-tn (load-store-offset (- (ash vector-data-offset word-shift)
+ other-pointer-lowtag)))))))))))))
+ (define-vref-vops ()
+ `(progn
+ ,@(loop for vref-record
+ in (sb-simd-internals:filter-available-function-records
+ #'sb-simd-internals:vref-record-p)
+ collect `(define-vref-vop ,(sb-simd-internals:vref-record-name vref-record))))))
+ (define-vref-vops))
diff --git a/contrib/sb-simd/code/instruction-sets/arm64.lisp b/contrib/sb-simd/code/instruction-sets/arm64.lisp
new file mode 100644
index 000000000..d6090c17e
--- /dev/null
+++ b/contrib/sb-simd/code/instruction-sets/arm64.lisp
@@ -0,0 +1,13 @@
+(in-package #:sb-simd-arm64)
+
+(define-instruction-set :arm64
+ (:include :sb-simd)
+ (:scalars
+ (imm1 1 (unsigned-byte 1) (:constant (unsigned-byte 1)))
+ (imm2 2 (unsigned-byte 2) (:constant (unsigned-byte 2)))
+ (imm3 3 (unsigned-byte 3) (:constant (unsigned-byte 3)))
+ (imm4 4 (unsigned-byte 4) (:constant (unsigned-byte 4)))
+ (imm5 5 (unsigned-byte 5) (:constant (unsigned-byte 5)))
+ (imm6 6 (unsigned-byte 6) (:constant (unsigned-byte 6)))
+ (imm7 7 (unsigned-byte 7) (:constant (unsigned-byte 7)))
+ (imm8 8 (unsigned-byte 8) (:constant (unsigned-byte 8)))))
diff --git a/contrib/sb-simd/code/instruction-sets/neon.lisp b/contrib/sb-simd/code/instruction-sets/neon.lisp
new file mode 100644
index 000000000..506702c2a
--- /dev/null
+++ b/contrib/sb-simd/code/instruction-sets/neon.lisp
@@ -0,0 +1,764 @@
+(in-package #:sb-simd-neon)
+
+(define-instruction-set :neon
+ (:test (neon-supported-p))
+ (:include :arm64)
+ (:simd-packs
+ (p128 nil 128 #:simd-pack (#:int-neon-reg #:double-neon-reg #:single-neon-reg))
+ (f32.4 f32 128 #:simd-pack-single (#:single-neon-reg))
+ (f64.2 f64 128 #:simd-pack-double (#:double-neon-reg))
+ (u8.16 u8 128 #:simd-pack-ub8 (#:int-neon-reg))
+ (u16.8 u16 128 #:simd-pack-ub16 (#:int-neon-reg))
+ (u32.4 u32 128 #:simd-pack-ub32 (#:int-neon-reg))
+ (u64.2 u64 128 #:simd-pack-ub64 (#:int-neon-reg))
+ (s8.16 s8 128 #:simd-pack-sb8 (#:int-neon-reg))
+ (s16.8 s16 128 #:simd-pack-sb16 (#:int-neon-reg))
+ (s32.4 s32 128 #:simd-pack-sb32 (#:int-neon-reg))
+ (s64.2 s64 128 #:simd-pack-sb64 (#:int-neon-reg)))
+ (:simd-casts
+ (f32.4 f32.4-broadcast)
+ (f64.2 f64.2-broadcast)
+ (u8.16 u8.16-broadcast)
+ (u16.8 u16.8-broadcast)
+ (u32.4 u32.4-broadcast)
+ (u64.2 u64.2-broadcast)
+ (s8.16 s8.16-broadcast)
+ (s16.8 s16.8-broadcast)
+ (s32.4 s32.4-broadcast)
+ (s64.2 s64.2-broadcast))
+ (:reinterpret-casts
+ (f32! f32!-from-p128)
+ (f64! f64!-from-p128)
+ (u8! u8!-from-p128)
+ (u16! u16!-from-p128)
+ (u32! u32!-from-p128)
+ (u64! u64!-from-p128)
+ (s8! s8!-from-p128)
+ (s16! s16!-from-p128)
+ (s32! s32!-from-p128)
+ (s64! s64!-from-p128)
+ (f32.4! f32.4!-from-f32 f32.4!-from-p128)
+ (f64.2! f64.2!-from-f64 f64.2!-from-p128)
+ (u8.16! u8.16!-from-u8 u8.16!-from-p128)
+ (u16.8! u16.8!-from-u16 u16.8!-from-p128)
+ (u32.4! u32.4!-from-u32 u32.4!-from-p128)
+ (u64.2! u64.2!-from-u64 u64.2!-from-p128)
+ (s8.16! s8.16!-from-s8 s8.16!-from-p128)
+ (s16.8! s16.8!-from-s16 s16.8!-from-p128)
+ (s32.4! s32.4!-from-s32 s32.4!-from-p128)
+ (s64.2! s64.2!-from-s64 s64.2!-from-p128))
+ (:instructions
+ ;; f32
+ (f32!-from-p128 nil (f32) (p128) :cost 1 :encoding :custom :always-translatable nil)
+ ;; f32.4
+ (f32.4!-from-f32 nil (f32.4) (f32) :cost 1 :encoding :custom)
+ (f32.4!-from-p128 #:mov (f32.4) (p128) :cost 1 :encoding :move :always-translatable nil :suffix '(:16b))
+ (f32.4-from-f64.2 #:fcvtn (f32.4) (f64.2) :cost 1)
+ (f32.4-from-f64.2-hi #:fcvtn2 (f32.4) (f32.4 f64.2) :cost 1 :encoding :neon-rmw)
+ (f32.4-from-s32.4 #:scvtf (f32.4) (s32.4) :cost 5 :suffix '(:4s))
+ (f32.4-from-u32.4 #:ucvtf (f32.4) (u32.4) :cost 5 :suffix '(:4s))
+ (make-f32.4 nil (f32.4) (f32 f32 f32 f32) :cost 1 :encoding :fake-vop)
+ (f32.4-values nil (f32 f32 f32 f32) (f32.4) :cost 1 :encoding :fake-vop)
+ (f32.4-broadcast #:dup (f32.4) (f32) :cost 1 :suffix '(:4s))
+ (two-arg-f32.4-and #:and (f32.4) (f32.4 f32.4) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-f32.4-or #:orr (f32.4) (f32.4 f32.4) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-f32.4-xor #:eor (f32.4) (f32.4 f32.4) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-f32.4-max #:fmax (f32.4) (f32.4 f32.4) :cost 3 :associative t :suffix '(:4s))
+ (two-arg-f32.4-min #:fmin (f32.4) (f32.4 f32.4) :cost 3 :associative t :suffix '(:4s))
+ (two-arg-f32.4+ #:fadd (f32.4) (f32.4 f32.4) :cost 2 :associative t :suffix '(:4s))
+ (two-arg-f32.4- #:fsub (f32.4) (f32.4 f32.4) :cost 2 :suffix '(:4s))
+ (two-arg-f32.4* #:fmul (f32.4) (f32.4 f32.4) :cost 2 :associative t :suffix '(:4s))
+ (two-arg-f32.4/ #:fdiv (f32.4) (f32.4 f32.4) :cost 8 :suffix '(:4s))
+ (two-arg-f32.4= #:fcmeq (u32.4) (f32.4 f32.4) :cost 4 :associative t :suffix '(:4s))
+ (two-arg-f32.4/= nil (u32.4) (f32.4 f32.4) :cost 4 :encoding :fake-vop)
+ (two-arg-f32.4< nil (u32.4) (f32.4 f32.4) :cost 4 :encoding :fake-vop)
+ (two-arg-f32.4<= nil (u32.4) (f32.4 f32.4) :cost 4 :encoding :fake-vop)
+ (two-arg-f32.4> #:fcmgt (u32.4) (f32.4 f32.4) :cost 4 :suffix '(:4s))
+ (two-arg-f32.4>= #:fcmge (u32.4) (f32.4 f32.4) :cost 4 :suffix '(:4s))
+ (f32.4-pair-min #:fminp (f32.4) (f32.4 f32.4) :cost 2 :suffix '(:4s) :associative t)
+ (f32.4-horizontal-min #:fminv (f32) (f32.4) :cost 2 :associative t)
+ (f32.4-pair-max #:fmaxp (f32.4) (f32.4 f32.4) :cost 2 :suffix '(:4s) :associative t)
+ (f32.4-horizontal-max #:fmaxv (f32) (f32.4) :cost 2 :associative t)
+ (f32.4-pair+ #:faddp (f32.4) (f32.4 f32.4) :cost 2 :suffix '(:4s) :associative t)
+ (f32.4-horizontal+ nil (f32) (f32.4) :cost 2 :encoding :fake-vop)
+ (f32.4-andc1 nil (f32.4) (f32.4 f32.4) :cost 1 :encoding :fake-vop)
+ (f32.4-andc2 #:bic (f32.4) (f32.4 f32.4) :cost 1 :suffix '(:16b))
+ (f32.4-not #:not (f32.4) (f32.4) :cost 1 :suffix '(:16b))
+ (f32.4-sqrt #:fsqrt (f32.4) (f32.4) :cost 15 :suffix '(:4s))
+ (f32.4-bit-select #:bsl (f32.4) (u32.4 f32.4 f32.4) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (f32.4-lane-extract nil (f32) (f32.4 imm2) :cost 1 :encoding :custom)
+ (f32.4-lane-insert nil (f32.4) (f32.4 f32 imm2) :cost 1 :encoding :custom)
+ (f32.4-dup nil (f32.4) (f32.4 imm2) :cost 1 :encoding :custom)
+ (f32.4-ins nil (f32.4) (f32.4 f32.4 imm2 imm2) :cost 1 :encoding :custom)
+ (f32.4-transpose-even #:trn1 (f32.4) (f32.4 f32.4) :cost 1 :suffix '(:4s))
+ (f32.4-transpose-odd #:trn2 (f32.4) (f32.4 f32.4) :cost 1 :suffix '(:4s))
+ (f32.4-unzip-even #:uzp1 (f32.4) (f32.4 f32.4) :cost 1 :suffix '(:4s))
+ (f32.4-unzip-odd #:uzp2 (f32.4) (f32.4 f32.4) :cost 1 :suffix '(:4s))
+ (f32.4-zip-even #:zip1 (f32.4) (f32.4 f32.4) :cost 1 :suffix '(:4s))
+ (f32.4-zip-odd #:zip2 (f32.4) (f32.4 f32.4) :cost 1 :suffix '(:4s))
+ ;; f64
+ (f64!-from-p128 nil (f64) (p128) :cost 1 :encoding :custom :always-translatable nil)
+ ;; f64.2
+ (f64.2!-from-f64 nil (f64.2) (f64) :cost 1 :encoding :custom)
+ (f64.2!-from-p128 #:mov (f64.2) (p128) :cost 1 :encoding :move :always-translatable nil :suffix '(:16b))
+ (f64.2-from-f32.4 #:fcvtl (f64.2) (f32.4) :cost 1)
+ (f64.2-from-f32.4-hi #:fcvtl2 (f64.2) (f32.4) :cost 1)
+ (f64.2-from-s64.2 #:scvtf (f64.2) (s64.2) :cost 5 :suffix '(:2d))
+ (f64.2-from-u64.2 #:ucvtf (f64.2) (u64.2) :cost 5 :suffix '(:2d))
+ (make-f64.2 nil (f64.2) (f64 f64) :cost 1 :encoding :fake-vop)
+ (f64.2-values nil (f64 f64) (f64.2) :cost 1 :encoding :fake-vop)
+ (f64.2-broadcast #:dup (f64.2) (f64) :cost 1 :suffix '(:2d))
+ (two-arg-f64.2-and #:and (f64.2) (f64.2 f64.2) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-f64.2-or #:orr (f64.2) (f64.2 f64.2) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-f64.2-xor #:eor (f64.2) (f64.2 f64.2) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-f64.2-max #:fmax (f64.2) (f64.2 f64.2) :cost 3 :associative t :suffix '(:2d))
+ (two-arg-f64.2-min #:fmin (f64.2) (f64.2 f64.2) :cost 3 :associative t :suffix '(:2d))
+ (two-arg-f64.2+ #:fadd (f64.2) (f64.2 f64.2) :cost 2 :associative t :suffix '(:2d))
+ (two-arg-f64.2- #:fsub (f64.2) (f64.2 f64.2) :cost 2 :suffix '(:2d))
+ (two-arg-f64.2* #:fmul (f64.2) (f64.2 f64.2) :cost 2 :associative t :suffix '(:2d))
+ (two-arg-f64.2/ #:fdiv (f64.2) (f64.2 f64.2) :cost 8 :suffix '(:2d))
+ (two-arg-f64.2= #:fcmeq (u64.2) (f64.2 f64.2) :cost 4 :associative t :suffix '(:2d))
+ (two-arg-f64.2/= nil (u64.2) (f64.2 f64.2) :cost 4 :encoding :fake-vop)
+ (two-arg-f64.2< nil (u64.2) (f64.2 f64.2) :cost 4 :encoding :fake-vop)
+ (two-arg-f64.2<= nil (u64.2) (f64.2 f64.2) :cost 4 :encoding :fake-vop)
+ (two-arg-f64.2> #:fcmgt (u64.2) (f64.2 f64.2) :cost 4 :suffix '(:2d))
+ (two-arg-f64.2>= #:fcmge (u64.2) (f64.2 f64.2) :cost 4 :suffix '(:2d))
+ (f64.2-pair-min #:fminp (f64.2) (f64.2 f64.2) :cost 2 :suffix '(:2d) :associative t)
+ (f64.2-horizontal-min #:fminp (f64) (f64.2) :cost 2 :suffix '(:2d))
+ (f64.2-pair-max #:fmaxp (f64.2) (f64.2 f64.2) :cost 2 :suffix '(:2d) :associative t)
+ (f64.2-horizontal-max #:fmaxp (f64) (f64.2) :cost 2 :suffix '(:2d))
+ (f64.2-pair+ #:faddp (f64.2) (f64.2 f64.2) :cost 2 :suffix '(:2d) :associative t)
+ (f64.2-horizontal+ #:faddp (f64) (f64.2) :cost 2 :suffix '(:2d))
+ (f64.2-andc1 nil (f64.2) (f64.2 f64.2) :cost 1 :encoding :fake-vop)
+ (f64.2-andc2 #:bic (f64.2) (f64.2 f64.2) :cost 1 :suffix '(:16b))
+ (f64.2-not #:not (f64.2) (f64.2) :cost 1 :suffix '(:16b))
+ (f64.2-sqrt #:fsqrt (f64.2) (f64.2) :cost 15 :suffix '(:2d))
+ (f64.2-bit-select #:bsl (f64.2) (u64.2 f64.2 f64.2) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (f64.2-lane-extract nil (f64) (f64.2 imm1) :cost 1 :encoding :custom)
+ (f64.2-lane-insert nil (f64.2) (f64.2 f64 imm1) :cost 1 :encoding :custom)
+ (f64.2-dup nil (f64.2) (f64.2 imm1) :cost 1 :encoding :custom)
+ (f64.2-ins nil (f64.2) (f64.2 f64.2 imm1 imm1) :cost 1 :encoding :custom)
+ (f64.2-transpose-even #:trn1 (f64.2) (f64.2 f64.2) :cost 1 :suffix '(:2d))
+ (f64.2-transpose-odd #:trn2 (f64.2) (f64.2 f64.2) :cost 1 :suffix '(:2d))
+ (f64.2-unzip-even #:uzp1 (f64.2) (f64.2 f64.2) :cost 1 :suffix '(:2d))
+ (f64.2-unzip-odd #:uzp2 (f64.2) (f64.2 f64.2) :cost 1 :suffix '(:2d))
+ (f64.2-zip-even #:zip1 (f64.2) (f64.2 f64.2) :cost 1 :suffix '(:2d))
+ (f64.2-zip-odd #:zip2 (f64.2) (f64.2 f64.2) :cost 1 :suffix '(:2d))
+ ;; u8
+ (u8!-from-p128 #:umov (u8) (p128) :cost 1 :encoding :move :suffix '(0 :b) :always-translatable nil)
+ ;; u8.16
+ (u8.16!-from-u8 nil (u8.16) (u8) :cost 1 :encoding :custom)
+ (u8.16!-from-p128 #:mov (u8.16) (p128) :cost 1 :encoding :move :always-translatable nil :suffix '(:16b))
+ (u8.16-from-u16.8 #:xtn (u8.16) (u16.8) :cost 1 :suffix '(:8b))
+ (u8.16-from-u16.8-hi #:xtn2 (u8.16) (u8.16 u16.8) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (u8.16-from-u16.8-saturating #:uqxtn (u8.16) (u16.8) :cost 1 :suffix '(:8b))
+ (u8.16-from-u16.8-saturating-hi #:uqxtn2 (u8.16) (u8.16 u16.8) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (make-u8.16 nil (u8.16) (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) :cost 1 :encoding :fake-vop)
+ (u8.16-values nil (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) (u8.16) :cost 1 :encoding :fake-vop)
+ (u8.16-broadcast #:dup (u8.16) (u8) :cost 1 :suffix '(:16b))
+ (two-arg-u8.16-and #:and (u8.16) (u8.16 u8.16) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u8.16-or #:orr (u8.16) (u8.16 u8.16) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u8.16-xor #:eor (u8.16) (u8.16 u8.16) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u8.16-max #:umax (u8.16) (u8.16 u8.16) :cost 3 :associative t :suffix '(:16b))
+ (two-arg-u8.16-min #:umin (u8.16) (u8.16 u8.16) :cost 3 :associative t :suffix '(:16b))
+ (u8.16-andc1 nil (u8.16) (u8.16 u8.16) :cost 1 :encoding :fake-vop)
+ (u8.16-andc2 #:bic (u8.16) (u8.16 u8.16) :cost 1 :suffix '(:16b))
+ (u8.16-not #:not (u8.16) (u8.16) :cost 1 :suffix '(:16b))
+ (two-arg-u8.16+ #:add (u8.16) (u8.16 u8.16) :cost 2 :suffix '(:16b) :associative t)
+ (two-arg-u8.16+-saturating #:uqadd (u8.16) (u8.16 u8.16) :cost 2 :suffix '(:16b) :associative t)
+ (two-arg-u8.16- #:sub (u8.16) (u8.16 u8.16) :cost 2 :suffix '(:16b))
+ (two-arg-u8.16--saturating #:uqsub (u8.16) (u8.16 u8.16) :cost 2 :suffix '(:16b) :associative t)
+ (two-arg-u8.16* #:mul (u8.16) (u8.16 u8.16) :cost 2 :suffix '(:16b) :associative t)
+ (u8.16*-long #:umull (u16.8) (u8.16 u8.16) :cost 2 :suffix '(:8b) :associative t)
+ (u8.16*-long-hi #:umull2 (u16.8) (u8.16 u8.16) :cost 2 :suffix '(:16b) :associative t)
+ (two-arg-u8.16= #:cmeq (u8.16) (u8.16 u8.16) :cost 1 :suffix '(:16b))
+ (two-arg-u8.16/= nil (u8.16) (u8.16 u8.16) :cost 2 :associative t :encoding :fake-vop)
+ (two-arg-u8.16> #:cmhi (u8.16) (u8.16 u8.16) :cost 1 :suffix '(:16b))
+ (two-arg-u8.16< nil (u8.16) (u8.16 u8.16) :cost 1 :encoding :fake-vop)
+ (two-arg-u8.16>= #:cmhs (u8.16) (u8.16 u8.16) :cost 1 :suffix '(:16b))
+ (two-arg-u8.16<= nil (u8.16) (u8.16 u8.16) :cost 1 :encoding :fake-vop)
+ (u8.16-pair-min #:uminp (u8.16) (u8.16 u8.16) :cost 2 :suffix '(:16b) :associative t)
+ (u8.16-horizontal-min #:uminv (u8) (u8.16) :cost 2 :encoding :neon-int-result :suffix '(:16b) :associative t)
+ (u8.16-pair-max #:umaxp (u8.16) (u8.16 u8.16) :cost 2 :suffix '(:16b) :associative t)
+ (u8.16-horizontal-max #:umaxv (u8) (u8.16) :cost 2 :encoding :neon-int-result :suffix '(:16b) :associative t)
+ (u8.16-pair+ #:addp (u8.16) (u8.16 u8.16) :cost 2 :suffix '(:16b) :associative t)
+ (u8.16-horizontal+ #:addv (u8) (u8.16) :cost 2 :encoding :neon-int-result :suffix '(:16b) :associative t)
+ (u8.16-shiftr nil (u8.16) (u8.16 imm3) :cost 1 :encoding :custom)
+ (u8.16-shiftl nil (u8.16) (u8.16 imm3) :cost 1 :encoding :custom)
+ (u8.16-shiftl-long nil (u16.8) (u8.16 imm3) :cost 1 :encoding :custom)
+ (u8.16-shiftl-long-hi nil (u16.8) (u8.16 imm3) :cost 1 :encoding :custom)
+ (u8.16-rev16 #:rev16 (u8.16) (u8.16) :cost 1 :suffix '(:16b))
+ (u8.16-rev32 #:rev32 (u8.16) (u8.16) :cost 1 :suffix '(:16b))
+ (u8.16-rev64 #:rev64 (u8.16) (u8.16) :cost 1 :suffix '(:16b))
+ (u8.16-bit-select #:bsl (u8.16) (u8.16 u8.16 u8.16) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (u8.16-lane-extract nil (u8) (u8.16 imm4) :cost 1 :encoding :custom)
+ (u8.16-lane-insert nil (u8.16) (u8.16 u8 imm4) :cost 1 :encoding :custom)
+ (u8.16-dup nil (u8.16) (u8.16 imm4) :cost 1 :encoding :custom)
+ (u8.16-ins nil (u8.16) (u8.16 u8.16 imm4 imm4) :cost 1 :encoding :custom)
+ (u8.16-transpose-even #:trn1 (u8.16) (u8.16 u8.16) :cost 1 :suffix '(:16b))
+ (u8.16-transpose-odd #:trn2 (u8.16) (u8.16 u8.16) :cost 1 :suffix '(:16b))
+ (u8.16-unzip-even #:uzp1 (u8.16) (u8.16 u8.16) :cost 1 :suffix '(:16b))
+ (u8.16-unzip-odd #:uzp2 (u8.16) (u8.16 u8.16) :cost 1 :suffix '(:16b))
+ (u8.16-zip-even #:zip1 (u8.16) (u8.16 u8.16) :cost 1 :suffix '(:16b))
+ (u8.16-zip-odd #:zip2 (u8.16) (u8.16 u8.16) :cost 1 :suffix '(:16b))
+ (u8.16-shuffle nil (u8.16) (u8.16 u8.16) :cost 1 :encoding :custom)
+ ;; u16
+ (u16!-from-p128 #:umov (u16) (p128) :cost 1 :encoding :move :suffix '(0 :h) :always-translatable nil)
+ ;; u16.8
+ (u16.8!-from-u16 nil (u16.8) (u16) :cost 1 :encoding :custom)
+ (u16.8!-from-p128 #:mov (u16.8) (p128) :cost 1 :encoding :move :always-translatable nil :suffix '(:16b))
+ (u16.8-from-u8.16 nil (u16.8) (u8.16) :cost 1 :encoding :custom)
+ (u16.8-from-u8.16-hi nil (u16.8) (u8.16) :cost 1 :encoding :custom)
+ (u16.8-from-u32.4 #:xtn (u16.8) (u32.4) :cost 1 :suffix '(:4h))
+ (u16.8-from-u32.4-hi #:xtn2 (u16.8) (u16.8 u32.4) :cost 1 :encoding :neon-rmw :suffix '(:8h))
+ (u16.8-from-u32.4-saturating #:uqxtn (u16.8) (u32.4) :cost 1 :suffix '(:4h))
+ (u16.8-from-u32.4-saturating-hi #:uqxtn2 (u16.8) (u16.8 u32.4) :cost 1 :encoding :neon-rmw :suffix '(:8h))
+ (make-u16.8 nil (u16.8) (u16 u16 u16 u16 u16 u16 u16 u16) :cost 1 :encoding :fake-vop)
+ (u16.8-values nil (u16 u16 u16 u16 u16 u16 u16 u16) (u16.8) :cost 1 :encoding :fake-vop)
+ (u16.8-broadcast #:dup (u16.8) (u16) :cost 1 :suffix '(:8h))
+ (two-arg-u16.8-and #:and (u16.8) (u16.8 u16.8) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u16.8-or #:orr (u16.8) (u16.8 u16.8) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u16.8-xor #:eor (u16.8) (u16.8 u16.8) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u16.8-max #:umax (u16.8) (u16.8 u16.8) :cost 3 :associative t :suffix '(:8h))
+ (two-arg-u16.8-min #:umin (u16.8) (u16.8 u16.8) :cost 3 :associative t :suffix '(:8h))
+ (u16.8-andc1 nil (u16.8) (u16.8 u16.8) :cost 1 :encoding :fake-vop)
+ (u16.8-andc2 #:bic (u16.8) (u16.8 u16.8) :cost 1 :suffix '(:16b))
+ (u16.8-not #:not (u16.8) (u16.8) :cost 1 :suffix '(:16b))
+ (two-arg-u16.8+ #:add (u16.8) (u16.8 u16.8) :cost 2 :suffix '(:8h) :associative t)
+ (two-arg-u16.8+-saturating #:uqadd (u16.8) (u16.8 u16.8) :cost 2 :suffix '(:8h) :associative t)
+ (two-arg-u16.8- #:sub (u16.8) (u16.8 u16.8) :cost 2 :suffix '(:8h))
+ (two-arg-u16.8--saturating #:uqsub (u16.8) (u16.8 u16.8) :cost 2 :suffix '(:8h))
+ (two-arg-u16.8* #:mul (u16.8) (u16.8 u16.8) :cost 2 :suffix '(:8h) :associative t)
+ (u16.8*-long #:umull (u32.4) (u16.8 u16.8) :cost 2 :suffix '(:4h) :associative t)
+ (u16.8*-long-hi #:umull2 (u32.4) (u16.8 u16.8) :cost 2 :suffix '(:8h) :associative t)
+ (two-arg-u16.8= #:cmeq (u16.8) (u16.8 u16.8) :cost 1 :suffix '(:8h))
+ (two-arg-u16.8/= nil (u16.8) (u16.8 u16.8) :cost 2 :associative t :encoding :fake-vop)
+ (two-arg-u16.8> #:cmhi (u16.8) (u16.8 u16.8) :cost 1 :suffix '(:8h))
+ (two-arg-u16.8< nil (u16.8) (u16.8 u16.8) :cost 1 :encoding :fake-vop)
+ (two-arg-u16.8>= #:cmhs (u16.8) (u16.8 u16.8) :cost 1 :suffix '(:8h))
+ (two-arg-u16.8<= nil (u16.8) (u16.8 u16.8) :cost 1 :encoding :fake-vop)
+ (u16.8-pair-min #:uminp (u16.8) (u16.8 u16.8) :cost 2 :suffix '(:8h) :associative t)
+ (u16.8-horizontal-min #:uminv (u16) (u16.8) :cost 2 :encoding :neon-int-result :suffix '(:8h) :associative t)
+ (u16.8-pair-max #:umaxp (u16.8) (u16.8 u16.8) :cost 2 :suffix '(:8h) :associative t)
+ (u16.8-horizontal-max #:umaxv (u16) (u16.8) :cost 2 :encoding :neon-int-result :suffix '(:8h) :associative t)
+ (u16.8-pair+ #:addp (u16.8) (u16.8 u16.8) :cost 2 :suffix '(:8h) :associative t)
+ (u16.8-horizontal+ #:addv (u16) (u16.8) :cost 2 :encoding :neon-int-result :suffix '(:8h) :associative t)
+ (u16.8-shiftr nil (u16.8) (u16.8 imm4) :cost 1 :encoding :custom)
+ (u16.8-shiftl nil (u16.8) (u16.8 imm4) :cost 1 :encoding :custom)
+ (u16.8-shiftl-long nil (u32.4) (u16.8 imm4) :cost 1 :encoding :custom)
+ (u16.8-shiftl-long-hi nil (u32.4) (u16.8 imm4) :cost 1 :encoding :custom)
+ (u16.8-bit-select #:bsl (u16.8) (u16.8 u16.8 u16.8) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (u16.8-lane-extract nil (u16) (u16.8 imm3) :cost 1 :encoding :custom)
+ (u16.8-lane-insert nil (u16.8) (u16.8 u16 imm3) :cost 1 :encoding :custom)
+ (u16.8-dup nil (u16.8) (u16.8 imm3) :cost 1 :encoding :custom)
+ (u16.8-ins nil (u16.8) (u16.8 u16.8 imm3 imm3) :cost 1 :encoding :custom)
+ (u16.8-transpose-even #:trn1 (u16.8) (u16.8 u16.8) :cost 1 :suffix '(:8h))
+ (u16.8-transpose-odd #:trn2 (u16.8) (u16.8 u16.8) :cost 1 :suffix '(:8h))
+ (u16.8-unzip-even #:uzp1 (u16.8) (u16.8 u16.8) :cost 1 :suffix '(:8h))
+ (u16.8-unzip-odd #:uzp2 (u16.8) (u16.8 u16.8) :cost 1 :suffix '(:8h))
+ (u16.8-zip-even #:zip1 (u16.8) (u16.8 u16.8) :cost 1 :suffix '(:8h))
+ (u16.8-zip-odd #:zip2 (u16.8) (u16.8 u16.8) :cost 1 :suffix '(:8h))
+ ;; u32
+ (u32!-from-p128 #:umov (u32) (p128) :cost 1 :encoding :move :suffix '(0 :s) :always-translatable nil)
+ ;; u32.4
+ (u32.4!-from-u32 nil (u32.4) (u32) :cost 1 :encoding :custom)
+ (u32.4!-from-p128 #:mov (u32.4) (p128) :cost 1 :encoding :move :always-translatable nil :suffix '(:16b))
+ (u32.4-from-f32.4 #:fcvtnu (u32.4) (f32.4) :cost 5 :suffix '(:4s))
+ (u32.4-from-u16.8 nil (u32.4) (u16.8) :cost 1 :encoding :custom)
+ (u32.4-from-u16.8-hi nil (u32.4) (u16.8) :cost 1 :encoding :custom)
+ (u32.4-from-u64.2 #:xtn (u32.4) (u64.2) :cost 1 :suffix '(:4h))
+ (u32.4-from-u64.2-hi #:xtn2 (u32.4) (u32.4 u64.2) :cost 1 :encoding :neon-rmw :suffix '(:8h))
+ (u32.4-from-u64.2-saturating #:uqxtn (u32.4) (u64.2) :cost 1 :suffix '(:4h))
+ (u32.4-from-u64.2-saturating-hi #:uqxtn2 (u32.4) (u32.4 u64.2) :cost 1 :encoding :neon-rmw :suffix '(:8h))
+ (make-u32.4 nil (u32.4) (u32 u32 u32 u32) :cost 1 :encoding :fake-vop)
+ (u32.4-values nil (u32 u32 u32 u32) (u32.4) :cost 1 :encoding :fake-vop)
+ (u32.4-broadcast #:dup (u32.4) (u32) :cost 1 :suffix '(:4s))
+ (two-arg-u32.4-and #:and (u32.4) (u32.4 u32.4) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u32.4-or #:orr (u32.4) (u32.4 u32.4) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u32.4-xor #:eor (u32.4) (u32.4 u32.4) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u32.4-max #:umax (u32.4) (u32.4 u32.4) :cost 3 :associative t :suffix '(:4s))
+ (two-arg-u32.4-min #:umin (u32.4) (u32.4 u32.4) :cost 3 :associative t :suffix '(:4s))
+ (u32.4-andc1 nil (u32.4) (u32.4 u32.4) :cost 1 :encoding :fake-vop)
+ (u32.4-andc2 #:bic (u32.4) (u32.4 u32.4) :cost 1 :suffix '(:16b))
+ (u32.4-not #:not (u32.4) (u32.4) :cost 1 :suffix '(:16b))
+ (two-arg-u32.4+ #:add (u32.4) (u32.4 u32.4) :cost 2 :suffix '(:4s) :associative t)
+ (two-arg-u32.4+-saturating #:uqadd (u32.4) (u32.4 u32.4) :cost 2 :suffix '(:4s) :associative t)
+ (two-arg-u32.4- #:sub (u32.4) (u32.4 u32.4) :cost 2 :suffix '(:4s))
+ (two-arg-u32.4--saturating #:uqsub (u32.4) (u32.4 u32.4) :cost 2 :suffix '(:4s))
+ (two-arg-u32.4* #:mul (u32.4) (u32.4 u32.4) :cost 2 :suffix '(:4s) :associative t)
+ (u32.4*-long #:umull (u64.2) (u32.4 u32.4) :cost 2 :suffix '(:2s) :associative t)
+ (u32.4*-long-hi #:umull2 (u64.2) (u32.4 u32.4) :cost 2 :suffix '(:4s) :associative t)
+ (two-arg-u32.4= #:cmeq (u32.4) (u32.4 u32.4) :cost 1 :suffix '(:4s))
+ (two-arg-u32.4/= nil (u32.4) (u32.4 u32.4) :cost 2 :associative t :encoding :fake-vop)
+ (two-arg-u32.4> #:cmhi (u32.4) (u32.4 u32.4) :cost 1 :suffix '(:4s))
+ (two-arg-u32.4< nil (u32.4) (u32.4 u32.4) :cost 1 :encoding :fake-vop)
+ (two-arg-u32.4>= #:cmhs (u32.4) (u32.4 u32.4) :cost 1 :suffix '(:4s))
+ (two-arg-u32.4<= nil (u32.4) (u32.4 u32.4) :cost 1 :encoding :fake-vop)
+ (u32.4-pair-min #:uminp (u32.4) (u32.4 u32.4) :cost 2 :suffix '(:4s) :associative t)
+ (u32.4-horizontal-min #:uminv (u32) (u32.4) :cost 2 :encoding :neon-int-result :suffix '(:4s) :associative t)
+ (u32.4-pair-max #:umaxp (u32.4) (u32.4 u32.4) :cost 2 :suffix '(:4s) :associative t)
+ (u32.4-horizontal-max #:umaxv (u32) (u32.4) :cost 2 :encoding :neon-int-result :suffix '(:4s) :associative t)
+ (u32.4-pair+ #:addp (u32.4) (u32.4 u32.4) :cost 2 :suffix '(:4s) :associative t)
+ (u32.4-horizontal+ #:addv (u32) (u32.4) :cost 2 :encoding :neon-int-result :suffix '(:4s) :associative t)
+ (u32.4-shiftr nil (u32.4) (u32.4 imm5) :cost 1 :encoding :custom)
+ (u32.4-shiftl nil (u32.4) (u32.4 imm5) :cost 1 :encoding :custom)
+ (u32.4-shiftl-long nil (u64.2) (u32.4 imm5) :cost 1 :encoding :custom)
+ (u32.4-shiftl-long-hi nil (u64.2) (u32.4 imm5) :cost 1 :encoding :custom)
+ (u32.4-bit-select #:bsl (u32.4) (u32.4 u32.4 u32.4) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (u32.4-lane-extract nil (u32) (u32.4 imm2) :cost 1 :encoding :custom)
+ (u32.4-lane-insert nil (u32.4) (u32.4 u32 imm2) :cost 1 :encoding :custom)
+ (u32.4-dup nil (u32.4) (u32.4 imm2) :cost 1 :encoding :custom)
+ (u32.4-ins nil (u32.4) (u32.4 u32.4 imm2 imm2) :cost 1 :encoding :custom)
+ (u32.4-transpose-even #:trn1 (u32.4) (u32.4 u32.4) :cost 1 :suffix '(:4s))
+ (u32.4-transpose-odd #:trn2 (u32.4) (u32.4 u32.4) :cost 1 :suffix '(:4s))
+ (u32.4-unzip-even #:uzp1 (u32.4) (u32.4 u32.4) :cost 1 :suffix '(:4s))
+ (u32.4-unzip-odd #:uzp2 (u32.4) (u32.4 u32.4) :cost 1 :suffix '(:4s))
+ (u32.4-zip-even #:zip1 (u32.4) (u32.4 u32.4) :cost 1 :suffix '(:4s))
+ (u32.4-zip-odd #:zip2 (u32.4) (u32.4 u32.4) :cost 1 :suffix '(:4s))
+ ;; u64
+ (u64!-from-p128 #:umov (u64) (p128) :cost 1 :encoding :move :suffix '(0 :d) :always-translatable nil)
+ ;; u64.2
+ (u64.2!-from-u64 nil (u64.2) (u64) :cost 1 :encoding :custom)
+ (u64.2!-from-p128 #:mov (u64.2) (p128) :cost 1 :encoding :move :always-translatable nil :suffix '(:16b))
+ (u64.2-from-f64.2 #:fcvtnu (u64.2) (f64.2) :cost 5 :suffix '(:2d))
+ (u64.2-from-u32.4 nil (u64.2) (u32.4) :cost 1 :encoding :custom)
+ (u64.2-from-u32.4-hi nil (u64.2) (u32.4) :cost 1 :encoding :custom)
+ (make-u64.2 nil (u64.2) (u64 u64) :cost 1 :encoding :fake-vop)
+ (u64.2-values nil (u64 u64) (u64.2) :cost 1 :encoding :fake-vop)
+ (u64.2-broadcast #:dup (u64.2) (u64) :cost 1 :suffix '(:2d))
+ (two-arg-u64.2-and #:and (u64.2) (u64.2 u64.2) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u64.2-or #:orr (u64.2) (u64.2 u64.2) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u64.2-xor #:eor (u64.2) (u64.2 u64.2) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-u64.2-max #:umax (u64.2) (u64.2 u64.2) :cost 3 :associative t :suffix '(:2d))
+ (two-arg-u64.2-min #:umin (u64.2) (u64.2 u64.2) :cost 3 :associative t :suffix '(:2d))
+ (u64.2-andc1 nil (u64.2) (u64.2 u64.2) :cost 1 :encoding :fake-vop)
+ (u64.2-andc2 #:bic (u64.2) (u64.2 u64.2) :cost 1 :suffix '(:16b))
+ (u64.2-not #:not (u64.2) (u64.2) :cost 1 :suffix '(:16b))
+ (two-arg-u64.2+ #:add (u64.2) (u64.2 u64.2) :cost 2 :suffix '(:2d) :associative t)
+ (two-arg-u64.2+-saturating #:uqadd (u64.2) (u64.2 u64.2) :cost 2 :suffix '(:2d) :associative t)
+ (two-arg-u64.2- #:sub (u64.2) (u64.2 u64.2) :cost 2 :suffix '(:2d))
+ (two-arg-u64.2--saturating #:uqsub (u64.2) (u64.2 u64.2) :cost 2 :suffix '(:2d))
+ (two-arg-u64.2* #:mul (u64.2) (u64.2 u64.2) :cost 2 :suffix '(:2d) :associative t)
+ (two-arg-u64.2= #:cmeq (u64.2) (u64.2 u64.2) :cost 1 :suffix '(:2d))
+ (two-arg-u64.2/= nil (u64.2) (u64.2 u64.2) :cost 2 :associative t :encoding :fake-vop)
+ (two-arg-u64.2> #:cmhi (u64.2) (u64.2 u64.2) :cost 1 :suffix '(:2d))
+ (two-arg-u64.2< nil (u64.2) (u64.2 u64.2) :cost 1 :encoding :fake-vop)
+ (two-arg-u64.2>= #:cmhs (u64.2) (u64.2 u64.2) :cost 1 :suffix '(:2d))
+ (two-arg-u64.2<= nil (u64.2) (u64.2 u64.2) :cost 1 :encoding :fake-vop)
+ (u64.2-pair-min nil (u64.2) (u64.2 u64.2) :cost 2 :encoding :fake-vop :associative t)
+ (u64.2-horizontal-min nil (u64) (u64.2) :cost 2 :encoding :fake-vop :associative t)
+ (u64.2-pair-max nil (u64.2) (u64.2 u64.2) :cost 2 :encoding :fake-vop :associative t)
+ (u64.2-horizontal-max nil (u64) (u64.2) :cost 2 :encoding :fake-vop :associative t)
+ (u64.2-pair+ nil (u64.2) (u64.2 u64.2) :cost 2 :encoding :fake-vop :associative t)
+ (u64.2-horizontal+ nil (u64) (u64.2) :cost 2 :encoding :fake-vop :associative t)
+ (u64.2-shiftr nil (u64.2) (u64.2 imm6) :cost 1 :encoding :custom)
+ (u64.2-shiftl nil (u64.2) (u64.2 imm6) :cost 1 :encoding :custom)
+ (u64.2-bit-select #:bsl (u64.2) (u64.2 u64.2 u64.2) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (u64.2-lane-extract nil (u64) (u64.2 imm1) :cost 1 :encoding :custom)
+ (u64.2-lane-insert nil (u64.2) (u64.2 u64 imm1) :cost 1 :encoding :custom)
+ (u64.2-dup nil (u64.2) (u64.2 imm1) :cost 1 :encoding :custom)
+ (u64.2-ins nil (u64.2) (u64.2 u64.2 imm1 imm1) :cost 1 :encoding :custom)
+ (u64.2-transpose-even #:trn1 (u64.2) (u64.2 u64.2) :cost 1 :suffix '(:2d))
+ (u64.2-transpose-odd #:trn2 (u64.2) (u64.2 u64.2) :cost 1 :suffix '(:2d))
+ (u64.2-unzip-even #:uzp1 (u64.2) (u64.2 u64.2) :cost 1 :suffix '(:2d))
+ (u64.2-unzip-odd #:uzp2 (u64.2) (u64.2 u64.2) :cost 1 :suffix '(:2d))
+ (u64.2-zip-even #:zip1 (u64.2) (u64.2 u64.2) :cost 1 :suffix '(:2d))
+ (u64.2-zip-odd #:zip2 (u64.2) (u64.2 u64.2) :cost 1 :suffix '(:2d))
+ ;; s8
+ (s8!-from-p128 #:smov (s8) (p128) :cost 1 :encoding :move :suffix '(0 :b) :always-translatable nil)
+ ;; s8.16
+ (s8.16!-from-s8 nil (s8.16) (s8) :cost 1 :encoding :custom)
+ (s8.16!-from-p128 #:mov (s8.16) (p128) :cost 1 :encoding :move :always-translatable nil :suffix '(:16b))
+ (s8.16-from-s16.8 #:xtn (s8.16) (s16.8) :cost 1 :suffix '(:8b))
+ (s8.16-from-s16.8-hi #:xtn2 (s8.16) (s8.16 s16.8) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (s8.16-from-s16.8-saturating #:sqxtn (s8.16) (s16.8) :cost 1 :suffix '(:8b))
+ (s8.16-from-s16.8-saturating-hi #:sqxtn2 (s8.16) (s8.16 s16.8) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (make-s8.16 nil (s8.16) (s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8) :cost 1 :encoding :fake-vop)
+ (s8.16-values nil (s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8) (s8.16) :cost 1 :encoding :fake-vop)
+ (s8.16-broadcast #:dup (s8.16) (s8) :cost 1 :suffix '(:16b))
+ (two-arg-s8.16-and #:and (s8.16) (s8.16 s8.16) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s8.16-or #:orr (s8.16) (s8.16 s8.16) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s8.16-xor #:eor (s8.16) (s8.16 s8.16) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s8.16-max #:smax (s8.16) (s8.16 s8.16) :cost 3 :associative t :suffix '(:16b))
+ (two-arg-s8.16-min #:smin (s8.16) (s8.16 s8.16) :cost 3 :associative t :suffix '(:16b))
+ (s8.16-andc1 nil (s8.16) (s8.16 s8.16) :cost 1 :encoding :fake-vop)
+ (s8.16-andc2 #:bic (s8.16) (s8.16 s8.16) :cost 1 :suffix '(:16b))
+ (s8.16-not #:not (s8.16) (s8.16) :cost 1 :suffix '(:16b))
+ (two-arg-s8.16+ #:add (s8.16) (s8.16 s8.16) :cost 2 :suffix '(:16b) :associative t)
+ (two-arg-s8.16+-saturating #:sqadd (s8.16) (s8.16 s8.16) :cost 2 :suffix '(:16b) :associative t)
+ (two-arg-s8.16- #:sub (s8.16) (s8.16 s8.16) :cost 2 :suffix '(:16b))
+ (two-arg-s8.16--saturating #:sqsub (s8.16) (s8.16 s8.16) :cost 2 :suffix '(:16b))
+ (two-arg-s8.16* #:mul (s8.16) (s8.16 s8.16) :cost 2 :suffix '(:16b) :associative t)
+ (two-arg-s8.16= #:cmeq (u8.16) (s8.16 s8.16) :cost 1 :suffix '(:16b))
+ (two-arg-s8.16/= nil (u8.16) (s8.16 s8.16) :cost 2 :associative t :encoding :fake-vop)
+ (two-arg-s8.16> #:cmgt (u8.16) (s8.16 s8.16) :cost 1 :suffix '(:16b))
+ (two-arg-s8.16< nil (u8.16) (s8.16 s8.16) :cost 1 :encoding :fake-vop)
+ (two-arg-s8.16>= #:cmge (u8.16) (s8.16 s8.16) :cost 1 :suffix '(:16b))
+ (two-arg-s8.16<= nil (u8.16) (s8.16 s8.16) :cost 1 :encoding :fake-vop)
+ (s8.16-pair-min #:sminp (s8.16) (s8.16 s8.16) :cost 2 :suffix '(:16b) :associative t)
+ (s8.16-horizontal-min #:sminv (s8) (s8.16) :cost 2 :encoding :neon-int-result :suffix '(:16b) :associative t)
+ (s8.16-pair-max #:smaxp (s8.16) (s8.16 s8.16) :cost 2 :suffix '(:16b) :associative t)
+ (s8.16-horizontal-max #:smaxv (s8) (s8.16) :cost 2 :encoding :neon-int-result :suffix '(:16b) :associative t)
+ (s8.16-pair+ #:addp (s8.16) (s8.16 s8.16) :cost 2 :suffix '(:16b) :associative t)
+ (s8.16-horizontal+ #:addv (s8) (s8.16) :cost 2 :encoding :neon-int-result :suffix '(:16b) :associative t)
+ (s8.16-shiftr nil (s8.16) (s8.16 imm3) :cost 1 :encoding :custom)
+ (s8.16-shiftl nil (s8.16) (s8.16 imm3) :cost 1 :encoding :custom)
+ (s8.16-shiftl-long nil (s16.8) (s8.16 imm3) :cost 1 :encoding :custom)
+ (s8.16-shiftl-long-hi nil (s16.8) (s8.16 imm3) :cost 1 :encoding :custom)
+ (s8.16-bit-select #:bsl (s8.16) (u8.16 s8.16 s8.16) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (s8.16-lane-extract nil (s8) (s8.16 imm4) :cost 1 :encoding :custom)
+ (s8.16-lane-insert nil (s8.16) (s8.16 s8 imm4) :cost 1 :encoding :custom)
+ (s8.16-dup nil (s8.16) (s8.16 imm4) :cost 1 :encoding :custom)
+ (s8.16-ins nil (s8.16) (s8.16 s8.16 imm4 imm4) :cost 1 :encoding :custom)
+ (s8.16-transpose-even #:trn1 (s8.16) (s8.16 s8.16) :cost 1 :suffix '(:16b))
+ (s8.16-transpose-odd #:trn2 (s8.16) (s8.16 s8.16) :cost 1 :suffix '(:16b))
+ (s8.16-unzip-even #:uzp1 (s8.16) (s8.16 s8.16) :cost 1 :suffix '(:16b))
+ (s8.16-unzip-odd #:uzp2 (s8.16) (s8.16 s8.16) :cost 1 :suffix '(:16b))
+ (s8.16-zip-even #:zip1 (s8.16) (s8.16 s8.16) :cost 1 :suffix '(:16b))
+ (s8.16-zip-odd #:zip2 (s8.16) (s8.16 s8.16) :cost 1 :suffix '(:16b))
+ ;; s16
+ (s16!-from-p128 #:smov (s16) (p128) :cost 1 :encoding :move :suffix '(0 :h) :always-translatable nil)
+ ;; s16.8
+ (s16.8!-from-s16 nil (s16.8) (s16) :cost 1 :encoding :custom)
+ (s16.8!-from-p128 #:mov (s16.8) (p128) :cost 1 :encoding :move :always-translatable nil :suffix '(:16b))
+ (s16.8-from-s8.16 nil (s16.8) (s8.16) :cost 1 :encoding :custom)
+ (s16.8-from-s8.16-hi nil (s16.8) (s8.16) :cost 1 :encoding :custom)
+ (s16.8-from-s32.4 #:xtn (s16.8) (s32.4) :cost 1 :suffix '(:4h))
+ (s16.8-from-s32.4-hi #:xtn2 (s16.8) (s16.8 s32.4) :cost 1 :encoding :neon-rmw :suffix '(:8h))
+ (s16.8-from-s32.4-saturating #:sqxtn (s16.8) (s32.4) :cost 1 :suffix '(:4h))
+ (s16.8-from-s32.4-saturating-hi #:sqxtn2 (s16.8) (s16.8 s32.4) :cost 1 :encoding :neon-rmw :suffix '(:8h))
+ (make-s16.8 nil (s16.8) (s16 s16 s16 s16 s16 s16 s16 s16) :cost 1 :encoding :fake-vop)
+ (s16.8-values nil (s16 s16 s16 s16 s16 s16 s16 s16) (s16.8) :cost 1 :encoding :fake-vop)
+ (s16.8-broadcast #:dup (s16.8) (s16) :cost 1 :suffix '(:8h))
+ (two-arg-s16.8-and #:and (s16.8) (s16.8 s16.8) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s16.8-or #:orr (s16.8) (s16.8 s16.8) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s16.8-xor #:eor (s16.8) (s16.8 s16.8) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s16.8-max #:smax (s16.8) (s16.8 s16.8) :cost 3 :associative t :suffix '(:8h))
+ (two-arg-s16.8-min #:smin (s16.8) (s16.8 s16.8) :cost 3 :associative t :suffix '(:8h))
+ (s16.8-andc1 nil (s16.8) (s16.8 s16.8) :cost 1 :encoding :fake-vop)
+ (s16.8-andc2 #:bic (s16.8) (s16.8 s16.8) :cost 1 :suffix '(:16b))
+ (s16.8-not #:not (s16.8) (s16.8) :cost 1 :suffix '(:16b))
+ (two-arg-s16.8+ #:add (s16.8) (s16.8 s16.8) :cost 2 :suffix '(:8h) :associative t)
+ (two-arg-s16.8+-saturating #:sqadd (s16.8) (s16.8 s16.8) :cost 2 :suffix '(:8h) :associative t)
+ (two-arg-s16.8- #:sub (s16.8) (s16.8 s16.8) :cost 2 :suffix '(:8h))
+ (two-arg-s16.8--saturating #:sqsub (s16.8) (s16.8 s16.8) :cost 2 :suffix '(:8h))
+ (two-arg-s16.8* #:mul (s16.8) (s16.8 s16.8) :cost 2 :suffix '(:8h) :associative t)
+ (two-arg-s16.8= #:cmeq (u16.8) (s16.8 s16.8) :cost 1 :suffix '(:8h))
+ (two-arg-s16.8/= nil (u16.8) (s16.8 s16.8) :cost 2 :associative t :encoding :fake-vop)
+ (two-arg-s16.8> #:cmgt (u16.8) (s16.8 s16.8) :cost 1 :suffix '(:8h))
+ (two-arg-s16.8< nil (u16.8) (s16.8 s16.8) :cost 1 :encoding :fake-vop)
+ (two-arg-s16.8>= #:cmge (u16.8) (s16.8 s16.8) :cost 1 :suffix '(:8h))
+ (two-arg-s16.8<= nil (u16.8) (s16.8 s16.8) :cost 1 :encoding :fake-vop)
+ (s16.8-pair-min #:sminp (s16.8) (s16.8 s16.8) :cost 2 :suffix '(:8h) :associative t)
+ (s16.8-horizontal-min #:sminv (s16) (s16.8) :cost 2 :encoding :neon-int-result :suffix '(:8h) :associative t)
+ (s16.8-pair-max #:smaxp (s16.8) (s16.8 s16.8) :cost 2 :suffix '(:8h) :associative t)
+ (s16.8-horizontal-max #:smaxv (s16) (s16.8) :cost 2 :encoding :neon-int-result :suffix '(:8h) :associative t)
+ (s16.8-pair+ #:addp (s16.8) (s16.8 s16.8) :cost 2 :suffix '(:8h) :associative t)
+ (s16.8-horizontal+ #:addv (s16) (s16.8) :cost 2 :encoding :neon-int-result :suffix '(:8h) :associative t)
+ (s16.8-shiftr nil (s16.8) (s16.8 imm4) :cost 1 :encoding :custom)
+ (s16.8-shiftl nil (s16.8) (s16.8 imm4) :cost 1 :encoding :custom)
+ (s16.8-shiftl-long nil (s32.4) (s16.8 imm4) :cost 1 :encoding :custom)
+ (s16.8-shiftl-long-hi nil (s32.4) (s16.8 imm4) :cost 1 :encoding :custom)
+ (s16.8-bit-select #:bsl (s16.8) (u16.8 s16.8 s16.8) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (s16.8-lane-extract nil (s16) (s16.8 imm3) :cost 1 :encoding :custom)
+ (s16.8-lane-insert nil (s16.8) (s16.8 s16 imm3) :cost 1 :encoding :custom)
+ (s16.8-dup nil (s16.8) (s16.8 imm3) :cost 1 :encoding :custom)
+ (s16.8-ins nil (s16.8) (s16.8 s16.8 imm3 imm3) :cost 1 :encoding :custom)
+ (s16.8-transpose-even #:trn1 (s16.8) (s16.8 s16.8) :cost 1 :suffix '(:8h))
+ (s16.8-transpose-odd #:trn2 (s16.8) (s16.8 s16.8) :cost 1 :suffix '(:8h))
+ (s16.8-unzip-even #:uzp1 (s16.8) (s16.8 s16.8) :cost 1 :suffix '(:8h))
+ (s16.8-unzip-odd #:uzp2 (s16.8) (s16.8 s16.8) :cost 1 :suffix '(:8h))
+ (s16.8-zip-even #:zip1 (s16.8) (s16.8 s16.8) :cost 1 :suffix '(:8h))
+ (s16.8-zip-odd #:zip2 (s16.8) (s16.8 s16.8) :cost 1 :suffix '(:8h))
+ ;; s32
+ (s32!-from-p128 #:smov (s32) (p128) :cost 1 :encoding :move :suffix '(0 :s) :always-translatable nil)
+ ;; s32.4
+ (s32.4!-from-s32 nil (s32.4) (s32) :cost 1 :encoding :custom)
+ (s32.4!-from-p128 #:mov (s32.4) (p128) :cost 1 :encoding :move :always-translatable nil :suffix '(:16b))
+ (s32.4-from-f32.4 #:fcvtns (s32.4) (f32.4) :cost 5 :suffix '(:4s))
+ (s32.4-from-s16.8 nil (s32.4) (s16.8) :cost 1 :encoding :custom)
+ (s32.4-from-s16.8-hi nil (s32.4) (s16.8) :cost 1 :encoding :custom)
+ (s32.4-from-s64.2 #:xtn (s32.4) (s64.2) :cost 1 :suffix '(:4h))
+ (s32.4-from-s64.2-hi #:xtn2 (s32.4) (s32.4 s64.2) :cost 1 :encoding :neon-rmw :suffix '(:8h))
+ (s32.4-from-s64.2-saturating #:sqxtn (s32.4) (s64.2) :cost 1 :suffix '(:4h))
+ (s32.4-from-s64.2-saturating-hi #:sqxtn2 (s32.4) (s32.4 s64.2) :cost 1 :encoding :neon-rmw :suffix '(:8h))
+ (make-s32.4 nil (s32.4) (s32 s32 s32 s32) :cost 1 :encoding :fake-vop)
+ (s32.4-values nil (s32 s32 s32 s32) (s32.4) :cost 1 :encoding :fake-vop)
+ (s32.4-broadcast #:dup (s32.4) (s32) :cost 1 :suffix '(:4s))
+ (two-arg-s32.4-and #:and (s32.4) (s32.4 s32.4) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s32.4-or #:orr (s32.4) (s32.4 s32.4) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s32.4-xor #:eor (s32.4) (s32.4 s32.4) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s32.4-max #:smax (s32.4) (s32.4 s32.4) :cost 3 :associative t :suffix '(:4s))
+ (two-arg-s32.4-min #:smin (s32.4) (s32.4 s32.4) :cost 3 :associative t :suffix '(:4s))
+ (s32.4-andc1 nil (s32.4) (s32.4 s32.4) :cost 1 :encoding :fake-vop)
+ (s32.4-andc2 #:bic (s32.4) (s32.4 s32.4) :cost 1 :suffix '(:16b))
+ (s32.4-not #:not (s32.4) (s32.4) :cost 1 :suffix '(:16b))
+ (two-arg-s32.4+ #:add (s32.4) (s32.4 s32.4) :cost 2 :suffix '(:4s) :associative t)
+ (two-arg-s32.4+-saturating #:sqadd (s32.4) (s32.4 s32.4) :cost 2 :suffix '(:4s) :associative t)
+ (two-arg-s32.4- #:sub (s32.4) (s32.4 s32.4) :cost 2 :suffix '(:4s))
+ (two-arg-s32.4--saturating #:sqsub (s32.4) (s32.4 s32.4) :cost 2 :suffix '(:4s))
+ (two-arg-s32.4* #:mul (s32.4) (s32.4 s32.4) :cost 2 :suffix '(:4s) :associative t)
+ (two-arg-s32.4= #:cmeq (u32.4) (s32.4 s32.4) :cost 1 :suffix '(:4s))
+ (two-arg-s32.4/= nil (u32.4) (s32.4 s32.4) :cost 2 :associative t :encoding :fake-vop)
+ (two-arg-s32.4> #:cmgt (u32.4) (s32.4 s32.4) :cost 1 :suffix '(:4s))
+ (two-arg-s32.4< nil (u32.4) (s32.4 s32.4) :cost 1 :encoding :fake-vop)
+ (two-arg-s32.4>= #:cmge (u32.4) (s32.4 s32.4) :cost 1 :suffix '(:4s))
+ (two-arg-s32.4<= nil (u32.4) (s32.4 s32.4) :cost 1 :encoding :fake-vop)
+ (s32.4-pair-min #:sminp (s32.4) (s32.4 s32.4) :cost 2 :suffix '(:4s) :associative t)
+ (s32.4-horizontal-min #:sminv (s32) (s32.4) :cost 2 :encoding :neon-int-result :suffix '(:4s) :associative t)
+ (s32.4-pair-max #:smaxp (s32.4) (s32.4 s32.4) :cost 2 :suffix '(:4s) :associative t)
+ (s32.4-horizontal-max #:smaxv (s32) (s32.4) :cost 2 :encoding :neon-int-result :suffix '(:4s) :associative t)
+ (s32.4-pair+ #:addp (s32.4) (s32.4 s32.4) :cost 2 :suffix '(:4s) :associative t)
+ (s32.4-horizontal+ #:addv (s32) (s32.4) :cost 2 :encoding :neon-int-result :suffix '(:4s) :associative t)
+ (s32.4-shiftr nil (s32.4) (s32.4 imm5) :cost 1 :encoding :custom)
+ (s32.4-shiftl nil (s32.4) (s32.4 imm5) :cost 1 :encoding :custom)
+ (s32.4-shiftl-long nil (s64.2) (s32.4 imm5) :cost 1 :encoding :custom)
+ (s32.4-shiftl-long-hi nil (s64.2) (s32.4 imm5) :cost 1 :encoding :custom)
+ (s32.4-bit-select #:bsl (s32.4) (u32.4 s32.4 s32.4) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (s32.4-lane-extract nil (s32) (s32.4 imm2) :cost 1 :encoding :custom)
+ (s32.4-lane-insert nil (s32.4) (s32.4 s32 imm2) :cost 1 :encoding :custom)
+ (s32.4-dup nil (s32.4) (s32.4 imm2) :cost 1 :encoding :custom)
+ (s32.4-ins nil (s32.4) (s32.4 s32.4 imm2 imm2) :cost 1 :encoding :custom)
+ (s32.4-transpose-even #:trn1 (s32.4) (s32.4 s32.4) :cost 1 :suffix '(:4s))
+ (s32.4-transpose-odd #:trn2 (s32.4) (s32.4 s32.4) :cost 1 :suffix '(:4s))
+ (s32.4-unzip-even #:uzp1 (s32.4) (s32.4 s32.4) :cost 1 :suffix '(:4s))
+ (s32.4-unzip-odd #:uzp2 (s32.4) (s32.4 s32.4) :cost 1 :suffix '(:4s))
+ (s32.4-zip-even #:zip1 (s32.4) (s32.4 s32.4) :cost 1 :suffix '(:4s))
+ (s32.4-zip-odd #:zip2 (s32.4) (s32.4 s32.4) :cost 1 :suffix '(:4s))
+ ;; s64
+ (s64!-from-p128 #:umov (s64) (p128) :cost 1 :encoding :move :suffix '(0 :d) :always-translatable nil)
+ ;; s64.2
+ (s64.2!-from-s64 nil (s64.2) (s64) :cost 1 :encoding :custom)
+ (s64.2!-from-p128 #:mov (s64.2) (p128) :cost 1 :encoding :move :always-translatable nil :suffix '(:16b))
+ (s64.2-from-f64.2 #:fcvtns (s64.2) (f64.2) :cost 5 :suffix '(:2d))
+ (s64.2-from-s32.4 nil (s64.2) (s32.4) :cost 1 :encoding :custom)
+ (s64.2-from-s32.4-hi nil (s64.2) (s32.4) :cost 1 :encoding :custom)
+ (make-s64.2 nil (s64.2) (s64 s64) :cost 1 :encoding :fake-vop)
+ (s64.2-values nil (s64 s64) (s64.2) :cost 1 :encoding :fake-vop)
+ (s64.2-broadcast #:dup (s64.2) (s64) :cost 1 :suffix '(:2d))
+ (two-arg-s64.2-and #:and (s64.2) (s64.2 s64.2) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s64.2-or #:orr (s64.2) (s64.2 s64.2) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s64.2-xor #:eor (s64.2) (s64.2 s64.2) :cost 1 :associative t :suffix '(:16b))
+ (two-arg-s64.2-max #:smax (s64.2) (s64.2 s64.2) :cost 3 :associative t :suffix '(:2d))
+ (two-arg-s64.2-min #:smin (s64.2) (s64.2 s64.2) :cost 3 :associative t :suffix '(:2d))
+ (s64.2-andc1 nil (s64.2) (s64.2 s64.2) :cost 1 :encoding :fake-vop)
+ (s64.2-andc2 #:bic (s64.2) (s64.2 s64.2) :cost 1 :suffix '(:16b))
+ (s64.2-not #:not (s64.2) (s64.2) :cost 1 :suffix '(:16b))
+ (two-arg-s64.2+ #:add (s64.2) (s64.2 s64.2) :cost 2 :suffix '(:2d) :associative t)
+ (two-arg-s64.2+-saturating #:sqadd (s64.2) (s64.2 s64.2) :cost 2 :suffix '(:2d) :associative t)
+ (two-arg-s64.2- #:sub (s64.2) (s64.2 s64.2) :cost 2 :suffix '(:2d))
+ (two-arg-s64.2--saturating #:sqsub (s64.2) (s64.2 s64.2) :cost 2 :suffix '(:2d))
+ (two-arg-s64.2* #:mul (s64.2) (s64.2 s64.2) :cost 2 :suffix '(:2d) :associative t)
+ (two-arg-s64.2= #:cmeq (u64.2) (s64.2 s64.2) :cost 1 :suffix '(:2d))
+ (two-arg-s64.2/= nil (u64.2) (s64.2 s64.2) :cost 2 :associative t :encoding :fake-vop)
+ (two-arg-s64.2> #:cmgt (u64.2) (s64.2 s64.2) :cost 1 :suffix '(:2d))
+ (two-arg-s64.2< nil (u64.2) (s64.2 s64.2) :cost 1 :encoding :fake-vop)
+ (two-arg-s64.2>= #:cmge (u64.2) (s64.2 s64.2) :cost 1 :suffix '(:2d))
+ (two-arg-s64.2<= nil (u64.2) (s64.2 s64.2) :cost 1 :encoding :fake-vop)
+ (s64.2-pair-min nil (s64.2) (s64.2 s64.2) :cost 2 :encoding :fake-vop :associative t)
+ (s64.2-horizontal-min nil (s64) (s64.2) :cost 2 :encoding :fake-vop :associative t)
+ (s64.2-pair-max nil (s64.2) (s64.2 s64.2) :cost 2 :encoding :fake-vop :associative t)
+ (s64.2-horizontal-max nil (s64) (s64.2) :cost 2 :encoding :fake-vop :associative t)
+ (s64.2-pair+ nil (s64.2) (s64.2 s64.2) :cost 2 :encoding :fake-vop :associative t)
+ (s64.2-horizontal+ nil (s64) (s64.2) :cost 2 :encoding :fake-vop :associative t)
+ (s64.2-shiftr nil (s64.2) (s64.2 imm6) :cost 1 :encoding :custom)
+ (s64.2-shiftl nil (s64.2) (s64.2 imm6) :cost 1 :encoding :custom)
+ (s64.2-bit-select #:bsl (s64.2) (u64.2 s64.2 s64.2) :cost 1 :encoding :neon-rmw :suffix '(:16b))
+ (s64.2-lane-extract nil (s64) (s64.2 imm1) :cost 1 :encoding :custom)
+ (s64.2-lane-insert nil (s64.2) (s64.2 s64 imm1) :cost 1 :encoding :custom)
+ (s64.2-dup nil (s64.2) (s64.2 imm1) :cost 1 :encoding :custom)
+ (s64.2-ins nil (s64.2) (s64.2 s64.2 imm1 imm1) :cost 1 :encoding :custom)
+ (s64.2-transpose-even #:trn1 (s64.2) (s64.2 s64.2) :cost 1 :suffix '(:2d))
+ (s64.2-transpose-odd #:trn2 (s64.2) (s64.2 s64.2) :cost 1 :suffix '(:2d))
+ (s64.2-unzip-even #:uzp1 (s64.2) (s64.2 s64.2) :cost 1 :suffix '(:2d))
+ (s64.2-unzip-odd #:uzp2 (s64.2) (s64.2 s64.2) :cost 1 :suffix '(:2d))
+ (s64.2-zip-even #:zip1 (s64.2) (s64.2 s64.2) :cost 1 :suffix '(:2d))
+ (s64.2-zip-odd #:zip2 (s64.2) (s64.2 s64.2) :cost 1 :suffix '(:2d)))
+ (:loads
+ #+sb-unicode
+ (u32.4-load-from-string #:ldr u32.4 charvec char-array u32.4-string-ref u32.4-row-major-string-ref)
+ (f32.4-load #:ldr f32.4 f32vec f32-array f32.4-aref f32.4-row-major-aref)
+ (f64.2-load #:ldr f64.2 f64vec f64-array f64.2-aref f64.2-row-major-aref)
+ (u8.16-load #:ldr u8.16 u8vec u8-array u8.16-aref u8.16-row-major-aref)
+ (u16.8-load #:ldr u16.8 u16vec u16-array u16.8-aref u16.8-row-major-aref)
+ (u32.4-load #:ldr u32.4 u32vec u32-array u32.4-aref u32.4-row-major-aref)
+ (u64.2-load #:ldr u64.2 u64vec u64-array u64.2-aref u64.2-row-major-aref)
+ (s8.16-load #:ldr s8.16 s8vec s8-array s8.16-aref s8.16-row-major-aref)
+ (s16.8-load #:ldr s16.8 s16vec s16-array s16.8-aref s16.8-row-major-aref)
+ (s32.4-load #:ldr s32.4 s32vec s32-array s32.4-aref s32.4-row-major-aref)
+ (s64.2-load #:ldr s64.2 s64vec s64-array s64.2-aref s64.2-row-major-aref))
+ (:stores
+ #+sb-unicode
+ (u32.4-store-into-string #:str u32.4 charvec char-array u32.4-string-ref u32.4-row-major-string-ref)
+ (f32.4-store #:str f32.4 f32vec f32-array f32.4-aref f32.4-row-major-aref)
+ (f64.2-store #:str f64.2 f64vec f64-array f64.2-aref f64.2-row-major-aref)
+ (u8.16-store #:str u8.16 u8vec u8-array u8.16-aref u8.16-row-major-aref)
+ (u16.8-store #:str u16.8 u16vec u16-array u16.8-aref u16.8-row-major-aref)
+ (u32.4-store #:str u32.4 u32vec u32-array u32.4-aref u32.4-row-major-aref)
+ (u64.2-store #:str u64.2 u64vec u64-array u64.2-aref u64.2-row-major-aref)
+ (s8.16-store #:str s8.16 s8vec s8-array s8.16-aref s8.16-row-major-aref)
+ (s16.8-store #:str s16.8 s16vec s16-array s16.8-aref s16.8-row-major-aref)
+ (s32.4-store #:str s32.4 s32vec s32-array s32.4-aref s32.4-row-major-aref)
+ (s64.2-store #:str s64.2 s64vec s64-array s64.2-aref s64.2-row-major-aref))
+ (:associatives
+ (f32.4-and two-arg-f32.4-and +f32-true+)
+ (f32.4-or two-arg-f32.4-or +f32-false+)
+ (f32.4-xor two-arg-f32.4-xor +f32-false+)
+ (f32.4-max two-arg-f32.4-max nil)
+ (f32.4-min two-arg-f32.4-min nil)
+ (f32.4+ two-arg-f32.4+ 0f0)
+ (f32.4* two-arg-f32.4* 1f0)
+ (f64.2-and two-arg-f64.2-and +f64-true+)
+ (f64.2-or two-arg-f64.2-or +f64-false+)
+ (f64.2-xor two-arg-f64.2-xor +f64-false+)
+ (f64.2-max two-arg-f64.2-max nil)
+ (f64.2-min two-arg-f64.2-min nil)
+ (f64.2+ two-arg-f64.2+ 0f0)
+ (f64.2* two-arg-f64.2* 1f0)
+ (u8.16-and two-arg-u8.16-and +u8-true+)
+ (u8.16-or two-arg-u8.16-or +u8-false+)
+ (u8.16-xor two-arg-u8.16-xor +u8-false+)
+ (u8.16-max two-arg-u8.16-max nil)
+ (u8.16-min two-arg-u8.16-min nil)
+ (u8.16+ two-arg-u8.16+ 0)
+ (u8.16+-saturating two-arg-u8.16+-saturating 0)
+ (u8.16* two-arg-u8.16* 1)
+ (u16.8-and two-arg-u16.8-and +u16-true+)
+ (u16.8-or two-arg-u16.8-or +u16-false+)
+ (u16.8-xor two-arg-u16.8-xor +u16-false+)
+ (u16.8-max two-arg-u16.8-max nil)
+ (u16.8-min two-arg-u16.8-min nil)
+ (u16.8+ two-arg-u16.8+ 0)
+ (u16.8+-saturating two-arg-u16.8+-saturating 0)
+ (u16.8* two-arg-u16.8* 1)
+ (u32.4-and two-arg-u32.4-and +u32-true+)
+ (u32.4-or two-arg-u32.4-or +u32-false+)
+ (u32.4-xor two-arg-u32.4-xor +u32-false+)
+ (u32.4-max two-arg-u32.4-max nil)
+ (u32.4-min two-arg-u32.4-min nil)
+ (u32.4+ two-arg-u32.4+ 0)
+ (u32.4+-saturating two-arg-u32.4+-saturating 0)
+ (u32.4* two-arg-u32.4* 1)
+ (u64.2-and two-arg-u64.2-and +u64-true+)
+ (u64.2-or two-arg-u64.2-or +u64-false+)
+ (u64.2-xor two-arg-u64.2-xor +u64-false+)
+ (u64.2-max two-arg-u64.2-max nil)
+ (u64.2-min two-arg-u64.2-min nil)
+ (u64.2+ two-arg-u64.2+ 0)
+ (u64.2+-saturating two-arg-u64.2+-saturating 0)
+ (u64.2* two-arg-u64.2* 1)
+ (s8.16-and two-arg-s8.16-and +s8-true+)
+ (s8.16-or two-arg-s8.16-or +s8-false+)
+ (s8.16-xor two-arg-s8.16-xor +s8-false+)
+ (s8.16-max two-arg-s8.16-max nil)
+ (s8.16-min two-arg-s8.16-min nil)
+ (s8.16+ two-arg-s8.16+ 0)
+ (s8.16+-saturating two-arg-s8.16+-saturating 0)
+ (s8.16* two-arg-s8.16* 1)
+ (s16.8-and two-arg-s16.8-and +s16-true+)
+ (s16.8-or two-arg-s16.8-or +s16-false+)
+ (s16.8-xor two-arg-s16.8-xor +s16-false+)
+ (s16.8-max two-arg-s16.8-max nil)
+ (s16.8-min two-arg-s16.8-min nil)
+ (s16.8+ two-arg-s16.8+ 0)
+ (s16.8+-saturating two-arg-s16.8+-saturating 0)
+ (s16.8* two-arg-s16.8* 1)
+ (s32.4-and two-arg-s32.4-and +s32-true+)
+ (s32.4-or two-arg-s32.4-or +s32-false+)
+ (s32.4-xor two-arg-s32.4-xor +s32-false+)
+ (s32.4-max two-arg-s32.4-max nil)
+ (s32.4-min two-arg-s32.4-min nil)
+ (s32.4+ two-arg-s32.4+ 0)
+ (s32.4+-saturating two-arg-s32.4+-saturating 0)
+ (s32.4* two-arg-s32.4* 1)
+ (s64.2-and two-arg-s64.2-and +s64-true+)
+ (s64.2-or two-arg-s64.2-or +s64-false+)
+ (s64.2-xor two-arg-s64.2-xor +s64-false+)
+ (s64.2-max two-arg-s64.2-max nil)
+ (s64.2-min two-arg-s64.2-min nil)
+ (s64.2+ two-arg-s64.2+ 0)
+ (s64.2+-saturating two-arg-s64.2+-saturating 0)
+ (s64.2* two-arg-s64.2* 1))
+ (:comparisons
+ (f32.4= two-arg-f32.4= u32.4-and +u32-true+)
+ (f32.4< two-arg-f32.4< u32.4-and +u32-true+)
+ (f32.4<= two-arg-f32.4<= u32.4-and +u32-true+)
+ (f32.4> two-arg-f32.4> u32.4-and +u32-true+)
+ (f32.4>= two-arg-f32.4>= u32.4-and +u32-true+)
+ (f64.2= two-arg-f64.2= u64.2-and +u64-true+)
+ (f64.2< two-arg-f64.2< u64.2-and +u64-true+)
+ (f64.2<= two-arg-f64.2<= u64.2-and +u64-true+)
+ (f64.2> two-arg-f64.2> u64.2-and +u64-true+)
+ (f64.2>= two-arg-f64.2>= u64.2-and +u64-true+)
+ (u8.16= two-arg-u8.16= u8.16-and +u8-true+)
+ (u8.16< two-arg-u8.16< u8.16-and +u8-true+)
+ (u8.16<= two-arg-u8.16<= u8.16-and +u8-true+)
+ (u8.16> two-arg-u8.16> u8.16-and +u8-true+)
+ (u8.16>= two-arg-u8.16>= u8.16-and +u8-true+)
+ (u16.8= two-arg-u16.8= u16.8-and +u16-true+)
+ (u16.8< two-arg-u16.8< u16.8-and +u16-true+)
+ (u16.8<= two-arg-u16.8<= u16.8-and +u16-true+)
+ (u16.8> two-arg-u16.8> u16.8-and +u16-true+)
+ (u16.8>= two-arg-u16.8>= u16.8-and +u16-true+)
+ (u32.4= two-arg-u32.4= u32.4-and +u32-true+)
+ (u32.4< two-arg-u32.4< u32.4-and +u32-true+)
+ (u32.4<= two-arg-u32.4<= u32.4-and +u32-true+)
+ (u32.4> two-arg-u32.4> u32.4-and +u32-true+)
+ (u32.4>= two-arg-u32.4>= u32.4-and +u32-true+)
+ (u64.2= two-arg-u64.2= u64.2-and +u64-true+)
+ (u64.2< two-arg-u64.2< u64.2-and +u64-true+)
+ (u64.2<= two-arg-u64.2<= u64.2-and +u64-true+)
+ (u64.2> two-arg-u64.2> u64.2-and +u64-true+)
+ (u64.2>= two-arg-u64.2>= u64.2-and +u64-true+)
+ (s8.16= two-arg-s8.16= u8.16-and +u8-true+)
+ (s8.16< two-arg-s8.16< u8.16-and +u8-true+)
+ (s8.16<= two-arg-s8.16<= u8.16-and +u8-true+)
+ (s8.16> two-arg-s8.16> u8.16-and +u8-true+)
+ (s8.16>= two-arg-s8.16>= u8.16-and +u8-true+)
+ (s16.8= two-arg-s16.8= u16.8-and +u16-true+)
+ (s16.8< two-arg-s16.8< u16.8-and +u16-true+)
+ (s16.8<= two-arg-s16.8<= u16.8-and +u16-true+)
+ (s16.8> two-arg-s16.8> u16.8-and +u16-true+)
+ (s16.8>= two-arg-s16.8>= u16.8-and +u16-true+)
+ (s32.4= two-arg-s32.4= u32.4-and +u32-true+)
+ (s32.4< two-arg-s32.4< u32.4-and +u32-true+)
+ (s32.4<= two-arg-s32.4<= u32.4-and +u32-true+)
+ (s32.4> two-arg-s32.4> u32.4-and +u32-true+)
+ (s32.4>= two-arg-s32.4>= u32.4-and +u32-true+)
+ (s64.2= two-arg-s64.2= u64.2-and +u64-true+)
+ (s64.2< two-arg-s64.2< u64.2-and +u64-true+)
+ (s64.2<= two-arg-s64.2<= u64.2-and +u64-true+)
+ (s64.2> two-arg-s64.2> u64.2-and +u64-true+)
+ (s64.2>= two-arg-s64.2>= u64.2-and +u64-true+))
+ (:reducers
+ (f32.4- two-arg-f32.4- 0f0)
+ (f32.4/ two-arg-f32.4/ 1f0)
+ (f64.2- two-arg-f64.2- 0d0)
+ (f64.2/ two-arg-f64.2/ 1d0)
+ (u8.16- two-arg-u8.16- 0)
+ (u8.16--saturating two-arg-u8.16--saturating 0)
+ (u16.8- two-arg-u16.8- 0)
+ (u16.8--saturating two-arg-u16.8--saturating 0)
+ (u32.4- two-arg-u32.4- 0)
+ (u32.4--saturating two-arg-u32.4--saturating 0)
+ (u64.2- two-arg-u64.2- 0)
+ (u64.2--saturating two-arg-u64.2--saturating 0)
+ (s8.16- two-arg-s8.16- 0)
+ (s8.16--saturating two-arg-s8.16--saturating 0)
+ (s16.8- two-arg-s16.8- 0)
+ (s16.8--saturating two-arg-s16.8--saturating 0)
+ (s32.4- two-arg-s32.4- 0)
+ (s32.4--saturating two-arg-s32.4--saturating 0)
+ (s64.2- two-arg-s64.2- 0)
+ (s64.2--saturating two-arg-s64.2--saturating 0))
+ (:unequals
+ (f32.4/= two-arg-f32.4/= u32.4-and +u32-true+)
+ (f64.2/= two-arg-f64.2/= u64.2-and +u64-true+)
+ (u8.16/= two-arg-u8.16/= u8.16-and +u8-true+)
+ (u16.8/= two-arg-u16.8/= u16.8-and +u16-true+)
+ (u32.4/= two-arg-u32.4/= u32.4-and +u32-true+)
+ (u64.2/= two-arg-u64.2/= u64.2-and +u64-true+)
+ (s8.16/= two-arg-s8.16/= u8.16-and +u8-true+)
+ (s16.8/= two-arg-s16.8/= u16.8-and +u16-true+)
+ (s32.4/= two-arg-s32.4/= u32.4-and +u32-true+)
+ (s64.2/= two-arg-s64.2/= u64.2-and +u64-true+)))
diff --git a/contrib/sb-simd/code/packages.lisp b/contrib/sb-simd/code/packages.lisp
index 9e8d38aa0..a0207f394 100644
--- a/contrib/sb-simd/code/packages.lisp
+++ b/contrib/sb-simd/code/packages.lisp
@@ -244,7 +244,8 @@
#:sse4.2-supported-p
#:avx-supported-p
#:avx2-supported-p
- #:fma-supported-p))
+ #:fma-supported-p
+ #:neon-supported-p))
(progn
(defpackage #:sb-simd
@@ -2079,9 +2080,555 @@
#:f64.4-fnmadd
#:f64.4-fmsub
#:f64.4-fmaddsub
- #:f64.4-fmsubadd)))
+ #:f64.4-fmsubadd))
-(dolist (p '("SB-SIMD" "SB-SIMD-AVX" "SB-SIMD-AVX2" "SB-SIMD-FMA"
+ (defpackage #:sb-simd-arm64
+ (:use #:common-lisp #:sb-simd-internals #:sb-simd)
+ #0#
+ #11=
+ (:export #:imm1 #:imm2 #:imm3 #:imm4 #:imm5 #:imm6 #:imm7 #:imm8))
+
+ (defpackage #:sb-simd-neon
+ (:use #:common-lisp #:sb-simd-internals #:sb-simd-arm64)
+ #0#
+ #11#
+ (:export
+ #:p128
+ #:f32!
+ #:f64!
+ #:u8!
+ #:u16!
+ #:u32!
+ #:u64!
+ ;; f32.4
+ #:make-f32.4
+ #:f32.4
+ #:f32.4!
+ #:f32.4-values
+ #:f32.4-broadcast
+ #:f32.4-bit-select
+ #:f32.4-lane-extract
+ #:f32.4-lane-insert
+ #:f32.4-dup
+ #:f32.4-ins
+ #:f32.4-transpose-even
+ #:f32.4-transpose-odd
+ #:f32.4-unzip-even
+ #:f32.4-unzip-odd
+ #:f32.4-zip-even
+ #:f32.4-zip-odd
+ #:f32.4-from-f64.2
+ #:f32.4-from-f64.2-hi
+ #:f32.4-from-s32.4
+ #:f32.4-from-u32.4
+ #:f32.4-and
+ #:f32.4-or
+ #:f32.4-xor
+ #:f32.4-andc1
+ #:f32.4-andc2
+ #:f32.4-not
+ #:f32.4-max
+ #:f32.4-min
+ #:f32.4-sqrt
+ #:f32.4+
+ #:f32.4-
+ #:f32.4*
+ #:f32.4/
+ #:f32.4=
+ #:f32.4/=
+ #:f32.4<
+ #:f32.4<=
+ #:f32.4>
+ #:f32.4>=
+ #:f32.4-pair-min
+ #:f32.4-horizontal-min
+ #:f32.4-pair-max
+ #:f32.4-horizontal-max
+ #:f32.4-pair+
+ #:f32.4-horizontal+
+ #:f32.4-incf
+ #:f32.4-decf
+ #:f32.4-aref
+ #:f32.4-row-major-aref
+ ;; f64.2
+ #:make-f64.2
+ #:f64.2
+ #:f64.2!
+ #:f64.2-values
+ #:f64.2-broadcast
+ #:f64.2-bit-select
+ #:f64.2-lane-extract
+ #:f64.2-lane-insert
+ #:f64.2-dup
+ #:f64.2-ins
+ #:f64.2-transpose-even
+ #:f64.2-transpose-odd
+ #:f64.2-unzip-even
+ #:f64.2-unzip-odd
+ #:f64.2-zip-even
+ #:f64.2-zip-odd
+ #:f64.2-from-f32.4
+ #:f64.2-from-f32.4-hi
+ #:f64.2-from-s64.2
+ #:f64.2-from-u64.2
+ #:f64.2-and
+ #:f64.2-or
+ #:f64.2-xor
+ #:f64.2-andc1
+ #:f64.2-andc2
+ #:f64.2-not
+ #:f64.2-max
+ #:f64.2-min
+ #:f64.2-sqrt
+ #:f64.2+
+ #:f64.2-
+ #:f64.2*
+ #:f64.2/
+ #:f64.2=
+ #:f64.2/=
+ #:f64.2<
+ #:f64.2<=
+ #:f64.2>
+ #:f64.2>=
+ #:f64.2-pair-min
+ #:f64.2-horizontal-min
+ #:f64.2-pair-max
+ #:f64.2-horizontal-max
+ #:f64.2-pair+
+ #:f64.2-horizontal+
+ #:f64.2-incf
+ #:f64.2-decf
+ #:f64.2-aref
+ #:f64.2-row-major-aref
+ ;; u8.16
+ #:make-u8.16
+ #:u8.16
+ #:u8.16!
+ #:u8.16-from-u16.8
+ #:u8.16-from-u16.8-hi
+ #:u8.16-from-u16.8-saturating
+ #:u8.16-from-u16.8-saturating-hi
+ #:u8.16-values
+ #:u8.16-broadcast
+ #:u8.16-bit-select
+ #:u8.16-lane-extract
+ #:u8.16-lane-insert
+ #:u8.16-dup
+ #:u8.16-ins
+ #:u8.16-transpose-even
+ #:u8.16-transpose-odd
+ #:u8.16-unzip-even
+ #:u8.16-unzip-odd
+ #:u8.16-zip-even
+ #:u8.16-zip-odd
+ #:u8.16-shuffle
+ #:u8.16-and
+ #:u8.16-or
+ #:u8.16-xor
+ #:u8.16-andc1
+ #:u8.16-andc2
+ #:u8.16-not
+ #:u8.16+
+ #:u8.16+-saturating
+ #:u8.16*
+ #:u8.16*-long
+ #:u8.16*-long-hi
+ #:u8.16-
+ #:u8.16--saturating
+ #:u8.16=
+ #:u8.16/=
+ #:u8.16<
+ #:u8.16<=
+ #:u8.16>
+ #:u8.16>=
+ #:u8.16-pair-min
+ #:u8.16-horizontal-min
+ #:u8.16-pair-max
+ #:u8.16-horizontal-max
+ #:u8.16-pair+
+ #:u8.16-horizontal+
+ #:u8.16-shiftr
+ #:u8.16-shiftl
+ #:u8.16-shiftl-long
+ #:u8.16-shiftl-long-hi
+ #:u8.16-rev16
+ #:u8.16-rev32
+ #:u8.16-rev64
+ #:u8.16-incf
+ #:u8.16-decf
+ #:u8.16-aref #:u8.16-row-major-aref
+ ;; u16.8
+ #:make-u16.8
+ #:u16.8
+ #:u16.8!
+ #:u16.8-from-u8.16
+ #:u16.8-from-u8.16-hi
+ #:u16.8-from-u32.4
+ #:u16.8-from-u32.4-hi
+ #:u16.8-from-u32.4-saturating
+ #:u16.8-from-u32.4-saturating-hi
+ #:u16.8-values
+ #:u16.8-broadcast
+ #:u16.8-bit-select
+ #:u16.8-lane-extract
+ #:u16.8-lane-insert
+ #:u16.8-dup
+ #:u16.8-ins
+ #:u16.8-transpose-even
+ #:u16.8-transpose-odd
+ #:u16.8-unzip-even
+ #:u16.8-unzip-odd
+ #:u16.8-zip-even
+ #:u16.8-zip-odd
+ #:u16.8-and
+ #:u16.8-or
+ #:u16.8-xor
+ #:u16.8-andc1
+ #:u16.8-andc2
+ #:u16.8-not
+ #:u16.8+
+ #:u16.8+-saturating
+ #:u16.8*
+ #:u16.8*-long
+ #:u16.8*-long-hi
+ #:u16.8-
+ #:u16.8--saturating
+ #:u16.8=
+ #:u16.8/=
+ #:u16.8<
+ #:u16.8<=
+ #:u16.8>
+ #:u16.8>=
+ #:u16.8-pair-min
+ #:u16.8-horizontal-min
+ #:u16.8-pair-max
+ #:u16.8-horizontal-max
+ #:u16.8-pair+
+ #:u16.8-horizontal+
+ #:u16.8-shiftr
+ #:u16.8-shiftl
+ #:u16.8-shiftl-long
+ #:u16.8-shiftl-long-hi
+ #:u16.8-incf
+ #:u16.8-decf
+ #:u16.8-aref #:u16.8-row-major-aref
+ ;; u32.4
+ #:make-u32.4
+ #:u32.4
+ #:u32.4!
+ #:u32.4-from-f32.4
+ #:u32.4-from-u16.8
+ #:u32.4-from-u16.8-hi
+ #:u32.4-from-u64.2
+ #:u32.4-from-u64.2-hi
+ #:u32.4-from-u64.2-saturating
+ #:u32.4-from-u64.2-saturating-hi
+ #:u32.4-values
+ #:u32.4-broadcast
+ #:u32.4-bit-select
+ #:u32.4-lane-extract
+ #:u32.4-lane-insert
+ #:u32.4-dup
+ #:u32.4-ins
+ #:u32.4-transpose-even
+ #:u32.4-transpose-odd
+ #:u32.4-unzip-even
+ #:u32.4-unzip-odd
+ #:u32.4-zip-even
+ #:u32.4-zip-odd
+ #:u32.4-and
+ #:u32.4-or
+ #:u32.4-xor
+ #:u32.4-andc1
+ #:u32.4-andc2
+ #:u32.4-not
+ #:u32.4+
+ #:u32.4+-saturating
+ #:u32.4*
+ #:u32.4*-long
+ #:u32.4*-long-hi
+ #:u32.4-
+ #:u32.4--saturating
+ #:u32.4=
+ #:u32.4/=
+ #:u32.4<
+ #:u32.4<=
+ #:u32.4>
+ #:u32.4>=
+ #:u32.4-pair-min
+ #:u32.4-horizontal-min
+ #:u32.4-pair-max
+ #:u32.4-horizontal-max
+ #:u32.4-pair+
+ #:u32.4-horizontal+
+ #:u32.4-shiftr
+ #:u32.4-shiftl
+ #:u32.4-shiftl-long
+ #:u32.4-shiftl-long-hi
+ #:u32.4-incf
+ #:u32.4-decf
+ #:u32.4-aref #:u32.4-row-major-aref
+ #+sb-unicode
+ #:u32.4-string-ref
+ #+sb-unicode
+ #:u32.4-row-major-string-ref
+ ;; u64.2
+ #:make-u64.2
+ #:u64.2
+ #:u64.2!
+ #:u64.2-from-f64.2
+ #:u64.2-from-u32.4
+ #:u64.2-from-u32.4-hi
+ #:u64.2-values
+ #:u64.2-broadcast
+ #:u64.2-bit-select
+ #:u64.2-lane-extract
+ #:u64.2-lane-insert
+ #:u64.2-dup
+ #:u64.2-ins
+ #:u64.2-transpose-even
+ #:u64.2-transpose-odd
+ #:u64.2-unzip-even
+ #:u64.2-unzip-odd
+ #:u64.2-zip-even
+ #:u64.2-zip-odd
+ #:u64.2-and
+ #:u64.2-or
+ #:u64.2-xor
+ #:u64.2-andc1
+ #:u64.2-andc2
+ #:u64.2-not
+ #:u64.2+
+ #:u64.2+-saturating
+ #:u64.2*
+ #:u64.2-
+ #:u64.2--saturating
+ #:u64.2=
+ #:u64.2/=
+ #:u64.2<
+ #:u64.2<=
+ #:u64.2>
+ #:u64.2>=
+ #:u64.2-pair-min
+ #:u64.2-horizontal-min
+ #:u64.2-pair-max
+ #:u64.2-horizontal-max
+ #:u64.2-pair+
+ #:u64.2-horizontal+
+ #:u64.2-shiftr
+ #:u64.2-shiftl
+ #:u64.2-incf
+ #:u64.2-decf
+ #:u64.2-aref #:u64.2-row-major-aref
+ ;; s8.16
+ #:make-s8.16
+ #:s8.16
+ #:s8.16!
+ #:s8.16-from-s16.8
+ #:s8.16-from-s16.8-hi
+ #:s8.16-from-s16.8-saturating
+ #:s8.16-from-s16.8-saturating-hi
+ #:s8.16-values
+ #:s8.16-broadcast
+ #:s8.16-bit-select
+ #:s8.16-lane-extract
+ #:s8.16-lane-insert
+ #:s8.16-dup
+ #:s8.16-ins
+ #:s8.16-transpose-even
+ #:s8.16-transpose-odd
+ #:s8.16-unzip-even
+ #:s8.16-unzip-odd
+ #:s8.16-zip-even
+ #:s8.16-zip-odd
+ #:s8.16-and
+ #:s8.16-or
+ #:s8.16-xor
+ #:s8.16-andc1
+ #:s8.16-andc2
+ #:s8.16-not
+ #:s8.16+
+ #:s8.16+-saturating
+ #:s8.16*
+ #:s8.16-
+ #:s8.16--saturating
+ #:s8.16=
+ #:s8.16/=
+ #:s8.16<
+ #:s8.16<=
+ #:s8.16>
+ #:s8.16>=
+ #:s8.16-pair-min
+ #:s8.16-horizontal-min
+ #:s8.16-pair-max
+ #:s8.16-horizontal-max
+ #:s8.16-pair+
+ #:s8.16-horizontal+
+ #:s8.16-shiftr
+ #:s8.16-shiftl
+ #:s8.16-shiftl-long
+ #:s8.16-shiftl-long-hi
+ #:s8.16-incf
+ #:s8.16-decf
+ #:s8.16-aref #:s8.16-row-major-aref
+ ;; s16.8
+ #:make-s16.8
+ #:s16.8
+ #:s16.8!
+ #:s16.8-from-s8.16
+ #:s16.8-from-s8.16-hi
+ #:s16.8-from-s32.4
+ #:s16.8-from-s32.4-hi
+ #:s16.8-from-s32.4-saturating
+ #:s16.8-from-s32.4-saturating-hi
+ #:s16.8-values
+ #:s16.8-broadcast
+ #:s16.8-bit-select
+ #:s16.8-lane-extract
+ #:s16.8-lane-insert
+ #:s16.8-dup
+ #:s16.8-ins
+ #:s16.8-transpose-even
+ #:s16.8-transpose-odd
+ #:s16.8-unzip-even
+ #:s16.8-unzip-odd
+ #:s16.8-zip-even
+ #:s16.8-zip-odd
+ #:s16.8-and
+ #:s16.8-or
+ #:s16.8-xor
+ #:s16.8-andc1
+ #:s16.8-andc2
+ #:s16.8-not
+ #:s16.8+
+ #:s16.8+-saturating
+ #:s16.8*
+ #:s16.8-
+ #:s16.8--saturating
+ #:s16.8=
+ #:s16.8/=
+ #:s16.8<
+ #:s16.8<=
+ #:s16.8>
+ #:s16.8>=
+ #:s16.8-pair-min
+ #:s16.8-horizontal-min
+ #:s16.8-pair-max
+ #:s16.8-horizontal-max
+ #:s16.8-pair+
+ #:s16.8-horizontal+
+ #:s16.8-shiftr
+ #:s16.8-shiftl
+ #:s16.8-shiftl-long
+ #:s16.8-shiftl-long-hi
+ #:s16.8-incf
+ #:s16.8-decf
+ #:s16.8-aref #:s16.8-row-major-aref
+ ;; s32.4
+ #:make-s32.4
+ #:s32.4
+ #:s32.4!
+ #:s32.4-from-f32.4
+ #:s32.4-from-s16.8
+ #:s32.4-from-s16.8-hi
+ #:s32.4-from-s64.2
+ #:s32.4-from-s64.2-hi
+ #:s32.4-from-s64.2-saturating
+ #:s32.4-from-s64.2-saturating-hi
+ #:s32.4-values
+ #:s32.4-broadcast
+ #:s32.4-bit-select
+ #:s32.4-lane-extract
+ #:s32.4-lane-insert
+ #:s32.4-dup
+ #:s32.4-ins
+ #:s32.4-transpose-even
+ #:s32.4-transpose-odd
+ #:s32.4-unzip-even
+ #:s32.4-unzip-odd
+ #:s32.4-zip-even
+ #:s32.4-zip-odd
+ #:s32.4-and
+ #:s32.4-or
+ #:s32.4-xor
+ #:s32.4-andc1
+ #:s32.4-andc2
+ #:s32.4-not
+ #:s32.4+
+ #:s32.4+-saturating
+ #:s32.4*
+ #:s32.4-
+ #:s32.4--saturating
+ #:s32.4=
+ #:s32.4/=
+ #:s32.4<
+ #:s32.4<=
+ #:s32.4>
+ #:s32.4>=
+ #:s32.4-pair-min
+ #:s32.4-horizontal-min
+ #:s32.4-pair-max
+ #:s32.4-horizontal-max
+ #:s32.4-pair+
+ #:s32.4-horizontal+
+ #:s32.4-shiftr
+ #:s32.4-shiftl
+ #:s32.4-shiftl-long
+ #:s32.4-shiftl-long-hi
+ #:s32.4-incf
+ #:s32.4-decf
+ #:s32.4-aref #:s32.4-row-major-aref
+ ;; s64.2
+ #:make-s64.2
+ #:s64.2
+ #:s64.2!
+ #:s64.2-from-f64.2
+ #:s64.2-from-s32.4
+ #:s64.2-from-s32.4-hi
+ #:s64.2-values
+ #:s64.2-broadcast
+ #:s64.2-bit-select
+ #:s64.2-lane-extract
+ #:s64.2-lane-insert
+ #:s64.2-dup
+ #:s64.2-ins
+ #:s64.2-transpose-even
+ #:s64.2-transpose-odd
+ #:s64.2-unzip-even
+ #:s64.2-unzip-odd
+ #:s64.2-zip-even
+ #:s64.2-zip-odd
+ #:s64.2-and
+ #:s64.2-or
+ #:s64.2-xor
+ #:s64.2-andc1
+ #:s64.2-andc2
+ #:s64.2-not
+ #:s64.2+
+ #:s64.2+-saturating
+ #:s64.2*
+ #:s64.2-
+ #:s64.2--saturating
+ #:s64.2=
+ #:s64.2/=
+ #:s64.2<
+ #:s64.2<=
+ #:s64.2>
+ #:s64.2>=
+ #:s64.2-pair-min
+ #:s64.2-horizontal-min
+ #:s64.2-pair-max
+ #:s64.2-horizontal-max
+ #:s64.2-pair+
+ #:s64.2-horizontal+
+ #:s64.2-shiftr
+ #:s64.2-shiftl
+ #:s64.2-incf
+ #:s64.2-decf
+ #:s64.2-aref #:s64.2-row-major-aref)))
+
+(dolist (p '("SB-SIMD" "SB-SIMD-NEON" "SB-SIMD-ARM64"
+ "SB-SIMD-AVX" "SB-SIMD-AVX2" "SB-SIMD-FMA"
"SB-SIMD-INTERNALS" "SB-SIMD-SSE" "SB-SIMD-SSE2"
"SB-SIMD-SSE3" "SB-SIMD-SSE4.1" "SB-SIMD-SSE4.2"
"SB-SIMD-SSSE3" "SB-SIMD-X86-64"))
diff --git a/contrib/sb-simd/code/record.lisp b/contrib/sb-simd/code/record.lisp
index 90eadebe7..ab8f7aa3f 100644
--- a/contrib/sb-simd/code/record.lisp
+++ b/contrib/sb-simd/code/record.lisp
@@ -181,21 +181,21 @@
(defun scalar-record-p (x)
(typep x '(and value-record (not simd-record))))
-#-simd-pack-256
+#-sb-simd-pack-256
(progn
(defstruct phony-simd-pack-256)
(deftype simd-pack-256 (&optional element-type)
(declare (ignore element-type))
'phony-simd-pack-256))
-(defmethod decode-record-definition ((_ (eql 'simd-record)) expr):w
+(defmethod decode-record-definition ((_ (eql 'simd-record)) expr)
(destructuring-bind (name scalar-record-name bits primitive-type scs) expr
(let ((simd-pack-type
(let ((base-type
(ecase bits
(128 (find-symbol "SIMD-PACK" "SB-EXT"))
(256 (or (find-symbol "SIMD-PACK-256" "SB-EXT")
- #-simd-pack-256
+ #-sb-simd-pack-256
'simd-pack-256)))))
(cond ((not base-type) 't)
((not scalar-record-name) base-type)
@@ -508,7 +508,7 @@
:reader instruction-record-always-translatable)
;; How the instruction is turned into a VOP.
(%encoding
- :type (member :standard :sse :sse+xmm0 :custom :fake-vop :move :fma)
+ :type (member :standard :sse :sse+xmm0 :custom :fake-vop :move :fma :neon-rmw)
:initarg :encoding
:initform :standard
:reader instruction-record-encoding)
diff --git a/contrib/sb-simd/sb-simd.asd b/contrib/sb-simd/sb-simd.asd
index 32e817749..f167ec608 100644
--- a/contrib/sb-simd/sb-simd.asd
+++ b/contrib/sb-simd/sb-simd.asd
@@ -30,7 +30,9 @@
(:file "sse4-2")
(:file "avx")
(:file "avx2")
- (:file "fma")))
+ (:file "fma")
+ (:file "arm64")
+ (:file "neon")))
(:file "define-types")
(:file "define-instruction-vops")
(:file "define-vref-vops")
diff --git a/contrib/sb-simd/test-suite/test-arefs.lisp b/contrib/sb-simd/test-suite/test-arefs.lisp
index 7f1926997..c0b7530c4 100644
--- a/contrib/sb-simd/test-suite/test-arefs.lisp
+++ b/contrib/sb-simd/test-suite/test-arefs.lisp
@@ -90,3 +90,16 @@
(sb-simd-test-suite:define-aref-test s16.16-aref (signed-byte 16) 16 s16.16-values)
(sb-simd-test-suite:define-aref-test s32.8-aref (signed-byte 32) 8 s32.8-values)
(sb-simd-test-suite:define-aref-test s64.4-aref (signed-byte 64) 4 s64.4-values)
+
+(in-package #:sb-simd-neon)
+
+(sb-simd-test-suite:define-aref-test f32.4-aref single-float 4 f32.4-values)
+(sb-simd-test-suite:define-aref-test f64.2-aref double-float 2 f64.2-values)
+(sb-simd-test-suite:define-aref-test u8.16-aref (unsigned-byte 8) 16 u8.16-values)
+(sb-simd-test-suite:define-aref-test u16.8-aref (unsigned-byte 16) 8 u16.8-values)
+(sb-simd-test-suite:define-aref-test u32.4-aref (unsigned-byte 32) 4 u32.4-values)
+(sb-simd-test-suite:define-aref-test u64.2-aref (unsigned-byte 64) 2 u64.2-values)
+(sb-simd-test-suite:define-aref-test s8.16-aref (signed-byte 8) 16 s8.16-values)
+(sb-simd-test-suite:define-aref-test s16.8-aref (signed-byte 16) 8 s16.8-values)
+(sb-simd-test-suite:define-aref-test s32.4-aref (signed-byte 32) 4 s32.4-values)
+(sb-simd-test-suite:define-aref-test s64.2-aref (signed-byte 64) 2 s64.2-values)
diff --git a/contrib/sb-simd/test-suite/test-hairy-simd-functions.lisp b/contrib/sb-simd/test-suite/test-hairy-simd-functions.lisp
index 24e36c71b..e2d9704dc 100644
--- a/contrib/sb-simd/test-suite/test-hairy-simd-functions.lisp
+++ b/contrib/sb-simd/test-suite/test-hairy-simd-functions.lisp
@@ -7,3 +7,5 @@
(in-package #:sb-simd-avx)
(in-package #:sb-simd-avx2)
+
+(in-package #:sb-simd-neon)
diff --git a/contrib/sb-simd/test-suite/test-horizontal-functions.lisp b/contrib/sb-simd/test-suite/test-horizontal-functions.lisp
index 3557e38d0..7e3d3ae0c 100644
--- a/contrib/sb-simd/test-suite/test-horizontal-functions.lisp
+++ b/contrib/sb-simd/test-suite/test-horizontal-functions.lisp
@@ -74,3 +74,45 @@
(sb-simd-test-suite:define-horizontal-test f64.4-horizontal* f64*)
(in-package #:sb-simd-avx2)
+
+(in-package #:sb-simd-neon)
+
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-max f32-max)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-min f32-min)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal+ f32+)
+
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-max f64-max)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-min f64-min)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal+ f64+)
+
+(sb-simd-test-suite:define-horizontal-test u8.16-horizontal-max u8-max)
+(sb-simd-test-suite:define-horizontal-test u8.16-horizontal-min u8-min)
+(sb-simd-test-suite:define-horizontal-test u8.16-horizontal+ u8+)
+
+(sb-simd-test-suite:define-horizontal-test u16.8-horizontal-max u16-max)
+(sb-simd-test-suite:define-horizontal-test u16.8-horizontal-min u16-min)
+(sb-simd-test-suite:define-horizontal-test u16.8-horizontal+ u16+)
+
+(sb-simd-test-suite:define-horizontal-test u32.4-horizontal-max u32-max)
+(sb-simd-test-suite:define-horizontal-test u32.4-horizontal-min u32-min)
+(sb-simd-test-suite:define-horizontal-test u32.4-horizontal+ u32+)
+
+(sb-simd-test-suite:define-horizontal-test u64.2-horizontal-max u64-max)
+(sb-simd-test-suite:define-horizontal-test u64.2-horizontal-min u64-min)
+(sb-simd-test-suite:define-horizontal-test u64.2-horizontal+ u64+)
+
+(sb-simd-test-suite:define-horizontal-test s8.16-horizontal-max s8-max)
+(sb-simd-test-suite:define-horizontal-test s8.16-horizontal-min s8-min)
+(sb-simd-test-suite:define-horizontal-test s8.16-horizontal+ s8+)
+
+(sb-simd-test-suite:define-horizontal-test s16.8-horizontal-max s16-max)
+(sb-simd-test-suite:define-horizontal-test s16.8-horizontal-min s16-min)
+(sb-simd-test-suite:define-horizontal-test s16.8-horizontal+ s16+)
+
+(sb-simd-test-suite:define-horizontal-test s32.4-horizontal-max s32-max)
+(sb-simd-test-suite:define-horizontal-test s32.4-horizontal-min s32-min)
+(sb-simd-test-suite:define-horizontal-test s32.4-horizontal+ s32+)
+
+(sb-simd-test-suite:define-horizontal-test s64.2-horizontal-max s64-max)
+(sb-simd-test-suite:define-horizontal-test s64.2-horizontal-min s64-min)
+(sb-simd-test-suite:define-horizontal-test s64.2-horizontal+ s64+)
diff --git a/contrib/sb-simd/test-suite/test-packages.lisp b/contrib/sb-simd/test-suite/test-packages.lisp
index 95ccb0e6e..ba18de9c2 100644
--- a/contrib/sb-simd/test-suite/test-packages.lisp
+++ b/contrib/sb-simd/test-suite/test-packages.lisp
@@ -14,6 +14,7 @@
(check-package '#:sb-simd-sse4.2)
(check-package '#:sb-simd-avx)
(check-package '#:sb-simd-avx2)
+ (check-package '#:sb-simd-neon)
;; Ensure that every instruction has a corresponding VOP.
(dolist (instruction-record (filter-available-function-records #'instruction-record-p))
(is (fboundp (instruction-record-vop instruction-record)))))
diff --git a/contrib/sb-simd/test-suite/test-simple-simd-functions.lisp b/contrib/sb-simd/test-suite/test-simple-simd-functions.lisp
index 9cbbd5b03..6ee0307aa 100644
--- a/contrib/sb-simd/test-suite/test-simple-simd-functions.lisp
+++ b/contrib/sb-simd/test-suite/test-simple-simd-functions.lisp
@@ -541,3 +541,152 @@
(sb-simd-test-suite:define-simple-simd-test f64.4-fmadd (f64.4) (f64.4 f64.4 f64.4) f64-fmadd)
(sb-simd-test-suite:define-simple-simd-test f64.4-fnmadd (f64.4) (f64.4 f64.4 f64.4) f64-fnmadd)
(sb-simd-test-suite:define-simple-simd-test f64.4-fmsub (f64.4) (f64.4 f64.4 f64.4) f64-fmsub)
+
+(in-package #:sb-simd-neon)
+
+(sb-simd-test-suite:define-simple-simd-test f32.4-and (f32.4) (&rest f32.4) f32-and)
+(sb-simd-test-suite:define-simple-simd-test f32.4-or (f32.4) (&rest f32.4) f32-or)
+(sb-simd-test-suite:define-simple-simd-test f32.4-xor (f32.4) (&rest f32.4) f32-xor)
+(sb-simd-test-suite:define-simple-simd-test f32.4-andc1 (f32.4) (f32.4 f32.4) f32-andc1)
+(sb-simd-test-suite:define-simple-simd-test f32.4-not (f32.4) (f32.4) f32-not)
+(sb-simd-test-suite:define-simple-simd-test f32.4-max (f32.4) (f32.4 &rest f32.4) f32-max)
+(sb-simd-test-suite:define-simple-simd-test f32.4-min (f32.4) (f32.4 &rest f32.4) f32-min)
+(sb-simd-test-suite:define-simple-simd-test f32.4+ (f32.4) (&rest f32.4) f32+)
+(sb-simd-test-suite:define-simple-simd-test f32.4- (f32.4) (f32.4 &rest f32.4) f32-)
+(sb-simd-test-suite:define-simple-simd-test f32.4* (f32.4) (&rest f32.4) f32*)
+(sb-simd-test-suite:define-simple-simd-test f32.4/ (f32.4) (f32.4 &rest f32.4) f32/)
+(sb-simd-test-suite:define-simple-simd-test f32.4= (u32.4) (f32.4 &rest f32.4) f32=)
+(sb-simd-test-suite:define-simple-simd-test f32.4/= (u32.4) (f32.4 &rest f32.4) f32/=)
+(sb-simd-test-suite:define-simple-simd-test f32.4< (u32.4) (f32.4 &rest f32.4) f32<)
+(sb-simd-test-suite:define-simple-simd-test f32.4<= (u32.4) (f32.4 &rest f32.4) f32<=)
+(sb-simd-test-suite:define-simple-simd-test f32.4> (u32.4) (f32.4 &rest f32.4) f32>)
+(sb-simd-test-suite:define-simple-simd-test f32.4>= (u32.4) (f32.4 &rest f32.4) f32>=)
+
+(sb-simd-test-suite:define-simple-simd-test f64.2-and (f64.2) (&rest f64.2) f64-and)
+(sb-simd-test-suite:define-simple-simd-test f64.2-or (f64.2) (&rest f64.2) f64-or)
+(sb-simd-test-suite:define-simple-simd-test f64.2-xor (f64.2) (&rest f64.2) f64-xor)
+(sb-simd-test-suite:define-simple-simd-test f64.2-andc1 (f64.2) (f64.2 f64.2) f64-andc1)
+(sb-simd-test-suite:define-simple-simd-test f64.2-not (f64.2) (f64.2) f64-not)
+(sb-simd-test-suite:define-simple-simd-test f64.2-max (f64.2) (f64.2 &rest f64.2) f64-max)
+(sb-simd-test-suite:define-simple-simd-test f64.2-min (f64.2) (f64.2 &rest f64.2) f64-min)
+(sb-simd-test-suite:define-simple-simd-test f64.2+ (f64.2) (&rest f64.2) f64+)
+(sb-simd-test-suite:define-simple-simd-test f64.2- (f64.2) (f64.2 &rest f64.2) f64-)
+(sb-simd-test-suite:define-simple-simd-test f64.2* (f64.2) (&rest f64.2) f64*)
+(sb-simd-test-suite:define-simple-simd-test f64.2/ (f64.2) (f64.2 &rest f64.2) f64/)
+(sb-simd-test-suite:define-simple-simd-test f64.2= (u64.2) (f64.2 &rest f64.2) f64=)
+(sb-simd-test-suite:define-simple-simd-test f64.2/= (u64.2) (f64.2 &rest f64.2) f64/=)
+(sb-simd-test-suite:define-simple-simd-test f64.2< (u64.2) (f64.2 &rest f64.2) f64<)
+(sb-simd-test-suite:define-simple-simd-test f64.2<= (u64.2) (f64.2 &rest f64.2) f64<=)
+(sb-simd-test-suite:define-simple-simd-test f64.2> (u64.2) (f64.2 &rest f64.2) f64>)
+(sb-simd-test-suite:define-simple-simd-test f64.2>= (u64.2) (f64.2 &rest f64.2) f64>=)
+
+(sb-simd-test-suite:define-simple-simd-test u8.16-and (u8.16) (&rest u8.16) u8-and)
+(sb-simd-test-suite:define-simple-simd-test u8.16-or (u8.16) (&rest u8.16) u8-or)
+(sb-simd-test-suite:define-simple-simd-test u8.16-xor (u8.16) (&rest u8.16) u8-xor)
+(sb-simd-test-suite:define-simple-simd-test u8.16-andc1 (u8.16) (u8.16 u8.16) u8-andc1)
+(sb-simd-test-suite:define-simple-simd-test u8.16-not (u8.16) (u8.16) u8-not)
+(sb-simd-test-suite:define-simple-simd-test u8.16+ (u8.16) (&rest u8.16) u8+)
+(sb-simd-test-suite:define-simple-simd-test u8.16- (u8.16) (u8.16 &rest u8.16) u8-)
+(sb-simd-test-suite:define-simple-simd-test u8.16= (u8.16) (u8.16 &rest u8.16) u8=)
+(sb-simd-test-suite:define-simple-simd-test u8.16/= (u8.16) (u8.16 &rest u8.16) u8/=)
+(sb-simd-test-suite:define-simple-simd-test u8.16< (u8.16) (u8.16 &rest u8.16) u8<)
+(sb-simd-test-suite:define-simple-simd-test u8.16<= (u8.16) (u8.16 &rest u8.16) u8<=)
+(sb-simd-test-suite:define-simple-simd-test u8.16> (u8.16) (u8.16 &rest u8.16) u8>)
+(sb-simd-test-suite:define-simple-simd-test u8.16>= (u8.16) (u8.16 &rest u8.16) u8>=)
+
+(sb-simd-test-suite:define-simple-simd-test u16.8-and (u16.8) (&rest u16.8) u16-and)
+(sb-simd-test-suite:define-simple-simd-test u16.8-or (u16.8) (&rest u16.8) u16-or)
+(sb-simd-test-suite:define-simple-simd-test u16.8-xor (u16.8) (&rest u16.8) u16-xor)
+(sb-simd-test-suite:define-simple-simd-test u16.8-andc1 (u16.8) (u16.8 u16.8) u16-andc1)
+(sb-simd-test-suite:define-simple-simd-test u16.8-not (u16.8) (u16.8) u16-not)
+(sb-simd-test-suite:define-simple-simd-test u16.8+ (u16.8) (&rest u16.8) u16+)
+(sb-simd-test-suite:define-simple-simd-test u16.8- (u16.8) (u16.8 &rest u16.8) u16-)
+(sb-simd-test-suite:define-simple-simd-test u16.8= (u16.8) (u16.8 &rest u16.8) u16=)
+(sb-simd-test-suite:define-simple-simd-test u16.8/= (u16.8) (u16.8 &rest u16.8) u16/=)
+(sb-simd-test-suite:define-simple-simd-test u16.8< (u16.8) (u16.8 &rest u16.8) u16<)
+(sb-simd-test-suite:define-simple-simd-test u16.8<= (u16.8) (u16.8 &rest u16.8) u16<=)
+(sb-simd-test-suite:define-simple-simd-test u16.8> (u16.8) (u16.8 &rest u16.8) u16>)
+(sb-simd-test-suite:define-simple-simd-test u16.8>= (u16.8) (u16.8 &rest u16.8) u16>=)
+
+(sb-simd-test-suite:define-simple-simd-test u32.4-and (u32.4) (&rest u32.4) u32-and)
+(sb-simd-test-suite:define-simple-simd-test u32.4-or (u32.4) (&rest u32.4) u32-or)
+(sb-simd-test-suite:define-simple-simd-test u32.4-xor (u32.4) (&rest u32.4) u32-xor)
+(sb-simd-test-suite:define-simple-simd-test u32.4-andc1 (u32.4) (u32.4 u32.4) u32-andc1)
+(sb-simd-test-suite:define-simple-simd-test u32.4-not (u32.4) (u32.4) u32-not)
+(sb-simd-test-suite:define-simple-simd-test u32.4+ (u32.4) (&rest u32.4) u32+)
+(sb-simd-test-suite:define-simple-simd-test u32.4- (u32.4) (u32.4 &rest u32.4) u32-)
+(sb-simd-test-suite:define-simple-simd-test u32.4= (u32.4) (u32.4 &rest u32.4) u32=)
+(sb-simd-test-suite:define-simple-simd-test u32.4/= (u32.4) (u32.4 &rest u32.4) u32/=)
+(sb-simd-test-suite:define-simple-simd-test u32.4< (u32.4) (u32.4 &rest u32.4) u32<)
+(sb-simd-test-suite:define-simple-simd-test u32.4<= (u32.4) (u32.4 &rest u32.4) u32<=)
+(sb-simd-test-suite:define-simple-simd-test u32.4> (u32.4) (u32.4 &rest u32.4) u32>)
+(sb-simd-test-suite:define-simple-simd-test u32.4>= (u32.4) (u32.4 &rest u32.4) u32>=)
+
+(sb-simd-test-suite:define-simple-simd-test u64.2-and (u64.2) (&rest u64.2) u64-and)
+(sb-simd-test-suite:define-simple-simd-test u64.2-or (u64.2) (&rest u64.2) u64-or)
+(sb-simd-test-suite:define-simple-simd-test u64.2-xor (u64.2) (&rest u64.2) u64-xor)
+(sb-simd-test-suite:define-simple-simd-test u64.2-andc1 (u64.2) (u64.2 u64.2) u64-andc1)
+(sb-simd-test-suite:define-simple-simd-test u64.2-not (u64.2) (u64.2) u64-not)
+(sb-simd-test-suite:define-simple-simd-test u64.2+ (u64.2) (&rest u64.2) u64+)
+(sb-simd-test-suite:define-simple-simd-test u64.2- (u64.2) (u64.2 &rest u64.2) u64-)
+(sb-simd-test-suite:define-simple-simd-test u64.2= (u64.2) (u64.2 &rest u64.2) u64=)
+(sb-simd-test-suite:define-simple-simd-test u64.2/= (u64.2) (u64.2 &rest u64.2) u64/=)
+(sb-simd-test-suite:define-simple-simd-test u64.2< (u64.2) (u64.2 &rest u64.2) u64<)
+(sb-simd-test-suite:define-simple-simd-test u64.2<= (u64.2) (u64.2 &rest u64.2) u64<=)
+(sb-simd-test-suite:define-simple-simd-test u64.2> (u64.2) (u64.2 &rest u64.2) u64>)
+(sb-simd-test-suite:define-simple-simd-test u64.2>= (u64.2) (u64.2 &rest u64.2) u64>=)
+
+(sb-simd-test-suite:define-simple-simd-test s8.16-and (s8.16) (&rest s8.16) s8-and)
+(sb-simd-test-suite:define-simple-simd-test s8.16-or (s8.16) (&rest s8.16) s8-or)
+(sb-simd-test-suite:define-simple-simd-test s8.16-xor (s8.16) (&rest s8.16) s8-xor)
+(sb-simd-test-suite:define-simple-simd-test s8.16-andc1 (s8.16) (s8.16 s8.16) s8-andc1)
+(sb-simd-test-suite:define-simple-simd-test s8.16-not (s8.16) (s8.16) s8-not)
+(sb-simd-test-suite:define-simple-simd-test s8.16+ (s8.16) (&rest s8.16) s8+)
+(sb-simd-test-suite:define-simple-simd-test s8.16- (s8.16) (s8.16 &rest s8.16) s8-)
+(sb-simd-test-suite:define-simple-simd-test s8.16= (u8.16) (s8.16 &rest s8.16) s8=)
+(sb-simd-test-suite:define-simple-simd-test s8.16/= (u8.16) (s8.16 &rest s8.16) s8/=)
+(sb-simd-test-suite:define-simple-simd-test s8.16< (u8.16) (s8.16 &rest s8.16) s8<)
+(sb-simd-test-suite:define-simple-simd-test s8.16<= (u8.16) (s8.16 &rest s8.16) s8<=)
+(sb-simd-test-suite:define-simple-simd-test s8.16> (u8.16) (s8.16 &rest s8.16) s8>)
+(sb-simd-test-suite:define-simple-simd-test s8.16>= (u8.16) (s8.16 &rest s8.16) s8>=)
+
+(sb-simd-test-suite:define-simple-simd-test s16.8-and (s16.8) (&rest s16.8) s16-and)
+(sb-simd-test-suite:define-simple-simd-test s16.8-or (s16.8) (&rest s16.8) s16-or)
+(sb-simd-test-suite:define-simple-simd-test s16.8-xor (s16.8) (&rest s16.8) s16-xor)
+(sb-simd-test-suite:define-simple-simd-test s16.8-andc1 (s16.8) (s16.8 s16.8) s16-andc1)
+(sb-simd-test-suite:define-simple-simd-test s16.8-not (s16.8) (s16.8) s16-not)
+(sb-simd-test-suite:define-simple-simd-test s16.8+ (s16.8) (&rest s16.8) s16+)
+(sb-simd-test-suite:define-simple-simd-test s16.8- (s16.8) (s16.8 &rest s16.8) s16-)
+(sb-simd-test-suite:define-simple-simd-test s16.8= (u16.8) (s16.8 &rest s16.8) s16=)
+(sb-simd-test-suite:define-simple-simd-test s16.8/= (u16.8) (s16.8 &rest s16.8) s16/=)
+(sb-simd-test-suite:define-simple-simd-test s16.8< (u16.8) (s16.8 &rest s16.8) s16<)
+(sb-simd-test-suite:define-simple-simd-test s16.8<= (u16.8) (s16.8 &rest s16.8) s16<=)
+(sb-simd-test-suite:define-simple-simd-test s16.8> (u16.8) (s16.8 &rest s16.8) s16>)
+(sb-simd-test-suite:define-simple-simd-test s16.8>= (u16.8) (s16.8 &rest s16.8) s16>=)
+
+(sb-simd-test-suite:define-simple-simd-test s32.4-and (s32.4) (&rest s32.4) s32-and)
+(sb-simd-test-suite:define-simple-simd-test s32.4-or (s32.4) (&rest s32.4) s32-or)
+(sb-simd-test-suite:define-simple-simd-test s32.4-xor (s32.4) (&rest s32.4) s32-xor)
+(sb-simd-test-suite:define-simple-simd-test s32.4-andc1 (s32.4) (s32.4 s32.4) s32-andc1)
+(sb-simd-test-suite:define-simple-simd-test s32.4-not (s32.4) (s32.4) s32-not)
+(sb-simd-test-suite:define-simple-simd-test s32.4+ (s32.4) (&rest s32.4) s32+)
+(sb-simd-test-suite:define-simple-simd-test s32.4- (s32.4) (s32.4 &rest s32.4) s32-)
+(sb-simd-test-suite:define-simple-simd-test s32.4= (u32.4) (s32.4 &rest s32.4) s32=)
+(sb-simd-test-suite:define-simple-simd-test s32.4/= (u32.4) (s32.4 &rest s32.4) s32/=)
+(sb-simd-test-suite:define-simple-simd-test s32.4< (u32.4) (s32.4 &rest s32.4) s32<)
+(sb-simd-test-suite:define-simple-simd-test s32.4<= (u32.4) (s32.4 &rest s32.4) s32<=)
+(sb-simd-test-suite:define-simple-simd-test s32.4> (u32.4) (s32.4 &rest s32.4) s32>)
+(sb-simd-test-suite:define-simple-simd-test s32.4>= (u32.4) (s32.4 &rest s32.4) s32>=)
+
+(sb-simd-test-suite:define-simple-simd-test s64.2-and (s64.2) (&rest s64.2) s64-and)
+(sb-simd-test-suite:define-simple-simd-test s64.2-or (s64.2) (&rest s64.2) s64-or)
+(sb-simd-test-suite:define-simple-simd-test s64.2-xor (s64.2) (&rest s64.2) s64-xor)
+(sb-simd-test-suite:define-simple-simd-test s64.2-andc1 (s64.2) (s64.2 s64.2) s64-andc1)
+(sb-simd-test-suite:define-simple-simd-test s64.2-not (s64.2) (s64.2) s64-not)
+(sb-simd-test-suite:define-simple-simd-test s64.2+ (s64.2) (&rest s64.2) s64+)
+(sb-simd-test-suite:define-simple-simd-test s64.2- (s64.2) (s64.2 &rest s64.2) s64-)
+(sb-simd-test-suite:define-simple-simd-test s64.2= (u64.2) (s64.2 &rest s64.2) s64=)
+(sb-simd-test-suite:define-simple-simd-test s64.2< (u64.2) (s64.2 &rest s64.2) s64<)
+(sb-simd-test-suite:define-simple-simd-test s64.2<= (u64.2) (s64.2 &rest s64.2) s64<=)
+(sb-simd-test-suite:define-simple-simd-test s64.2> (u64.2) (s64.2 &rest s64.2) s64>)
+(sb-simd-test-suite:define-simple-simd-test s64.2>= (u64.2) (s64.2 &rest s64.2) s64>=)
diff --git a/contrib/sb-simd/test-suite/test-suite.lisp b/contrib/sb-simd/test-suite/test-suite.lisp
index 79fba2e9f..93250adfe 100644
--- a/contrib/sb-simd/test-suite/test-suite.lisp
+++ b/contrib/sb-simd/test-suite/test-suite.lisp
@@ -59,13 +59,19 @@
(defmacro define-test (test-name &body body)
"Define a test function and add it to *TESTS*."
- (let ((name (intern-test-name test-name)))
+ (let ((name (intern-test-name test-name))
+ (instruction-set (sb-simd-internals:find-instruction-set
+ (symbol-package test-name)
+ nil)))
`(prog1 ',name
(defun ,name ()
(declare (optimize (debug 3) (safety 3)))
(with-test-harness
(enter-test ',name)
- ,@body))
+ ,(if (or (not instruction-set)
+ (sb-simd-internals:instruction-set-available-p instruction-set))
+ `(progn ,@body)
+ nil)))
(pushnew ',name *tests*))))
(defun enter-test (test-name)
diff --git a/contrib/sb-simd/test-suite/utilities.lisp b/contrib/sb-simd/test-suite/utilities.lisp
index 7e77ef3ea..7751abab6 100644
--- a/contrib/sb-simd/test-suite/utilities.lisp
+++ b/contrib/sb-simd/test-suite/utilities.lisp
@@ -40,6 +40,7 @@
(multiple-value-bind (a0 a1) (sb-ext:%simd-pack-ub64s a)
(multiple-value-bind (b0 b1) (sb-ext:%simd-pack-ub64s b)
(and (= a0 b0) (= a1 b1))))))
+ #+sb-simd-pack-256
(sb-ext:simd-pack-256
(when (sb-ext:simd-pack-256-p b)
(multiple-value-bind (a0 a1 a2 a3) (sb-ext:%simd-pack-256-ub64s a)
diff --git a/doc/manual/beyond-ansi.texinfo b/doc/manual/beyond-ansi.texinfo
index 8f2d8466f..4f21d2fa2 100644
--- a/doc/manual/beyond-ansi.texinfo
+++ b/doc/manual/beyond-ansi.texinfo
@@ -664,7 +664,7 @@ For structures:
@item @code{slot-value} and @code{slot-boundp} function as expected, including (for
@code{slot-value}) calling and respecting the return value of
- @code{slot-unbound} if the slot is unbound;
+ @code{slot-unbound} if the slot is unbound;
@item @code{(setf slot-value)} functions as expected, including performing
type checks to verify that the new value is of an appropriate type
diff --git a/doc/manual/compiler.texinfo b/doc/manual/compiler.texinfo
index e3afe66ef..0ab8479b0 100644
--- a/doc/manual/compiler.texinfo
+++ b/doc/manual/compiler.texinfo
@@ -393,7 +393,7 @@ Note that @code{do} appears in the processing path. This is because
@cindex macroexpansion
@cindex source transform
The rest of the processing path results from the macroexpansion of
-@code{do}:
+@code{do}:
@example
(block nil
diff --git a/make-config.sh b/make-config.sh
index 06f33bd04..b8d44b6a8 100755
--- a/make-config.sh
+++ b/make-config.sh
@@ -555,7 +555,9 @@ echo "(lambda (features) (set-difference (union features (list :${sbcl_arch}$WIT
# Automatically block sb-simd on non-x86 platforms, at least for now.
case "$sbcl_arch" in
- x86-64) ;; *) SBCL_CONTRIB_BLOCKLIST="$SBCL_CONTRIB_BLOCKLIST sb-simd" ;;
+ x86-64) ;;
+ arm64) ;;
+ *) SBCL_CONTRIB_BLOCKLIST="$SBCL_CONTRIB_BLOCKLIST sb-simd" ;;
esac
case "$sbcl_os" in
linux) ;; *) SBCL_CONTRIB_BLOCKLIST="$SBCL_CONTRIB_BLOCKLIST sb-perf" ;;
@@ -751,6 +753,9 @@ case "$sbcl_arch" in
printf ' :immobile-space' >> $ltf
esac
;;
+ arm64)
+ printf ' :sb-simd-pack' >> $ltf # not mandatory
+ ;;
ppc)
if [ "$sbcl_os" = "darwin" ]; then
# We provide a dlopen shim, so a little lie won't hurt
diff --git a/tests/sb-simd.impure.lisp b/tests/sb-simd.impure.lisp
index 9e6197e78..8668fcf3d 100644
--- a/tests/sb-simd.impure.lisp
+++ b/tests/sb-simd.impure.lisp
@@ -14,10 +14,6 @@
(t
(error "Unexpected error: ~A" c)))))
-(when (zerop (sb-alien:extern-alien "avx2_supported" int))
- (format t "~&Skipping test of sb-simd~%")
- (invoke-restart 'run-tests::skip-file))
-
(with-compilation-unit ()
(dolist (file '("packages.lisp"
"numbers.lisp"
diff --git a/xperfecthash63.lisp-expr b/xperfecthash63.lisp-expr
index 6c3da591e..2b29fc6ec 100644
--- a/xperfecthash63.lisp-expr
+++ b/xperfecthash63.lisp-expr
@@ -1746,5 +1746,79 @@
(let ((b (& val #x7)))
(let ((a (>> (u32+ val (<< val 25)) 29)))
(^ a (aref tab b))))))")
+(#(11131BE7 168C60FD 2D45A4CF 2E16A006 32030766 3AA6053C 444AF531 47AA782C 5302EDCF 5CB9F779 6964D143 7058717C 83A1A47F
+ 86C51BD4 977E07FB B9D57393 CFFE5CF3)
+ "#(((:TYPE VALUES-TYPE)) ((:TYPE FUN-DESIGNATOR-TYPE) (:TYPE FUN-TYPE)) ((:TYPE NUMERIC-UNION-TYPE)) ((:TYPE UNION-TYPE) (:TYPE INTERSECTION-TYPE) (:TYPE COMPOUND-TYPE)) ((:TYPE MEMBER-TYPE)) ((:TYPE ARRAY-TYPE)) ((:TYPE UNKNOWN-TYPE) (:TYPE HAIRY-TYPE)) ((:TYPE NEGATION-TYPE)) ((:TYPE CONSTANT-TYPE)) ((:TYPE CONS-TYPE)) ((:TYPE CHARACTER-SET-TYPE)) ((:TYPE SIMD-PACK-TYPE)) ((:TYPE ALIEN-TYPE-TYPE)))"
+ "((let ((tab #a((16) (unsigned-byte 8) 0 13 0 0 9 0 13 1 0 15 0 4 0 17 0 0)))
+ (let ((b (& val #xf)))
+ (let ((a (>> val 28)))
+ (^ a (aref tab b))))))")
+(#(11131BE7 168C60FD 2D45A4CF 2E16A006 3AA6053C 444AF531 5302EDCF 5CB9F779 6964D143 7058717C 83A1A47F 86C51BD4 977E07FB
+ B9D57393 CFFE5CF3 D3774C34)
+ "#(((:TYPE UNKNOWN-TYPE) (:TYPE HAIRY-TYPE) (:TYPE SIMD-PACK-TYPE) (:TYPE CHARACTER-SET-TYPE) (:TYPE MEMBER-TYPE) (:TYPE NUMERIC-UNION-TYPE)) ((:TYPE VALUES-TYPE) (:TYPE FUN-TYPE) (:TYPE FUN-DESIGNATOR-TYPE) (:TYPE ARGS-TYPE)) ((:TYPE CONS-TYPE)) ((:TYPE ARRAY-TYPE)) ((:TYPE UNION-TYPE) (:TYPE INTERSECTION-TYPE) (:TYPE COMPOUND-TYPE)) ((:TYPE NEGATION-TYPE)))"
+ "((let ((tab #a((8) (unsigned-byte 8) 3 12 0 12 0 12 15 0)))
+ (let ((b (& (>> val 6) #x7)))
+ (let ((a (>> (<< val 5) 29)))
+ (^ a (aref tab b))))))")
+(#(11131BE7 168C60FD 2D45A4CF 2E16A006 32030766 3AA6053C 44273B23 444AF531 47AA782C 5302EDCF 538C4003 5595EEC3 58D40BEE
+ 5CB9F779 634EB941 646135D9 6964D143 7058717C 7CFF98EB 7F255BE9 83A1A47F 86C51BD4 90EC2354 977E07FB 99AE692E B9D57393
+ CFFE5CF3 D3774C34 E8C239EF FC8EAFE7)
+ "(UNDEFINED-CLASSOID BUILT-IN-CLASSOID CONDITION-CLASSOID STRUCTURE-CLASSOID STANDARD-CLASSOID STATIC-CLASSOID CLASSOID SIMD-PACK-TYPE CONSTANT-TYPE VALUES-TYPE FUN-TYPE FUN-DESIGNATOR-TYPE ARGS-TYPE CONS-TYPE NUMERIC-UNION-TYPE NEGATION-TYPE ALIEN-TYPE-TYPE NAMED-TYPE UNKNOWN-TYPE HAIRY-TYPE MEMBER-TYPE ARRAY-TYPE CHARACTER-SET-TYPE COMPOUND-TYPE UNION-TYPE INTERSECTION-TYPE CTYPE SB-C::LVAR SB-C::LAMBDA-VAR CONSTANT)"
+ "((let ((tab #a((16) (unsigned-byte 8) 21 31 22 3 15 7 31 22 11 24 13 21 0 13 18 0)))
+ (let ((b (& (>> val 9) #xf)))
+ (let ((a (>> (<< val 7) 28)))
+ (^ a (aref tab b))))))")
+(#(90D0AD8D A836E4F7 BE9D3FB8 D3F36C41)
+ "(SB-VM::NEON-REG SB-VM::INT-NEON-REG SB-VM::SINGLE-NEON-REG SB-VM::DOUBLE-NEON-REG)"
+ "((& (>> val 2) 3))")
+(#(1607B33C 380E1F3F 90D0AD8D A4EDA9A1 A836E4F7 BE9D3FB8 D3F36C41 FD00B976)
+ "(SB-VM::COMPLEX-DOUBLE-REG SB-VM::SINGLE-NEON-REG SB-VM::DOUBLE-NEON-REG SB-VM::INT-NEON-REG SB-VM::NEON-REG SB-VM::COMPLEX-SINGLE-REG SB-VM::DOUBLE-REG SB-VM::SINGLE-REG)"
+ "((& (- (>> val 2) (>> val 9)) 7))")
+(#(C 2E 34 36 38 3A 3C)
+ "(26 30 29 28 27 23 6)"
+ "((& (+ (>> val 1) (>> val 2) (>> val 4)) 7))")
+(#(29F3ECDD 32F3FB08 38F4047A)
+ "#(:B :H :S)"
+ "((& val 3))")
+(#(6 8 A 16 2C)
+ "(5 4 3 11 22)"
+ "((let ((tab #a((4) (unsigned-byte 8) 2 3 6 0)))
+ (let ((b (& (>> val 1) #x3)))
+ (let ((a (>> (<< val 26) 30)))
+ (^ a (aref tab b))))))")
+(#(6 8 A 18 2C)
+ "(5 4 3 12 22)"
+ "((let ((tab #a((4) (unsigned-byte 8) 0 2 1 4)))
+ (let ((b (& (>> val 1) #x3)))
+ (let ((a (>> (<< val 26) 30)))
+ (^ a (aref tab b))))))")
+(#(1E 20 22 38 3A 3C)
+ "(17 16 15 30 29 28)"
+ "((let ((tab #a((4) (unsigned-byte 8) 0 7 2 1)))
+ (let ((b (& (>> val 1) #x3)))
+ (let ((a (>> (<< val 26) 30)))
+ (^ a (aref tab b))))))")
+(#(8 E 10 16 2C)
+ "(11 22 7 8 4)"
+ "((let ((tab #a((4) (unsigned-byte 8) 0 0 6 2)))
+ (let ((b (& (>> val 1) #x3)))
+ (let ((a (>> (<< val 26) 30)))
+ (^ a (aref tab b))))))")
+(#(0 2 4 24 26 28 2E 30)
+ "(18 19 23 24 2 20 1 0)"
+ "((let ((tab #a((4) (unsigned-byte 8) 7 0 1 7)))
+ (+= val #x762d9085)
+ (^= val (>> val 4))
+ (let ((b (& val #x3)))
+ (let ((a (>> (u32+ val (<< val 26)) 30)))
+ (^ a (aref tab b))))))")
+(#(0 2 4 6 8 A E 10 12 14 16 18 1A 1C 1E 20 22 24 26 28 2A 2E 30 32 34 36 3A 3C 7A)
+ "(61 4 3 10 7 8 9 5 14 13 12 11 26 25 24 23 16 17 15 29 30 27 21 20 2 19 18 0 1)"
+ "((let ((tab #a((16) (unsigned-byte 8) 15 0 21 16 9 3 18 13 1 11 20 23 24 4 26 29)))
+ (+= val #x2c45f2cb)
+ (^= val (>> val 4))
+ (let ((b (& val #xf)))
+ (let ((a (>> (u32+ val (<< val 23)) 28)))
+ (^ a (aref tab b))))))")
)
;; EOF
-----------------------------------------------------------------------
hooks/post-receive
--
SBCL