[PR] swscale/aarch64: split ops_static.c out of ops_asmgen.c (PR #23858)
Ramiro Polla via ffmpeg-devel <[email protected]> Sun, 19 Jul 2026 21:11:25 -0000
| Newsgroups | gmane.comp.video.ffmpeg.devel |
|---|---|
| Message-ID | <178449548631.59.14428571186218240033@29965ddac10e> |
PR #23858 opened by Ramiro Polla (ramiro) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23858 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23858.patch This PR splits the CPS-specific code out of `ops_asmgen.c` and into `ops_static.c` (which now becomes the standalone tool to generate `ops_neon.gen.S`). It also shapes `SwsAArch64Context` and `ops_asmgen.c` to support the JIT backend. The split files might still need a few tweaks for complete JIT support, but this will come with the JIT PR once it's finished. >From 116c097a7cf176120412c83d552230da898fb432 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Mon, 6 Jul 2026 20:07:29 +0200 Subject: [PATCH 01/23] swscale/aarch64/rasm: fill AArch64VecViews by value Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 77 +++++++++------------------------ libswscale/aarch64/rasm.c | 33 +++++++------- libswscale/aarch64/rasm.h | 9 +++- 3 files changed, 46 insertions(+), 73 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 5faa8166b8..09bf8f95a1 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -379,13 +379,9 @@ static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParam RasmContext *r = s->rctx; RasmOp bitmask_vec = s->vt[1]; RasmOp wtmp = a64op_w(s->tmp0); - AArch64VecViews vl[1]; - AArch64VecViews vtmp; - AArch64VecViews shift_vec; - - a64op_vec_views(s->vt[0], &shift_vec); - a64op_vec_views(s->vl[0], &vl[0]); - a64op_vec_views(s->vt[2], &vtmp); + AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; + AArch64VecViews vtmp = a64op_vec_views(s->vt[2]); + AArch64VecViews shift_vec = a64op_vec_views(s->vt[0]); /* Note that shift_vec has negative values, so that using it with * ushl actually performs a right shift. */ @@ -415,11 +411,8 @@ static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplPa { RasmContext *r = s->rctx; RasmOp nibble_mask = v_8b(s->vt[0]); - AArch64VecViews vl[1]; - AArch64VecViews vtmp; - - a64op_vec_views(s->vl[0], &vl[0]); - a64op_vec_views(s->vt[1], &vtmp); + AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; + AArch64VecViews vtmp = a64op_vec_views(s->vt[1]); rasm_annotate_next(r, "v128 nibble_mask = {0xf <repeats 8 times>, 0x0 <repeats 8 times>};"); i_movi(r, nibble_mask, IMM(0x0f)); @@ -459,13 +452,8 @@ static void asmgen_op_read_packed(SwsAArch64Context *s, const SwsAArch64OpImplPa static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - AArch64VecViews vl[4]; - AArch64VecViews vh[4]; - - for (int i = 0; i < 4; i++) { - a64op_vec_views(s->vl[i], &vl[i]); - a64op_vec_views(s->vh[i], &vh[i]); - } + AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(s->vl); + AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(s->vh); LOOP_MASK(p, i) { switch ((s->use_vh ? 0x100 : 0) | s->vec_size) { @@ -487,15 +475,10 @@ static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplPa static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - AArch64VecViews vl[1]; - AArch64VecViews shift_vec; - AArch64VecViews vtmp0; - AArch64VecViews vtmp1; - - a64op_vec_views(s->vl[0], &vl[0]); - a64op_vec_views(s->vt[0], &shift_vec); - a64op_vec_views(s->vt[1], &vtmp0); - a64op_vec_views(s->vt[2], &vtmp1); + AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; + AArch64VecViews shift_vec = a64op_vec_views(s->vt[0]); + AArch64VecViews vtmp0 = a64op_vec_views(s->vt[1]); + AArch64VecViews vtmp1 = a64op_vec_views(s->vt[2]); rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); i_ldr(r, shift_vec.q, a64op_off(s->impl, offsetof_impl_priv)); @@ -518,14 +501,9 @@ static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplPara static void asmgen_op_write_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - AArch64VecViews vl[4]; - AArch64VecViews vtmp0; - AArch64VecViews vtmp1; - - for (int i = 0; i < 4; i++) - a64op_vec_views(s->vl[i], &vl[i]); - a64op_vec_views(s->vt[0], &vtmp0); - a64op_vec_views(s->vt[1], &vtmp1); + AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(s->vl); + AArch64VecViews vtmp0 = a64op_vec_views(s->vt[0]); + AArch64VecViews vtmp1 = a64op_vec_views(s->vt[1]); if (p->block_size == 8) { i_shl (r, vtmp0.h4, vl[0].h4, IMM(4)); @@ -564,13 +542,8 @@ static void asmgen_op_write_packed(SwsAArch64Context *s, const SwsAArch64OpImplP static void asmgen_op_write_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - AArch64VecViews vl[4]; - AArch64VecViews vh[4]; - - for (int i = 0; i < 4; i++) { - a64op_vec_views(s->vl[i], &vl[i]); - a64op_vec_views(s->vh[i], &vh[i]); - } + AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(s->vl); + AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(s->vh); LOOP_MASK(p, i) { switch ((s->use_vh ? 0x100 : 0) | s->vec_size) { @@ -589,13 +562,8 @@ static void asmgen_op_write_planar(SwsAArch64Context *s, const SwsAArch64OpImplP static void asmgen_op_swap_bytes(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - AArch64VecViews vl[4]; - AArch64VecViews vh[4]; - - for (int i = 0; i < 4; i++) { - a64op_vec_views(s->vl[i], &vl[i]); - a64op_vec_views(s->vh[i], &vh[i]); - } + AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(s->vl); + AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(s->vh); switch (ff_sws_pixel_type_size(p->type)) { case sizeof(uint16_t): @@ -847,8 +815,8 @@ static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams * static void asmgen_op_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - AArch64VecViews vl[4]; - AArch64VecViews vh[4]; + AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(s->vl); + AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(s->vh); /** * Since each instruction in the convert operation needs specific @@ -856,11 +824,6 @@ static void asmgen_op_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams * each operand instead of reshaping all vectors. */ - for (int i = 0; i < 4; i++) { - a64op_vec_views(s->vl[i], &vl[i]); - a64op_vec_views(s->vh[i], &vh[i]); - } - size_t src_el_size = s->el_size; SwsPixelType to_type; switch (p->uop) { diff --git a/libswscale/aarch64/rasm.c b/libswscale/aarch64/rasm.c index abea0e8dce..b866e1d3ff 100644 --- a/libswscale/aarch64/rasm.c +++ b/libswscale/aarch64/rasm.c @@ -327,23 +327,26 @@ int rasm_new_labelf(RasmContext *rctx, char *s, size_t n, const char *fmt, ...) /*********************************************************************/ /* AArch64-specific */ -void a64op_vec_views(RasmOp op, AArch64VecViews *out) +AArch64VecViews a64op_vec_views(RasmOp op) { uint8_t n = a64op_vec_n(op); - out->b = a64op_vecb (n); - out->h = a64op_vech (n); - out->s = a64op_vecs (n); - out->d = a64op_vecd (n); - out->q = a64op_vecq (n); - out->b8 = a64op_vec8b (n); - out->b16 = a64op_vec16b(n); - out->h4 = a64op_vec4h (n); - out->h8 = a64op_vec8h (n); - out->s2 = a64op_vec2s (n); - out->s4 = a64op_vec4s (n); - out->d2 = a64op_vec2d (n); + AArch64VecViews out = { + .b = a64op_vecb (n), + .h = a64op_vech (n), + .s = a64op_vecs (n), + .d = a64op_vecd (n), + .q = a64op_vecq (n), + .b8 = a64op_vec8b (n), + .b16 = a64op_vec16b(n), + .h4 = a64op_vec4h (n), + .h8 = a64op_vec8h (n), + .s2 = a64op_vec2s (n), + .s4 = a64op_vec4s (n), + .d2 = a64op_vec2d (n), + }; for (int i = 0; i < 2; i++) - out->be[i] = a64op_elem(out->b, i); + out.be[i] = a64op_elem(out.b, i); for (int i = 0; i < 2; i++) - out->de[i] = a64op_elem(out->d, i); + out.de[i] = a64op_elem(out.d, i); + return out; } diff --git a/libswscale/aarch64/rasm.h b/libswscale/aarch64/rasm.h index ac37efed86..8b1c8efcb2 100644 --- a/libswscale/aarch64/rasm.h +++ b/libswscale/aarch64/rasm.h @@ -474,7 +474,14 @@ typedef struct AArch64VecViews { } AArch64VecViews; /* Fill vector view struct for given op. */ -void a64op_vec_views(RasmOp op, AArch64VecViews *out); +AArch64VecViews a64op_vec_views(RasmOp op); + +#define A64OP_VEC_VIEWS4(op) { \ + a64op_vec_views((op)[0]), \ + a64op_vec_views((op)[1]), \ + a64op_vec_views((op)[2]), \ + a64op_vec_views((op)[3]), \ +} /*********************************************************************/ /* AARCH64_OP_BASE */ -- 2.52.0 >From 8a56d90f6fdccd9490fa6d938658981730c9c584 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 19 Jul 2026 18:21:58 +0200 Subject: [PATCH 02/23] swscale/aarch64/rasm: add extra helpers for instructions that take all 16b operands Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/rasm.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libswscale/aarch64/rasm.h b/libswscale/aarch64/rasm.h index 8b1c8efcb2..3d28f8099a 100644 --- a/libswscale/aarch64/rasm.h +++ b/libswscale/aarch64/rasm.h @@ -623,6 +623,8 @@ static inline RasmOp a64cond_nv(void) { return a64op_cond(AARCH64_COND_NV); } #define i_ble(rctx, id) i_bcond(rctx, a64cond_le(), rasm_op_label(id)) /* Extra helpers. */ -#define i_mov16b(rctx, op0, op1) i_mov(rctx, v_16b(op0), v_16b(op1)) +#define i_and16b(rctx, op0, op1, op2) i_and(rctx, v_16b(op0), v_16b(op1), v_16b(op2)) +#define i_mov16b(rctx, op0, op1 ) i_mov(rctx, v_16b(op0), v_16b(op1) ) +#define i_orr16b(rctx, op0, op1, op2) i_orr(rctx, v_16b(op0), v_16b(op1), v_16b(op2)) #endif /* SWSCALE_AARCH64_RASM_H */ -- 2.52.0 >From 2accecb3460d4cab503178f11548a2c924ed3243 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 19 Jul 2026 18:23:37 +0200 Subject: [PATCH 03/23] swscale/aarch64/ops_asmgen: use and/orr arrangement specifier helpers from rasm This way, we don't depend on the RasmOp having been correctly reshaped prior to being used. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 09bf8f95a1..6a526da07e 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -684,9 +684,8 @@ static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams } /* Apply masks. */ - reshape_all_vectors(s, 16, 1); - LOOP_MASK_BWD (p, i) { i_and(r, vl[i], vl[i], vt[mask_idx[i]]); CMTF("vl[%u] &= 0x%x;", i, mask_val[i]); } - LOOP_MASK_BWD_VH(s, p, i) { i_and(r, vh[i], vh[i], vt[mask_idx[i]]); CMTF("vh[%u] &= 0x%x;", i, mask_val[i]); } + LOOP_MASK_BWD (p, i) { i_and16b(r, vl[i], vl[i], vt[mask_idx[i]]); CMTF("vl[%u] &= 0x%x;", i, mask_val[i]); } + LOOP_MASK_BWD_VH(s, p, i) { i_and16b(r, vh[i], vh[i], vt[mask_idx[i]]); CMTF("vh[%u] &= 0x%x;", i, mask_val[i]); } } /*********************************************************************/ @@ -716,12 +715,11 @@ static void asmgen_op_pack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p LOOP_VH(s, offset_mask, i) { i_shl(r, vh[i], vh[i], IMM(offsets[i])); CMTF("vh[%u] <<= %u;", i, offsets[i]); } /* Combine components. */ - reshape_all_vectors(s, 16, 1); LOOP_MASK (p, i) { if (i != 0) { - i_orr (r, vl[0], vl[0], vl[i]); CMTF("vl[0] |= vl[%u];", i); + i_orr16b (r, vl[0], vl[0], vl[i]); CMTF("vl[0] |= vl[%u];", i); if (s->use_vh) { - i_orr(r, vh[0], vh[0], vh[i]); CMTF("vh[0] |= vh[%u];", i); + i_orr16b(r, vh[0], vh[0], vh[i]); CMTF("vh[0] |= vh[%u];", i); } } } -- 2.52.0 >From 44deef563cd3dc463087e7ede0556ed37e92377e Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sat, 18 Jul 2026 01:53:21 +0200 Subject: [PATCH 04/23] swscale/aarch64/ops_asmgen: use arrangement specifiers for vector registers with known size This way, we don't depend on the RasmOp having been correctly reshaped prior to being used. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 6a526da07e..94ee61c725 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -377,7 +377,7 @@ static void asmgen_set_load_cont_node(SwsAArch64Context *s) static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - RasmOp bitmask_vec = s->vt[1]; + AArch64VecViews bitmask_vec = a64op_vec_views(s->vt[1]); RasmOp wtmp = a64op_w(s->tmp0); AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; AArch64VecViews vtmp = a64op_vec_views(s->vt[2]); @@ -391,41 +391,41 @@ static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParam if (p->block_size == 16) { i_ldrh(r, wtmp, a64op_post(s->in[0], 2)); CMT("uint16_t tmp = *in[0]++;"); - i_movi(r, bitmask_vec, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 16 times>};"); + i_movi(r, bitmask_vec.b16, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 16 times>};"); i_dup (r, vl[0].b8, wtmp); CMT("vl[0].lo = broadcast(tmp);"); i_lsr (r, wtmp, wtmp, IMM(8)); CMT("tmp >>= 8;"); i_dup (r, vtmp.b8, wtmp); CMT("vtmp.lo = broadcast(tmp);"); i_ins (r, vl[0].de[1], vtmp.de[0]); CMT("vl[0].hi = vtmp.lo;"); i_ushl(r, vl[0].b16, vl[0].b16, shift_vec.b16); CMT("vl[0] <<= shift_vec;"); - i_and (r, vl[0].b16, vl[0].b16, bitmask_vec); CMT("vl[0] &= bitmask_vec;"); + i_and (r, vl[0].b16, vl[0].b16, bitmask_vec.b16); CMT("vl[0] &= bitmask_vec;"); } else { i_ldrb(r, wtmp, a64op_post(s->in[0], 1)); CMT("uint8_t tmp = *in[0]++;"); - i_movi(r, bitmask_vec, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 8 times>, 0 <repeats 8 times>};"); + i_movi(r, bitmask_vec.b8, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 8 times>, 0 <repeats 8 times>};"); i_dup (r, vl[0].b8, wtmp); CMT("vl[0].lo = broadcast(tmp);"); i_ushl(r, vl[0].b8, vl[0].b8, shift_vec.b8); CMT("vl[0] <<= shift_vec;"); - i_and (r, vl[0].b8, vl[0].b8, bitmask_vec); CMT("vl[0] &= bitmask_vec;"); + i_and (r, vl[0].b8, vl[0].b8, bitmask_vec.b8); CMT("vl[0] &= bitmask_vec;"); } } static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - RasmOp nibble_mask = v_8b(s->vt[0]); + AArch64VecViews nibble_mask = a64op_vec_views(s->vt[0]); AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; AArch64VecViews vtmp = a64op_vec_views(s->vt[1]); rasm_annotate_next(r, "v128 nibble_mask = {0xf <repeats 8 times>, 0x0 <repeats 8 times>};"); - i_movi(r, nibble_mask, IMM(0x0f)); + i_movi(r, nibble_mask.b8, IMM(0x0f)); if (p->block_size == 8) { i_ldr (r, vl[0].s, a64op_post(s->in[0], 4)); CMT("vl[0] = *in[0]++;"); i_ushr(r, vtmp.b8, vl[0].b8, IMM(4)); CMT("vtmp.lo = vl[0] >> 4;"); - i_and (r, vl[0].b8, vl[0].b8, nibble_mask); CMT("vl[0].lo &= nibble_mask;"); + i_and (r, vl[0].b8, vl[0].b8, nibble_mask.b8); CMT("vl[0].lo &= nibble_mask;"); i_zip1(r, vl[0].b8, vtmp.b8, vl[0].b8); CMT("interleave"); } else { i_ldr (r, vl[0].d, a64op_post(s->in[0], 8)); CMT("vl[0] = *in[0]++;"); i_ushr(r, vtmp.b8, vl[0].b8, IMM(4)); CMT("vtmp.lo = vl[0] >> 4;"); - i_and (r, vl[0].b8, vl[0].b8, nibble_mask); CMT("vl[0].lo &= nibble_mask;"); + i_and (r, vl[0].b8, vl[0].b8, nibble_mask.b8); CMT("vl[0].lo &= nibble_mask;"); i_zip1(r, vl[0].b16, vtmp.b16, vl[0].b16); CMT("interleave"); } } -- 2.52.0 >From daaa6e78bf5b208a2193a753dc9136fa245f9da1 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Fri, 17 Jul 2026 00:09:47 +0200 Subject: [PATCH 05/23] swscale/aarch64/ops_asmgen: use RasmOp directly when generating masks for SWS_UOP_UNPACK There is no need to pack the vector register usage for this function. This change creates holes in the usage of const vector registers, but populates the RasmOp const array directly, which will help for the implementation of the JIT compiler. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 94ee61c725..46e10f80d9 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -626,11 +626,9 @@ static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams RasmContext *r = s->rctx; RasmOp *vl = s->vl; RasmOp *vh = s->vh; - RasmOp *vt = s->vt; + RasmOp *vmask = s->vt; RasmOp mask_gpr = a64op_w(s->tmp0); uint32_t mask_val[4] = { 0 }; - uint8_t mask_idx[4] = { 0 }; - uint8_t cur_vt = 0; const int offsets[4] = { p->par.pack.pattern[3] + p->par.pack.pattern[2] + p->par.pack.pattern[1], @@ -646,7 +644,7 @@ static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams for (int j = 0; j < 4; j++) { if (mask_val[j] == val) { mask_val[i] = mask_val[j]; - mask_idx[i] = mask_idx[j]; + vmask[i] = vmask[j]; break; } } @@ -657,13 +655,13 @@ static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams * like 10-bit. In those cases, we use mov + dup instead. */ if (val <= 0xff || val == 0xffff) { - i_movi(r, vt[cur_vt], IMM(val)); + i_movi(r, vmask[i], IMM(val)); } else { - i_mov (r, mask_gpr, IMM(val)); - i_dup (r, vt[cur_vt], mask_gpr); + i_mov (r, mask_gpr, IMM(val)); + i_dup (r, vmask[i], mask_gpr); } mask_val[i] = val; - mask_idx[i] = cur_vt++; + vmask[i] = v_16b(vmask[i]); } } @@ -684,8 +682,8 @@ static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams } /* Apply masks. */ - LOOP_MASK_BWD (p, i) { i_and16b(r, vl[i], vl[i], vt[mask_idx[i]]); CMTF("vl[%u] &= 0x%x;", i, mask_val[i]); } - LOOP_MASK_BWD_VH(s, p, i) { i_and16b(r, vh[i], vh[i], vt[mask_idx[i]]); CMTF("vh[%u] &= 0x%x;", i, mask_val[i]); } + LOOP_MASK_BWD (p, i) { i_and16b(r, vl[i], vl[i], vmask[i]); CMTF("vl[%u] &= 0x%x;", i, (1u << p->par.pack.pattern[i]) - 1); } + LOOP_MASK_BWD_VH(s, p, i) { i_and16b(r, vh[i], vh[i], vmask[i]); CMTF("vh[%u] &= 0x%x;", i, (1u << p->par.pack.pattern[i]) - 1); } } /*********************************************************************/ -- 2.52.0 >From cca5a60cf9ad7512adf16aad927af0d12ba93591 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Thu, 16 Jul 2026 23:57:46 +0200 Subject: [PATCH 06/23] swscale/aarch64/ops_asmgen: add RasmOp impl_priv helper to SwsAArch64Context This reduces the line length for instructions that access impl->priv. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 46e10f80d9..cdd7a564c3 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -140,6 +140,7 @@ typedef struct SwsAArch64Context { RasmOp op0_func; RasmOp op1_impl; RasmOp cont; + RasmOp impl_priv; RasmNode *load_cont_node; /* Vector registers. Two banks (low and high) are used. */ @@ -386,7 +387,7 @@ static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParam /* Note that shift_vec has negative values, so that using it with * ushl actually performs a right shift. */ rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); - i_ldr(r, shift_vec.q, a64op_off(s->impl, offsetof_impl_priv)); + i_ldr(r, shift_vec.q, s->impl_priv); asmgen_set_load_cont_node(s); if (p->block_size == 16) { @@ -481,7 +482,7 @@ static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplPara AArch64VecViews vtmp1 = a64op_vec_views(s->vt[2]); rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); - i_ldr(r, shift_vec.q, a64op_off(s->impl, offsetof_impl_priv)); + i_ldr(r, shift_vec.q, s->impl_priv); asmgen_set_load_cont_node(s); if (p->block_size == 8) { @@ -793,7 +794,7 @@ static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams * load_priv = true; } if (load_priv) { - i_ldr(r, v_q(clear_vec), a64op_off(s->impl, offsetof_impl_priv)); CMT("v128 clear_vec = impl->priv.v128;"); + i_ldr(r, v_q(clear_vec), s->impl_priv); CMT("v128 clear_vec = impl->priv.v128;"); asmgen_set_load_cont_node(s); } @@ -940,7 +941,7 @@ static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) RasmOp *vt = s->vt; RasmOp min_vec = s->vt[4]; - i_ldr(r, v_q(min_vec), a64op_off(s->impl, offsetof_impl_priv)); CMT("v128 min_vec = impl->priv.v128;"); + i_ldr(r, v_q(min_vec), s->impl_priv); CMT("v128 min_vec = impl->priv.v128;"); asmgen_set_load_cont_node(s); LOOP_MASK(p, i) { i_dup(r, vt[i], a64op_elem(min_vec, i)); CMTF("v128 vmin%u = min_vec[%u];", i, i); } @@ -965,7 +966,7 @@ static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) RasmOp *vt = s->vt; RasmOp max_vec = s->vt[4]; - i_ldr(r, v_q(max_vec), a64op_off(s->impl, offsetof_impl_priv)); CMT("v128 max_vec = impl->priv.v128;"); + i_ldr(r, v_q(max_vec), s->impl_priv); CMT("v128 max_vec = impl->priv.v128;"); asmgen_set_load_cont_node(s); LOOP_MASK(p, i) { i_dup(r, vt[i], a64op_elem(max_vec, i)); CMTF("v128 vmax%u = max_vec[%u];", i, i); } @@ -1115,7 +1116,7 @@ static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams case 3: coeff_veclist = vv_3(vc[0], vc[1], vc[2]); break; case 4: coeff_veclist = vv_4(vc[0], vc[1], vc[2], vc[3]); break; } - i_ldr(r, ptr, a64op_off(s->impl, offsetof_impl_priv)); CMT("v128 *vcoeff_ptr = impl->priv.ptr;"); + i_ldr(r, ptr, s->impl_priv); CMT("v128 *vcoeff_ptr = impl->priv.ptr;"); asmgen_set_load_cont_node(s); i_ld1(r, coeff_veclist, a64op_base(ptr)); CMT("coeff_veclist = *vcoeff_ptr;"); @@ -1183,7 +1184,7 @@ static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams } } - i_ldr(r, ptr, a64op_off(s->impl, offsetof_impl_priv)); CMT("void *ptr = impl->priv.ptr;"); + i_ldr(r, ptr, s->impl_priv); CMT("void *ptr = impl->priv.ptr;"); asmgen_set_load_cont_node(s); /** @@ -1306,6 +1307,7 @@ static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) reshape_all_vectors(s, s->el_count, el_size); /* Common start for continuation-passing style (CPS) functions. */ + s->impl_priv = a64op_off(s->impl, offsetof_impl_priv); asmgen_set_load_cont_node(s); switch (p->uop) { -- 2.52.0 >From 72d5b1f4efabe774fc4b543d582bfa631e816d89 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Mon, 6 Jul 2026 22:13:31 +0200 Subject: [PATCH 07/23] swscale/aarch64/ops_asmgen: split constant vector registers out of temp This makes each vector register's purpose clearer, and will let the JIT compiler later factor loading of constant values out of the main loop. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 101 ++++++++++++++++---------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index cdd7a564c3..6a6a530a8c 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -144,9 +144,10 @@ typedef struct SwsAArch64Context { RasmNode *load_cont_node; /* Vector registers. Two banks (low and high) are used. */ - RasmOp vl[ 4]; - RasmOp vh[ 4]; - RasmOp vt[12]; + RasmOp vl[4]; + RasmOp vh[4]; + RasmOp vt[8]; + RasmOp vk[4]; /* Read/Write data pointers and padding. */ RasmOp in[4]; @@ -176,26 +177,26 @@ typedef struct SwsAArch64Context { /* Reshape all vector registers for current SwsOp. */ static void reshape_all_vectors(SwsAArch64Context *s, int el_count, int el_size) { - s->vl[ 0] = a64op_make_vec( 0, el_count, el_size); - s->vl[ 1] = a64op_make_vec( 1, el_count, el_size); - s->vl[ 2] = a64op_make_vec( 2, el_count, el_size); - s->vl[ 3] = a64op_make_vec( 3, el_count, el_size); - s->vh[ 0] = a64op_make_vec( 4, el_count, el_size); - s->vh[ 1] = a64op_make_vec( 5, el_count, el_size); - s->vh[ 2] = a64op_make_vec( 6, el_count, el_size); - s->vh[ 3] = a64op_make_vec( 7, el_count, el_size); - s->vt[ 0] = a64op_make_vec(16, el_count, el_size); - s->vt[ 1] = a64op_make_vec(17, el_count, el_size); - s->vt[ 2] = a64op_make_vec(18, el_count, el_size); - s->vt[ 3] = a64op_make_vec(19, el_count, el_size); - s->vt[ 4] = a64op_make_vec(20, el_count, el_size); - s->vt[ 5] = a64op_make_vec(21, el_count, el_size); - s->vt[ 6] = a64op_make_vec(22, el_count, el_size); - s->vt[ 7] = a64op_make_vec(23, el_count, el_size); - s->vt[ 8] = a64op_make_vec(24, el_count, el_size); - s->vt[ 9] = a64op_make_vec(25, el_count, el_size); - s->vt[10] = a64op_make_vec(26, el_count, el_size); - s->vt[11] = a64op_make_vec(27, el_count, el_size); + s->vl[0] = a64op_make_vec( 0, el_count, el_size); + s->vl[1] = a64op_make_vec( 1, el_count, el_size); + s->vl[2] = a64op_make_vec( 2, el_count, el_size); + s->vl[3] = a64op_make_vec( 3, el_count, el_size); + s->vh[0] = a64op_make_vec( 4, el_count, el_size); + s->vh[1] = a64op_make_vec( 5, el_count, el_size); + s->vh[2] = a64op_make_vec( 6, el_count, el_size); + s->vh[3] = a64op_make_vec( 7, el_count, el_size); + s->vt[0] = a64op_make_vec(16, el_count, el_size); + s->vt[1] = a64op_make_vec(17, el_count, el_size); + s->vt[2] = a64op_make_vec(18, el_count, el_size); + s->vt[3] = a64op_make_vec(19, el_count, el_size); + s->vt[4] = a64op_make_vec(20, el_count, el_size); + s->vt[5] = a64op_make_vec(21, el_count, el_size); + s->vt[6] = a64op_make_vec(22, el_count, el_size); + s->vt[7] = a64op_make_vec(23, el_count, el_size); + s->vk[0] = a64op_make_vec(24, el_count, el_size); + s->vk[1] = a64op_make_vec(25, el_count, el_size); + s->vk[2] = a64op_make_vec(26, el_count, el_size); + s->vk[3] = a64op_make_vec(27, el_count, el_size); } /*********************************************************************/ @@ -378,11 +379,11 @@ static void asmgen_set_load_cont_node(SwsAArch64Context *s) static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - AArch64VecViews bitmask_vec = a64op_vec_views(s->vt[1]); + AArch64VecViews bitmask_vec = a64op_vec_views(s->vk[0]); RasmOp wtmp = a64op_w(s->tmp0); AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; - AArch64VecViews vtmp = a64op_vec_views(s->vt[2]); - AArch64VecViews shift_vec = a64op_vec_views(s->vt[0]); + AArch64VecViews vtmp = a64op_vec_views(s->vt[0]); + AArch64VecViews shift_vec = a64op_vec_views(s->vk[1]); /* Note that shift_vec has negative values, so that using it with * ushl actually performs a right shift. */ @@ -411,9 +412,9 @@ static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParam static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - AArch64VecViews nibble_mask = a64op_vec_views(s->vt[0]); + AArch64VecViews nibble_mask = a64op_vec_views(s->vk[0]); AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; - AArch64VecViews vtmp = a64op_vec_views(s->vt[1]); + AArch64VecViews vtmp = a64op_vec_views(s->vt[0]); rasm_annotate_next(r, "v128 nibble_mask = {0xf <repeats 8 times>, 0x0 <repeats 8 times>};"); i_movi(r, nibble_mask.b8, IMM(0x0f)); @@ -477,9 +478,9 @@ static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplPara { RasmContext *r = s->rctx; AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; - AArch64VecViews shift_vec = a64op_vec_views(s->vt[0]); - AArch64VecViews vtmp0 = a64op_vec_views(s->vt[1]); - AArch64VecViews vtmp1 = a64op_vec_views(s->vt[2]); + AArch64VecViews shift_vec = a64op_vec_views(s->vk[0]); + AArch64VecViews vtmp0 = a64op_vec_views(s->vt[0]); + AArch64VecViews vtmp1 = a64op_vec_views(s->vt[1]); rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); i_ldr(r, shift_vec.q, s->impl_priv); @@ -627,7 +628,7 @@ static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams RasmContext *r = s->rctx; RasmOp *vl = s->vl; RasmOp *vh = s->vh; - RasmOp *vmask = s->vt; + RasmOp *vmask = s->vk; RasmOp mask_gpr = a64op_w(s->tmp0); uint32_t mask_val[4] = { 0 }; @@ -762,7 +763,7 @@ static void emit_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vx, int i, const char *vx_str) { RasmContext *r = s->rctx; - RasmOp clear_vec = s->vt[0]; + RasmOp clear_vec = s->vk[0]; if (p->par.clear.zero & SWS_COMP(i)) { i_movi(r, vx[i], IMM(0)); CMTF("%s[%u] = 0;", vx_str, i); } else if (p->par.clear.one & SWS_COMP(i)) { @@ -780,7 +781,7 @@ static void emit_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) { RasmContext *r = s->rctx; - RasmOp clear_vec = s->vt[0]; + RasmOp clear_vec = s->vk[0]; /** * TODO @@ -938,19 +939,19 @@ static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) RasmContext *r = s->rctx; RasmOp *vl = s->vl; RasmOp *vh = s->vh; - RasmOp *vt = s->vt; - RasmOp min_vec = s->vt[4]; + RasmOp *vk = s->vk; + RasmOp min_vec = s->vt[0]; i_ldr(r, v_q(min_vec), s->impl_priv); CMT("v128 min_vec = impl->priv.v128;"); asmgen_set_load_cont_node(s); - LOOP_MASK(p, i) { i_dup(r, vt[i], a64op_elem(min_vec, i)); CMTF("v128 vmin%u = min_vec[%u];", i, i); } + LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(min_vec, i)); CMTF("v128 vmin%u = min_vec[%u];", i, i); } if (p->type == SWS_PIXEL_F32) { - LOOP_MASK (p, i) { i_fmin(r, vl[i], vl[i], vt[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); } - LOOP_MASK_VH(s, p, i) { i_fmin(r, vh[i], vh[i], vt[i]); CMTF("vh[%u] = min(vh[%u], vmin%u);", i, i, i); } + LOOP_MASK (p, i) { i_fmin(r, vl[i], vl[i], vk[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); } + LOOP_MASK_VH(s, p, i) { i_fmin(r, vh[i], vh[i], vk[i]); CMTF("vh[%u] = min(vh[%u], vmin%u);", i, i, i); } } else { - LOOP_MASK (p, i) { i_umin(r, vl[i], vl[i], vt[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); } - LOOP_MASK_VH(s, p, i) { i_umin(r, vh[i], vh[i], vt[i]); CMTF("vh[%u] = min(vh[%u], vmin%u);", i, i, i); } + LOOP_MASK (p, i) { i_umin(r, vl[i], vl[i], vk[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); } + LOOP_MASK_VH(s, p, i) { i_umin(r, vh[i], vh[i], vk[i]); CMTF("vh[%u] = min(vh[%u], vmin%u);", i, i, i); } } } @@ -963,19 +964,19 @@ static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) RasmContext *r = s->rctx; RasmOp *vl = s->vl; RasmOp *vh = s->vh; - RasmOp *vt = s->vt; - RasmOp max_vec = s->vt[4]; + RasmOp *vk = s->vk; + RasmOp max_vec = s->vt[0]; i_ldr(r, v_q(max_vec), s->impl_priv); CMT("v128 max_vec = impl->priv.v128;"); asmgen_set_load_cont_node(s); - LOOP_MASK(p, i) { i_dup(r, vt[i], a64op_elem(max_vec, i)); CMTF("v128 vmax%u = max_vec[%u];", i, i); } + LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(max_vec, i)); CMTF("v128 vmax%u = max_vec[%u];", i, i); } if (p->type == SWS_PIXEL_F32) { - LOOP_MASK (p, i) { i_fmax(r, vl[i], vl[i], vt[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); } - LOOP_MASK_VH(s, p, i) { i_fmax(r, vh[i], vh[i], vt[i]); CMTF("vh[%u] = max(vh[%u], vmax%u);", i, i, i); } + LOOP_MASK (p, i) { i_fmax(r, vl[i], vl[i], vk[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); } + LOOP_MASK_VH(s, p, i) { i_fmax(r, vh[i], vh[i], vk[i]); CMTF("vh[%u] = max(vh[%u], vmax%u);", i, i, i); } } else { - LOOP_MASK (p, i) { i_umax(r, vl[i], vl[i], vt[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); } - LOOP_MASK_VH(s, p, i) { i_umax(r, vh[i], vh[i], vt[i]); CMTF("vh[%u] = max(vh[%u], vmax%u);", i, i, i); } + LOOP_MASK (p, i) { i_umax(r, vl[i], vl[i], vk[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); } + LOOP_MASK_VH(s, p, i) { i_umax(r, vh[i], vh[i], vk[i]); CMTF("vh[%u] = max(vh[%u], vmax%u);", i, i, i); } } } @@ -989,7 +990,7 @@ static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams * RasmOp *vl = s->vl; RasmOp *vh = s->vh; RasmOp priv_ptr = s->tmp0; - RasmOp scale_vec = s->vt[0]; + RasmOp scale_vec = s->vk[0]; i_add (r, priv_ptr, s->impl, IMM(offsetof_impl_priv)); CMT("v128 *scale_vec_ptr = &impl->priv;"); asmgen_set_load_cont_node(s); @@ -1103,7 +1104,7 @@ static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams { RasmContext *r = s->rctx; RasmOp *vt = s->vt; - RasmOp *vc = &vt[8]; /* The coefficients are loaded starting from temp vector 8 */ + RasmOp *vc = s->vk; RasmOp ptr = s->tmp0; RasmOp coeff_veclist; -- 2.52.0 >From c8d0fb86f1c1a5e309a8b63e8722e2c37260be14 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Fri, 17 Jul 2026 00:12:22 +0200 Subject: [PATCH 08/23] swscale/aarch64/ops_asmgen: split reshape_all_vectors() into vector register classes This reduces some unnecessary vector reshaping, and also makes the intention clearer at the call-sites. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 6a6a530a8c..aec800df2c 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -174,8 +174,8 @@ typedef struct SwsAArch64Context { #define CMT(comment) rasm_annotate(r, comment) #define CMTF(fmt, ...) rasm_annotatef(r, (char[128]){0}, 128, fmt, __VA_ARGS__) -/* Reshape all vector registers for current SwsOp. */ -static void reshape_all_vectors(SwsAArch64Context *s, int el_count, int el_size) +/* Reshape input/output vector registers for current SwsOp. */ +static void reshape_io_vectors(SwsAArch64Context *s, int el_count, int el_size) { s->vl[0] = a64op_make_vec( 0, el_count, el_size); s->vl[1] = a64op_make_vec( 1, el_count, el_size); @@ -185,6 +185,11 @@ static void reshape_all_vectors(SwsAArch64Context *s, int el_count, int el_size) s->vh[1] = a64op_make_vec( 5, el_count, el_size); s->vh[2] = a64op_make_vec( 6, el_count, el_size); s->vh[3] = a64op_make_vec( 7, el_count, el_size); +} + +/* Reshape temp vector registers for current SwsOp. */ +static void reshape_tmp_vectors(SwsAArch64Context *s, int el_count, int el_size) +{ s->vt[0] = a64op_make_vec(16, el_count, el_size); s->vt[1] = a64op_make_vec(17, el_count, el_size); s->vt[2] = a64op_make_vec(18, el_count, el_size); @@ -193,6 +198,11 @@ static void reshape_all_vectors(SwsAArch64Context *s, int el_count, int el_size) s->vt[5] = a64op_make_vec(21, el_count, el_size); s->vt[6] = a64op_make_vec(22, el_count, el_size); s->vt[7] = a64op_make_vec(23, el_count, el_size); +} + +/* Reshape const vector registers for current SwsOp. */ +static void reshape_const_vectors(SwsAArch64Context *s, int el_count, int el_size) +{ s->vk[0] = a64op_make_vec(24, el_count, el_size); s->vk[1] = a64op_make_vec(25, el_count, el_size); s->vk[2] = a64op_make_vec(26, el_count, el_size); @@ -918,13 +928,13 @@ static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams if (src_el_size == 1) { rasm_add_comment(r, "u8 -> u16"); - reshape_all_vectors(s, 16, 1); + reshape_io_vectors(s, 16, 1); LOOP_MASK_VH(s, p, i) i_zip2(r, vh[i], vl[i], vl[i]); LOOP_MASK (p, i) i_zip1(r, vl[i], vl[i], vl[i]); } if (dst_el_size == 4) { rasm_add_comment(r, "u16 -> u32"); - reshape_all_vectors(s, 8, 2); + reshape_io_vectors(s, 8, 2); LOOP_MASK_VH(s, p, i) i_zip2(r, vh[i], vl[i], vl[i]); LOOP_MASK (p, i) i_zip1(r, vl[i], vl[i], vl[i]); } @@ -1305,7 +1315,9 @@ static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) s->el_size = el_size; s->el_count = s->vec_size / el_size; - reshape_all_vectors(s, s->el_count, el_size); + reshape_io_vectors(s, s->el_count, el_size); + reshape_tmp_vectors(s, s->el_count, el_size); + reshape_const_vectors(s, s->el_count, el_size); /* Common start for continuation-passing style (CPS) functions. */ s->impl_priv = a64op_off(s->impl, offsetof_impl_priv); -- 2.52.0 >From 8897e61646da0359b63d667349969c2920344c01 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Mon, 6 Jul 2026 23:05:49 +0200 Subject: [PATCH 09/23] swscale/aarch64/ops_asmgen: pass SwsAArch64OpRegs as a separate argument to asmgen_op_*() This will help the JIT compiler by letting us provide a separate set of registers for each operation. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 340 +++++++++++++++++--------------- 1 file changed, 186 insertions(+), 154 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index aec800df2c..c96bc4d727 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -116,6 +116,14 @@ static const SwsAArch64OpEntry ops_entries[] = { { NULL } }; +/*********************************************************************/ +typedef struct SwsAArch64OpRegs { + RasmOp vl[4]; /* input/output vector registers (low bank) */ + RasmOp vh[4]; /* input/output vector registers (high bank) */ + RasmOp vt[8]; /* temp vector registers */ + RasmOp vk[4]; /* constant data (may be gprs) */ +} SwsAArch64OpRegs; + /*********************************************************************/ typedef struct SwsAArch64Context { RasmContext *rctx; @@ -142,12 +150,7 @@ typedef struct SwsAArch64Context { RasmOp cont; RasmOp impl_priv; RasmNode *load_cont_node; - - /* Vector registers. Two banks (low and high) are used. */ - RasmOp vl[4]; - RasmOp vh[4]; - RasmOp vt[8]; - RasmOp vk[4]; + SwsAArch64OpRegs regs; /* Read/Write data pointers and padding. */ RasmOp in[4]; @@ -175,38 +178,38 @@ typedef struct SwsAArch64Context { #define CMTF(fmt, ...) rasm_annotatef(r, (char[128]){0}, 128, fmt, __VA_ARGS__) /* Reshape input/output vector registers for current SwsOp. */ -static void reshape_io_vectors(SwsAArch64Context *s, int el_count, int el_size) +static void reshape_io_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size) { - s->vl[0] = a64op_make_vec( 0, el_count, el_size); - s->vl[1] = a64op_make_vec( 1, el_count, el_size); - s->vl[2] = a64op_make_vec( 2, el_count, el_size); - s->vl[3] = a64op_make_vec( 3, el_count, el_size); - s->vh[0] = a64op_make_vec( 4, el_count, el_size); - s->vh[1] = a64op_make_vec( 5, el_count, el_size); - s->vh[2] = a64op_make_vec( 6, el_count, el_size); - s->vh[3] = a64op_make_vec( 7, el_count, el_size); + regs->vl[0] = a64op_make_vec( 0, el_count, el_size); + regs->vl[1] = a64op_make_vec( 1, el_count, el_size); + regs->vl[2] = a64op_make_vec( 2, el_count, el_size); + regs->vl[3] = a64op_make_vec( 3, el_count, el_size); + regs->vh[0] = a64op_make_vec( 4, el_count, el_size); + regs->vh[1] = a64op_make_vec( 5, el_count, el_size); + regs->vh[2] = a64op_make_vec( 6, el_count, el_size); + regs->vh[3] = a64op_make_vec( 7, el_count, el_size); } /* Reshape temp vector registers for current SwsOp. */ -static void reshape_tmp_vectors(SwsAArch64Context *s, int el_count, int el_size) +static void reshape_temp_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size) { - s->vt[0] = a64op_make_vec(16, el_count, el_size); - s->vt[1] = a64op_make_vec(17, el_count, el_size); - s->vt[2] = a64op_make_vec(18, el_count, el_size); - s->vt[3] = a64op_make_vec(19, el_count, el_size); - s->vt[4] = a64op_make_vec(20, el_count, el_size); - s->vt[5] = a64op_make_vec(21, el_count, el_size); - s->vt[6] = a64op_make_vec(22, el_count, el_size); - s->vt[7] = a64op_make_vec(23, el_count, el_size); + regs->vt[0] = a64op_make_vec(16, el_count, el_size); + regs->vt[1] = a64op_make_vec(17, el_count, el_size); + regs->vt[2] = a64op_make_vec(18, el_count, el_size); + regs->vt[3] = a64op_make_vec(19, el_count, el_size); + regs->vt[4] = a64op_make_vec(20, el_count, el_size); + regs->vt[5] = a64op_make_vec(21, el_count, el_size); + regs->vt[6] = a64op_make_vec(22, el_count, el_size); + regs->vt[7] = a64op_make_vec(23, el_count, el_size); } /* Reshape const vector registers for current SwsOp. */ -static void reshape_const_vectors(SwsAArch64Context *s, int el_count, int el_size) +static void reshape_const_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size) { - s->vk[0] = a64op_make_vec(24, el_count, el_size); - s->vk[1] = a64op_make_vec(25, el_count, el_size); - s->vk[2] = a64op_make_vec(26, el_count, el_size); - s->vk[3] = a64op_make_vec(27, el_count, el_size); + regs->vk[0] = a64op_make_vec(24, el_count, el_size); + regs->vk[1] = a64op_make_vec(25, el_count, el_size); + regs->vk[2] = a64op_make_vec(26, el_count, el_size); + regs->vk[3] = a64op_make_vec(27, el_count, el_size); } /*********************************************************************/ @@ -386,14 +389,15 @@ static void asmgen_set_load_cont_node(SwsAArch64Context *s) /* SWS_UOP_READ_PACKED */ /* SWS_UOP_READ_PLANAR */ -static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews bitmask_vec = a64op_vec_views(s->vk[0]); + AArch64VecViews bitmask_vec = a64op_vec_views(regs->vk[0]); RasmOp wtmp = a64op_w(s->tmp0); - AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; - AArch64VecViews vtmp = a64op_vec_views(s->vt[0]); - AArch64VecViews shift_vec = a64op_vec_views(s->vk[1]); + AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; + AArch64VecViews vtmp = a64op_vec_views(regs->vt[0]); + AArch64VecViews shift_vec = a64op_vec_views(regs->vk[1]); /* Note that shift_vec has negative values, so that using it with * ushl actually performs a right shift. */ @@ -419,12 +423,13 @@ static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParam } } -static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews nibble_mask = a64op_vec_views(s->vk[0]); - AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; - AArch64VecViews vtmp = a64op_vec_views(s->vt[0]); + AArch64VecViews nibble_mask = a64op_vec_views(regs->vk[0]); + AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; + AArch64VecViews vtmp = a64op_vec_views(regs->vt[0]); rasm_annotate_next(r, "v128 nibble_mask = {0xf <repeats 8 times>, 0x0 <repeats 8 times>};"); i_movi(r, nibble_mask.b8, IMM(0x0f)); @@ -453,19 +458,21 @@ static void asmgen_op_read_packed_n(SwsAArch64Context *s, const SwsAArch64OpImpl } } -static void asmgen_op_read_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_read_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { av_assert0(p->mask != 0x0001); - asmgen_op_read_packed_n(s, p, s->vl); + asmgen_op_read_packed_n(s, p, regs->vl); if (s->use_vh) - asmgen_op_read_packed_n(s, p, s->vh); + asmgen_op_read_packed_n(s, p, regs->vh); } -static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(s->vl); - AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(s->vh); + AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(regs->vl); + AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(regs->vh); LOOP_MASK(p, i) { switch ((s->use_vh ? 0x100 : 0) | s->vec_size) { @@ -484,13 +491,14 @@ static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplPa /* SWS_UOP_WRITE_PACKED */ /* SWS_UOP_WRITE_PLANAR */ -static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[1] = { a64op_vec_views(s->vl[0]) }; - AArch64VecViews shift_vec = a64op_vec_views(s->vk[0]); - AArch64VecViews vtmp0 = a64op_vec_views(s->vt[0]); - AArch64VecViews vtmp1 = a64op_vec_views(s->vt[1]); + AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; + AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); + AArch64VecViews vtmp0 = a64op_vec_views(regs->vt[0]); + AArch64VecViews vtmp1 = a64op_vec_views(regs->vt[1]); rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); i_ldr(r, shift_vec.q, s->impl_priv); @@ -510,12 +518,13 @@ static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplPara } } -static void asmgen_op_write_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_write_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(s->vl); - AArch64VecViews vtmp0 = a64op_vec_views(s->vt[0]); - AArch64VecViews vtmp1 = a64op_vec_views(s->vt[1]); + AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(regs->vl); + AArch64VecViews vtmp0 = a64op_vec_views(regs->vt[0]); + AArch64VecViews vtmp1 = a64op_vec_views(regs->vt[1]); if (p->block_size == 8) { i_shl (r, vtmp0.h4, vl[0].h4, IMM(4)); @@ -543,19 +552,21 @@ static void asmgen_op_write_packed_n(SwsAArch64Context *s, const SwsAArch64OpImp } } -static void asmgen_op_write_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_write_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { av_assert0(p->mask != 0x0001); - asmgen_op_write_packed_n(s, p, s->vl); + asmgen_op_write_packed_n(s, p, regs->vl); if (s->use_vh) - asmgen_op_write_packed_n(s, p, s->vh); + asmgen_op_write_packed_n(s, p, regs->vh); } -static void asmgen_op_write_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_write_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(s->vl); - AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(s->vh); + AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(regs->vl); + AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(regs->vh); LOOP_MASK(p, i) { switch ((s->use_vh ? 0x100 : 0) | s->vec_size) { @@ -571,11 +582,12 @@ static void asmgen_op_write_planar(SwsAArch64Context *s, const SwsAArch64OpImplP /* swap byte order (for differing endianness) */ /* SWS_UOP_SWAP_BYTES */ -static void asmgen_op_swap_bytes(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_swap_bytes(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(s->vl); - AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(s->vh); + AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(regs->vl); + AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(regs->vh); switch (ff_sws_pixel_type_size(p->type)) { case sizeof(uint16_t): @@ -604,18 +616,19 @@ static const char *print_swizzle_v(char buf[8], int8_t n, uint8_t vh) } #define PRINT_SWIZZLE_V(n, vh) print_swizzle_v((char[8]){ 0 }, n, vh) -static RasmOp swizzle_a64op(SwsAArch64Context *s, int8_t n, uint8_t vh) +static RasmOp swizzle_a64op(SwsAArch64OpRegs *regs, int8_t n, uint8_t vh) { if (n == -1) - return s->vt[vh]; - return vh ? s->vh[n] : s->vl[n]; + return regs->vt[vh]; + return vh ? regs->vh[n] : regs->vl[n]; } -static void swizzle_emit(SwsAArch64Context *s, int8_t dst, int8_t src) +static void swizzle_emit(SwsAArch64Context *s, SwsAArch64OpRegs *regs, + int8_t dst, int8_t src) { RasmContext *r = s->rctx; - RasmOp src_op[2] = { swizzle_a64op(s, src, 0), swizzle_a64op(s, src, 1) }; - RasmOp dst_op[2] = { swizzle_a64op(s, dst, 0), swizzle_a64op(s, dst, 1) }; + RasmOp src_op[2] = { swizzle_a64op(regs, src, 0), swizzle_a64op(regs, src, 1) }; + RasmOp dst_op[2] = { swizzle_a64op(regs, dst, 0), swizzle_a64op(regs, dst, 1) }; i_mov (r, dst_op[0], src_op[0]); CMTF("%s = %s;", PRINT_SWIZZLE_V(dst, 0), PRINT_SWIZZLE_V(src, 0)); if (s->use_vh) { @@ -623,22 +636,25 @@ static void swizzle_emit(SwsAArch64Context *s, int8_t dst, int8_t src) } } -static void asmgen_op_move(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_move(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { for (int i = 0; i < p->par.move.num_moves; i++) - swizzle_emit(s, p->par.move.dst[i], p->par.move.src[i]); + swizzle_emit(s, regs, p->par.move.dst[i], p->par.move.src[i]); } /*********************************************************************/ /* split tightly packed data into components */ /* SWS_UOP_UNPACK */ -static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = s->vl; - RasmOp *vh = s->vh; - RasmOp *vmask = s->vk; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; + RasmOp *vmask = regs->vk; + RasmOp mask_gpr = a64op_w(s->tmp0); uint32_t mask_val[4] = { 0 }; @@ -702,11 +718,12 @@ static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams /* compress components into tightly packed data */ /* SWS_UOP_PACK */ -static void asmgen_op_pack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_pack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = s->vl; - RasmOp *vh = s->vh; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; const int offsets[4] = { p->par.pack.pattern[3] + p->par.pack.pattern[2] + p->par.pack.pattern[1], @@ -739,12 +756,13 @@ static void asmgen_op_pack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p /* logical left shift of raw pixel values */ /* SWS_UOP_LSHIFT */ -static void asmgen_op_lshift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_lshift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { uint8_t shift = p->par.shift.amount; RasmContext *r = s->rctx; - RasmOp *vl = s->vl; - RasmOp *vh = s->vh; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; LOOP_MASK (p, i) { i_shl(r, vl[i], vl[i], IMM(shift)); CMTF("vl[%u] <<= %u;", i, shift); } LOOP_MASK_VH(s, p, i) { i_shl(r, vh[i], vh[i], IMM(shift)); CMTF("vh[%u] <<= %u;", i, shift); } @@ -754,12 +772,13 @@ static void asmgen_op_lshift(SwsAArch64Context *s, const SwsAArch64OpImplParams /* right shift of raw pixel values */ /* SWS_UOP_RSHIFT */ -static void asmgen_op_rshift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_rshift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { uint8_t shift = p->par.shift.amount; RasmContext *r = s->rctx; - RasmOp *vl = s->vl; - RasmOp *vh = s->vh; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; LOOP_MASK (p, i) { i_ushr(r, vl[i], vl[i], IMM(shift)); CMTF("vl[%u] >>= %u;", i, shift); } LOOP_MASK_VH(s, p, i) { i_ushr(r, vh[i], vh[i], IMM(shift)); CMTF("vh[%u] >>= %u;", i, shift); } @@ -770,10 +789,10 @@ static void asmgen_op_rshift(SwsAArch64Context *s, const SwsAArch64OpImplParams /* SWS_UOP_CLEAR */ static void emit_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - RasmOp *vx, int i, const char *vx_str) + RasmOp *vx, RasmOp *vk, int i, const char *vx_str) { RasmContext *r = s->rctx; - RasmOp clear_vec = s->vk[0]; + RasmOp clear_vec = vk[0]; if (p->par.clear.zero & SWS_COMP(i)) { i_movi(r, vx[i], IMM(0)); CMTF("%s[%u] = 0;", vx_str, i); } else if (p->par.clear.one & SWS_COMP(i)) { @@ -788,10 +807,14 @@ static void emit_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, } } -static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp clear_vec = s->vk[0]; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; + RasmOp *vk = regs->vk; + RasmOp clear_vec = vk[0]; /** * TODO @@ -809,8 +832,8 @@ static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams * asmgen_set_load_cont_node(s); } - LOOP_MASK (p, i) { emit_clear(s, p, s->vl, i, "vl"); } - LOOP_MASK_VH(s, p, i) { emit_clear(s, p, s->vh, i, "vh"); } + LOOP_MASK (p, i) { emit_clear(s, p, vl, vk, i, "vl"); } + LOOP_MASK_VH(s, p, i) { emit_clear(s, p, vh, vk, i, "vh"); } } /*********************************************************************/ @@ -820,11 +843,12 @@ static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams * /* SWS_UOP_TO_U32 */ /* SWS_UOP_TO_F32 */ -static void asmgen_op_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(s->vl); - AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(s->vh); + AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(regs->vl); + AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(regs->vh); /** * Since each instruction in the convert operation needs specific @@ -904,11 +928,12 @@ static void asmgen_op_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams /* SWS_UOP_EXPAND_PAIR */ /* SWS_UOP_EXPAND_QUAD */ -static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = s->vl; - RasmOp *vh = s->vh; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; size_t src_el_size = s->el_size; SwsPixelType to_type; @@ -928,13 +953,13 @@ static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams if (src_el_size == 1) { rasm_add_comment(r, "u8 -> u16"); - reshape_io_vectors(s, 16, 1); + reshape_io_vectors(regs, 16, 1); LOOP_MASK_VH(s, p, i) i_zip2(r, vh[i], vl[i], vl[i]); LOOP_MASK (p, i) i_zip1(r, vl[i], vl[i], vl[i]); } if (dst_el_size == 4) { rasm_add_comment(r, "u16 -> u32"); - reshape_io_vectors(s, 8, 2); + reshape_io_vectors(regs, 8, 2); LOOP_MASK_VH(s, p, i) i_zip2(r, vh[i], vl[i], vl[i]); LOOP_MASK (p, i) i_zip1(r, vl[i], vl[i], vl[i]); } @@ -944,13 +969,14 @@ static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams /* numeric minimum */ /* SWS_UOP_MIN */ -static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = s->vl; - RasmOp *vh = s->vh; - RasmOp *vk = s->vk; - RasmOp min_vec = s->vt[0]; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; + RasmOp *vk = regs->vk; + RasmOp min_vec = regs->vt[0]; i_ldr(r, v_q(min_vec), s->impl_priv); CMT("v128 min_vec = impl->priv.v128;"); asmgen_set_load_cont_node(s); @@ -969,13 +995,14 @@ static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) /* numeric maximum */ /* SWS_UOP_MAX */ -static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = s->vl; - RasmOp *vh = s->vh; - RasmOp *vk = s->vk; - RasmOp max_vec = s->vt[0]; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; + RasmOp *vk = regs->vk; + RasmOp max_vec = regs->vt[0]; i_ldr(r, v_q(max_vec), s->impl_priv); CMT("v128 max_vec = impl->priv.v128;"); asmgen_set_load_cont_node(s); @@ -994,13 +1021,14 @@ static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) /* multiplication by scalar */ /* SWS_UOP_SCALE */ -static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = s->vl; - RasmOp *vh = s->vh; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; RasmOp priv_ptr = s->tmp0; - RasmOp scale_vec = s->vk[0]; + RasmOp scale_vec = regs->vk[0]; i_add (r, priv_ptr, s->impl, IMM(offsetof_impl_priv)); CMT("v128 *scale_vec_ptr = &impl->priv;"); asmgen_set_load_cont_node(s); @@ -1025,7 +1053,7 @@ static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams * * (low or high). */ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - RasmOp *vt, RasmOp *vc, + SwsAArch64OpRegs *regs, SwsCompMask save_mask, bool vh_pass) { RasmContext *r = s->rctx; @@ -1033,8 +1061,10 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, * The intermediate registers for fmul+fadd (for when SWS_BITEXACT * is set) start from temp vector 4. */ + RasmOp *vt = regs->vt; + RasmOp *vc = regs->vk; RasmOp *vtmp = &vt[4]; - RasmOp *vx = vh_pass ? s->vh : s->vl; + RasmOp *vx = vh_pass ? regs->vh : regs->vl; char cvh = vh_pass ? 'h' : 'l'; if (vh_pass && !s->use_vh) @@ -1110,11 +1140,12 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, } } -static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vt = s->vt; - RasmOp *vc = s->vk; + RasmOp *vc = regs->vk; + RasmOp ptr = s->tmp0; RasmOp coeff_veclist; @@ -1147,24 +1178,25 @@ static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams } /* Perform linear passes for low and high vector banks. */ - linear_pass(s, p, vt, vc, save_mask, false); - linear_pass(s, p, vt, vc, save_mask, true); + linear_pass(s, p, regs, save_mask, false); + linear_pass(s, p, regs, save_mask, true); } /*********************************************************************/ /* add dithering noise */ /* SWS_UOP_DITHER */ -static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = s->vl; - RasmOp *vh = s->vh; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; RasmOp ptr = s->tmp0; RasmOp tmp1 = s->tmp1; RasmOp wtmp1 = a64op_w(tmp1); - RasmOp dither_vl = s->vt[0]; - RasmOp dither_vh = s->vt[1]; + RasmOp dither_vl = regs->vt[0]; + RasmOp dither_vh = regs->vt[1]; RasmOp bx64 = a64op_x(s->bx); RasmOp y64 = a64op_x(s->y); @@ -1315,43 +1347,43 @@ static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) s->el_size = el_size; s->el_count = s->vec_size / el_size; - reshape_io_vectors(s, s->el_count, el_size); - reshape_tmp_vectors(s, s->el_count, el_size); - reshape_const_vectors(s, s->el_count, el_size); + reshape_io_vectors(&s->regs, s->el_count, el_size); + reshape_temp_vectors(&s->regs, s->el_count, el_size); + reshape_const_vectors(&s->regs, s->el_count, el_size); /* Common start for continuation-passing style (CPS) functions. */ s->impl_priv = a64op_off(s->impl, offsetof_impl_priv); asmgen_set_load_cont_node(s); switch (p->uop) { - case SWS_UOP_READ_BIT: asmgen_op_read_bit(s, p); break; - case SWS_UOP_READ_NIBBLE: asmgen_op_read_nibble(s, p); break; - case SWS_UOP_READ_PACKED: asmgen_op_read_packed(s, p); break; - case SWS_UOP_READ_PLANAR: asmgen_op_read_planar(s, p); break; - case SWS_UOP_WRITE_BIT: asmgen_op_write_bit(s, p); break; - case SWS_UOP_WRITE_NIBBLE: asmgen_op_write_nibble(s, p); break; - case SWS_UOP_WRITE_PACKED: asmgen_op_write_packed(s, p); break; - case SWS_UOP_WRITE_PLANAR: asmgen_op_write_planar(s, p); break; - case SWS_UOP_SWAP_BYTES: asmgen_op_swap_bytes(s, p); break; - case SWS_UOP_PERMUTE: asmgen_op_move(s, p); break; - case SWS_UOP_COPY: asmgen_op_move(s, p); break; - case SWS_UOP_UNPACK: asmgen_op_unpack(s, p); break; - case SWS_UOP_PACK: asmgen_op_pack(s, p); break; - case SWS_UOP_LSHIFT: asmgen_op_lshift(s, p); break; - case SWS_UOP_RSHIFT: asmgen_op_rshift(s, p); break; - case SWS_UOP_CLEAR: asmgen_op_clear(s, p); break; - case SWS_UOP_TO_U8: asmgen_op_convert(s, p); break; - case SWS_UOP_TO_U16: asmgen_op_convert(s, p); break; - case SWS_UOP_TO_U32: asmgen_op_convert(s, p); break; - case SWS_UOP_TO_F32: asmgen_op_convert(s, p); break; - case SWS_UOP_EXPAND_PAIR: asmgen_op_expand(s, p); break; - case SWS_UOP_EXPAND_QUAD: asmgen_op_expand(s, p); break; - case SWS_UOP_MIN: asmgen_op_min(s, p); break; - case SWS_UOP_MAX: asmgen_op_max(s, p); break; - case SWS_UOP_SCALE: asmgen_op_scale(s, p); break; - case SWS_UOP_LINEAR: asmgen_op_linear(s, p); break; - case SWS_UOP_LINEAR_FMA: asmgen_op_linear(s, p); break; - case SWS_UOP_DITHER: asmgen_op_dither(s, p); break; + case SWS_UOP_READ_BIT: asmgen_op_read_bit(s, p, &s->regs); break; + case SWS_UOP_READ_NIBBLE: asmgen_op_read_nibble(s, p, &s->regs); break; + case SWS_UOP_READ_PACKED: asmgen_op_read_packed(s, p, &s->regs); break; + case SWS_UOP_READ_PLANAR: asmgen_op_read_planar(s, p, &s->regs); break; + case SWS_UOP_WRITE_BIT: asmgen_op_write_bit(s, p, &s->regs); break; + case SWS_UOP_WRITE_NIBBLE: asmgen_op_write_nibble(s, p, &s->regs); break; + case SWS_UOP_WRITE_PACKED: asmgen_op_write_packed(s, p, &s->regs); break; + case SWS_UOP_WRITE_PLANAR: asmgen_op_write_planar(s, p, &s->regs); break; + case SWS_UOP_SWAP_BYTES: asmgen_op_swap_bytes(s, p, &s->regs); break; + case SWS_UOP_PERMUTE: asmgen_op_move(s, p, &s->regs); break; + case SWS_UOP_COPY: asmgen_op_move(s, p, &s->regs); break; + case SWS_UOP_UNPACK: asmgen_op_unpack(s, p, &s->regs); break; + case SWS_UOP_PACK: asmgen_op_pack(s, p, &s->regs); break; + case SWS_UOP_LSHIFT: asmgen_op_lshift(s, p, &s->regs); break; + case SWS_UOP_RSHIFT: asmgen_op_rshift(s, p, &s->regs); break; + case SWS_UOP_CLEAR: asmgen_op_clear(s, p, &s->regs); break; + case SWS_UOP_TO_U8: asmgen_op_convert(s, p, &s->regs); break; + case SWS_UOP_TO_U16: asmgen_op_convert(s, p, &s->regs); break; + case SWS_UOP_TO_U32: asmgen_op_convert(s, p, &s->regs); break; + case SWS_UOP_TO_F32: asmgen_op_convert(s, p, &s->regs); break; + case SWS_UOP_EXPAND_PAIR: asmgen_op_expand(s, p, &s->regs); break; + case SWS_UOP_EXPAND_QUAD: asmgen_op_expand(s, p, &s->regs); break; + case SWS_UOP_MIN: asmgen_op_min(s, p, &s->regs); break; + case SWS_UOP_MAX: asmgen_op_max(s, p, &s->regs); break; + case SWS_UOP_SCALE: asmgen_op_scale(s, p, &s->regs); break; + case SWS_UOP_LINEAR: asmgen_op_linear(s, p, &s->regs); break; + case SWS_UOP_LINEAR_FMA: asmgen_op_linear(s, p, &s->regs); break; + case SWS_UOP_DITHER: asmgen_op_dither(s, p, &s->regs); break; /* TODO implement SWS_UOP_SHUFFLE */ default: break; -- 2.52.0 >From 7702d2768aa9ffba217c3b06434f719c3a1e69b4 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sat, 11 Jul 2026 14:19:55 +0200 Subject: [PATCH 10/23] swscale/aarch64/ops_asmgen: split asmgen_setup_*() out of asmgen_op_*() This paves the way for performing operation setup differently for the JIT backend, which will have constant values factored out of the loop. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 280 +++++++++++++++++++++----------- 1 file changed, 189 insertions(+), 91 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index c96bc4d727..6db8f368dd 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -122,6 +122,11 @@ typedef struct SwsAArch64OpRegs { RasmOp vh[4]; /* input/output vector registers (high bank) */ RasmOp vt[8]; /* temp vector registers */ RasmOp vk[4]; /* constant data (may be gprs) */ + + /* Op-specific registers. */ + union { + RasmOp dither_ptr; + }; } SwsAArch64OpRegs; /*********************************************************************/ @@ -389,25 +394,38 @@ static void asmgen_set_load_cont_node(SwsAArch64Context *s) /* SWS_UOP_READ_PACKED */ /* SWS_UOP_READ_PLANAR */ +static void asmgen_setup_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); + AArch64VecViews bitmask_vec = a64op_vec_views(regs->vk[1]); + + rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); + i_ldr(r, shift_vec.q, s->impl_priv); + asmgen_set_load_cont_node(s); + if (p->block_size == 16) { + i_movi(r, bitmask_vec.b16, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 16 times>};"); + } else { + i_movi(r, bitmask_vec.b8, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 8 times>, 0 <repeats 8 times>};"); + } +} + static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews bitmask_vec = a64op_vec_views(regs->vk[0]); - RasmOp wtmp = a64op_w(s->tmp0); - AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; - AArch64VecViews vtmp = a64op_vec_views(regs->vt[0]); - AArch64VecViews shift_vec = a64op_vec_views(regs->vk[1]); + AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; + AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); + AArch64VecViews bitmask_vec = a64op_vec_views(regs->vk[1]); + + AArch64VecViews vtmp = a64op_vec_views(regs->vt[0]); + RasmOp wtmp = a64op_w(s->tmp0); /* Note that shift_vec has negative values, so that using it with * ushl actually performs a right shift. */ - rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); - i_ldr(r, shift_vec.q, s->impl_priv); - asmgen_set_load_cont_node(s); - if (p->block_size == 16) { i_ldrh(r, wtmp, a64op_post(s->in[0], 2)); CMT("uint16_t tmp = *in[0]++;"); - i_movi(r, bitmask_vec.b16, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 16 times>};"); i_dup (r, vl[0].b8, wtmp); CMT("vl[0].lo = broadcast(tmp);"); i_lsr (r, wtmp, wtmp, IMM(8)); CMT("tmp >>= 8;"); i_dup (r, vtmp.b8, wtmp); CMT("vtmp.lo = broadcast(tmp);"); @@ -416,23 +434,30 @@ static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParam i_and (r, vl[0].b16, vl[0].b16, bitmask_vec.b16); CMT("vl[0] &= bitmask_vec;"); } else { i_ldrb(r, wtmp, a64op_post(s->in[0], 1)); CMT("uint8_t tmp = *in[0]++;"); - i_movi(r, bitmask_vec.b8, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 8 times>, 0 <repeats 8 times>};"); i_dup (r, vl[0].b8, wtmp); CMT("vl[0].lo = broadcast(tmp);"); i_ushl(r, vl[0].b8, vl[0].b8, shift_vec.b8); CMT("vl[0] <<= shift_vec;"); i_and (r, vl[0].b8, vl[0].b8, bitmask_vec.b8); CMT("vl[0] &= bitmask_vec;"); } } +static void asmgen_setup_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + AArch64VecViews nibble_mask = a64op_vec_views(regs->vk[0]); + + rasm_annotate_next(r, "v128 nibble_mask = {0xf <repeats 8 times>, 0x0 <repeats 8 times>};"); + i_movi(r, nibble_mask.b8, IMM(0x0f)); +} + static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; + AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; AArch64VecViews nibble_mask = a64op_vec_views(regs->vk[0]); - AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; - AArch64VecViews vtmp = a64op_vec_views(regs->vt[0]); - rasm_annotate_next(r, "v128 nibble_mask = {0xf <repeats 8 times>, 0x0 <repeats 8 times>};"); - i_movi(r, nibble_mask.b8, IMM(0x0f)); + AArch64VecViews vtmp = a64op_vec_views(regs->vt[0]); if (p->block_size == 8) { i_ldr (r, vl[0].s, a64op_post(s->in[0], 4)); CMT("vl[0] = *in[0]++;"); @@ -491,18 +516,26 @@ static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplPa /* SWS_UOP_WRITE_PACKED */ /* SWS_UOP_WRITE_PLANAR */ +static void asmgen_setup_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); + + rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); + i_ldr(r, shift_vec.q, s->impl_priv); + asmgen_set_load_cont_node(s); +} + static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); - AArch64VecViews vtmp0 = a64op_vec_views(regs->vt[0]); - AArch64VecViews vtmp1 = a64op_vec_views(regs->vt[1]); - rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); - i_ldr(r, shift_vec.q, s->impl_priv); - asmgen_set_load_cont_node(s); + AArch64VecViews vtmp0 = a64op_vec_views(regs->vt[0]); + AArch64VecViews vtmp1 = a64op_vec_views(regs->vt[1]); if (p->block_size == 8) { i_ushl(r, vl[0].b8, vl[0].b8, shift_vec.b8); CMT("vl[0] <<= shift_vec;"); @@ -647,24 +680,14 @@ static void asmgen_op_move(SwsAArch64Context *s, const SwsAArch64OpImplParams *p /* split tightly packed data into components */ /* SWS_UOP_UNPACK */ -static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) +static void asmgen_setup_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; RasmOp *vmask = regs->vk; - RasmOp mask_gpr = a64op_w(s->tmp0); uint32_t mask_val[4] = { 0 }; - const int offsets[4] = { - p->par.pack.pattern[3] + p->par.pack.pattern[2] + p->par.pack.pattern[1], - p->par.pack.pattern[3] + p->par.pack.pattern[2], - p->par.pack.pattern[3], - 0 - }; - /* Generate masks. */ rasm_add_comment(r, "generate masks"); LOOP_MASK(p, i) { @@ -692,6 +715,22 @@ static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams vmask[i] = v_16b(vmask[i]); } } +} + +static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; + RasmOp *vmask = regs->vk; + + const int offsets[4] = { + p->par.pack.pattern[3] + p->par.pack.pattern[2] + p->par.pack.pattern[1], + p->par.pack.pattern[3] + p->par.pack.pattern[2], + p->par.pack.pattern[3], + 0 + }; /* Loop backwards to avoid clobbering component 0. */ LOOP_MASK_BWD (p, i) { @@ -788,6 +827,29 @@ static void asmgen_op_rshift(SwsAArch64Context *s, const SwsAArch64OpImplParams /* clear pixel values */ /* SWS_UOP_CLEAR */ +static void asmgen_setup_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp *vk = regs->vk; + + /** + * TODO + * - pack elements in impl->priv and perform smaller loads + * - if only 1 element and not vh, load directly with ld1r + */ + + bool load_priv = false; + LOOP_MASK(p, i) { + if (!((p->par.clear.zero | p->par.clear.one) & SWS_COMP(i))) + load_priv = true; + } + if (load_priv) { + i_ldr(r, v_q(vk[0]), s->impl_priv); CMT("v128 clear_vec = impl->priv.v128;"); + asmgen_set_load_cont_node(s); + } +} + static void emit_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vx, RasmOp *vk, int i, const char *vx_str) { @@ -810,27 +872,9 @@ static void emit_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { - RasmContext *r = s->rctx; RasmOp *vl = regs->vl; RasmOp *vh = regs->vh; RasmOp *vk = regs->vk; - RasmOp clear_vec = vk[0]; - - /** - * TODO - * - pack elements in impl->priv and perform smaller loads - * - if only 1 element and not vh, load directly with ld1r - */ - - bool load_priv = false; - LOOP_MASK(p, i) { - if (!((p->par.clear.zero | p->par.clear.one) & SWS_COMP(i))) - load_priv = true; - } - if (load_priv) { - i_ldr(r, v_q(clear_vec), s->impl_priv); CMT("v128 clear_vec = impl->priv.v128;"); - asmgen_set_load_cont_node(s); - } LOOP_MASK (p, i) { emit_clear(s, p, vl, vk, i, "vl"); } LOOP_MASK_VH(s, p, i) { emit_clear(s, p, vh, vk, i, "vh"); } @@ -969,6 +1013,18 @@ static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams /* numeric minimum */ /* SWS_UOP_MIN */ +static void asmgen_setup_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp *vk = regs->vk; + + RasmOp min_vec = regs->vt[0]; + i_ldr(r, v_q(min_vec), s->impl_priv); CMT("v128 min_vec = impl->priv.v128;"); + asmgen_set_load_cont_node(s); + LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(min_vec, i)); CMTF("v128 vmin%u = min_vec[%u];", i, i); } +} + static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { @@ -976,11 +1032,6 @@ static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vl = regs->vl; RasmOp *vh = regs->vh; RasmOp *vk = regs->vk; - RasmOp min_vec = regs->vt[0]; - - i_ldr(r, v_q(min_vec), s->impl_priv); CMT("v128 min_vec = impl->priv.v128;"); - asmgen_set_load_cont_node(s); - LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(min_vec, i)); CMTF("v128 vmin%u = min_vec[%u];", i, i); } if (p->type == SWS_PIXEL_F32) { LOOP_MASK (p, i) { i_fmin(r, vl[i], vl[i], vk[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); } @@ -995,6 +1046,18 @@ static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, /* numeric maximum */ /* SWS_UOP_MAX */ +static void asmgen_setup_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp *vk = regs->vk; + + RasmOp max_vec = regs->vt[0]; + i_ldr(r, v_q(max_vec), s->impl_priv); CMT("v128 max_vec = impl->priv.v128;"); + asmgen_set_load_cont_node(s); + LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(max_vec, i)); CMTF("v128 vmax%u = max_vec[%u];", i, i); } +} + static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { @@ -1002,11 +1065,6 @@ static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vl = regs->vl; RasmOp *vh = regs->vh; RasmOp *vk = regs->vk; - RasmOp max_vec = regs->vt[0]; - - i_ldr(r, v_q(max_vec), s->impl_priv); CMT("v128 max_vec = impl->priv.v128;"); - asmgen_set_load_cont_node(s); - LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(max_vec, i)); CMTF("v128 vmax%u = max_vec[%u];", i, i); } if (p->type == SWS_PIXEL_F32) { LOOP_MASK (p, i) { i_fmax(r, vl[i], vl[i], vk[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); } @@ -1021,19 +1079,26 @@ static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, /* multiplication by scalar */ /* SWS_UOP_SCALE */ +static void asmgen_setup_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp scale_vec = regs->vk[0]; + + RasmOp priv_ptr = s->tmp0; + i_add (r, priv_ptr, s->impl, IMM(offsetof_impl_priv)); CMT("v128 *scale_vec_ptr = &impl->priv;"); + asmgen_set_load_cont_node(s); + i_ld1r(r, vv_1(scale_vec), a64op_base(priv_ptr)); CMT("v128 scale_vec = broadcast(*scale_vec_ptr);"); +} + static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; - RasmOp priv_ptr = s->tmp0; + RasmOp *vl = regs->vl; + RasmOp *vh = regs->vh; RasmOp scale_vec = regs->vk[0]; - i_add (r, priv_ptr, s->impl, IMM(offsetof_impl_priv)); CMT("v128 *scale_vec_ptr = &impl->priv;"); - asmgen_set_load_cont_node(s); - i_ld1r(r, vv_1(scale_vec), a64op_base(priv_ptr)); CMT("v128 scale_vec = broadcast(*scale_vec_ptr);"); - if (p->type == SWS_PIXEL_F32) { LOOP_MASK (p, i) { i_fmul(r, vl[i], vl[i], scale_vec); CMTF("vl[%u] *= scale_vec;", i); } LOOP_MASK_VH(s, p, i) { i_fmul(r, vh[i], vh[i], scale_vec); CMTF("vh[%u] *= scale_vec;", i); } @@ -1048,6 +1113,29 @@ static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams * /* SWS_UOP_LINEAR */ /* SWS_UOP_LINEAR_FMA */ +static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp *vc = regs->vk; + + RasmOp ptr = s->tmp0; + RasmOp coeff_veclist; + + /* Preload coefficients from impl->priv. */ + const int num_vregs = linear_num_vregs(p); + av_assert0(num_vregs <= 4); + switch (num_vregs) { + case 1: coeff_veclist = vv_1(vc[0]); break; + case 2: coeff_veclist = vv_2(vc[0], vc[1]); break; + case 3: coeff_veclist = vv_3(vc[0], vc[1], vc[2]); break; + case 4: coeff_veclist = vv_4(vc[0], vc[1], vc[2], vc[3]); break; + } + i_ldr(r, ptr, s->impl_priv); CMT("v128 *vcoeff_ptr = impl->priv.ptr;"); + asmgen_set_load_cont_node(s); + i_ld1(r, coeff_veclist, a64op_base(ptr)); CMT("coeff_veclist = *vcoeff_ptr;"); +} + /** * Performs one pass of the linear transform over a single vector bank * (low or high). @@ -1143,25 +1231,6 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { - RasmContext *r = s->rctx; - RasmOp *vc = regs->vk; - - RasmOp ptr = s->tmp0; - RasmOp coeff_veclist; - - /* Preload coefficients from impl->priv. */ - const int num_vregs = linear_num_vregs(p); - av_assert0(num_vregs <= 4); - switch (num_vregs) { - case 1: coeff_veclist = vv_1(vc[0]); break; - case 2: coeff_veclist = vv_2(vc[0], vc[1]); break; - case 3: coeff_veclist = vv_3(vc[0], vc[1], vc[2]); break; - case 4: coeff_veclist = vv_4(vc[0], vc[1], vc[2], vc[3]); break; - } - i_ldr(r, ptr, s->impl_priv); CMT("v128 *vcoeff_ptr = impl->priv.ptr;"); - asmgen_set_load_cont_node(s); - i_ld1(r, coeff_veclist, a64op_base(ptr)); CMT("coeff_veclist = *vcoeff_ptr;"); - /* Compute mask for rows that must be saved before being overwritten. */ SwsCompMask save_mask = 0; bool overwritten[4] = { false, false, false, false }; @@ -1186,12 +1255,25 @@ static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams /* add dithering noise */ /* SWS_UOP_DITHER */ +static void asmgen_setup_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp src_ptr = s->tmp0; + + regs->dither_ptr = src_ptr; + i_ldr(r, src_ptr, s->impl_priv); CMT("void *ptr = impl->priv.ptr;"); + asmgen_set_load_cont_node(s); +} + static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; RasmOp *vl = regs->vl; RasmOp *vh = regs->vh; + RasmOp src_ptr = regs->dither_ptr; + RasmOp ptr = s->tmp0; RasmOp tmp1 = s->tmp1; RasmOp wtmp1 = a64op_w(tmp1); @@ -1227,9 +1309,6 @@ static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams } } - i_ldr(r, ptr, s->impl_priv); CMT("void *ptr = impl->priv.ptr;"); - asmgen_set_load_cont_node(s); - /** * We use ubfiz to mask and shift left in one single instruction: * ubfiz <Wd>, <Wn>, #<lsb>, #<width> @@ -1257,7 +1336,8 @@ static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams RasmOp lsb = IMM(block_size_log2 + sizeof_float_log2); RasmOp width = IMM(dither_size_log2 - block_size_log2); i_ubfiz(r, tmp1, bx64, lsb, width); CMT("tmp1 = (bx & ((dither_size / block_size) - 1)) * block_size * sizeof(float);"); - i_add (r, ptr, ptr, tmp1); CMT("ptr += tmp1;"); + i_add (r, ptr, src_ptr, tmp1); CMT("ptr += tmp1;"); + src_ptr = ptr; } int last_y_off = -1; @@ -1281,7 +1361,7 @@ static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams i_add (r, wtmp1, s->y, IMM(y_off)); CMTF("tmp1 = y + y_off[%u];", i); i_ubfiz(r, tmp1, tmp1, lsb, width); CMT("tmp1 = (tmp1 & (dither_size - 1)) * dither_size * sizeof(float);"); } - i_add(r, ptr, ptr, tmp1); CMT("ptr += tmp1;"); + i_add(r, ptr, src_ptr, tmp1); CMT("ptr += tmp1;"); } else if (do_load) { /** * On subsequent runs, just increment the pointer. @@ -1355,6 +1435,24 @@ static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) s->impl_priv = a64op_off(s->impl, offsetof_impl_priv); asmgen_set_load_cont_node(s); + /* Set up constants. */ + switch (p->uop) { + case SWS_UOP_READ_BIT: asmgen_setup_read_bit(s, p, &s->regs); break; + case SWS_UOP_READ_NIBBLE: asmgen_setup_read_nibble(s, p, &s->regs); break; + case SWS_UOP_WRITE_BIT: asmgen_setup_write_bit(s, p, &s->regs); break; + case SWS_UOP_UNPACK: asmgen_setup_unpack(s, p, &s->regs); break; + case SWS_UOP_CLEAR: asmgen_setup_clear(s, p, &s->regs); break; + case SWS_UOP_MIN: asmgen_setup_min(s, p, &s->regs); break; + case SWS_UOP_MAX: asmgen_setup_max(s, p, &s->regs); break; + case SWS_UOP_SCALE: asmgen_setup_scale(s, p, &s->regs); break; + case SWS_UOP_LINEAR: asmgen_setup_linear(s, p, &s->regs); break; + case SWS_UOP_LINEAR_FMA: asmgen_setup_linear(s, p, &s->regs); break; + case SWS_UOP_DITHER: asmgen_setup_dither(s, p, &s->regs); break; + default: + break; + } + + /* Emit uop kernel. */ switch (p->uop) { case SWS_UOP_READ_BIT: asmgen_op_read_bit(s, p, &s->regs); break; case SWS_UOP_READ_NIBBLE: asmgen_op_read_nibble(s, p, &s->regs); break; -- 2.52.0 >From 158ed00523c54236692687c32d2ab48a4b748593 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sat, 11 Jul 2026 19:26:43 +0200 Subject: [PATCH 11/23] swscale/aarch64/ops_asmgen: split cps code out of asmgen_process() The asmgen_process() function is now more generic, which will allow us to use it from the JIT compiler. The naming of the function, along with the setup code and main loop, are moved to asmgen_process_cps(). Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 50 +++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 6db8f368dd..2366a659ae 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -163,6 +163,10 @@ typedef struct SwsAArch64Context { RasmOp in_bump[4]; RasmOp out_bump[4]; + /* Process function. */ + RasmNode *setup; + RasmNode *loop; + /* Vector register dimensions. */ size_t el_size; size_t el_count; @@ -297,7 +301,6 @@ static unsigned clobbered_gprs(const SwsAArch64Context *s, static void asmgen_process(SwsAArch64Context *s, SwsCompMask mask) { RasmContext *r = s->rctx; - char func_name[128]; char buf[64]; /** @@ -305,19 +308,14 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask mask) * The description in x86/ops_include.asm mostly holds as well here. */ - snprintf(func_name, sizeof(func_name), "ff_sws_process_%04x_neon", nibble_mask(mask)); - - rasm_func_begin(r, func_name, true, false); - /* Function prologue */ RasmOp saved_regs[MAX_SAVED_REGS]; unsigned nsaved = clobbered_gprs(s, mask, saved_regs); if (nsaved) asmgen_prologue(s, saved_regs, nsaved); - /* Load values from impl. */ - i_ldr(r, s->op0_func, a64op_off(s->impl, offsetof_impl_cont)); CMT("SwsFuncPtr op0_func = impl->cont;"); - i_add(r, s->op1_impl, s->impl, IMM(sizeof_impl)); CMT("SwsOpImpl *op1_impl = impl + 1;"); + /* Setup. */ + s->setup = rasm_get_current_node(r); /* Load values from exec. */ LOOP(mask, i) { @@ -353,10 +351,9 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask mask) rasm_add_label(r, first_row); CMT("first_row:"); i_mov(r, s->bx, s->bx_start); CMT("bx = bx_start;"); - /* Reset impl and call first kernel. */ + /* Main loop. */ rasm_add_label(r, next_block); CMT("next_block:"); - i_mov(r, s->impl, s->op1_impl); CMT("impl = op1_impl;"); - i_blr(r, s->op0_func); CMT("op0_func();"); + s->loop = rasm_get_current_node(r); /* Perform horizontal loop. */ i_add(r, s->bx, s->bx, IMM(1)); CMT("bx += 1;"); @@ -1388,6 +1385,29 @@ static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams } } +/*********************************************************************/ +static void asmgen_process_cps(SwsAArch64Context *s, SwsCompMask mask) +{ + RasmContext *r = s->rctx; + char func_name[128]; + + snprintf(func_name, sizeof(func_name), "ff_sws_process_%04x_neon", nibble_mask(mask)); + rasm_func_begin(r, func_name, true, false); + + asmgen_process(s, mask); + + /* Load values from impl. */ + rasm_set_current_node(r, s->setup); + RasmOp impl_cont = a64op_off(s->impl, offsetof_impl_cont); + i_ldr(r, s->op0_func, impl_cont); CMT("SwsFuncPtr op0_func = impl->cont;"); + i_add(r, s->op1_impl, s->impl, IMM(sizeof_impl)); CMT("SwsOpImpl *op1_impl = impl + 1;"); + + /* Reset impl and call first kernel. */ + rasm_set_current_node(r, s->loop); + i_mov(r, s->impl, s->op1_impl); CMT("impl = op1_impl;"); + i_blr(r, s->op0_func); CMT("op0_func();"); +} + /*********************************************************************/ static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) { @@ -1590,10 +1610,10 @@ static int asmgen(void) s.out_bump[3] = a64op_gpx(27); /* Generate all process functions using rasm. */ - asmgen_process(&s, SWS_COMP_ELEMS(1)); - asmgen_process(&s, SWS_COMP_ELEMS(2)); - asmgen_process(&s, SWS_COMP_ELEMS(3)); - asmgen_process(&s, SWS_COMP_ELEMS(4)); + asmgen_process_cps(&s, SWS_COMP_ELEMS(1)); + asmgen_process_cps(&s, SWS_COMP_ELEMS(2)); + asmgen_process_cps(&s, SWS_COMP_ELEMS(3)); + asmgen_process_cps(&s, SWS_COMP_ELEMS(4)); /* Generate all functions from ops_entries.c using rasm. */ const SwsAArch64OpEntry *entries = ops_entries; -- 2.52.0 >From 19711a47af98c442c06cf572296ae019f850c0de Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sat, 18 Jul 2026 00:18:07 +0200 Subject: [PATCH 12/23] swscale/aarch64/ops_asmgen: pass both input and output masks to asmgen_process() Currently, for CPS code, both masks are the same, similar to how the C and x86 backends do it. But the JIT compiler will have more precise input and output masks. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 2366a659ae..38cac1d67c 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -284,21 +284,23 @@ static void clobber_gpr(RasmOp regs[MAX_SAVED_REGS], unsigned *count, } static unsigned clobbered_gprs(const SwsAArch64Context *s, - SwsCompMask mask, + SwsCompMask imask, SwsCompMask omask, RasmOp regs[MAX_SAVED_REGS]) { unsigned count = 0; clobber_gpr(regs, &count, a64op_lr()); - LOOP(mask, i) { + LOOP(imask, i) { clobber_gpr(regs, &count, s->in[i]); - clobber_gpr(regs, &count, s->out[i]); clobber_gpr(regs, &count, s->in_bump[i]); + } + LOOP(omask, i) { + clobber_gpr(regs, &count, s->out[i]); clobber_gpr(regs, &count, s->out_bump[i]); } return count; } -static void asmgen_process(SwsAArch64Context *s, SwsCompMask mask) +static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) { RasmContext *r = s->rctx; char buf[64]; @@ -310,7 +312,7 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask mask) /* Function prologue */ RasmOp saved_regs[MAX_SAVED_REGS]; - unsigned nsaved = clobbered_gprs(s, mask, saved_regs); + unsigned nsaved = clobbered_gprs(s, imask, omask, saved_regs); if (nsaved) asmgen_prologue(s, saved_regs, nsaved); @@ -318,19 +320,19 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask mask) s->setup = rasm_get_current_node(r); /* Load values from exec. */ - LOOP(mask, i) { + LOOP(imask, i) { rasm_annotate_nextf(r, buf, sizeof(buf), "in[%u] = exec->in[%u];", i, i); i_ldr(r, s->in[i], a64op_off(s->exec, offsetof_exec_in + (i * sizeof(uint8_t *)))); } - LOOP(mask, i) { + LOOP(omask, i) { rasm_annotate_nextf(r, buf, sizeof(buf), "out[%u] = exec->out[%u];", i, i); i_ldr(r, s->out[i], a64op_off(s->exec, offsetof_exec_out + (i * sizeof(uint8_t *)))); } - LOOP(mask, i) { + LOOP(imask, i) { rasm_annotate_nextf(r, buf, sizeof(buf), "in_bump[%u] = exec->in_bump[%u];", i, i); i_ldr(r, s->in_bump[i], a64op_off(s->exec, offsetof_exec_in_bump + (i * sizeof(ptrdiff_t)))); } - LOOP(mask, i) { + LOOP(omask, i) { rasm_annotate_nextf(r, buf, sizeof(buf), "out_bump[%u] = exec->out_bump[%u];", i, i); i_ldr(r, s->out_bump[i], a64op_off(s->exec, offsetof_exec_out_bump + (i * sizeof(ptrdiff_t)))); } @@ -344,8 +346,8 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask mask) /* Perform padding, preparing for next row. */ rasm_add_label(r, next_row); CMT("next_row:"); - LOOP(mask, i) { i_add(r, s->in[i], s->in[i], s->in_bump[i]); CMTF("in[%u] += in_bump[%u];", i, i); } - LOOP(mask, i) { i_add(r, s->out[i], s->out[i], s->out_bump[i]); CMTF("out[%u] += out_bump[%u];", i, i); } + LOOP(imask, i) { i_add(r, s->in[i], s->in[i], s->in_bump[i]); CMTF("in[%u] += in_bump[%u];", i, i); } + LOOP(omask, i) { i_add(r, s->out[i], s->out[i], s->out_bump[i]); CMTF("out[%u] += out_bump[%u];", i, i); } /* First row (reset x). */ rasm_add_label(r, first_row); CMT("first_row:"); @@ -1394,7 +1396,7 @@ static void asmgen_process_cps(SwsAArch64Context *s, SwsCompMask mask) snprintf(func_name, sizeof(func_name), "ff_sws_process_%04x_neon", nibble_mask(mask)); rasm_func_begin(r, func_name, true, false); - asmgen_process(s, mask); + asmgen_process(s, mask, mask); /* Load values from impl. */ rasm_set_current_node(r, s->setup); -- 2.52.0 >From f1c468411592570fa53c83521f6ccd512e4b66ea Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 12 Jul 2026 18:22:44 +0200 Subject: [PATCH 13/23] swscale/aarch64/ops_asmgen: use more (but simpler) loops for exec load generation Same behaviour, just looks neater. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 38cac1d67c..e6c4f8e3bd 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -303,7 +303,6 @@ static unsigned clobbered_gprs(const SwsAArch64Context *s, static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) { RasmContext *r = s->rctx; - char buf[64]; /** * The process function for aarch64 works similarly to the x86 backend. @@ -320,22 +319,18 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask s->setup = rasm_get_current_node(r); /* Load values from exec. */ - LOOP(imask, i) { - rasm_annotate_nextf(r, buf, sizeof(buf), "in[%u] = exec->in[%u];", i, i); - i_ldr(r, s->in[i], a64op_off(s->exec, offsetof_exec_in + (i * sizeof(uint8_t *)))); - } - LOOP(omask, i) { - rasm_annotate_nextf(r, buf, sizeof(buf), "out[%u] = exec->out[%u];", i, i); - i_ldr(r, s->out[i], a64op_off(s->exec, offsetof_exec_out + (i * sizeof(uint8_t *)))); - } - LOOP(imask, i) { - rasm_annotate_nextf(r, buf, sizeof(buf), "in_bump[%u] = exec->in_bump[%u];", i, i); - i_ldr(r, s->in_bump[i], a64op_off(s->exec, offsetof_exec_in_bump + (i * sizeof(ptrdiff_t)))); - } - LOOP(omask, i) { - rasm_annotate_nextf(r, buf, sizeof(buf), "out_bump[%u] = exec->out_bump[%u];", i, i); - i_ldr(r, s->out_bump[i], a64op_off(s->exec, offsetof_exec_out_bump + (i * sizeof(ptrdiff_t)))); - } + RasmOp exec_in[4]; + RasmOp exec_in_bump[4]; + RasmOp exec_out[4]; + RasmOp exec_out_bump[4]; + LOOP(imask, i) { exec_in [i] = a64op_off(s->exec, offsetof_exec_in + (i * sizeof(uint8_t *))); } + LOOP(imask, i) { exec_in_bump [i] = a64op_off(s->exec, offsetof_exec_in_bump + (i * sizeof(uint8_t *))); } + LOOP(omask, i) { exec_out [i] = a64op_off(s->exec, offsetof_exec_out + (i * sizeof(uint8_t *))); } + LOOP(omask, i) { exec_out_bump[i] = a64op_off(s->exec, offsetof_exec_out_bump + (i * sizeof(uint8_t *))); } + LOOP(imask, i) { i_ldr(r, s->in[i], exec_in [i]); CMTF("in[%u] = exec->in[%u];", i, i); } + LOOP(omask, i) { i_ldr(r, s->out[i], exec_out[i]); CMTF("out[%u] = exec->out[%u];", i, i); } + LOOP(imask, i) { i_ldr(r, s->in_bump[i], exec_in_bump[i]); CMTF("in_bump[%u] = exec->in_bump[%u];", i, i); } + LOOP(omask, i) { i_ldr(r, s->out_bump[i], exec_out_bump[i]); CMTF("out_bump[%u] = exec->out_bump[%u];", i, i); } int first_row = rasm_new_label(r, NULL); int next_row = rasm_new_label(r, NULL); -- 2.52.0 >From 5516b2d94d070c41dac20b2ba97258708271ac84 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sat, 18 Jul 2026 00:24:21 +0200 Subject: [PATCH 14/23] swscale/aarch64/ops_asmgen: load values from exec before performing setup This changes little for the current CPS code, which has fairly simple setup code. But in JIT we will factor out much more code from the main loop into the setup section. This also frees up the register used by exec (x0) earlier on, so it may be reused earlier in JIT code. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index e6c4f8e3bd..9f72266272 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -315,9 +315,6 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask if (nsaved) asmgen_prologue(s, saved_regs, nsaved); - /* Setup. */ - s->setup = rasm_get_current_node(r); - /* Load values from exec. */ RasmOp exec_in[4]; RasmOp exec_in_bump[4]; @@ -332,6 +329,9 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask LOOP(imask, i) { i_ldr(r, s->in_bump[i], exec_in_bump[i]); CMTF("in_bump[%u] = exec->in_bump[%u];", i, i); } LOOP(omask, i) { i_ldr(r, s->out_bump[i], exec_out_bump[i]); CMTF("out_bump[%u] = exec->out_bump[%u];", i, i); } + /* Setup. */ + s->setup = rasm_get_current_node(r); + int first_row = rasm_new_label(r, NULL); int next_row = rasm_new_label(r, NULL); int next_block = rasm_new_label(r, NULL); -- 2.52.0 >From a89b4a2a67ae14bc56b4e7b96ae650e435893970 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 12 Jul 2026 16:10:13 +0200 Subject: [PATCH 15/23] swscale/aarch64/ops_asmgen: setup frame for each function Currently this is redundant, since all CPS functions share the same frame. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 163 ++++++++++++++++++-------------- 1 file changed, 90 insertions(+), 73 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 9f72266272..0530c0f5ae 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -1382,6 +1382,94 @@ static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams } } +/*********************************************************************/ +/** + * Register assignment for CPS functions. + * + * The entry point of the SwsOpFunc is the `process` function. The + * first kernel function is called from `process`, and subsequent + * kernel functions are chained by directly branching to the next + * operation, using a continuation-passing style design. The last + * operation must be a write operation, which returns from the call + * to the `process` function. + * + * The GPRs used by the entire call-chain are listed below. + * + * Function arguments are passed in r0-r5. After the parameters from + * `exec` have been read, r0 is reused to branch to the continuation + * functions. After the original parameters from `impl` have been + * computed, r1 is reused as the `impl` pointer for each operation. + * + * Loop iterators are r6 for `bx` and r3 for `y`, reused from + * `y_start`, which doesn't need to be preserved. + * + * The intra-procedure-call temporary registers (r16 and r17) are used + * as scratch registers. They may be used by call veneers and PLT code + * inserted by the linker, so we cannot expect them to persist across + * branches between functions. + * + * The Platform Register (r18) is not used. + * + * The read/write data pointers and padding values first use up the + * remaining free caller-saved registers, and only then are the + * caller-saved registers (r19-r29) used. + * + * The Link Register (r30) is used when calling the first kernel, so it + * must be saved. + */ + +static const int rw_gprs[] = { + 9, 10, 11, 12, + 13, 14, 15, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, +}; + +static void asmgen_common_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) +{ + /* Loop iterator variables. */ + s->bx = a64op_gpw(6); + s->y = a64op_gpw(3); /* Reused from SwsOpFunc.y_start argument. */ + + /* Scratch registers. */ + s->tmp0 = a64op_gpx(16); /* IP0 */ + s->tmp1 = a64op_gpx(17); /* IP1 */ + + /* Read/Write data pointers. */ + LOOP(imask, i) { s->in [i] = a64op_gpx(rw_gprs[(i * 4) + 0]); } + LOOP(omask, i) { s->out[i] = a64op_gpx(rw_gprs[(i * 4) + 1]); } +} + +static void asmgen_process_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) +{ + asmgen_common_frame(s, imask, omask); + + /* SwsOpFunc arguments. */ + s->exec = a64op_gpx(0); // const SwsOpExec *exec + s->impl = a64op_gpx(1); // const void *priv + s->bx_start = a64op_gpw(2); // int bx_start + s->y_start = a64op_gpw(3); // int y_start + s->bx_end = a64op_gpw(4); // int bx_end + s->y_end = a64op_gpw(5); // int y_end + + /* CPS-related variables. */ + s->op0_func = a64op_gpx(7); + s->op1_impl = a64op_gpx(8); + + /* Read/Write data pointer padding. */ + LOOP(imask, i) { s->in_bump [i] = a64op_gpx(rw_gprs[(i * 4) + 2]); } + LOOP(omask, i) { s->out_bump[i] = a64op_gpx(rw_gprs[(i * 4) + 3]); } +} + +static void asmgen_op_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) +{ + asmgen_common_frame(s, imask, omask); + + /* CPS-related variables. */ + s->cont = a64op_gpx(0); /* Reused from SwsOpFunc.exec argument. */ + s->impl = a64op_gpx(1); /* Same as SwsOpFunc.impl argument. */ +} + /*********************************************************************/ static void asmgen_process_cps(SwsAArch64Context *s, SwsCompMask mask) { @@ -1390,6 +1478,7 @@ static void asmgen_process_cps(SwsAArch64Context *s, SwsCompMask mask) snprintf(func_name, sizeof(func_name), "ff_sws_process_%04x_neon", nibble_mask(mask)); rasm_func_begin(r, func_name, true, false); + asmgen_process_frame(s, mask, mask); asmgen_process(s, mask, mask); @@ -1431,6 +1520,7 @@ static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) } rasm_func_begin(r, entry->name, true, !is_read); + asmgen_op_frame(s, is_read ? p->mask : 0, is_write ? p->mask : 0); /** * Set up vector register dimensions and reshape all vectors @@ -1533,79 +1623,6 @@ static int asmgen(void) av_bprint_init(&bp, 0, AV_BPRINT_SIZE_UNLIMITED); - /** - * The entry point of the SwsOpFunc is the `process` function. The - * first kernel function is called from `process`, and subsequent - * kernel functions are chained by directly branching to the next - * operation, using a continuation-passing style design. The last - * operation must be a write operation, which returns from the call - * to the `process` function. - * - * The GPRs used by the entire call-chain are listed below. - * - * Function arguments are passed in r0-r5. After the parameters - * from `exec` have been read, r0 is reused to branch to the - * continuation functions. After the original parameters from - * `impl` have been computed, r1 is reused as the `impl` pointer - * for each operation. - * - * Loop iterators are r6 for `bx` and r3 for `y`, reused from - * `y_start`, which doesn't need to be preserved. - * - * The intra-procedure-call temporary registers (r16 and r17) are - * used as scratch registers. They may be used by call veneers and - * PLT code inserted by the linker, so we cannot expect them to - * persist across branches between functions. - * - * The Platform Register (r18) is not used. - * - * The read/write data pointers and padding values first use up the - * remaining free caller-saved registers, and only then are the - * caller-saved registers (r19-r28) used. - * - * The Link Register (r30) is used when calling the first kernel, - * so it must be saved. - */ - - /* SwsOpFunc arguments. */ - s.exec = a64op_gpx(0); // const SwsOpExec *exec - s.impl = a64op_gpx(1); // const void *priv - s.bx_start = a64op_gpw(2); // int bx_start - s.y_start = a64op_gpw(3); // int y_start - s.bx_end = a64op_gpw(4); // int bx_end - s.y_end = a64op_gpw(5); // int y_end - - /* Loop iterator variables. */ - s.bx = a64op_gpw(6); - s.y = s.y_start; /* Reused from SwsOpFunc argument. */ - - /* Scratch registers. */ - s.tmp0 = a64op_gpx(16); /* IP0 */ - s.tmp1 = a64op_gpx(17); /* IP1 */ - - /* CPS-related variables. */ - s.op0_func = a64op_gpx(7); - s.op1_impl = a64op_gpx(8); - s.cont = s.exec; /* Reused from SwsOpFunc argument. */ - - /* Read/Write data pointers and padding. */ - s.in [0] = a64op_gpx(9); - s.out [0] = a64op_gpx(10); - s.in_bump [0] = a64op_gpx(11); - s.out_bump[0] = a64op_gpx(12); - s.in [1] = a64op_gpx(13); - s.out [1] = a64op_gpx(14); - s.in_bump [1] = a64op_gpx(15); - s.out_bump[1] = a64op_gpx(19); - s.in [2] = a64op_gpx(20); - s.out [2] = a64op_gpx(21); - s.in_bump [2] = a64op_gpx(22); - s.out_bump[2] = a64op_gpx(23); - s.in [3] = a64op_gpx(24); - s.out [3] = a64op_gpx(25); - s.in_bump [3] = a64op_gpx(26); - s.out_bump[3] = a64op_gpx(27); - /* Generate all process functions using rasm. */ asmgen_process_cps(&s, SWS_COMP_ELEMS(1)); asmgen_process_cps(&s, SWS_COMP_ELEMS(2)); -- 2.52.0 >From bb989984cafad3803f394483b4f2d17d4949a77b Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 19 Jul 2026 18:50:19 +0200 Subject: [PATCH 16/23] swscale/aarch64/ops_asmgen: split vector init from reshaping This changes nothing for the CPS code, but will allow the JIT compiler to allocate and reorder vector registers without them being affected by reshaping. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 55 +++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 0530c0f5ae..f343871737 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -189,36 +189,24 @@ typedef struct SwsAArch64Context { /* Reshape input/output vector registers for current SwsOp. */ static void reshape_io_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size) { - regs->vl[0] = a64op_make_vec( 0, el_count, el_size); - regs->vl[1] = a64op_make_vec( 1, el_count, el_size); - regs->vl[2] = a64op_make_vec( 2, el_count, el_size); - regs->vl[3] = a64op_make_vec( 3, el_count, el_size); - regs->vh[0] = a64op_make_vec( 4, el_count, el_size); - regs->vh[1] = a64op_make_vec( 5, el_count, el_size); - regs->vh[2] = a64op_make_vec( 6, el_count, el_size); - regs->vh[3] = a64op_make_vec( 7, el_count, el_size); + for (int i = 0; i < 4; i++) { + regs->vl[i] = a64op_make_vec(a64op_vec_n(regs->vl[i]), el_count, el_size); + regs->vh[i] = a64op_make_vec(a64op_vec_n(regs->vh[i]), el_count, el_size); + } } /* Reshape temp vector registers for current SwsOp. */ static void reshape_temp_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size) { - regs->vt[0] = a64op_make_vec(16, el_count, el_size); - regs->vt[1] = a64op_make_vec(17, el_count, el_size); - regs->vt[2] = a64op_make_vec(18, el_count, el_size); - regs->vt[3] = a64op_make_vec(19, el_count, el_size); - regs->vt[4] = a64op_make_vec(20, el_count, el_size); - regs->vt[5] = a64op_make_vec(21, el_count, el_size); - regs->vt[6] = a64op_make_vec(22, el_count, el_size); - regs->vt[7] = a64op_make_vec(23, el_count, el_size); + for (int i = 0; i < 8; i++) + regs->vt[i] = a64op_make_vec(a64op_vec_n(regs->vt[i]), el_count, el_size); } /* Reshape const vector registers for current SwsOp. */ static void reshape_const_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size) { - regs->vk[0] = a64op_make_vec(24, el_count, el_size); - regs->vk[1] = a64op_make_vec(25, el_count, el_size); - regs->vk[2] = a64op_make_vec(26, el_count, el_size); - regs->vk[3] = a64op_make_vec(27, el_count, el_size); + for (int i = 0; i < 4; i++) + regs->vk[i] = a64op_make_vec(a64op_vec_n(regs->vk[i]), el_count, el_size); } /*********************************************************************/ @@ -1470,6 +1458,32 @@ static void asmgen_op_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask s->impl = a64op_gpx(1); /* Same as SwsOpFunc.impl argument. */ } +/*********************************************************************/ +/* Vector register assignment. */ +static void init_vectors_cps(SwsAArch64Context *s, SwsAArch64OpRegs *regs) +{ + regs->vl[0] = a64op_vec( 0); + regs->vl[1] = a64op_vec( 1); + regs->vl[2] = a64op_vec( 2); + regs->vl[3] = a64op_vec( 3); + regs->vh[0] = a64op_vec( 4); + regs->vh[1] = a64op_vec( 5); + regs->vh[2] = a64op_vec( 6); + regs->vh[3] = a64op_vec( 7); + regs->vt[0] = a64op_vec(16); + regs->vt[1] = a64op_vec(17); + regs->vt[2] = a64op_vec(18); + regs->vt[3] = a64op_vec(19); + regs->vt[4] = a64op_vec(20); + regs->vt[5] = a64op_vec(21); + regs->vt[6] = a64op_vec(22); + regs->vt[7] = a64op_vec(23); + regs->vk[0] = a64op_vec(24); + regs->vk[1] = a64op_vec(25); + regs->vk[2] = a64op_vec(26); + regs->vk[3] = a64op_vec(27); +} + /*********************************************************************/ static void asmgen_process_cps(SwsAArch64Context *s, SwsCompMask mask) { @@ -1534,6 +1548,7 @@ static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) s->el_size = el_size; s->el_count = s->vec_size / el_size; + init_vectors_cps(s, &s->regs); reshape_io_vectors(&s->regs, s->el_count, el_size); reshape_temp_vectors(&s->regs, s->el_count, el_size); reshape_const_vectors(&s->regs, s->el_count, el_size); -- 2.52.0 >From db3bdff760f8cf7cbb11d1323b339286724fd720 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sat, 18 Jul 2026 02:43:25 +0200 Subject: [PATCH 17/23] swscale/aarch64/ops_asmgen: split input/output vector banks into separate register variables For CPS these will continue being the same vector register numbers, because of the fixed ABI, but for JIT we will be able to use different input/output vector registers to implicitly perform swizzles. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 343 +++++++++++++++++++------------- 1 file changed, 200 insertions(+), 143 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index f343871737..2fb9c040b1 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -118,8 +118,10 @@ static const SwsAArch64OpEntry ops_entries[] = { /*********************************************************************/ typedef struct SwsAArch64OpRegs { - RasmOp vl[4]; /* input/output vector registers (low bank) */ - RasmOp vh[4]; /* input/output vector registers (high bank) */ + RasmOp sl[4]; /* input vector registers (low bank) */ + RasmOp sh[4]; /* input vector registers (high bank) */ + RasmOp dl[4]; /* output vector registers (low bank) */ + RasmOp dh[4]; /* output vector registers (high bank) */ RasmOp vt[8]; /* temp vector registers */ RasmOp vk[4]; /* constant data (may be gprs) */ @@ -190,8 +192,10 @@ typedef struct SwsAArch64Context { static void reshape_io_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size) { for (int i = 0; i < 4; i++) { - regs->vl[i] = a64op_make_vec(a64op_vec_n(regs->vl[i]), el_count, el_size); - regs->vh[i] = a64op_make_vec(a64op_vec_n(regs->vh[i]), el_count, el_size); + regs->sl[i] = a64op_make_vec(a64op_vec_n(regs->sl[i]), el_count, el_size); + regs->sh[i] = a64op_make_vec(a64op_vec_n(regs->sh[i]), el_count, el_size); + regs->dl[i] = a64op_make_vec(a64op_vec_n(regs->dl[i]), el_count, el_size); + regs->dh[i] = a64op_make_vec(a64op_vec_n(regs->dh[i]), el_count, el_size); } } @@ -397,7 +401,7 @@ static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParam SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; + AArch64VecViews dl[1] = { a64op_vec_views(regs->dl[0]) }; AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); AArch64VecViews bitmask_vec = a64op_vec_views(regs->vk[1]); @@ -408,17 +412,17 @@ static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParam * ushl actually performs a right shift. */ if (p->block_size == 16) { i_ldrh(r, wtmp, a64op_post(s->in[0], 2)); CMT("uint16_t tmp = *in[0]++;"); - i_dup (r, vl[0].b8, wtmp); CMT("vl[0].lo = broadcast(tmp);"); + i_dup (r, dl[0].b8, wtmp); CMT("vl[0].lo = broadcast(tmp);"); i_lsr (r, wtmp, wtmp, IMM(8)); CMT("tmp >>= 8;"); i_dup (r, vtmp.b8, wtmp); CMT("vtmp.lo = broadcast(tmp);"); - i_ins (r, vl[0].de[1], vtmp.de[0]); CMT("vl[0].hi = vtmp.lo;"); - i_ushl(r, vl[0].b16, vl[0].b16, shift_vec.b16); CMT("vl[0] <<= shift_vec;"); - i_and (r, vl[0].b16, vl[0].b16, bitmask_vec.b16); CMT("vl[0] &= bitmask_vec;"); + i_ins (r, dl[0].de[1], vtmp.de[0]); CMT("vl[0].hi = vtmp.lo;"); + i_ushl(r, dl[0].b16, dl[0].b16, shift_vec.b16); CMT("vl[0] <<= shift_vec;"); + i_and (r, dl[0].b16, dl[0].b16, bitmask_vec.b16); CMT("vl[0] &= bitmask_vec;"); } else { i_ldrb(r, wtmp, a64op_post(s->in[0], 1)); CMT("uint8_t tmp = *in[0]++;"); - i_dup (r, vl[0].b8, wtmp); CMT("vl[0].lo = broadcast(tmp);"); - i_ushl(r, vl[0].b8, vl[0].b8, shift_vec.b8); CMT("vl[0] <<= shift_vec;"); - i_and (r, vl[0].b8, vl[0].b8, bitmask_vec.b8); CMT("vl[0] &= bitmask_vec;"); + i_dup (r, dl[0].b8, wtmp); CMT("vl[0].lo = broadcast(tmp);"); + i_ushl(r, dl[0].b8, dl[0].b8, shift_vec.b8); CMT("vl[0] <<= shift_vec;"); + i_and (r, dl[0].b8, dl[0].b8, bitmask_vec.b8); CMT("vl[0] &= bitmask_vec;"); } } @@ -436,21 +440,21 @@ static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplPa SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; + AArch64VecViews dl[1] = { a64op_vec_views(regs->dl[0]) }; AArch64VecViews nibble_mask = a64op_vec_views(regs->vk[0]); AArch64VecViews vtmp = a64op_vec_views(regs->vt[0]); if (p->block_size == 8) { - i_ldr (r, vl[0].s, a64op_post(s->in[0], 4)); CMT("vl[0] = *in[0]++;"); - i_ushr(r, vtmp.b8, vl[0].b8, IMM(4)); CMT("vtmp.lo = vl[0] >> 4;"); - i_and (r, vl[0].b8, vl[0].b8, nibble_mask.b8); CMT("vl[0].lo &= nibble_mask;"); - i_zip1(r, vl[0].b8, vtmp.b8, vl[0].b8); CMT("interleave"); + i_ldr (r, dl[0].s, a64op_post(s->in[0], 4)); CMT("vl[0] = *in[0]++;"); + i_ushr(r, vtmp.b8, dl[0].b8, IMM(4)); CMT("vtmp.lo = vl[0] >> 4;"); + i_and (r, dl[0].b8, dl[0].b8, nibble_mask.b8); CMT("vl[0].lo &= nibble_mask;"); + i_zip1(r, dl[0].b8, vtmp.b8, dl[0].b8); CMT("interleave"); } else { - i_ldr (r, vl[0].d, a64op_post(s->in[0], 8)); CMT("vl[0] = *in[0]++;"); - i_ushr(r, vtmp.b8, vl[0].b8, IMM(4)); CMT("vtmp.lo = vl[0] >> 4;"); - i_and (r, vl[0].b8, vl[0].b8, nibble_mask.b8); CMT("vl[0].lo &= nibble_mask;"); - i_zip1(r, vl[0].b16, vtmp.b16, vl[0].b16); CMT("interleave"); + i_ldr (r, dl[0].d, a64op_post(s->in[0], 8)); CMT("vl[0] = *in[0]++;"); + i_ushr(r, vtmp.b8, dl[0].b8, IMM(4)); CMT("vtmp.lo = vl[0] >> 4;"); + i_and (r, dl[0].b8, dl[0].b8, nibble_mask.b8); CMT("vl[0].lo &= nibble_mask;"); + i_zip1(r, dl[0].b16, vtmp.b16, dl[0].b16); CMT("interleave"); } } @@ -469,24 +473,24 @@ static void asmgen_op_read_packed(SwsAArch64Context *s, const SwsAArch64OpImplPa SwsAArch64OpRegs *regs) { av_assert0(p->mask != 0x0001); - asmgen_op_read_packed_n(s, p, regs->vl); + asmgen_op_read_packed_n(s, p, regs->dl); if (s->use_vh) - asmgen_op_read_packed_n(s, p, regs->vh); + asmgen_op_read_packed_n(s, p, regs->dh); } static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(regs->vl); - AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(regs->vh); + AArch64VecViews dl[4] = A64OP_VEC_VIEWS4(regs->dl); + AArch64VecViews dh[4] = A64OP_VEC_VIEWS4(regs->dh); LOOP_MASK(p, i) { switch ((s->use_vh ? 0x100 : 0) | s->vec_size) { - case 0x008: i_ldr(r, vl[i].d, a64op_post(s->in[i], s->vec_size * 1)); break; - case 0x010: i_ldr(r, vl[i].q, a64op_post(s->in[i], s->vec_size * 1)); break; - case 0x108: i_ldp(r, vl[i].d, vh[i].d, a64op_post(s->in[i], s->vec_size * 2)); break; - case 0x110: i_ldp(r, vl[i].q, vh[i].q, a64op_post(s->in[i], s->vec_size * 2)); break; + case 0x008: i_ldr(r, dl[i].d, a64op_post(s->in[i], s->vec_size * 1)); break; + case 0x010: i_ldr(r, dl[i].q, a64op_post(s->in[i], s->vec_size * 1)); break; + case 0x108: i_ldp(r, dl[i].d, dh[i].d, a64op_post(s->in[i], s->vec_size * 2)); break; + case 0x110: i_ldp(r, dl[i].q, dh[i].q, a64op_post(s->in[i], s->vec_size * 2)); break; } } } @@ -513,20 +517,20 @@ static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplPara SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[1] = { a64op_vec_views(regs->vl[0]) }; + AArch64VecViews sl[1] = { a64op_vec_views(regs->sl[0]) }; AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); AArch64VecViews vtmp0 = a64op_vec_views(regs->vt[0]); AArch64VecViews vtmp1 = a64op_vec_views(regs->vt[1]); if (p->block_size == 8) { - i_ushl(r, vl[0].b8, vl[0].b8, shift_vec.b8); CMT("vl[0] <<= shift_vec;"); - i_addv(r, vtmp0.b, vl[0].b8); CMT("vtmp0[0] = add_across(vl[0].lo);"); + i_ushl(r, sl[0].b8, sl[0].b8, shift_vec.b8); CMT("vl[0] <<= shift_vec;"); + i_addv(r, vtmp0.b, sl[0].b8); CMT("vtmp0[0] = add_across(vl[0].lo);"); i_str (r, vtmp0.b, a64op_post(s->out[0], 1)); CMT("*out[0]++ = vtmp0;"); } else { - i_ushl(r, vl[0].b16, vl[0].b16, shift_vec.b16); CMT("vl[0] <<= shift_vec;"); - i_addv(r, vtmp0.b, vl[0].b8); CMT("vtmp0[0] = add_across(vl[0].lo);"); - i_ins (r, vtmp1.de[0], vl[0].de[1]); CMT("vtmp1.lo = vl[0].hi;"); + i_ushl(r, sl[0].b16, sl[0].b16, shift_vec.b16); CMT("vl[0] <<= shift_vec;"); + i_addv(r, vtmp0.b, sl[0].b8); CMT("vtmp0[0] = add_across(vl[0].lo);"); + i_ins (r, vtmp1.de[0], sl[0].de[1]); CMT("vtmp1.lo = vl[0].hi;"); i_addv(r, vtmp1.b, vtmp1.b8); CMT("vtmp1[0] = add_across(vtmp1);"); i_ins (r, vtmp0.be[1], vtmp1.be[0]); CMT("vtmp0[1] = vtmp1[0];"); i_str (r, vtmp0.h, a64op_post(s->out[0], 2)); CMT("*out[0]++ = vtmp0;"); @@ -537,21 +541,21 @@ static void asmgen_op_write_nibble(SwsAArch64Context *s, const SwsAArch64OpImplP SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(regs->vl); + AArch64VecViews sl[4] = A64OP_VEC_VIEWS4(regs->sl); AArch64VecViews vtmp0 = a64op_vec_views(regs->vt[0]); AArch64VecViews vtmp1 = a64op_vec_views(regs->vt[1]); if (p->block_size == 8) { - i_shl (r, vtmp0.h4, vl[0].h4, IMM(4)); - i_ushr(r, vtmp1.h4, vl[0].h4, IMM(8)); - i_orr (r, vl[0].b8, vtmp0.b8, vtmp1.b8); - i_xtn (r, vtmp0.b8, vl[0].h8); + i_shl (r, vtmp0.h4, sl[0].h4, IMM(4)); + i_ushr(r, vtmp1.h4, sl[0].h4, IMM(8)); + i_orr (r, sl[0].b8, vtmp0.b8, vtmp1.b8); + i_xtn (r, vtmp0.b8, sl[0].h8); i_str (r, vtmp0.s, a64op_post(s->out[0], 4)); } else { - i_shl (r, vtmp0.h8, vl[0].h8, IMM(4)); - i_ushr(r, vtmp1.h8, vl[0].h8, IMM(8)); - i_orr (r, vl[0].b16, vtmp0.b16, vtmp1.b16); - i_xtn (r, vtmp0.b8, vl[0].h8); + i_shl (r, vtmp0.h8, sl[0].h8, IMM(4)); + i_ushr(r, vtmp1.h8, sl[0].h8, IMM(8)); + i_orr (r, sl[0].b16, vtmp0.b16, vtmp1.b16); + i_xtn (r, vtmp0.b8, sl[0].h8); i_str (r, vtmp0.d, a64op_post(s->out[0], 8)); } } @@ -571,24 +575,24 @@ static void asmgen_op_write_packed(SwsAArch64Context *s, const SwsAArch64OpImplP SwsAArch64OpRegs *regs) { av_assert0(p->mask != 0x0001); - asmgen_op_write_packed_n(s, p, regs->vl); + asmgen_op_write_packed_n(s, p, regs->sl); if (s->use_vh) - asmgen_op_write_packed_n(s, p, regs->vh); + asmgen_op_write_packed_n(s, p, regs->sh); } static void asmgen_op_write_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(regs->vl); - AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(regs->vh); + AArch64VecViews sl[4] = A64OP_VEC_VIEWS4(regs->sl); + AArch64VecViews sh[4] = A64OP_VEC_VIEWS4(regs->sh); LOOP_MASK(p, i) { switch ((s->use_vh ? 0x100 : 0) | s->vec_size) { - case 0x008: i_str(r, vl[i].d, a64op_post(s->out[i], s->vec_size * 1)); break; - case 0x010: i_str(r, vl[i].q, a64op_post(s->out[i], s->vec_size * 1)); break; - case 0x108: i_stp(r, vl[i].d, vh[i].d, a64op_post(s->out[i], s->vec_size * 2)); break; - case 0x110: i_stp(r, vl[i].q, vh[i].q, a64op_post(s->out[i], s->vec_size * 2)); break; + case 0x008: i_str(r, sl[i].d, a64op_post(s->out[i], s->vec_size * 1)); break; + case 0x010: i_str(r, sl[i].q, a64op_post(s->out[i], s->vec_size * 1)); break; + case 0x108: i_stp(r, sl[i].d, sh[i].d, a64op_post(s->out[i], s->vec_size * 2)); break; + case 0x110: i_stp(r, sl[i].q, sh[i].q, a64op_post(s->out[i], s->vec_size * 2)); break; } } } @@ -601,17 +605,19 @@ static void asmgen_op_swap_bytes(SwsAArch64Context *s, const SwsAArch64OpImplPar SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(regs->vl); - AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(regs->vh); + AArch64VecViews sl[4] = A64OP_VEC_VIEWS4(regs->sl); + AArch64VecViews sh[4] = A64OP_VEC_VIEWS4(regs->sh); + AArch64VecViews dl[4] = A64OP_VEC_VIEWS4(regs->dl); + AArch64VecViews dh[4] = A64OP_VEC_VIEWS4(regs->dh); switch (ff_sws_pixel_type_size(p->type)) { case sizeof(uint16_t): - LOOP_MASK (p, i) i_rev16(r, vl[i].b16, vl[i].b16); - LOOP_MASK_VH(s, p, i) i_rev16(r, vh[i].b16, vh[i].b16); + LOOP_MASK (p, i) i_rev16(r, dl[i].b16, sl[i].b16); + LOOP_MASK_VH(s, p, i) i_rev16(r, dh[i].b16, sh[i].b16); break; case sizeof(uint32_t): - LOOP_MASK (p, i) i_rev32(r, vl[i].b16, vl[i].b16); - LOOP_MASK_VH(s, p, i) i_rev32(r, vh[i].b16, vh[i].b16); + LOOP_MASK (p, i) i_rev32(r, dl[i].b16, sl[i].b16); + LOOP_MASK_VH(s, p, i) i_rev32(r, dh[i].b16, sh[i].b16); break; } } @@ -631,19 +637,21 @@ static const char *print_swizzle_v(char buf[8], int8_t n, uint8_t vh) } #define PRINT_SWIZZLE_V(n, vh) print_swizzle_v((char[8]){ 0 }, n, vh) -static RasmOp swizzle_a64op(SwsAArch64OpRegs *regs, int8_t n, uint8_t vh) +static RasmOp swizzle_a64op(SwsAArch64OpRegs *regs, int8_t n, uint8_t vh, bool dst) { if (n == -1) return regs->vt[vh]; - return vh ? regs->vh[n] : regs->vl[n]; + if (vh) + return dst ? regs->dh[n] : regs->sh[n]; + return dst ? regs->dl[n] : regs->sl[n]; } static void swizzle_emit(SwsAArch64Context *s, SwsAArch64OpRegs *regs, int8_t dst, int8_t src) { RasmContext *r = s->rctx; - RasmOp src_op[2] = { swizzle_a64op(regs, src, 0), swizzle_a64op(regs, src, 1) }; - RasmOp dst_op[2] = { swizzle_a64op(regs, dst, 0), swizzle_a64op(regs, dst, 1) }; + RasmOp src_op[2] = { swizzle_a64op(regs, src, 0, false), swizzle_a64op(regs, src, 1, false) }; + RasmOp dst_op[2] = { swizzle_a64op(regs, dst, 0, true), swizzle_a64op(regs, dst, 1, true) }; i_mov (r, dst_op[0], src_op[0]); CMTF("%s = %s;", PRINT_SWIZZLE_V(dst, 0), PRINT_SWIZZLE_V(src, 0)); if (s->use_vh) { @@ -703,8 +711,10 @@ static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; + RasmOp *sl = regs->sl; + RasmOp *sh = regs->sh; + RasmOp *dl = regs->dl; + RasmOp *dh = regs->dh; RasmOp *vmask = regs->vk; const int offsets[4] = { @@ -717,22 +727,22 @@ static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams /* Loop backwards to avoid clobbering component 0. */ LOOP_MASK_BWD (p, i) { if (offsets[i]) { - i_ushr (r, vl[i], vl[0], IMM(offsets[i])); CMTF("vl[%u] >>= %u;", i, offsets[i]); + i_ushr (r, dl[i], sl[0], IMM(offsets[i])); CMTF("vl[%u] >>= %u;", i, offsets[i]); } else if (i) { - i_mov16b(r, vl[i], vl[0]); CMTF("vl[%u] = vl[0];", i); + i_mov16b(r, dl[i], sl[0]); CMTF("vl[%u] = vl[0];", i); } } LOOP_MASK_BWD_VH(s, p, i) { if (offsets[i]) { - i_ushr (r, vh[i], vh[0], IMM(offsets[i])); CMTF("vh[%u] >>= %u;", i, offsets[i]); + i_ushr (r, dh[i], sh[0], IMM(offsets[i])); CMTF("vh[%u] >>= %u;", i, offsets[i]); } else if (i) { - i_mov16b(r, vh[i], vh[0]); CMTF("vh[%u] = vh[0];", i); + i_mov16b(r, dh[i], sh[0]); CMTF("vh[%u] = vh[0];", i); } } /* Apply masks. */ - LOOP_MASK_BWD (p, i) { i_and16b(r, vl[i], vl[i], vmask[i]); CMTF("vl[%u] &= 0x%x;", i, (1u << p->par.pack.pattern[i]) - 1); } - LOOP_MASK_BWD_VH(s, p, i) { i_and16b(r, vh[i], vh[i], vmask[i]); CMTF("vh[%u] &= 0x%x;", i, (1u << p->par.pack.pattern[i]) - 1); } + LOOP_MASK_BWD (p, i) { i_and16b(r, dl[i], dl[i], vmask[i]); CMTF("vl[%u] &= 0x%x;", i, (1u << p->par.pack.pattern[i]) - 1); } + LOOP_MASK_BWD_VH(s, p, i) { i_and16b(r, dh[i], dh[i], vmask[i]); CMTF("vh[%u] &= 0x%x;", i, (1u << p->par.pack.pattern[i]) - 1); } } /*********************************************************************/ @@ -743,8 +753,10 @@ static void asmgen_op_pack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; + RasmOp *sl = regs->sl; + RasmOp *sh = regs->sh; + RasmOp *dl = regs->dl; + RasmOp *dh = regs->dh; const int offsets[4] = { p->par.pack.pattern[3] + p->par.pack.pattern[2] + p->par.pack.pattern[1], @@ -759,15 +771,23 @@ static void asmgen_op_pack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p } /* Perform left shift. */ - LOOP (offset_mask, i) { i_shl(r, vl[i], vl[i], IMM(offsets[i])); CMTF("vl[%u] <<= %u;", i, offsets[i]); } - LOOP_VH(s, offset_mask, i) { i_shl(r, vh[i], vh[i], IMM(offsets[i])); CMTF("vh[%u] <<= %u;", i, offsets[i]); } + LOOP (offset_mask, i) { i_shl(r, dl[i], sl[i], IMM(offsets[i])); CMTF("vl[%u] <<= %u;", i, offsets[i]); } + LOOP_VH(s, offset_mask, i) { i_shl(r, dh[i], sh[i], IMM(offsets[i])); CMTF("vh[%u] <<= %u;", i, offsets[i]); } + LOOP (offset_mask, i) { sl[i] = dl[i]; } + LOOP_VH(s, offset_mask, i) { sh[i] = dh[i]; } /* Combine components. */ + for (int i = 0; i < 4; i++) { + sl[i] = v_16b(sl[i]); + sh[i] = v_16b(sh[i]); + dl[i] = v_16b(dl[i]); + dh[i] = v_16b(dh[i]); + } LOOP_MASK (p, i) { if (i != 0) { - i_orr16b (r, vl[0], vl[0], vl[i]); CMTF("vl[0] |= vl[%u];", i); + i_orr16b (r, dl[0], sl[0], sl[i]); CMTF("vl[0] |= vl[%u];", i); if (s->use_vh) { - i_orr16b(r, vh[0], vh[0], vh[i]); CMTF("vh[0] |= vh[%u];", i); + i_orr16b(r, dh[0], sh[0], sh[i]); CMTF("vh[0] |= vh[%u];", i); } } } @@ -782,11 +802,13 @@ static void asmgen_op_lshift(SwsAArch64Context *s, const SwsAArch64OpImplParams { uint8_t shift = p->par.shift.amount; RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; + RasmOp *sl = regs->sl; + RasmOp *sh = regs->sh; + RasmOp *dl = regs->dl; + RasmOp *dh = regs->dh; - LOOP_MASK (p, i) { i_shl(r, vl[i], vl[i], IMM(shift)); CMTF("vl[%u] <<= %u;", i, shift); } - LOOP_MASK_VH(s, p, i) { i_shl(r, vh[i], vh[i], IMM(shift)); CMTF("vh[%u] <<= %u;", i, shift); } + LOOP_MASK (p, i) { i_shl(r, dl[i], sl[i], IMM(shift)); CMTF("vl[%u] <<= %u;", i, shift); } + LOOP_MASK_VH(s, p, i) { i_shl(r, dh[i], sh[i], IMM(shift)); CMTF("vh[%u] <<= %u;", i, shift); } } /*********************************************************************/ @@ -798,11 +820,13 @@ static void asmgen_op_rshift(SwsAArch64Context *s, const SwsAArch64OpImplParams { uint8_t shift = p->par.shift.amount; RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; + RasmOp *sl = regs->sl; + RasmOp *sh = regs->sh; + RasmOp *dl = regs->dl; + RasmOp *dh = regs->dh; - LOOP_MASK (p, i) { i_ushr(r, vl[i], vl[i], IMM(shift)); CMTF("vl[%u] >>= %u;", i, shift); } - LOOP_MASK_VH(s, p, i) { i_ushr(r, vh[i], vh[i], IMM(shift)); CMTF("vh[%u] >>= %u;", i, shift); } + LOOP_MASK (p, i) { i_ushr(r, dl[i], sl[i], IMM(shift)); CMTF("vl[%u] >>= %u;", i, shift); } + LOOP_MASK_VH(s, p, i) { i_ushr(r, dh[i], sh[i], IMM(shift)); CMTF("vh[%u] >>= %u;", i, shift); } } /*********************************************************************/ @@ -854,12 +878,12 @@ static void emit_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; + RasmOp *dl = regs->dl; + RasmOp *dh = regs->dh; RasmOp *vk = regs->vk; - LOOP_MASK (p, i) { emit_clear(s, p, vl, vk, i, "vl"); } - LOOP_MASK_VH(s, p, i) { emit_clear(s, p, vh, vk, i, "vh"); } + LOOP_MASK (p, i) { emit_clear(s, p, dl, vk, i, "vl"); } + LOOP_MASK_VH(s, p, i) { emit_clear(s, p, dh, vk, i, "vh"); } } /*********************************************************************/ @@ -873,8 +897,10 @@ static void asmgen_op_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - AArch64VecViews vl[4] = A64OP_VEC_VIEWS4(regs->vl); - AArch64VecViews vh[4] = A64OP_VEC_VIEWS4(regs->vh); + AArch64VecViews sl[4] = A64OP_VEC_VIEWS4(regs->sl); + AArch64VecViews sh[4] = A64OP_VEC_VIEWS4(regs->sh); + AArch64VecViews dl[4] = A64OP_VEC_VIEWS4(regs->dl); + AArch64VecViews dh[4] = A64OP_VEC_VIEWS4(regs->dh); /** * Since each instruction in the convert operation needs specific @@ -902,50 +928,61 @@ static void asmgen_op_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams */ if (p->type == SWS_PIXEL_F32) { rasm_add_comment(r, "f32 -> u32"); - LOOP_MASK(p, i) i_fcvtzu(r, vl[i].s4, vl[i].s4); - LOOP_MASK(p, i) i_fcvtzu(r, vh[i].s4, vh[i].s4); + LOOP_MASK(p, i) i_fcvtzu(r, dl[i].s4, sl[i].s4); + LOOP_MASK(p, i) i_fcvtzu(r, dh[i].s4, sh[i].s4); + memcpy(sl, dl, sizeof(sl)); + memcpy(sh, dh, sizeof(sh)); } if (p->block_size == 8) { if (src_el_size == 1 && dst_el_size > src_el_size) { rasm_add_comment(r, "u8 -> u16"); - LOOP_MASK(p, i) i_uxtl (r, vl[i].h8, vl[i].b8); + LOOP_MASK(p, i) i_uxtl (r, dl[i].h8, sl[i].b8); + memcpy(sl, dl, sizeof(sl)); src_el_size = 2; } else if (src_el_size == 4 && dst_el_size < src_el_size) { rasm_add_comment(r, "u32 -> u16"); - LOOP_MASK(p, i) i_xtn (r, vl[i].h4, vl[i].s4); - LOOP_MASK(p, i) i_xtn (r, vh[i].h4, vh[i].s4); - LOOP_MASK(p, i) i_ins (r, vl[i].de[1], vh[i].de[0]); + LOOP_MASK(p, i) i_xtn (r, dl[i].h4, sl[i].s4); + LOOP_MASK(p, i) i_xtn (r, dh[i].h4, sh[i].s4); + LOOP_MASK(p, i) i_ins (r, dl[i].de[1], sh[i].de[0]); + memcpy(sl, dl, sizeof(sl)); + memcpy(sh, dh, sizeof(sh)); src_el_size = 2; } if (src_el_size == 2 && dst_el_size == 4) { rasm_add_comment(r, "u16 -> u32"); - LOOP_MASK(p, i) i_uxtl2(r, vh[i].s4, vl[i].h8); - LOOP_MASK(p, i) i_uxtl (r, vl[i].s4, vl[i].h4); + LOOP_MASK(p, i) i_uxtl2(r, dh[i].s4, sl[i].h8); + LOOP_MASK(p, i) i_uxtl (r, dl[i].s4, sl[i].h4); + memcpy(sl, dl, sizeof(sl)); + memcpy(sh, dh, sizeof(sh)); src_el_size = 4; } else if (src_el_size == 2 && dst_el_size == 1) { rasm_add_comment(r, "u16 -> u8"); - LOOP_MASK(p, i) i_xtn (r, vl[i].b8, vl[i].h8); + LOOP_MASK(p, i) i_xtn (r, dl[i].b8, sl[i].h8); + memcpy(sl, dl, sizeof(sl)); src_el_size = 1; } } else /* if (p->block_size == 16) */ { if (src_el_size == 1 && dst_el_size == 2) { rasm_add_comment(r, "u8 -> u16"); - LOOP_MASK(p, i) i_uxtl2(r, vh[i].h8, vl[i].b16); - LOOP_MASK(p, i) i_uxtl (r, vl[i].h8, vl[i].b8); + LOOP_MASK(p, i) i_uxtl2(r, dh[i].h8, sl[i].b16); + LOOP_MASK(p, i) i_uxtl (r, dl[i].h8, sl[i].b8); + memcpy(sl, dl, sizeof(sl)); + memcpy(sh, dh, sizeof(sh)); } else if (src_el_size == 2 && dst_el_size == 1) { rasm_add_comment(r, "u16 -> u8"); - LOOP_MASK(p, i) i_xtn (r, vl[i].b8, vl[i].h8); - LOOP_MASK(p, i) i_xtn (r, vh[i].b8, vh[i].h8); - LOOP_MASK(p, i) i_ins (r, vl[i].de[1], vh[i].de[0]); + LOOP_MASK(p, i) i_xtn (r, dl[i].b8, sl[i].h8); + LOOP_MASK(p, i) i_xtn (r, dh[i].b8, sh[i].h8); + LOOP_MASK(p, i) i_ins (r, dl[i].de[1], sh[i].de[0]); + memcpy(sl, dl, sizeof(sl)); } } /* See comment above for high vector bank usage for u32. */ if (to_type == SWS_PIXEL_F32) { rasm_add_comment(r, "u32 -> f32"); - LOOP_MASK(p, i) i_ucvtf(r, vl[i].s4, vl[i].s4); - LOOP_MASK(p, i) i_ucvtf(r, vh[i].s4, vh[i].s4); + LOOP_MASK(p, i) i_ucvtf(r, dl[i].s4, sl[i].s4); + LOOP_MASK(p, i) i_ucvtf(r, dh[i].s4, sh[i].s4); } } @@ -958,8 +995,9 @@ static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; + RasmOp *sl = regs->sl; + RasmOp *dl = regs->dl; + RasmOp *dh = regs->dh; size_t src_el_size = s->el_size; SwsPixelType to_type; @@ -980,14 +1018,15 @@ static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams if (src_el_size == 1) { rasm_add_comment(r, "u8 -> u16"); reshape_io_vectors(regs, 16, 1); - LOOP_MASK_VH(s, p, i) i_zip2(r, vh[i], vl[i], vl[i]); - LOOP_MASK (p, i) i_zip1(r, vl[i], vl[i], vl[i]); + LOOP_MASK_VH(s, p, i) i_zip2(r, dh[i], sl[i], sl[i]); + LOOP_MASK (p, i) i_zip1(r, dl[i], sl[i], sl[i]); + sl = dl; } if (dst_el_size == 4) { rasm_add_comment(r, "u16 -> u32"); reshape_io_vectors(regs, 8, 2); - LOOP_MASK_VH(s, p, i) i_zip2(r, vh[i], vl[i], vl[i]); - LOOP_MASK (p, i) i_zip1(r, vl[i], vl[i], vl[i]); + LOOP_MASK_VH(s, p, i) i_zip2(r, dh[i], sl[i], sl[i]); + LOOP_MASK (p, i) i_zip1(r, dl[i], sl[i], sl[i]); } } @@ -1011,16 +1050,18 @@ static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; + RasmOp *sl = regs->sl; + RasmOp *sh = regs->sh; + RasmOp *dl = regs->dl; + RasmOp *dh = regs->dh; RasmOp *vk = regs->vk; if (p->type == SWS_PIXEL_F32) { - LOOP_MASK (p, i) { i_fmin(r, vl[i], vl[i], vk[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); } - LOOP_MASK_VH(s, p, i) { i_fmin(r, vh[i], vh[i], vk[i]); CMTF("vh[%u] = min(vh[%u], vmin%u);", i, i, i); } + LOOP_MASK (p, i) { i_fmin(r, dl[i], sl[i], vk[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); } + LOOP_MASK_VH(s, p, i) { i_fmin(r, dh[i], sh[i], vk[i]); CMTF("vh[%u] = min(vh[%u], vmin%u);", i, i, i); } } else { - LOOP_MASK (p, i) { i_umin(r, vl[i], vl[i], vk[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); } - LOOP_MASK_VH(s, p, i) { i_umin(r, vh[i], vh[i], vk[i]); CMTF("vh[%u] = min(vh[%u], vmin%u);", i, i, i); } + LOOP_MASK (p, i) { i_umin(r, dl[i], sl[i], vk[i]); CMTF("vl[%u] = min(vl[%u], vmin%u);", i, i, i); } + LOOP_MASK_VH(s, p, i) { i_umin(r, dh[i], sh[i], vk[i]); CMTF("vh[%u] = min(vh[%u], vmin%u);", i, i, i); } } } @@ -1044,16 +1085,18 @@ static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; + RasmOp *sl = regs->sl; + RasmOp *sh = regs->sh; + RasmOp *dl = regs->dl; + RasmOp *dh = regs->dh; RasmOp *vk = regs->vk; if (p->type == SWS_PIXEL_F32) { - LOOP_MASK (p, i) { i_fmax(r, vl[i], vl[i], vk[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); } - LOOP_MASK_VH(s, p, i) { i_fmax(r, vh[i], vh[i], vk[i]); CMTF("vh[%u] = max(vh[%u], vmax%u);", i, i, i); } + LOOP_MASK (p, i) { i_fmax(r, dl[i], sl[i], vk[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); } + LOOP_MASK_VH(s, p, i) { i_fmax(r, dh[i], sh[i], vk[i]); CMTF("vh[%u] = max(vh[%u], vmax%u);", i, i, i); } } else { - LOOP_MASK (p, i) { i_umax(r, vl[i], vl[i], vk[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); } - LOOP_MASK_VH(s, p, i) { i_umax(r, vh[i], vh[i], vk[i]); CMTF("vh[%u] = max(vh[%u], vmax%u);", i, i, i); } + LOOP_MASK (p, i) { i_umax(r, dl[i], sl[i], vk[i]); CMTF("vl[%u] = max(vl[%u], vmax%u);", i, i, i); } + LOOP_MASK_VH(s, p, i) { i_umax(r, dh[i], sh[i], vk[i]); CMTF("vh[%u] = max(vh[%u], vmax%u);", i, i, i); } } } @@ -1077,16 +1120,18 @@ static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams * SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; + RasmOp *sl = regs->sl; + RasmOp *sh = regs->sh; + RasmOp *dl = regs->dl; + RasmOp *dh = regs->dh; RasmOp scale_vec = regs->vk[0]; if (p->type == SWS_PIXEL_F32) { - LOOP_MASK (p, i) { i_fmul(r, vl[i], vl[i], scale_vec); CMTF("vl[%u] *= scale_vec;", i); } - LOOP_MASK_VH(s, p, i) { i_fmul(r, vh[i], vh[i], scale_vec); CMTF("vh[%u] *= scale_vec;", i); } + LOOP_MASK (p, i) { i_fmul(r, dl[i], sl[i], scale_vec); CMTF("vl[%u] *= scale_vec;", i); } + LOOP_MASK_VH(s, p, i) { i_fmul(r, dh[i], sh[i], scale_vec); CMTF("vh[%u] *= scale_vec;", i); } } else { - LOOP_MASK (p, i) { i_mul (r, vl[i], vl[i], scale_vec); CMTF("vl[%u] *= scale_vec;", i); } - LOOP_MASK_VH(s, p, i) { i_mul (r, vh[i], vh[i], scale_vec); CMTF("vh[%u] *= scale_vec;", i); } + LOOP_MASK (p, i) { i_mul (r, dl[i], sl[i], scale_vec); CMTF("vl[%u] *= scale_vec;", i); } + LOOP_MASK_VH(s, p, i) { i_mul (r, dh[i], sh[i], scale_vec); CMTF("vh[%u] *= scale_vec;", i); } } } @@ -1134,7 +1179,7 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vt = regs->vt; RasmOp *vc = regs->vk; RasmOp *vtmp = &vt[4]; - RasmOp *vx = vh_pass ? regs->vh : regs->vl; + RasmOp *vx = vh_pass ? regs->dh : regs->dl; char cvh = vh_pass ? 'h' : 'l'; if (vh_pass && !s->use_vh) @@ -1252,8 +1297,10 @@ static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; - RasmOp *vl = regs->vl; - RasmOp *vh = regs->vh; + RasmOp *sl = regs->sl; + RasmOp *sh = regs->sh; + RasmOp *dl = regs->dl; + RasmOp *dh = regs->dh; RasmOp src_ptr = regs->dither_ptr; RasmOp ptr = s->tmp0; @@ -1360,10 +1407,12 @@ static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams i_ldp (r, dither_vlq, dither_vhq, a64op_base(ptr)); CMT("{ ditherl, ditherh } = *ptr;"); } - i_fadd (r, vl[i], vl[i], dither_vl); CMTF("vl[%u] += vditherl;", i); + i_fadd (r, dl[i], sl[i], dither_vl); CMTF("vl[%u] += vditherl;", i); if (s->use_vh) { - i_fadd(r, vh[i], vh[i], dither_vh); CMTF("vh[%u] += vditherh;", i); + i_fadd(r, dh[i], sh[i], dither_vh); CMTF("vh[%u] += vditherh;", i); } + sl = dl; + sh = dh; last_y_off = y_off; prev_i = i; @@ -1462,14 +1511,22 @@ static void asmgen_op_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask /* Vector register assignment. */ static void init_vectors_cps(SwsAArch64Context *s, SwsAArch64OpRegs *regs) { - regs->vl[0] = a64op_vec( 0); - regs->vl[1] = a64op_vec( 1); - regs->vl[2] = a64op_vec( 2); - regs->vl[3] = a64op_vec( 3); - regs->vh[0] = a64op_vec( 4); - regs->vh[1] = a64op_vec( 5); - regs->vh[2] = a64op_vec( 6); - regs->vh[3] = a64op_vec( 7); + regs->sl[0] = a64op_vec( 0); + regs->sl[1] = a64op_vec( 1); + regs->sl[2] = a64op_vec( 2); + regs->sl[3] = a64op_vec( 3); + regs->sh[0] = a64op_vec( 4); + regs->sh[1] = a64op_vec( 5); + regs->sh[2] = a64op_vec( 6); + regs->sh[3] = a64op_vec( 7); + regs->dl[0] = a64op_vec( 0); + regs->dl[1] = a64op_vec( 1); + regs->dl[2] = a64op_vec( 2); + regs->dl[3] = a64op_vec( 3); + regs->dh[0] = a64op_vec( 4); + regs->dh[1] = a64op_vec( 5); + regs->dh[2] = a64op_vec( 6); + regs->dh[3] = a64op_vec( 7); regs->vt[0] = a64op_vec(16); regs->vt[1] = a64op_vec(17); regs->vt[2] = a64op_vec(18); -- 2.52.0 >From 8a99ce8ee3ff7ca3c88aa320dc0a4366fdc3ba47 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 19 Jul 2026 19:08:25 +0200 Subject: [PATCH 18/23] swscale/aarch64/ops_asmgen: increase the number of temp vectors to 12 This change is done separately from the next commit (which uses the new temp vectors in the linear op) to make it easier to track changes to ops_neon.gen.S. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 74 +++++++++++++++++---------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 2fb9c040b1..9f7f2842fa 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -118,12 +118,12 @@ static const SwsAArch64OpEntry ops_entries[] = { /*********************************************************************/ typedef struct SwsAArch64OpRegs { - RasmOp sl[4]; /* input vector registers (low bank) */ - RasmOp sh[4]; /* input vector registers (high bank) */ - RasmOp dl[4]; /* output vector registers (low bank) */ - RasmOp dh[4]; /* output vector registers (high bank) */ - RasmOp vt[8]; /* temp vector registers */ - RasmOp vk[4]; /* constant data (may be gprs) */ + RasmOp sl[ 4]; /* input vector registers (low bank) */ + RasmOp sh[ 4]; /* input vector registers (high bank) */ + RasmOp dl[ 4]; /* output vector registers (low bank) */ + RasmOp dh[ 4]; /* output vector registers (high bank) */ + RasmOp vt[12]; /* temp vector registers */ + RasmOp vk[ 4]; /* constant data (may be gprs) */ /* Op-specific registers. */ union { @@ -202,7 +202,7 @@ static void reshape_io_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size /* Reshape temp vector registers for current SwsOp. */ static void reshape_temp_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size) { - for (int i = 0; i < 8; i++) + for (int i = 0; i < 12; i++) regs->vt[i] = a64op_make_vec(a64op_vec_n(regs->vt[i]), el_count, el_size); } @@ -1511,34 +1511,38 @@ static void asmgen_op_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask /* Vector register assignment. */ static void init_vectors_cps(SwsAArch64Context *s, SwsAArch64OpRegs *regs) { - regs->sl[0] = a64op_vec( 0); - regs->sl[1] = a64op_vec( 1); - regs->sl[2] = a64op_vec( 2); - regs->sl[3] = a64op_vec( 3); - regs->sh[0] = a64op_vec( 4); - regs->sh[1] = a64op_vec( 5); - regs->sh[2] = a64op_vec( 6); - regs->sh[3] = a64op_vec( 7); - regs->dl[0] = a64op_vec( 0); - regs->dl[1] = a64op_vec( 1); - regs->dl[2] = a64op_vec( 2); - regs->dl[3] = a64op_vec( 3); - regs->dh[0] = a64op_vec( 4); - regs->dh[1] = a64op_vec( 5); - regs->dh[2] = a64op_vec( 6); - regs->dh[3] = a64op_vec( 7); - regs->vt[0] = a64op_vec(16); - regs->vt[1] = a64op_vec(17); - regs->vt[2] = a64op_vec(18); - regs->vt[3] = a64op_vec(19); - regs->vt[4] = a64op_vec(20); - regs->vt[5] = a64op_vec(21); - regs->vt[6] = a64op_vec(22); - regs->vt[7] = a64op_vec(23); - regs->vk[0] = a64op_vec(24); - regs->vk[1] = a64op_vec(25); - regs->vk[2] = a64op_vec(26); - regs->vk[3] = a64op_vec(27); + regs->sl[ 0] = a64op_vec( 0); + regs->sl[ 1] = a64op_vec( 1); + regs->sl[ 2] = a64op_vec( 2); + regs->sl[ 3] = a64op_vec( 3); + regs->sh[ 0] = a64op_vec( 4); + regs->sh[ 1] = a64op_vec( 5); + regs->sh[ 2] = a64op_vec( 6); + regs->sh[ 3] = a64op_vec( 7); + regs->dl[ 0] = a64op_vec( 0); + regs->dl[ 1] = a64op_vec( 1); + regs->dl[ 2] = a64op_vec( 2); + regs->dl[ 3] = a64op_vec( 3); + regs->dh[ 0] = a64op_vec( 4); + regs->dh[ 1] = a64op_vec( 5); + regs->dh[ 2] = a64op_vec( 6); + regs->dh[ 3] = a64op_vec( 7); + regs->vt[ 0] = a64op_vec(16); + regs->vt[ 1] = a64op_vec(17); + regs->vt[ 2] = a64op_vec(18); + regs->vt[ 3] = a64op_vec(19); + regs->vt[ 4] = a64op_vec(20); + regs->vt[ 5] = a64op_vec(21); + regs->vt[ 6] = a64op_vec(22); + regs->vt[ 7] = a64op_vec(23); + regs->vt[ 8] = a64op_vec(24); + regs->vt[ 9] = a64op_vec(25); + regs->vt[10] = a64op_vec(26); + regs->vt[11] = a64op_vec(27); + regs->vk[ 0] = a64op_vec(28); + regs->vk[ 1] = a64op_vec(29); + regs->vk[ 2] = a64op_vec(30); + regs->vk[ 3] = a64op_vec(31); } /*********************************************************************/ -- 2.52.0 >From f485cbad5c5ff4f8aed5436055060efac94a6e8c Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sat, 18 Jul 2026 03:27:10 +0200 Subject: [PATCH 19/23] swscale/aarch64/ops_asmgen: move save_mask setup out of asmgen_op_linear() and into asmgen_setup_linear() The save_mask setup makes copies of input data that would be clobbered by the linear operation itself. This setup is not entirely necessary with JIT, since we may have separate input and output vector registers, so the code is moved into the setup function which will be CPS-only. This doubles the amount of temp registers needed, since they must all be copied before both loops. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 196 ++++++++++++++++---------------- 1 file changed, 98 insertions(+), 98 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 9f7f2842fa..8916c2e4f0 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -1144,7 +1144,10 @@ static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplPara SwsAArch64OpRegs *regs) { RasmContext *r = s->rctx; + RasmOp *sl = regs->sl; + RasmOp *sh = regs->sh; RasmOp *vc = regs->vk; + RasmOp *vt = regs->vt; RasmOp ptr = s->tmp0; RasmOp coeff_veclist; @@ -1161,103 +1164,7 @@ static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplPara i_ldr(r, ptr, s->impl_priv); CMT("v128 *vcoeff_ptr = impl->priv.ptr;"); asmgen_set_load_cont_node(s); i_ld1(r, coeff_veclist, a64op_base(ptr)); CMT("coeff_veclist = *vcoeff_ptr;"); -} -/** - * Performs one pass of the linear transform over a single vector bank - * (low or high). - */ -static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs, - SwsCompMask save_mask, bool vh_pass) -{ - RasmContext *r = s->rctx; - /** - * The intermediate registers for fmul+fadd (for when SWS_BITEXACT - * is set) start from temp vector 4. - */ - RasmOp *vt = regs->vt; - RasmOp *vc = regs->vk; - RasmOp *vtmp = &vt[4]; - RasmOp *vx = vh_pass ? regs->dh : regs->dl; - char cvh = vh_pass ? 'h' : 'l'; - - if (vh_pass && !s->use_vh) - return; - - /** - * Save rows that need to be used as input after they have been already - * written to. - */ - RasmOp src_vx[4] = { vx[0], vx[1], vx[2], vx[3] }; - if (save_mask) { - for (int i = 0; i < 4; i++) { - if (save_mask & SWS_COMP(i)) { - src_vx[i] = vt[i]; - i_mov16b(r, vt[i], vx[i]); CMTF("vsrc[%u] = v%c[%u];", i, cvh, i); - } - } - } - - /** - * The non-zero coefficients have been packed in aarch64_setup_linear() - * in sequential order into the individual lanes of the coefficient - * vector registers. We must follow the same order of execution here. - */ - int i_coeff = 0; - LOOP_MASK(p, i) { - bool first = true; - RasmNode *pre_mul = rasm_get_current_node(r); - for (int j = 0; j < 5; j++) { - bool is_offset = (j == 0); - int src_j = is_offset ? 4 : (j - 1); - if (p->par.lin.zero & SWS_MASK(i, src_j)) - continue; - RasmOp vsrc = src_vx[src_j]; - uint8_t vc_i = i_coeff / 4; - uint8_t vc_j = i_coeff & 3; - RasmOp vcoeff = a64op_elem(vc[vc_i], vc_j); - i_coeff++; - if (first && is_offset) { - i_dup (r, vx[i], vcoeff); CMTF("v%c[%u] = broadcast(vc[%u][%u]);", cvh, i, vc_i, vc_j); - } else if (first && !is_offset) { - if (p->par.lin.one & SWS_MASK(i, src_j)) { - i_mov16b(r, vx[i], vsrc); CMTF("v%c[%u] = vsrc[%u];", cvh, i, src_j); - } else { - i_fmul (r, vx[i], vsrc, vcoeff); CMTF("v%c[%u] = vsrc[%u] * vc[%u][%u];", cvh, i, src_j, vc_i, vc_j); - } - } else if (p->uop == SWS_UOP_LINEAR_FMA) { - /** - * Most modern aarch64 cores have a fastpath for sequences - * of fmla instructions. This means that even if the coefficient - * is 1, it is still faster to use fmla by 1 instead of fadd. - */ - i_fmla(r, vx[i], vsrc, vcoeff); CMTF("v%c[%u] += vsrc[%u] * vc[%u][%u];", cvh, i, src_j, vc_i, vc_j); - } else { - /** - * Split the multiply-accumulate into fmul+fadd. All - * multiplications are performed first into temporary - * registers, and only then added to the destination, - * to reduce the dependency chain. - * There is no need to perform multiplications by 1. - */ - if (!(p->par.lin.one & SWS_MASK(i, src_j))) { - pre_mul = rasm_set_current_node(r, pre_mul); - i_fmul(r, vtmp[vc_j], vsrc, vcoeff); CMTF("vtmp[%u] = vsrc[%u] * vc[%u][%u];", vc_j, src_j, vc_i, vc_j); - pre_mul = rasm_set_current_node(r, pre_mul); - i_fadd(r, vx[i], vx[i], vtmp[vc_j]); CMTF("v%c[%u] += vtmp[%u];", cvh, i, vc_j); - } else { - i_fadd(r, vx[i], vx[i], vsrc); CMTF("v%c[%u] += vsrc[%u];", cvh, i, vc_j); - } - } - first = false; - } - } -} - -static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ /* Compute mask for rows that must be saved before being overwritten. */ SwsCompMask save_mask = 0; bool overwritten[4] = { false, false, false, false }; @@ -1273,9 +1180,102 @@ static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams } } + /** + * Save rows that need to be used as input after they have been already + * written to. + */ + RasmOp *tl = &vt[0]; + RasmOp *th = &vt[4]; + LOOP (save_mask, i) { i_mov16b(r, tl[i], sl[i]); CMTF("vsrcl[%u] = vl[%u];", i, i); } + LOOP_VH(s, save_mask, i) { i_mov16b(r, th[i], sh[i]); CMTF("vsrch[%u] = vh[%u];", i, i); } + LOOP (save_mask, i) { sl[i] = tl[i]; } + LOOP_VH(s, save_mask, i) { sh[i] = th[i]; } +} + +/** + * Performs one pass of the linear transform over a single vector bank + * (low or high). + */ +static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs, bool vh_pass) +{ + RasmContext *r = s->rctx; + /** + * The intermediate registers for fmul+fadd (for when SWS_BITEXACT + * is set) start from temp vector 8. + */ + RasmOp *vt = regs->vt; + RasmOp *vc = regs->vk; + RasmOp *vtmp = &vt[8]; + RasmOp *sx = vh_pass ? regs->sh : regs->sl; + RasmOp *dx = vh_pass ? regs->dh : regs->dl; + char cvh = vh_pass ? 'h' : 'l'; + + if (vh_pass && !s->use_vh) + return; + + /** + * The non-zero coefficients have been packed in aarch64_setup_linear() + * in sequential order into the individual lanes of the coefficient + * vector registers. We must follow the same order of execution here. + */ + int i_coeff = 0; + LOOP_MASK(p, i) { + bool first = true; + RasmNode *pre_mul = rasm_get_current_node(r); + for (int j = 0; j < 5; j++) { + bool is_offset = (j == 0); + int src_j = is_offset ? 4 : (j - 1); + if (p->par.lin.zero & SWS_MASK(i, src_j)) + continue; + RasmOp vsrc = sx[src_j]; + uint8_t vc_i = i_coeff / 4; + uint8_t vc_j = i_coeff & 3; + RasmOp vcoeff = a64op_elem(vc[vc_i], vc_j); + i_coeff++; + if (first && is_offset) { + i_dup (r, dx[i], vcoeff); CMTF("v%c[%u] = broadcast(vc[%u][%u]);", cvh, i, vc_i, vc_j); + } else if (first && !is_offset) { + if (p->par.lin.one & SWS_MASK(i, src_j)) { + i_mov16b(r, dx[i], vsrc); CMTF("v%c[%u] = vsrc%c[%u];", cvh, i, cvh, src_j); + } else { + i_fmul (r, dx[i], vsrc, vcoeff); CMTF("v%c[%u] = vsrc%c[%u] * vc[%u][%u];", cvh, i, cvh, src_j, vc_i, vc_j); + } + } else if (p->uop == SWS_UOP_LINEAR_FMA) { + /** + * Most modern aarch64 cores have a fastpath for sequences + * of fmla instructions. This means that even if the coefficient + * is 1, it is still faster to use fmla by 1 instead of fadd. + */ + i_fmla(r, dx[i], vsrc, vcoeff); CMTF("v%c[%u] += vsrc%c[%u] * vc[%u][%u];", cvh, i, cvh, src_j, vc_i, vc_j); + } else { + /** + * Split the multiply-accumulate into fmul+fadd. All + * multiplications are performed first into temporary + * registers, and only then added to the destination, + * to reduce the dependency chain. + * There is no need to perform multiplications by 1. + */ + if (!(p->par.lin.one & SWS_MASK(i, src_j))) { + pre_mul = rasm_set_current_node(r, pre_mul); + i_fmul(r, vtmp[vc_j], vsrc, vcoeff); CMTF("vtmp[%u] = vsrc%c[%u] * vc[%u][%u];", vc_j, cvh, src_j, vc_i, vc_j); + pre_mul = rasm_set_current_node(r, pre_mul); + i_fadd(r, dx[i], dx[i], vtmp[vc_j]); CMTF("v%c[%u] += vtmp[%u];", cvh, i, vc_j); + } else { + i_fadd(r, dx[i], dx[i], vsrc); CMTF("v%c[%u] += vsrc%c[%u];", cvh, i, cvh, vc_j); + } + } + first = false; + } + } +} + +static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ /* Perform linear passes for low and high vector banks. */ - linear_pass(s, p, regs, save_mask, false); - linear_pass(s, p, regs, save_mask, true); + linear_pass(s, p, regs, false); + linear_pass(s, p, regs, true); } /*********************************************************************/ -- 2.52.0 >From 49e5a995fd4653b7171558c2a4101a3d2c5848e4 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 19 Jul 2026 21:02:09 +0200 Subject: [PATCH 20/23] swscale/aarch64/ops_asmgen: reword annotations for linear operation Refer to offset[i] or coeff[i][j] instead of position of the value from the packed data. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 8916c2e4f0..9c534683a3 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -1234,12 +1234,12 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp vcoeff = a64op_elem(vc[vc_i], vc_j); i_coeff++; if (first && is_offset) { - i_dup (r, dx[i], vcoeff); CMTF("v%c[%u] = broadcast(vc[%u][%u]);", cvh, i, vc_i, vc_j); + i_dup (r, dx[i], vcoeff); CMTF("v%c[%u] = broadcast(offset[%u]);", cvh, i, i); } else if (first && !is_offset) { if (p->par.lin.one & SWS_MASK(i, src_j)) { i_mov16b(r, dx[i], vsrc); CMTF("v%c[%u] = vsrc%c[%u];", cvh, i, cvh, src_j); } else { - i_fmul (r, dx[i], vsrc, vcoeff); CMTF("v%c[%u] = vsrc%c[%u] * vc[%u][%u];", cvh, i, cvh, src_j, vc_i, vc_j); + i_fmul (r, dx[i], vsrc, vcoeff); CMTF("v%c[%u] = vsrc%c[%u] * coeff[%u][%u];", cvh, i, cvh, src_j, i, src_j); } } else if (p->uop == SWS_UOP_LINEAR_FMA) { /** @@ -1247,7 +1247,7 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, * of fmla instructions. This means that even if the coefficient * is 1, it is still faster to use fmla by 1 instead of fadd. */ - i_fmla(r, dx[i], vsrc, vcoeff); CMTF("v%c[%u] += vsrc%c[%u] * vc[%u][%u];", cvh, i, cvh, src_j, vc_i, vc_j); + i_fmla(r, dx[i], vsrc, vcoeff); CMTF("v%c[%u] += vsrc%c[%u] * coeff[%u][%u];", cvh, i, cvh, src_j, i, src_j); } else { /** * Split the multiply-accumulate into fmul+fadd. All @@ -1258,7 +1258,7 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, */ if (!(p->par.lin.one & SWS_MASK(i, src_j))) { pre_mul = rasm_set_current_node(r, pre_mul); - i_fmul(r, vtmp[vc_j], vsrc, vcoeff); CMTF("vtmp[%u] = vsrc%c[%u] * vc[%u][%u];", vc_j, cvh, src_j, vc_i, vc_j); + i_fmul(r, vtmp[vc_j], vsrc, vcoeff); CMTF("vtmp[%u] = vsrc%c[%u] * coeff[%u][%u];", vc_j, cvh, src_j, i, src_j); pre_mul = rasm_set_current_node(r, pre_mul); i_fadd(r, dx[i], dx[i], vtmp[vc_j]); CMTF("v%c[%u] += vtmp[%u];", cvh, i, vc_j); } else { -- 2.52.0 >From 5b7f37c268e068e9c8eb0374d463fbf07bfb5228 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 19 Jul 2026 21:04:02 +0200 Subject: [PATCH 21/23] swscale/aarch64/ops_asmgen: use different value for vtmp register index Use vtmp register index relative to the current matrix column instead of reusing the position of the value from the packed data. The i_coeff and derived values will be removed in the following commit. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 9c534683a3..79cbb7cc5d 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -1258,11 +1258,11 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, */ if (!(p->par.lin.one & SWS_MASK(i, src_j))) { pre_mul = rasm_set_current_node(r, pre_mul); - i_fmul(r, vtmp[vc_j], vsrc, vcoeff); CMTF("vtmp[%u] = vsrc%c[%u] * coeff[%u][%u];", vc_j, cvh, src_j, i, src_j); + i_fmul(r, vtmp[src_j], vsrc, vcoeff); CMTF("vtmp[%u] = vsrc%c[%u] * coeff[%u][%u];", src_j, cvh, src_j, i, src_j); pre_mul = rasm_set_current_node(r, pre_mul); - i_fadd(r, dx[i], dx[i], vtmp[vc_j]); CMTF("v%c[%u] += vtmp[%u];", cvh, i, vc_j); + i_fadd(r, dx[i], dx[i], vtmp[src_j]); CMTF("v%c[%u] += vtmp[%u];", cvh, i, src_j); } else { - i_fadd(r, dx[i], dx[i], vsrc); CMTF("v%c[%u] += vsrc%c[%u];", cvh, i, cvh, vc_j); + i_fadd(r, dx[i], dx[i], vsrc); CMTF("v%c[%u] += vsrc%c[%u];", cvh, i, cvh, src_j); } } first = false; -- 2.52.0 >From c0deb4579baa79fa6df1463f114de8b2bf92cb44 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 19 Jul 2026 20:23:45 +0200 Subject: [PATCH 22/23] swscale/aarch64/ops_asmgen: populate operands matrix in asmgen_setup_linear() This will serve multiple purposes: - Allow JIT code to populate operands from a deduplicated data pool; - Simplify the future implementation of integer linear operation, since mul/mla take full vector registers as coefficients, and not elements. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 79cbb7cc5d..78bc703b38 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -128,6 +128,7 @@ typedef struct SwsAArch64OpRegs { /* Op-specific registers. */ union { RasmOp dither_ptr; + RasmOp linear_vcoeff[4][5]; }; } SwsAArch64OpRegs; @@ -1165,15 +1166,23 @@ static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplPara asmgen_set_load_cont_node(s); i_ld1(r, coeff_veclist, a64op_base(ptr)); CMT("coeff_veclist = *vcoeff_ptr;"); - /* Compute mask for rows that must be saved before being overwritten. */ + /** + * Populate operands matrix from packed data into linear_vcoeff matrix + * and compute mask for rows that must be saved before being overwritten. + */ SwsCompMask save_mask = 0; bool overwritten[4] = { false, false, false, false }; + int i_coeff = 0; LOOP_MASK(p, i) { for (int j = 0; j < 5; j++) { bool is_offset = (j == 0); int src_j = is_offset ? 4 : (j - 1); if (p->par.lin.zero & SWS_MASK(i, src_j)) continue; + uint8_t vc_i = i_coeff / 4; + uint8_t vc_j = i_coeff & 3; + regs->linear_vcoeff[i][j] = a64op_elem(vc[vc_i], vc_j); + i_coeff++; if (!is_offset && overwritten[src_j]) save_mask |= SWS_COMP(src_j); overwritten[i] = true; @@ -1205,7 +1214,6 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, * is set) start from temp vector 8. */ RasmOp *vt = regs->vt; - RasmOp *vc = regs->vk; RasmOp *vtmp = &vt[8]; RasmOp *sx = vh_pass ? regs->sh : regs->sl; RasmOp *dx = vh_pass ? regs->dh : regs->dl; @@ -1219,7 +1227,6 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, * in sequential order into the individual lanes of the coefficient * vector registers. We must follow the same order of execution here. */ - int i_coeff = 0; LOOP_MASK(p, i) { bool first = true; RasmNode *pre_mul = rasm_get_current_node(r); @@ -1229,10 +1236,7 @@ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, if (p->par.lin.zero & SWS_MASK(i, src_j)) continue; RasmOp vsrc = sx[src_j]; - uint8_t vc_i = i_coeff / 4; - uint8_t vc_j = i_coeff & 3; - RasmOp vcoeff = a64op_elem(vc[vc_i], vc_j); - i_coeff++; + RasmOp vcoeff = regs->linear_vcoeff[i][j]; if (first && is_offset) { i_dup (r, dx[i], vcoeff); CMTF("v%c[%u] = broadcast(offset[%u]);", cvh, i, i); } else if (first && !is_offset) { -- 2.52.0 >From 151a7276d4945cb2e01b3c19c40b627a809a665c Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Mon, 6 Jul 2026 23:59:58 +0200 Subject: [PATCH 23/23] swscale/aarch64: split ops_static.c out of ops_asmgen.c ops_asmgen.c contains common aarch64 asm generation code that will be used by CPS backend and the JIT backend. The standalone tool with CPS-specific code is now ops_static.c. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/.gitignore | 2 +- libswscale/aarch64/Makefile | 4 +- libswscale/aarch64/ops.c | 4 +- libswscale/aarch64/ops_asmgen.c | 700 +------------------------------- libswscale/aarch64/ops_asmgen.h | 85 ++++ libswscale/aarch64/ops_static.c | 663 ++++++++++++++++++++++++++++++ 6 files changed, 754 insertions(+), 704 deletions(-) create mode 100644 libswscale/aarch64/ops_asmgen.h create mode 100644 libswscale/aarch64/ops_static.c diff --git a/libswscale/aarch64/.gitignore b/libswscale/aarch64/.gitignore index 667c77c085..949c25f5c0 100644 --- a/libswscale/aarch64/.gitignore +++ b/libswscale/aarch64/.gitignore @@ -1 +1 @@ -/ops_asmgen +/ops_static diff --git a/libswscale/aarch64/Makefile b/libswscale/aarch64/Makefile index e311f5446a..8fdcf000cc 100644 --- a/libswscale/aarch64/Makefile +++ b/libswscale/aarch64/Makefile @@ -14,7 +14,7 @@ NEON-OBJS += aarch64/hscale.o \ NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops.o NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops_neon.gen.o -$(SUBDIR)aarch64/ops_neon.gen.S: $(SUBDIR)aarch64/ops_asmgen$(HOSTEXESUF) +$(SUBDIR)aarch64/ops_neon.gen.S: $(SUBDIR)aarch64/ops_static$(HOSTEXESUF) $(M)$< > [email protected] $(CP) [email protected] $@ $(RM) [email protected] @@ -22,4 +22,4 @@ $(SUBDIR)aarch64/ops_neon.gen.S: $(SUBDIR)aarch64/ops_asmgen$(HOSTEXESUF) clean:: $(RM) $(CLEANSUFFIXES:%=libswscale/aarch64/%) -HOSTPROGS = aarch64/ops_asmgen +HOSTPROGS = aarch64/ops_static diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index 6e1de6a458..96058a282f 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -28,9 +28,9 @@ /** * Check that there is no mismatch for the SwsOpExec/SwsOpImpl offset - * values used by ops_asmgen. + * values used by ops_static. * NOTE: The check is performed here since this file only ever targets - * aarch64, differently from ops_asmgen which may be built on any + * aarch64, differently from ops_static which may be built on any * host. */ static_assert(offsetof_exec_in == offsetof(SwsOpExec, in), "SwsOpExec layout mismatch"); diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 78bc703b38..3d3cc9155a 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -18,164 +18,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -#include <assert.h> -#include <limits.h> -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#ifdef _WIN32 -#include <io.h> -#include <fcntl.h> -#endif - -/** - * This file is compiled as a standalone build-time tool and must not depend - * on internal FFmpeg libraries. The necessary utils are redefined below using - * standard C equivalents. - */ - -#define AVUTIL_AVASSERT_H -#define AVUTIL_LOG_H -#define AVUTIL_MACROS_H -#define AVUTIL_MEM_H -#define av_assert0(cond) assert(cond) -#define av_malloc(s) malloc(s) -#define av_mallocz(s) calloc(1, s) -#define av_realloc(p, s) realloc(p, s) -#define av_strdup(s) strdup(s) -#define av_free(p) free(p) -#define FFMAX(a,b) ((a) > (b) ? (a) : (b)) -#define FFMIN(a,b) ((a) > (b) ? (b) : (a)) -#define MKTAG(a,b,c,d) ((a) | ((b) << 8) | ((c) << 16) | ((unsigned)(d) << 24)) - -static void av_freep(void *ptr) -{ - void **pptr = (void **) ptr; - if (pptr) { - ptr = *pptr; - if (ptr) - free(ptr); - *pptr = NULL; - } -} - -static void *av_memdup(const void *p, size_t size) -{ - void *ptr = NULL; - if (p) { - ptr = av_malloc(size); - if (ptr) - memcpy(ptr, p, size); - } - return ptr; -} - -#include "libavutil/dynarray.h" - -static void *av_dynarray2_add(void **tab_ptr, int *nb_ptr, size_t elem_size, - const uint8_t *elem_data) -{ - uint8_t *tab_elem_data = NULL; - - FF_DYNARRAY_ADD(INT_MAX, elem_size, *tab_ptr, *nb_ptr, { - tab_elem_data = (uint8_t *)*tab_ptr + (*nb_ptr) * elem_size; - if (elem_data) - memcpy(tab_elem_data, elem_data, elem_size); - }, { - av_freep(tab_ptr); - *nb_ptr = 0; - }); - return tab_elem_data; -} - -#include "libavutil/bprint.c" - -/*********************************************************************/ -#include "rasm.c" -#include "rasm_print.c" -#include "ops_impl.h" - -/** - * Implementation parameters for all exported functions. This list is - * compiled by performing a dummy run of all conversions in sws_ops and - * collecting all functions that need to be generated. This is achieved - * by running: - * make fate-sws-ops-entries-aarch64 GEN=1 - */ -typedef struct SwsAArch64OpEntry { - const char *name; - SwsAArch64OpImplParams params; -} SwsAArch64OpEntry; - -static const SwsAArch64OpEntry ops_entries[] = { -#define ENTRY(fname, ...) { .name = #fname, .params = __VA_ARGS__ }, -#include "ops_entries.c" -#undef ENTRY - { NULL } -}; - -/*********************************************************************/ -typedef struct SwsAArch64OpRegs { - RasmOp sl[ 4]; /* input vector registers (low bank) */ - RasmOp sh[ 4]; /* input vector registers (high bank) */ - RasmOp dl[ 4]; /* output vector registers (low bank) */ - RasmOp dh[ 4]; /* output vector registers (high bank) */ - RasmOp vt[12]; /* temp vector registers */ - RasmOp vk[ 4]; /* constant data (may be gprs) */ - - /* Op-specific registers. */ - union { - RasmOp dither_ptr; - RasmOp linear_vcoeff[4][5]; - }; -} SwsAArch64OpRegs; - -/*********************************************************************/ -typedef struct SwsAArch64Context { - RasmContext *rctx; - - /* SwsOpFunc arguments. */ - RasmOp exec; - RasmOp impl; - RasmOp bx_start; - RasmOp y_start; - RasmOp bx_end; - RasmOp y_end; - - /* Loop iterator variables. */ - RasmOp bx; - RasmOp y; - - /* Scratch registers. */ - RasmOp tmp0; - RasmOp tmp1; - - /* CPS-related variables. */ - RasmOp op0_func; - RasmOp op1_impl; - RasmOp cont; - RasmOp impl_priv; - RasmNode *load_cont_node; - SwsAArch64OpRegs regs; - - /* Read/Write data pointers and padding. */ - RasmOp in[4]; - RasmOp out[4]; - RasmOp in_bump[4]; - RasmOp out_bump[4]; - - /* Process function. */ - RasmNode *setup; - RasmNode *loop; - - /* Vector register dimensions. */ - size_t el_size; - size_t el_count; - size_t vec_size; - bool use_vh; -} SwsAArch64Context; +#include "ops_asmgen.h" /*********************************************************************/ /* Helpers functions. */ @@ -362,18 +205,6 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask i_ret(r); } -/*********************************************************************/ -/** - * Set node where the continuation address will be loaded and impl will - * be incremented. This should be done right after impl->priv has been - * used. - */ -static void asmgen_set_load_cont_node(SwsAArch64Context *s) -{ - RasmContext *r = s->rctx; - s->load_cont_node = rasm_get_current_node(r); -} - /*********************************************************************/ /* gather raw pixels from planes */ /* SWS_UOP_READ_BIT */ @@ -381,23 +212,6 @@ static void asmgen_set_load_cont_node(SwsAArch64Context *s) /* SWS_UOP_READ_PACKED */ /* SWS_UOP_READ_PLANAR */ -static void asmgen_setup_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ - RasmContext *r = s->rctx; - AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); - AArch64VecViews bitmask_vec = a64op_vec_views(regs->vk[1]); - - rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); - i_ldr(r, shift_vec.q, s->impl_priv); - asmgen_set_load_cont_node(s); - if (p->block_size == 16) { - i_movi(r, bitmask_vec.b16, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 16 times>};"); - } else { - i_movi(r, bitmask_vec.b8, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 8 times>, 0 <repeats 8 times>};"); - } -} - static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { @@ -427,16 +241,6 @@ static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParam } } -static void asmgen_setup_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ - RasmContext *r = s->rctx; - AArch64VecViews nibble_mask = a64op_vec_views(regs->vk[0]); - - rasm_annotate_next(r, "v128 nibble_mask = {0xf <repeats 8 times>, 0x0 <repeats 8 times>};"); - i_movi(r, nibble_mask.b8, IMM(0x0f)); -} - static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { @@ -503,17 +307,6 @@ static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplPa /* SWS_UOP_WRITE_PACKED */ /* SWS_UOP_WRITE_PLANAR */ -static void asmgen_setup_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ - RasmContext *r = s->rctx; - AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); - - rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); - i_ldr(r, shift_vec.q, s->impl_priv); - asmgen_set_load_cont_node(s); -} - static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { @@ -671,43 +464,6 @@ static void asmgen_op_move(SwsAArch64Context *s, const SwsAArch64OpImplParams *p /* split tightly packed data into components */ /* SWS_UOP_UNPACK */ -static void asmgen_setup_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ - RasmContext *r = s->rctx; - RasmOp *vmask = regs->vk; - RasmOp mask_gpr = a64op_w(s->tmp0); - uint32_t mask_val[4] = { 0 }; - - /* Generate masks. */ - rasm_add_comment(r, "generate masks"); - LOOP_MASK(p, i) { - uint32_t val = (1u << p->par.pack.pattern[i]) - 1; - for (int j = 0; j < 4; j++) { - if (mask_val[j] == val) { - mask_val[i] = mask_val[j]; - vmask[i] = vmask[j]; - break; - } - } - if (!mask_val[i]) { - /** - * All-one values in movi only work up to 8-bit, and then - * at full 16- or 32-bit, but not for intermediate values - * like 10-bit. In those cases, we use mov + dup instead. - */ - if (val <= 0xff || val == 0xffff) { - i_movi(r, vmask[i], IMM(val)); - } else { - i_mov (r, mask_gpr, IMM(val)); - i_dup (r, vmask[i], mask_gpr); - } - mask_val[i] = val; - vmask[i] = v_16b(vmask[i]); - } - } -} - static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { @@ -834,29 +590,6 @@ static void asmgen_op_rshift(SwsAArch64Context *s, const SwsAArch64OpImplParams /* clear pixel values */ /* SWS_UOP_CLEAR */ -static void asmgen_setup_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ - RasmContext *r = s->rctx; - RasmOp *vk = regs->vk; - - /** - * TODO - * - pack elements in impl->priv and perform smaller loads - * - if only 1 element and not vh, load directly with ld1r - */ - - bool load_priv = false; - LOOP_MASK(p, i) { - if (!((p->par.clear.zero | p->par.clear.one) & SWS_COMP(i))) - load_priv = true; - } - if (load_priv) { - i_ldr(r, v_q(vk[0]), s->impl_priv); CMT("v128 clear_vec = impl->priv.v128;"); - asmgen_set_load_cont_node(s); - } -} - static void emit_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, RasmOp *vx, RasmOp *vk, int i, const char *vx_str) { @@ -1035,18 +768,6 @@ static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams /* numeric minimum */ /* SWS_UOP_MIN */ -static void asmgen_setup_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ - RasmContext *r = s->rctx; - RasmOp *vk = regs->vk; - - RasmOp min_vec = regs->vt[0]; - i_ldr(r, v_q(min_vec), s->impl_priv); CMT("v128 min_vec = impl->priv.v128;"); - asmgen_set_load_cont_node(s); - LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(min_vec, i)); CMTF("v128 vmin%u = min_vec[%u];", i, i); } -} - static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { @@ -1070,18 +791,6 @@ static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, /* numeric maximum */ /* SWS_UOP_MAX */ -static void asmgen_setup_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ - RasmContext *r = s->rctx; - RasmOp *vk = regs->vk; - - RasmOp max_vec = regs->vt[0]; - i_ldr(r, v_q(max_vec), s->impl_priv); CMT("v128 max_vec = impl->priv.v128;"); - asmgen_set_load_cont_node(s); - LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(max_vec, i)); CMTF("v128 vmax%u = max_vec[%u];", i, i); } -} - static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { @@ -1105,18 +814,6 @@ static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, /* multiplication by scalar */ /* SWS_UOP_SCALE */ -static void asmgen_setup_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ - RasmContext *r = s->rctx; - RasmOp scale_vec = regs->vk[0]; - - RasmOp priv_ptr = s->tmp0; - i_add (r, priv_ptr, s->impl, IMM(offsetof_impl_priv)); CMT("v128 *scale_vec_ptr = &impl->priv;"); - asmgen_set_load_cont_node(s); - i_ld1r(r, vv_1(scale_vec), a64op_base(priv_ptr)); CMT("v128 scale_vec = broadcast(*scale_vec_ptr);"); -} - static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { @@ -1141,70 +838,6 @@ static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams * /* SWS_UOP_LINEAR */ /* SWS_UOP_LINEAR_FMA */ -static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ - RasmContext *r = s->rctx; - RasmOp *sl = regs->sl; - RasmOp *sh = regs->sh; - RasmOp *vc = regs->vk; - RasmOp *vt = regs->vt; - - RasmOp ptr = s->tmp0; - RasmOp coeff_veclist; - - /* Preload coefficients from impl->priv. */ - const int num_vregs = linear_num_vregs(p); - av_assert0(num_vregs <= 4); - switch (num_vregs) { - case 1: coeff_veclist = vv_1(vc[0]); break; - case 2: coeff_veclist = vv_2(vc[0], vc[1]); break; - case 3: coeff_veclist = vv_3(vc[0], vc[1], vc[2]); break; - case 4: coeff_veclist = vv_4(vc[0], vc[1], vc[2], vc[3]); break; - } - i_ldr(r, ptr, s->impl_priv); CMT("v128 *vcoeff_ptr = impl->priv.ptr;"); - asmgen_set_load_cont_node(s); - i_ld1(r, coeff_veclist, a64op_base(ptr)); CMT("coeff_veclist = *vcoeff_ptr;"); - - /** - * Populate operands matrix from packed data into linear_vcoeff matrix - * and compute mask for rows that must be saved before being overwritten. - */ - SwsCompMask save_mask = 0; - bool overwritten[4] = { false, false, false, false }; - int i_coeff = 0; - LOOP_MASK(p, i) { - for (int j = 0; j < 5; j++) { - bool is_offset = (j == 0); - int src_j = is_offset ? 4 : (j - 1); - if (p->par.lin.zero & SWS_MASK(i, src_j)) - continue; - uint8_t vc_i = i_coeff / 4; - uint8_t vc_j = i_coeff & 3; - regs->linear_vcoeff[i][j] = a64op_elem(vc[vc_i], vc_j); - i_coeff++; - if (!is_offset && overwritten[src_j]) - save_mask |= SWS_COMP(src_j); - overwritten[i] = true; - } - } - - /** - * Save rows that need to be used as input after they have been already - * written to. - */ - RasmOp *tl = &vt[0]; - RasmOp *th = &vt[4]; - LOOP (save_mask, i) { i_mov16b(r, tl[i], sl[i]); CMTF("vsrcl[%u] = vl[%u];", i, i); } - LOOP_VH(s, save_mask, i) { i_mov16b(r, th[i], sh[i]); CMTF("vsrch[%u] = vh[%u];", i, i); } - LOOP (save_mask, i) { sl[i] = tl[i]; } - LOOP_VH(s, save_mask, i) { sh[i] = th[i]; } -} - -/** - * Performs one pass of the linear transform over a single vector bank - * (low or high). - */ static void linear_pass(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs, bool vh_pass) { @@ -1286,17 +919,6 @@ static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams /* add dithering noise */ /* SWS_UOP_DITHER */ -static void asmgen_setup_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, - SwsAArch64OpRegs *regs) -{ - RasmContext *r = s->rctx; - RasmOp src_ptr = s->tmp0; - - regs->dither_ptr = src_ptr; - i_ldr(r, src_ptr, s->impl_priv); CMT("void *ptr = impl->priv.ptr;"); - asmgen_set_load_cont_node(s); -} - static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs) { @@ -1422,323 +1044,3 @@ static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams prev_i = i; } } - -/*********************************************************************/ -/** - * Register assignment for CPS functions. - * - * The entry point of the SwsOpFunc is the `process` function. The - * first kernel function is called from `process`, and subsequent - * kernel functions are chained by directly branching to the next - * operation, using a continuation-passing style design. The last - * operation must be a write operation, which returns from the call - * to the `process` function. - * - * The GPRs used by the entire call-chain are listed below. - * - * Function arguments are passed in r0-r5. After the parameters from - * `exec` have been read, r0 is reused to branch to the continuation - * functions. After the original parameters from `impl` have been - * computed, r1 is reused as the `impl` pointer for each operation. - * - * Loop iterators are r6 for `bx` and r3 for `y`, reused from - * `y_start`, which doesn't need to be preserved. - * - * The intra-procedure-call temporary registers (r16 and r17) are used - * as scratch registers. They may be used by call veneers and PLT code - * inserted by the linker, so we cannot expect them to persist across - * branches between functions. - * - * The Platform Register (r18) is not used. - * - * The read/write data pointers and padding values first use up the - * remaining free caller-saved registers, and only then are the - * caller-saved registers (r19-r29) used. - * - * The Link Register (r30) is used when calling the first kernel, so it - * must be saved. - */ - -static const int rw_gprs[] = { - 9, 10, 11, 12, - 13, 14, 15, 19, - 20, 21, 22, 23, - 24, 25, 26, 27, -}; - -static void asmgen_common_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) -{ - /* Loop iterator variables. */ - s->bx = a64op_gpw(6); - s->y = a64op_gpw(3); /* Reused from SwsOpFunc.y_start argument. */ - - /* Scratch registers. */ - s->tmp0 = a64op_gpx(16); /* IP0 */ - s->tmp1 = a64op_gpx(17); /* IP1 */ - - /* Read/Write data pointers. */ - LOOP(imask, i) { s->in [i] = a64op_gpx(rw_gprs[(i * 4) + 0]); } - LOOP(omask, i) { s->out[i] = a64op_gpx(rw_gprs[(i * 4) + 1]); } -} - -static void asmgen_process_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) -{ - asmgen_common_frame(s, imask, omask); - - /* SwsOpFunc arguments. */ - s->exec = a64op_gpx(0); // const SwsOpExec *exec - s->impl = a64op_gpx(1); // const void *priv - s->bx_start = a64op_gpw(2); // int bx_start - s->y_start = a64op_gpw(3); // int y_start - s->bx_end = a64op_gpw(4); // int bx_end - s->y_end = a64op_gpw(5); // int y_end - - /* CPS-related variables. */ - s->op0_func = a64op_gpx(7); - s->op1_impl = a64op_gpx(8); - - /* Read/Write data pointer padding. */ - LOOP(imask, i) { s->in_bump [i] = a64op_gpx(rw_gprs[(i * 4) + 2]); } - LOOP(omask, i) { s->out_bump[i] = a64op_gpx(rw_gprs[(i * 4) + 3]); } -} - -static void asmgen_op_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) -{ - asmgen_common_frame(s, imask, omask); - - /* CPS-related variables. */ - s->cont = a64op_gpx(0); /* Reused from SwsOpFunc.exec argument. */ - s->impl = a64op_gpx(1); /* Same as SwsOpFunc.impl argument. */ -} - -/*********************************************************************/ -/* Vector register assignment. */ -static void init_vectors_cps(SwsAArch64Context *s, SwsAArch64OpRegs *regs) -{ - regs->sl[ 0] = a64op_vec( 0); - regs->sl[ 1] = a64op_vec( 1); - regs->sl[ 2] = a64op_vec( 2); - regs->sl[ 3] = a64op_vec( 3); - regs->sh[ 0] = a64op_vec( 4); - regs->sh[ 1] = a64op_vec( 5); - regs->sh[ 2] = a64op_vec( 6); - regs->sh[ 3] = a64op_vec( 7); - regs->dl[ 0] = a64op_vec( 0); - regs->dl[ 1] = a64op_vec( 1); - regs->dl[ 2] = a64op_vec( 2); - regs->dl[ 3] = a64op_vec( 3); - regs->dh[ 0] = a64op_vec( 4); - regs->dh[ 1] = a64op_vec( 5); - regs->dh[ 2] = a64op_vec( 6); - regs->dh[ 3] = a64op_vec( 7); - regs->vt[ 0] = a64op_vec(16); - regs->vt[ 1] = a64op_vec(17); - regs->vt[ 2] = a64op_vec(18); - regs->vt[ 3] = a64op_vec(19); - regs->vt[ 4] = a64op_vec(20); - regs->vt[ 5] = a64op_vec(21); - regs->vt[ 6] = a64op_vec(22); - regs->vt[ 7] = a64op_vec(23); - regs->vt[ 8] = a64op_vec(24); - regs->vt[ 9] = a64op_vec(25); - regs->vt[10] = a64op_vec(26); - regs->vt[11] = a64op_vec(27); - regs->vk[ 0] = a64op_vec(28); - regs->vk[ 1] = a64op_vec(29); - regs->vk[ 2] = a64op_vec(30); - regs->vk[ 3] = a64op_vec(31); -} - -/*********************************************************************/ -static void asmgen_process_cps(SwsAArch64Context *s, SwsCompMask mask) -{ - RasmContext *r = s->rctx; - char func_name[128]; - - snprintf(func_name, sizeof(func_name), "ff_sws_process_%04x_neon", nibble_mask(mask)); - rasm_func_begin(r, func_name, true, false); - asmgen_process_frame(s, mask, mask); - - asmgen_process(s, mask, mask); - - /* Load values from impl. */ - rasm_set_current_node(r, s->setup); - RasmOp impl_cont = a64op_off(s->impl, offsetof_impl_cont); - i_ldr(r, s->op0_func, impl_cont); CMT("SwsFuncPtr op0_func = impl->cont;"); - i_add(r, s->op1_impl, s->impl, IMM(sizeof_impl)); CMT("SwsOpImpl *op1_impl = impl + 1;"); - - /* Reset impl and call first kernel. */ - rasm_set_current_node(r, s->loop); - i_mov(r, s->impl, s->op1_impl); CMT("impl = op1_impl;"); - i_blr(r, s->op0_func); CMT("op0_func();"); -} - -/*********************************************************************/ -static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) -{ - const SwsAArch64OpImplParams *p = &entry->params; - RasmContext *r = s->rctx; - - bool is_read = false; - bool is_write = false; - switch (p->uop) { - case SWS_UOP_READ_BIT: - case SWS_UOP_READ_NIBBLE: - case SWS_UOP_READ_PACKED: - case SWS_UOP_READ_PLANAR: - is_read = true; - break; - case SWS_UOP_WRITE_BIT: - case SWS_UOP_WRITE_NIBBLE: - case SWS_UOP_WRITE_PACKED: - case SWS_UOP_WRITE_PLANAR: - is_write = true; - break; - default: - break; - } - - rasm_func_begin(r, entry->name, true, !is_read); - asmgen_op_frame(s, is_read ? p->mask : 0, is_write ? p->mask : 0); - - /** - * Set up vector register dimensions and reshape all vectors - * accordingly. - */ - size_t el_size = ff_sws_pixel_type_size(p->type); - size_t total_size = p->block_size * el_size; - - s->vec_size = FFMIN(total_size, 16); - s->use_vh = (s->vec_size != total_size); - - s->el_size = el_size; - s->el_count = s->vec_size / el_size; - init_vectors_cps(s, &s->regs); - reshape_io_vectors(&s->regs, s->el_count, el_size); - reshape_temp_vectors(&s->regs, s->el_count, el_size); - reshape_const_vectors(&s->regs, s->el_count, el_size); - - /* Common start for continuation-passing style (CPS) functions. */ - s->impl_priv = a64op_off(s->impl, offsetof_impl_priv); - asmgen_set_load_cont_node(s); - - /* Set up constants. */ - switch (p->uop) { - case SWS_UOP_READ_BIT: asmgen_setup_read_bit(s, p, &s->regs); break; - case SWS_UOP_READ_NIBBLE: asmgen_setup_read_nibble(s, p, &s->regs); break; - case SWS_UOP_WRITE_BIT: asmgen_setup_write_bit(s, p, &s->regs); break; - case SWS_UOP_UNPACK: asmgen_setup_unpack(s, p, &s->regs); break; - case SWS_UOP_CLEAR: asmgen_setup_clear(s, p, &s->regs); break; - case SWS_UOP_MIN: asmgen_setup_min(s, p, &s->regs); break; - case SWS_UOP_MAX: asmgen_setup_max(s, p, &s->regs); break; - case SWS_UOP_SCALE: asmgen_setup_scale(s, p, &s->regs); break; - case SWS_UOP_LINEAR: asmgen_setup_linear(s, p, &s->regs); break; - case SWS_UOP_LINEAR_FMA: asmgen_setup_linear(s, p, &s->regs); break; - case SWS_UOP_DITHER: asmgen_setup_dither(s, p, &s->regs); break; - default: - break; - } - - /* Emit uop kernel. */ - switch (p->uop) { - case SWS_UOP_READ_BIT: asmgen_op_read_bit(s, p, &s->regs); break; - case SWS_UOP_READ_NIBBLE: asmgen_op_read_nibble(s, p, &s->regs); break; - case SWS_UOP_READ_PACKED: asmgen_op_read_packed(s, p, &s->regs); break; - case SWS_UOP_READ_PLANAR: asmgen_op_read_planar(s, p, &s->regs); break; - case SWS_UOP_WRITE_BIT: asmgen_op_write_bit(s, p, &s->regs); break; - case SWS_UOP_WRITE_NIBBLE: asmgen_op_write_nibble(s, p, &s->regs); break; - case SWS_UOP_WRITE_PACKED: asmgen_op_write_packed(s, p, &s->regs); break; - case SWS_UOP_WRITE_PLANAR: asmgen_op_write_planar(s, p, &s->regs); break; - case SWS_UOP_SWAP_BYTES: asmgen_op_swap_bytes(s, p, &s->regs); break; - case SWS_UOP_PERMUTE: asmgen_op_move(s, p, &s->regs); break; - case SWS_UOP_COPY: asmgen_op_move(s, p, &s->regs); break; - case SWS_UOP_UNPACK: asmgen_op_unpack(s, p, &s->regs); break; - case SWS_UOP_PACK: asmgen_op_pack(s, p, &s->regs); break; - case SWS_UOP_LSHIFT: asmgen_op_lshift(s, p, &s->regs); break; - case SWS_UOP_RSHIFT: asmgen_op_rshift(s, p, &s->regs); break; - case SWS_UOP_CLEAR: asmgen_op_clear(s, p, &s->regs); break; - case SWS_UOP_TO_U8: asmgen_op_convert(s, p, &s->regs); break; - case SWS_UOP_TO_U16: asmgen_op_convert(s, p, &s->regs); break; - case SWS_UOP_TO_U32: asmgen_op_convert(s, p, &s->regs); break; - case SWS_UOP_TO_F32: asmgen_op_convert(s, p, &s->regs); break; - case SWS_UOP_EXPAND_PAIR: asmgen_op_expand(s, p, &s->regs); break; - case SWS_UOP_EXPAND_QUAD: asmgen_op_expand(s, p, &s->regs); break; - case SWS_UOP_MIN: asmgen_op_min(s, p, &s->regs); break; - case SWS_UOP_MAX: asmgen_op_max(s, p, &s->regs); break; - case SWS_UOP_SCALE: asmgen_op_scale(s, p, &s->regs); break; - case SWS_UOP_LINEAR: asmgen_op_linear(s, p, &s->regs); break; - case SWS_UOP_LINEAR_FMA: asmgen_op_linear(s, p, &s->regs); break; - case SWS_UOP_DITHER: asmgen_op_dither(s, p, &s->regs); break; - /* TODO implement SWS_UOP_SHUFFLE */ - default: - break; - } - - if (is_write) { - /* Write functions return directly. */ - i_ret(r); - } else { - /* Load continuation address and increment impl pointer. */ - RasmNode *node = rasm_set_current_node(r, s->load_cont_node); - RasmOp impl_post = a64op_post(s->impl, sizeof_impl); - i_ldr(r, s->cont, impl_post); CMT("SwsFuncPtr cont = (impl++)->cont;"); - rasm_set_current_node(r, node); - /* Common end for remaining CPS functions. */ - i_br (r, s->cont); CMT("jump to cont"); - } -} - -/*********************************************************************/ - -/* Generate all functions described by ops_entries.c */ -static int asmgen(void) -{ - RasmContext *rctx = rasm_alloc(); - if (!rctx) - return AVERROR(ENOMEM); - - SwsAArch64Context s = { .rctx = rctx }; - AVBPrint bp; - int ret; - - av_bprint_init(&bp, 0, AV_BPRINT_SIZE_UNLIMITED); - - /* Generate all process functions using rasm. */ - asmgen_process_cps(&s, SWS_COMP_ELEMS(1)); - asmgen_process_cps(&s, SWS_COMP_ELEMS(2)); - asmgen_process_cps(&s, SWS_COMP_ELEMS(3)); - asmgen_process_cps(&s, SWS_COMP_ELEMS(4)); - - /* Generate all functions from ops_entries.c using rasm. */ - const SwsAArch64OpEntry *entries = ops_entries; - while (entries->name) { - asmgen_op_cps(&s, entries++); - if (rctx->error) { - ret = rctx->error; - goto error; - } - } - - /* Print all rasm functions to stdout. */ - printf("#include \"libavutil/aarch64/asm.S\"\n"); - printf("\n"); - ret = rasm_print(s.rctx, &bp); - if (ret < 0) - goto error; - fputs(bp.str, stdout); - -error: - av_bprint_finalize(&bp, NULL); - rasm_free(&s.rctx); - return ret; -} - -/*********************************************************************/ -int main(int argc, char *argv[]) -{ -#ifdef _WIN32 - _setmode(_fileno(stdout), _O_BINARY); -#endif - - return asmgen(); -} diff --git a/libswscale/aarch64/ops_asmgen.h b/libswscale/aarch64/ops_asmgen.h new file mode 100644 index 0000000000..f2ea32628b --- /dev/null +++ b/libswscale/aarch64/ops_asmgen.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2026 Ramiro Polla + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SWSCALE_AARCH64_OPS_ASMGEN_H +#define SWSCALE_AARCH64_OPS_ASMGEN_H + +/*********************************************************************/ +typedef struct SwsAArch64OpRegs { + RasmOp sl[ 4]; /* input vector registers (low bank) */ + RasmOp sh[ 4]; /* input vector registers (high bank) */ + RasmOp dl[ 4]; /* output vector registers (low bank) */ + RasmOp dh[ 4]; /* output vector registers (high bank) */ + RasmOp vt[12]; /* temp vector registers */ + RasmOp vk[ 4]; /* constant data (may be gprs) */ + + /* Op-specific registers. */ + union { + RasmOp dither_ptr; + RasmOp linear_vcoeff[4][5]; + }; +} SwsAArch64OpRegs; + +/*********************************************************************/ +typedef struct SwsAArch64Context { + RasmContext *rctx; + + /* SwsOpFunc arguments. */ + RasmOp exec; + RasmOp impl; + RasmOp bx_start; + RasmOp y_start; + RasmOp bx_end; + RasmOp y_end; + + /* Loop iterator variables. */ + RasmOp bx; + RasmOp y; + + /* Scratch registers. */ + RasmOp tmp0; + RasmOp tmp1; + + /* CPS-related variables. */ + RasmOp op0_func; + RasmOp op1_impl; + RasmOp cont; + RasmOp impl_priv; + RasmNode *load_cont_node; + SwsAArch64OpRegs regs; + + /* Read/Write data pointers and padding. */ + RasmOp in[4]; + RasmOp out[4]; + RasmOp in_bump[4]; + RasmOp out_bump[4]; + + /* Process function. */ + RasmNode *setup; + RasmNode *loop; + + /* Vector register dimensions. */ + size_t el_size; + size_t el_count; + size_t vec_size; + bool use_vh; +} SwsAArch64Context; + +#endif /* SWSCALE_AARCH64_OPS_ASMGEN_H */ diff --git a/libswscale/aarch64/ops_static.c b/libswscale/aarch64/ops_static.c new file mode 100644 index 0000000000..23aefd2033 --- /dev/null +++ b/libswscale/aarch64/ops_static.c @@ -0,0 +1,663 @@ +/* + * Copyright (C) 2026 Ramiro Polla + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <assert.h> +#include <limits.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifdef _WIN32 +#include <io.h> +#include <fcntl.h> +#endif + +/** + * This file is compiled as a standalone build-time tool and must not depend + * on internal FFmpeg libraries. The necessary utils are redefined below using + * standard C equivalents. + */ + +#define AVUTIL_AVASSERT_H +#define AVUTIL_LOG_H +#define AVUTIL_MACROS_H +#define AVUTIL_MEM_H +#define av_assert0(cond) assert(cond) +#define av_malloc(s) malloc(s) +#define av_mallocz(s) calloc(1, s) +#define av_realloc(p, s) realloc(p, s) +#define av_strdup(s) strdup(s) +#define av_free(p) free(p) +#define FFMAX(a,b) ((a) > (b) ? (a) : (b)) +#define FFMIN(a,b) ((a) > (b) ? (b) : (a)) +#define MKTAG(a,b,c,d) ((a) | ((b) << 8) | ((c) << 16) | ((unsigned)(d) << 24)) + +static void av_freep(void *ptr) +{ + void **pptr = (void **) ptr; + if (pptr) { + ptr = *pptr; + if (ptr) + free(ptr); + *pptr = NULL; + } +} + +static void *av_memdup(const void *p, size_t size) +{ + void *ptr = NULL; + if (p) { + ptr = av_malloc(size); + if (ptr) + memcpy(ptr, p, size); + } + return ptr; +} + +#include "libavutil/dynarray.h" + +static void *av_dynarray2_add(void **tab_ptr, int *nb_ptr, size_t elem_size, + const uint8_t *elem_data) +{ + uint8_t *tab_elem_data = NULL; + + FF_DYNARRAY_ADD(INT_MAX, elem_size, *tab_ptr, *nb_ptr, { + tab_elem_data = (uint8_t *)*tab_ptr + (*nb_ptr) * elem_size; + if (elem_data) + memcpy(tab_elem_data, elem_data, elem_size); + }, { + av_freep(tab_ptr); + *nb_ptr = 0; + }); + return tab_elem_data; +} + +#include "libavutil/bprint.c" + +/*********************************************************************/ +#include "rasm.c" +#include "rasm_print.c" +#include "ops_impl.h" + +/** + * Implementation parameters for all exported functions. This list is + * compiled by performing a dummy run of all conversions in sws_ops and + * collecting all functions that need to be generated. This is achieved + * by running: + * make fate-sws-ops-entries-aarch64 GEN=1 + */ +typedef struct SwsAArch64OpEntry { + const char *name; + SwsAArch64OpImplParams params; +} SwsAArch64OpEntry; + +static const SwsAArch64OpEntry ops_entries[] = { +#define ENTRY(fname, ...) { .name = #fname, .params = __VA_ARGS__ }, +#include "ops_entries.c" +#undef ENTRY + { NULL } +}; + +#include "ops_asmgen.c" + +/*********************************************************************/ +/** + * Set node where the continuation address will be loaded and impl will + * be incremented. This should be done right after impl->priv has been + * used. + */ +static void asmgen_set_load_cont_node(SwsAArch64Context *s) +{ + RasmContext *r = s->rctx; + s->load_cont_node = rasm_get_current_node(r); +} + +/*********************************************************************/ +/* gather raw pixels from planes */ +/* SWS_UOP_READ_BIT */ +/* SWS_UOP_READ_NIBBLE */ +/* SWS_UOP_READ_PACKED */ +/* SWS_UOP_READ_PLANAR */ + +static void asmgen_setup_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); + AArch64VecViews bitmask_vec = a64op_vec_views(regs->vk[1]); + + rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); + i_ldr(r, shift_vec.q, s->impl_priv); + asmgen_set_load_cont_node(s); + if (p->block_size == 16) { + i_movi(r, bitmask_vec.b16, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 16 times>};"); + } else { + i_movi(r, bitmask_vec.b8, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 8 times>, 0 <repeats 8 times>};"); + } +} + +static void asmgen_setup_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + AArch64VecViews nibble_mask = a64op_vec_views(regs->vk[0]); + + rasm_annotate_next(r, "v128 nibble_mask = {0xf <repeats 8 times>, 0x0 <repeats 8 times>};"); + i_movi(r, nibble_mask.b8, IMM(0x0f)); +} + +static void asmgen_setup_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]); + + rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;"); + i_ldr(r, shift_vec.q, s->impl_priv); + asmgen_set_load_cont_node(s); +} + +static void asmgen_setup_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp *vmask = regs->vk; + RasmOp mask_gpr = a64op_w(s->tmp0); + uint32_t mask_val[4] = { 0 }; + + /* Generate masks. */ + rasm_add_comment(r, "generate masks"); + LOOP_MASK(p, i) { + uint32_t val = (1u << p->par.pack.pattern[i]) - 1; + for (int j = 0; j < 4; j++) { + if (mask_val[j] == val) { + mask_val[i] = mask_val[j]; + vmask[i] = vmask[j]; + break; + } + } + if (!mask_val[i]) { + /** + * All-one values in movi only work up to 8-bit, and then + * at full 16- or 32-bit, but not for intermediate values + * like 10-bit. In those cases, we use mov + dup instead. + */ + if (val <= 0xff || val == 0xffff) { + i_movi(r, vmask[i], IMM(val)); + } else { + i_mov (r, mask_gpr, IMM(val)); + i_dup (r, vmask[i], mask_gpr); + } + mask_val[i] = val; + vmask[i] = v_16b(vmask[i]); + } + } +} + +static void asmgen_setup_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp *vk = regs->vk; + + /** + * TODO + * - pack elements in impl->priv and perform smaller loads + * - if only 1 element and not vh, load directly with ld1r + */ + + bool load_priv = false; + LOOP_MASK(p, i) { + if (!((p->par.clear.zero | p->par.clear.one) & SWS_COMP(i))) + load_priv = true; + } + if (load_priv) { + i_ldr(r, v_q(vk[0]), s->impl_priv); CMT("v128 clear_vec = impl->priv.v128;"); + asmgen_set_load_cont_node(s); + } +} + +static void asmgen_setup_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp *vk = regs->vk; + + RasmOp min_vec = regs->vt[0]; + i_ldr(r, v_q(min_vec), s->impl_priv); CMT("v128 min_vec = impl->priv.v128;"); + asmgen_set_load_cont_node(s); + LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(min_vec, i)); CMTF("v128 vmin%u = min_vec[%u];", i, i); } +} + +static void asmgen_setup_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp *vk = regs->vk; + + RasmOp max_vec = regs->vt[0]; + i_ldr(r, v_q(max_vec), s->impl_priv); CMT("v128 max_vec = impl->priv.v128;"); + asmgen_set_load_cont_node(s); + LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(max_vec, i)); CMTF("v128 vmax%u = max_vec[%u];", i, i); } +} + +static void asmgen_setup_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp scale_vec = regs->vk[0]; + + RasmOp priv_ptr = s->tmp0; + i_add (r, priv_ptr, s->impl, IMM(offsetof_impl_priv)); CMT("v128 *scale_vec_ptr = &impl->priv;"); + asmgen_set_load_cont_node(s); + i_ld1r(r, vv_1(scale_vec), a64op_base(priv_ptr)); CMT("v128 scale_vec = broadcast(*scale_vec_ptr);"); +} + +static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp *sl = regs->sl; + RasmOp *sh = regs->sh; + RasmOp *vc = regs->vk; + RasmOp *vt = regs->vt; + + RasmOp ptr = s->tmp0; + RasmOp coeff_veclist; + + /* Preload coefficients from impl->priv. */ + const int num_vregs = linear_num_vregs(p); + av_assert0(num_vregs <= 4); + switch (num_vregs) { + case 1: coeff_veclist = vv_1(vc[0]); break; + case 2: coeff_veclist = vv_2(vc[0], vc[1]); break; + case 3: coeff_veclist = vv_3(vc[0], vc[1], vc[2]); break; + case 4: coeff_veclist = vv_4(vc[0], vc[1], vc[2], vc[3]); break; + } + i_ldr(r, ptr, s->impl_priv); CMT("v128 *vcoeff_ptr = impl->priv.ptr;"); + asmgen_set_load_cont_node(s); + i_ld1(r, coeff_veclist, a64op_base(ptr)); CMT("coeff_veclist = *vcoeff_ptr;"); + + /** + * Populate operands matrix from packed data into linear_vcoeff matrix + * and compute mask for rows that must be saved before being overwritten. + */ + SwsCompMask save_mask = 0; + bool overwritten[4] = { false, false, false, false }; + int i_coeff = 0; + LOOP_MASK(p, i) { + for (int j = 0; j < 5; j++) { + bool is_offset = (j == 0); + int src_j = is_offset ? 4 : (j - 1); + if (p->par.lin.zero & SWS_MASK(i, src_j)) + continue; + uint8_t vc_i = i_coeff / 4; + uint8_t vc_j = i_coeff & 3; + regs->linear_vcoeff[i][j] = a64op_elem(vc[vc_i], vc_j); + i_coeff++; + if (!is_offset && overwritten[src_j]) + save_mask |= SWS_COMP(src_j); + overwritten[i] = true; + } + } + + /** + * Save rows that need to be used as input after they have been already + * written to. + */ + RasmOp *tl = &vt[0]; + RasmOp *th = &vt[4]; + LOOP (save_mask, i) { i_mov16b(r, tl[i], sl[i]); CMTF("vsrcl[%u] = vl[%u];", i, i); } + LOOP_VH(s, save_mask, i) { i_mov16b(r, th[i], sh[i]); CMTF("vsrch[%u] = vh[%u];", i, i); } + LOOP (save_mask, i) { sl[i] = tl[i]; } + LOOP_VH(s, save_mask, i) { sh[i] = th[i]; } +} + +static void asmgen_setup_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + RasmOp src_ptr = s->tmp0; + + regs->dither_ptr = src_ptr; + i_ldr(r, src_ptr, s->impl_priv); CMT("void *ptr = impl->priv.ptr;"); + asmgen_set_load_cont_node(s); +} + +/*********************************************************************/ +/** + * Register assignment for CPS functions. + * + * The entry point of the SwsOpFunc is the `process` function. The + * first kernel function is called from `process`, and subsequent + * kernel functions are chained by directly branching to the next + * operation, using a continuation-passing style design. The last + * operation must be a write operation, which returns from the call + * to the `process` function. + * + * The GPRs used by the entire call-chain are listed below. + * + * Function arguments are passed in r0-r5. After the parameters from + * `exec` have been read, r0 is reused to branch to the continuation + * functions. After the original parameters from `impl` have been + * computed, r1 is reused as the `impl` pointer for each operation. + * + * Loop iterators are r6 for `bx` and r3 for `y`, reused from + * `y_start`, which doesn't need to be preserved. + * + * The intra-procedure-call temporary registers (r16 and r17) are used + * as scratch registers. They may be used by call veneers and PLT code + * inserted by the linker, so we cannot expect them to persist across + * branches between functions. + * + * The Platform Register (r18) is not used. + * + * The read/write data pointers and padding values first use up the + * remaining free caller-saved registers, and only then are the + * caller-saved registers (r19-r29) used. + * + * The Link Register (r30) is used when calling the first kernel, so it + * must be saved. + */ + +static const int rw_gprs[] = { + 9, 10, 11, 12, + 13, 14, 15, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, +}; + +static void asmgen_common_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) +{ + /* Loop iterator variables. */ + s->bx = a64op_gpw(6); + s->y = a64op_gpw(3); /* Reused from SwsOpFunc.y_start argument. */ + + /* Scratch registers. */ + s->tmp0 = a64op_gpx(16); /* IP0 */ + s->tmp1 = a64op_gpx(17); /* IP1 */ + + /* Read/Write data pointers. */ + LOOP(imask, i) { s->in [i] = a64op_gpx(rw_gprs[(i * 4) + 0]); } + LOOP(omask, i) { s->out[i] = a64op_gpx(rw_gprs[(i * 4) + 1]); } +} + +static void asmgen_process_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) +{ + asmgen_common_frame(s, imask, omask); + + /* SwsOpFunc arguments. */ + s->exec = a64op_gpx(0); // const SwsOpExec *exec + s->impl = a64op_gpx(1); // const void *priv + s->bx_start = a64op_gpw(2); // int bx_start + s->y_start = a64op_gpw(3); // int y_start + s->bx_end = a64op_gpw(4); // int bx_end + s->y_end = a64op_gpw(5); // int y_end + + /* CPS-related variables. */ + s->op0_func = a64op_gpx(7); + s->op1_impl = a64op_gpx(8); + + /* Read/Write data pointer padding. */ + LOOP(imask, i) { s->in_bump [i] = a64op_gpx(rw_gprs[(i * 4) + 2]); } + LOOP(omask, i) { s->out_bump[i] = a64op_gpx(rw_gprs[(i * 4) + 3]); } +} + +static void asmgen_op_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) +{ + asmgen_common_frame(s, imask, omask); + + /* CPS-related variables. */ + s->cont = a64op_gpx(0); /* Reused from SwsOpFunc.exec argument. */ + s->impl = a64op_gpx(1); /* Same as SwsOpFunc.impl argument. */ +} + +/*********************************************************************/ +/* Vector register assignment. */ +static void init_vectors_cps(SwsAArch64Context *s, SwsAArch64OpRegs *regs) +{ + regs->sl[ 0] = a64op_vec( 0); + regs->sl[ 1] = a64op_vec( 1); + regs->sl[ 2] = a64op_vec( 2); + regs->sl[ 3] = a64op_vec( 3); + regs->sh[ 0] = a64op_vec( 4); + regs->sh[ 1] = a64op_vec( 5); + regs->sh[ 2] = a64op_vec( 6); + regs->sh[ 3] = a64op_vec( 7); + regs->dl[ 0] = a64op_vec( 0); + regs->dl[ 1] = a64op_vec( 1); + regs->dl[ 2] = a64op_vec( 2); + regs->dl[ 3] = a64op_vec( 3); + regs->dh[ 0] = a64op_vec( 4); + regs->dh[ 1] = a64op_vec( 5); + regs->dh[ 2] = a64op_vec( 6); + regs->dh[ 3] = a64op_vec( 7); + regs->vt[ 0] = a64op_vec(16); + regs->vt[ 1] = a64op_vec(17); + regs->vt[ 2] = a64op_vec(18); + regs->vt[ 3] = a64op_vec(19); + regs->vt[ 4] = a64op_vec(20); + regs->vt[ 5] = a64op_vec(21); + regs->vt[ 6] = a64op_vec(22); + regs->vt[ 7] = a64op_vec(23); + regs->vt[ 8] = a64op_vec(24); + regs->vt[ 9] = a64op_vec(25); + regs->vt[10] = a64op_vec(26); + regs->vt[11] = a64op_vec(27); + regs->vk[ 0] = a64op_vec(28); + regs->vk[ 1] = a64op_vec(29); + regs->vk[ 2] = a64op_vec(30); + regs->vk[ 3] = a64op_vec(31); +} + +/*********************************************************************/ +static void asmgen_process_cps(SwsAArch64Context *s, SwsCompMask mask) +{ + RasmContext *r = s->rctx; + char func_name[128]; + + snprintf(func_name, sizeof(func_name), "ff_sws_process_%04x_neon", nibble_mask(mask)); + rasm_func_begin(r, func_name, true, false); + asmgen_process_frame(s, mask, mask); + + asmgen_process(s, mask, mask); + + /* Load values from impl. */ + rasm_set_current_node(r, s->setup); + RasmOp impl_cont = a64op_off(s->impl, offsetof_impl_cont); + i_ldr(r, s->op0_func, impl_cont); CMT("SwsFuncPtr op0_func = impl->cont;"); + i_add(r, s->op1_impl, s->impl, IMM(sizeof_impl)); CMT("SwsOpImpl *op1_impl = impl + 1;"); + + /* Reset impl and call first kernel. */ + rasm_set_current_node(r, s->loop); + i_mov(r, s->impl, s->op1_impl); CMT("impl = op1_impl;"); + i_blr(r, s->op0_func); CMT("op0_func();"); +} + +/*********************************************************************/ +static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) +{ + const SwsAArch64OpImplParams *p = &entry->params; + RasmContext *r = s->rctx; + + bool is_read = false; + bool is_write = false; + switch (p->uop) { + case SWS_UOP_READ_BIT: + case SWS_UOP_READ_NIBBLE: + case SWS_UOP_READ_PACKED: + case SWS_UOP_READ_PLANAR: + is_read = true; + break; + case SWS_UOP_WRITE_BIT: + case SWS_UOP_WRITE_NIBBLE: + case SWS_UOP_WRITE_PACKED: + case SWS_UOP_WRITE_PLANAR: + is_write = true; + break; + default: + break; + } + + rasm_func_begin(r, entry->name, true, !is_read); + asmgen_op_frame(s, is_read ? p->mask : 0, is_write ? p->mask : 0); + + /** + * Set up vector register dimensions and reshape all vectors + * accordingly. + */ + size_t el_size = ff_sws_pixel_type_size(p->type); + size_t total_size = p->block_size * el_size; + + s->vec_size = FFMIN(total_size, 16); + s->use_vh = (s->vec_size != total_size); + + s->el_size = el_size; + s->el_count = s->vec_size / el_size; + init_vectors_cps(s, &s->regs); + reshape_io_vectors(&s->regs, s->el_count, el_size); + reshape_temp_vectors(&s->regs, s->el_count, el_size); + reshape_const_vectors(&s->regs, s->el_count, el_size); + + /* Common start for continuation-passing style (CPS) functions. */ + s->impl_priv = a64op_off(s->impl, offsetof_impl_priv); + asmgen_set_load_cont_node(s); + + /* Set up constants. */ + switch (p->uop) { + case SWS_UOP_READ_BIT: asmgen_setup_read_bit(s, p, &s->regs); break; + case SWS_UOP_READ_NIBBLE: asmgen_setup_read_nibble(s, p, &s->regs); break; + case SWS_UOP_WRITE_BIT: asmgen_setup_write_bit(s, p, &s->regs); break; + case SWS_UOP_UNPACK: asmgen_setup_unpack(s, p, &s->regs); break; + case SWS_UOP_CLEAR: asmgen_setup_clear(s, p, &s->regs); break; + case SWS_UOP_MIN: asmgen_setup_min(s, p, &s->regs); break; + case SWS_UOP_MAX: asmgen_setup_max(s, p, &s->regs); break; + case SWS_UOP_SCALE: asmgen_setup_scale(s, p, &s->regs); break; + case SWS_UOP_LINEAR: asmgen_setup_linear(s, p, &s->regs); break; + case SWS_UOP_LINEAR_FMA: asmgen_setup_linear(s, p, &s->regs); break; + case SWS_UOP_DITHER: asmgen_setup_dither(s, p, &s->regs); break; + default: + break; + } + + /* Emit uop kernel. */ + switch (p->uop) { + case SWS_UOP_READ_BIT: asmgen_op_read_bit(s, p, &s->regs); break; + case SWS_UOP_READ_NIBBLE: asmgen_op_read_nibble(s, p, &s->regs); break; + case SWS_UOP_READ_PACKED: asmgen_op_read_packed(s, p, &s->regs); break; + case SWS_UOP_READ_PLANAR: asmgen_op_read_planar(s, p, &s->regs); break; + case SWS_UOP_WRITE_BIT: asmgen_op_write_bit(s, p, &s->regs); break; + case SWS_UOP_WRITE_NIBBLE: asmgen_op_write_nibble(s, p, &s->regs); break; + case SWS_UOP_WRITE_PACKED: asmgen_op_write_packed(s, p, &s->regs); break; + case SWS_UOP_WRITE_PLANAR: asmgen_op_write_planar(s, p, &s->regs); break; + case SWS_UOP_SWAP_BYTES: asmgen_op_swap_bytes(s, p, &s->regs); break; + case SWS_UOP_PERMUTE: asmgen_op_move(s, p, &s->regs); break; + case SWS_UOP_COPY: asmgen_op_move(s, p, &s->regs); break; + case SWS_UOP_UNPACK: asmgen_op_unpack(s, p, &s->regs); break; + case SWS_UOP_PACK: asmgen_op_pack(s, p, &s->regs); break; + case SWS_UOP_LSHIFT: asmgen_op_lshift(s, p, &s->regs); break; + case SWS_UOP_RSHIFT: asmgen_op_rshift(s, p, &s->regs); break; + case SWS_UOP_CLEAR: asmgen_op_clear(s, p, &s->regs); break; + case SWS_UOP_TO_U8: asmgen_op_convert(s, p, &s->regs); break; + case SWS_UOP_TO_U16: asmgen_op_convert(s, p, &s->regs); break; + case SWS_UOP_TO_U32: asmgen_op_convert(s, p, &s->regs); break; + case SWS_UOP_TO_F32: asmgen_op_convert(s, p, &s->regs); break; + case SWS_UOP_EXPAND_PAIR: asmgen_op_expand(s, p, &s->regs); break; + case SWS_UOP_EXPAND_QUAD: asmgen_op_expand(s, p, &s->regs); break; + case SWS_UOP_MIN: asmgen_op_min(s, p, &s->regs); break; + case SWS_UOP_MAX: asmgen_op_max(s, p, &s->regs); break; + case SWS_UOP_SCALE: asmgen_op_scale(s, p, &s->regs); break; + case SWS_UOP_LINEAR: asmgen_op_linear(s, p, &s->regs); break; + case SWS_UOP_LINEAR_FMA: asmgen_op_linear(s, p, &s->regs); break; + case SWS_UOP_DITHER: asmgen_op_dither(s, p, &s->regs); break; + /* TODO implement SWS_UOP_SHUFFLE */ + default: + break; + } + + if (is_write) { + /* Write functions return directly. */ + i_ret(r); + } else { + /* Load continuation address and increment impl pointer. */ + RasmNode *node = rasm_set_current_node(r, s->load_cont_node); + RasmOp impl_post = a64op_post(s->impl, sizeof_impl); + i_ldr(r, s->cont, impl_post); CMT("SwsFuncPtr cont = (impl++)->cont;"); + rasm_set_current_node(r, node); + /* Common end for remaining CPS functions. */ + i_br (r, s->cont); CMT("jump to cont"); + } +} + +/*********************************************************************/ + +/* Generate all functions described by ops_entries.c */ +static int asmgen(void) +{ + RasmContext *rctx = rasm_alloc(); + if (!rctx) + return AVERROR(ENOMEM); + + SwsAArch64Context s = { .rctx = rctx }; + AVBPrint bp; + int ret; + + av_bprint_init(&bp, 0, AV_BPRINT_SIZE_UNLIMITED); + + /* Generate all process functions using rasm. */ + asmgen_process_cps(&s, SWS_COMP_ELEMS(1)); + asmgen_process_cps(&s, SWS_COMP_ELEMS(2)); + asmgen_process_cps(&s, SWS_COMP_ELEMS(3)); + asmgen_process_cps(&s, SWS_COMP_ELEMS(4)); + + /* Generate all functions from ops_entries.c using rasm. */ + const SwsAArch64OpEntry *entries = ops_entries; + while (entries->name) { + asmgen_op_cps(&s, entries++); + if (rctx->error) { + ret = rctx->error; + goto error; + } + } + + /* Print all rasm functions to stdout. */ + printf("#include \"libavutil/aarch64/asm.S\"\n"); + printf("\n"); + ret = rasm_print(s.rctx, &bp); + if (ret < 0) + goto error; + fputs(bp.str, stdout); + +error: + av_bprint_finalize(&bp, NULL); + rasm_free(&s.rctx); + return ret; +} + +/*********************************************************************/ +int main(int argc, char *argv[]) +{ +#ifdef _WIN32 + _setmode(_fileno(stdout), _O_BINARY); +#endif + + return asmgen(); +} -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]