[PR] swscale/aarch64: add NEON JIT backend (PR #23923)
Ramiro Polla via ffmpeg-devel <[email protected]> Mon, 27 Jul 2026 12:37:56 -0000
| Newsgroups | gmane.comp.video.ffmpeg.devel |
|---|---|
| Message-ID | <178515587782.59.17500427320283342782@29965ddac10e> |
PR #23923 opened by Ramiro Polla (ramiro) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23923 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23923.patch This PR implements the AArch64 NEON JIT backend. A single specialized function is generated for a pair of src/dst pixel formats. The kernels themselves use the same code as the CPS backend (ops_asmgen.c), but the loading of constants is factored out of the main loop, and the continuation-passing itself is avoided since the JIT compiler will stitch them together at runtime. The backend depends on LLVM to assemble the code generated at runtime. The following speedup is observed from the CPS backend to JIT: A55: Overall speedup=1.300x faster, min=0.521x max=3.154x A76: Overall speedup=1.166x faster, min=0.602x max=2.433x The 0.521x and 0.602x outliers happen on pathological cases of instruction cache misses. Sponsored-by: Sovereign Tech Fund >From 4368cc260c8d83bf9df31898e664dc354756691b Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Fri, 3 Jul 2026 16:21:48 +0200 Subject: [PATCH 01/10] swscale/aarch64/rasm: add support for const data Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/rasm.c | 74 ++++++++++++++++++++++++++++++++ libswscale/aarch64/rasm.h | 54 ++++++++++++++++++++++- libswscale/aarch64/rasm_print.c | 76 ++++++++++++++++++++++++++++++++- 3 files changed, 200 insertions(+), 4 deletions(-) diff --git a/libswscale/aarch64/rasm.c b/libswscale/aarch64/rasm.c index b866e1d3ff..0cc7b4542c 100644 --- a/libswscale/aarch64/rasm.c +++ b/libswscale/aarch64/rasm.c @@ -51,6 +51,9 @@ void rasm_free(RasmContext **prctx) case RASM_NODE_DIRECTIVE: av_freep(&node->directive.text); break; + case RASM_NODE_DATA: + av_freep(&node->data.data); + break; default: break; } @@ -191,6 +194,46 @@ RasmNode *rasm_add_directive(RasmContext *rctx, const char *text) return node; } +RasmNode *rasm_add_data(RasmContext *rctx, const void *data, unsigned count, + RasmDataType type) +{ + if (rctx->error) + return NULL; + + size_t size = count * rasm_data_type_size(type); + void *dup = av_memdup(data, size); + if (!dup) { + rctx->error = AVERROR(ENOMEM); + return NULL; + } + + RasmNode *node = add_node(rctx, RASM_NODE_DATA); + if (node) { + node->data.data = dup; + node->data.count = count; + node->data.type = type; + } else { + av_freep(&dup); + } + return node; +} + +RasmNode *rasm_add_const(RasmContext *rctx, int id) +{ + RasmNode *node = add_node(rctx, RASM_NODE_CONST); + if (node) { + av_assert0(id >= 0 && id < rctx->num_labels); + node->konst.name = rctx->labels[id]; + } + return node; +} + +RasmNode *rasm_add_endconst(RasmContext *rctx) +{ + RasmNode *node = add_node(rctx, RASM_NODE_ENDCONST); + return node; +} + RasmNode *rasm_get_current_node(RasmContext *rctx) { return rctx->current_node; @@ -239,6 +282,37 @@ int rasm_func_begin(RasmContext *rctx, const char *name, bool export, return id; } +int rasm_const_begin(RasmContext *rctx, const char *name) +{ + if (rctx->error) + return rctx->error; + + /* Grow entries array. */ + RasmEntry *entry = av_dynarray2_add((void **) &rctx->entries, + &rctx->num_entries, + sizeof(*rctx->entries), NULL); + if (!entry) { + rctx->error = AVERROR(ENOMEM); + return rctx->error; + } + + entry->type = RASM_ENTRY_CONST; + + int id = rasm_new_label(rctx, name); + + rasm_set_current_node(rctx, NULL); + entry->start = rasm_add_const(rctx, id); + entry->end = rasm_add_endconst(rctx); + rasm_set_current_node(rctx, entry->start); + + entry->konst.label_id = id; + + if (rctx->error) + return rctx->error; + + return id; +} + /*********************************************************************/ void rasm_annotate(RasmContext *rctx, const char *comment) { diff --git a/libswscale/aarch64/rasm.h b/libswscale/aarch64/rasm.h index 3d28f8099a..4ff2557436 100644 --- a/libswscale/aarch64/rasm.h +++ b/libswscale/aarch64/rasm.h @@ -105,6 +105,32 @@ static inline int rasm_op_label_id(RasmOp op) return (int) op.u16[0]; } +/*********************************************************************/ +/* Data types */ + +typedef enum RasmDataType { + RASM_DATA_BYTE = 0, + RASM_DATA_SHORT, + RASM_DATA_WORD, + RASM_DATA_QUAD, + + RASM_DATA_NB, +} RasmDataType; + +static inline unsigned rasm_data_type_size(RasmDataType type) +{ + switch (type) { + case RASM_DATA_BYTE: return 1; break; + case RASM_DATA_SHORT: return 2; break; + case RASM_DATA_WORD: return 4; break; + case RASM_DATA_QUAD: return 8; break; + default: + break; + } + av_assert0(0); + return 0; +} + /*********************************************************************/ /* IR Nodes */ @@ -115,7 +141,9 @@ typedef enum RasmNodeType { RASM_NODE_FUNCTION, RASM_NODE_ENDFUNC, RASM_NODE_DIRECTIVE, - RASM_NODE_DATA, /* NOTE not yet implemented */ + RASM_NODE_DATA, + RASM_NODE_CONST, + RASM_NODE_ENDCONST, } RasmNodeType; typedef struct RasmNodeInsn { @@ -141,6 +169,16 @@ typedef struct RasmNodeDirective { char *text; } RasmNodeDirective; +typedef struct RasmNodeData { + void *data; + unsigned count; + RasmDataType type; +} RasmNodeData; + +typedef struct RasmNodeConst { + char *name; +} RasmNodeConst; + /* A single node in the IR. */ typedef struct RasmNode { RasmNodeType type; @@ -150,6 +188,8 @@ typedef struct RasmNode { RasmNodeLabel label; RasmNodeFunc func; RasmNodeDirective directive; + RasmNodeData data; + RasmNodeConst konst; }; char *inline_comment; struct RasmNode *prev; @@ -161,7 +201,7 @@ typedef struct RasmNode { typedef enum RasmEntryType { RASM_ENTRY_FUNC, - RASM_ENTRY_DATA, /* NOTE not yet implemented */ + RASM_ENTRY_CONST, } RasmEntryType; typedef struct RasmFunction { @@ -169,6 +209,10 @@ typedef struct RasmFunction { int label_id; } RasmFunction; +typedef struct RasmConst { + int label_id; +} RasmConst; + /* A contiguous range of nodes. */ typedef struct RasmEntry { RasmEntryType type; @@ -176,6 +220,7 @@ typedef struct RasmEntry { RasmNode *end; union { RasmFunction func; + RasmConst konst; }; } RasmEntry; @@ -206,6 +251,10 @@ RasmNode *rasm_add_func(RasmContext *rctx, int id, bool export, bool jumpable); RasmNode *rasm_add_endfunc(RasmContext *rctx); RasmNode *rasm_add_directive(RasmContext *rctx, const char *text); +RasmNode *rasm_add_data(RasmContext *rctx, const void *data, unsigned count, + RasmDataType type); +RasmNode *rasm_add_const(RasmContext *rctx, int id); +RasmNode *rasm_add_endconst(RasmContext *rctx); RasmNode *rasm_get_current_node(RasmContext *rctx); RasmNode *rasm_set_current_node(RasmContext *rctx, RasmNode *node); @@ -213,6 +262,7 @@ RasmNode *rasm_set_current_node(RasmContext *rctx, RasmNode *node); /* Top-level IR entries */ int rasm_func_begin(RasmContext *rctx, const char *name, bool export, bool jumpable); +int rasm_const_begin(RasmContext *rctx, const char *name); /** * Allocate a new label ID with the given name. diff --git a/libswscale/aarch64/rasm_print.c b/libswscale/aarch64/rasm_print.c index c32edaa491..6993f6fb31 100644 --- a/libswscale/aarch64/rasm_print.c +++ b/libswscale/aarch64/rasm_print.c @@ -18,6 +18,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include <inttypes.h> #include <stdarg.h> #include <string.h> @@ -418,6 +419,68 @@ static void print_node_directive(const RasmContext *rctx, av_bprintf(bp, "%s", node->directive.text); } +/*********************************************************************/ +/* RASM_NODE_DATA */ + +static const char data_type_names[RASM_DATA_NB][8] = { + [RASM_DATA_BYTE] = ".byte", + [RASM_DATA_SHORT] = ".short", + [RASM_DATA_WORD] = ".word", + [RASM_DATA_QUAD] = ".quad", +}; + +static unsigned data_type_wrap[RASM_DATA_NB] = { + [RASM_DATA_BYTE] = 16, + [RASM_DATA_SHORT] = 8, + [RASM_DATA_WORD] = 4, + [RASM_DATA_QUAD] = 2, +}; + +static void print_node_data(const RasmContext *rctx, + AVBPrint *bp, unsigned line_start, + const RasmNode *node) +{ + for (unsigned i = 0; i < node->data.count; i++) { + if (!(i & (data_type_wrap[node->data.type] - 1))) { + if (i > 0) + av_bprintf(bp, "\n"); + indent_to(bp, bp->len, INSTR_INDENT); + av_bprintf(bp, "%s", data_type_names[node->data.type]); + } else { + av_bprintf(bp, ","); + } + + switch (node->data.type) { + case RASM_DATA_BYTE: av_bprintf(bp, " 0x%02x", ((const uint8_t *) node->data.data)[i]); break; + case RASM_DATA_SHORT: av_bprintf(bp, " 0x%04x", ((const uint16_t *) node->data.data)[i]); break; + case RASM_DATA_WORD: av_bprintf(bp, " 0x%08x", ((const uint32_t *) node->data.data)[i]); break; + case RASM_DATA_QUAD: av_bprintf(bp, " 0x%016" PRIx64, ((const uint64_t *) node->data.data)[i]); break; + default: + break; + } + } +} + +/*********************************************************************/ +/* RASM_NODE_CONST */ + +static void print_node_const(const RasmContext *rctx, + AVBPrint *bp, unsigned line_start, + const RasmNode *node) +{ + av_bprintf(bp, "const %s", node->konst.name); +} + +/*********************************************************************/ +/* RASM_NODE_ENDCONST */ + +static void print_node_endconst(const RasmContext *rctx, + AVBPrint *bp, unsigned line_start, + const RasmNode *node) +{ + av_bprintf(bp, "endconst"); +} + /*********************************************************************/ int rasm_print(RasmContext *rctx, AVBPrint *bp) { @@ -470,6 +533,15 @@ int rasm_print(RasmContext *rctx, AVBPrint *bp) case RASM_NODE_DIRECTIVE: print_node_directive(rctx, bp, line_start, node); break; + case RASM_NODE_DATA: + print_node_data(rctx, bp, line_start, node); + break; + case RASM_NODE_CONST: + print_node_const(rctx, bp, line_start, node); + break; + case RASM_NODE_ENDCONST: + print_node_endconst(rctx, bp, line_start, node); + break; default: break; } @@ -480,8 +552,8 @@ int rasm_print(RasmContext *rctx, AVBPrint *bp) } av_bprintf(bp, "\n"); - /* Add extra line after end of functions. */ - if (node->type == RASM_NODE_ENDFUNC) + /* Add extra line after end of function/const blocks. */ + if (node->type == RASM_NODE_ENDFUNC || node->type == RASM_NODE_ENDCONST) av_bprintf(bp, "\n"); } } -- 2.52.0 >From 5377cf11c47ef7a8dd46326aae594f933d210d65 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 26 Jul 2026 20:12:01 +0200 Subject: [PATCH 02/10] swscale/aarch64/rasm: remove vv_n() veclist helpers Convert a64op_veclist() to take a const RasmOp * pointer along with the number of registers to use from the array instead. Also split out a64op_contiguous_vecs(), which will come in handy for the JIT compiler. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 12 +++++------ libswscale/aarch64/ops_static.c | 10 ++------- libswscale/aarch64/rasm.h | 38 +++++++++++++-------------------- 3 files changed, 23 insertions(+), 37 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 6068fc327a..c449a26fe6 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -268,9 +268,9 @@ static void asmgen_op_read_packed_n(SwsAArch64Context *s, const SwsAArch64OpImpl RasmContext *r = s->rctx; switch (p->mask) { - case SWS_COMP_ELEMS(2): i_ld2(r, vv_2(vx[0], vx[1]), a64op_post(s->in[0], s->vec_size * 2)); break; - case SWS_COMP_ELEMS(3): i_ld3(r, vv_3(vx[0], vx[1], vx[2]), a64op_post(s->in[0], s->vec_size * 3)); break; - case SWS_COMP_ELEMS(4): i_ld4(r, vv_4(vx[0], vx[1], vx[2], vx[3]), a64op_post(s->in[0], s->vec_size * 4)); break; + case SWS_COMP_ELEMS(2): i_ld2(r, a64op_veclist(vx, 2), a64op_post(s->in[0], s->vec_size * 2)); break; + case SWS_COMP_ELEMS(3): i_ld3(r, a64op_veclist(vx, 3), a64op_post(s->in[0], s->vec_size * 3)); break; + case SWS_COMP_ELEMS(4): i_ld4(r, a64op_veclist(vx, 4), a64op_post(s->in[0], s->vec_size * 4)); break; } } @@ -359,9 +359,9 @@ static void asmgen_op_write_packed_n(SwsAArch64Context *s, const SwsAArch64OpImp RasmContext *r = s->rctx; switch (p->mask) { - case SWS_COMP_ELEMS(2): i_st2(r, vv_2(vx[0], vx[1]), a64op_post(s->out[0], s->vec_size * 2)); break; - case SWS_COMP_ELEMS(3): i_st3(r, vv_3(vx[0], vx[1], vx[2]), a64op_post(s->out[0], s->vec_size * 3)); break; - case SWS_COMP_ELEMS(4): i_st4(r, vv_4(vx[0], vx[1], vx[2], vx[3]), a64op_post(s->out[0], s->vec_size * 4)); break; + case SWS_COMP_ELEMS(2): i_st2(r, a64op_veclist(vx, 2), a64op_post(s->out[0], s->vec_size * 2)); break; + case SWS_COMP_ELEMS(3): i_st3(r, a64op_veclist(vx, 3), a64op_post(s->out[0], s->vec_size * 3)); break; + case SWS_COMP_ELEMS(4): i_st4(r, a64op_veclist(vx, 4), a64op_post(s->out[0], s->vec_size * 4)); break; } } diff --git a/libswscale/aarch64/ops_static.c b/libswscale/aarch64/ops_static.c index 9986dd3032..ff8d26be15 100644 --- a/libswscale/aarch64/ops_static.c +++ b/libswscale/aarch64/ops_static.c @@ -271,7 +271,7 @@ static void asmgen_setup_scale(SwsAArch64Context *s, const SwsAArch64OpImplParam RasmOp priv_ptr = s->tmp0; i_add (r, priv_ptr, s->impl, IMM(offsetof_impl_priv)); CMT("v128 *scale_vec_ptr = &impl->priv;"); asmgen_set_load_cont_node(s); - i_ld1r(r, vv_1(scale_vec), a64op_base(priv_ptr)); CMT("v128 scale_vec = broadcast(*scale_vec_ptr);"); + i_ld1r(r, a64op_veclist(&scale_vec, 1), a64op_base(priv_ptr)); CMT("v128 scale_vec = broadcast(*scale_vec_ptr);"); } static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, @@ -284,17 +284,11 @@ static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplPara RasmOp *vt = regs->vt; RasmOp ptr = s->tmp0; - RasmOp coeff_veclist; /* Preload coefficients from impl->priv. */ const int num_vregs = linear_num_vregs(p); av_assert0(num_vregs <= 4); - switch (num_vregs) { - case 1: coeff_veclist = vv_1(vc[0]); break; - case 2: coeff_veclist = vv_2(vc[0], vc[1]); break; - case 3: coeff_veclist = vv_3(vc[0], vc[1], vc[2]); break; - case 4: coeff_veclist = vv_4(vc[0], vc[1], vc[2], vc[3]); break; - } + RasmOp coeff_veclist = a64op_veclist(vc, num_vregs); i_ldr(r, ptr, IMPL_PRIV(s)); CMT("v128 *vcoeff_ptr = impl->priv.ptr;"); asmgen_set_load_cont_node(s); i_ld1(r, coeff_veclist, a64op_base(ptr)); CMT("coeff_veclist = *vcoeff_ptr;"); diff --git a/libswscale/aarch64/rasm.h b/libswscale/aarch64/rasm.h index 4ff2557436..625e9b8022 100644 --- a/libswscale/aarch64/rasm.h +++ b/libswscale/aarch64/rasm.h @@ -444,28 +444,26 @@ static inline RasmOp a64op_vec2s (uint8_t n) { return a64op_make_vec(n, 2, 4); static inline RasmOp a64op_vec4s (uint8_t n) { return a64op_make_vec(n, 4, 4); } static inline RasmOp a64op_vec2d (uint8_t n) { return a64op_make_vec(n, 2, 8); } +/* Check whether the vectors in ops are contiguous. */ +static inline bool a64op_contiguous_vecs(const RasmOp *ops, uint8_t num_regs) +{ + for (int i = 1; i < num_regs; i++) { + if (((a64op_vec_n(ops[i - 1]) + 1) & 0x1f) != a64op_vec_n(ops[i])) + return false; + } + return true; +} + /** * Create register-list operand for structured load/store instructions. * Registers must be consecutive. */ -static inline RasmOp a64op_veclist(RasmOp op0, RasmOp op1, RasmOp op2, RasmOp op3) +static inline RasmOp a64op_veclist(const RasmOp *ops, uint8_t num_regs) { - av_assert0(rasm_op_type(op0) != RASM_OP_NONE); - uint8_t num_regs = 1; - if (rasm_op_type(op1) != RASM_OP_NONE) { - av_assert0(((a64op_vec_n(op0) + 1) & 0x1f) == a64op_vec_n(op1)); - num_regs++; - if (rasm_op_type(op2) != RASM_OP_NONE) { - av_assert0(((a64op_vec_n(op1) + 1) & 0x1f) == a64op_vec_n(op2)); - num_regs++; - if (rasm_op_type(op3) != RASM_OP_NONE) { - av_assert0(((a64op_vec_n(op2) + 1) & 0x1f) == a64op_vec_n(op3)); - num_regs++; - } - } - } - op0.u8[3] = num_regs; - return op0; + av_assert0(a64op_contiguous_vecs(ops, num_regs)); + RasmOp op = ops[0]; + op.u8[3] = num_regs; + return op; } /* by-element modifier */ @@ -492,12 +490,6 @@ static inline RasmOp v_2s (RasmOp op) { return a64op_vec2s (a64op_vec_n(op)); } static inline RasmOp v_4s (RasmOp op) { return a64op_vec4s (a64op_vec_n(op)); } static inline RasmOp v_2d (RasmOp op) { return a64op_vec2d (a64op_vec_n(op)); } -/* register-list modifiers */ -static inline RasmOp vv_1(RasmOp op0) { return a64op_veclist(op0, OPN, OPN, OPN); } -static inline RasmOp vv_2(RasmOp op0, RasmOp op1) { return a64op_veclist(op0, op1, OPN, OPN); } -static inline RasmOp vv_3(RasmOp op0, RasmOp op1, RasmOp op2) { return a64op_veclist(op0, op1, op2, OPN); } -static inline RasmOp vv_4(RasmOp op0, RasmOp op1, RasmOp op2, RasmOp op3) { return a64op_veclist(op0, op1, op2, op3); } - /** * This helper structure is used to mimic the assembler syntax for vector * register modifiers. This simplifies writing code with expressions such -- 2.52.0 >From 5240f086b242fac0a703114f85c8b2c88fb6872b Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sun, 12 Jul 2026 18:02:49 +0200 Subject: [PATCH 03/10] swscale/aarch64/rasm: add a register state tracker and prologue/epilogue emitter The emitter is a refined version from the code in ops_asmgen.c, which also supports vector registers. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.h | 1 + libswscale/aarch64/ops_static.c | 25 +++++ libswscale/aarch64/rasm.c | 175 ++++++++++++++++++++++++++++++++ libswscale/aarch64/rasm.h | 68 +++++++++++++ 4 files changed, 269 insertions(+) diff --git a/libswscale/aarch64/ops_asmgen.h b/libswscale/aarch64/ops_asmgen.h index d77fcabe4d..88836c6ac3 100644 --- a/libswscale/aarch64/ops_asmgen.h +++ b/libswscale/aarch64/ops_asmgen.h @@ -73,6 +73,7 @@ typedef struct SwsAArch64Context { RasmOp out_bump[4]; /* Process function. */ + AArch64RegState regstate; RasmNode *setup; RasmNode *loop; diff --git a/libswscale/aarch64/ops_static.c b/libswscale/aarch64/ops_static.c index ff8d26be15..dabf2758f0 100644 --- a/libswscale/aarch64/ops_static.c +++ b/libswscale/aarch64/ops_static.c @@ -37,6 +37,7 @@ */ #define AVUTIL_AVASSERT_H +#define AVUTIL_INTMATH_H #define AVUTIL_LOG_H #define AVUTIL_MACROS_H #define AVUTIL_MEM_H @@ -51,6 +52,30 @@ #define FF_ARRAY_ELEMS(a) (sizeof(a) / sizeof((a)[0])) #define MKTAG(a,b,c,d) ((a) | ((b) << 8) | ((c) << 16) | ((unsigned)(d) << 24)) +static int ff_ctz(uint32_t mask) +{ + if (!mask) + return 32; + int n = 0; + while (!(mask & 1)) { + mask >>= 1; + n++; + } + return n; +} + +static int ff_clz(uint32_t mask) +{ + if (!mask) + return 32; + int n = 0; + while (!(mask & 0x80000000u)) { + mask <<= 1; + n++; + } + return n; +} + static void av_freep(void *ptr) { void **pptr = (void **) ptr; diff --git a/libswscale/aarch64/rasm.c b/libswscale/aarch64/rasm.c index 0cc7b4542c..e6af3d4254 100644 --- a/libswscale/aarch64/rasm.c +++ b/libswscale/aarch64/rasm.c @@ -23,6 +23,7 @@ #include <stdarg.h> #include "libavutil/error.h" +#include "libavutil/intmath.h" #include "libavutil/macros.h" #include "libavutil/mem.h" @@ -424,3 +425,177 @@ AArch64VecViews a64op_vec_views(RasmOp op) out.de[i] = a64op_elem(out.d, i); return out; } + +/*********************************************************************/ +/* AArch64 register state tracker */ + +#define AARCH64_GPR_PR (1 << 18u) /* Platform Register */ +#define AARCH64_GPR_SP (1 << 31u) /* Stack Pointer */ + +/* Callee-saved GPRs (r19-r28, fp, and lr). */ +#define AARCH64_GPR_CALLEE_SAVED 0x7ff80000u +#define AARCH64_GPR_CALLEE_SAVED_COUNT 12 + +/* Callee-saved vector registers (bottom 64-bit of v8-v15). */ +#define AARCH64_VEC_CALLEE_SAVED 0x0000ff00u +#define AARCH64_VEC_CALLEE_SAVED_COUNT 8 + +typedef enum AArch64RegPick { + AARCH64_REG_PICK_LOWEST, + AARCH64_REG_PICK_HIGHEST, +} AArch64RegPick; + +static int pick_avail(uint32_t avail, AArch64RegPick pick) +{ + switch (pick) { + case AARCH64_REG_PICK_LOWEST: + return ff_ctz(avail); + case AARCH64_REG_PICK_HIGHEST: + return 31 - ff_clz(avail); + } + return -1; +} + +/* GPRs */ +static int pick_gpr(uint32_t mask, AArch64RegPick pick) +{ + uint32_t avail = ~(mask | AARCH64_GPR_PR | AARCH64_GPR_SP); + av_assert0(avail); + /* Use callee-saved registers last. */ + if (avail & ~AARCH64_GPR_CALLEE_SAVED) + return pick_avail(avail & ~AARCH64_GPR_CALLEE_SAVED, pick); + return pick_avail(avail, pick); +} + +int a64reg_pick_unused_gpr(AArch64RegState *rs, int r) +{ + if (r < 0) { + r = pick_gpr(rs->gpr_used, AARCH64_REG_PICK_LOWEST); + } else { + av_assert0(r >= 0 && r <= 30); + } + rs->gpr_used |= 1u << r; + rs->gpr_clobbered |= 1u << r; + return r; +} + +int a64reg_pick_unclobbered_gpr(AArch64RegState *rs) +{ + int r = pick_gpr(rs->gpr_clobbered, AARCH64_REG_PICK_HIGHEST); + rs->gpr_used |= 1u << r; + rs->gpr_clobbered |= 1u << r; + return r; +} + +/* Vector registers */ +static int pick_vec(uint32_t mask, AArch64RegPick pick) +{ + uint32_t avail = ~mask; + av_assert0(avail); + /* Use callee-saved registers last. */ + if (avail & ~AARCH64_VEC_CALLEE_SAVED) + return pick_avail(avail & ~AARCH64_VEC_CALLEE_SAVED, pick); + return pick_avail(avail, pick); +} + +int a64reg_pick_unused_vec(AArch64RegState *rs, int r) +{ + if (r < 0) { + r = pick_vec(rs->vec_used, AARCH64_REG_PICK_LOWEST); + } else { + av_assert0(r >= 0 && r <= 31); + } + rs->vec_used |= 1u << r; + rs->vec_clobbered |= 1u << r; + return r; +} + +int a64reg_pick_unclobbered_vec(AArch64RegState *rs) +{ + int r = pick_vec(rs->vec_clobbered, AARCH64_REG_PICK_HIGHEST); + rs->vec_used |= 1u << r; + rs->vec_clobbered |= 1u << r; + return r; +} + +static int pick_contiguous_vec(uint32_t avail, int num_regs) +{ + uint32_t mask = (1u << num_regs) - 1; + for (int i = 0; i <= (32 - num_regs); i++) { + if ((avail & (mask << i)) == (mask << i)) + return i; + } + return -1; +} + +int a64reg_pick_unused_veclist(AArch64RegState *rs, int num_regs) +{ + uint32_t avail = ~rs->vec_used; + av_assert0(avail); + /* Use callee-saved registers last. */ + int r = pick_contiguous_vec(avail & ~AARCH64_VEC_CALLEE_SAVED, num_regs); + if (r < 0) + r = pick_contiguous_vec(avail, num_regs); + av_assert0(r >= 0); + rs->vec_used |= ((1u << num_regs) - 1) << r; + rs->vec_clobbered |= ((1u << num_regs) - 1) << r; + return r; +} + +void a64reg_emit(RasmContext *rctx, const AArch64RegState *rs, + RasmNode *prologue, RasmNode *epilogue) +{ + /* Collect clobbered registers and compute frame size. */ + RasmOp regs[AARCH64_GPR_CALLEE_SAVED_COUNT + AARCH64_VEC_CALLEE_SAVED_COUNT]; + unsigned n = 0; + for (unsigned i = 0; i <= 30; i++) { + if (rs->gpr_clobbered & AARCH64_GPR_CALLEE_SAVED & (1u << i)) + regs[n++] = a64op_gpx(i); + } + if (n & 1) + regs[n++] = rasm_op_none(); + for (unsigned i = 0; i <= 31; i++) { + if (rs->vec_clobbered & AARCH64_VEC_CALLEE_SAVED & (1u << i)) + regs[n++] = a64op_vecd(i); + } + if (n & 1) + regs[n++] = rasm_op_none(); + if (!n) + return; + unsigned frame_size = n * sizeof(uint64_t); + + RasmNode *saved = rasm_get_current_node(rctx); + RasmOp sp = a64op_sp(); + RasmOp sp_pre = a64op_pre(sp, -frame_size); + RasmOp sp_post = a64op_post(sp, frame_size); + + /* Emit prologue. */ + rasm_set_current_node(rctx, prologue); + rasm_add_comment(rctx, "prologue"); + if (rasm_op_type(regs[1]) == RASM_OP_NONE) + i_str(rctx, regs[0], sp_pre); + else + i_stp(rctx, regs[0], regs[1], sp_pre); + for (unsigned i = 2; i < n; i += 2) { + if (rasm_op_type(regs[i + 1]) == RASM_OP_NONE) + i_str(rctx, regs[i], a64op_off(sp, i * sizeof(uint64_t))); + else + i_stp(rctx, regs[i], regs[i + 1], a64op_off(sp, i * sizeof(uint64_t))); + } + + /* Emit epilogue. */ + rasm_set_current_node(rctx, epilogue); + rasm_add_comment(rctx, "epilogue"); + for (unsigned i = n - 2; i >= 2; i -= 2) { + if (rasm_op_type(regs[i + 1]) == RASM_OP_NONE) + i_ldr(rctx, regs[i], a64op_off(sp, i * sizeof(uint64_t))); + else + i_ldp(rctx, regs[i], regs[i + 1], a64op_off(sp, i * sizeof(uint64_t))); + } + if (rasm_op_type(regs[1]) == RASM_OP_NONE) + i_ldr(rctx, regs[0], sp_post); + else + i_ldp(rctx, regs[0], regs[1], sp_post); + + rasm_set_current_node(rctx, saved); +} diff --git a/libswscale/aarch64/rasm.h b/libswscale/aarch64/rasm.h index 625e9b8022..92e2378256 100644 --- a/libswscale/aarch64/rasm.h +++ b/libswscale/aarch64/rasm.h @@ -581,6 +581,74 @@ static inline RasmOp a64cond_le(void) { return a64op_cond(AARCH64_COND_LE); } static inline RasmOp a64cond_al(void) { return a64op_cond(AARCH64_COND_AL); } static inline RasmOp a64cond_nv(void) { return a64op_cond(AARCH64_COND_NV); } +/*********************************************************************/ +/* AArch64 register state tracker */ + +typedef struct AArch64RegState { + uint32_t gpr_used; + uint32_t gpr_clobbered; + uint32_t vec_used; + uint32_t vec_clobbered; +} AArch64RegState; + +/* GPRs */ +int a64reg_pick_unused_gpr (AArch64RegState *rs, int r); +int a64reg_pick_unclobbered_gpr(AArch64RegState *rs); + +static inline void a64reg_gpr_free(AArch64RegState *rs, RasmOp op) +{ + rs->gpr_used &= ~(1u << a64op_gpr_n(op)); +} + +static inline void a64reg_gpr_clobber(AArch64RegState *rs, RasmOp op) +{ + rs->gpr_clobbered |= 1u << a64op_gpr_n(op); +} + +static inline RasmOp a64reg_gpx (AArch64RegState *rs, int r){ return a64op_gpx(a64reg_pick_unused_gpr(rs, r)); } +static inline RasmOp a64reg_gpw (AArch64RegState *rs, int r){ return a64op_gpw(a64reg_pick_unused_gpr(rs, r)); } +static inline RasmOp a64reg_unclobbered_gpx(AArch64RegState *rs) { return a64op_gpx(a64reg_pick_unclobbered_gpr(rs)); } +static inline RasmOp a64reg_unclobbered_gpw(AArch64RegState *rs) { return a64op_gpw(a64reg_pick_unclobbered_gpr(rs)); } + +/* Function arguments */ +static inline int a64reg_arg(AArch64RegState *rs, int argnum) +{ + av_assert0(argnum >= 0 && argnum < 8); + return a64reg_pick_unused_gpr(rs, argnum); +} + +static inline RasmOp a64reg_argx(AArch64RegState *rs, int argnum) { return a64op_gpx(a64reg_arg(rs, argnum)); } +static inline RasmOp a64reg_argw(AArch64RegState *rs, int argnum) { return a64op_gpw(a64reg_arg(rs, argnum)); } + +/* Vector registers */ +int a64reg_pick_unused_vec (AArch64RegState *rs, int r); +int a64reg_pick_unclobbered_vec(AArch64RegState *rs); +int a64reg_pick_unused_veclist (AArch64RegState *rs, int num_regs); + +static inline void a64reg_vec_free(AArch64RegState *rs, RasmOp op) +{ + rs->vec_used &= ~(1u << a64op_vec_n(op)); +} + +static inline void a64reg_vec_clobber(AArch64RegState *rs, RasmOp op) +{ + rs->vec_clobbered |= 1u << a64op_vec_n(op); +} + +static inline RasmOp a64reg_vec (AArch64RegState *rs, int r) { return a64op_vec(a64reg_pick_unused_vec(rs, r)); } +static inline RasmOp a64reg_unclobbered_vec(AArch64RegState *rs) { return a64op_vec(a64reg_pick_unclobbered_vec(rs)); } + +static inline void a64reg_veclist_ops(AArch64RegState *rs, int num_regs, RasmOp *ops) +{ + int r = a64reg_pick_unused_veclist(rs, num_regs); + for (int i = 0; i < num_regs; i++) + ops[i] = a64op_vec(r + i); +} + +/* Emits prologue and epilogue for callee-saved clobbered registers. */ +void a64reg_emit(RasmContext *rctx, const AArch64RegState *rs, + RasmNode *prologue, RasmNode *epilogue); + /*********************************************************************/ /* Helpers to add instructions. */ -- 2.52.0 >From beea8aab2a6538a4a81e594edad157fabedbe234 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Sat, 18 Jul 2026 01:41:18 +0200 Subject: [PATCH 04/10] swscale/aarch64/ops_asmgen: use register state tracker and prologue/epilogue emitter from rasm Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 88 ++---------------------- libswscale/aarch64/ops_static.c | 115 +++++++++++++++++--------------- 2 files changed, 67 insertions(+), 136 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index c449a26fe6..6e922bc2b4 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -58,84 +58,6 @@ static void reshape_const_vectors(SwsAArch64OpRegs *regs, int el_count, int el_s } /*********************************************************************/ -/* Function frame */ - -static unsigned clobbered_frame_size(unsigned n) -{ - return ((n + 1) >> 1) * 16; -} - -static void asmgen_prologue(SwsAArch64Context *s, const RasmOp *regs, unsigned n) -{ - RasmContext *r = s->rctx; - RasmOp sp = a64op_sp(); - unsigned frame_size = clobbered_frame_size(n); - RasmOp sp_pre = a64op_pre(sp, -frame_size); - - rasm_add_comment(r, "prologue"); - if (n == 0) { - /* no-op */ - } else if (n == 1) { - i_str(r, regs[0], sp_pre); - } else { - i_stp(r, regs[0], regs[1], sp_pre); - for (unsigned i = 2; i + 1 < n; i += 2) - i_stp(r, regs[i], regs[i + 1], a64op_off(sp, i * sizeof(uint64_t))); - if (n & 1) - i_str(r, regs[n - 1], a64op_off(sp, (n - 1) * sizeof(uint64_t))); - } -} - -static void asmgen_epilogue(SwsAArch64Context *s, const RasmOp *regs, unsigned n) -{ - RasmContext *r = s->rctx; - RasmOp sp = a64op_sp(); - unsigned frame_size = clobbered_frame_size(n); - RasmOp sp_post = a64op_post(sp, frame_size); - - rasm_add_comment(r, "epilogue"); - if (n == 0) { - /* no-op */ - } else if (n == 1) { - i_ldr(r, regs[0], sp_post); - } else { - if (n & 1) - i_ldr(r, regs[n - 1], a64op_off(sp, (n - 1) * sizeof(uint64_t))); - for (unsigned i = (n & ~1u) - 2; i >= 2; i -= 2) - i_ldp(r, regs[i], regs[i + 1], a64op_off(sp, i * sizeof(uint64_t))); - i_ldp(r, regs[0], regs[1], sp_post); - } -} - -/*********************************************************************/ -/* Callee-saved registers (r19-r28, fp, and lr). */ -#define MAX_SAVED_REGS 12 - -static void clobber_gpr(RasmOp regs[MAX_SAVED_REGS], unsigned *count, - RasmOp gpr) -{ - const int n = a64op_gpr_n(gpr); - if (n >= 19 && n <= 30) - regs[(*count)++] = gpr; -} - -static unsigned clobbered_gprs(const SwsAArch64Context *s, - SwsCompMask imask, SwsCompMask omask, - RasmOp regs[MAX_SAVED_REGS]) -{ - unsigned count = 0; - clobber_gpr(regs, &count, a64op_lr()); - LOOP(imask, i) { - clobber_gpr(regs, &count, s->in[i]); - clobber_gpr(regs, &count, s->in_bump[i]); - } - LOOP(omask, i) { - clobber_gpr(regs, &count, s->out[i]); - clobber_gpr(regs, &count, s->out_bump[i]); - } - return count; -} - static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) { RasmContext *r = s->rctx; @@ -146,10 +68,7 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask */ /* Function prologue */ - RasmOp saved_regs[MAX_SAVED_REGS]; - unsigned nsaved = clobbered_gprs(s, imask, omask, saved_regs); - if (nsaved) - asmgen_prologue(s, saved_regs, nsaved); + RasmNode *prologue = rasm_get_current_node(r); /* Load values from exec. */ RasmOp exec_in[4]; @@ -199,10 +118,11 @@ static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask i_bne(r, next_row); CMT(" goto next_row;"); /* Function epilogue */ - if (nsaved) - asmgen_epilogue(s, saved_regs, nsaved); + RasmNode *epilogue = rasm_get_current_node(r); i_ret(r); + + a64reg_emit(r, &s->regstate, prologue, epilogue); } /*********************************************************************/ diff --git a/libswscale/aarch64/ops_static.c b/libswscale/aarch64/ops_static.c index dabf2758f0..5c292054fd 100644 --- a/libswscale/aarch64/ops_static.c +++ b/libswscale/aarch64/ops_static.c @@ -409,85 +409,96 @@ static const int rw_gprs[] = { static void asmgen_common_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) { + AArch64RegState *rs = &s->regstate; + + /* Reset register state. */ + *rs = (AArch64RegState) { 0 }; + /* Loop iterator variables. */ - s->bx = a64op_gpw(6); - s->y = a64op_gpw(3); /* Reused from SwsOpFunc.y_start argument. */ + s->bx = a64reg_gpw(rs, 6); + s->y = a64reg_gpw(rs, 3); /* Reused from SwsOpFunc.y_start argument. */ /* Scratch registers. */ - s->tmp0 = a64op_gpx(16); /* IP0 */ - s->tmp1 = a64op_gpx(17); /* IP1 */ + s->tmp0 = a64reg_gpx(rs, 16); /* IP0 */ + s->tmp1 = a64reg_gpx(rs, 17); /* IP1 */ /* Read/Write data pointers. */ - LOOP(imask, i) { s->in [i] = a64op_gpx(rw_gprs[(i * 4) + 0]); } - LOOP(omask, i) { s->out[i] = a64op_gpx(rw_gprs[(i * 4) + 1]); } + LOOP(imask, i) { s->in [i] = a64reg_gpx(rs, rw_gprs[(i * 4) + 0]); } + LOOP(omask, i) { s->out[i] = a64reg_gpx(rs, rw_gprs[(i * 4) + 1]); } } static void asmgen_process_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) { + AArch64RegState *rs = &s->regstate; + asmgen_common_frame(s, imask, omask); /* SwsOpFunc arguments. */ - s->exec = a64op_gpx(0); // const SwsOpExec *exec - s->impl = a64op_gpx(1); // const void *priv - s->bx_start = a64op_gpw(2); // int bx_start - s->y_start = a64op_gpw(3); // int y_start - s->bx_end = a64op_gpw(4); // int bx_end - s->y_end = a64op_gpw(5); // int y_end + s->exec = a64reg_argx(rs, 0); // const SwsOpExec *exec + s->impl = a64reg_argx(rs, 1); // const void *priv + s->bx_start = a64reg_argw(rs, 2); // int bx_start + s->y_start = a64reg_argw(rs, 3); // int y_start + s->bx_end = a64reg_argw(rs, 4); // int bx_end + s->y_end = a64reg_argw(rs, 5); // int y_end /* CPS-related variables. */ - s->op0_func = a64op_gpx(7); - s->op1_impl = a64op_gpx(8); + s->op0_func = a64reg_gpx(rs, 7); + s->op1_impl = a64reg_gpx(rs, 8); + + /* The link register is clobbered by the call to the first kernel. */ + a64reg_gpr_clobber(rs, a64op_lr()); /* Read/Write data pointer padding. */ - LOOP(imask, i) { s->in_bump [i] = a64op_gpx(rw_gprs[(i * 4) + 2]); } - LOOP(omask, i) { s->out_bump[i] = a64op_gpx(rw_gprs[(i * 4) + 3]); } + LOOP(imask, i) { s->in_bump [i] = a64reg_gpx(rs, rw_gprs[(i * 4) + 2]); } + LOOP(omask, i) { s->out_bump[i] = a64reg_gpx(rs, rw_gprs[(i * 4) + 3]); } } static void asmgen_op_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) { + AArch64RegState *rs = &s->regstate; + asmgen_common_frame(s, imask, omask); /* CPS-related variables. */ - s->cont = a64op_gpx(0); /* Reused from SwsOpFunc.exec argument. */ - s->impl = a64op_gpx(1); /* Same as SwsOpFunc.impl argument. */ + s->cont = a64reg_argx(rs, 0); /* Reused from SwsOpFunc.exec argument. */ + s->impl = a64reg_argx(rs, 1); /* Same as SwsOpFunc.impl argument. */ } -/*********************************************************************/ -/* Vector register assignment. */ static void init_vectors_cps(SwsAArch64Context *s, SwsAArch64OpRegs *regs) { - regs->sl[ 0] = a64op_vec( 0); - regs->sl[ 1] = a64op_vec( 1); - regs->sl[ 2] = a64op_vec( 2); - regs->sl[ 3] = a64op_vec( 3); - regs->sh[ 0] = a64op_vec( 4); - regs->sh[ 1] = a64op_vec( 5); - regs->sh[ 2] = a64op_vec( 6); - regs->sh[ 3] = a64op_vec( 7); - regs->dl[ 0] = a64op_vec( 0); - regs->dl[ 1] = a64op_vec( 1); - regs->dl[ 2] = a64op_vec( 2); - regs->dl[ 3] = a64op_vec( 3); - regs->dh[ 0] = a64op_vec( 4); - regs->dh[ 1] = a64op_vec( 5); - regs->dh[ 2] = a64op_vec( 6); - regs->dh[ 3] = a64op_vec( 7); - regs->vt[ 0] = a64op_vec(16); - regs->vt[ 1] = a64op_vec(17); - regs->vt[ 2] = a64op_vec(18); - regs->vt[ 3] = a64op_vec(19); - regs->vt[ 4] = a64op_vec(20); - regs->vt[ 5] = a64op_vec(21); - regs->vt[ 6] = a64op_vec(22); - regs->vt[ 7] = a64op_vec(23); - regs->vt[ 8] = a64op_vec(24); - regs->vt[ 9] = a64op_vec(25); - regs->vt[10] = a64op_vec(26); - regs->vt[11] = a64op_vec(27); - regs->vk[ 0] = a64op_vec(28); - regs->vk[ 1] = a64op_vec(29); - regs->vk[ 2] = a64op_vec(30); - regs->vk[ 3] = a64op_vec(31); + AArch64RegState *rs = &s->regstate; + regs->sl[ 0] = a64reg_vec(rs, 0); + regs->sl[ 1] = a64reg_vec(rs, 1); + regs->sl[ 2] = a64reg_vec(rs, 2); + regs->sl[ 3] = a64reg_vec(rs, 3); + regs->sh[ 0] = a64reg_vec(rs, 4); + regs->sh[ 1] = a64reg_vec(rs, 5); + regs->sh[ 2] = a64reg_vec(rs, 6); + regs->sh[ 3] = a64reg_vec(rs, 7); + regs->dl[ 0] = a64reg_vec(rs, 0); + regs->dl[ 1] = a64reg_vec(rs, 1); + regs->dl[ 2] = a64reg_vec(rs, 2); + regs->dl[ 3] = a64reg_vec(rs, 3); + regs->dh[ 0] = a64reg_vec(rs, 4); + regs->dh[ 1] = a64reg_vec(rs, 5); + regs->dh[ 2] = a64reg_vec(rs, 6); + regs->dh[ 3] = a64reg_vec(rs, 7); + regs->vt[ 0] = a64reg_vec(rs, 16); + regs->vt[ 1] = a64reg_vec(rs, 17); + regs->vt[ 2] = a64reg_vec(rs, 18); + regs->vt[ 3] = a64reg_vec(rs, 19); + regs->vt[ 4] = a64reg_vec(rs, 20); + regs->vt[ 5] = a64reg_vec(rs, 21); + regs->vt[ 6] = a64reg_vec(rs, 22); + regs->vt[ 7] = a64reg_vec(rs, 23); + regs->vt[ 8] = a64reg_vec(rs, 24); + regs->vt[ 9] = a64reg_vec(rs, 25); + regs->vt[10] = a64reg_vec(rs, 26); + regs->vt[11] = a64reg_vec(rs, 27); + regs->vk[ 0] = a64reg_vec(rs, 28); + regs->vk[ 1] = a64reg_vec(rs, 29); + regs->vk[ 2] = a64reg_vec(rs, 30); + regs->vk[ 3] = a64reg_vec(rs, 31); } /*********************************************************************/ -- 2.52.0 >From 7c3985c6525a17c064aaf27b4250dd2e5c7606ae Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Mon, 27 Jul 2026 11:56:16 +0200 Subject: [PATCH 05/10] swscale/aarch64/ops_asmgen: include string.h for memcpy Fixes 1b66342df27. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 6e922bc2b4..7bf4425d0b 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -18,6 +18,8 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include <string.h> + #include "ops_asmgen.h" /*********************************************************************/ -- 2.52.0 >From 72159fe3613995699f36d02757554130cbd3e2eb Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Mon, 27 Jul 2026 12:08:46 +0200 Subject: [PATCH 06/10] swscale/aarch64/ops_asmgen: export some useful functions These common functions will be used by the JIT compiler as well. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 67 ++++++++++++++++++++++++++++++--- libswscale/aarch64/ops_asmgen.h | 13 +++++++ libswscale/aarch64/ops_static.c | 49 ++---------------------- 3 files changed, 78 insertions(+), 51 deletions(-) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index 7bf4425d0b..c908d82fa4 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -25,11 +25,6 @@ /*********************************************************************/ /* Helpers functions. */ -/* Looping when s->use_vh is set. */ -#define LOOP_VH(s, mask, idx) if (s->use_vh) LOOP(mask, idx) -#define LOOP_MASK_VH(s, p, idx) if (s->use_vh) LOOP_MASK(p, idx) -#define LOOP_MASK_BWD_VH(s, p, idx) if (s->use_vh) LOOP_MASK_BWD(p, idx) - /* Inline rasm comments. */ #define CMT(comment) rasm_annotate(r, comment) #define CMTF(fmt, ...) rasm_annotatef(r, (char[128]){0}, 128, fmt, __VA_ARGS__) @@ -60,7 +55,28 @@ static void reshape_const_vectors(SwsAArch64OpRegs *regs, int el_count, int el_s } /*********************************************************************/ -static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) +void ff_sws_aarch64_asmgen_setup_vecs(SwsAArch64Context *s, const SwsAArch64OpImplParams *p) +{ + size_t el_size = ff_sws_pixel_type_size(p->type); + size_t total_size = p->block_size * el_size; + + s->vec_size = FFMIN(total_size, 16); + s->use_vh = (s->vec_size != total_size); + + s->el_size = el_size; + s->el_count = s->vec_size / el_size; +} + +/*********************************************************************/ +void ff_sws_aarch64_asmgen_reshape_vecs(SwsAArch64Context *s, SwsAArch64OpRegs *regs) +{ + reshape_io_vectors(regs, s->el_count, s->el_size); + reshape_temp_vectors(regs, s->el_count, s->el_size); + reshape_const_vectors(regs, s->el_count, s->el_size); +} + +/*********************************************************************/ +void ff_sws_aarch64_asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) { RasmContext *r = s->rctx; @@ -966,3 +982,42 @@ static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams prev_i = i; } } + +/*********************************************************************/ +void ff_sws_aarch64_asmgen_op(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + switch (p->uop) { + case SWS_UOP_READ_BIT: asmgen_op_read_bit(s, p, regs); break; + case SWS_UOP_READ_NIBBLE: asmgen_op_read_nibble(s, p, regs); break; + case SWS_UOP_READ_PACKED: asmgen_op_read_packed(s, p, regs); break; + case SWS_UOP_READ_PLANAR: asmgen_op_read_planar(s, p, regs); break; + case SWS_UOP_WRITE_BIT: asmgen_op_write_bit(s, p, regs); break; + case SWS_UOP_WRITE_NIBBLE: asmgen_op_write_nibble(s, p, regs); break; + case SWS_UOP_WRITE_PACKED: asmgen_op_write_packed(s, p, regs); break; + case SWS_UOP_WRITE_PLANAR: asmgen_op_write_planar(s, p, regs); break; + case SWS_UOP_SWAP_BYTES: asmgen_op_swap_bytes(s, p, regs); break; + case SWS_UOP_PERMUTE: asmgen_op_move(s, p, regs); break; + case SWS_UOP_COPY: asmgen_op_move(s, p, regs); break; + case SWS_UOP_UNPACK: asmgen_op_unpack(s, p, regs); break; + case SWS_UOP_PACK: asmgen_op_pack(s, p, regs); break; + case SWS_UOP_LSHIFT: asmgen_op_lshift(s, p, regs); break; + case SWS_UOP_RSHIFT: asmgen_op_rshift(s, p, regs); break; + case SWS_UOP_CLEAR: asmgen_op_clear(s, p, regs); break; + case SWS_UOP_TO_U8: asmgen_op_convert(s, p, regs); break; + case SWS_UOP_TO_U16: asmgen_op_convert(s, p, regs); break; + case SWS_UOP_TO_U32: asmgen_op_convert(s, p, regs); break; + case SWS_UOP_TO_F32: asmgen_op_convert(s, p, regs); break; + case SWS_UOP_EXPAND_PAIR: asmgen_op_expand(s, p, regs); break; + case SWS_UOP_EXPAND_QUAD: asmgen_op_expand(s, p, regs); break; + case SWS_UOP_MIN: asmgen_op_min(s, p, regs); break; + case SWS_UOP_MAX: asmgen_op_max(s, p, regs); break; + case SWS_UOP_SCALE: asmgen_op_scale(s, p, regs); break; + case SWS_UOP_LINEAR: asmgen_op_linear(s, p, regs); break; + case SWS_UOP_LINEAR_FMA: asmgen_op_linear(s, p, regs); break; + case SWS_UOP_DITHER: asmgen_op_dither(s, p, regs); break; + /* TODO implement SWS_UOP_SHUFFLE */ + default: + break; + } +} diff --git a/libswscale/aarch64/ops_asmgen.h b/libswscale/aarch64/ops_asmgen.h index 88836c6ac3..9bc135d6eb 100644 --- a/libswscale/aarch64/ops_asmgen.h +++ b/libswscale/aarch64/ops_asmgen.h @@ -22,6 +22,7 @@ #define SWSCALE_AARCH64_OPS_ASMGEN_H #include "rasm.h" +#include "ops_impl.h" /*********************************************************************/ typedef struct SwsAArch64OpRegs { @@ -84,4 +85,16 @@ typedef struct SwsAArch64Context { bool use_vh; } SwsAArch64Context; +void ff_sws_aarch64_asmgen_setup_vecs(SwsAArch64Context *s, const SwsAArch64OpImplParams *p); +void ff_sws_aarch64_asmgen_reshape_vecs(SwsAArch64Context *s, SwsAArch64OpRegs *regs); + +void ff_sws_aarch64_asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask); +void ff_sws_aarch64_asmgen_op(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs); + +/* Looping when s->use_vh is set. */ +#define LOOP_VH(s, mask, idx) if (s->use_vh) LOOP(mask, idx) +#define LOOP_MASK_VH(s, p, idx) if (s->use_vh) LOOP_MASK(p, idx) +#define LOOP_MASK_BWD_VH(s, p, idx) if (s->use_vh) LOOP_MASK_BWD(p, idx) + #endif /* SWSCALE_AARCH64_OPS_ASMGEN_H */ diff --git a/libswscale/aarch64/ops_static.c b/libswscale/aarch64/ops_static.c index 5c292054fd..8aa4c79a6f 100644 --- a/libswscale/aarch64/ops_static.c +++ b/libswscale/aarch64/ops_static.c @@ -511,7 +511,7 @@ static void asmgen_process_cps(SwsAArch64Context *s, SwsCompMask mask) rasm_func_begin(r, func_name, true, false); asmgen_process_frame(s, mask, mask); - asmgen_process(s, mask, mask); + ff_sws_aarch64_asmgen_process(s, mask, mask); /* Load values from impl. */ rasm_set_current_node(r, s->setup); @@ -557,18 +557,9 @@ static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) * Set up vector register dimensions and reshape all vectors * accordingly. */ - size_t el_size = ff_sws_pixel_type_size(p->type); - size_t total_size = p->block_size * el_size; - - s->vec_size = FFMIN(total_size, 16); - s->use_vh = (s->vec_size != total_size); - - s->el_size = el_size; - s->el_count = s->vec_size / el_size; + ff_sws_aarch64_asmgen_setup_vecs(s, p); init_vectors_cps(s, &s->regs); - reshape_io_vectors(&s->regs, s->el_count, el_size); - reshape_temp_vectors(&s->regs, s->el_count, el_size); - reshape_const_vectors(&s->regs, s->el_count, el_size); + ff_sws_aarch64_asmgen_reshape_vecs(s, &s->regs); /* Common start for continuation-passing style (CPS) functions. */ asmgen_set_load_cont_node(s); @@ -591,39 +582,7 @@ static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) } /* Emit uop kernel. */ - switch (p->uop) { - case SWS_UOP_READ_BIT: asmgen_op_read_bit(s, p, &s->regs); break; - case SWS_UOP_READ_NIBBLE: asmgen_op_read_nibble(s, p, &s->regs); break; - case SWS_UOP_READ_PACKED: asmgen_op_read_packed(s, p, &s->regs); break; - case SWS_UOP_READ_PLANAR: asmgen_op_read_planar(s, p, &s->regs); break; - case SWS_UOP_WRITE_BIT: asmgen_op_write_bit(s, p, &s->regs); break; - case SWS_UOP_WRITE_NIBBLE: asmgen_op_write_nibble(s, p, &s->regs); break; - case SWS_UOP_WRITE_PACKED: asmgen_op_write_packed(s, p, &s->regs); break; - case SWS_UOP_WRITE_PLANAR: asmgen_op_write_planar(s, p, &s->regs); break; - case SWS_UOP_SWAP_BYTES: asmgen_op_swap_bytes(s, p, &s->regs); break; - case SWS_UOP_PERMUTE: asmgen_op_move(s, p, &s->regs); break; - case SWS_UOP_COPY: asmgen_op_move(s, p, &s->regs); break; - case SWS_UOP_UNPACK: asmgen_op_unpack(s, p, &s->regs); break; - case SWS_UOP_PACK: asmgen_op_pack(s, p, &s->regs); break; - case SWS_UOP_LSHIFT: asmgen_op_lshift(s, p, &s->regs); break; - case SWS_UOP_RSHIFT: asmgen_op_rshift(s, p, &s->regs); break; - case SWS_UOP_CLEAR: asmgen_op_clear(s, p, &s->regs); break; - case SWS_UOP_TO_U8: asmgen_op_convert(s, p, &s->regs); break; - case SWS_UOP_TO_U16: asmgen_op_convert(s, p, &s->regs); break; - case SWS_UOP_TO_U32: asmgen_op_convert(s, p, &s->regs); break; - case SWS_UOP_TO_F32: asmgen_op_convert(s, p, &s->regs); break; - case SWS_UOP_EXPAND_PAIR: asmgen_op_expand(s, p, &s->regs); break; - case SWS_UOP_EXPAND_QUAD: asmgen_op_expand(s, p, &s->regs); break; - case SWS_UOP_MIN: asmgen_op_min(s, p, &s->regs); break; - case SWS_UOP_MAX: asmgen_op_max(s, p, &s->regs); break; - case SWS_UOP_SCALE: asmgen_op_scale(s, p, &s->regs); break; - case SWS_UOP_LINEAR: asmgen_op_linear(s, p, &s->regs); break; - case SWS_UOP_LINEAR_FMA: asmgen_op_linear(s, p, &s->regs); break; - case SWS_UOP_DITHER: asmgen_op_dither(s, p, &s->regs); break; - /* TODO implement SWS_UOP_SHUFFLE */ - default: - break; - } + ff_sws_aarch64_asmgen_op(s, p, &s->regs); if (is_write) { /* Write functions return directly. */ -- 2.52.0 >From c3a50d3a635b5ef93eec2a835e4c731b1d8a6bfd Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Mon, 27 Jul 2026 12:28:30 +0200 Subject: [PATCH 07/10] swscale/aarch64/ops_asmgen: free exec register after it's done being used This makes no difference for the CPS code, but will allow the JIT compiler to reuse x0. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops_asmgen.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libswscale/aarch64/ops_asmgen.c b/libswscale/aarch64/ops_asmgen.c index c908d82fa4..9877091243 100644 --- a/libswscale/aarch64/ops_asmgen.c +++ b/libswscale/aarch64/ops_asmgen.c @@ -79,6 +79,7 @@ void ff_sws_aarch64_asmgen_reshape_vecs(SwsAArch64Context *s, SwsAArch64OpRegs * void ff_sws_aarch64_asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask) { RasmContext *r = s->rctx; + AArch64RegState *rs = &s->regstate; /** * The process function for aarch64 works similarly to the x86 backend. @@ -101,6 +102,7 @@ void ff_sws_aarch64_asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsC LOOP(omask, i) { i_ldr(r, s->out[i], exec_out[i]); CMTF("out[%u] = exec->out[%u];", i, i); } LOOP(imask, i) { i_ldr(r, s->in_bump[i], exec_in_bump[i]); CMTF("in_bump[%u] = exec->in_bump[%u];", i, i); } LOOP(omask, i) { i_ldr(r, s->out_bump[i], exec_out_bump[i]); CMTF("out_bump[%u] = exec->out_bump[%u];", i, i); } + a64reg_gpr_free(rs, s->exec); /* Setup. */ s->setup = rasm_get_current_node(r); -- 2.52.0 >From e3f5a3f0f358a22575b7f75f78af2ba82fbc6085 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Fri, 3 Jul 2026 17:38:52 +0200 Subject: [PATCH 08/10] swscale/aarch64/ops: export convert_to_aarch64_impl() to ff_sws_aarch64_ops_translate() This function will be used by both the CPS and JIT aarch64 backends. Note that the function still needs to be #include'd directly when not targetting aarch64, since the symbol will be absent from libswscale itself. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/Makefile | 1 + libswscale/aarch64/ops.c | 4 ++-- libswscale/aarch64/ops_impl.h | 4 ++++ libswscale/aarch64/ops_impl_conv.c | 13 +++++-------- libswscale/tests/sws_ops_aarch64.c | 6 +++++- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/libswscale/aarch64/Makefile b/libswscale/aarch64/Makefile index 8fdcf000cc..b3d18a0f21 100644 --- a/libswscale/aarch64/Makefile +++ b/libswscale/aarch64/Makefile @@ -12,6 +12,7 @@ NEON-OBJS += aarch64/hscale.o \ aarch64/yuv2rgb_neon.o \ NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops.o +NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops_impl_conv.o NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops_neon.gen.o $(SUBDIR)aarch64/ops_neon.gen.S: $(SUBDIR)aarch64/ops_static$(HOSTEXESUF) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index 96058a282f..bb86eb60ed 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -24,7 +24,7 @@ #include "libavutil/avstring.h" #include "libavutil/tree.h" -#include "ops_impl_conv.c" +#include "ops_impl.h" /** * Check that there is no mismatch for the SwsOpExec/SwsOpImpl offset @@ -220,7 +220,7 @@ static int aarch64_compile(SwsContext *ctx, const SwsOpList *ops, /* Look up kernel functions. */ for (int i = 0; i < ops->num_ops; i++) { SwsAArch64OpImplParams params = { 0 }; - ret = convert_to_aarch64_impl(ctx, ops, i, block_size, ¶ms); + ret = ff_sws_aarch64_ops_translate(ctx, ops, i, block_size, ¶ms); if (ret < 0) goto error; SwsFuncPtr func = aarch64_lookup(¶ms); diff --git a/libswscale/aarch64/ops_impl.h b/libswscale/aarch64/ops_impl.h index 304b68e44c..38ea99ed6b 100644 --- a/libswscale/aarch64/ops_impl.h +++ b/libswscale/aarch64/ops_impl.h @@ -52,6 +52,10 @@ typedef struct SwsAArch64OpImplParams { SwsUOpParams par; } SwsAArch64OpImplParams; +/* Convert SwsOp to a SwsAArch64OpImplParams. */ +int ff_sws_aarch64_ops_translate(SwsContext *ctx, const SwsOpList *ops, int n, + int block_size, SwsAArch64OpImplParams *out); + /* SwsCompMask-related helpers. */ #define LOOP(mask, idx) \ for (int idx = 0; idx < 4; idx++) \ diff --git a/libswscale/aarch64/ops_impl_conv.c b/libswscale/aarch64/ops_impl_conv.c index 21360e51c7..e330c853d4 100644 --- a/libswscale/aarch64/ops_impl_conv.c +++ b/libswscale/aarch64/ops_impl_conv.c @@ -19,13 +19,14 @@ */ /** - * NOTE: This file is #include'd directly by both the NEON backend and - * the sws_ops_aarch64 tool. + * NOTE: This file is #include'd directly by the sws_ops_aarch64 tool, + * and also built as part of libswscale when targetting aarch64. */ #include "libavutil/error.h" #include "libavutil/rational.h" #include "libswscale/ops.h" +#include "libswscale/ops_chain.h" #include "ops_impl.h" @@ -91,12 +92,8 @@ static void convert_swizzle_to_moves(const SwsOp *op, SwsAArch64OpImplParams *ou } } -/** - * Convert SwsOp to a SwsAArch64OpImplParams. Read the comments regarding - * SwsAArch64OpImplParams in ops_impl.h for more information. - */ -static int convert_to_aarch64_impl(SwsContext *ctx, const SwsOpList *ops, int n, - int block_size, SwsAArch64OpImplParams *out) +int ff_sws_aarch64_ops_translate(SwsContext *ctx, const SwsOpList *ops, int n, + int block_size, SwsAArch64OpImplParams *out) { const SwsOp *op = &ops->ops[n]; diff --git a/libswscale/tests/sws_ops_aarch64.c b/libswscale/tests/sws_ops_aarch64.c index cb47f42037..c8b198ab54 100644 --- a/libswscale/tests/sws_ops_aarch64.c +++ b/libswscale/tests/sws_ops_aarch64.c @@ -30,7 +30,11 @@ #include "libswscale/op_list_gen_template.c" #include "libswscale/ops_dispatch.h" +#if ARCH_AARCH64 +#include "libswscale/aarch64/ops_impl.h" +#else #include "libswscale/aarch64/ops_impl_conv.c" +#endif #ifdef _WIN32 #include <io.h> @@ -202,7 +206,7 @@ static int collect_ops_compile(SwsContext *ctx, const SwsOpList *ops, for (int i = 0; i < ops->num_ops; i++) { SwsAArch64OpImplParams params = { 0 }; - ret = convert_to_aarch64_impl(ctx, ops, i, block_size, ¶ms); + ret = ff_sws_aarch64_ops_translate(ctx, ops, i, block_size, ¶ms); if (ret == AVERROR(ENOTSUP)) continue; if (ret < 0) -- 2.52.0 >From c50da6c645d5ee048d0048bc09cf63369bf34857 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Mon, 27 Jul 2026 12:45:04 +0200 Subject: [PATCH 09/10] swscale/aarch64/ops: export aarch64_setup() to ff_sws_aarch64_setup() This function will be used by both the CPS and JIT aarch64 backends. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- libswscale/aarch64/ops.c | 5 +++-- libswscale/aarch64/ops.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 libswscale/aarch64/ops.h diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index bb86eb60ed..772f5e1b8c 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -25,6 +25,7 @@ #include "libavutil/tree.h" #include "ops_impl.h" +#include "ops.h" /** * Check that there is no mismatch for the SwsOpExec/SwsOpImpl offset @@ -147,7 +148,7 @@ static int aarch64_setup_dither(const SwsAArch64OpImplParams *p, } /*********************************************************************/ -static int aarch64_setup(const SwsOpList *ops, int block_size, int n, +int ff_sws_aarch64_setup(const SwsOpList *ops, int block_size, int n, const SwsAArch64OpImplParams *p, SwsImplResult *out) { const SwsOp *op = &ops->ops[n]; @@ -229,7 +230,7 @@ static int aarch64_compile(SwsContext *ctx, const SwsOpList *ops, goto error; } SwsImplResult res = { 0 }; - ret = aarch64_setup(ops, block_size, i, ¶ms, &res); + ret = ff_sws_aarch64_setup(ops, block_size, i, ¶ms, &res); if (ret < 0) goto error; ret = ff_sws_op_chain_append(chain, func, res.free, &res.priv); diff --git a/libswscale/aarch64/ops.h b/libswscale/aarch64/ops.h new file mode 100644 index 0000000000..8be27b3479 --- /dev/null +++ b/libswscale/aarch64/ops.h @@ -0,0 +1,29 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SWSCALE_AARCH64_OPS_H +#define SWSCALE_AARCH64_OPS_H + +#include "libswscale/ops_chain.h" + +#include "ops_impl.h" + +int ff_sws_aarch64_setup(const SwsOpList *ops, int block_size, int n, + const SwsAArch64OpImplParams *p, SwsImplResult *out); + +#endif /* SWSCALE_AARCH64_OPS_H */ -- 2.52.0 >From 0dcad062cd50a5bf7305d1302f64a6514d223b11 Mon Sep 17 00:00:00 2001 From: Ramiro Polla <[email protected]> Date: Wed, 24 Jun 2026 22:40:12 +0200 Subject: [PATCH 10/10] swscale/aarch64: add NEON JIT backend This commit pieces together the previous few commits to implement the NEON JIT backend. A single specialized function is generated for a pair of src/dst pixel formats. The kernels themselves use the same code as the CPS backend (ops_asmgen.c), but the loading of constants is factored out of the main loop, and the continuation-passing itself is avoided since the JIT compiler will stitch them together at runtime. The backend depends on LLVM to assemble the code generated at runtime. The following speedup is observed from the CPS backend to JIT: A55: Overall speedup=1.300x faster, min=0.521x max=3.154x A76: Overall speedup=1.166x faster, min=0.602x max=2.433x The 0.521x and 0.602x outliers happen on pathological cases of instruction cache misses. Sponsored-by: Sovereign Tech Fund Signed-off-by: Ramiro Polla <[email protected]> --- configure | 16 +- libswscale/aarch64/Makefile | 8 + libswscale/aarch64/ops_asmgen.h | 17 + libswscale/aarch64/ops_jit.c | 859 ++++++++++++++++++++++++++++ libswscale/aarch64/ops_jit_llvm.cpp | 222 +++++++ libswscale/aarch64/ops_jit_llvm.h | 34 ++ libswscale/ops.c | 4 + libswscale/options.c | 23 +- libswscale/swscale.h | 4 +- tests/checkasm/Makefile | 2 +- 10 files changed, 1175 insertions(+), 14 deletions(-) create mode 100644 libswscale/aarch64/ops_jit.c create mode 100644 libswscale/aarch64/ops_jit_llvm.cpp create mode 100644 libswscale/aarch64/ops_jit_llvm.h diff --git a/configure b/configure index d51809931c..513101fd63 100755 --- a/configure +++ b/configure @@ -319,6 +319,7 @@ External library support: --enable-libzimg enable z.lib, needed for zscale filter [no] --enable-libzmq enable message passing via libzmq [no] --enable-libzvbi enable teletext support via libzvbi [no] + --enable-llvm enable LLVM-based JIT assembler for aarch64 swscale [no] --enable-lv2 enable LV2 audio filtering [no] --disable-lzma disable lzma [autodetect] --enable-decklink enable Blackmagic DeckLink I/O support [no] @@ -403,6 +404,7 @@ Toolchain options: --objcc=OCC use ObjC compiler OCC [$cc_default] --dep-cc=DEPCC use dependency generator DEPCC [$cc_default] --glslc=GLSLC use GLSL compiler GLSLC [$glslc_default] + --llvm-config=LLVMCONFIG use LLVM configuration tool LLVMCONFIG [$llvm_config_default] --nvcc=NVCC use Nvidia CUDA compiler NVCC or clang [$nvcc_default] --ld=LD use linker LD [$ld_default] --metalcc=METALCC use metal compiler METALCC [$metalcc_default] @@ -2119,6 +2121,7 @@ EXTERNAL_LIBRARY_LIST=" libzimg libzmq libzvbi + llvm lv2 mediacodec ohcodec @@ -2885,6 +2888,7 @@ CMDLINE_SET=" ignore_tests install ld + llvm_config ln_s logfile malloc_prefix @@ -4363,6 +4367,7 @@ shader_compression_suggest="zlib" avcodec_extralibs="pthreads_extralibs iconv_extralibs dxva2_extralibs liblcevc_dec_extralibs lcms2_extralibs" avfilter_extralibs="pthreads_extralibs" avutil_extralibs="d3d11va_extralibs d3d12va_extralibs mediacodec_extralibs nanosleep_extralibs pthreads_extralibs vaapi_drm_extralibs vaapi_x11_extralibs vaapi_win32_extralibs vdpau_x11_extralibs" +swscale_extralibs="llvm_extralibs" # programs ffmpeg_deps="avcodec avfilter avformat threads" @@ -4409,6 +4414,7 @@ makeinfo_default="makeinfo" install="install" ln_s_default="ln -s -f" glslc_default="glslc" +llvm_config_default="llvm-config" metalcc_default="xcrun -sdk macosx metal" metallib_default="xcrun -sdk macosx metallib" nm_default="nm -g" @@ -5054,7 +5060,7 @@ if enabled cuda_nvcc; then fi set_default arch cc cxx doxygen pkg_config ranlib strip sysinclude \ - target_exec x86asmexe glslc metalcc metallib stdc stdcxx makeinfo + target_exec x86asmexe glslc llvm_config metalcc metallib stdc stdcxx makeinfo enabled cross_compile || host_cc_default=$cc set_default host_cc @@ -7444,6 +7450,14 @@ enabled libvo_amrwbenc && { check_pkg_config libvo_amrwbenc vo-amrwbenc vo-am enabled libvorbis && require_pkg_config libvorbis vorbis vorbis/codec.h vorbis_info_init && require_pkg_config libvorbisenc vorbisenc vorbis/vorbisenc.h vorbis_encode_init +enabled llvm && { + enabled aarch64 || die "ERROR: --enable-llvm is only supported on aarch64" + llvm_cflags=$($llvm_config --cflags 2>/dev/null) + llvm_libs=$($llvm_config --ldflags --libs --system-libs AArch64 MC MCParser Object 2>/dev/null) + check_lib_cxx llvm "llvm/MC/MCAsmInfo.h" "llvm::MCAsmInfo" $llvm_cflags $llvm_libs "-lstdc++" || die "ERROR: LLVM not found" + add_cxxflags $llvm_cflags + llvm_extralibs="$llvm_libs -lstdc++" +} enabled whisper && require_pkg_config whisper "whisper >= 1.7.5" whisper.h whisper_init_from_file_with_params enabled libvpx && { diff --git a/libswscale/aarch64/Makefile b/libswscale/aarch64/Makefile index b3d18a0f21..8d0904ecf2 100644 --- a/libswscale/aarch64/Makefile +++ b/libswscale/aarch64/Makefile @@ -15,6 +15,14 @@ NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops.o NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops_impl_conv.o NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops_neon.gen.o +ifeq ($(CONFIG_LLVM),yes) +NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops_asmgen.o +NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops_jit.o +NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/ops_jit_llvm.o +NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/rasm.o +NEON-OBJS-$(CONFIG_UNSTABLE) += aarch64/rasm_print.o +endif + $(SUBDIR)aarch64/ops_neon.gen.S: $(SUBDIR)aarch64/ops_static$(HOSTEXESUF) $(M)$< > [email protected] $(CP) [email protected] $@ diff --git a/libswscale/aarch64/ops_asmgen.h b/libswscale/aarch64/ops_asmgen.h index 9bc135d6eb..74a329664a 100644 --- a/libswscale/aarch64/ops_asmgen.h +++ b/libswscale/aarch64/ops_asmgen.h @@ -40,6 +40,18 @@ typedef struct SwsAArch64OpRegs { }; } SwsAArch64OpRegs; +/*********************************************************************/ +typedef union SwsAArch64Vector { + uint32_t u32[4]; + uint64_t u64[2]; +} SwsAArch64Vector; + +typedef struct SwsAArch64ConstVec { + SwsAArch64Vector vec; + RasmOp op; + int op_idx; /* index of last 32-bit element used. */ +} SwsAArch64ConstVec; + /*********************************************************************/ typedef struct SwsAArch64Context { RasmContext *rctx; @@ -67,6 +79,11 @@ typedef struct SwsAArch64Context { RasmNode *load_cont_node; SwsAArch64OpRegs regs; + /* JIT-related variables. */ +#define SWS_AARCH64_MAX_CONST_VECS 16 + SwsAArch64ConstVec data[SWS_AARCH64_MAX_CONST_VECS]; + int data_count; + /* Read/Write data pointers and padding. */ RasmOp in[4]; RasmOp out[4]; diff --git a/libswscale/aarch64/ops_jit.c b/libswscale/aarch64/ops_jit.c new file mode 100644 index 0000000000..8dbfd9a9ec --- /dev/null +++ b/libswscale/aarch64/ops_jit.c @@ -0,0 +1,859 @@ +/* + * Copyright (C) 2026 Ramiro Polla + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <string.h> + +#include "../ops_chain.h" +#include "../uops_list.h" +#include "../jit.h" + +#include "ops.h" +#include "ops_asmgen.h" +#include "ops_impl.h" +#include "ops_jit_llvm.h" +#include "rasm.h" + +/*********************************************************************/ +typedef struct SwsAArch64JITContext { + SwsAArch64Context s; + + SwsAArch64OpImplParams params[SWS_MAX_OPS]; + SwsAArch64OpRegs regs[SWS_MAX_OPS]; + SwsImplResult res[SWS_MAX_OPS]; + + uint8_t *code; + size_t code_size; +} SwsAArch64JITContext; + +static void aarch64_jit_free(void *priv) +{ + if (priv) { + SwsAArch64JITContext *ctx = priv; + rasm_free(&ctx->s.rctx); + for (int i = 0; i < SWS_MAX_OPS; i++) { + if (ctx->res[i].free) + ctx->res[i].free(&ctx->res[i].priv); + } + if (ctx->code) + ff_sws_jit_free(ctx->code, ctx->code_size); + av_free(ctx); + } +} + +/*********************************************************************/ +static const char asm_function_macros[] = + ".macro function name, export=0, jumpable=0, align=4\n" + " .text\n" + " .align \\align\n" + "\\name:\n" + ".endm\n" + ".macro endfunc\n" + ".endm\n"; + +static const char asm_const_macros[] = + ".macro const name, align=4, relocate=0\n" + " .align \\align\n" + "\\name:\n" + ".endm\n" + ".macro endconst\n" + ".endm\n"; + +/*********************************************************************/ +/** + * Structured read and write instructions (ld2/ld3/ld4/st2/st3/st4), + * used by the packed read and write operations, require contiguous + * vectors. The read operation already has contiguous vectors because + * it is the first operation to be emitted, but there is no guarantee + * that the write operation will have contiguous vectors. We fix this + * here by ensuring the vectors are contiguous, allocating and moving + * to new registers if necessary. + */ + +static void jit_make_contiguous_vecs(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + RasmOp *sx) +{ + int num_regs = SWS_COMP_COUNT(p->mask); + if (a64op_contiguous_vecs(sx, num_regs)) + return; + + RasmContext *r = s->rctx; + AArch64RegState *rs = &s->regstate; + RasmOp new_sx[4] = { 0 }; + a64reg_veclist_ops(rs, num_regs, new_sx); + LOOP_MASK(p, i) { + new_sx[i] = a64op_make_vec(a64op_vec_n(new_sx[i]), s->el_count, s->el_size); + i_mov(r, new_sx[i], sx[i]); + a64reg_vec_free(rs, sx[i]); + sx[i] = new_sx[i]; + } +} + +static void jit_write_packed_fixup(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + jit_make_contiguous_vecs (s, p, regs->sl); + if (s->use_vh) + jit_make_contiguous_vecs(s, p, regs->sh); +} + +/*********************************************************************/ +/* Constant data */ + +/* Returns a vector operand that holds the entire 128-bit sequence. */ +static RasmOp jit_push_v128(SwsAArch64Context *s, void *val) +{ + /* Check whether we already have it. */ + for (int i = 0; i < s->data_count; i++) { + if (rasm_op_type(s->data[i].op) != AARCH64_OP_VEC) + continue; + if (s->data[i].op_idx == 4 && !memcmp(&s->data[i].vec, val, 16)) { + return s->data[i].op; + } + } + + /* Add it to our data and create a new vector. */ + av_assert0(s->data_count < SWS_AARCH64_MAX_CONST_VECS); + int idx = s->data_count++; + memcpy(&s->data[idx].vec, val, 16); + s->data[idx].op = v_q(a64reg_unclobbered_vec(&s->regstate)); + s->data[idx].op_idx = 4; + + return s->data[idx].op; +} + +/* Returns a vector operand that holds the 32-bit value broadcast to all elements. */ +static RasmOp jit_push_vimm(SwsAArch64Context *s, SwsPixelType type, uint32_t val) +{ + /* Expand to u32. */ + switch (type) { + case SWS_PIXEL_U8: val = val | (val << 8); av_fallthrough; + case SWS_PIXEL_U16: val = val | (val << 16); break; + } + + /* TODO use movi for movi-encodable immediates. */ + /* TODO use mov+dup instead of taking up data. */ + + SwsAArch64Vector vec = { .u32 = { val, val, val, val } }; + return jit_push_v128(s, &vec); +} + +/* Returns a vector by-element operand that holds the 32-bit value. */ +static RasmOp jit_push_elem(SwsAArch64Context *s, SwsPixelType type, uint32_t val) +{ + /* Check whether we already have it in data. */ + for (int i = 0; i < s->data_count; i++) { + if (rasm_op_type(s->data[i].op) != AARCH64_OP_VEC) + continue; + for (int j = 0; j < s->data[i].op_idx; j++) { + if (s->data[i].vec.u32[j] == val) + return a64op_elem(v_4s(s->data[i].op), j); + } + } + + /* Check whether there's space left in a previously allocated vector. */ + for (int i = 0; i < s->data_count; i++) { + if (rasm_op_type(s->data[i].op) != AARCH64_OP_VEC) + continue; + if (s->data[i].op_idx != 4) { + int jdx = s->data[i].op_idx++; + s->data[i].vec.u32[jdx] = val; + return a64op_elem(v_4s(s->data[i].op), jdx); + } + } + + /* Add it to our data and create a new vector. */ + av_assert0(s->data_count < SWS_AARCH64_MAX_CONST_VECS); + int idx = s->data_count++; + s->data[idx].vec.u32[0] = val; + s->data[idx].op = v_q(a64reg_unclobbered_vec(&s->regstate)); + s->data[idx].op_idx = 1; + + return a64op_elem(v_4s(s->data[idx].op), 0); +} + +/* Returns a GPR that holds the 64-bit value. */ +static RasmOp jit_push_u64(SwsAArch64Context *s, uint64_t val) +{ + /* Add it to our data and create a new GPR. */ + av_assert0(s->data_count < SWS_AARCH64_MAX_CONST_VECS); + int idx = s->data_count++; + s->data[idx].vec.u64[0] = val; + s->data[idx].op = a64reg_unclobbered_gpx(&s->regstate); + s->data[idx].op_idx = 2; + return s->data[idx].op; +} + +static void aarch64_jit_data(SwsAArch64Context *s) +{ + RasmContext *r = s->rctx; + AArch64RegState *rs = &s->regstate; + + /* Emit data. */ + int ldata = rasm_const_begin(r, "ldata"); + for (int i = 0; i < s->data_count; i++) { + switch (rasm_op_type(s->data[i].op)) { + case AARCH64_OP_GPR: + rasm_add_data(r, &s->data[i].vec, 2, RASM_DATA_QUAD); + break; + case AARCH64_OP_VEC: + default: + rasm_add_data(r, &s->data[i].vec, 4, RASM_DATA_WORD); + break; + } + } + + /* Load data. */ + RasmNode *saved = rasm_set_current_node(r, s->setup); + rasm_add_comment(r, "load constants"); + RasmOp ptr = a64reg_gpx(rs, -1); + i_adr(r, ptr, rasm_op_label(ldata)); + for (int i = 0; i < s->data_count; i++) { + RasmOp base = a64op_off(ptr, i * 16); + i_ldr(r, s->data[i].op, base); + } + a64reg_gpr_free(rs, ptr); + s->setup = rasm_set_current_node(r, saved); +} + +/*********************************************************************/ +/* Setup helpers. */ + +/** + * Recompute op_mask from SwsOp because SwsAArch64OpImplParams has dropped + * passthrough information to prevent duplicates. + */ +static SwsCompMask recompute_op_mask(const SwsOp *op) +{ + SwsCompMask op_mask = 0; + for (int i = 0; i < 4; i++) { + if (SWS_OP_NEEDED(op, i)) + op_mask |= SWS_COMP(i); + } + return op_mask; +} + +/* Get value from SwsOpPriv based on the pixel type. */ +static uint32_t get_priv_val(const SwsOpPriv *priv, SwsPixelType type, int i) +{ + return (type == SWS_PIXEL_U8) ? priv->u8[i] + : (type == SWS_PIXEL_U16) ? priv->u16[i] + : priv->u32[i]; +} + +/** + * Allocate registers and immediately free them. + * NOTE: this should be done as the last step in register allocation, + * to prevent these temporary registers from being reused. + */ +static void jit_alloc_vt(AArch64RegState *rs, int n, RasmOp *out) +{ + for (int i = 0; i < n; i++) + out[i] = a64reg_vec(rs, -1); + for (int i = 0; i < n; i++) + a64reg_vec_free(rs, out[i]); +} + +static void setup_mask_alloc(SwsAArch64Context *s, SwsCompMask mask, + SwsAArch64OpRegs *regs) +{ + AArch64RegState *rs = &s->regstate; + LOOP (mask, i) { regs->dl[i] = a64reg_vec(rs, -1); } + LOOP_VH(s, mask, i) { regs->dh[i] = a64reg_vec(rs, -1); } +} + +static void setup_mask_free(SwsAArch64Context *s, SwsCompMask mask, + SwsAArch64OpRegs *regs) +{ + AArch64RegState *rs = &s->regstate; + LOOP (mask, i) { a64reg_vec_free(rs, regs->sl[i]); } + LOOP_VH(s, mask, i) { a64reg_vec_free(rs, regs->sh[i]); } +} + +static void setup_mask_write(SwsAArch64Context *s, SwsCompMask mask, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs) +{ + LOOP (mask, i) { regs->sl[i] = prev->dl[i]; } + LOOP_VH(s, mask, i) { regs->sh[i] = prev->dh[i]; } +} + +static void setup_mask_passthrough(SwsAArch64Context *s, SwsCompMask mask, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs) +{ + LOOP (mask, i) { regs->dl[i] = regs->sl[i] = prev->dl[i]; } + LOOP_VH(s, mask, i) { regs->dh[i] = regs->sh[i] = prev->dh[i]; } +} + +/*********************************************************************/ +static void asmgen_setup_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_alloc(s, p->mask, regs); + + AArch64RegState *rs = &s->regstate; + jit_alloc_vt(rs, 1, regs->vt); + + /* constants */ + regs->vk[0] = jit_push_v128(s, res->priv.data); + regs->vk[1] = jit_push_vimm(s, SWS_PIXEL_U8, 1); +} + +static void asmgen_setup_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_alloc(s, p->mask, regs); + + AArch64RegState *rs = &s->regstate; + jit_alloc_vt(rs, 1, regs->vt); + + /* constants */ + regs->vk[0] = jit_push_vimm(s, SWS_PIXEL_U8, 0x0f); +} + +static void asmgen_setup_read_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + int num_regs = SWS_COMP_COUNT(p->mask); + AArch64RegState *rs = &s->regstate; + a64reg_veclist_ops (rs, num_regs, regs->dl); + if (s->use_vh) + a64reg_veclist_ops(rs, num_regs, regs->dh); +} + +static void asmgen_setup_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_alloc(s, p->mask, regs); +} + +static void asmgen_setup_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_write(s, p->mask, prev, regs); + + AArch64RegState *rs = &s->regstate; + jit_alloc_vt(rs, 2, regs->vt); + + /* constants */ + regs->vk[0] = jit_push_v128(s, res->priv.data); +} + +static void asmgen_setup_write_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_write(s, p->mask, prev, regs); + + AArch64RegState *rs = &s->regstate; + jit_alloc_vt(rs, 2, regs->vt); +} + +static void asmgen_setup_write_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + /* TODO See jit_write_packed_fixup(). */ + setup_mask_write(s, p->mask, prev, regs); +} + +static void asmgen_setup_write_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_write(s, p->mask, prev, regs); +} + +static void asmgen_setup_swap_bytes(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_passthrough(s, p->mask, prev, regs); +} + +static void asmgen_setup_swizzle(SwsAArch64Context *s, SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res, const SwsOp *op) +{ + /* Split original swizzle into identity, renames, and copies. */ + SwsCompMask identity = 0; + SwsMoveUOp rename = { 0 }; + SwsMoveUOp copy = { 0 }; + bool overwritten[4] = { false, false, false, false }; + SwsCompMask op_mask = recompute_op_mask(op); + LOOP(op_mask, i) { + int src = op->swizzle.in[i]; + if (src == i) { + identity |= SWS_COMP(i); + } else { + SwsMoveUOp *list = overwritten[src] ? © : &rename; + list->dst[list->num_moves] = i; + list->src[list->num_moves] = src; + list->num_moves++; + } + overwritten[src] = true; + } + + /* Identity passthrough. */ + setup_mask_passthrough(s, identity, prev, regs); + + /* Perform simple renames. */ + for (int i = 0; i < rename.num_moves; i++) { + int src = rename.src[i]; + int dst = rename.dst[i]; + regs->dl[dst] = regs->sl[src] = prev->dl[src]; + if (s->use_vh) + regs->dh[dst] = regs->sh[src] = prev->dh[src]; + } + + /* Replace moves list with remaining copies. */ + p->par.move = copy; + p->mask = 0; + for (int i = 0; i < copy.num_moves; i++) { + int dst = copy.dst[i]; + p->mask |= SWS_COMP(dst); + } + setup_mask_alloc(s, p->mask, regs); +} + +static void asmgen_setup_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_passthrough(s, 1u, prev, regs); + setup_mask_alloc(s, p->mask & ~1u, regs); + + /* constants */ + LOOP_MASK (p, i) { + uint32_t val = (1u << p->par.pack.pattern[i]) - 1; + regs->vk[i] = jit_push_vimm(s, p->type, val); + } +} + +static void asmgen_setup_pack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_passthrough(s, p->mask, prev, regs); + setup_mask_free(s, p->mask & ~1u, regs); +} + +static void asmgen_setup_shift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_passthrough(s, p->mask, prev, regs); +} + +static void asmgen_setup_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res, const SwsOp *op) +{ + /* TODO factor clear into setup instead of performing dup. */ + + SwsCompMask op_mask = recompute_op_mask(op); + SwsCompMask identity = op_mask & ~p->mask; + + if (prev) { + /* Reuse registers that have already been allocated. */ + LOOP_MASK(p, i) { identity |= (rasm_op_type(prev->dl[i]) != RASM_OP_NONE) ? SWS_COMP(i) : 0; } + setup_mask_passthrough(s, identity, prev, regs); + } + + setup_mask_alloc(s, p->mask & ~identity, regs); + + /* constants */ + regs->vk[0] = jit_push_v128(s, res->priv.data); +} + +static void asmgen_setup_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + SwsPixelType to_type = (p->uop == SWS_UOP_EXPAND_PAIR) ? SWS_PIXEL_U16 + : (p->uop == SWS_UOP_EXPAND_QUAD) ? SWS_PIXEL_U32 + : (p->uop == SWS_UOP_TO_U8) ? SWS_PIXEL_U8 + : (p->uop == SWS_UOP_TO_U16) ? SWS_PIXEL_U16 + : (p->uop == SWS_UOP_TO_U32) ? SWS_PIXEL_U32 + : SWS_PIXEL_F32; + size_t src_el_size = s->el_size; + size_t dst_el_size = ff_sws_pixel_type_size(to_type); + bool src_use_vh = (p->block_size * src_el_size) > 16; + bool dst_use_vh = (p->block_size * dst_el_size) > 16; + + setup_mask_passthrough(s, p->mask, prev, regs); + AArch64RegState *rs = &s->regstate; + LOOP_MASK(p, i) { + if (src_use_vh && dst_use_vh) { + regs->dh[i] = regs->sh[i] = prev->dh[i]; + } else if (!src_use_vh && dst_use_vh) { + regs->dh[i] = a64reg_vec(rs, -1); + } else if (src_use_vh && !dst_use_vh) { + regs->sh[i] = prev->dh[i]; + a64reg_vec_free(rs, regs->sh[i]); + } + } +} + +static void asmgen_setup_clamp(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_passthrough(s, p->mask, prev, regs); + + /* constants */ + LOOP_MASK(p, i) { + uint32_t val = get_priv_val(&res->priv, p->type, i); + regs->vk[i] = jit_push_vimm(s, p->type, val); + } +} + +static void asmgen_setup_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + setup_mask_passthrough(s, p->mask, prev, regs); + + /* constants */ + uint32_t val = get_priv_val(&res->priv, p->type, 0); + regs->vk[0] = jit_push_vimm(s, p->type, val); +} + +static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res) +{ + AArch64RegState *rs = &s->regstate; + + /* Start passing through all components. */ + setup_mask_passthrough(s, SWS_COMP_ALL, prev, regs); + + /** + * Relocate sources that would be clobbered while still being used. + * Also collect coefficients. + */ + SwsCompMask save_mask = 0; + bool overwritten[4] = { false, false, false, false }; + const SwsPixel *coeffs = (const SwsPixel *) res->priv.ptr; + int i_coeff = 0; + LOOP_MASK(p, i) { + for (int j = 0; j < 5; j++) { + bool is_offset = (j == 0); + int src_j = is_offset ? 4 : (j - 1); + if (p->par.lin.zero & SWS_MASK(i, src_j)) + continue; + if (!is_offset && overwritten[src_j]) + save_mask |= SWS_COMP(src_j); + overwritten[i] = true; + /* constants */ + regs->linear_vcoeff[i][j] = jit_push_elem(s, SWS_PIXEL_U32, coeffs[i_coeff++].u32); + } + } + setup_mask_alloc(s, save_mask, regs); + if (p->uop == SWS_UOP_LINEAR) + jit_alloc_vt(rs, 4, ®s->vt[8]); + setup_mask_free(s, save_mask, regs); +} + +static void asmgen_setup_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + const SwsAArch64OpRegs *prev, SwsAArch64OpRegs *regs, + SwsImplResult *res, const SwsOp *op) +{ + SwsCompMask op_mask = recompute_op_mask(op); + setup_mask_passthrough(s, op_mask, prev, regs); + + AArch64RegState *rs = &s->regstate; + jit_alloc_vt(rs, 2, regs->vt); + + /* constants */ + regs->dither_ptr = jit_push_u64(s, (uint64_t) res->priv.ptr); +} + +/* Set up registers for operation. */ +static void aarch64_jit_op_setup(SwsAArch64JITContext *ctx, const SwsOpList *ops, int n) +{ + SwsAArch64Context *s = &ctx->s; + SwsAArch64OpImplParams *p = &ctx->params[n]; + const SwsAArch64OpRegs *prev = n ? &ctx->regs[n - 1] : NULL; + SwsAArch64OpRegs *regs = &ctx->regs[n]; + SwsImplResult *res = &ctx->res[n]; + const SwsOp *op = &ops->ops[n]; + + ff_sws_aarch64_asmgen_setup_vecs(s, p); + + switch (p->uop) { + case SWS_UOP_READ_BIT: asmgen_setup_read_bit(s, p, prev, regs, res); break; + case SWS_UOP_READ_NIBBLE: asmgen_setup_read_nibble(s, p, prev, regs, res); break; + case SWS_UOP_READ_PACKED: asmgen_setup_read_packed(s, p, prev, regs, res); break; + case SWS_UOP_READ_PLANAR: asmgen_setup_read_planar(s, p, prev, regs, res); break; + case SWS_UOP_WRITE_BIT: asmgen_setup_write_bit(s, p, prev, regs, res); break; + case SWS_UOP_WRITE_NIBBLE: asmgen_setup_write_nibble(s, p, prev, regs, res); break; + case SWS_UOP_WRITE_PACKED: asmgen_setup_write_packed(s, p, prev, regs, res); break; + case SWS_UOP_WRITE_PLANAR: asmgen_setup_write_planar(s, p, prev, regs, res); break; + case SWS_UOP_SWAP_BYTES: asmgen_setup_swap_bytes(s, p, prev, regs, res); break; + case SWS_UOP_PERMUTE: asmgen_setup_swizzle(s, p, prev, regs, res, op); break; + case SWS_UOP_COPY: asmgen_setup_swizzle(s, p, prev, regs, res, op); break; + case SWS_UOP_UNPACK: asmgen_setup_unpack(s, p, prev, regs, res); break; + case SWS_UOP_PACK: asmgen_setup_pack(s, p, prev, regs, res); break; + case SWS_UOP_LSHIFT: asmgen_setup_shift(s, p, prev, regs, res); break; + case SWS_UOP_RSHIFT: asmgen_setup_shift(s, p, prev, regs, res); break; + case SWS_UOP_CLEAR: asmgen_setup_clear(s, p, prev, regs, res, op); break; + case SWS_UOP_TO_U8: asmgen_setup_convert(s, p, prev, regs, res); break; + case SWS_UOP_TO_U16: asmgen_setup_convert(s, p, prev, regs, res); break; + case SWS_UOP_TO_U32: asmgen_setup_convert(s, p, prev, regs, res); break; + case SWS_UOP_TO_F32: asmgen_setup_convert(s, p, prev, regs, res); break; + case SWS_UOP_EXPAND_PAIR: asmgen_setup_convert(s, p, prev, regs, res); break; + case SWS_UOP_EXPAND_QUAD: asmgen_setup_convert(s, p, prev, regs, res); break; + case SWS_UOP_MIN: asmgen_setup_clamp(s, p, prev, regs, res); break; + case SWS_UOP_MAX: asmgen_setup_clamp(s, p, prev, regs, res); break; + case SWS_UOP_SCALE: asmgen_setup_scale(s, p, prev, regs, res); break; + case SWS_UOP_LINEAR: asmgen_setup_linear(s, p, prev, regs, res); break; + case SWS_UOP_LINEAR_FMA: asmgen_setup_linear(s, p, prev, regs, res); break; + case SWS_UOP_DITHER: asmgen_setup_dither(s, p, prev, regs, res, op); break; + default: + break; + } +} + +/*********************************************************************/ +static void aarch64_jit_process_frame(SwsAArch64Context *s, + SwsCompMask imask, SwsCompMask omask) +{ + AArch64RegState *rs = &s->regstate; + + /* SwsOpFunc arguments. */ + s->exec = a64reg_argx(rs, 0); // const SwsOpExec *exec + s->impl = a64reg_argx(rs, 1); // const void *priv + s->bx_start = a64reg_argw(rs, 2); // int bx_start + s->y_start = a64reg_argw(rs, 3); // int y_start + s->bx_end = a64reg_argw(rs, 4); // int bx_end + s->y_end = a64reg_argw(rs, 5); // int y_end + + /* Loop iterator variables. */ + s->bx = a64reg_gpw(rs, 6); + s->y = a64reg_gpw(rs, 3); /* Reused from SwsOpFunc.y_start argument. */ + + /* Scratch registers. */ + s->tmp0 = a64reg_gpx(rs, 16); /* IP0 */ + s->tmp1 = a64reg_gpx(rs, 17); /* IP1 */ + + /* GPRs */ + LOOP(imask, i) { s->in [i] = a64reg_gpx(rs, -1); } + LOOP(imask, i) { s->in_bump [i] = a64reg_gpx(rs, -1); } + LOOP(omask, i) { s->out [i] = a64reg_gpx(rs, -1); } + LOOP(omask, i) { s->out_bump[i] = a64reg_gpx(rs, -1); } +} + +/*********************************************************************/ +static void aarch64_jit_process(SwsAArch64Context *s, const SwsOpList *ops, + SwsCompMask imask, SwsCompMask omask) +{ + RasmContext *r = s->rctx; + char func_name[128]; + + snprintf(func_name, sizeof(func_name), "jit_process_%s_%s_neon", + av_get_pix_fmt_name(ops->src.format), + av_get_pix_fmt_name(ops->dst.format)); + rasm_func_begin(r, func_name, true, false); + + ff_sws_aarch64_asmgen_process(s, imask, omask); +} + +/*********************************************************************/ +static const char *const op_type_names[SWS_UOP_TYPE_NB] = { +#define UOP_NAME(OP, ABBR) [OP] = ABBR, + UOPS_LIST(UOP_NAME) +#undef UOP_NAME +}; + +static void aarch64_jit_op(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, + SwsAArch64OpRegs *regs) +{ + RasmContext *r = s->rctx; + + ff_sws_aarch64_asmgen_setup_vecs(s, p); + ff_sws_aarch64_asmgen_reshape_vecs(s, regs); + + rasm_add_commentf(r, (char[32]){0}, 32, "%s() {", op_type_names[p->uop]); + + /** + * Ensure packed write operations work on contiguous vectors. + * TODO fix this by implementing proper register tracking and automatic + * register allocation. + */ + if (p->uop == SWS_UOP_WRITE_PACKED) + jit_write_packed_fixup(s, p, regs); + + ff_sws_aarch64_asmgen_op(s, p, regs); + + rasm_add_comment(r, "}"); +} + +/*********************************************************************/ +/* Debug logging. */ + +static const char *print_reg(char buf[8], RasmOp op) +{ + switch (rasm_op_type(op)) { + case AARCH64_OP_GPR: + snprintf(buf, 8, "x%-2d", a64op_gpr_n(op)); + break; + case AARCH64_OP_VEC: + snprintf(buf, 8, "v%-2d", a64op_vec_n(op)); + break; + default: + snprintf(buf, 8, "___"); + break; + } + return buf; +} +#define PRINT_REG(op) print_reg((char[8]){0}, op) + +static void print_io_regs(SwsContext *sws, const SwsAArch64OpImplParams *p, const SwsAArch64OpRegs *regs) +{ + av_log(sws, AV_LOG_TRACE, "[%-18s] { %s %s %s %s } { %s %s %s %s } -> { %s %s %s %s } { %s %s %s %s }\n", + op_type_names[p->uop], + PRINT_REG(regs->sl[0]), PRINT_REG(regs->sl[1]), PRINT_REG(regs->sl[2]), PRINT_REG(regs->sl[3]), + PRINT_REG(regs->sh[0]), PRINT_REG(regs->sh[1]), PRINT_REG(regs->sh[2]), PRINT_REG(regs->sh[3]), + PRINT_REG(regs->dl[0]), PRINT_REG(regs->dl[1]), PRINT_REG(regs->dl[2]), PRINT_REG(regs->dl[3]), + PRINT_REG(regs->dh[0]), PRINT_REG(regs->dh[1]), PRINT_REG(regs->dh[2]), PRINT_REG(regs->dh[3])); +} + +/*********************************************************************/ +static int aarch64_jit_asmgen(SwsContext *sws, SwsAArch64JITContext *ctx, const SwsOpList *ops, + SwsCompMask imask, SwsCompMask omask, AVBPrint *errstr) +{ + /* Create process function. */ + aarch64_jit_process(&ctx->s, ops, imask, omask); + + /* Add all ops into main loop. */ + rasm_set_current_node(ctx->s.rctx, ctx->s.loop); + for (int i = 0; i < ops->num_ops; i++) + aarch64_jit_op(&ctx->s, &ctx->params[i], &ctx->regs[i]); + + /* Emit const data and load it into registers at setup time. */ + if (ctx->s.data_count) + aarch64_jit_data(&ctx->s); + + AVBPrint bp; + av_bprint_init(&bp, 0, AV_BPRINT_SIZE_UNLIMITED); + av_bprintf(&bp, "%s\n", asm_function_macros); + if (ctx->s.data_count) + av_bprintf(&bp, "%s\n", asm_const_macros); + rasm_print(ctx->s.rctx, &bp); + + /* Debug generated code. */ + if (av_log_get_level() >= AV_LOG_TRACE) + av_log(sws, AV_LOG_TRACE, "JIT generated code:\n%s", bp.str); + + int ret = ff_sws_jit_assemble_llvm(sws, bp.str, &ctx->code, &ctx->code_size, errstr); + av_bprint_finalize(&bp, NULL); + + return ret; +} + +/*********************************************************************/ +static int aarch64_jit_compile(SwsContext *sws, const SwsOpList *ops, + SwsCompiledOp *out) +{ + int ret; + + const int cpu_flags = av_get_cpu_flags(); + if (!(cpu_flags & AV_CPU_FLAG_NEON)) + return AVERROR(ENOTSUP); + + SwsAArch64JITContext *ctx = av_mallocz(sizeof(*ctx)); + if (!ctx) + return AVERROR(ENOMEM); + + ctx->s.rctx = rasm_alloc(); + if (!ctx->s.rctx) { + ret = AVERROR(ENOMEM); + goto error; + } + + av_log(sws, AV_LOG_DEBUG, "JIT compile: %s -> %s\n", + av_get_pix_fmt_name(ops->src.format), + av_get_pix_fmt_name(ops->dst.format)); + + /* Use at most two full vregs during the widest precision section */ + int block_size = (ff_sws_op_list_max_size(ops) == 4) ? 8 : 16; + + /* Setup process frame. */ + const SwsOp *read = ff_sws_op_list_input(ops); + const SwsOp *write = ff_sws_op_list_output(ops); + const int read_planes = read ? ff_sws_rw_op_planes(read) : 0; + const int write_planes = ff_sws_rw_op_planes(write); + SwsCompMask imask = SWS_COMP_ELEMS(read_planes); + SwsCompMask omask = SWS_COMP_ELEMS(write_planes); + aarch64_jit_process_frame(&ctx->s, imask, omask); + + /* Translate all ops into implementation parameters and setup registers. */ + for (int i = 0; i < ops->num_ops; i++) { + ret = ff_sws_aarch64_ops_translate(sws, ops, i, block_size, &ctx->params[i]); + if (ret < 0) + goto error; + ret = ff_sws_aarch64_setup(ops, block_size, i, &ctx->params[i], &ctx->res[i]); + if (ret < 0) + goto error; + aarch64_jit_op_setup(ctx, ops, i); + } + /* Debug registers. */ + if (av_log_get_level() >= AV_LOG_TRACE) { + av_log(sws, AV_LOG_TRACE, "JIT I/O register allocation:\n"); + for (int i = 0; i < ops->num_ops; i++) + print_io_regs(sws, &ctx->params[i], &ctx->regs[i]); + } + + /* Generate JIT code and assemble it. */ + AVBPrint errstr; + av_bprint_init(&errstr, 0, AV_BPRINT_SIZE_UNLIMITED); + ret = aarch64_jit_asmgen(sws, ctx, ops, imask, omask, &errstr); + if (ret < 0) + av_log(sws, AV_LOG_TRACE, "LLVM JIT error log:\n%s", errstr.str); + av_bprint_finalize(&errstr, NULL); + + if (!ret) { + *out = (SwsCompiledOp) { + .priv = ctx, + .slice_align = 1, + .free = aarch64_jit_free, + .block_size = block_size, + .func = (SwsOpFunc) ctx->code, + .cpu_flags = AV_CPU_FLAG_NEON, + }; + } + +error: + if (ret < 0) + aarch64_jit_free(ctx); + return ret; +} + +/*********************************************************************/ +const SwsOpBackend backend_aarch64_jit = { + .name = "aarch64_jit", + .flags = SWS_BACKEND_AARCH64_JIT, + .compile = aarch64_jit_compile, + .hw_format = AV_PIX_FMT_NONE, +}; diff --git a/libswscale/aarch64/ops_jit_llvm.cpp b/libswscale/aarch64/ops_jit_llvm.cpp new file mode 100644 index 0000000000..3a82f762ae --- /dev/null +++ b/libswscale/aarch64/ops_jit_llvm.cpp @@ -0,0 +1,222 @@ +/* + * Copyright (C) 2026 Ramiro Polla + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Prevent collision with LLVM's usage of PIC as a parameter name. */ +#undef PIC + +#include <llvm/MC/MCAsmBackend.h> +#include <llvm/MC/MCAsmInfo.h> +#include <llvm/MC/MCCodeEmitter.h> +#include <llvm/MC/MCContext.h> +#include <llvm/MC/MCInstrInfo.h> +#include <llvm/MC/MCObjectFileInfo.h> +#include <llvm/MC/MCObjectWriter.h> +#include <llvm/MC/MCParser/MCAsmParser.h> +#include <llvm/MC/MCParser/MCTargetAsmParser.h> +#include <llvm/MC/MCRegisterInfo.h> +#include <llvm/MC/MCStreamer.h> +#include <llvm/MC/MCSubtargetInfo.h> +#include <llvm/MC/MCTargetOptions.h> +#include <llvm/MC/TargetRegistry.h> +#include <llvm/Object/ObjectFile.h> +#include <llvm/Support/MemoryBuffer.h> +#include <llvm/Support/SourceMgr.h> +#include <llvm/Support/TargetSelect.h> +#include <llvm/Support/raw_ostream.h> +#include <llvm/TargetParser/Triple.h> + +extern "C" { +#include "libavutil/bprint.h" +#include "libavutil/error.h" +#include "libavutil/log.h" +#include "../jit.h" +#include "ops_jit_llvm.h" +} + +using namespace llvm; + +/*********************************************************************/ +static void asm_diag_handler(const SMDiagnostic &diag, void *context) +{ + AVBPrint *errstr = (AVBPrint *) context; + int n_line = diag.getLineNo(); + int n_col = diag.getColumnNo(); + const char *kind = "unknown"; + std::string message = std::string(diag.getMessage()); + std::string line = std::string(diag.getLineContents()); + + switch (diag.getKind()) { + case SourceMgr::DK_Error: kind = "error"; break; + case SourceMgr::DK_Warning: kind = "warning"; break; + case SourceMgr::DK_Remark: kind = "remark"; break; + case SourceMgr::DK_Note: kind = "note"; break; + } + + av_bprintf(errstr, "%d:%d: %s: %s\n", n_line, n_col + 1, kind, message.c_str()); + if (!line.empty()) { + av_bprintf(errstr, "%s\n", line.c_str()); + av_bprintf(errstr, "%*s^\n", n_col, " "); + } +} + +/*********************************************************************/ +extern "C" +int ff_sws_jit_assemble_llvm(void *logctx, const char *src, + uint8_t **out_text, size_t *out_size, + AVBPrint *errstr) +{ + static const char triple_name[] = "aarch64-unknown-linux-gnu"; + + LLVMInitializeAArch64TargetInfo(); + LLVMInitializeAArch64Target(); + LLVMInitializeAArch64TargetMC(); + LLVMInitializeAArch64AsmParser(); + + std::string err; + const Target *target = TargetRegistry::lookupTarget(triple_name, err); + if (!target) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: failed to find target %s: %s\n", + triple_name, err.c_str()); + return AVERROR_EXTERNAL; + } + + std::unique_ptr<MCRegisterInfo> mri(target->createMCRegInfo(triple_name)); + if (!mri) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: createMCRegInfo() failed\n"); + return AVERROR_EXTERNAL; + } + + MCTargetOptions options; + std::unique_ptr<MCAsmInfo> mai(target->createMCAsmInfo(*mri, triple_name, options)); + if (!mai) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: createMCAsmInfo() failed\n"); + return AVERROR_EXTERNAL; + } + + std::unique_ptr<MCInstrInfo> mcii(target->createMCInstrInfo()); + if (!mcii) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: createMCInstrInfo() failed\n"); + return AVERROR_EXTERNAL; + } + + std::unique_ptr<MCSubtargetInfo> sti(target->createMCSubtargetInfo(triple_name, "", "")); + if (!sti) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: createMCSubtargetInfo() failed\n"); + return AVERROR_EXTERNAL; + } + + MCContext ctx(Triple(triple_name), mai.get(), mri.get(), sti.get()); + + std::unique_ptr<MCObjectFileInfo> mofi(target->createMCObjectFileInfo(ctx, false)); + if (!mofi) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: createMCObjectFileInfo() failed\n"); + return AVERROR_EXTERNAL; + } + ctx.setObjectFileInfo(mofi.get()); + + std::unique_ptr<MCCodeEmitter> ce(target->createMCCodeEmitter(*mcii, ctx)); + if (!ce) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: createMCCodeEmitter() failed\n"); + return AVERROR_EXTERNAL; + } + + std::unique_ptr<MCAsmBackend> mab(target->createMCAsmBackend(*sti, *mri, options)); + if (!mab) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: createMCAsmBackend() failed\n"); + return AVERROR_EXTERNAL; + } + + SmallVector<char, 4096> obj_buf; + raw_svector_ostream ostream(obj_buf); + std::unique_ptr<MCObjectWriter> ow(mab->createObjectWriter(ostream)); + if (!ow) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: createObjectWriter() failed\n"); + return AVERROR_EXTERNAL; + } + + std::unique_ptr<MCStreamer> streamer(target->createMCObjectStreamer(Triple(triple_name), ctx, std::move(mab), std::move(ow), std::move(ce), *sti)); + if (!streamer) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: createMCObjectStreamer() failed\n"); + return AVERROR_EXTERNAL; + } + + SourceMgr src_mgr; + src_mgr.setDiagHandler(asm_diag_handler, errstr); + src_mgr.AddNewSourceBuffer(MemoryBuffer::getMemBuffer(src, "<asm>"), SMLoc()); + + std::unique_ptr<MCAsmParser> parser(createMCAsmParser(src_mgr, ctx, *streamer, *mai)); + if (!parser) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: createMCAsmParser() failed\n"); + return AVERROR_EXTERNAL; + } + + MCTargetAsmParser *tap = target->createMCAsmParser(*sti, *parser, *mcii, options); + if (!tap) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: target createMCAsmParser() failed\n"); + return AVERROR_EXTERNAL; + } + parser->setTargetParser(*tap); + + if (parser->Run(false)) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: assembly failed\n"); + return AVERROR(EINVAL); + } + + MemoryBufferRef obj_membuf(StringRef(obj_buf.data(), obj_buf.size()), "obj"); + Expected<std::unique_ptr<object::ObjectFile>> obj = object::ObjectFile::createObjectFile(obj_membuf); + if (!obj) { + consumeError(obj.takeError()); + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: failed to parse assembled object\n"); + return AVERROR_INVALIDDATA; + } + + for (const object::SectionRef §ion: (*obj)->sections()) { + Expected<StringRef> name = section.getName(); + if (!name) { + consumeError(name.takeError()); + continue; + } + if (*name != ".text") + continue; + + Expected<StringRef> contents = section.getContents(); + if (!contents) { + consumeError(contents.takeError()); + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: failed to read .text section\n"); + return AVERROR_INVALIDDATA; + } + if (contents->empty()) { + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: .text section is empty\n"); + return AVERROR_INVALIDDATA; + } + + *out_size = contents->size(); + *out_text = (uint8_t *) ff_sws_jit_alloc(*out_size); + if (!*out_text) + return AVERROR(ENOMEM); + + memcpy(*out_text, contents->data(), *out_size); + return ff_sws_jit_protect(*out_text, *out_size); + } + + av_log(logctx, AV_LOG_WARNING, "LLVM JIT: no .text section in assembled output\n"); + + return AVERROR_INVALIDDATA; +} diff --git a/libswscale/aarch64/ops_jit_llvm.h b/libswscale/aarch64/ops_jit_llvm.h new file mode 100644 index 0000000000..f03741c287 --- /dev/null +++ b/libswscale/aarch64/ops_jit_llvm.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2026 Ramiro Polla + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SWSCALE_AARCH64_OPS_JIT_LLVM_H +#define SWSCALE_AARCH64_OPS_JIT_LLVM_H + +#include <stddef.h> +#include <stdint.h> + +typedef struct AVBPrint AVBPrint; + +/* Assemble AArch64 GAS-syntax source using LLVM. */ +int ff_sws_jit_assemble_llvm(void *logctx, const char *asm_src, + uint8_t **out_text, size_t *out_size, + AVBPrint *errstr); + +#endif /* SWSCALE_AARCH64_OPS_JIT_LLVM_H */ diff --git a/libswscale/ops.c b/libswscale/ops.c index 0582a32345..05c57fa046 100644 --- a/libswscale/ops.c +++ b/libswscale/ops.c @@ -34,6 +34,7 @@ extern const SwsOpBackend backend_c; extern const SwsOpBackend backend_murder; extern const SwsOpBackend backend_aarch64; +extern const SwsOpBackend backend_aarch64_jit; extern const SwsOpBackend backend_x86; #if HAVE_SPIRV_HEADERS_SPIRV_H || HAVE_SPIRV_UNIFIED1_SPIRV_H extern const SwsOpBackend backend_spirv; @@ -42,6 +43,9 @@ extern const SwsOpBackend backend_spirv; const SwsOpBackend * const ff_sws_op_backends[] = { &backend_murder, #if ARCH_AARCH64 && HAVE_NEON +#if CONFIG_LLVM + &backend_aarch64_jit, +#endif &backend_aarch64, #elif ARCH_X86_64 && HAVE_X86ASM &backend_x86, diff --git a/libswscale/options.c b/libswscale/options.c index fe6c5a6eac..8a7a2bd2c0 100644 --- a/libswscale/options.c +++ b/libswscale/options.c @@ -106,17 +106,18 @@ static const AVOption swscale_options[] = { { "saturation", "saturation mapping", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_INTENT_SATURATION }, .flags = VE, .unit = "intent" }, { "absolute_colorimetric", "absolute colorimetric clipping", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_INTENT_ABSOLUTE_COLORIMETRIC }, .flags = VE, .unit = "intent" }, - { "sws_backends", "set allowed swscale backends", OFFSET(backends), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, .flags = VE, .unit = "sws_backend", .max = UINT_MAX }, - { "auto", "automatic selection", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, .flags = VE, .unit = "sws_backend" }, - { "stable", "All stable backends", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_STABLE }, .flags = VE, .unit = "sws_backend" }, - { "unstable", "All unstable backends", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_UNSTABLE }, .flags = VE, .unit = "sws_backend" }, - { "all", "All available backends", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_ALL }, .flags = VE, .unit = "sws_backend" }, - { "legacy", "legacy swscale code", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_LEGACY }, .flags = VE, .unit = "sws_backend" }, - { "c", "template-based reference code", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_C }, .flags = VE, .unit = "sws_backend" }, - { "memcpy", "fast path using libc memcpy", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_MEMCPY }, .flags = VE, .unit = "sws_backend" }, - { "x86", "x86 SIMD kernels", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_X86 }, .flags = VE, .unit = "sws_backend" }, - { "aarch64", "AArch64 NEON kernels", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_AARCH64 }, .flags = VE, .unit = "sws_backend" }, - { "spirv", "Vulkan SPIR-V backend", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_SPIRV }, .flags = VE, .unit = "sws_backend" }, + { "sws_backends", "set allowed swscale backends", OFFSET(backends), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, .flags = VE, .unit = "sws_backend", .max = UINT_MAX }, + { "auto", "automatic selection", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, .flags = VE, .unit = "sws_backend" }, + { "stable", "All stable backends", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_STABLE }, .flags = VE, .unit = "sws_backend" }, + { "unstable", "All unstable backends", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_UNSTABLE }, .flags = VE, .unit = "sws_backend" }, + { "all", "All available backends", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_ALL }, .flags = VE, .unit = "sws_backend" }, + { "legacy", "legacy swscale code", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_LEGACY }, .flags = VE, .unit = "sws_backend" }, + { "c", "template-based reference code", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_C }, .flags = VE, .unit = "sws_backend" }, + { "memcpy", "fast path using libc memcpy", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_MEMCPY }, .flags = VE, .unit = "sws_backend" }, + { "x86", "x86 SIMD kernels", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_X86 }, .flags = VE, .unit = "sws_backend" }, + { "aarch64", "AArch64 NEON kernels", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_AARCH64 }, .flags = VE, .unit = "sws_backend" }, + { "aarch64_jit", "AArch64 JIT NEON kernels", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_AARCH64_JIT }, .flags = VE, .unit = "sws_backend" }, + { "spirv", "Vulkan SPIR-V backend", 0, AV_OPT_TYPE_CONST, { .i64 = SWS_BACKEND_SPIRV }, .flags = VE, .unit = "sws_backend" }, { NULL } }; diff --git a/libswscale/swscale.h b/libswscale/swscale.h index c913e76985..16a8421a66 100644 --- a/libswscale/swscale.h +++ b/libswscale/swscale.h @@ -117,11 +117,13 @@ typedef enum SwsBackend { SWS_BACKEND_MEMCPY = (1 << 2), ///< Fast path using libc memcpy() / memset() SWS_BACKEND_X86 = (1 << 3), ///< Chained x86 SIMD kernels SWS_BACKEND_AARCH64 = (1 << 4), ///< Chained AArch64 NEON kernels - SWS_BACKEND_SPIRV = (1 << 5), ///< Vulkan SPIR-V backend + SWS_BACKEND_AARCH64_JIT = (1 << 5), ///< JIT AArch64 NEON kernels + SWS_BACKEND_SPIRV = (1 << 6), ///< Vulkan SPIR-V backend SWS_BACKEND_UNSTABLE = SWS_BACKEND_C | SWS_BACKEND_MEMCPY | SWS_BACKEND_X86 | SWS_BACKEND_AARCH64 | + SWS_BACKEND_AARCH64_JIT | SWS_BACKEND_SPIRV, SWS_BACKEND_ALL = SWS_BACKEND_STABLE | SWS_BACKEND_UNSTABLE, diff --git a/tests/checkasm/Makefile b/tests/checkasm/Makefile index c154d19ed4..774fcbd7d0 100644 --- a/tests/checkasm/Makefile +++ b/tests/checkasm/Makefile @@ -162,7 +162,7 @@ tests/checkasm/checkasm.o: CFLAGS += -Umain CHECKASM := tests/checkasm/checkasm$(EXESUF) $(CHECKASM): $(CHECKASMOBJS) $(FF_STATIC_DEP_LIBS) - $(call LINK,$(LDFLAGS) $(LDEXEFLAGS) $(LD_O) $(CHECKASMOBJS) $(FF_STATIC_DEP_LIBS) $(EXTRALIBS-avcodec) $(EXTRALIBS-avfilter) $(EXTRALIBS-avformat) $(EXTRALIBS-avutil) $(EXTRALIBS-swresample) $(EXTRALIBS) $(EXTRALIBS-checkasm)) + $(call LINK,$(LDFLAGS) $(LDEXEFLAGS) $(LD_O) $(CHECKASMOBJS) $(FF_STATIC_DEP_LIBS) $(EXTRALIBS-avcodec) $(EXTRALIBS-avfilter) $(EXTRALIBS-avformat) $(EXTRALIBS-avutil) $(EXTRALIBS-swresample) $(EXTRALIBS-swscale) $(EXTRALIBS) $(EXTRALIBS-checkasm)) run-checkasm: $(CHECKASM) run-checkasm: -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]