[PR] avfilter/vf_scale_cuda: various fixes (PR #24154)

PR #24154 opened by Timo Rothenpieler (BtbN)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24154
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24154.patch

This should fix the crash/malfunction observed in #24126
The filter would run intermediate passes even when none were needed.
Likewise, a passthrough-pass would not setup any passes, but not disable filtering, resulting in the same.

Also fixes a bunch of things I ran into while working on it, like the filtering logic not taking chroma subsampling into account for what it considers downscaling.


>From 1a29cba1520e01e05eacea57770331ca3ece8a93 Mon Sep 17 00:00:00 2001
From: Timo Rothenpieler <[email protected]>
Date: Fri, 14 Aug 2026 22:47:47 +0200
Subject: [PATCH 1/5] avfilter/vf_scale_cuda: don't try to execute disabled
 filter passes

---
 libavfilter/vf_scale_cuda.c | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c
index 240e40d43b..95ecde2774 100644
--- a/libavfilter/vf_scale_cuda.c
+++ b/libavfilter/vf_scale_cuda.c
@@ -157,6 +157,7 @@ typedef struct CUDAScaleContext {
     CUDAScaleFilter filters_uv[FILTER_NB];
     CUDATex inter_tex;
     int use_filters; /* -1 for auto */
+    int pass_x, pass_y;
 
     float param;
 } CUDAScaleContext;
@@ -407,6 +408,8 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int
         s->frames_ctx = av_buffer_ref(inl->hw_frames_ctx);
         if (!s->frames_ctx)
             return AVERROR(ENOMEM);
+
+        s->use_filters = 0;
     } else {
         s->passthrough = 0;
 
@@ -451,9 +454,7 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx)
 
     if (s->use_filters) {
         /* Final pass is always vertical unless not vertically scaling */
-        AVFilterLink  *inlink = ctx->inputs[0];
-        AVFilterLink *outlink = ctx->outputs[0];
-        function_infix = inlink->h == outlink->h ? "Generic_h" : "Generic_v";
+        function_infix = s->pass_y == FILTER_OUT ? "Generic_v" : "Generic_h";
         s->interp_use_linear = 0;
         s->interp_as_integer = 0;
     } else {
@@ -623,19 +624,19 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx)
     if (ret < 0)
         return ret;
 
-    int pass_x = -1, pass_y = -1;
+    s->pass_x = s->pass_y = -1;
     if (inlink->w != outlink->w && inlink->h != outlink->h) {
         /* Always perform the horizontal scaling pass first */
-        pass_x = FILTER_TMP;
-        pass_y = FILTER_OUT;
+        s->pass_x = FILTER_TMP;
+        s->pass_y = FILTER_OUT;
     } else if (inlink->w != outlink->w) {
-        pass_x = FILTER_OUT;
+        s->pass_x = FILTER_OUT;
     } else if (inlink->h != outlink->h) {
-        pass_y = FILTER_OUT;
+        s->pass_y = FILTER_OUT;
     }
 
-    if (pass_x >= 0) {
-        ret = cudascale_filter_init(ctx, &s->filters[pass_x],
+    if (s->pass_x >= 0) {
+        ret = cudascale_filter_init(ctx, &s->filters[s->pass_x],
                                     inlink->w, outlink->w, 0.0);
         if (ret < 0)
             goto fail;
@@ -643,15 +644,15 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx)
             const int src_size = AV_CEIL_RSHIFT(inlink->w,  in_sub_x);
             const int dst_size = AV_CEIL_RSHIFT(outlink->w, out_sub_x);
             const double virtual_size = (double) outlink->w / (1 << out_sub_x);
-            ret = cudascale_filter_init(ctx, &s->filters_uv[pass_x],
+            ret = cudascale_filter_init(ctx, &s->filters_uv[s->pass_x],
                                         src_size, dst_size, virtual_size);
             if (ret < 0)
                 goto fail;
         }
     }
 
-    if (pass_y >= 0) {
-        ret = cudascale_filter_init(ctx, &s->filters[pass_y],
+    if (s->pass_y >= 0) {
+        ret = cudascale_filter_init(ctx, &s->filters[s->pass_y],
                                     inlink->h, outlink->h, 0.0);
         if (ret < 0)
             goto fail;
@@ -659,7 +660,7 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx)
             const int src_size = AV_CEIL_RSHIFT(inlink->h,  in_sub_y);
             const int dst_size = AV_CEIL_RSHIFT(outlink->h, out_sub_y);
             const double virtual_size = (double) outlink->h / (1 << out_sub_y);
-            ret = cudascale_filter_init(ctx, &s->filters_uv[pass_y],
+            ret = cudascale_filter_init(ctx, &s->filters_uv[s->pass_y],
                                         src_size, dst_size, virtual_size);
             if (ret < 0)
                 goto fail;
@@ -915,7 +916,7 @@ static int cudascale_scale(AVFilterContext *ctx, AVFrame *out, AVFrame *in)
         goto fail;
 
     const CUDATex *src = &in_tex;
-    if (s->use_filters) {
+    if (s->pass_x == FILTER_TMP) {
         /* Handle first pass separately */
         s->inter_tex.color_range = in->color_range;
         ret = scalecuda_resize(ctx, FILTER_TMP, &s->inter_tex, src);
-- 
2.52.0


>From 403d17af1a95c21929be8a6e4761c9d075c36e5b Mon Sep 17 00:00:00 2001
From: Timo Rothenpieler <[email protected]>
Date: Fri, 14 Aug 2026 22:56:20 +0200
Subject: [PATCH 2/5] avfilter/vf_scale_cuda: skip allocating intermediate
 texture if not needed

---
 libavfilter/vf_scale_cuda.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c
index 95ecde2774..2c22ec1aa4 100644
--- a/libavfilter/vf_scale_cuda.c
+++ b/libavfilter/vf_scale_cuda.c
@@ -667,9 +667,11 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx)
         }
     }
 
-    ret = inter_buf_init(ctx, outlink->w, inlink->h);
-    if (ret < 0)
-        goto fail;
+    if (s->pass_x == FILTER_TMP) {
+        ret = inter_buf_init(ctx, outlink->w, inlink->h);
+        if (ret < 0)
+            goto fail;
+    }
 
     ret = 0;
 
-- 
2.52.0


>From 11042b0d0c7958b0a10a66afea40fd5202d020fd Mon Sep 17 00:00:00 2001
From: Timo Rothenpieler <[email protected]>
Date: Fri, 14 Aug 2026 23:13:16 +0200
Subject: [PATCH 3/5] avfilter/vf_scale_cuda: extract pass determination into
 its own function

---
 libavfilter/vf_scale_cuda.c | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c
index 2c22ec1aa4..fc3a91d299 100644
--- a/libavfilter/vf_scale_cuda.c
+++ b/libavfilter/vf_scale_cuda.c
@@ -369,6 +369,27 @@ static av_cold void set_format_info(AVFilterContext *ctx, enum AVPixelFormat in_
     }
 }
 
+static av_cold void cudascale_setup_passes(AVFilterContext *ctx)
+{
+    CUDAScaleContext *s = ctx->priv;
+    AVFilterLink  *inlink = ctx->inputs[0];
+    AVFilterLink *outlink = ctx->outputs[0];
+
+    s->pass_x = s->pass_y = -1;
+    if (!s->use_filters)
+        return;
+
+    if (inlink->w != outlink->w && inlink->h != outlink->h) {
+        /* Always perform the horizontal scaling pass first */
+        s->pass_x = FILTER_TMP;
+        s->pass_y = FILTER_OUT;
+    } else if (inlink->w != outlink->w) {
+        s->pass_x = FILTER_OUT;
+    } else if (inlink->h != outlink->h) {
+        s->pass_y = FILTER_OUT;
+    }
+}
+
 static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int in_height,
                                          int out_width, int out_height)
 {
@@ -433,6 +454,8 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int
     if (!outl->hw_frames_ctx)
         return AVERROR(ENOMEM);
 
+    cudascale_setup_passes(ctx);
+
     return 0;
 }
 
@@ -624,17 +647,6 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx)
     if (ret < 0)
         return ret;
 
-    s->pass_x = s->pass_y = -1;
-    if (inlink->w != outlink->w && inlink->h != outlink->h) {
-        /* Always perform the horizontal scaling pass first */
-        s->pass_x = FILTER_TMP;
-        s->pass_y = FILTER_OUT;
-    } else if (inlink->w != outlink->w) {
-        s->pass_x = FILTER_OUT;
-    } else if (inlink->h != outlink->h) {
-        s->pass_y = FILTER_OUT;
-    }
-
     if (s->pass_x >= 0) {
         ret = cudascale_filter_init(ctx, &s->filters[s->pass_x],
                                     inlink->w, outlink->w, 0.0);
-- 
2.52.0


>From f55ab33a1c8cbf7943f6e21f0ba9fcf881ec4858 Mon Sep 17 00:00:00 2001
From: Timo Rothenpieler <[email protected]>
Date: Fri, 14 Aug 2026 23:26:14 +0200
Subject: [PATCH 4/5] avfilter/vf_scale_cuda: always have valid passes setup

---
 libavfilter/vf_scale_cuda.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c
index fc3a91d299..7b001c9f5b 100644
--- a/libavfilter/vf_scale_cuda.c
+++ b/libavfilter/vf_scale_cuda.c
@@ -385,7 +385,7 @@ static av_cold void cudascale_setup_passes(AVFilterContext *ctx)
         s->pass_y = FILTER_OUT;
     } else if (inlink->w != outlink->w) {
         s->pass_x = FILTER_OUT;
-    } else if (inlink->h != outlink->h) {
+    } else {
         s->pass_y = FILTER_OUT;
     }
 }
-- 
2.52.0


>From 80c5c1ccb382d4274002ad0c000ce5f19da069d6 Mon Sep 17 00:00:00 2001
From: Timo Rothenpieler <[email protected]>
Date: Fri, 14 Aug 2026 23:52:00 +0200
Subject: [PATCH 5/5] avfilter/vf_scale_cuda: consider chroma subsampling for
 scaling decisions

---
 libavfilter/vf_scale_cuda.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c
index 7b001c9f5b..36c53288be 100644
--- a/libavfilter/vf_scale_cuda.c
+++ b/libavfilter/vf_scale_cuda.c
@@ -379,11 +379,16 @@ static av_cold void cudascale_setup_passes(AVFilterContext *ctx)
     if (!s->use_filters)
         return;
 
-    if (inlink->w != outlink->w && inlink->h != outlink->h) {
+    const int scale_x = inlink->w != outlink->w ||
+                        s->in_desc->log2_chroma_w != s->out_desc->log2_chroma_w;
+    const int scale_y = inlink->h != outlink->h ||
+                        s->in_desc->log2_chroma_h != s->out_desc->log2_chroma_h;
+
+    if (scale_x && scale_y) {
         /* Always perform the horizontal scaling pass first */
         s->pass_x = FILTER_TMP;
         s->pass_y = FILTER_OUT;
-    } else if (inlink->w != outlink->w) {
+    } else if (scale_x) {
         s->pass_x = FILTER_OUT;
     } else {
         s->pass_y = FILTER_OUT;
@@ -444,7 +449,11 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int
 
         if (s->interp_algo == INTERP_ALGO_NEAREST) {
             s->use_filters = 0;
-        } else if (s->use_filters < 0 && (out_width < in_width || out_height < in_height))
+        } else if (s->use_filters < 0 && (
+                       out_width < in_width || out_height < in_height ||
+                       AV_CEIL_RSHIFT(out_width, s->out_desc->log2_chroma_w) < AV_CEIL_RSHIFT(in_width, s->in_desc->log2_chroma_w) ||
+                       AV_CEIL_RSHIFT(out_height, s->out_desc->log2_chroma_h) < AV_CEIL_RSHIFT(in_height, s->in_desc->log2_chroma_h)
+                   ))
             s->use_filters = 1; /* downscaling; needed for anti-aliasing */
         else if (s->use_filters < 0)
             s->use_filters = 0;
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]