[PR] avfilter/yadif_videotoolbox: reject unsupported pixel layouts (PR #24017)

PR #24017 opened by Qingzheng Li (iSoldLeo)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24017
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24017.patch

yadif_videotoolbox negotiates the outer AV_PIX_FMT_VIDEOTOOLBOX
format, while the actual CVPixelBuffer layout is described by
AVHWFramesContext.sw_format.

The current Metal path maps each processed plane only to one- or
two-channel 8/16-bit textures. Previously, incompatible layouts such
as BGRA, AYUV and AYUV64 were detected only in the per-frame backend
callback. Since that callback returns void, returning before running
the kernel did not propagate an error to the common YADIF code, which
then forwarded the already allocated but unwritten destination frame.

Validate sw_format during input configuration, before retaining the
input frames context or creating the output frame pool. Move texture
format selection into a shared helper used by both the configuration
check and the runtime guard. The helper also rejects unsupported
component sizes and non-integral channel layouts before division.

This preserves every layout accepted by the previous per-frame Metal
texture-layout check. In particular, packed UYVY/YUYV behavior is
deliberately unchanged; their component-semantics issue can be handled
separately.

Validation on Apple M4 / macOS 27:

- BGRA, AYUV and AYUV64 fail during configuration with AVERROR(ENOSYS)
  and produce no frames.
- A directly uploaded bayer_rggb16le VideoToolbox context is also
  rejected during configuration with no output.
- NV12, P010, YUV420P, NV16, NV24, P210, P216, P410, P416 and UYVY
  produce three frames each, with frame checksums identical to their
  matching baselines.
- A 66x34 NV12 send_field / BFF control produces four frames.
- A full configured FATE run completes with 5548 tests, 5548 reports,
  no missing or extra reports, and no failures.
- fate-hw passes.

Successful Metal controls encounter the independent Objective-C
teardown issue tracked by #23850 on an unmodified tree. Its ownership
fix was applied only as a validation overlay to confirm clean exit and
is not included in this change.



>From 0064f927d1861cf3ac431be8fd6f5f01722d1397 Mon Sep 17 00:00:00 2001
From: Qingzheng Li <[email protected]>
Date: Wed, 5 Aug 2026 20:15:18 +0800
Subject: [PATCH] avfilter/yadif_videotoolbox: reject unsupported input formats

The actual layout of VideoToolbox frames is stored in
AVHWFramesContext.sw_format. It was previously checked only in the
per-frame callback, after the destination frame had been allocated.

For unsupported layouts such as BGRA, AYUV and AYUV64, the callback
returned without writing the destination. Since the callback returns
void, common YADIF code then forwarded the unwritten frame downstream.

Factor Metal texture format selection into a shared helper and validate
sw_format during input configuration. Validate the pixel size before
division and reuse the helper in the per-frame path.

Signed-off-by: Qingzheng Li <[email protected]>
---
 libavfilter/vf_yadif_videotoolbox.m | 67 +++++++++++++++++++++++------
 1 file changed, 53 insertions(+), 14 deletions(-)

diff --git a/libavfilter/vf_yadif_videotoolbox.m b/libavfilter/vf_yadif_videotoolbox.m
index f8eb0c6bfa..1904dc87fa 100644
--- a/libavfilter/vf_yadif_videotoolbox.m
+++ b/libavfilter/vf_yadif_videotoolbox.m
@@ -63,6 +63,49 @@ struct mtlYadifParams {
     int field_mode;
 };
 
+static int get_texture_format(const AVComponentDescriptor *comp,
+                              int *pixel_size, int *channels,
+                              MTLPixelFormat *format)
+                              API_AVAILABLE(macos(10.11), ios(8.0))
+{
+    *pixel_size = (comp->depth + comp->shift) / 8;
+    if ((*pixel_size != 1 && *pixel_size != 2) ||
+        comp->step % *pixel_size)
+        return AVERROR(ENOSYS);
+
+    *channels = comp->step / *pixel_size;
+    if (*channels < 1 || *channels > 2)
+        return AVERROR(ENOSYS);
+
+    if (*pixel_size == 1)
+        *format = *channels == 1 ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
+    else
+        *format = *channels == 1 ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
+
+    return 0;
+}
+
+static int format_has_supported_texture_layout(const AVPixFmtDescriptor *desc)
+                                               API_AVAILABLE(macos(10.11), ios(8.0))
+{
+    if (!desc)
+        return 0;
+
+    for (int i = 0; i < desc->nb_components; i++) {
+        const AVComponentDescriptor *comp = &desc->comp[i];
+        MTLPixelFormat format;
+        int pixel_size, channels;
+
+        if (comp->plane < i)
+            continue;
+
+        if (get_texture_format(comp, &pixel_size, &channels, &format) < 0)
+            return 0;
+    }
+
+    return 1;
+}
+
 static void call_kernel(AVFilterContext *ctx,
                         id<MTLTexture> dst,
                         id<MTLTexture> prev,
@@ -120,20 +163,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
             continue;
         }
 
-        pixel_size = (comp->depth + comp->shift) / 8;
-        channels = comp->step / pixel_size;
-        if (pixel_size > 2 || channels > 2) {
-            av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
-            goto exit;
-        }
-        switch (pixel_size) {
-        case 1:
-            format = channels == 1 ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
-            break;
-        case 2:
-            format = channels == 1 ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
-            break;
-        default:
+        if (get_texture_format(comp, &pixel_size, &channels, &format) < 0) {
             av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
             goto exit;
         }
@@ -291,6 +321,7 @@ static int do_config_input(AVFilterLink *inlink) API_AVAILABLE(macos(10.11), ios
     FilterLink *l = ff_filter_link(inlink);
     AVFilterContext *ctx = inlink->dst;
     YADIFVTContext *s = ctx->priv;
+    AVHWFramesContext *input_frames;
 
     if (!l->hw_frames_ctx) {
         av_log(ctx, AV_LOG_ERROR, "A hardware frames reference is "
@@ -298,6 +329,14 @@ static int do_config_input(AVFilterLink *inlink) API_AVAILABLE(macos(10.11), ios
         return AVERROR(EINVAL);
     }
 
+    input_frames = (AVHWFramesContext *)l->hw_frames_ctx->data;
+    if (!format_has_supported_texture_layout(
+            av_pix_fmt_desc_get(input_frames->sw_format))) {
+        av_log(ctx, AV_LOG_ERROR, "Unsupported input format: %s\n",
+               av_get_pix_fmt_name(input_frames->sw_format));
+        return AVERROR(ENOSYS);
+    }
+
     s->input_frames_ref = av_buffer_ref(l->hw_frames_ctx);
     if (!s->input_frames_ref) {
         av_log(ctx, AV_LOG_ERROR, "A input frames reference create "
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]