[PR] avcodec/vulkan_decode: set H.264 pictureLayout from SPS frame_mbs_only_flag (PR #24224)

PR #24224 opened by t-boiko
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24224
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24224.patch

Set VkVideoDecodeH264ProfileInfoKHR::pictureLayout from SPS frame_mbs_only_flag: 
- progressive when frame_mbs_only_flag == 1 / no SPS;
- interlaced when frame_mbs_only_flag == 0 (field/MBAFF allowed per ITU-T H.264).
Do not use avctx->field_order: MBAFF may still report progressive.

The commit fixes a decoding of the ITU-T H.264.1 MBAFF samples:

CAMA1_Sony_C
CAMA3_Sand_E
CAMACI3_Sony_C
CAMANL3_Sand_E
CAMASL3_Sony_B
CANLMA2_Sony_C
CANLMA3_Sony_C
CVCANLMA2_Sony_C
CVMA1_Sony_D
CVMAQP2_Sony_G
CVMAQP3_Sony_D


>From 4e0b062770e10d5d245a962caffb2b43f6e8a2f8 Mon Sep 17 00:00:00 2001
From: Tymur Boiko <[email protected]>
Date: Thu, 20 Aug 2026 22:40:18 +0200
Subject: [PATCH] avcodec/vulkan_decode: set H.264 pictureLayout from SPS
 frame_mbs_only_flag

VkVideoDecodeH264ProfileInfoKHR::pictureLayout must match the sequence.
ITU-T H.264 frame_mbs_only_flag==1 means frames only; ==0 allows field
pictures and MBAFF. Vulkan PROGRESSIVE sessions are progressive-only;
non-PROGRESSIVE enables interlaced field DPB (h264_decode / videocoding).

pictureLayout values are mutually exclusive video profiles (vk.xml).
Map with !frame_mbs_only_flag, then validate with
vkGetPhysicalDeviceVideoCapabilitiesKHR:

- frame_mbs_only_flag == 1, or no SPS -> PROGRESSIVE (one query)
- frame_mbs_only_flag == 0 -> prefer INTERLEAVED_LINES, then SEPARATE_PLANES
  only on VK_ERROR_VIDEO_PICTURE_LAYOUT_NOT_SUPPORTED_KHR

Do not use avctx->field_order: MBAFF may still report progressive.

Signed-off-by: Tymur Boiko <[email protected]>
---
 libavcodec/vulkan_decode.c | 42 ++++++++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 6 deletions(-)

diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c
index b954c78250..635fdeeeea 100644
--- a/libavcodec/vulkan_decode.c
+++ b/libavcodec/vulkan_decode.c
@@ -24,6 +24,9 @@
 #include "libavutil/avassert.h"
 #include "libavutil/mem.h"
 #include "libavutil/vulkan_loader.h"
+#if CONFIG_H264_VULKAN_HWACCEL
+#include "h264dec.h"
+#endif
 
 #define DECODER_IS_SDR(codec_id) \
     (((codec_id) == AV_CODEC_ID_FFV1) || \
@@ -745,6 +748,7 @@ static VkResult vulkan_setup_profile(AVCodecContext *avctx,
     VkVideoDecodeVP9ProfileInfoKHR *vp9_profile  = &prof->vp9_profile;
 #endif
     VkVideoDecodeAV1ProfileInfoKHR *av1_profile  = &prof->av1_profile;
+    int h264_interlaced = 0;
 
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->sw_pix_fmt);
     if (!desc)
@@ -760,10 +764,15 @@ static VkResult vulkan_setup_profile(AVCodecContext *avctx,
         h264_profile->stdProfileIdc = cur_profile & ~(AV_PROFILE_H264_CONSTRAINED |
                                                       AV_PROFILE_H264_INTRA);
 
-        h264_profile->pictureLayout = avctx->field_order == AV_FIELD_UNKNOWN ||
-                                      avctx->field_order == AV_FIELD_PROGRESSIVE ?
-                                      VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_PROGRESSIVE_KHR :
-                                      VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_INTERLEAVED_LINES_BIT_KHR;
+        h264_profile->pictureLayout =
+            VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_PROGRESSIVE_KHR;
+#if CONFIG_H264_VULKAN_HWACCEL
+        {
+            const H264Context *h = avctx->priv_data;
+            if (h && h->ps.sps && !h->ps.sps->frame_mbs_only_flag)
+                h264_interlaced = 1;
+        }
+#endif
     } else if (avctx->codec_id == AV_CODEC_ID_H265) {
         dec_caps->pNext = h265_caps;
         usage->pNext = h265_profile;
@@ -804,8 +813,25 @@ static VkResult vulkan_setup_profile(AVCodecContext *avctx,
     dec_caps->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_CAPABILITIES_KHR;
     /* dec_caps->pNext already filled in */
 
-    return vk->GetPhysicalDeviceVideoCapabilitiesKHR(hwctx->phys_dev, profile,
-                                                     caps);
+    if (h264_interlaced) {
+        static const VkVideoDecodeH264PictureLayoutFlagBitsKHR layouts[] = {
+            VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_INTERLEAVED_LINES_BIT_KHR,
+            VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_SEPARATE_PLANES_BIT_KHR,
+        };
+        VkResult result = VK_ERROR_VIDEO_PICTURE_LAYOUT_NOT_SUPPORTED_KHR;
+
+        for (int i = 0; i < FF_ARRAY_ELEMS(layouts); i++) {
+            h264_profile->pictureLayout = layouts[i];
+            result = vk->GetPhysicalDeviceVideoCapabilitiesKHR(hwctx->phys_dev,
+                                                               profile, caps);
+            if (result != VK_ERROR_VIDEO_PICTURE_LAYOUT_NOT_SUPPORTED_KHR)
+                return result;
+        }
+        return result;
+    } else {
+        return vk->GetPhysicalDeviceVideoCapabilitiesKHR(hwctx->phys_dev, profile,
+                                                         caps);
+    }
 }
 
 static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_ref,
@@ -904,6 +930,10 @@ static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_
                avcodec_get_name(avctx->codec_id),
                avcodec_profile_name(avctx->codec_id, cur_profile));
         return AVERROR(EINVAL);
+    } else if (ret == VK_ERROR_VIDEO_PICTURE_LAYOUT_NOT_SUPPORTED_KHR) {
+        av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
+               "H.264 pictureLayout not supported!\n");
+        return AVERROR(EINVAL);
     } else if (ret == VK_ERROR_VIDEO_PROFILE_FORMAT_NOT_SUPPORTED_KHR) {
         av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
                "format (%s) not supported!\n",
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]