[PR] avformat & h264: avoid-decode-for-parse (PR #24320)

PR #24320 opened by michaelni
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24320
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24320.patch

Previously av_find_stream_info() did decode a h264 frame to obtain all needed info, 
this patchset attempts to extract that information without fully decoding frame(s)
this depends on all SPS matching as without a frame we cannot know which is correct otherwise


>From 54635d2abf4b792cedf4b53588d0d658fe95436b Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <[email protected]>
Date: Sat, 29 Aug 2026 19:45:07 +0200
Subject: [PATCH 1/3] h264: factor the software pixel format choice out of
 get_pixel_format()

---
 libavcodec/h264_slice.c | 64 +++++------------------------------------
 libavcodec/h264dec.h    | 30 +++++++++++++++++++
 2 files changed, 37 insertions(+), 57 deletions(-)

diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 9b5ed8f77e..95911cfdef 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -799,15 +799,6 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback)
 
     switch (h->ps.sps->bit_depth_luma) {
     case 9:
-        if (CHROMA444(h)) {
-            if (h->avctx->colorspace == AVCOL_SPC_RGB) {
-                *fmt++ = AV_PIX_FMT_GBRP9;
-            } else
-                *fmt++ = AV_PIX_FMT_YUV444P9;
-        } else if (CHROMA422(h))
-            *fmt++ = AV_PIX_FMT_YUV422P9;
-        else
-            *fmt++ = AV_PIX_FMT_YUV420P9;
         break;
     case 10:
 #if CONFIG_H264_VIDEOTOOLBOX_HWACCEL
@@ -823,47 +814,20 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback)
 #if CONFIG_H264_NVDEC_CUARRAY_HWACCEL
         *fmt++ = AV_PIX_FMT_CUARRAY;
 #endif
-        if (CHROMA444(h)) {
-            if (h->avctx->colorspace == AVCOL_SPC_RGB) {
-                *fmt++ = AV_PIX_FMT_GBRP10;
-            } else
-                *fmt++ = AV_PIX_FMT_YUV444P10;
-        } else if (CHROMA422(h))
-            *fmt++ = AV_PIX_FMT_YUV422P10;
-        else {
 #if CONFIG_H264_VAAPI_HWACCEL
-            // Just add as candidate. Whether VAProfileH264High10 usable or
-            // not is decided by vaapi_decode_make_config() defined in FFmpeg
-            // and vaQueryCodingProfile() defined in libva.
+        // Just add as candidate. Whether VAProfileH264High10 usable or
+        // not is decided by vaapi_decode_make_config() defined in FFmpeg
+        // and vaQueryCodingProfile() defined in libva.
+        if (!CHROMA444(h) && !CHROMA422(h))
             *fmt++ = AV_PIX_FMT_VAAPI;
 #endif
-            *fmt++ = AV_PIX_FMT_YUV420P10;
-        }
         break;
     case 12:
 #if CONFIG_H264_VULKAN_HWACCEL
         *fmt++ = AV_PIX_FMT_VULKAN;
 #endif
-        if (CHROMA444(h)) {
-            if (h->avctx->colorspace == AVCOL_SPC_RGB) {
-                *fmt++ = AV_PIX_FMT_GBRP12;
-            } else
-                *fmt++ = AV_PIX_FMT_YUV444P12;
-        } else if (CHROMA422(h))
-            *fmt++ = AV_PIX_FMT_YUV422P12;
-        else
-            *fmt++ = AV_PIX_FMT_YUV420P12;
         break;
     case 14:
-        if (CHROMA444(h)) {
-            if (h->avctx->colorspace == AVCOL_SPC_RGB) {
-                *fmt++ = AV_PIX_FMT_GBRP14;
-            } else
-                *fmt++ = AV_PIX_FMT_YUV444P14;
-        } else if (CHROMA422(h))
-            *fmt++ = AV_PIX_FMT_YUV422P14;
-        else
-            *fmt++ = AV_PIX_FMT_YUV420P14;
         break;
     case 8:
 #if CONFIG_H264_VDPAU_HWACCEL
@@ -882,19 +846,7 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback)
         if (h->avctx->colorspace != AVCOL_SPC_RGB)
             *fmt++ = AV_PIX_FMT_VIDEOTOOLBOX;
 #endif
-        if (CHROMA444(h)) {
-            if (h->avctx->colorspace == AVCOL_SPC_RGB)
-                *fmt++ = AV_PIX_FMT_GBRP;
-            else if (h->avctx->color_range == AVCOL_RANGE_JPEG)
-                *fmt++ = AV_PIX_FMT_YUVJ444P;
-            else
-                *fmt++ = AV_PIX_FMT_YUV444P;
-        } else if (CHROMA422(h)) {
-            if (h->avctx->color_range == AVCOL_RANGE_JPEG)
-                *fmt++ = AV_PIX_FMT_YUVJ422P;
-            else
-                *fmt++ = AV_PIX_FMT_YUV422P;
-        } else {
+        if (!CHROMA444(h) && !CHROMA422(h)) {
 #if CONFIG_H264_DXVA2_HWACCEL
             *fmt++ = AV_PIX_FMT_DXVA2_VLD;
 #endif
@@ -908,10 +860,6 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback)
 #if CONFIG_H264_VAAPI_HWACCEL
             *fmt++ = AV_PIX_FMT_VAAPI;
 #endif
-            if (h->avctx->color_range == AVCOL_RANGE_JPEG)
-                *fmt++ = AV_PIX_FMT_YUVJ420P;
-            else
-                *fmt++ = AV_PIX_FMT_YUV420P;
         }
         break;
     default:
@@ -920,6 +868,8 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback)
         return AVERROR_INVALIDDATA;
     }
 
+    *fmt++ = h264_sw_pix_fmt(h->ps.sps, h->avctx);
+
     *fmt = AV_PIX_FMT_NONE;
 
     for (int i = 0; pix_fmts[i] != AV_PIX_FMT_NONE; i++)
diff --git a/libavcodec/h264dec.h b/libavcodec/h264dec.h
index 74fd09dfaa..5817f5381f 100644
--- a/libavcodec/h264dec.h
+++ b/libavcodec/h264dec.h
@@ -703,4 +703,34 @@ void ff_h264_free_tables(H264Context *h);
 
 void ff_h264_set_erpic(ERPicture *dst, const H264Picture *src);
 
+static inline enum AVPixelFormat h264_sw_pix_fmt(const SPS *sps,
+                                                 const AVCodecContext *avctx)
+{
+    const int chroma444 = sps->chroma_format_idc == 3;
+    const int chroma422 = sps->chroma_format_idc == 2;
+    const int rgb       = avctx->colorspace  == AVCOL_SPC_RGB;
+    const int jpeg      = avctx->color_range == AVCOL_RANGE_JPEG;
+
+    switch (sps->bit_depth_luma) {
+    case 9:
+        return chroma444 ? (rgb ? AV_PIX_FMT_GBRP9  : AV_PIX_FMT_YUV444P9)  :
+               chroma422 ?        AV_PIX_FMT_YUV422P9  : AV_PIX_FMT_YUV420P9;
+    case 10:
+        return chroma444 ? (rgb ? AV_PIX_FMT_GBRP10 : AV_PIX_FMT_YUV444P10) :
+               chroma422 ?        AV_PIX_FMT_YUV422P10 : AV_PIX_FMT_YUV420P10;
+    case 12:
+        return chroma444 ? (rgb ? AV_PIX_FMT_GBRP12 : AV_PIX_FMT_YUV444P12) :
+               chroma422 ?        AV_PIX_FMT_YUV422P12 : AV_PIX_FMT_YUV420P12;
+    case 14:
+        return chroma444 ? (rgb ? AV_PIX_FMT_GBRP14 : AV_PIX_FMT_YUV444P14) :
+               chroma422 ?        AV_PIX_FMT_YUV422P14 : AV_PIX_FMT_YUV420P14;
+    case 8:
+        return chroma444 ? (rgb  ? AV_PIX_FMT_GBRP     :
+                            jpeg ? AV_PIX_FMT_YUVJ444P : AV_PIX_FMT_YUV444P) :
+               chroma422 ? (jpeg ? AV_PIX_FMT_YUVJ422P : AV_PIX_FMT_YUV422P) :
+                           (jpeg ? AV_PIX_FMT_YUVJ420P : AV_PIX_FMT_YUV420P);
+    }
+    return AV_PIX_FMT_NONE;
+}
+
 #endif /* AVCODEC_H264DEC_H */
-- 
2.52.0


>From 6e79ab0fb1c99a1a75bf63daaaa17452a4e36108 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <[email protected]>
Date: Sat, 29 Aug 2026 15:13:52 +0200
Subject: [PATCH 2/3] h264dec: export stream parameters from the extradata at
 init

avformat_find_stream_info() then sees the complete parameter set
without decoding a frame.
---
 libavcodec/h264dec.c | 70 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 66 insertions(+), 4 deletions(-)

diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c
index b78b7989ea..09d1153b2f 100644
--- a/libavcodec/h264dec.c
+++ b/libavcodec/h264dec.c
@@ -59,6 +59,71 @@
 
 const uint16_t ff_h264_mb_sizes[4] = { 256, 384, 512, 768 };
 
+static const SPS *h264_effective_sps(const H264ParamSets *ps)
+{
+    const SPS *sps = ps->sps;
+
+    if (sps)
+        return sps;
+
+    for (int i = 0; i < MAX_SPS_COUNT; i++) {
+        const SPS *s = ps->sps_list[i];
+        if (!s)
+            continue;
+        if (!sps) {
+            sps = s;
+            continue;
+        }
+        if (s->chroma_format_idc          != sps->chroma_format_idc          ||
+            s->bit_depth_luma             != sps->bit_depth_luma             ||
+            s->level_idc                  != sps->level_idc                  ||
+            ff_h264_get_profile(s)        != ff_h264_get_profile(sps)        ||
+            s->bitstream_restriction_flag != sps->bitstream_restriction_flag ||
+            s->num_reorder_frames         != sps->num_reorder_frames         ||
+            s->vui.sar.num                != sps->vui.sar.num                ||
+            s->vui.sar.den                != sps->vui.sar.den                ||
+            s->vui.video_signal_type_present_flag  != sps->vui.video_signal_type_present_flag  ||
+            s->vui.video_full_range_flag           != sps->vui.video_full_range_flag           ||
+            s->vui.colour_description_present_flag != sps->vui.colour_description_present_flag ||
+            s->vui.colour_primaries       != sps->vui.colour_primaries       ||
+            s->vui.transfer_characteristics != sps->vui.transfer_characteristics ||
+            s->vui.matrix_coeffs          != sps->vui.matrix_coeffs          ||
+            s->vui.chroma_location        != sps->vui.chroma_location)
+            return NULL;
+    }
+    return sps;
+}
+
+static void h264_export_extradata_params(AVCodecContext *avctx, const H264Context *h)
+{
+    const SPS *sps = h264_effective_sps(&h->ps);
+
+    if (!sps)
+        return;
+
+    avctx->profile = ff_h264_get_profile(sps);
+    avctx->level   = sps->level_idc;
+    if (sps->vui.video_signal_type_present_flag) {
+        avctx->color_range = sps->vui.video_full_range_flag > 0 ? AVCOL_RANGE_JPEG
+                                                                : AVCOL_RANGE_MPEG;
+        if (sps->vui.colour_description_present_flag) {
+            avctx->color_primaries = sps->vui.colour_primaries;
+            avctx->color_trc       = sps->vui.transfer_characteristics;
+            avctx->colorspace      = sps->vui.matrix_coeffs;
+        }
+    }
+    avctx->chroma_sample_location = sps->vui.chroma_location;
+    avctx->bits_per_raw_sample    = sps->bit_depth_luma;
+    if (!avctx->sample_aspect_ratio.num && sps->vui.sar.num > 0)
+        avctx->sample_aspect_ratio = sps->vui.sar;
+    if (sps->bitstream_restriction_flag &&
+        avctx->has_b_frames < sps->num_reorder_frames)
+        avctx->has_b_frames = sps->num_reorder_frames;
+    if (avctx->pix_fmt == AV_PIX_FMT_NONE &&
+        avctx->get_format == avcodec_default_get_format)
+        avctx->pix_fmt = h264_sw_pix_fmt(sps, avctx);
+}
+
 int avpriv_h264_has_num_reorder_frames(AVCodecContext *avctx)
 {
     H264Context *h = avctx->priv_data;
@@ -413,10 +478,7 @@ static av_cold int h264_decode_init(AVCodecContext *avctx)
         }
     }
 
-    if (h->ps.sps && h->ps.sps->bitstream_restriction_flag &&
-        h->avctx->has_b_frames < h->ps.sps->num_reorder_frames) {
-        h->avctx->has_b_frames = h->ps.sps->num_reorder_frames;
-    }
+    h264_export_extradata_params(avctx, h);
 
     ff_h264_flush_change(h);
 
-- 
2.52.0


>From 36a1338569eb3a7fd98ad5a6c051432e6a890335 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <[email protected]>
Date: Sat, 29 Aug 2026 15:13:52 +0200
Subject: [PATCH 3/3] avformat/demux: settle the h264 reorder-delay guess
 without decoding

---
 libavcodec/h264dec.c | 4 +++-
 libavformat/demux.c  | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c
index 09d1153b2f..8c2878e03f 100644
--- a/libavcodec/h264dec.c
+++ b/libavcodec/h264dec.c
@@ -127,7 +127,9 @@ static void h264_export_extradata_params(AVCodecContext *avctx, const H264Contex
 int avpriv_h264_has_num_reorder_frames(AVCodecContext *avctx)
 {
     H264Context *h = avctx->priv_data;
-    return h && h->ps.sps ? h->ps.sps->num_reorder_frames : 0;
+    const SPS *sps = h ? h264_effective_sps(&h->ps) : NULL;
+
+    return sps ? sps->num_reorder_frames : 0;
 }
 
 static void h264_er_decode_mb(void *opaque, int ref, int mv_dir, int mv_type,
diff --git a/libavformat/demux.c b/libavformat/demux.c
index da0f5853cc..c789a06bdf 100644
--- a/libavformat/demux.c
+++ b/libavformat/demux.c
@@ -764,7 +764,7 @@ static int has_decode_delay_been_guessed(AVStream *st)
         return 1;
     av_assert0(sti->avctx->codec_id == AV_CODEC_ID_H264 || (sti->avctx->codec_id == AV_CODEC_ID_NONE && !avcodec_is_open(sti->avctx)));
 #if CONFIG_H264_DECODER
-    if (sti->avctx->has_b_frames && avcodec_is_open(sti->avctx) &&
+    if (avcodec_is_open(sti->avctx) &&
         avpriv_h264_has_num_reorder_frames(sti->avctx) == sti->avctx->has_b_frames)
         return 1;
 #endif
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]