[PR] avformat/mpegenc: reject stream counts that overflow the system header (PR #24087)

PR #24087 opened by michaelni
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24087
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24087.patch

put_system_header() writes 12 + 3*N bytes after the pack header into a
fixed 128-byte stack buffer, but is handed a PutBitContext sized past the
real buffer, so its own bounds check never fires; ~35+ streams overflow the
stack. Reject at mux init when the system header would not fit.

Fixes: out of array access
Fixes: many.mkv


>From 0f019ade7141b29933604c885b07471bc3900a96 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <[email protected]>
Date: Tue, 11 Aug 2026 20:42:16 +0200
Subject: [PATCH 1/2] avformat/mpegenc: pass buffer size into
 put_system_header()

Fixes: out of array access
Fixes: many.mkv

Found-by: Joshua Rogers <[email protected]>
Signed-off-by: Michael Niedermayer <[email protected]>
---
 libavformat/mpegenc.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/libavformat/mpegenc.c b/libavformat/mpegenc.c
index c41aa1095c..128dfe2885 100644
--- a/libavformat/mpegenc.c
+++ b/libavformat/mpegenc.c
@@ -128,14 +128,14 @@ static int put_pack_header(AVFormatContext *ctx, uint8_t *buf,
     return put_bytes_output(&pb);
 }
 
-static int put_system_header(AVFormatContext *ctx, uint8_t *buf,
+static int put_system_header(AVFormatContext *ctx, uint8_t *buf, int buf_size,
                              int only_for_stream_id)
 {
     MpegMuxContext *s = ctx->priv_data;
     int size, i, private_stream_coded, id;
     PutBitContext pb;
 
-    init_put_bits(&pb, buf, 128);
+    init_put_bits(&pb, buf, buf_size);
 
     put_bits32(&pb, SYSTEM_HEADER_START_CODE);
     put_bits(&pb, 16, 0);
@@ -657,6 +657,7 @@ static int flush_packet(AVFormatContext *ctx, int stream_index,
     int size, payload_size, startcode, id, stuffing_size, header_len;
     int packet_size;
     uint8_t buffer[128];
+    uint8_t *buf_end = buffer + sizeof(buffer);
     int zero_trail_bytes = 0;
     int pad_packet_bytes = 0;
     int pes_flags;
@@ -682,7 +683,7 @@ static int flush_packet(AVFormatContext *ctx, int stream_index,
              * audio packet (see VCD standard p. IV-7 and IV-8). */
 
             if (stream->packet_number == 0) {
-                size     = put_system_header(ctx, buf_ptr, id);
+                size     = put_system_header(ctx, buf_ptr, buf_end - buf_ptr, id);
                 buf_ptr += size;
             }
         } else if (s->is_dvd) {
@@ -697,7 +698,7 @@ static int flush_packet(AVFormatContext *ctx, int stream_index,
                 }
 
                 if (stream->bytes_to_iframe == 0 || s->packet_number == 0) {
-                    size     = put_system_header(ctx, buf_ptr, 0);
+                    size     = put_system_header(ctx, buf_ptr, buf_end - buf_ptr, 0);
                     buf_ptr += size;
                     size     = buf_ptr - buffer;
                     avio_write(ctx->pb, buffer, size);
@@ -730,7 +731,7 @@ static int flush_packet(AVFormatContext *ctx, int stream_index,
             }
         } else {
             if ((s->packet_number % s->system_header_freq) == 0) {
-                size     = put_system_header(ctx, buf_ptr, 0);
+                size     = put_system_header(ctx, buf_ptr, buf_end - buf_ptr, 0);
                 buf_ptr += size;
             }
         }
-- 
2.52.0


>From 8bd1d425c30dbec9aac17fed5e305cceb3d04bac Mon Sep 17 00:00:00 2001
From: Joshua Rogers <[email protected]>
Date: Tue, 4 Aug 2026 12:11:55 +0000
Subject: [PATCH 2/2] avformat/mpegenc: reject stream counts that overflow the
 system header

put_system_header() writes 12 + 3*N bytes after the pack header into a
fixed 128-byte stack buffer, but is handed a PutBitContext sized past the
real buffer, so its own bounds check never fires; ~35+ streams overflow the
stack. Reject at mux init when the system header would not fit.

Fixes: out of array access
Fixes: many.mkv
---
 libavformat/mpegenc.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/libavformat/mpegenc.c b/libavformat/mpegenc.c
index 128dfe2885..cea91d80da 100644
--- a/libavformat/mpegenc.c
+++ b/libavformat/mpegenc.c
@@ -473,6 +473,16 @@ static av_cold int mpeg_mux_init(AVFormatContext *ctx)
         if (!stream->fifo)
             return AVERROR(ENOMEM);
     }
+
+    /* The system header is emitted, right after the pack header (which is at
+     * most 14 bytes), into the fixed 128-byte buffer used by flush_packet().
+     * Reject configurations whose system header would not fit. */
+    if (get_system_header_size(ctx) > 128 - 14) {
+        av_log(ctx, AV_LOG_ERROR,
+               "Too many streams to fit the MPEG program stream system header\n");
+        return AVERROR(EINVAL);
+    }
+
     bitrate       = 0;
     audio_bitrate = 0;
     video_bitrate = 0;
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]