[PR] avformat/rawutils: reject raw RGB frames that do not fit an AVPacket / avformat/mux: check dimensions in init_muxer() (PR #23900)

PR #23900 opened by michaelni
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23900
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23900.patch

avformat/rawutils: reject raw RGB frames that do not fit an AVPacket
Fixes: integer overflow
Fixes: out of array access
Fixes: payload.film
Fixes: czK1F83k3zvT
Found-by: Clouditera Security, Z.ai Security, NSFOCUS


>From bf6394d031ef8d701b1ad3a113f1fec18b5bb3f3 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <[email protected]>
Date: Wed, 22 Jul 2026 05:57:51 +0200
Subject: [PATCH 1/2] avformat/rawutils: reject raw RGB frames that do not fit
 an AVPacket

Fixes: integer overflow
Fixes: out of array access
Fixes: payload.film
Fixes: czK1F83k3zvT
Found-by: Clouditera Security, Z.ai Security, NSFOCUS
---
 libavformat/rawutils.c | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/libavformat/rawutils.c b/libavformat/rawutils.c
index e44c946d47..35b2c9913c 100644
--- a/libavformat/rawutils.c
+++ b/libavformat/rawutils.c
@@ -29,15 +29,21 @@ int ff_reshuffle_raw_rgb(AVFormatContext *s, AVPacket **ppkt, AVCodecParameters
     int ret;
     AVPacket *pkt = *ppkt;
     int64_t bpc = par->bits_per_coded_sample != 15 ? par->bits_per_coded_sample : 16;
-    int min_stride = (par->width * bpc + 7) >> 3;
-    int with_pal_size = min_stride * par->height + 1024;
-    int contains_pal = bpc == 8 && pkt->size == with_pal_size;
-    int size = contains_pal ? min_stride * par->height : pkt->size;
-    int stride = size / par->height;
-    int padding = expected_stride - FFMIN(expected_stride, stride);
-    int y;
+    int64_t min_stride = (par->width * bpc + 7) >> 3;
+    int with_pal_size, contains_pal, size, stride, padding, y;
     AVPacket *new_pkt;
 
+    if (par->height <= 0 || min_stride <= 0 ||
+        min_stride      > (INT_MAX - 1024) / par->height ||
+        expected_stride > (INT_MAX - AV_INPUT_BUFFER_PADDING_SIZE) / par->height)
+        return 0;
+
+    with_pal_size = min_stride * par->height + 1024;
+    contains_pal  = bpc == 8 && pkt->size == with_pal_size;
+    size          = contains_pal ? min_stride * par->height : pkt->size;
+    stride        = size / par->height;
+    padding       = expected_stride - FFMIN(expected_stride, stride);
+
     if (pkt->size == expected_stride * par->height)
         return 0;
     if (size != stride * par->height)
-- 
2.52.0


>From cbe48bdab51047e5015783eca667d6cb1acef123 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <[email protected]>
Date: Fri, 24 Jul 2026 22:41:05 +0200
Subject: [PATCH 2/2] avformat/mux: check dimensions in init_muxer()

Fixes: czK1F83k3zvT
---
 libavformat/mux.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libavformat/mux.c b/libavformat/mux.c
index ea9838e380..898331cdfa 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -34,6 +34,7 @@
 #include "libavutil/timestamp.h"
 #include "libavutil/avassert.h"
 #include "libavutil/frame.h"
+#include "libavutil/imgutils.h"
 #include "libavutil/internal.h"
 #include "libavutil/mathematics.h"
 
@@ -252,6 +253,9 @@ static int init_muxer(AVFormatContext *s, AVDictionary **options)
                 ret = AVERROR(EINVAL);
                 goto fail;
             }
+            if (!(of->p.flags & AVFMT_NODIMENSIONS) &&
+                (ret = av_image_check_size(par->width, par->height, 0, s)) < 0)
+                goto fail;
             if (av_cmp_q(st->sample_aspect_ratio, par->sample_aspect_ratio)
                 && fabs(av_q2d(st->sample_aspect_ratio) - av_q2d(par->sample_aspect_ratio)) > 0.004*av_q2d(st->sample_aspect_ratio)
             ) {
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]