[PATCH 0/3] avformat: systematically harden integer arithmetic in network-reachable demuxers

From 49ce529eb897f01735a74d35ceacbccc511885ab Mon Sep 17 00:00:00 2001
From: Hardik-369 <[email protected]>
Date: Sun, 5 Jul 2026 10:39:29 +0530
Subject: [PATCH 1/3] avformat/rtp: replace unsafe multiplication with
 overflow-checked variants

Replace raw av_malloc(a * b) and av_realloc(NULL, a * b) calls with
av_malloc_array() and av_realloc_array() which perform proper overflow
checking internally.

rtpdec_rfc4175.c: use uint64_t arithmetic for frame_size and copy_offset
computations derived from SDP fmtp attributes (width, height, sampling)
and RTP packet headers (line, offset). Previously, signed int
multiplication could overflow with undefined behavior, potentially
bypassing downstream bounds checks.

rtpdec_asf.c: add overflow check for base64 decoded size computation
before allocation.

Signed-off-by: Siddharth <[email protected]>
---
 libavformat/rtpdec_asf.c     | 11 +++++++++--
 libavformat/rtpdec_mpeg4.c   |  2 +-
 libavformat/rtpdec_qt.c      |  2 +-
 libavformat/rtpdec_rfc4175.c |  9 ++++++---
 4 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/libavformat/rtpdec_asf.c b/libavformat/rtpdec_asf.c
index f7fa69e..d11c2e3 100644
--- a/libavformat/rtpdec_asf.c
+++ b/libavformat/rtpdec_asf.c
@@ -106,8 +106,15 @@ int ff_wms_parse_sdp_a_line(AVFormatContext *s, const char *p)
         FFIOContext pb;
         RTSPState *rt = s->priv_data;
         AVDictionary *opts = NULL;
-        int len = strlen(p) * 6 / 8;
-        char *buf = av_mallocz(len);
+        size_t p_len = strlen(p);
+        size_t decoded_size = p_len * 6 / 8;
+        int len;
+        char *buf;
+
+        if (decoded_size > INT_MAX)
+            return AVERROR(ENOMEM);
+        len = decoded_size;
+        buf = av_mallocz(len);
         const AVInputFormat *iformat;
 
         if (!buf)
diff --git a/libavformat/rtpdec_mpeg4.c b/libavformat/rtpdec_mpeg4.c
index 2a9ee99..70d1018 100644
--- a/libavformat/rtpdec_mpeg4.c
+++ b/libavformat/rtpdec_mpeg4.c
@@ -162,7 +162,7 @@ static int rtp_parse_mp4_au(PayloadContext *data, const uint8_t *buf, int len)
     data->nb_au_headers = au_headers_length / au_header_size;
     if (!data->au_headers || data->au_headers_allocated < data->nb_au_headers) {
         av_free(data->au_headers);
-        data->au_headers = av_malloc(sizeof(struct AUHeaders) * data->nb_au_headers);
+        data->au_headers = av_malloc_array(data->nb_au_headers, sizeof(struct AUHeaders));
         if (!data->au_headers)
             return AVERROR(ENOMEM);
         data->au_headers_allocated = data->nb_au_headers;
diff --git a/libavformat/rtpdec_qt.c b/libavformat/rtpdec_qt.c
index af00a7e..0ef998b 100644
--- a/libavformat/rtpdec_qt.c
+++ b/libavformat/rtpdec_qt.c
@@ -229,7 +229,7 @@ static int qt_rtp_parse_packet(AVFormatContext *s, PayloadContext *qt,
         pkt->stream_index = st->index;
         if (qt->remaining > 0) {
             av_freep(&qt->pkt->data);
-            qt->pkt->data = av_realloc(NULL, qt->remaining * qt->bytes_per_frame);
+            qt->pkt->data = av_realloc_array(NULL, qt->remaining, qt->bytes_per_frame);
             if (!qt->pkt->data) {
                 av_packet_unref(pkt);
                 return AVERROR(ENOMEM);
diff --git a/libavformat/rtpdec_rfc4175.c b/libavformat/rtpdec_rfc4175.c
index b49fc55..5d62d71 100644
--- a/libavformat/rtpdec_rfc4175.c
+++ b/libavformat/rtpdec_rfc4175.c
@@ -107,7 +107,10 @@ static int rfc4175_parse_format(AVStream *stream, PayloadContext *data)
     stream->codecpar->format = pixfmt;
     stream->codecpar->codec_tag = tag;
     stream->codecpar->bits_per_coded_sample = av_get_bits_per_pixel(desc);
-    data->frame_size = data->width * data->height * data->pgroup / data->xinc;
+    uint64_t frame_size = (uint64_t)data->width * data->height * data->pgroup / data->xinc;
+    if (frame_size == 0 || frame_size > UINT_MAX)
+        return AVERROR_INVALIDDATA;
+    data->frame_size = frame_size;
 
     if (data->interlaced)
         stream->codecpar->field_order = AV_FIELD_TT;
@@ -311,8 +314,8 @@ static int rfc4175_handle_packet(AVFormatContext *ctx, PayloadContext *data,
             return AVERROR_INVALIDDATA;
 
         /* prevent ill-formed packets to write after buffer's end */
-        copy_offset = (line * data->width + offset) * data->pgroup / data->xinc;
-        if (copy_offset + length > data->frame_size || !data->frame)
+        copy_offset = ((uint64_t)line * data->width + offset) * data->pgroup / data->xinc;
+        if (copy_offset >= data->frame_size || copy_offset + length > data->frame_size || !data->frame)
             return AVERROR_INVALIDDATA;
 
         dest = data->frame + copy_offset;
-- 
2.53.0.windows.1

From eec3ddebe6e65dce141940bbd0a0f4b0b6a3ac34 Mon Sep 17 00:00:00 2001
From: Hardik-369 <[email protected]>
Date: Sun, 5 Jul 2026 10:39:33 +0530
Subject: [PATCH 2/3] avformat/ogg: add integer overflow checks in memory
 allocation

Use av_size_mult() for the flexible array member calculation in
ogg_save() to prevent overflow when nstreams is large. In buf_realloc(),
check that doubling bufsize does not overflow before calling av_realloc.

Signed-off-by: Siddharth <[email protected]>
---
 libavformat/oggdec.c | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/libavformat/oggdec.c b/libavformat/oggdec.c
index 48279f2..1ad6cf9 100644
--- a/libavformat/oggdec.c
+++ b/libavformat/oggdec.c
@@ -83,11 +83,17 @@ static void free_stream(AVFormatContext *s, int i)
 static int ogg_save(AVFormatContext *s)
 {
     struct ogg *ogg = s->priv_data;
-    struct ogg_state *ost =
-        av_malloc(sizeof(*ost) + (ogg->nstreams - 1) * sizeof(*ogg->streams));
+    size_t streams_size;
+    struct ogg_state *ost;
     int i;
     int ret = 0;
 
+    if (ogg->nstreams > 0 &&
+        av_size_mult(ogg->nstreams - 1, sizeof(*ogg->streams), &streams_size) < 0)
+        return AVERROR(ENOMEM);
+    if (streams_size > SIZE_MAX - sizeof(*ost))
+        return AVERROR(ENOMEM);
+    ost = av_malloc(sizeof(*ost) + streams_size);
     if (!ost)
         return AVERROR(ENOMEM);
 
@@ -95,7 +101,7 @@ static int ogg_save(AVFormatContext *s)
     ost->curidx   = ogg->curidx;
     ost->next     = ogg->state;
     ost->nstreams = ogg->nstreams;
-    memcpy(ost->streams, ogg->streams, ogg->nstreams * sizeof(*ogg->streams));
+    memcpy(ost->streams, ogg->streams, streams_size + sizeof(*ogg->streams));
 
     for (i = 0; i < ogg->nstreams; i++) {
         struct ogg_stream *os = ogg->streams + i;
@@ -304,11 +310,18 @@ static int buf_realloc(struct ogg_stream *os, int size)
 {
     /* Even if invalid guarantee there's enough memory to read the page */
     if (os->bufsize - os->bufpos < size) {
-        uint8_t *nb = av_realloc(os->buf, 2*os->bufsize + AV_INPUT_BUFFER_PADDING_SIZE);
+        size_t new_size;
+
+        if (os->bufsize > (SIZE_MAX - AV_INPUT_BUFFER_PADDING_SIZE) / 2)
+            return AVERROR(ENOMEM);
+        new_size = 2 * (size_t)os->bufsize + AV_INPUT_BUFFER_PADDING_SIZE;
+        if (new_size > INT_MAX)
+            return AVERROR(ENOMEM);
+        uint8_t *nb = av_realloc(os->buf, new_size);
         if (!nb)
             return AVERROR(ENOMEM);
         os->buf = nb;
-        os->bufsize *= 2;
+        os->bufsize = new_size;
     }
 
     return 0;
-- 
2.53.0.windows.1

From 6d27f78166b5a46466155389f822efe27383d777 Mon Sep 17 00:00:00 2001
From: Hardik-369 <[email protected]>
Date: Sun, 5 Jul 2026 10:39:35 +0530
Subject: [PATCH 3/3] avformat/asf: replace debug-only assert with runtime
 check

The av_assert0() macro is only active in debug builds. In optimized
builds, the unchecked multiplication 2 * len could overflow, producing
an undersized heap allocation. Replace with a runtime check.

Signed-off-by: Siddharth <[email protected]>
---
 libavformat/asfdec_f.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libavformat/asfdec_f.c b/libavformat/asfdec_f.c
index a1a0428..f1a05d5 100644
--- a/libavformat/asfdec_f.c
+++ b/libavformat/asfdec_f.c
@@ -225,7 +225,8 @@ static void get_tag(AVFormatContext *s, const char *key, int type, int len)
     int64_t off = avio_tell(s->pb);
 #define LEN 22
 
-    av_assert0((unsigned)len < (INT_MAX - LEN) / 2);
+    if ((unsigned)len >= (INT_MAX - LEN) / 2)
+        goto finish;
 
     if (!asf->export_xmp && !strncmp(key, "xmp", 3))
         goto finish;
-- 
2.53.0.windows.1