[PR] avcodec/dovi: multiple fixes (PR #23911)

PR #23911 opened by michaelni
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23911
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23911.patch


>From 3548661ea4af200104e7ff0f85e8013e493a622a Mon Sep 17 00:00:00 2001
From: Kenan Alghythee <[email protected]>
Date: Sat, 11 Jul 2026 16:57:38 +0200
Subject: [PATCH 1/5] avcodec/bsf/dovi_rpu: handle update_rpu() returning no
 RPU

Fixes: out of array access
Fixes: SUcVEyk7r3Gc
Found-by: Kenan Alghythee <[email protected]>
---
 libavcodec/bsf/dovi_rpu.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libavcodec/bsf/dovi_rpu.c b/libavcodec/bsf/dovi_rpu.c
index 88da4417b5..e8aa3eadcd 100644
--- a/libavcodec/bsf/dovi_rpu.c
+++ b/libavcodec/bsf/dovi_rpu.c
@@ -99,6 +99,8 @@ static int dovi_rpu_update_fragment_hevc(AVBSFContext *bsf, AVPacket *pkt,
     ret = update_rpu(bsf, pkt, 0, nal->data + 2, nal->data_size - 2, &rpu, &rpu_size);
     if (ret < 0)
         return ret;
+    if (!rpu || rpu_size <= 0)
+        return 0;
 
     /* NAL unit header + NAL prefix */
     if (rpu_size + 3 <= nal->data_size && av_buffer_is_writable(nal->data_ref)) {
@@ -158,6 +160,10 @@ static int dovi_rpu_update_fragment_av1(AVBSFContext *bsf, AVPacket *pkt,
                          &rpu, &rpu_size);
         if (ret < 0)
             return ret;
+        if (!rpu || rpu_size <= 1) {
+            av_free(rpu);
+            continue;
+        }
 
         ref = av_buffer_create(rpu, rpu_size, av_buffer_default_free, NULL, 0);
         if (!ref) {
-- 
2.52.0


>From 314a840cd0960809f02a85c4ded1f0f5c7f41347 Mon Sep 17 00:00:00 2001
From: Kenan Alghythee <[email protected]>
Date: Sat, 11 Jul 2026 16:57:38 +0200
Subject: [PATCH 2/5] avcodec/dovi_rpuenc: validate vdr_rpu_id from the input
 metadata

Fixes: out of array access
Fixes: SUcVEyk7r3Gc
Found-by: Kenan Alghythee <[email protected]>
---
 libavcodec/dovi_rpuenc.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libavcodec/dovi_rpuenc.c b/libavcodec/dovi_rpuenc.c
index dc63c68405..9d3db33071 100644
--- a/libavcodec/dovi_rpuenc.c
+++ b/libavcodec/dovi_rpuenc.c
@@ -599,6 +599,10 @@ int ff_dovi_rpu_generate(DOVIContext *s, const AVDOVIMetadata *metadata,
         dv_md_compression = AV_DOVI_COMPRESSION_NONE;
 
     vdr_rpu_id = mapping->vdr_rpu_id;
+    if (vdr_rpu_id < 0 || vdr_rpu_id > DOVI_MAX_DM_ID) {
+        av_log(s->logctx, AV_LOG_ERROR, "Invalid VDR RPU id %d\n", vdr_rpu_id);
+        return AVERROR_INVALIDDATA;
+    }
     use_prev_vdr_rpu = 0;
 
     if (!s->vdr[vdr_rpu_id]) {
-- 
2.52.0


>From f8b42ad77709cae2ccb318283e4ba9ee0258cee3 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <[email protected]>
Date: Sat, 11 Jul 2026 17:01:30 +0200
Subject: [PATCH 3/5] avcodec/dovi_rpudec: bound num_x/y_partitions

Fixes: out of array access
Fixes: SUcVEyk7r3Gc
Found-by: Kenan Alghythee <[email protected]>
---
 libavcodec/dovi_rpudec.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libavcodec/dovi_rpudec.c b/libavcodec/dovi_rpudec.c
index d210ba52cf..ba4e7cd214 100644
--- a/libavcodec/dovi_rpudec.c
+++ b/libavcodec/dovi_rpudec.c
@@ -582,6 +582,8 @@ int ff_dovi_rpu_parse(DOVIContext *s, const uint8_t *rpu, size_t rpu_size,
 
         mapping->num_x_partitions = get_ue_golomb_long(gb) + 1;
         mapping->num_y_partitions = get_ue_golomb_long(gb) + 1;
+        VALIDATE(mapping->num_x_partitions, 1, 0xFFFF);
+        VALIDATE(mapping->num_y_partitions, 1, 0xFFFF);
         /* End of rpu_data_header(), start of vdr_rpu_data_payload() */
 
         for (int c = 0; c < 3; c++) {
-- 
2.52.0


>From 37faa69b6ffd13dc9e59a4c26efe9f78faf94675 Mon Sep 17 00:00:00 2001
From: Kenan Alghythee <[email protected]>
Date: Wed, 22 Jul 2026 23:48:29 +0200
Subject: [PATCH 4/5] avcodec/dovi_rpuenc: validate the data mapping before
 generation

Fixes: out of array access
Fixes: SUcVEyk7r3Gc
Found-by: Kenan Alghythee <[email protected]>
---
 libavcodec/dovi_rpuenc.c | 86 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/libavcodec/dovi_rpuenc.c b/libavcodec/dovi_rpuenc.c
index 9d3db33071..8b7a74f313 100644
--- a/libavcodec/dovi_rpuenc.c
+++ b/libavcodec/dovi_rpuenc.c
@@ -367,6 +367,87 @@ static inline void put_se_coef(PutBitContext *pb, const AVDOVIRpuDataHeader *hdr
     }
 }
 
+static int validate_ue_golomb_value(uint64_t value)
+{
+    return value <= 0xFFFE;
+}
+
+static int validate_se_golomb_value(int64_t value)
+{
+    return value >= -0x7FFF && value <= 0x7FFF;
+}
+
+static int validate_ue_coef(const AVDOVIRpuDataHeader *hdr, uint64_t coef)
+{
+    if (hdr->coef_log2_denom >= 63)
+        return 0;
+    return validate_ue_golomb_value(coef >> hdr->coef_log2_denom);
+}
+
+static int validate_se_coef(const AVDOVIRpuDataHeader *hdr, int64_t coef)
+{
+    if (hdr->coef_log2_denom >= 63)
+        return 0;
+    return validate_se_golomb_value(coef >> hdr->coef_log2_denom);
+}
+
+static int validate_mapping_for_generation(const AVDOVIRpuDataHeader *hdr,
+                                           const AVDOVIDataMapping *mapping)
+{
+    if (!mapping->num_x_partitions || mapping->num_x_partitions > 0xFFFF ||
+        !mapping->num_y_partitions || mapping->num_y_partitions > 0xFFFF)
+        return 0;
+
+    for (int c = 0; c < 3; c++) {
+        const AVDOVIReshapingCurve *curve = &mapping->curves[c];
+
+        if (curve->num_pivots < 2 || curve->num_pivots > AV_DOVI_MAX_PIECES + 1)
+            return 0;
+
+        for (int i = 1; i < curve->num_pivots; i++)
+            if (curve->pivots[i] < curve->pivots[i - 1])
+                return 0;
+
+        for (int i = 0; i < curve->num_pivots - 1; i++) {
+            switch (curve->mapping_idc[i]) {
+            case AV_DOVI_MAPPING_POLYNOMIAL:
+                if (curve->poly_order[i] < 1 || curve->poly_order[i] > 2)
+                    return 0;
+                for (int k = 0; k <= curve->poly_order[i]; k++)
+                    if (!validate_se_coef(hdr, curve->poly_coef[i][k]))
+                        return 0;
+                break;
+            case AV_DOVI_MAPPING_MMR:
+                if (curve->mmr_order[i] < 1 || curve->mmr_order[i] > 3)
+                    return 0;
+                if (!validate_se_coef(hdr, curve->mmr_constant[i]))
+                    return 0;
+                for (int j = 0; j < curve->mmr_order[i]; j++)
+                    for (int k = 0; k < 7; k++)
+                        if (!validate_se_coef(hdr, curve->mmr_coef[i][j][k]))
+                            return 0;
+                break;
+            default:
+                return 0;
+            }
+        }
+    }
+
+    if (mapping->nlq_method_idc != AV_DOVI_NLQ_NONE) {
+        if (mapping->nlq_method_idc != AV_DOVI_NLQ_LINEAR_DZ)
+            return 0;
+        for (int c = 0; c < 3; c++) {
+            const AVDOVINLQParams *nlq = &mapping->nlq[c];
+            if (!validate_ue_coef(hdr, nlq->vdr_in_max) ||
+                !validate_ue_coef(hdr, nlq->linear_deadzone_slope) ||
+                !validate_ue_coef(hdr, nlq->linear_deadzone_threshold))
+                return 0;
+        }
+    }
+
+    return 1;
+}
+
 static int av_q2den(AVRational q, int den)
 {
     if (!q.den || q.den == den)
@@ -595,6 +676,11 @@ int ff_dovi_rpu_generate(DOVIContext *s, const AVDOVIMetadata *metadata,
         return AVERROR_INVALIDDATA;
     }
 
+    if (!validate_mapping_for_generation(hdr, mapping)) {
+        av_log(s->logctx, AV_LOG_ERROR, "Coefficient out of range for RPU\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     if (!(flags & FF_DOVI_COMPRESS_RPU))
         dv_md_compression = AV_DOVI_COMPRESSION_NONE;
 
-- 
2.52.0


>From 4e4a06a98715d853b7af180bb45cc9ae2bad7d4b Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <[email protected]>
Date: Wed, 22 Jul 2026 22:46:55 +0200
Subject: [PATCH 5/5] avcodec/dovi_rpuenc: normalize vdr_dm_metadata_present to
 0/1

---
 libavcodec/dovi_rpuenc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libavcodec/dovi_rpuenc.c b/libavcodec/dovi_rpuenc.c
index 8b7a74f313..d0abcc0d9e 100644
--- a/libavcodec/dovi_rpuenc.c
+++ b/libavcodec/dovi_rpuenc.c
@@ -726,7 +726,7 @@ int ff_dovi_rpu_generate(DOVIContext *s, const AVDOVIMetadata *metadata,
             return AVERROR(ENOMEM);
     }
 
-    vdr_dm_metadata_present = memcmp(color, &ff_dovi_color_default, sizeof(*color));
+    vdr_dm_metadata_present = !!memcmp(color, &ff_dovi_color_default, sizeof(*color));
     if (metadata->num_ext_blocks)
         vdr_dm_metadata_present = 1;
 
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]