[PR] avfilter/swaprect: validate expression results before integer conversion (PR #24023)

PR #24023 opened by Qingzheng Li (iSoldLeo)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24023
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24023.patch

## Summary

- validate all six evaluated `swaprect` size and position expressions before converting them from `double` to `int`
- preserve the filter's existing truncation-toward-zero semantics, then reject converted non-positive dimensions
- free the owned input frame on expression-evaluation and validation failures
- add FATE coverage for negative, sub-unit, finite out-of-range, and infinite values

## Background

Supersedes #23088.

The earlier patch by @un-tag added the initial non-positive-size check and
regression test for #22866. This version is based on current master and also
addresses the blocking review concerning all six unchecked
`double`-to-`int` conversions.

`av_expr_parse_and_eval()` rejects a final NaN result, but expressions such
as `1/0` can still produce infinity and return successfully. Large finite
values are also possible. Converting either class to `int` is undefined, and
the same problem affects all four coordinates before they are clipped.

The conversion helper first rejects non-finite values, applies `trunc()`,
and then checks the truncated result against the `int` range. Applying
`trunc()` first preserves the existing conversion semantics for valid
fractional expressions.

The positivity check intentionally applies after conversion. Values such as
`w=0.5` therefore truncate to zero and are rejected as non-positive
rectangle dimensions. Negative coordinates remain valid and continue to be
clipped by the existing `av_clip()` calls.

All expression-evaluation and validation failures use a common cleanup path
because `filter_frame()` owns the input frame until it forwards that frame
with `ff_filter_frame()`.

## Tests

The new FATE cases require both the expected diagnostic and the exit status
corresponding to `AVERROR(EINVAL)`, so a signal termination cannot satisfy
the tests:

- `w=-1`: original negative-width reproducer
- `w=0.5`: positive expression that truncates to a zero width
- `x1=1e100`: finite coordinate outside the `int` range
- `x1=1/0`: reachable non-finite coordinate

Validated on macOS arm64:

- `fate-filter-swaprect`
- `fate-filter-swaprect-invalid-size`
- `fate-filter-swaprect-invalid-fractional-size`
- `fate-filter-swaprect-invalid-coordinate`
- `fate-filter-swaprect-nonfinite-coordinate`

All tests pass. A targeted build and `git diff --check` also pass.


>From 1252ac789d71181666f64007d256bbde5dc26dcb Mon Sep 17 00:00:00 2001
From: Qingzheng Li <[email protected]>
Date: Thu, 6 Aug 2026 01:22:57 +0800
Subject: [PATCH 1/2] avfilter/swaprect: validate rectangle expression results

swaprect evaluates its dimensions and coordinates as doubles before
converting them to int. Non-finite or out-of-range results make that
conversion undefined, and negative dimensions can later become invalid
copy sizes.

Truncate each of the six results before checking the int range, preserving
the filter's existing conversion semantics. Reject non-finite and
unrepresentable results, then validate the converted dimensions so values
such as 0.5 cannot become zero-sized rectangles.

Route expression and validation failures through a common cleanup path,
since filter_frame() owns the input frame until it is forwarded.

Fixes: #22866
Reported-by: cxxz16
Based-on-patch-by: un-tag <[email protected]>
Signed-off-by: Qingzheng Li <[email protected]>
---
 libavfilter/vf_swaprect.c | 48 +++++++++++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 7 deletions(-)

diff --git a/libavfilter/vf_swaprect.c b/libavfilter/vf_swaprect.c
index fe007ee5e7..0c8b8cd846 100644
--- a/libavfilter/vf_swaprect.c
+++ b/libavfilter/vf_swaprect.c
@@ -21,6 +21,7 @@
 #include "libavutil/avassert.h"
 #include "libavutil/eval.h"
 #include "libavutil/imgutils.h"
+#include "libavutil/libm.h"
 #include "libavutil/mem.h"
 #include "libavutil/opt.h"
 
@@ -71,6 +72,19 @@ static int query_formats(const AVFilterContext *ctx,
 static const char *const var_names[] = {   "w",   "h",   "a",   "n",   "t",   "sar",   "dar",        NULL };
 enum                                   { VAR_W, VAR_H, VAR_A, VAR_N, VAR_T, VAR_SAR, VAR_DAR, VAR_VARS_NB };
 
+static int trunc_double_to_int(int *dst, double value)
+{
+    if (!isfinite(value))
+        return AVERROR(EINVAL);
+
+    value = trunc(value);
+    if (value < INT_MIN || value > INT_MAX)
+        return AVERROR(EINVAL);
+
+    *dst = value;
+    return 0;
+}
+
 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
 {
     FilterLink *inl = ff_filter_link(inlink);
@@ -101,44 +115,60 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
                                  NULL, NULL, NULL, NULL,
                                  0, 0, ctx);
     if (ret < 0)
-        return ret;
+        goto fail;
 
     ret = av_expr_parse_and_eval(&dh, s->h,
                                  var_names, &var_values[0],
                                  NULL, NULL, NULL, NULL,
                                  0, 0, ctx);
     if (ret < 0)
-        return ret;
+        goto fail;
 
     ret = av_expr_parse_and_eval(&dx1, s->x1,
                                  var_names, &var_values[0],
                                  NULL, NULL, NULL, NULL,
                                  0, 0, ctx);
     if (ret < 0)
-        return ret;
+        goto fail;
 
     ret = av_expr_parse_and_eval(&dy1, s->y1,
                                  var_names, &var_values[0],
                                  NULL, NULL, NULL, NULL,
                                  0, 0, ctx);
     if (ret < 0)
-        return ret;
+        goto fail;
 
     ret = av_expr_parse_and_eval(&dx2, s->x2,
                                  var_names, &var_values[0],
                                  NULL, NULL, NULL, NULL,
                                  0, 0, ctx);
     if (ret < 0)
-        return ret;
+        goto fail;
 
     ret = av_expr_parse_and_eval(&dy2, s->y2,
                                  var_names, &var_values[0],
                                  NULL, NULL, NULL, NULL,
                                  0, 0, ctx);
     if (ret < 0)
-        return ret;
+        goto fail;
 
-    w = dw; h = dh; x1[0] = dx1; y1[0] = dy1; x2[0] = dx2; y2[0] = dy2;
+    if (trunc_double_to_int(&w, dw)       < 0 ||
+        trunc_double_to_int(&h, dh)       < 0 ||
+        trunc_double_to_int(&x1[0], dx1)  < 0 ||
+        trunc_double_to_int(&y1[0], dy1)  < 0 ||
+        trunc_double_to_int(&x2[0], dx2)  < 0 ||
+        trunc_double_to_int(&y2[0], dy2)  < 0) {
+        av_log(ctx, AV_LOG_ERROR,
+               "Rectangle expression result is not representable as an integer.\n");
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
+
+    if (w <= 0 || h <= 0) {
+        av_log(ctx, AV_LOG_ERROR, "Rectangle dimensions must be positive.\n");
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
 
     x1[0] = av_clip(x1[0], 0, inlink->w - 1);
     y1[0] = av_clip(y1[0], 0, inlink->h - 1);
@@ -194,6 +224,10 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
     }
 
     return ff_filter_frame(outlink, in);
+
+fail:
+    av_frame_free(&in);
+    return ret;
 }
 
 static int config_input(AVFilterLink *inlink)
-- 
2.52.0


>From 794fc1dd0e5117728656b3a01f7b63aa75d5101e Mon Sep 17 00:00:00 2001
From: Qingzheng Li <[email protected]>
Date: Thu, 6 Aug 2026 01:23:33 +0800
Subject: [PATCH 2/2] fate/filter-video: test invalid swaprect expressions

Cover negative and sub-unit dimensions, as well as finite out-of-range
and infinite coordinates.

Require the expected EINVAL process status and diagnostic so a crash does
not satisfy the rejection tests.

Based-on-patch-by: un-tag <[email protected]>
Signed-off-by: Qingzheng Li <[email protected]>
---
 tests/fate/filter-video.mak | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/fate/filter-video.mak b/tests/fate/filter-video.mak
index 0eb2e7076c..18afdf45a1 100644
--- a/tests/fate/filter-video.mak
+++ b/tests/fate/filter-video.mak
@@ -376,6 +376,20 @@ fate-filter-swaprect: CMD = framecrc -c:v pgmyuv -i $(SRC) -vf swaprect
 
 FATE_FILTER_VSYNTH_PGMYUV-$(CONFIG_SWAPRECT_FILTER) += $(FATE_SWAPRECT)
 
+FATE_FILTER-$(call ALLYES, COLOR_FILTER LAVFI_INDEV SWAPRECT_FILTER WRAPPED_AVFRAME_ENCODER NULL_MUXER) += fate-filter-swaprect-invalid-size fate-filter-swaprect-invalid-fractional-size fate-filter-swaprect-invalid-coordinate fate-filter-swaprect-nonfinite-coordinate
+fate-filter-swaprect-invalid-size: CMD = run $(FFMPEG) -nostdin -hide_banner -f lavfi -i "color=c=black:s=16x16:r=1" -vf "swaprect=w=-1:h=1:x1=0:y1=0:x2=1:y2=0" -frames:v 1 -f null none ; ret=$$?; test $$ret -eq 234
+fate-filter-swaprect-invalid-size: CMP = grep
+fate-filter-swaprect-invalid-size: REF = Rectangle dimensions must be positive
+fate-filter-swaprect-invalid-fractional-size: CMD = run $(FFMPEG) -nostdin -hide_banner -f lavfi -i "color=c=black:s=16x16:r=1" -vf "swaprect=w=0.5:h=1:x1=0:y1=0:x2=1:y2=0" -frames:v 1 -f null none ; ret=$$?; test $$ret -eq 234
+fate-filter-swaprect-invalid-fractional-size: CMP = grep
+fate-filter-swaprect-invalid-fractional-size: REF = Rectangle dimensions must be positive
+fate-filter-swaprect-invalid-coordinate: CMD = run $(FFMPEG) -nostdin -hide_banner -f lavfi -i "color=c=black:s=16x16:r=1" -vf "swaprect=w=1:h=1:x1=1e100:y1=0:x2=1:y2=0" -frames:v 1 -f null none ; ret=$$?; test $$ret -eq 234
+fate-filter-swaprect-invalid-coordinate: CMP = grep
+fate-filter-swaprect-invalid-coordinate: REF = Rectangle expression result is not representable as an integer
+fate-filter-swaprect-nonfinite-coordinate: CMD = run $(FFMPEG) -nostdin -hide_banner -f lavfi -i "color=c=black:s=16x16:r=1" -vf "swaprect=w=1:h=1:x1=1/0:y1=0:x2=1:y2=0" -frames:v 1 -f null none ; ret=$$?; test $$ret -eq 234
+fate-filter-swaprect-nonfinite-coordinate: CMP = grep
+fate-filter-swaprect-nonfinite-coordinate: REF = Rectangle expression result is not representable as an integer
+
 FATE_FILTER_VSYNTH_PGMYUV-$(CONFIG_TBLEND_FILTER) += fate-filter-tblend
 fate-filter-tblend: CMD = framecrc -c:v pgmyuv -i $(SRC) -vf tblend=all_mode=difference128
 
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]