[PR] swscale: avoid overflow in fast bilinear edge handling (PR #24000)

PR #24000 opened by iSoldLeo
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24000
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24000.patch

The final edge clamp computes the source position using int
multiplication before shifting. With sufficiently wide inputs this
overflows, which may suppress the clamp and leave the last output
pixels interpolated with the padding byte.

Promote the multiplication to int64_t in the C, MMXEXT and VSX
implementations.

Add a FATE regression test covering the rightmost pixel of a wide
upscale. It fails before this change on both the C and MMXEXT paths.

Fixes: signed integer overflow: 15 * 255918080 cannot be represented in type 'int'
Fixes: #21591

Tested on an Apple M4 Mac:

- make fate-filter-scale-fast-bilinear-wide-edge
- make fate-checkasm-sw_scale
- UBSan reproduction on the C path
- x86_64 MMXEXT path under Rosetta


>From e40455a16435c8578c0f36ad2334cb26176eea75 Mon Sep 17 00:00:00 2001
From: iSold Leo <[email protected]>
Date: Tue, 4 Aug 2026 14:31:44 +0800
Subject: [PATCH] swscale: avoid overflow in fast bilinear edge handling

The final edge clamp computes the source position using int
multiplication before shifting. With sufficiently wide inputs this
overflows, which may suppress the clamp and leave the last output
pixels interpolated with the padding byte.

Promote the multiplication to int64_t in the C, MMXEXT and VSX
implementations.

Add a regression test covering the rightmost pixel of a wide upscale,
which is wrong before this change on both the C and the MMXEXT path.

Fixes: signed integer overflow: 15 * 255918080 cannot be represented in type 'int'
Fixes: #21591
---
 libswscale/hscale_fast_bilinear.c                   | 4 ++--
 libswscale/ppc/swscale_vsx.c                        | 4 ++--
 libswscale/x86/hscale_fast_bilinear_simd.c          | 4 ++--
 tests/fate/filter-video.mak                         | 3 +++
 tests/ref/fate/filter-scale-fast-bilinear-wide-edge | 6 ++++++
 5 files changed, 15 insertions(+), 6 deletions(-)
 create mode 100644 tests/ref/fate/filter-scale-fast-bilinear-wide-edge

diff --git a/libswscale/hscale_fast_bilinear.c b/libswscale/hscale_fast_bilinear.c
index abcfb95e2c..331a3c95e0 100644
--- a/libswscale/hscale_fast_bilinear.c
+++ b/libswscale/hscale_fast_bilinear.c
@@ -31,7 +31,7 @@ void ff_hyscale_fast_c(SwsInternal *c, int16_t *dst, int dstWidth,
         dst[i] = (src[xx] << 7) + (src[xx + 1] - src[xx]) * xalpha;
         xpos  += xInc;
     }
-    for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
+    for (i=dstWidth-1; (i*(int64_t)xInc)>>16 >=srcW-1; i--)
         dst[i] = src[srcW-1]*128;
 }
 
@@ -48,7 +48,7 @@ void ff_hcscale_fast_c(SwsInternal *c, int16_t *dst1, int16_t *dst2,
         dst2[i] = (src2[xx] * (xalpha ^ 127) + src2[xx + 1] * xalpha);
         xpos   += xInc;
     }
-    for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
+    for (i=dstWidth-1; (i*(int64_t)xInc)>>16 >=srcW-1; i--) {
         dst1[i] = src1[srcW-1]*128;
         dst2[i] = src2[srcW-1]*128;
     }
diff --git a/libswscale/ppc/swscale_vsx.c b/libswscale/ppc/swscale_vsx.c
index 21fb43092d..8bbe6cde65 100644
--- a/libswscale/ppc/swscale_vsx.c
+++ b/libswscale/ppc/swscale_vsx.c
@@ -1754,7 +1754,7 @@ static void hyscale_fast_vsx(SwsInternal *c, int16_t *dst, int dstWidth,
 
         xpos += xInc * 16;
     }
-    for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
+    for (i=dstWidth-1; (i*(int64_t)xInc)>>16 >=srcW-1; i--)
         dst[i] = src[srcW-1]*128;
 }
 
@@ -1850,7 +1850,7 @@ static void hcscale_fast_vsx(SwsInternal *c, int16_t *dst1, int16_t *dst2,
 
         xpos += xInc * 16;
     }
-    for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
+    for (i=dstWidth-1; (i*(int64_t)xInc)>>16 >=srcW-1; i--) {
         dst1[i] = src1[srcW-1]*128;
         dst2[i] = src2[srcW-1]*128;
     }
diff --git a/libswscale/x86/hscale_fast_bilinear_simd.c b/libswscale/x86/hscale_fast_bilinear_simd.c
index d8a4e444b4..3f3c799a33 100644
--- a/libswscale/x86/hscale_fast_bilinear_simd.c
+++ b/libswscale/x86/hscale_fast_bilinear_simd.c
@@ -275,7 +275,7 @@ void ff_hyscale_fast_mmxext(SwsInternal *c, int16_t *dst,
 #endif
     );
 
-    for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
+    for (i=dstWidth-1; (i*(int64_t)xInc)>>16 >=srcW-1; i--)
         dst[i] = src[srcW-1]*128;
 }
 
@@ -352,7 +352,7 @@ void ff_hcscale_fast_mmxext(SwsInternal *c, int16_t *dst1, int16_t *dst2,
 #endif
     );
 
-    for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
+    for (i=dstWidth-1; (i*(int64_t)xInc)>>16 >=srcW-1; i--) {
         dst1[i] = src1[srcW-1]*128;
         dst2[i] = src2[srcW-1]*128;
     }
diff --git a/tests/fate/filter-video.mak b/tests/fate/filter-video.mak
index 0eb2e7076c..f8c0ffc3cb 100644
--- a/tests/fate/filter-video.mak
+++ b/tests/fate/filter-video.mak
@@ -181,6 +181,9 @@ FATE_FILTER-$(call FILTERFRAMECRC, TESTSRC FORMAT CONCAT SCALE, LAVFI_INDEV FILE
 fate-filter-lavd-scalenorm: tests/data/filtergraphs/scalenorm
 fate-filter-lavd-scalenorm: CMD = framecrc -f lavfi -graph_file $(TARGET_PATH)/tests/data/filtergraphs/scalenorm -i dummy
 
+FATE_FILTER-$(call FILTERFRAMECRC, COLOR FORMAT SCALE CROP) += fate-filter-scale-fast-bilinear-wide-edge
+fate-filter-scale-fast-bilinear-wide-edge: CMD = framecrc -flags bitexact -lavfi color=c=red:s=40000x1:r=1:d=1,format=yuv444p,scale=40032:1:flags=fast_bilinear,crop=1:1:40031:0 -frames:v 1
+
 FATE_FILTER-$(call FILTERFRAMECRC, TESTSRC2 FEEDBACK HFLIP, LAVFI_INDEV) += fate-filter-feedback-hflip
 fate-filter-feedback-hflip: CMD = framecrc -f lavfi -i testsrc2=d=1 -vf "[in][hflipin]feedback=x=0:y=0:w=100:h=100[out][hflipout];[hflipout]hflip[hflipin]"
 
diff --git a/tests/ref/fate/filter-scale-fast-bilinear-wide-edge b/tests/ref/fate/filter-scale-fast-bilinear-wide-edge
new file mode 100644
index 0000000000..8b38823e97
--- /dev/null
+++ b/tests/ref/fate/filter-scale-fast-bilinear-wide-edge
@@ -0,0 +1,6 @@
+#tb 0: 1/1
+#media_type 0: video
+#codec_id 0: rawvideo
+#dimensions 0: 1x1
+#sar 0: 0/1
+0,          0,          0,        1,        3, 0x0297019b
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]