[PATCH] idct_sse2_skal: fix overread in TEST_ROW on x86_64

Peter Ross <[email protected]> Fri, 20 Dec 2024 20:35:27 +1100
Newsgroups gmane.comp.video.xvid.devel
Message-ID <f76efb9ac6d647e89a40f0dde9751ce72701476b.1734687219.git.pross@xvid.org>
on x86_64 the _EAX/_EDX macros expand to rax/rdx registers and are 8 bytes
wide. it is therefore sufficient to test only the %1 and %1+8 pixel offsets in
TEST_ROW macro.

testing the offsets %1+4 and %1+12 is unnecessary on x86_64. the later case also
causes an overread when processing the final pixel block (chroma v block). this
overread is detected by valgrind.
---
 xvidcore/src/dct/x86_asm/fdct_sse2_skal.asm | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/xvidcore/src/dct/x86_asm/fdct_sse2_skal.asm b/xvidcore/src/dct/x86_asm/fdct_sse2_skal.asm
index 4816ea6..a172e14 100644
--- a/xvidcore/src/dct/x86_asm/fdct_sse2_skal.asm
+++ b/xvidcore/src/dct/x86_asm/fdct_sse2_skal.asm
@@ -365,10 +365,15 @@ cglobal fdct_sse2_skal
 ;-----------------------------------------------------------------------------
 
 %macro TEST_ROW 2     ; %1:src,  %2:label x8
+%ifdef ARCH_IS_X86_64
+  mov _EAX, [%1   ]
+  mov _EDX, [%1+ 8]
+%else
   mov _EAX, [%1   ]
   mov _EDX, [%1+ 8]
   or  _EAX, [%1+ 4]
   or  _EDX, [%1+12]
+%endif
   or  _EAX, _EDX
   jz near %2
 %endmacro
-- 
2.45.2

-- Peter
(A907 E02F A6E5 0CD2 34CD 20D2 6760 79C5 AC40 DD6B)