[PATCH] selftests/prctl: Fix non-anonymous VMA mapping in set-anon-vma-name test

Hongfu Li <[email protected]> Mon, 3 Aug 2026 18:30:46 +0800
Newsgroups org.kernel.vger.linux-kselftest,org.kernel.vger.linux-kernel
Message-ID <[email protected]>
From: Hongfu Li <[email protected]>

The test creates a non-anonymous VMA (ptr_not_anon) via mmap() with
MAP_PRIVATE but without MAP_ANONYMOUS, using fd=0 (stdin) as the file
descriptor.  This always fails because fd=0 is not a regular file,
and the failure was hidden because ASSERT_NE() incorrectly checked
for NULL instead of MAP_FAILED.

Fix by using mkstemp() + ftruncate() to create a real temporary file,
then mapping it with MAP_PRIVATE to obtain a genuine file-backed VMA.
Also fix the mmap() error checks to use MAP_FAILED instead of NULL,
and pass fd=-1 for the anonymous mapping for clarity.  The temp file
is unlinked immediately so it does not persist on disk.

Signed-off-by: Hongfu Li <[email protected]>
---
 .../selftests/prctl/set-anon-vma-name-test.c  | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/tools/testing/selftests/prctl/set-anon-vma-name-test.c b/tools/testing/selftests/prctl/set-anon-vma-name-test.c
index ac6721b184a6..5f9589534da8 100644
--- a/tools/testing/selftests/prctl/set-anon-vma-name-test.c
+++ b/tools/testing/selftests/prctl/set-anon-vma-name-test.c
@@ -4,11 +4,13 @@
  */
 
 #include <errno.h>
+#include <fcntl.h>
 #include <sys/prctl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <sys/mman.h>
 #include <string.h>
+#include <unistd.h>
 
 #include "kselftest_harness.h"
 
@@ -73,15 +75,24 @@ int was_renaming_successful(char *target_name, unsigned long ptr)
 
 FIXTURE(vma) {
 	void *ptr_anon, *ptr_not_anon;
+	int fd_not_anon;
 };
 
 FIXTURE_SETUP(vma) {
+	char template[] = "./set-anon-vma-test-XXXXXX";
+
 	self->ptr_anon = mmap(NULL, AREA_SIZE, PROT_READ | PROT_WRITE,
-					MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
-	ASSERT_NE(self->ptr_anon, NULL);
+					MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	ASSERT_NE(self->ptr_anon, MAP_FAILED);
+
+	self->fd_not_anon = mkstemp(template);
+	ASSERT_NE(self->fd_not_anon, -1);
+	unlink(template);
+	ASSERT_EQ(ftruncate(self->fd_not_anon, AREA_SIZE), 0);
 	self->ptr_not_anon = mmap(NULL, AREA_SIZE, PROT_READ | PROT_WRITE,
-					MAP_PRIVATE, 0, 0);
-	ASSERT_NE(self->ptr_not_anon, NULL);
+					MAP_PRIVATE, self->fd_not_anon, 0);
+	ASSERT_NE(self->ptr_not_anon, MAP_FAILED);
+	close(self->fd_not_anon);
 }
 
 FIXTURE_TEARDOWN(vma) {
@@ -98,7 +109,7 @@ TEST_F(vma, renaming) {
 	EXPECT_EQ(rename_vma((unsigned long)self->ptr_anon, AREA_SIZE, BAD_NAME), -EINVAL);
 
 	TH_LOG("Try to rename non-anonymous VMA");
-	EXPECT_EQ(rename_vma((unsigned long) self->ptr_not_anon, AREA_SIZE, GOOD_NAME), -EINVAL);
+	EXPECT_EQ(rename_vma((unsigned long) self->ptr_not_anon, AREA_SIZE, GOOD_NAME), -EBADF);
 }
 
 TEST_HARNESS_MAIN
-- 
2.54.0