[PATCH 06/16] selftests/mm: add khugepaged race harness

From: "Kiryl Shutsemau (Meta)" <[email protected]>

Collapse serializes against faults, GUP, fork, mremap and zapping
through a protocol of locks, TLB flushes and refcount checks; none of
the khugepaged selftests exercise it under contention.

Add khugepaged_race: two faulters, MADV_DONTNEED, transient FOLL_PIN
(gup_test), fork and mremap threads race one of three collapse
drivers over the same ranges:

  stepped   khugepaged driven one full pass at a time via
            khugepaged_full_pass() — deterministic extent per step;
  free      free-running khugepaged (scan_sleep_millisecs=0) — soak;
  madvise   MADV_COLLAPSE + MADV_DONTNEED loop — the PMD-order axis.

All anon THP orders are enabled (inherit) and max_ptes_none is 0, so
a window collapses only once fully populated; the racing
MADV_DONTNEED then steers selection across orders as pages come
and go. Every racing page must
read as its pattern or zero, never anything else — checked
continuously by the faulters and fork children, and in a final sweep.
The kernel-side assertions (DEBUG_VM, page_table_check, KASAN,
lockdep) are the other half of the oracle; runners should inspect
dmesg.

The racing threads share three PMD-sized areas by default, plus one
owned by the mremap thread; -a overrides the count for
memory-constrained or emulated hosts, where a 512M PMD (arm64/64K)
makes the default playground multi-gigabyte.

Short runs are wired into run_vmtests.sh; soak length is -d.

Assisted-by: Claude-Code:claude-opus-5
Signed-off-by: Kiryl Shutsemau (Meta) <[email protected]>
---
 tools/testing/selftests/mm/.gitignore        |   1 +
 tools/testing/selftests/mm/Makefile          |   1 +
 tools/testing/selftests/mm/khugepaged_race.c | 354 +++++++++++++++++++
 tools/testing/selftests/mm/run_vmtests.sh    |   7 +
 4 files changed, 363 insertions(+)
 create mode 100644 tools/testing/selftests/mm/khugepaged_race.c

diff --git a/tools/testing/selftests/mm/.gitignore b/tools/testing/selftests/mm/.gitignore
index 54eefd8f97bc..b3b26447cd23 100644
--- a/tools/testing/selftests/mm/.gitignore
+++ b/tools/testing/selftests/mm/.gitignore
@@ -67,3 +67,4 @@ prctl_thp_disable
 rmap
 folio_split_race_test
 folio_order_check
+khugepaged_race
diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile
index 33917c76b871..046bae8d1eff 100644
--- a/tools/testing/selftests/mm/Makefile
+++ b/tools/testing/selftests/mm/Makefile
@@ -105,6 +105,7 @@ TEST_GEN_FILES += merge
 TEST_GEN_FILES += rmap
 TEST_GEN_FILES += folio_split_race_test
 TEST_GEN_FILES += folio_order_check
+TEST_GEN_FILES += khugepaged_race
 
 ifneq ($(ARCH),arm64)
 TEST_GEN_FILES += soft-dirty
diff --git a/tools/testing/selftests/mm/khugepaged_race.c b/tools/testing/selftests/mm/khugepaged_race.c
new file mode 100644
index 000000000000..b586a114e4cd
--- /dev/null
+++ b/tools/testing/selftests/mm/khugepaged_race.c
@@ -0,0 +1,354 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * khugepaged race harness.
+ *
+ * Runs collapse against concurrent faults, transient GUP pins
+ * (gup_test), fork, mremap and MADV_DONTNEED over the same ranges, in
+ * one of three driver modes:
+ *
+ *   stepped	khugepaged, driven synchronously one full pass at a time
+ *		through khugepaged_full_pass() — the primary driver: the
+ *		full scan+collapse path with a deterministic extent per
+ *		step;
+ *   free	free-running khugepaged (scan_sleep_millisecs=0) — the
+ *		soak;
+ *   madvise	MADV_COLLAPSE in a loop — the legacy-PMD regression
+ *		axis.
+ *
+ * All anon THP orders are enabled (inherit) and max_ptes_none is set
+ * mid-range, so the MADV_DONTNEED holes steer selection across orders.
+ *
+ * Correctness signals: every racing page must read as its pattern or
+ * zero (MADV_DONTNEED), never anything else — checked continuously by
+ * the faulters and the fork children and once at the end — plus
+ * whatever DEBUG_VM / page_table_check / KASAN / lockdep report in
+ * dmesg, which the caller is expected to inspect.
+ */
+#define _GNU_SOURCE
+#include <errno.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "kselftest.h"
+#include "vm_util.h"
+#include "hugepage_settings.h"
+#include "../../../../mm/gup_test.h"
+
+#define BASE_ADDR	((void *)(1UL << 30))
+/*
+ * Shared playground for faults/pins/fork/dontneed: several PMD-sized
+ * areas the racing threads spread across, plus one area owned by the
+ * mremap thread. More areas means more independent regions collapsing
+ * at once; the default suits a normal machine. On a memory-constrained
+ * host -- or under emulation, where a 512M PMD (arm64/64K) makes the
+ * default playground multi-gigabyte -- pass -a to shrink it.
+ */
+#define DEFAULT_SHARED_AREAS	3
+static int nr_shared_areas;
+static int nr_areas;
+
+static unsigned long hpage_pmd_size;
+static unsigned long page_size;
+static char *region;		/* NR_AREAS * hpage_pmd_size */
+static char *mremap_area;	/* region + NR_SHARED_AREAS areas */
+static char *mremap_scratch;	/* well above the region */
+static int gup_fd = -1;
+static volatile int stop;
+static volatile int corrupted;
+
+static unsigned int pattern(unsigned long page_idx)
+{
+	unsigned int val = (unsigned int)page_idx * 2654435761U;
+
+	return val ? val : 1;	/* never collides with the zero-fill */
+}
+
+static void check_page(unsigned long page_idx)
+{
+	unsigned int val = *(unsigned int *)(region + page_idx * page_size);
+
+	if (val && val != pattern(page_idx)) {
+		corrupted = 1;
+		ksft_print_msg("Corruption at page %lu: %#x != %#x\n",
+			       page_idx, val, pattern(page_idx));
+	}
+}
+
+static unsigned long rand_page(unsigned int *seed)
+{
+	return (unsigned long)rand_r(seed) %
+	       (nr_shared_areas * hpage_pmd_size / page_size);
+}
+
+static void *faulter_fn(void *arg)
+{
+	unsigned int seed = (unsigned long)arg;
+
+	while (!stop) {
+		unsigned long page_idx = rand_page(&seed);
+
+		if (rand_r(&seed) & 1)
+			*(unsigned int *)(region + page_idx * page_size) =
+				pattern(page_idx);
+		else
+			check_page(page_idx);
+	}
+	return NULL;
+}
+
+static void *dontneed_fn(void *arg)
+{
+	unsigned int seed = (unsigned long)arg;
+
+	while (!stop) {
+		unsigned long page_idx = rand_page(&seed);
+		unsigned long nr = 1UL << (rand_r(&seed) % 6);	/* 1..32 pages */
+
+		madvise(region + page_idx * page_size, nr * page_size,
+			MADV_DONTNEED);
+		usleep(rand_r(&seed) % 500);
+	}
+	return NULL;
+}
+
+static void *pinner_fn(void *arg)
+{
+	unsigned int seed = (unsigned long)arg;
+
+	while (!stop) {
+		struct gup_test gup = {};
+		unsigned long page_idx = rand_page(&seed);
+
+		gup.addr = (unsigned long)(region + page_idx * page_size);
+		gup.size = 16 * page_size;
+		gup.nr_pages_per_call = 16;
+		gup.gup_flags = 1;	/* FOLL_WRITE */
+		/* Racing MADV_DONTNEED makes transient failures expected. */
+		ioctl(gup_fd, PIN_FAST_BENCHMARK, &gup);
+		usleep(rand_r(&seed) % 200);
+	}
+	return NULL;
+}
+
+static void *forker_fn(void *arg)
+{
+	unsigned int seed = (unsigned long)arg;
+
+	while (!stop) {
+		pid_t pid = fork();
+
+		if (pid == 0) {
+			for (int i = 0; i < 16; i++)
+				check_page(rand_page(&seed));
+			_exit(corrupted);
+		}
+		if (pid > 0) {
+			int wstatus;
+
+			waitpid(pid, &wstatus, 0);
+			if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus))
+				corrupted = 1;
+		}
+		usleep(rand_r(&seed) % 2000);
+	}
+	return NULL;
+}
+
+static void *mremapper_fn(void *arg)
+{
+	unsigned int seed = (unsigned long)arg;
+
+	while (!stop) {
+		void *p;
+
+		p = mremap(mremap_area, hpage_pmd_size, hpage_pmd_size,
+			   MREMAP_MAYMOVE | MREMAP_FIXED, mremap_scratch);
+		if (p == MAP_FAILED)
+			ksft_exit_fail_perror("mremap() away");
+		for (int i = 0; i < 8; i++)
+			mremap_scratch[(rand_r(&seed) %
+				(hpage_pmd_size / page_size)) * page_size] = 1;
+		p = mremap(mremap_scratch, hpage_pmd_size, hpage_pmd_size,
+			   MREMAP_MAYMOVE | MREMAP_FIXED, mremap_area);
+		if (p == MAP_FAILED)
+			ksft_exit_fail_perror("mremap() back");
+		usleep(rand_r(&seed) % 2000);
+	}
+	return NULL;
+}
+
+static unsigned long now_ms(void)
+{
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	return tv.tv_sec * 1000UL + tv.tv_usec / 1000;
+}
+
+static void usage(void)
+{
+	fprintf(stderr,
+		"Usage: khugepaged_race [-d seconds] [-m stepped|free|madvise] [-a areas]\n"
+		"\t-a: number of shared PMD-sized playground areas (default 3)\n");
+	exit(1);
+}
+
+int main(int argc, char **argv)
+{
+	static const char * const thread_names[] = {
+		"faulter", "faulter2", "dontneed", "pinner", "forker",
+		"mremapper",
+	};
+	void *(*const thread_fns[])(void *) = {
+		faulter_fn, faulter_fn, dontneed_fn, pinner_fn, forker_fn,
+		mremapper_fn,
+	};
+	const int nr_threads = ARRAY_SIZE(thread_names);
+	pthread_t threads[ARRAY_SIZE(thread_names)];
+	const char *mode = "stepped";
+	struct thp_settings settings;
+	unsigned long end_ms;
+	int duration_s = 10;
+	unsigned long thread_mask = ~0UL;
+	int nr_areas_arg = 0;
+	unsigned long i;
+	int steps = 0;
+	int opt;
+
+	while ((opt = getopt(argc, argv, "a:d:m:t:h")) != -1) {
+		switch (opt) {
+		case 'a':
+			nr_areas_arg = atoi(optarg);
+			break;
+		case 'd':
+			duration_s = atoi(optarg);
+			break;
+		case 'm':
+			mode = optarg;
+			break;
+		case 't':
+			/* debug: bitmask of racing threads to start */
+			thread_mask = strtoul(optarg, NULL, 0);
+			break;
+		default:
+			usage();
+		}
+	}
+	if (strcmp(mode, "stepped") && strcmp(mode, "free") &&
+	    strcmp(mode, "madvise"))
+		usage();
+
+	ksft_print_header();
+	if (!thp_available())
+		ksft_exit_skip("Transparent Hugepages not available\n");
+
+	page_size = getpagesize();
+	hpage_pmd_size = read_pmd_pagesize();
+	if (!hpage_pmd_size)
+		ksft_exit_fail_msg("Reading PMD pagesize failed\n");
+
+	gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
+	if (gup_fd < 0)
+		ksft_exit_skip("/sys/kernel/debug/gup_test requires CONFIG_GUP_TEST and root\n");
+
+	nr_shared_areas = nr_areas_arg > 0 ? nr_areas_arg : DEFAULT_SHARED_AREAS;
+	nr_areas = nr_shared_areas + 1;
+
+	ksft_set_plan(1);
+
+	thp_save_settings();
+	thp_read_settings(&settings);
+	settings.thp_enabled = THP_MADVISE;
+	settings.thp_defrag = THP_DEFRAG_ALWAYS;
+	settings.shmem_enabled = SHMEM_NEVER;
+	settings.khugepaged.defrag = 1;
+	settings.khugepaged.scan_sleep_millisecs =
+		strcmp(mode, "free") ? 1000 : 0;
+	settings.khugepaged.alloc_sleep_millisecs = 10;
+	/*
+	 * Strict occupancy: mTHP collapse only supports 0 or
+	 * HPAGE_PMD_NR - 1 and coerces anything else to 0 anyway, and 0
+	 * also keeps khugepaged from burning the whole step in doomed
+	 * PMD-sized allocations on 512M-PMD configs: under racing
+	 * MADV_DONTNEED a fully populated PMD area is rare.
+	 */
+	settings.khugepaged.max_ptes_none = 0;
+	settings.khugepaged.pages_to_scan =
+		nr_areas * (hpage_pmd_size / page_size) * 8;
+	for (i = 0; i < NR_ORDERS; i++) {
+		if (thp_supported_orders() & (1UL << i))
+			settings.hugepages[i].enabled = THP_INHERIT;
+	}
+	/* Base of the settings stack; the bottom entry is never popped. */
+	thp_push_settings(&settings);
+
+	region = mmap(BASE_ADDR, nr_areas * hpage_pmd_size,
+		      PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE,
+		      -1, 0);
+	if (region != BASE_ADDR)
+		ksft_exit_fail_msg("Failed to allocate VMA at %p\n", BASE_ADDR);
+	mremap_area = region + nr_shared_areas * hpage_pmd_size;
+	mremap_scratch = (char *)BASE_ADDR + 2 * nr_areas * hpage_pmd_size;
+
+	/* Populate so the first pass has something to collapse. */
+	for (i = 0; i < nr_shared_areas * hpage_pmd_size / page_size; i++)
+		*(unsigned int *)(region + i * page_size) = pattern(i);
+	memset(mremap_area, 1, hpage_pmd_size);
+	madvise(region, nr_areas * hpage_pmd_size, MADV_HUGEPAGE);
+
+	for (i = 0; i < nr_threads; i++) {
+		if (!(thread_mask & (1UL << i))) {
+			threads[i] = 0;
+			continue;
+		}
+		if (pthread_create(&threads[i], NULL, thread_fns[i],
+				   (void *)(i + 1)))
+			ksft_exit_fail_perror("pthread_create()");
+	}
+
+	end_ms = now_ms() + duration_s * 1000UL;
+	if (!strcmp(mode, "stepped")) {
+		while (now_ms() < end_ms && !corrupted) {
+			if (!khugepaged_full_pass(600))
+				ksft_exit_fail_msg("khugepaged pass timed out\n");
+			steps++;
+		}
+	} else if (!strcmp(mode, "free")) {
+		while (now_ms() < end_ms && !corrupted)
+			usleep(100 * 1000);
+	} else {	/* madvise */
+		while (now_ms() < end_ms && !corrupted) {
+			for (i = 0; i < nr_shared_areas; i++) {
+				madvise(region + i * hpage_pmd_size,
+					hpage_pmd_size, MADV_COLLAPSE);
+			}
+			madvise(region, nr_shared_areas * hpage_pmd_size,
+				MADV_DONTNEED);
+			steps++;
+		}
+	}
+
+	stop = 1;
+	for (i = 0; i < nr_threads; i++) {
+		if (threads[i])
+			pthread_join(threads[i], NULL);
+	}
+
+	/* Final integrity sweep. */
+	for (i = 0; i < nr_shared_areas * hpage_pmd_size / page_size; i++)
+		check_page(i);
+
+	thp_restore_settings();
+
+	ksft_test_result(!corrupted, "%s: %ds, %d steps, no corruption\n",
+			 mode, duration_s, steps);
+	ksft_finished();
+}
diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh
index 6f42e35b5227..a8b6b839cb97 100755
--- a/tools/testing/selftests/mm/run_vmtests.sh
+++ b/tools/testing/selftests/mm/run_vmtests.sh
@@ -404,6 +404,13 @@ CATEGORY="cow" run_test ./cow
 
 CATEGORY="thp" run_test ./folio_order_check
 
+
+CATEGORY="thp" run_test ./khugepaged_race -d 5 -m stepped
+
+CATEGORY="thp" run_test ./khugepaged_race -d 5 -m free
+
+CATEGORY="thp" run_test ./khugepaged_race -d 5 -m madvise
+
 CATEGORY="thp" run_test ./khugepaged
 
 CATEGORY="thp" run_test ./khugepaged -s 2
-- 
2.54.0