gh-75876: Report the peak memory use of a bigmem test (GH-156024)

https://github.com/python/cpython/commit/3cfc87e8bdb4711a0757d59000459202fb1a82f6
commit: 3cfc87e8bdb4711a0757d59000459202fb1a82f6
branch: main
author: Serhiy Storchaka <[email protected]>
committer: serhiy-storchaka <[email protected]>
date: 2026-08-19T11:42:08+03:00
summary:

gh-75876: Report the peak memory use of a bigmem test (GH-156024)

In verbose mode, report how much memory the test really used, next to how
much it declared.  The subprocess measures its own peak; if it died before
reporting it, the parent reports what it sampled, as a lower bound.

Report also the number of major page faults, if any: the test did not fit in
memory, so its timing means little.

Co-authored-by: Claude Opus 5 (1M context) <[email protected]>

files:
A Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst
M Lib/test/support/__init__.py
M Lib/test/support/subprocess_runner.py

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 31e5508dd9b790..be71575a6ea06a 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -1271,15 +1271,20 @@ def set_memlimit(limit: str) -> None:
 
 
 def _memory_watchdog(pid):
-    """Return a function printing the memory usage of process *pid*."""
+    """Return a function printing the memory usage of process *pid*.
+
+    The largest value it saw is kept in its ``peak`` attribute.
+    """
     # Imported here: test.support does not depend on test.libregrtest.
     from test.libregrtest.utils import get_process_memory_usage
 
     def watch():
         mem = get_process_memory_usage(pid)
         if mem is not None:
+            watch.peak = max(watch.peak, mem)
             print(f" ... process data size: {mem / (1024 ** 3):.1f} GiB",
                   flush=True)
+    watch.peak = 0
     return watch
 
 
@@ -1333,7 +1338,23 @@ def wrapper(self):
                 qualname = f'{cls.__qualname__}.{f.__name__}'
                 proc = isolation._start_test(cls.__module__, qualname)
                 watchdog = _memory_watchdog(proc.pid) if verbose else None
-                isolation._replay_test(self, *proc.wait(tick=watchdog))
+                payload, output, returncode = proc.wait(tick=watchdog)
+                if watchdog:
+                    # The subprocess measures its own peak exactly.  What the
+                    # parent sampled is only a lower bound.
+                    maxrss = payload and payload.get('maxrss')
+                    peak = maxrss or watchdog.peak
+                    if peak:
+                        print(f" ... peak memory use: "
+                              f"{peak / (1024 ** 3):.1f} GiB"
+                              f"{'' if maxrss else ' or more'}", flush=True)
+                    majflt = payload and payload.get('majflt')
+                    if majflt:
+                        # The test did not fit in memory, so its timing means
+                        # little.
+                        print(f" ... {majflt} major page faults: the test "
+                              f"waited for the disk", flush=True)
+                isolation._replay_test(self, payload, output, returncode)
                 return
 
             return f(self, maxsize)
diff --git a/Lib/test/support/subprocess_runner.py b/Lib/test/support/subprocess_runner.py
index 90d74cc757d878..bb4ab059e2b8b6 100644
--- a/Lib/test/support/subprocess_runner.py
+++ b/Lib/test/support/subprocess_runner.py
@@ -72,7 +72,45 @@ def _outcome(kind, test, detail):
              for t, tb in result.expectedFailures]
 outcomes += [_outcome('skipped', t, reason) for t, reason in result.skipped]
 
-payload = {'outcomes': outcomes, 'durations': result.id_durations}
+def _usage():
+    """What this process used: peak resident set size in bytes, and the
+    number of major page faults it took, either of which can be None.
+
+    A major page fault is served from disk, so a non-zero count means swapping.
+
+    The modules are imported here, after the test has run, so that the test
+    does not see them.
+    """
+    try:
+        import resource
+    except ImportError:
+        pass
+    else:
+        usage = resource.getrusage(resource.RUSAGE_SELF)
+        # Solaris and illumos leave these fields at 0, which no live process
+        # has, so treat it as "not supported".
+        if not usage.ru_maxrss:
+            return {'maxrss': None, 'majflt': None}
+        # ru_maxrss is in bytes on macOS, in kilobytes on Linux and the BSDs.
+        maxrss = usage.ru_maxrss
+        return {'maxrss': maxrss if sys.platform == 'darwin' else maxrss * 1024,
+                'majflt': usage.ru_majflt}
+    try:
+        import os
+        import _winapi
+        handle = _winapi.OpenProcess(
+            _winapi.PROCESS_QUERY_LIMITED_INFORMATION, False, os.getpid())
+    except (ImportError, OSError):
+        return {'maxrss': None, 'majflt': None}
+    try:
+        info = _winapi.GetProcessMemoryInfo(handle)
+    finally:
+        _winapi.CloseHandle(handle)
+    # PageFaultCount counts all faults, not only the ones served from disk.
+    return {'maxrss': info['PeakWorkingSetSize'], 'majflt': None}
+
+
+payload = {'outcomes': outcomes, 'durations': result.id_durations, **_usage()}
 with open(outfile, 'wb') as f:
     marshal.dump(payload, f)
 
diff --git a/Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst b/Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst
new file mode 100644
index 00000000000000..863b49d2f91d8c
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2026-08-18-19-40-22.gh-issue-75876.IOiCcK.rst
@@ -0,0 +1,3 @@
+In verbose mode, a test decorated with :func:`~test.support.bigmemtest` now
+reports how much memory it really used, and the number of major page faults
+it took, if any.

_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]