[PATCH v4] shell: enable OOM protection by default

From: Andrea Cervesato <andrea.cervesato-IBi9RG/[email protected]>

The shell harness shields itself from the OOM killer and runs the
test in a child process, so it survives memory pressure and can
still report results (e.g. during memcg stress tests).

Suggested-by: Li Wang <[email protected]>
Signed-off-by: Andrea Cervesato <andrea.cervesato-IBi9RG/[email protected]>
---
Under Li's idea, implement a OOM protection mechanism for the shell
tests so we can avoid OOM for memcg stress tests.
---
Changes in v4:
- fix with Petr suggestions
- fix with AI agent suggestions
- Link to v3: https://lore.kernel.org/20260804-shell_oom_protection-v3-1-fe42b15d034c-IBi9RG/[email protected]

Changes in v3:
- simplify oom code
- enable OOM protection by default
- Link to v2: https://lore.kernel.org/20260730-shell_oom_protection-v2-0-be1de2baa83d-IBi9RG/[email protected]

Changes in v2:
- update shell OOM protection functional test
- Link to v1: https://lore.kernel.org/20260713-shell_oom_protection-v1-0-b732e8647894-IBi9RG/[email protected]

To: Linux Test Project <[email protected]>
---
 lib/newlib_tests/runtest.sh                  |  1 +
 lib/newlib_tests/shell/tst_oom_protection.sh | 34 +++++++++++
 testcases/lib/tst_test.sh                    | 84 ++++++++++++++++++++++++++--
 3 files changed, 115 insertions(+), 4 deletions(-)

diff --git a/lib/newlib_tests/runtest.sh b/lib/newlib_tests/runtest.sh
index 71808ef8b..7e2d0a2ac 100755
--- a/lib/newlib_tests/runtest.sh
+++ b/lib/newlib_tests/runtest.sh
@@ -44,6 +44,7 @@ shell/tst_check_driver.sh
 shell/tst_check_kconfig0[1-5].sh
 shell/tst_mount_device.sh
 shell/tst_mount_device_tmpfs.sh
+shell/tst_oom_protection.sh
 shell/tst_skip_filesystems.sh
 shell/net/*.sh}"
 
diff --git a/lib/newlib_tests/shell/tst_oom_protection.sh b/lib/newlib_tests/shell/tst_oom_protection.sh
new file mode 100755
index 000000000..564680700
--- /dev/null
+++ b/lib/newlib_tests/shell/tst_oom_protection.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (c) 2026 Linux Test Project
+
+TST_TESTFUNC=do_test
+
+read_oom_score_adj() {
+	cat "/proc/$1/oom_score_adj"
+}
+
+do_test() {
+	local harness_score body_score
+
+	# $$ points to the protected harness, while /proc/self is the
+	# unprotected child that actually runs the test body.
+	harness_score=$(read_oom_score_adj "$$")
+	body_score=$(read_oom_score_adj self)
+
+	if [ "$harness_score" != -1000 ]; then
+		tst_res TCONF "shell harness OOM protection unavailable"
+		return
+	fi
+
+	tst_res TPASS "shell harness is protected from OOM by default"
+
+	if [ "$body_score" = 0 ]; then
+		tst_res TPASS "test body runs in an unprotected child (oom_score_adj=0)"
+	else
+		tst_res TFAIL "test body oom_score_adj is $body_score, expected 0"
+	fi
+}
+
+. tst_test.sh
+tst_run
diff --git a/testcases/lib/tst_test.sh b/testcases/lib/tst_test.sh
index b3e7e29bb..e42e562f5 100644
--- a/testcases/lib/tst_test.sh
+++ b/testcases/lib/tst_test.sh
@@ -28,6 +28,48 @@ export TST_USR_GID="${LTP_USR_GID:-65534}"
 trap "tst_brk TBROK 'test interrupted'" INT
 trap "unset _tst_setup_timer_pid; tst_brk TBROK 'test terminated'" TERM
 
+_tst_set_oom_score_adj()
+{
+	local value="$1"
+	local path="/proc/self/oom_score_adj"
+
+	if [ ! -e "$path" ]; then
+		tst_res TINFO "oom_score_adj does not exist, skipping the adjustment"
+		return
+	fi
+
+	if ! echo "$value" > "$path"; then
+		tst_res TWARN "Can't adjust score"
+	fi
+}
+
+_tst_enable_oom_protection()
+{
+	_tst_set_oom_score_adj -1000
+}
+
+_tst_disable_oom_protection()
+{
+	_tst_set_oom_score_adj 0
+}
+
+_tst_write_results()
+{
+	[ "$TST_CHILD" = 1 ] || return 0
+	[ -n "$TST_RESULTS_FILE" ] || return 0
+
+	echo "$TST_PASS $TST_FAIL $TST_BROK $TST_WARN $TST_CONF $TST_COUNT" \
+		> "$TST_RESULTS_FILE"
+}
+
+_tst_read_results()
+{
+	[ -s "$TST_RESULTS_FILE" ] || return 0
+
+	read TST_PASS TST_FAIL TST_BROK TST_WARN TST_CONF TST_COUNT \
+		< "$TST_RESULTS_FILE"
+}
+
 _tst_do_cleanup()
 {
 	if [ -n "$TST_DO_CLEANUP" -a -n "$TST_CLEANUP" -a -z "$LTP_NO_CLEANUP" ]; then
@@ -48,6 +90,15 @@ _tst_do_exit()
 
 	_tst_do_cleanup
 
+	# When running as the unprotected test child, only propagate the
+	# results back to the protected harness which does the teardown and
+	# prints the summary.
+	if [ "$TST_CHILD" = 1 ]; then
+		_tst_cleanup_timer
+		_tst_write_results
+		exit 0
+	fi
+
 	cd "$LTPROOT"
 	[ "$TST_MOUNT_FLAG" = 1 ] && tst_umount
 
@@ -788,10 +839,34 @@ tst_run()
 
 	TST_MNTPOINT="${TST_MNTPOINT:-$PWD/mntpoint}"
 
-	if [ "$TST_ALL_FILESYSTEMS" = 1 ]; then
-		_tst_run_tcases_per_fs
-	else
-		_tst_run_iterations
+	# Protect the harness so it survives memory pressure and can
+	# still report results, then run the test body in a child that
+	# resets OOM protection. This way workloads forked by the test
+	# remain killable, mirroring the C test harness.
+	_tst_enable_oom_protection
+
+	TST_RESULTS_FILE=$(mktemp "${TMPDIR:-/tmp}/ltp_${TST_ID}_res.XXXXXX")
+
+	(
+		TST_CHILD=1
+		_tst_disable_oom_protection
+		trap '_tst_cleanup_timer; _tst_write_results' EXIT
+
+		if [ "$TST_ALL_FILESYSTEMS" = 1 ]; then
+			_tst_run_tcases_per_fs
+		else
+			_tst_run_iterations
+		fi
+	) &
+
+	wait $!
+	ret=$?
+
+	_tst_read_results
+	rm -f "$TST_RESULTS_FILE"
+
+	if [ $ret -gt 128 ]; then
+		tst_res TBROK "Test killed by signal $((ret - 128))!"
 	fi
 
 	_tst_do_exit
@@ -865,6 +940,7 @@ _tst_run_test()
 	$_tst_fnc "$@"
 	_tst_rescmp "$_tst_res"
 	TST_COUNT=$((TST_COUNT+1))
+	_tst_write_results
 }
 
 export LC_ALL=C

---
base-commit: ee945bb71ba7015dace7d983cf98a530b3eb53a4
change-id: 20260713-shell_oom_protection-6221ab825220

Best regards,
--  
Andrea Cervesato <andrea.cervesato-IBi9RG/[email protected]>


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp