svn commit: r1936270 - in httpd/httpd/trunk/test: . pyhttpd pytest_suite

Author: jim
Date: Fri Jul 17 15:49:01 2026
New Revision: 1936270

Log:
Make the Python test suites more robust:
  o Call location independence
  o Fully support pip or uv
  o Better reporting of test results when both suites are run
  o Support postitional args for test cases in a reliable manner

Modified:
   httpd/httpd/trunk/test/README
   httpd/httpd/trunk/test/pyhttpd/runtests.sh
   httpd/httpd/trunk/test/pytest_suite/runtests.sh
   httpd/httpd/trunk/test/run-all-tests.sh

Modified: httpd/httpd/trunk/test/README
==============================================================================
--- httpd/httpd/trunk/test/README	Fri Jul 17 14:42:31 2026	(r1936269)
+++ httpd/httpd/trunk/test/README	Fri Jul 17 15:49:01 2026	(r1936270)
@@ -73,20 +73,24 @@ The runner exits non-zero if either suit
 Running a suite directly
 ------------------------
 
-pytest_suite (from its own directory; it creates its own virtualenv):
-
-  cd pytest_suite
-  uv sync                                   # one-time: create the venv
-  ./runtests.sh --apxs=/path/to/apxs        # all tests
-  ./runtests.sh --php-fpm=/path/to/php-fpm tests/t/php   # PHP tests
-  ./runtests.sh -k rewrite -v               # any pytest args pass through
+Both runtests.sh scripts create their own virtualenv on first run and can be
+invoked from any directory -- the paths below are just the convenient way to
+type them. The venv is (re)built automatically whenever it is missing or its
+pyproject.toml has changed, using `uv sync` if uv is installed and otherwise
+`python3 -m venv` + pip (reading the dependency list from pyproject.toml -- no
+uv required). To force a clean rebuild yourself, `rm -rf <suite>/.venv`.
+
+pytest_suite (self-contained; the venv holds only pytest + httpx):
+
+  ./pytest_suite/runtests.sh --apxs=/path/to/apxs            # all tests
+  ./pytest_suite/runtests.sh --php-fpm=/path/to/php-fpm tests/t/php   # PHP tests
+  ./pytest_suite/runtests.sh -k rewrite -v                   # any pytest args pass through
 
 pyhttpd tests (need pyhttpd/config.ini from httpd's configure, plus curl,
 nghttp2/h2load, and -- for modules/md -- pyOpenSSL and an ACME test server):
 
-  pytest modules/http2                      # all HTTP/2 tests
-  pytest modules/core -k test_001           # a subset
-
+  ./pyhttpd/runtests.sh modules/http2       # all HTTP/2 tests
+  ./pyhttpd/runtests.sh modules/core -k test_001   # a subset
 
 Other contents
 --------------

Modified: httpd/httpd/trunk/test/pyhttpd/runtests.sh
==============================================================================
--- httpd/httpd/trunk/test/pyhttpd/runtests.sh	Fri Jul 17 14:42:31 2026	(r1936269)
+++ httpd/httpd/trunk/test/pyhttpd/runtests.sh	Fri Jul 17 15:49:01 2026	(r1936270)
@@ -18,21 +18,37 @@ set -eu
 
 here="$(cd "$(dirname "$0")" && pwd)"
 
+# --- ensure the venv exists and is current ----------------------------------
+# We invoke .venv/bin/pytest directly rather than `uv run` so the suite works
+# even where `uv run` is shimmed/unavailable.
+#
+# Create $here/.venv on first run, and rebuild it when pyproject.toml is newer
+# than the venv (i.e. dependencies changed). Prefer uv (which reads
+# pyproject.toml + uv.lock); otherwise fall back to python3 -m venv + pip,
+# taking the dependency list straight from pyproject.toml so there is no second
+# copy to keep in sync. Absolute paths throughout, so this behaves identically
+# regardless of the caller's cwd. This block is kept byte-for-byte identical in
+# pytest_suite/runtests.sh and pyhttpd/runtests.sh -- edit both together.
 PYTEST="$here/.venv/bin/pytest"
-if [ ! -x "$PYTEST" ]; then
+if [ ! -x "$PYTEST" ] || [ "$here/pyproject.toml" -nt "$here/.venv" ]; then
     if command -v uv >/dev/null 2>&1; then
-        echo "runtests.sh: .venv not found; running 'uv sync' to create it..." >&2
+        echo "runtests.sh: (re)creating $here/.venv via 'uv sync'..." >&2
         uv sync --project "$here"
     elif command -v python3 >/dev/null 2>&1; then
-        echo "runtests.sh: .venv not found; creating with python3 + pip..." >&2
+        echo "runtests.sh: (re)creating $here/.venv via python3 + pip..." >&2
         python3 -m venv "$here/.venv"
-         # Keep this list in sync with pyproject.toml [project].dependencies
-         "$here/.venv/bin/pip" install --quiet \
-             "pytest>=7.0" cryptography filelock "python-multipart" pyopenssl packaging websockets
+        # Read [project].dependencies from pyproject.toml (one entry per line,
+        # double-quoted) so the install list never drifts from the manifest.
+        deps=$(awk -F'"' '/^dependencies = \[/{f=1; next} f && /^\]/{f=0} f && NF>=2 {print $2}' "$here/pyproject.toml")
+        # shellcheck disable=SC2086  # deps is an intentional word-split list
+        "$here/.venv/bin/pip" install --quiet $deps
     else
-        echo "runtests.sh: ERROR: $PYTEST not found and neither 'uv' nor 'python3' is on PATH." >&2
+        echo "runtests.sh: ERROR: $PYTEST not found and neither 'uv' nor 'python3' is installed." >&2
         exit 1
     fi
+    # Mark the venv as freshly built so the staleness check above won't retrigger
+    # until pyproject.toml changes again.
+    touch "$here/.venv"
 fi
 
 # Prepend the venv's bin dir so that CGI scripts forked by httpd also resolve
@@ -40,8 +56,40 @@ fi
 # that any shim wrappers earlier on PATH are shadowed.
 export PATH="$here/.venv/bin:$PATH"
 
-targets="${PYHTTPD_TARGETS:-modules}"
+# The modules/ test suite lives in test/, a sibling of this script's directory
+# (test/pyhttpd/) -- cd there so both the default target and any
+# PYHTTPD_TARGETS/positional path the caller supplies resolve the same way
+# regardless of where runtests.sh was invoked from.
+cd "$(dirname "$here")"
+
+# Only fall back to the "modules" default when the caller gave no positional
+# test path of their own -- otherwise it would always tag along after theirs
+# (`pytest modules modules/http1`), silently widening any subset selection
+# back out to the full suite. A positional path is recognized by actually
+# existing on disk (relative to test/, our cwd at this point) -- this avoids
+# both having to enumerate every pytest flag that takes a separate-word value
+# (-k, -m, -p, --tb, --maxfail, -n from pytest-xdist, ...) and misdetecting a
+# -k/-m expression that happens to contain '/' (this suite's own parametrize
+# IDs look like "/006/006.css", so "-k 006/006" is a realistic selector, and
+# it does not exist as a path).
+have_path=0
+for arg in "$@"; do
+    case "$arg" in
+        -*) ;;
+        # Strip a trailing ::nodeid (pytest's file::Class::test node-selector
+        # syntax) before checking existence -- only the file/dir part is real.
+        *) [ -e "${arg%%::*}" ] && have_path=1 ;;
+    esac
+done
+
+if [ -n "${PYHTTPD_TARGETS:-}" ]; then
+    targets="$PYHTTPD_TARGETS"
+elif [ "$have_path" = 1 ]; then
+    targets=""
+else
+    targets="modules"
+fi
 
-# shellcheck disable=SC2086
 echo "runtests.sh: $PYTEST $targets $*" >&2
+# shellcheck disable=SC2086  # $targets is an intentional word-split path list
 exec "$PYTEST" $targets "$@"

Modified: httpd/httpd/trunk/test/pytest_suite/runtests.sh
==============================================================================
--- httpd/httpd/trunk/test/pytest_suite/runtests.sh	Fri Jul 17 14:42:31 2026	(r1936269)
+++ httpd/httpd/trunk/test/pytest_suite/runtests.sh	Fri Jul 17 15:49:01 2026	(r1936270)
@@ -26,23 +26,37 @@ set -eu
 here="$(cd "$(dirname "$0")" && pwd)"
 cd "$here"
 
-# --- locate the virtualenv's pytest -----------------------------------------
+# --- ensure the venv exists and is current ----------------------------------
 # We invoke .venv/bin/pytest directly rather than `uv run` so the suite works
-# even where `uv run` is shimmed/unavailable. Create the venv with `uv sync`
-# (or `python -m venv .venv && .venv/bin/pip install -e .`) if it's missing.
+# even where `uv run` is shimmed/unavailable.
+#
+# Create $here/.venv on first run, and rebuild it when pyproject.toml is newer
+# than the venv (i.e. dependencies changed). Prefer uv (which reads
+# pyproject.toml + uv.lock); otherwise fall back to python3 -m venv + pip,
+# taking the dependency list straight from pyproject.toml so there is no second
+# copy to keep in sync. Absolute paths throughout, so this behaves identically
+# regardless of the caller's cwd. This block is kept byte-for-byte identical in
+# pytest_suite/runtests.sh and pyhttpd/runtests.sh -- edit both together.
 PYTEST="$here/.venv/bin/pytest"
-if [ ! -x "$PYTEST" ]; then
+if [ ! -x "$PYTEST" ] || [ "$here/pyproject.toml" -nt "$here/.venv" ]; then
     if command -v uv >/dev/null 2>&1; then
-        echo "runtests.sh: .venv not found; running 'uv sync' to create it..." >&2
-        uv sync
+        echo "runtests.sh: (re)creating $here/.venv via 'uv sync'..." >&2
+        uv sync --project "$here"
     elif command -v python3 >/dev/null 2>&1; then
-        echo "runtests.sh: .venv not found; creating it with python3 + pip..." >&2
-        python3 -m venv .venv
-        .venv/bin/pip install --quiet -e .
+        echo "runtests.sh: (re)creating $here/.venv via python3 + pip..." >&2
+        python3 -m venv "$here/.venv"
+        # Read [project].dependencies from pyproject.toml (one entry per line,
+        # double-quoted) so the install list never drifts from the manifest.
+        deps=$(awk -F'"' '/^dependencies = \[/{f=1; next} f && /^\]/{f=0} f && NF>=2 {print $2}' "$here/pyproject.toml")
+        # shellcheck disable=SC2086  # deps is an intentional word-split list
+        "$here/.venv/bin/pip" install --quiet $deps
     else
         echo "runtests.sh: ERROR: $PYTEST not found and neither 'uv' nor 'python3' is installed." >&2
         exit 1
     fi
+    # Mark the venv as freshly built so the staleness check above won't retrigger
+    # until pyproject.toml changes again.
+    touch "$here/.venv"
 fi
 
 # --- discover apxs / httpd / php-fpm ----------------------------------------
@@ -93,6 +107,6 @@ esac
 rm -f "$here/t/logs/cgisock"* 2>/dev/null || true
 
 # --- run --------------------------------------------------------------------
-# shellcheck disable=SC2086  # auto_args is an intentional word-split flag list
 echo "runtests.sh: $PYTEST $auto_args $*" >&2
+# shellcheck disable=SC2086  # auto_args is an intentional word-split flag list
 exec "$PYTEST" $auto_args "$@"

Modified: httpd/httpd/trunk/test/run-all-tests.sh
==============================================================================
--- httpd/httpd/trunk/test/run-all-tests.sh	Fri Jul 17 14:42:31 2026	(r1936269)
+++ httpd/httpd/trunk/test/run-all-tests.sh	Fri Jul 17 15:49:01 2026	(r1936270)
@@ -70,7 +70,14 @@ config_ini="$here/pyhttpd/config.ini"
 #             paths and go ONLY to pytest_suite. The pyhttpd side selects its
 #             tests via PYHTTPD_TARGETS (or its auto-detected default), since a
 #             pytest_suite path is meaningless there.
-# A flag that takes a separate-word value (-k NAME) keeps the value as a flag.
+#
+# The hard part is telling a positional test path from the value of a flag that
+# takes a separate word (e.g. `--tb short`, `--maxfail 3`, `-n 4`). We handle it
+# two ways: (a) the common value-flags -k/-m/-p are known to consume the next
+# word, and (b) any OTHER bare word is treated as a pysuite path only if it
+# actually exists on disk -- a flag value like "short"/"3"/"no" never does, so
+# it stays with `flags` (attached to its preceding flag) instead of being
+# misrouted to pysuite-only paths and stripped from what pyhttpd receives.
 only=""
 apxs_opt=""
 flags=""
@@ -89,7 +96,15 @@ for arg in "$@"; do
         --clean-modules) pysuite_flags="$pysuite_flags $arg" ;;  # pysuite-only; pyhttpd has no C modules
         -k|-m|-p) flags="$flags $arg"; expect_flagval=1 ;;  # take a value next
         -*)       flags="$flags $arg" ;;
-        *)        paths="$paths $arg" ;;
+        # A real pysuite path exists relative to pytest_suite/ (how users type
+        # it, e.g. "tests/t/php") or to our cwd; strip any ::nodeid suffix
+        # first. Anything else is a stray flag value -> keep it with the flags.
+        *)  if [ -e "$suite_dir/${arg%%::*}" ] || [ -e "${arg%%::*}" ]; then
+                paths="$paths $arg"
+            else
+                flags="$flags $arg"
+            fi
+            ;;
     esac
 done
 
@@ -110,6 +125,7 @@ php_args=""
 [ -n "${PHP_FPM:-}" ] && php_args="--php-fpm=$PHP_FPM"
 
 rc=0
+skipped=""   # names of suites that did NOT run (so we never report them "passed")
 
 run_pysuite() {
     echo "=========================================================="
@@ -134,6 +150,7 @@ run_pyhttpd() {
     if [ ! -f "$config_ini" ]; then
         echo "run-all-tests.sh: note: pyhttpd/config.ini not found;" >&2
         echo "  build httpd with its test config (configure) to run these." >&2
+        skipped="$skipped pyhttpd"
         return 0
     fi
     # runtests.sh manages the venv, prepends its bin/ to PATH (so CGI
@@ -159,6 +176,21 @@ case "$only" in
 esac
 
 echo "=========================================================="
-[ "$rc" -eq 0 ] && echo "ALL SUITES PASSED" || echo "SOME TESTS FAILED (rc=$rc)"
+if [ "$rc" -ne 0 ]; then
+    echo "SOME TESTS FAILED (rc=$rc)"
+elif [ -n "$skipped" ]; then
+    # Nothing failed, but at least one suite never ran -- don't claim success
+    # for a suite that was skipped (e.g. pyhttpd with no config.ini).
+    echo "PASSED, BUT SKIPPED:$skipped (not run -- see notes above)"
+else
+    echo "ALL SUITES PASSED"
+fi
 echo "=========================================================="
+
+# If a suite was skipped and the user explicitly asked for ONLY that suite,
+# treat "ran nothing" as a failure -- otherwise --only=pyhttpd could exit 0
+# having executed zero tests.
+if [ -n "$skipped" ] && [ -n "$only" ] && [ "$rc" -eq 0 ]; then
+    exit 3
+fi
 exit "$rc"