Re: [PATCH] Refactor selftests

On 3/3/26 1:13 AM, Alan Maguire wrote:
> On 02/03/2026 17:10, Bruce McCulloch wrote:
>> Hello All,
>>
>> Here is a small patch to clean up common functions in the testsuite and
>> introduce a few quality of life features in a common library. Please let
>> me know if you have any concerns or would like to see any additional
>> features.
>>
>> Bruce
> hi Bruce, thanks for the patch! I ran it through dwarves CI and we see
> a fatal failure in [1] due the the fact that /usr/bin/perf isn't a
> program with DWARF but rather a shell wrapper. That's been a long-standing
> issue in the selftests/CI, but was non-fatal previously:
>
> Pretty printing of files using DWARF type information: libbpf: failed to get EHDR from /usr/sbin/perf
> pahole: /usr/sbin/perf: Invalid argument
> skip: /usr/sbin/perf doesn't have 'struct perf_event_header' type info
>
> If you want to run CI yourself you can clone https://github.com/acmel/dwarves
> on github and push a branch; you will see the CI run under the "Actions" tab
> of your repo.
>
>
> [1] https://github.com/alan-maguire/dwarves/actions/runs/22597958001
Hi Alan,

Thanks for the heads up, I'll make the change and re-submit after testing.

Bruce
>> ---
>> Simple refactoring of the testsuite with the creation of a shared
>> library (tests/test_lib.sh).
>> Additions include:
>> * Getters to find vmlinux.
>> * Functions to open tmp dirs and tmp files.
>> * Functions to automatically clean tmp dirs and files on pass.
>> * On fail, tmp dirs are preserved.
>> * Functions to pass/fail/softfail tests (works with CI).
>> * Logging functions with varying verbosity levels.
>> * Colors.
>>
>> Signed-off-by: Bruce McCulloch <[email protected]>
>> ---
>>   tests/btf_functions.sh        | 143 +++++++++++-------------------
>>   tests/default_vmlinux_btf.sh  |  18 ++--
>>   tests/flexible_arrays.sh      |  48 +++++-----
>>   tests/pfunct-btf-decl-tags.sh |  34 +++----
>>   tests/prettify_perf.data.sh   |  53 ++++++-----
>>   tests/reproducible_build.sh   |  47 +++++-----
>>   tests/test_lib.sh             | 161 ++++++++++++++++++++++++++++++++++
>>   tests/tests                   |  29 +++---
>>   8 files changed, 332 insertions(+), 201 deletions(-)
>>   create mode 100755 tests/test_lib.sh
>>
>> diff --git a/tests/btf_functions.sh b/tests/btf_functions.sh
>> index ee0f9ff..127640e 100755
>> --- a/tests/btf_functions.sh
>> +++ b/tests/btf_functions.sh
>> @@ -8,45 +8,22 @@
>>   # also should have been.
>>   #
>>   
>> -outdir=
>> -
>> -fail()
>> -{
>> -	# Do not remove test dir; might be useful for analysis
>> -	trap - EXIT
>> -	if [[ -d "$outdir" ]]; then
>> -		echo "Test data is in $outdir"
>> -	fi
>> -	exit 1
>> -}
>> -
>> -cleanup()
>> -{
>> -	rm ${outdir}/*
>> -	rmdir $outdir
>> -}
>> -
>> -vmlinux=${vmlinux:-$1}
>> -
>> -if [ -z "$vmlinux" ] ; then
>> -	vmlinux=$(pahole --running_kernel_vmlinux)
>> -	if [ -z "$vmlinux" ] ; then
>> -		echo "Please specify a vmlinux file to operate on"
>> -		exit 2
>> -	fi
>> -fi
>> +source test_lib.sh
>>   
>> -if [ ! -f "$vmlinux" ] ; then
>> -	echo "$vmlinux file not available, please specify another"
>> -	exit 2
>> +vmlinux=$(get_vmlinux $1)
>> +if [ $? -ne 0 ] ; then
>> +	info_log "$vmlinux"
>> +	test_fail
>>   fi
>>   
>> -outdir=$(mktemp -d /tmp/btf_functions.sh.XXXXXX)
>> +outdir=$(make_tmpdir)
>>   
>> +# Comment this out to save test data.
>>   trap cleanup EXIT
>>   
>> -echo -n "Validation of BTF encoding of functions; this may take some time: "
>> -test -n "$VERBOSE" && printf "\nEncoding..."
>> +title_log "Validation of BTF encoding of functions."
>> +info_log "This may take some time."
>> +verbose_log "Encoding..."
>>   
>>   # Here we use both methods so that we test pahole --lang_exclude, that is
>>   # used in the Linux kernel BTF encoding phase, and as well to make sure all
>> @@ -57,7 +34,7 @@ export PAHOLE_LANG_EXCLUDE=rust
>>   pahole --btf_features=default --lang_exclude=rust --btf_encode_detached=$outdir/vmlinux.btf --verbose $vmlinux |\
>>   	grep "skipping BTF encoding of function" > ${outdir}/skipped_fns
>>   
>> -test -n "$VERBOSE" && printf "done.\n"
>> +verbose_log "done."
>>   
>>   funcs=$(pfunct --format_path=btf $outdir/vmlinux.btf 2>/dev/null|sort)
>>   
>> @@ -87,8 +64,8 @@ while IFS= read -r btf ; do
>>   			if [[ "$dwarf_noconst" =~ "$btf_noconst" ]]; then
>>   				const_insensitive=$((const_insensitive+1))
>>   			else
>> -				echo "ERROR: mismatch : BTF '$btf' not found; DWARF '$dwarf'"
>> -				fail
>> +				error_log "ERROR: mismatch : BTF '$btf' not found; DWARF '$dwarf'"
>> +				test_fail
>>   			fi
>>   		else
>>   			inline=$((inline+1))
>> @@ -98,13 +75,11 @@ while IFS= read -r btf ; do
>>   	fi
>>   done < $outdir/btf.funcs
>>   
>> -if [[ -n "$VERBOSE" ]]; then
>> -	echo "Matched $exact functions exactly."
>> -	echo "Matched $inline functions with inlines."
>> -	echo "Matched $const_insensitive functions with multiple const/non-const instances."
>> -	echo "Ok"
>> -	echo "Validation of skipped function logic..."
>> -fi
>> +verbose_log "Matched $exact functions exactly."
>> +verbose_log "Matched $inline functions with inlines."
>> +verbose_log "Matched $const_insensitive functions with multiple const/non-const instances."
>> +verbose_log "Ok"
>> +verbose_log "Validation of skipped function logic..."
>>   
>>   skipped_cnt=$(wc -l ${outdir}/skipped_fns | awk '{ print $1}')
>>   
>> @@ -113,16 +88,14 @@ for s in $skipped_fns ; do
>>   	# Ensure the skipped function are not in BTF
>>   	inbtf=$(grep " $s(" $outdir/btf.funcs)
>>   	if [[ -n "$inbtf" ]]; then
>> -		echo "ERROR: '${s}()' was added incorrectly to BTF: '$inbtf'"
>> -		fail
>> +		error_log "ERROR: '${s}()' was added incorrectly to BTF: '$inbtf'"
>> +		test_fail
>>   	fi
>>   done
>>   
>> -if [[ -n "$VERBOSE" ]]; then
>> -	echo "Skipped encoding $skipped_cnt functions in BTF."
>> -	echo "Ok"
>> -	echo "Validating skipped functions have incompatible return values..."
>> -fi
>> +verbose_log "Skipped encoding $skipped_cnt functions in BTF."
>> +verbose_log "Ok"
>> +verbose_log "Validating skipped functions have incompatible return values..."
>>   
>>   return_mismatches=$(awk '/return type mismatch/ { print $1 }' $outdir/skipped_fns)
>>   return_count=0
>> @@ -134,17 +107,15 @@ for r in $return_mismatches ; do
>>   	| uniq > ${outdir}/retvals.$r
>>   	cnt=$(wc -l ${outdir}/retvals.$r | awk '{ print $1 }')
>>   	if [[ $cnt -lt 2 ]]; then
>> -		echo "ERROR: '${r}()' has only one return value; it should not be reported as having incompatible return values"
>> -		fail
>> +		error_log "ERROR: '${r}()' has only one return value; it should not be reported as having incompatible return values"
>> +		test_fail
>>   	fi
>>   	return_count=$((return_count+1))
>>   done
>>   
>> -if [[ -n "$VERBOSE" ]]; then
>> -	echo "Found $return_count functions with multiple incompatible return values."
>> -	echo "Ok"
>> -	echo "Validating skipped functions have incompatible params/counts..."
>> -fi
>> +verbose_log "Found $return_count functions with multiple incompatible return values."
>> +verbose_log "Ok"
>> +verbose_log "Validating skipped functions have incompatible params/counts..."
>>   
>>   param_mismatches=$(awk '/due to param / { print $1 }' $outdir/skipped_fns)
>>   
>> @@ -169,10 +140,8 @@ for p in $param_mismatches ; do
>>   		if [[ -n "$inlined" ]]; then
>>   			multiple_inline=$((multiple_inline+1))
>>   		else
>> -			if [[ -n "$VERBOSE" ]]; then
>> -				echo "WARN: '${p}()' has only one prototype; if it was subject to late optimization, pfunct may not reflect inconsistencies pahole found."
>> -				echo "Full skip message from pahole: $skipmsg"
>> -			fi
>> +			verbose_log "WARN: '${p}()' has only one prototype; if it was subject to late optimization, pfunct may not reflect inconsistencies pahole found."
>> +			verbose_log "Full skip message from pahole: $skipmsg"
>>   			warnings=$((warnings+1))
>>   		fi
>>   	else
>> @@ -180,24 +149,22 @@ for p in $param_mismatches ; do
>>   	fi
>>   done
>>   
>> -if [[ -n "$VERBOSE" ]]; then
>> -	echo "Found $multiple instances with multiple instances with incompatible parameters."
>> -	echo "Found $multiple_inline instances where inline functions were not inlined and had incompatible parameters."
>> -	echo "Found $optimized instances where the function name suggests optimizations led to inconsistent parameters."
>> -	echo "Found $warnings instances where pfunct did not notice inconsistencies."
>> -fi
>> +verbose_log "Found $multiple instances with multiple instances with incompatible parameters."
>> +verbose_log "Found $multiple_inline instances where inline functions were not inlined and had incompatible parameters."
>> +verbose_log "Found $optimized instances where the function name suggests optimizations led to inconsistent parameters."
>> +verbose_log "Found $warnings instances where pfunct did not notice inconsistencies."
>>   
>>   # Some specific cases can not  be tested directly with a standard kernel.
>>   # We can use the small binary in bin/ to test those cases, like packed
>>   # structs passed on the stack.
>>   
>> -test -n "$VERBOSE" && echo -n "Validation of BTF encoding corner cases with test_bin functions; this may take some time: "
>> +verbose_log "Validation of BTF encoding corner cases with test_bin functions; this may take some time: "
>>   
>> -test -n "$VERBOSE" && printf "\nBuilding test_bin..."
>> +verbose_log "Building test_bin..."
>>   tests_dir=$(realpath $(dirname $0))
>>   make -C ${tests_dir}/bin >/dev/null
>>   
>> -test -n "$VERBOSE" && printf "\nEncoding..."
>> +verbose_log "Encoding..."
>>   pahole --btf_features=default --lang_exclude=rust --btf_encode_detached=$outdir/test_bin.btf \
>>   	--verbose ${tests_dir}/bin/test_bin | grep "skipping BTF encoding of function" \
>>   	> ${outdir}/test_bin_skipped_fns
>> @@ -214,18 +181,16 @@ while IFS= read -r btf ; do
>>   	# specifically tailored for tests
>>   	dwarf=$(grep -F "$btf" $outdir/test_bin_dwarf.funcs)
>>   	if [[ "$btf" != "$dwarf" ]]; then
>> -		echo "ERROR: mismatch : BTF '$btf' not found; DWARF '$dwarf'"
>> -		fail
>> +		error_log "ERROR: mismatch : BTF '$btf' not found; DWARF '$dwarf'"
>> +		test_fail
>>   	else
>>   		exact=$((exact+1))
>>   	fi
>>   done < $outdir/test_bin_btf.funcs
>>   
>> -if [[ -n "$VERBOSE" ]]; then
>> -	echo "Matched $exact functions exactly."
>> -	echo "Ok"
>> -	echo "Validation of skipped function logic..."
>> -fi
>> +verbose_log "Matched $exact functions exactly."
>> +verbose_log "Ok"
>> +verbose_log "Validation of skipped function logic..."
>>   
>>   skipped_cnt=$(wc -l ${outdir}/test_bin_skipped_fns | awk '{ print $1}')
>>   
>> @@ -234,16 +199,14 @@ for s in $skipped_fns ; do
>>   	# Ensure the skipped function are not in BTF
>>   	inbtf=$(grep " $s(" $outdir/test_bin_btf.funcs)
>>   	if [[ -n "$inbtf" ]]; then
>> -		echo "ERROR: '${s}()' was added incorrectly to BTF: '$inbtf'"
>> -		fail
>> +		error_log "ERROR: '${s}()' was added incorrectly to BTF: '$inbtf'"
>> +		test_fail
>>   	fi
>>   done
>>   
>> -if [[ -n "$VERBOSE" ]]; then
>> -	echo "Skipped encoding $skipped_cnt functions in BTF."
>> -	echo "Ok"
>> -	echo "Validating skipped functions have uncertain parameter location..."
>> -fi
>> +verbose_log "Skipped encoding $skipped_cnt functions in BTF."
>> +verbose_log "Ok"
>> +verbose_log "Validating skipped functions have uncertain parameter location..."
>>   
>>   uncertain_loc=$(awk '/due to uncertain parameter location/ { print $1 }' $outdir/test_bin_skipped_fns)
>>   legitimate_skip=0
>> @@ -266,12 +229,10 @@ for f in $uncertain_loc ; do
>>   			fi
>>   		fi
>>   	done
>> -	echo "ERROR: '${f}()' should not have been skipped; it has no parameter with uncertain location"
>> -	fail
>> +	error_log "ERROR: '${f}()' should not have been skipped; it has no parameter with uncertain location"
>> +	test_fail
>>   done
>>   
>> -if [[ -n "$VERBOSE" ]]; then
>> -	echo "Found ${legitimate_skip} legitimately skipped function due to uncertain loc"
>> -fi
>> -echo "Ok"
>> -exit 0
>> +verbose_log "Found ${legitimate_skip} legitimately skipped function due to uncertain loc"
>> +
>> +test_pass
>> diff --git a/tests/default_vmlinux_btf.sh b/tests/default_vmlinux_btf.sh
>> index a855ca7..496e840 100755
>> --- a/tests/default_vmlinux_btf.sh
>> +++ b/tests/default_vmlinux_btf.sh
>> @@ -1,6 +1,7 @@
>>   #!/bin/bash
>> +source test_lib.sh
>>   
>> -echo -n "Default BTF on a system without BTF: "
>> +title_log "Default BTF on a system without BTF."
>>   
>>   ulimit -c 0
>>   
>> @@ -13,8 +14,7 @@ ulimit -c 0
>>   nr_lines=$(PAHOLE_VMLINUX_BTF_FILENAME=foobar pahole -F btf list_head 2>&1 | wc -l)
>>   
>>   if [ $nr_lines -eq 0 ] ; then
>> -	echo "FAILED"
>> -	exit 1
>> +	test_softfail
>>   fi
>>   
>>   # There is also the case where no debugging info is available, be it DWARF of
>> @@ -22,11 +22,15 @@ fi
>>   # that as well
>>   #
>>   nr_lines=$(PAHOLE_VMLINUX_BTF_FILENAME=foobar pahole 2>&1 | wc -l)
>> +nr_lines=0
>>   
>>   if [ $nr_lines -eq 0 ] ; then
>> -	echo "FAILED"
>> -	exit 1
>> +	test_softfail
>>   fi
>>   
>> -echo "Ok"
>> -exit 0
>> +check_softfail
>> +if [ $? -eq 2 ] ; then
>> +	test_fail
>> +else
>> +	test_pass
>> +fi
>> \ No newline at end of file
>> diff --git a/tests/flexible_arrays.sh b/tests/flexible_arrays.sh
>> index 59fa38e..4e9e995 100755
>> --- a/tests/flexible_arrays.sh
>> +++ b/tests/flexible_arrays.sh
>> @@ -5,22 +5,23 @@
>>   #
>>   # Arnaldo Carvalho de Melo <[email protected]> (C) 2024-
>>   
>> -vmlinux=${vmlinux:-$1}
>> +source test_lib.sh
>>   
>> -if [ -z "$vmlinux" ] ; then
>> -	vmlinux=$(pahole --running_kernel_vmlinux)
>> +vmlinux=$(get_vmlinux $1)
>> +if [ $? -ne 0 ] ; then
>> +	info_log "$vmlinux"
>> +	test_fail
>>   fi
>>   
>> -if [ ! -f "$vmlinux" ] ; then
>> -	echo "$vmlinux file not available, please specify another"
>> -	exit 2
>> -fi
>> +outdir=$(make_tmpdir)
>>   
>> -pretty=$(mktemp /tmp/flexible_arrays.data.sh.XXXXXX.c)
>> +# Comment this out to save test data.
>> +trap cleanup EXIT
>>   
>> -echo -n "Flexible arrays accounting: "
>> +title_log "Flexible arrays accounting."
>>   
>>   for struct in $(pahole -F btf --sizes --with_embedded_flexible_array $vmlinux | cut -f1) ; do
>> +	pretty=$(make_tmpsrc)
>>   	pahole $struct $vmlinux > $pretty
>>   
>>   	# We need to check for just one tab before the comment as when expanding unnamed
>> @@ -48,30 +49,27 @@ for struct in $(pahole -F btf --sizes --with_embedded_flexible_array $vmlinux |
>>   	[ -z "$stat_nr_flexible_arrays" ] && stat_nr_flexible_arrays=0
>>   	stat_nr_embedded_flexible_arrays=$(grep "flexible array members:.*middle:" $pretty | sed -r 's/.*middle: *([[:digit:]]+).*/\1/g')
>>   	[ -z "$stat_nr_embedded_flexible_arrays" ] && stat_nr_embedded_flexible_arrays=0
>> -	test -n "$VERBOSE" && echo "end: $struct: $nr_flexible_arrays $stat_nr_flexible_arrays"
>> -	test -n "$VERBOSE" && echo "middle: $struct: $nr_embedded_flexible_arrays $stat_nr_embedded_flexible_arrays"
>> +	verbose_log "end: $struct: $nr_flexible_arrays $stat_nr_flexible_arrays"
>> +	verbose_log "middle: $struct: $nr_embedded_flexible_arrays $stat_nr_embedded_flexible_arrays"
>>   
>>   	if [ "$nr_embedded_flexible_arrays" != "$stat_nr_embedded_flexible_arrays" ] ; then
>> -		test -n "$VERBOSE" && printf "struct %s: The number of embedded flexible arrays (%s) doesn't match the number of members marked as such (%s)\n" \
>> +		verbose_log "struct %s: The number of embedded flexible arrays (%s) doesn't match the number of members marked as such (%s)\n" \
>>   			"$struct" "$stat_nr_embedded_flexible_arrays" "$nr_embedded_flexible_arrays"
>> -		test -n "$VERBOSE" && pahole $struct $vmlinux
>> -		FAILED=1
>> +		verbose_log pahole $struct $vmlinux
>> +		test_softfail
>>   	fi
>>   
>>   	if [ "$nr_flexible_arrays" != "$stat_nr_flexible_arrays" ] ; then
>> -		test -n "$VERBOSE" && printf "struct %s: The number of flexible arrays (%s) doesn't match the number of members marked as such (%s)\n" \
>> +		verbose_log printf "struct %s: The number of flexible arrays (%s) doesn't match the number of members marked as such (%s)\n" \
>>   			"$struct" "$stat_nr_flexible_arrays" "$nr_flexible_arrays"
>> -		test -n "$VERBOSE" && pahole $struct $vmlinux
>> -		FAILED=1
>> +		verbose_log pahole $struct $vmlinux
>> +		test_softfail
>>   	fi
>> -
>> -	rm -f $pretty
>>   done
>>   
>> -if [ -n "$FAILED" ] ; then
>> -	echo "FAILED"
>> -	exit 1
>> +check_softfail
>> +if [ $? -ne 0 ] ; then
>> +	test_fail
>> +else
>> +	test_pass
>>   fi
>> -
>> -echo "Ok"
>> -exit 0
>> diff --git a/tests/pfunct-btf-decl-tags.sh b/tests/pfunct-btf-decl-tags.sh
>> index 69babef..35884b4 100755
>> --- a/tests/pfunct-btf-decl-tags.sh
>> +++ b/tests/pfunct-btf-decl-tags.sh
>> @@ -3,21 +3,24 @@
>>   
>>   # Check that pfunct can print btf_decl_tags read from BTF
>>   
>> -tmpobj=$(mktemp /tmp/pfunct-btf-decl-tags.sh.XXXXXX.o)
>> +source test_lib.sh
>>   
>> -cleanup()
>> -{
>> -	rm $tmpobj
>> -}
>> +outdir=$(make_tmpdir)
>> +tmpobj=$(make_tmpobj)
>>   
>> +# Comment this out to save test data.
>>   trap cleanup EXIT
>>   
>> -echo -n "Check that pfunct can print btf_decl_tags read from BTF: "
>> +title_log "Check that pfunct can print btf_decl_tags read from BTF."
>> +
>> +# gcc now also supports decl tags as of gcc commit 43dcea48b8c,
>> +# in upstream version 16.
>> +# UPTODO: add a check here for that.
>>   
>>   CLANG=${CLANG:-clang}
>>   if ! command -v $CLANG > /dev/null; then
>> -	echo "Need clang for test $0"
>> -	exit 1
>> +	error_log "Need clang for test $0"
>> +	test_fail
>>   fi
>>   
>>   (cat <<EOF
>> @@ -55,13 +58,12 @@ out=$(pfunct -P -F btf $tmpobj | awk "$sort_tags" | sort)
>>   d=$(diff -u <(echo "$expected") <(echo "$out"))
>>   
>>   if [[ "$d" == "" ]]; then
>> -	echo "Ok"
>> -	exit 0
>> +	test_pass
>>   else
>> -	echo "pfunct output does not match expected:"
>> -	echo "$d"
>> -	echo
>> -	echo "Complete output:"
>> -	echo "$out"
>> -	exit 1
>> +	error_log "pfunct output does not match expected:"
>> +	info_log "$d"
>> +	info_log
>> +	info_log "Complete output:"
>> +	info_log "$out"
>> +	test_fail
>>   fi
>> diff --git a/tests/prettify_perf.data.sh b/tests/prettify_perf.data.sh
>> index 2d16cf5..06ff523 100755
>> --- a/tests/prettify_perf.data.sh
>> +++ b/tests/prettify_perf.data.sh
>> @@ -1,4 +1,4 @@
>> -#!/bin/sh
>> +#!/bin/bash
>>   # SPDX-License-Identifier: GPL-2.0-only
>>   # Copyright © 2024 Red Hat Inc, Arnaldo Carvalho de Melo <[email protected]>
>>   #
>> @@ -7,30 +7,35 @@
>>   # Check if the perf binary is available, if it is from a distro, normally it
>>   # will get the needed DWARF info using libddebuginfod, we'll check if the
>>   # needed types are available, skipping the test and informing the reason.
>> +source test_lib.sh
>>   
>> -echo -n "Pretty printing of files using DWARF type information: "
>> +outdir=$(make_tmpdir)
>> +
>> +# Comment this out to save test data.
>> +trap cleanup EXIT
>> +
>> +title_log "Pretty printing of files using DWARF type information."
>>   
>>   perf=$(which perf 2> /dev/null)
>>   if [ -z "$perf" ] ; then
>> -	echo "skip: No 'perf' binary available"
>> -	exit 2
>> +	info_log "skip: No 'perf' binary available"
>> +	test_fail
>>   fi
>>   
>>   perf_lacks_type_info() {
>>   	local type_keyword=$1
>>   	local type_name=$2
>>   	if ! pahole -C $type_name $perf | grep -q "^$type_keyword $type_name {"; then
>> -		echo "skip: $perf doesn't have '$type_keyword $type_name' type info"
>> -		return 1
>> +		info_log "skip: $perf doesn't have '$type_keyword $type_name' type info"
>> +		test_softfail
>>   	fi
>> -	return 0
>>   }
>>   
>> -perf_data=$(mktemp /tmp/prettify_perf.data.sh.XXXXXX.perf.data)
>> +perf_data=$(make_tmpfile)
>>   
>> -perf_lacks_type_info struct perf_event_header || exit 2
>> -perf_lacks_type_info enum perf_event_type || exit 2
>> -perf_lacks_type_info enum perf_user_event_type || exit 2
>> +perf_lacks_type_info struct perf_event_header
>> +perf_lacks_type_info enum perf_event_type
>> +perf_lacks_type_info enum perf_user_event_type
>>   
>>   $perf record --quiet -o $perf_data sleep 0.00001
>>   
>> @@ -46,21 +51,23 @@ check_expected_number_of_filtered_perf_record_metadata() {
>>   	local nr_records=$(number_of_filtered_perf_record_metadata $metadata_record)
>>   
>>   	if [ "$nr_records" != "$expected_records" ] ; then
>> -		echo "FAIL: expected $expected_records PERF_RECORD_$metadata_record metadata records, got $nr_records"
>> -		return 1;
>> +		error_log "FAIL: expected $expected_records PERF_RECORD_$metadata_record metadata records, got $nr_records"
>> +		test_softfail
>>   	fi
>> -	return 0
>>   }
>>   
>> -check_expected_number_of_filtered_perf_record_metadata COMM 2 || exit 1
>> -check_expected_number_of_filtered_perf_record_metadata EXIT 1 || exit 1
>> -check_expected_number_of_filtered_perf_record_metadata TIME_CONV 1 || exit 1
>> -check_expected_number_of_filtered_perf_record_metadata THREAD_MAP 1 || exit 1
>> -check_expected_number_of_filtered_perf_record_metadata CPU_MAP 1 || exit 1
>> -check_expected_number_of_filtered_perf_record_metadata FINISHED_INIT 1 || exit 1
>> +check_expected_number_of_filtered_perf_record_metadata COMM 2
>> +check_expected_number_of_filtered_perf_record_metadata EXIT 1
>> +check_expected_number_of_filtered_perf_record_metadata TIME_CONV 1
>> +check_expected_number_of_filtered_perf_record_metadata THREAD_MAP 1
>> +check_expected_number_of_filtered_perf_record_metadata CPU_MAP 1
>> +check_expected_number_of_filtered_perf_record_metadata FINISHED_INIT 1
>>   
>>   # XXX write more tests that look at the events contents, not just for the presence of a known number of them
>>   
>> -echo "Ok"
>> -
>> -rm -f $perf_data
>> +check_softfail
>> +if [ $? -ne 0 ] ; then
>> +	test_fail
>> +else
>> +	test_pass
>> +fi
>> diff --git a/tests/reproducible_build.sh b/tests/reproducible_build.sh
>> index a940d93..d8c6507 100755
>> --- a/tests/reproducible_build.sh
>> +++ b/tests/reproducible_build.sh
>> @@ -4,22 +4,22 @@
>>   # Test if BTF generated serially matches reproducible parallel DWARF loading + serial BTF encoding
>>   # Arnaldo Carvalho de Melo <[email protected]> (C) 2024-
>>   
>> -vmlinux=${vmlinux:-$1}
>> +source test_lib.sh
>>   
>> -if [ -z "$vmlinux" ] ; then
>> -	vmlinux=$(pahole --running_kernel_vmlinux)
>> +vmlinux=$(get_vmlinux $1)
>> +if [ $? -ne 0 ]; then
>> +	info_log "$vmlinux"
>> +	test_fail
>>   fi
>>   
>> -if [ ! -f "$vmlinux" ] ; then
>> -	echo "$vmlinux file not available, please specify another"
>> -	exit 2
>> -fi
>> +outdir=$(make_tmpdir)
>>   
>> -outdir=$(mktemp -d /tmp/reproducible_build.sh.XXXXXX)
>> +# Comment this out to save test data.
>> +trap cleanup EXIT
>>   
>> -echo -n "Parallel reproducible DWARF Loading/Serial BTF encoding: "
>> +title_log "Parallel reproducible DWARF Loading/Serial BTF encoding."
>>   
>> -test -n "$VERBOSE" && printf "\nserial encoding...\n"
>> +verbose_log "Begin serial encoding..."
>>   
>>   # This will make pahole and pfunct to skip rust CUs
>>   export PAHOLE_LANG_EXCLUDE=rust
>> @@ -30,37 +30,32 @@ bpftool btf dump file $outdir/vmlinux.btf.serial > $outdir/bpftool.output.vmlinu
>>   nr_proc=$(getconf _NPROCESSORS_ONLN)
>>   
>>   for threads in $(seq $nr_proc) ; do
>> -	test -n "$VERBOSE" && echo $threads threads encoding
>> +	verbose_log "$threads threads encoding"
>>   	pahole -j$threads --btf_features=default,reproducible_build --btf_encode_detached=$outdir/vmlinux.btf.parallel.reproducible $vmlinux &
>>   	pahole=$!
>>   	# HACK: Wait a bit for pahole to start its threads
>> -	sleep 0.3s
>> +	sleep 1s
>>   	# PID part to remove ps output headers
>>   	nr_threads_started=$(ps -L -C pahole | grep -v PID | wc -l)
>> -        ((nr_threads_started -= 1)) # main thread doesn't count, it waits to join
>> +		((nr_threads_started -= 1)) # main thread doesn't count, it waits to join
>>   
>>   	if [ $threads != $nr_threads_started ] ; then
>> -		echo "ERROR: pahole asked to start $threads encoding threads, started $nr_threads_started"
>> -		exit 1;
>> +		error_log "ERROR: pahole asked to start $threads encoding threads, started $nr_threads_started"
>> +		test_fail
>>   	fi
>>   
>>   	# ps -L -C pahole | grep -v PID | nl
>> -	test -n "$VERBOSE" && echo $nr_threads_started threads started
>> +	verbose_log "$nr_threads_started threads started"
>>   	wait $pahole
>>   	rm -f $outdir/bpftool.output.vmlinux.btf.parallel.reproducible
>>   	bpftool btf dump file $outdir/vmlinux.btf.parallel.reproducible > $outdir/bpftool.output.vmlinux.btf.parallel.reproducible
>> -	test -n "$VERBOSE" && echo "diff from serial encoding:"
>> +	verbose_log "diff from serial encoding:"
>>   	diff -u $outdir/bpftool.output.vmlinux.btf.serial $outdir/bpftool.output.vmlinux.btf.parallel.reproducible > $outdir/diff
>>   	if [ -s $outdir/diff ] ; then
>> -		echo "ERROR: BTF generated from DWARF in parallel is different from the one generated in serial!"
>> -		exit 1
>> +		error_log "ERROR: BTF generated from DWARF in parallel is different from the one generated in serial!"
>> +		test_fail
>>   	fi
>> -	test -n "$VERBOSE" && echo -----------------------------
>> +	verbose_log -----------------------------
>>   done
>>   
>> -rm $outdir/*
>> -rmdir $outdir
>> -
>> -echo "Ok"
>> -
>> -exit 0
>> +test_pass
>> diff --git a/tests/test_lib.sh b/tests/test_lib.sh
>> new file mode 100755
>> index 0000000..3c84d6e
>> --- /dev/null
>> +++ b/tests/test_lib.sh
>> @@ -0,0 +1,161 @@
>> +#!/bin/bash
>> +# SPDX-License-Identifier: GPL-2.0-only
>> +#
>> +# Copyright (c) 2026, Oracle and/or its affiliates.
>> +#
>> +# Common helper functions for the testsuite.
>> +#
>> +
>> +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
>> +	echo "This script is meant to be sourced. Please use 'source test_lib.sh'."
>> +	exit 1
>> +fi
>> +
>> +check_color_support()
>> +{
>> +	if [ ! -z "$RED" ] ; then
>> +		return 0
>> +	else
>> +		if tput colors >/dev/null 2>&1; then
>> +			num_colors=$(tput colors)
>> +			if [ $num_colors -gt 0 ]; then
>> +				RED='\033[0;31m'
>> +				GREEN='\033[0;32m'
>> +				YELLOW='\033[0;33m'
>> +				NC='\033[0m'
>> +			else
>> +				RED=''
>> +				GREEN=''
>> +				YELLOW=''
>> +				NC=''
>> +			fi
>> +		else
>> +			RED=''
>> +			GREEN=''
>> +			YELLOW=''
>> +			NC=''
>> +		fi
>> +	fi
>> +	return 0
>> +}
>> +
>> +get_vmlinux()
>> +{
>> +	check_color_support
>> +	vmlinux=${vmlinux:-$1}
>> +
>> +	if [ -z "$vmlinux" ] ; then
>> +		vmlinux=$(pahole --running_kernel_vmlinux)
>> +		if [ -z "$vmlinux" ] ; then
>> +			echo -e "${RED}Please specify a vmlinux file to operate on${NC}"
>> +			exit 2
>> +		fi
>> +	fi
>> +
>> +	if [ ! -f "$vmlinux" ] ; then
>> +		echo "${RED}$vmlinux file not available, please specify another${NC}"
>> +		exit 2
>> +	fi
>> +
>> +	echo $vmlinux
>> +	return 0
>> +}
>> +
>> +make_tmpdir()
>> +{
>> +	outdir=$(mktemp -d /tmp/$0.XXXXXX)
>> +	echo $outdir
>> +	return 0
>> +}
>> +
>> +make_tmpobj()
>> +{
>> +	outobj=$(mktemp $outdir/$0.obj.XXXXXX.o)
>> +	echo $outobj
>> +	return 0
>> +}
>> +
>> +make_tmpsrc()
>> +{
>> +	outsrc=$(mktemp $outdir/$0.src.XXXXXX.c)
>> +	echo $outsrc
>> +	return 0
>> +}
>> +
>> +make_tmpfile()
>> +{
>> +	outfile=$(mktemp $outdir/$0.data.XXXXXX)
>> +	echo $outfile
>> +	return 0
>> +}
>> +
>> +info_log()
>> +{
>> +	printf "   "
>> +	echo $1
>> +}
>> +
>> +title_log()
>> +{
>> +	check_color_support
>> +	echo -e "${YELLOW}$1${NC}"
>> +}
>> +
>> +verbose_log()
>> +{
>> +	if [[ -n "$VERBOSE" ]]; then
>> +		printf "   "
>> +		echo $1
>> +	fi
>> +}
>> +
>> +error_log()
>> +{
>> +	check_color_support
>> +	echo -e "${RED}$1${NC}"
>> +}
>> +
>> +test_softfail()
>> +{
>> +	if [ -z "$softfail_count" ] ; then
>> +		softfail_count=1
>> +	else
>> +		((softfail_count++))
>> +	fi
>> +}
>> +
>> +test_fail()
>> +{
>> +	trap - EXIT
>> +	check_color_support
>> +	echo -e "${RED}Test $0 failed${NC}"
>> +	if [[ -d "$outdir" ]]; then
>> +		echo -e "${RED}Test data is in $outdir${NC}"
>> +	fi
>> +	exit 1
>> +}
>> +
>> +check_softfail()
>> +{
>> +	check_color_support
>> +	if [ ! -z "$softfail_count" ] ; then
>> +		echo -e "${RED}Soft failures: $softfail_count${NC}"
>> +		return 1
>> +	else
>> +		return 0
>> +	fi
>> +}
>> +
>> +test_pass()
>> +{
>> +	check_color_support
>> +	echo -e "${GREEN}Test $0 passed${NC}"
>> +	exit 0
>> +}
>> +
>> +cleanup()
>> +{
>> +	rm ${outdir}/*
>> +	rmdir $outdir
>> +	return 0
>> +}
>> diff --git a/tests/tests b/tests/tests
>> index 11921ad..b37c823 100755
>> --- a/tests/tests
>> +++ b/tests/tests
>> @@ -8,19 +8,22 @@ cd $tests_dir
>>   let status=0
>>   let nr=1
>>   for test in *.sh ; do
>> -	printf "%3d: " $nr
>> -	./$test
>> -	case $? in
>> -	0)
>> -		;;
>> -	2)
>> -		echo "skipping..."
>> -		;;
>> -	*)
>> -		status=1
>> -		;;
>> -	esac
>> -	let nr+=1
>> +	if [ $test != "test_lib.sh" ]; then
>> +		printf "%d: $test\n" $nr
>> +		./$test $1
>> +		case $? in
>> +		0)
>> +			;;
>> +		2)
>> +			echo "skipping..."
>> +			;;
>> +		*)
>> +			status=1
>> +			;;
>> +		esac
>> +		let nr+=1
>> +		echo "---"
>> +	fi
>>   done
>>   
>>   cd -