[gcc r17-2956] Run the bootstrap object comparison in parallel
Kyrylo Tkachov via Gcc-cvs <[email protected]> Tue, 4 Aug 2026 17:07:38 +0000 (GMT)
| Newsgroups | gmane.comp.gcc.cvs |
|---|---|
| Message-ID | <[email protected]> |
https://gcc.gnu.org/g:54c3bdc8ad643f3d8692cfd9d738ce2aed935b3f commit r17-2956-g54c3bdc8ad643f3d8692cfd9d738ce2aed935b3f Author: Kyrylo Tkachov <[email protected]> Date: Mon Jul 6 02:42:14 2026 -0700 Run the bootstrap object comparison in parallel The stage 2 and stage 3 object comparison runs one command per object in a serial shell loop. Everything after it waits while an otherwise idle machine compares thousands of objects. Generate one make target per object and run the comparisons through $(MAKE), so they share the original jobserver. Keep the generated makefile and result shards invocation-specific, clean them on normal and signal exits, and sort the shards before creating the traditional .bad_compare diagnostic. Files in the exclusion list still only warn. Comparator statuses other than zero or one now report an operational error instead of being treated as equal. The tail-based fallback for cmp implementations without byte skipping used fixed temporary names. Give those files process-specific names, clean them with a trap, and preserve the same status contract for tail failures and signals. On highly parallel AArch64 and x86_64 systems this reduces comparison wall time by about 95%. ChangeLog: * Makefile.tpl ([+compare-target+]): Run object comparisons through an invocation-specific parallel sub-make. Collect sorted failure shards and clean temporary state on exit. * Makefile.in: Regenerate. * configure: Regenerate. config/ChangeLog: * acx.m4 (ACX_PROG_CMP_IGNORE_INITIAL): Use process-specific temporary files and clean them on exit. Diagnose tail failures as errors. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- Makefile.in | 116 ++++++++++++++++++++++++++++++++++++++++++---------------- Makefile.tpl | 58 +++++++++++++++++++++-------- config/acx.m4 | 11 +++++- configure | 11 +++++- 4 files changed, 146 insertions(+), 50 deletions(-) diff --git a/Makefile.in b/Makefile.in index f34142cc077d..5cca69c3f523 100644 --- a/Makefile.in +++ b/Makefile.in @@ -66022,6 +66022,9 @@ do-clean: clean-stage3 # only possibility, but now it conflicts with no-bootstrap rules @if gcc-bootstrap +# Run the comparisons in parallel through a generated sub-makefile under +# the jobserver. Each failing recipe writes a separate result shard that +# is collected after the sub-make finishes. compare: @r=`${PWD_COMMAND}`; export r; \ s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \ @@ -66030,29 +66033,52 @@ compare: exit 0; \ fi; \ : $(MAKE); $(stage); \ - rm -f .bad_compare; \ + compare_id=$$$$; \ + bad_compare=.bad_compare.$$compare_id; export bad_compare; \ + compare_makefile=compare.$$compare_id.mk; \ + trap 'st=$$?; rm -f "$$compare_makefile" "$$bad_compare" \ + "$$bad_compare".*; trap - 0; exit $$st' 0; \ + trap 'exit 1' 1 2 3 15; \ echo Comparing stages 2 and 3; \ sed=`echo stage3 | sed 's,^stage,,;s,.,.,g'`; \ files=`find stage3-* -name "*$(objext)" -print | \ sed -n s,^stage$$sed-,,p`; \ - for file in $${files} ${extra-compare}; do \ - f1=$$r/stage2-$$file; f2=$$r/stage3-$$file; \ - if test ! -f $$f1; then continue; fi; \ - $(do-compare) > /dev/null 2>&1; \ - if test $$? -eq 1; then \ - case $$file in \ - @compare_exclusions@) \ - echo warning: $$file differs ;; \ - *) \ - echo $$file differs >> .bad_compare ;; \ - esac; \ - fi; \ - done; \ - if [ -f .bad_compare ]; then \ + cmp_raw='$(do-compare)'; \ + { \ + echo 'all:'; \ + echo '.PHONY: all FORCE'; \ + echo 'FORCE:'; \ + printf 'compare/%%: FORCE ; @'; \ + printf 'f1=$$$$r/stage2-$$*; '; \ + printf 'f2=$$$$r/stage3-$$*; '; \ + printf '%s' "$$cmp_raw" | sed 's,\$$,$$$$,g'; \ + printf ' > /dev/null 2>&1; st=$$$$?; '; \ + printf 'if test $$$$st -eq 1; then '; \ + printf 'case $$* in '; \ + printf '@compare_exclusions@) echo warning: $$* differs ;; '; \ + printf '*) echo $$* differs >> "$$$$bad_compare.$$$$$$$$" ;; '; \ + printf 'esac; '; \ + printf 'elif test $$$$st -ne 0; then '; \ + printf 'echo "$$* compare: error status $$$$st" '; \ + printf '>> "$$$$bad_compare.$$$$$$$$"; fi\n'; \ + for file in $${files} ${extra-compare}; do \ + if test ! -f $$r/stage2-$$file; then continue; fi; \ + echo "all: compare/$$file"; \ + done; \ + } > "$$compare_makefile"; \ + $(MAKE) -s -f "$$compare_makefile" all; compare_status=$$?; \ + if test $$compare_status -ne 0; then \ + exit $$compare_status; \ + fi; \ + set -- "$$bad_compare".*; \ + if test -f "$$1"; then \ echo "Bootstrap comparison failure!"; \ - cat .bad_compare; \ + LC_ALL=C sort "$$bad_compare".* > "$$bad_compare" || exit 1; \ + cat "$$bad_compare"; \ + mv -f "$$bad_compare" .bad_compare; \ exit 1; \ else \ + rm -f .bad_compare; \ echo Comparison successful.; \ fi; \ $(STAMP) compare @@ -66456,6 +66482,9 @@ do-clean: clean-stage4 # only possibility, but now it conflicts with no-bootstrap rules @if gcc-bootstrap +# Run the comparisons in parallel through a generated sub-makefile under +# the jobserver. Each failing recipe writes a separate result shard that +# is collected after the sub-make finishes. compare3: @r=`${PWD_COMMAND}`; export r; \ s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \ @@ -66464,29 +66493,52 @@ compare3: exit 0; \ fi; \ : $(MAKE); $(stage); \ - rm -f .bad_compare; \ + compare_id=$$$$; \ + bad_compare=.bad_compare.$$compare_id; export bad_compare; \ + compare_makefile=compare3.$$compare_id.mk; \ + trap 'st=$$?; rm -f "$$compare_makefile" "$$bad_compare" \ + "$$bad_compare".*; trap - 0; exit $$st' 0; \ + trap 'exit 1' 1 2 3 15; \ echo Comparing stages 3 and 4; \ sed=`echo stage4 | sed 's,^stage,,;s,.,.,g'`; \ files=`find stage4-* -name "*$(objext)" -print | \ sed -n s,^stage$$sed-,,p`; \ - for file in $${files} ${extra-compare}; do \ - f1=$$r/stage3-$$file; f2=$$r/stage4-$$file; \ - if test ! -f $$f1; then continue; fi; \ - $(do-compare3) > /dev/null 2>&1; \ - if test $$? -eq 1; then \ - case $$file in \ - @compare_exclusions@) \ - echo warning: $$file differs ;; \ - *) \ - echo $$file differs >> .bad_compare ;; \ - esac; \ - fi; \ - done; \ - if [ -f .bad_compare ]; then \ + cmp_raw='$(do-compare3)'; \ + { \ + echo 'all:'; \ + echo '.PHONY: all FORCE'; \ + echo 'FORCE:'; \ + printf 'compare3/%%: FORCE ; @'; \ + printf 'f1=$$$$r/stage3-$$*; '; \ + printf 'f2=$$$$r/stage4-$$*; '; \ + printf '%s' "$$cmp_raw" | sed 's,\$$,$$$$,g'; \ + printf ' > /dev/null 2>&1; st=$$$$?; '; \ + printf 'if test $$$$st -eq 1; then '; \ + printf 'case $$* in '; \ + printf '@compare_exclusions@) echo warning: $$* differs ;; '; \ + printf '*) echo $$* differs >> "$$$$bad_compare.$$$$$$$$" ;; '; \ + printf 'esac; '; \ + printf 'elif test $$$$st -ne 0; then '; \ + printf 'echo "$$* compare: error status $$$$st" '; \ + printf '>> "$$$$bad_compare.$$$$$$$$"; fi\n'; \ + for file in $${files} ${extra-compare}; do \ + if test ! -f $$r/stage3-$$file; then continue; fi; \ + echo "all: compare3/$$file"; \ + done; \ + } > "$$compare_makefile"; \ + $(MAKE) -s -f "$$compare_makefile" all; compare_status=$$?; \ + if test $$compare_status -ne 0; then \ + exit $$compare_status; \ + fi; \ + set -- "$$bad_compare".*; \ + if test -f "$$1"; then \ echo "Bootstrap comparison failure!"; \ - cat .bad_compare; \ + LC_ALL=C sort "$$bad_compare".* > "$$bad_compare" || exit 1; \ + cat "$$bad_compare"; \ + mv -f "$$bad_compare" .bad_compare; \ exit 1; \ else \ + rm -f .bad_compare; \ echo Comparison successful.; \ fi; \ $(STAMP) compare3 diff --git a/Makefile.tpl b/Makefile.tpl index 5891b67b6974..98bd03e9a331 100644 --- a/Makefile.tpl +++ b/Makefile.tpl @@ -1821,6 +1821,9 @@ do-clean: clean-stage[+id+] # only possibility, but now it conflicts with no-bootstrap rules @if gcc-bootstrap [+ IF compare-target +] +# Run the comparisons in parallel through a generated sub-makefile under +# the jobserver. Each failing recipe writes a separate result shard that +# is collected after the sub-make finishes. [+compare-target+]: @r=`${PWD_COMMAND}`; export r; \ s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \ @@ -1829,29 +1832,52 @@ do-clean: clean-stage[+id+] exit 0; \ fi; \ : $(MAKE); $(stage); \ - rm -f .bad_compare; \ + compare_id=$$$$; \ + bad_compare=.bad_compare.$$compare_id; export bad_compare; \ + compare_makefile=[+compare-target+].$$compare_id.mk; \ + trap 'st=$$?; rm -f "$$compare_makefile" "$$bad_compare" \ + "$$bad_compare".*; trap - 0; exit $$st' 0; \ + trap 'exit 1' 1 2 3 15; \ echo Comparing stages [+prev+] and [+id+]; \ sed=`echo stage[+id+] | sed 's,^stage,,;s,.,.,g'`; \ files=`find stage[+id+]-* -name "*$(objext)" -print | \ sed -n s,^stage$$sed-,,p`; \ - for file in $${files} ${extra-compare}; do \ - f1=$$r/stage[+prev+]-$$file; f2=$$r/stage[+id+]-$$file; \ - if test ! -f $$f1; then continue; fi; \ - $(do-[+compare-target+]) > /dev/null 2>&1; \ - if test $$? -eq 1; then \ - case $$file in \ - @compare_exclusions@) \ - echo warning: $$file differs ;; \ - *) \ - echo $$file differs >> .bad_compare ;; \ - esac; \ - fi; \ - done; \ - if [ -f .bad_compare ]; then \ + cmp_raw='$(do-[+compare-target+])'; \ + { \ + echo 'all:'; \ + echo '.PHONY: all FORCE'; \ + echo 'FORCE:'; \ + printf '[+compare-target+]/%%: FORCE ; @'; \ + printf 'f1=$$$$r/stage[+prev+]-$$*; '; \ + printf 'f2=$$$$r/stage[+id+]-$$*; '; \ + printf '%s' "$$cmp_raw" | sed 's,\$$,$$$$,g'; \ + printf ' > /dev/null 2>&1; st=$$$$?; '; \ + printf 'if test $$$$st -eq 1; then '; \ + printf 'case $$* in '; \ + printf '@compare_exclusions@) echo warning: $$* differs ;; '; \ + printf '*) echo $$* differs >> "$$$$bad_compare.$$$$$$$$" ;; '; \ + printf 'esac; '; \ + printf 'elif test $$$$st -ne 0; then '; \ + printf 'echo "$$* compare: error status $$$$st" '; \ + printf '>> "$$$$bad_compare.$$$$$$$$"; fi\n'; \ + for file in $${files} ${extra-compare}; do \ + if test ! -f $$r/stage[+prev+]-$$file; then continue; fi; \ + echo "all: [+compare-target+]/$$file"; \ + done; \ + } > "$$compare_makefile"; \ + $(MAKE) -s -f "$$compare_makefile" all; compare_status=$$?; \ + if test $$compare_status -ne 0; then \ + exit $$compare_status; \ + fi; \ + set -- "$$bad_compare".*; \ + if test -f "$$1"; then \ echo "Bootstrap comparison failure!"; \ - cat .bad_compare; \ + LC_ALL=C sort "$$bad_compare".* > "$$bad_compare" || exit 1; \ + cat "$$bad_compare"; \ + mv -f "$$bad_compare" .bad_compare; \ exit 1; \ else \ + rm -f .bad_compare; \ echo Comparison successful.; \ fi; \ $(STAMP) [+compare-target+][+ IF prev +] diff --git a/config/acx.m4 b/config/acx.m4 index ef5468bdd356..5547cbeb9b8f 100644 --- a/config/acx.m4 +++ b/config/acx.m4 @@ -477,7 +477,16 @@ AC_DEFUN([ACX_PROG_CMP_IGNORE_INITIAL], [AC_CACHE_CHECK([how to compare bootstrapped objects], gcc_cv_prog_cmp_skip, [ echo abfoo >t1 echo cdfoo >t2 - gcc_cv_prog_cmp_skip='tail -c +17 $$f1 > tmp-foo1; tail -c +17 $$f2 > tmp-foo2; cmp tmp-foo1 tmp-foo2' + gcc_cv_prog_cmp_skip='(trap "st=\$$?; rm -f tmp-foo1.$$$$ ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tmp-foo2.$$$$; ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'trap - 0; exit \$$st" 0; ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'trap "exit 2" 1 2 3 15; ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tail -c +17 $$f1 ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'> tmp-foo1.$$$$ || exit 2; ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tail -c +17 $$f2 ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'> tmp-foo2.$$$$ || exit 2; ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'cmp tmp-foo1.$$$$ ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tmp-foo2.$$$$)' if cmp t1 t2 2 2 > /dev/null 2>&1; then if cmp t1 t2 1 1 > /dev/null 2>&1; then : diff --git a/configure b/configure index ad490083741e..4db0edbdccdb 100755 --- a/configure +++ b/configure @@ -6123,7 +6123,16 @@ if ${gcc_cv_prog_cmp_skip+:} false; then : else echo abfoo >t1 echo cdfoo >t2 - gcc_cv_prog_cmp_skip='tail -c +17 $$f1 > tmp-foo1; tail -c +17 $$f2 > tmp-foo2; cmp tmp-foo1 tmp-foo2' + gcc_cv_prog_cmp_skip='(trap "st=\$$?; rm -f tmp-foo1.$$$$ ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tmp-foo2.$$$$; ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'trap - 0; exit \$$st" 0; ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'trap "exit 2" 1 2 3 15; ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tail -c +17 $$f1 ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'> tmp-foo1.$$$$ || exit 2; ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tail -c +17 $$f2 ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'> tmp-foo2.$$$$ || exit 2; ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'cmp tmp-foo1.$$$$ ' + gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tmp-foo2.$$$$)' if cmp t1 t2 2 2 > /dev/null 2>&1; then if cmp t1 t2 1 1 > /dev/null 2>&1; then :