Re: gzimg hashes

Brook Milligan <[email protected]> Wed, 15 Jul 2026 11:54:46 -0600
Newsgroups	gmane.os.netbsd.current
Message-ID	<[email protected]>
--Apple-Mail=_7253959E-E726-4324-8310-38C5CA7AE055
Content-Transfer-Encoding: quoted-printable
Content-Type: text/plain;
	charset=us-ascii


> On Jul 14, 2026, at 06:30, David H. Gutteridge <[email protected]> =
wrote:
>=20
> Looks like this is a parallelism issue, as I suspected. My example =
above
> is with "-j 8", where these checksums are spun up in parallel quite
> quickly. So something would have to deal with this.

Yes, indeed a parallelism issue.  That is now fixed.

I believe this now fixes all the issues with creating hashes for =
multiple, compressed release files in the same directory, and does the =
same for all (that I know of) uses in the tree.

Please review the patch below.  Once this is committed, the mkhashes =
script will become an integral part of the release process.  Therefore, =
I am particularly interested that it be as portable as possible.  I =
believe I have picked up all the relevant tools that might differ among =
build environments, but more eyes would be helpful.

Thanks for any suggestions on offer.

Cheers,
Brook



--Apple-Mail=_7253959E-E726-4324-8310-38C5CA7AE055
Content-Disposition: attachment;
	filename=mkhashes.patch
Content-Type: application/octet-stream;
	x-unix-mode=0644;
	name="mkhashes.patch"
Content-Transfer-Encoding: 7bit

Index: distrib/utils/embedded/mkimage
===================================================================
RCS file: /cvsroot/src/distrib/utils/embedded/mkimage,v
retrieving revision 1.97
diff -u -r1.97 mkimage
--- distrib/utils/embedded/mkimage	22 May 2026 14:44:16 -0000	1.97
+++ distrib/utils/embedded/mkimage	15 Jul 2026 16:57:03 -0000
@@ -425,8 +425,4 @@
 	image="${image}.gz"
 fi
 
-cd "${IMAGEDIR}"
-${CKSUM} -a MD5 "$(basename "${image}")" > MD5
-${CKSUM} -a SHA512 "$(basename "${image}")" > SHA512
-
 echo ${bar} Image is ${image} ${bar}
Index: etc/etc.evbarm/Makefile.inc
===================================================================
RCS file: /cvsroot/src/etc/etc.evbarm/Makefile.inc,v
retrieving revision 1.138
diff -u -r1.138 Makefile.inc
--- etc/etc.evbarm/Makefile.inc	20 Feb 2026 07:20:28 -0000	1.138
+++ etc/etc.evbarm/Makefile.inc	15 Jul 2026 16:57:03 -0000
@@ -3,6 +3,7 @@
 #	etc.evbarm/Makefile.inc -- evbarm-specific etc Makefile targets
 #
 
+MKHASHES=${NETBSDSRCDIR}/distrib/utils/embedded/mkhashes
 MKIMAGE= ${NETBSDSRCDIR}/distrib/utils/embedded/mkimage
 
 # If you change the list of distributed kernels, don't forget
@@ -96,21 +97,27 @@
 	MACHINE=${MACHINE} \
 	MKDTB=${MKDTB} \
 	MKPOSTFIX=${MKPOSTFIX} \
-	CKSUM=${CKSUM} \
-	IMAGEDIR=${IMAGE.dir} \
 	${HOST_SH} ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
 	    -S ${NETBSDSRCDIR} -B ${IMAGEENDIAN} ${MKI_OPTS.${.TARGET}} \
 	    ${IMAGE.dir}/${.TARGET:S/smp_//}.img
 .for f in ${INSTALLBOOT_BOARDS}
-	@echo "===> Making bootable image ${IMAGE.dir:T}/${.TARGET:S/smp_//}-${f}.img.gz"
+	@echo "=== Bootable image: ${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img.gz ==="
 	@cp ${IMAGE.dir}/${.TARGET:S/smp_//}.img ${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img
 	@${TOOL_INSTALLBOOT} -m ${MACHINE} -o board=${f} \
 			${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img \
 		&& ${TOOL_GZIP} -f ${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img \
+		&& TOOL_CAT=${TOOL_CAT} \
+		TOOL_GREP=${TOOL_GREP} \
+		TOOL_MKTEMP=${TOOL_MKTEMP} \
+		${MKHASHES} -c ${TOOL_CKSUM} ${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img.gz \
 		|| rm ${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img
 .endfor
 	@echo "===> Compressing image ${IMAGE.dir}/${.TARGET:S/smp_//}.img"
 	${TOOL_GZIP} -f ${GZIP_N_FLAG} -9 ${IMAGE.dir}/${.TARGET:S/smp_//}.img
+	TOOL_CAT=${TOOL_CAT} \
+	TOOL_GREP=${TOOL_GREP} \
+	TOOL_MKTEMP=${TOOL_MKTEMP} \
+	${MKHASHES} -c ${TOOL_CKSUM} ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz 
 
 ARCHES.earmv4=		armv4
 ARCHES.earmv4eb=	armv4
@@ -137,42 +144,42 @@
 .if !empty(MACHINE_ARCH:Maarch64*)
 smp_efibootaa64:
 	cd ${KERNSRCDIR}/stand/efiboot/bootaa64 && ${MAKE} release
-SNAP_MD_POST_DEPS+=	smp_efibootaa64
+SNAP_MD_POST_DEPS+=	.WAIT smp_efibootaa64
 INSTALLATION_DIRS+=	installation/misc
 .endif
 
 .if !empty(MACHINE_ARCH:Mearmv7hf)
 smp_efibootarm:
 	cd ${KERNSRCDIR}/stand/efiboot/bootarm && ${MAKE} release
-SNAP_MD_POST_DEPS+=	smp_efibootarm
+SNAP_MD_POST_DEPS+=	.WAIT smp_efibootarm
 INSTALLATION_DIRS+=	installation/misc
 .endif
 
 .if !empty(MACHINE_ARCH:M*armv7*) && empty(ALL_KERNELS)
 smp_armv7: __mkimage
 MKI_OPTS.smp_armv7=	-K ${IMAGE.kern} -x
-SNAP_MD_POST_DEPS+=	smp_armv7
+SNAP_MD_POST_DEPS+=	.WAIT smp_armv7
 .endif
 
 .if !empty(MACHINE_ARCH:Maarch64*) && empty(ALL_KERNELS)
 smp_arm64: __mkimage
 MKI_OPTS.smp_arm64=	-K ${IMAGE.kern} -x
-SNAP_MD_POST_DEPS+=	smp_arm64
+SNAP_MD_POST_DEPS+=	.WAIT smp_arm64
 smp_arm64mbr: __mkimage
 MKI_OPTS.smp_arm64mbr=	-K ${IMAGE.kern} -x
-SNAP_MD_POST_DEPS+=	smp_arm64mbr
+SNAP_MD_POST_DEPS+=	.WAIT smp_arm64mbr
 .endif
 
 .if !empty(KERNEL_SETS:MRPI)
 smp_rpi: __mkimage
 .if empty(ALL_KERNELS) || !empty(ALL_KERNELS:MRPI)
 MKI_OPTS.smp_rpi=	-K ${IMAGE.kern}/netbsd-RPI.img.gz -x
-SNAP_MD_POST_DEPS+=	smp_rpi
+SNAP_MD_POST_DEPS+=	.WAIT smp_rpi
 .endif
 smp_rpi_inst: ${IMAGE.instk}/netbsd-RPI_INSTALL.img.gz __mkimage
 .if empty(ALL_KERNELS) || !empty(ALL_KERNELS:MRPI_INSTALL)
 MKI_OPTS.smp_rpi_inst=	-b -K ${IMAGE.instk}/netbsd-RPI_INSTALL.img.gz -x
-SNAP_MD_POST_DEPS+=	smp_rpi_inst
+SNAP_MD_POST_DEPS+=	.WAIT smp_rpi_inst
 .endif
 .endif
 
Index: etc/etc.evbmips/Makefile.inc
===================================================================
RCS file: /cvsroot/src/etc/etc.evbmips/Makefile.inc,v
retrieving revision 1.32
diff -u -r1.32 Makefile.inc
--- etc/etc.evbmips/Makefile.inc	22 Mar 2025 09:25:14 -0000	1.32
+++ etc/etc.evbmips/Makefile.inc	15 Jul 2026 16:57:03 -0000
@@ -52,6 +52,9 @@
 IMAGEENDIAN=	le
 .endif
 
+# Support for mkhashes
+MKHASHES= 	${NETBSDSRCDIR}/distrib/utils/embedded/mkhashes
+
 __mkimage: .USE
 	TOOL_MAKE=${MAKE} \
 	TOOL_MAKEFS=${TOOL_MAKEFS} \
@@ -70,16 +73,18 @@
 	MACHINE=${MACHINE} \
 	MKDTB=${MKDTB} \
 	MKPOSTFIX=${MKPOSTFIX} \
-	CKSUM=${CKSUM} \
-	IMAGEDIR=${IMAGE.dir} \
 	${HOST_SH} ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
             -S ${NETBSDSRCDIR} -B ${IMAGEENDIAN} ${MKI_OPTS.${.TARGET}} \
             ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
+	TOOL_CAT=${TOOL_CAT} \
+	TOOL_GREP=${TOOL_GREP} \
+	TOOL_MKTEMP=${TOOL_MKTEMP} \
+	${MKHASHES} -c ${CKSUM} ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
 
 .if !empty(MACHINE_ARCH:Mmips*64eb)
 smp_octeon: __mkimage
 MKI_OPTS.smp_octeon=	-K ${IMAGE.kern}
-SNAP_MD_POST_DEPS+=	smp_octeon
+SNAP_MD_POST_DEPS+=	.WAIT smp_octeon
 INSTALLATION_DIRS+=	binary/gzimg
 .endif
 
Index: etc/etc.evbppc/Makefile.inc
===================================================================
RCS file: /cvsroot/src/etc/etc.evbppc/Makefile.inc,v
retrieving revision 1.22
diff -u -r1.22 Makefile.inc
--- etc/etc.evbppc/Makefile.inc	10 Jan 2026 06:56:58 -0000	1.22
+++ etc/etc.evbppc/Makefile.inc	15 Jul 2026 16:57:03 -0000
@@ -39,6 +39,9 @@
 IMAGE.kern=     ${IMAGE.rel}/binary/kernel
 IMAGEENDIAN=	be
 
+# Support for mkhashes
+MKHASHES= 	${NETBSDSRCDIR}/distrib/utils/embedded/mkhashes
+
 .if ${MACHINE_ARCH} == "powerpc"
 __mkimage: .USE
 	TOOL_MAKE=${MAKE} \
@@ -58,16 +61,18 @@
 	MACHINE=${MACHINE} \
 	MKDTB=${MKDTB} \
 	MKPOSTFIX=${MKPOSTFIX} \
-	CKSUM=${CKSUM} \
-	IMAGEDIR=${IMAGE.dir} \
 	${HOST_SH} -x ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
             -S ${NETBSDSRCDIR} -B ${IMAGEENDIAN} ${MKI_OPTS.${.TARGET}} \
             ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
+	TOOL_CAT=${TOOL_CAT} \
+	TOOL_GREP=${TOOL_GREP} \
+	TOOL_MKTEMP=${TOOL_MKTEMP} \
+	${MKHASHES} -c ${CKSUM} ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
 
 smp_nintendo: __mkimage
 	cd ${KERNSRCDIR}/arch/evbppc/stand/wii && ${MAKE} release
 MKI_OPTS.smp_nintendo=		-K ${IMAGE.kern} -x
-SNAP_MD_POST_DEPS+=		smp_nintendo
+SNAP_MD_POST_DEPS+=		.WAIT smp_nintendo
 INSTALLATION_DIRS+=		binary/gzimg
 INSTALLATION_DIRS+=		installation/misc
 .endif
Index: etc/etc.riscv/Makefile.inc
===================================================================
RCS file: /cvsroot/src/etc/etc.riscv/Makefile.inc,v
retrieving revision 1.7
diff -u -r1.7 Makefile.inc
--- etc/etc.riscv/Makefile.inc	22 Mar 2025 09:25:14 -0000	1.7
+++ etc/etc.riscv/Makefile.inc	15 Jul 2026 16:57:03 -0000
@@ -15,6 +15,9 @@
 IMAGE.dir=      ${IMAGE.rel}/binary/gzimg
 IMAGE.kern=     ${IMAGE.rel}/binary/kernel
 
+# Support for mkhashes
+MKHASHES= 	${NETBSDSRCDIR}/distrib/utils/embedded/mkhashes
+
 #	TOOL_MKUBOOTIMAGE=${TOOL_MKUBOOTIMAGE} \
 
 __mkimage: .USE
@@ -34,12 +37,13 @@
 	MACHINE=${MACHINE} \
 	MKDTB=${MKDTB} \
 	MKPOSTFIX=${MKPOSTFIX} \
-	CKSUM=${CKSUM} \
-	IMAGEDIR=${IMAGE.dir} \
 	${HOST_SH} ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
             -S ${NETBSDSRCDIR} ${MKI_OPTS.${.TARGET}} \
             ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
-
+	TOOL_CAT=${TOOL_CAT} \
+	TOOL_GREP=${TOOL_GREP} \
+	TOOL_MKTEMP=${TOOL_MKTEMP} \
+	${MKHASHES} -c ${CKSUM} ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
 
 .if ${MACHINE_ARCH} == "riscv64"
 .if ${HAVE_UEFI} == "yes"
@@ -52,14 +56,14 @@
 
 smp_riscv64: __mkimage
 MKI_OPTS.smp_riscv64=	-K ${IMAGE.kern}
-SNAP_MD_POST_DEPS+=	smp_riscv64
+SNAP_MD_POST_DEPS+=	.WAIT smp_riscv64
 .endif
 
 
 .if ${MACHINE_ARCH} == "riscv32"
 smp_riscv32: __mkimage
 MKI_OPTS.smp_riscv32=	-K ${IMAGE.kern}
-SNAP_MD_POST_DEPS+=	smp_riscv32
+SNAP_MD_POST_DEPS+=	.WAIT smp_riscv32
 .endif
 
 snap_md_post: check_DESTDIR check_RELEASEDIR snap_post .WAIT ${SNAP_MD_POST_DEPS}



--- /dev/null	2026-07-15 10:54:37.944990496 -0600
+++ distrib/utils/embedded/mkhashes	2026-07-15 11:42:40.719381839 -0600
@@ -0,0 +1,112 @@
+#!/bin/sh
+# $NetBSD: mkimage,v 1.97 2026/05/22 14:44:16 riastradh Exp $
+#
+# Copyright (c) Brook Milligan.  All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+
+#
+# Makes a set of hashes for a file.
+#
+
+set -e
+
+PROG="$(basename "$0")"
+
+cat=${TOOL_CAT:-cat}
+cksum=${TOOL_CKSUM:-cksum}
+grep=${TOOL_GREP:-grep}
+mktemp=${TOOL_MKTEMP:-mktemp}
+
+hashes=${HASHES:-"MD5 SHA512"}
+imagedir=${IMAGEDIR}
+
+bar="==="
+
+tmp="$(${mktemp} -d "${TMPDIR:-/tmp}/$PROG.XXXXXX")"
+
+trap "cleanup" 0 1 2 3 15
+
+cleanup() {
+	case "$tmp" in
+	"${TMPDIR:-/tmp}/$PROG."*)	rm -fr "$tmp";;
+	esac
+}
+
+usage() {
+	${cat} << EOF 1>&2
+Usage: $PROG [-H <hashes>] [-c <cksum>] [-d <imagedir>] <image>
+
+-H	Hash algorithms [${hashes}]
+-c	Checksum program [${cksum}]
+-d	Image directory [dirname \${image}]
+EOF
+	exit 1
+}
+
+OPTS="H:c:d:"
+while getopts "$OPTS" f
+do
+	case $f in
+	H)	hashes="$OPTARG";;
+	c)	cksum="$OPTARG";;
+	d)	imagedir="$OPTARG";;
+	*)	usage;;
+	esac
+done
+
+shift $(( $OPTIND - 1 ))
+if [ -n "$1" ]; then
+	# take the next argument as being the image name
+	image="$1"
+	shift
+fi
+
+if [ -z "${image}" ]; then
+	echo "${PROG}: image undefined"
+	usage
+fi
+
+if [ -z "${imagedir}" ]; then
+	imagedir="$(dirname "${image}")"
+fi
+
+image="$(basename "${image}")"
+
+cd "${imagedir}"
+
+if [ ! -f ${image} ]; then
+	echo "${PROG}: ${imagedir}/${image} does not exist"
+	usage
+fi
+
+for h in ${hashes}; do
+	echo "${bar} ${h} hash: ${imagedir}/${image} ${bar}"
+	if [ -f ${h} ]; then
+		${grep} -v "${image}" ${h} > ${tmp}/${h} || true
+	else
+		cp /dev/null ${tmp}/${h}
+	fi
+	${cksum} -a ${h} "${image}" >> ${tmp}/${h}
+	mv ${tmp}/${h} ${h}
+done

--Apple-Mail=_7253959E-E726-4324-8310-38C5CA7AE055
Content-Transfer-Encoding: 7bit
Content-Type: text/plain;
	charset=us-ascii




--Apple-Mail=_7253959E-E726-4324-8310-38C5CA7AE055--