Re: gzimg hashes

Brook Milligan <[email protected]> Sat, 18 Jul 2026 09:10:23 -0600
Newsgroups	gmane.os.netbsd.current
Message-ID	<[email protected]>
> On Jul 15, 2026, at 16:51, Valery Ushakov <[email protected]> wrote:
>=20
> On Wed, Jul 15, 2026 at 11:54:46 -0600, Brook Milligan wrote:
>=20
>> Yes, indeed a parallelism issue.  That is now fixed.
>=20
> mkhashes still carries stale $NetBSD$ from its prototype
>=20
> mkhashes serializes image creation, can't we exploit make parallelism
> istead of fighting it?
>=20
> foo.img.gz.$(HASH) depends on foo.img.gz
>=20
> $(HASH) depends on all *.img.gz.$(HASH) with a trivial cat $> > $@
> rule

Good point.  At first, I was trying to minimize the changes.  However, =
here is another solution along the lines you suggested.

The basic idea is to add a new target (snap_md_post_images) to the end =
of snap_md_post that takes care of the (up to) four steps: (i) create an =
uncompressed image, (ii) create compressed bootable images + =
corresponding hash fiies from it, (iii) compress and hash the original =
image, and (iv) collect all the hashes into a single file.  Each step =
can be done in parallel, something that itself is an improvement.  Also, =
all these steps are not necessary for every architecture, so some =
changes skip or merge these four into fewer.

Please look over the patch below and provide comments.

Thanks a lot.

Cheers,
Brook


Index: distrib/utils/embedded/mkimage
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
RCS file: /cvsroot/src/distrib/utils/embedded/mkimage,v
retrieving revision 1.97
diff -u -r1.97 mkimage
--- distrib/utils/embedded/mkimage	22 May 2026 14:44:16 -0000	=
1.97
+++ distrib/utils/embedded/mkimage	18 Jul 2026 14:39:36 -0000
@@ -425,8 +425,4 @@
	image=3D"${image}.gz"
fi

-cd "${IMAGEDIR}"
-${CKSUM} -a MD5 "$(basename "${image}")" > MD5
-${CKSUM} -a SHA512 "$(basename "${image}")" > SHA512
-
echo ${bar} Image is ${image} ${bar}
Index: etc/Makefile
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
RCS file: /cvsroot/src/etc/Makefile,v
retrieving revision 1.476
diff -u -r1.476 Makefile
--- etc/Makefile	10 May 2026 14:19:10 -0000	1.476
+++ etc/Makefile	18 Jul 2026 14:39:36 -0000
@@ -48,6 +48,9 @@

GZIP_FLAGS=3D -9 ${GZIP_N_FLAG}

+HASHES+=3D	MD5
+HASHES+=3D	SHA512
+
# Flags for creating ISO CDROM image for mac68k and macppc that require
# tools mkhybrid(8) to create hybrid ISO9660/HFS images.
MKISOFS?=3D	${TOOL_MKHYBRID}
@@ -61,7 +64,7 @@
.if ${MKISOFS_FLAGS:N-v}
MKISOFS_FLAGS+=3D	-quiet
.endif
-=20
+
# Reproducible build parameters
.if ${MKREPRO:Uno} =3D=3D "no"
BUILDER!=3D	echo $${USER-root}@$$(hostname)
@@ -697,7 +700,7 @@
	done
.	for k in ${ALL_KERNELS}
install-debugkernels: ${DESTDIR}${DEBUGDIR}/netbsd-${k}.debug
-	=09
+
${DESTDIR}${DEBUGDIR}/netbsd-${k}.debug:
	${_MKTARGET_INSTALL}
	touch ${.TARGET}
Index: etc/etc.evbarm/Makefile.inc
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
RCS file: /cvsroot/src/etc/etc.evbarm/Makefile.inc,v
retrieving revision 1.138
diff -u -r1.138 Makefile.inc
--- etc/etc.evbarm/Makefile.inc	20 Feb 2026 07:20:28 -0000	1.138
+++ etc/etc.evbarm/Makefile.inc	18 Jul 2026 14:39:37 -0000
@@ -96,21 +96,27 @@
	MACHINE=3D${MACHINE} \
	MKDTB=3D${MKDTB} \
	MKPOSTFIX=3D${MKPOSTFIX} \
-	CKSUM=3D${CKSUM} \
-	IMAGEDIR=3D${IMAGE.dir} \
-	${HOST_SH} ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
+	${HOST_SH} ${MKIMAGE} -h ${.TARGET:R:S/smp_//} -D ${DESTDIR} \
	    -S ${NETBSDSRCDIR} -B ${IMAGEENDIAN} ${MKI_OPTS.${.TARGET}} =
\
-	    ${IMAGE.dir}/${.TARGET:S/smp_//}.img
-.for f in ${INSTALLBOOT_BOARDS}
-	@echo "=3D=3D=3D> Making bootable image =
${IMAGE.dir:T}/${.TARGET:S/smp_//}-${f}.img.gz"
-	@cp ${IMAGE.dir}/${.TARGET:S/smp_//}.img =
${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img
-	@${TOOL_INSTALLBOOT} -m ${MACHINE} -o board=3D${f} \
-			${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img \
-		&& ${TOOL_GZIP} -f =
${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img \
-		|| rm ${IMAGE.dir}/${.TARGET:S/smp_//}-${f}.img
+	    ${IMAGE.dir}/${.TARGET:S/smp_//}
+
+__mkbootimage: .USE
+	@echo "=3D=3D=3D> Making bootable image =
${IMAGE.dir:T}/${.TARGET:S/smp_//}"
+	cp ${IMAGE.dir}/${.TARGET:R:S/smp_//:C/-.*//}.img =
${IMAGE.dir}/${.TARGET:R:S/smp_//}
+	${TOOL_INSTALLBOOT} -m ${MACHINE} -o =
board=3D${.TARGET:R:R:C/smp_[^-]*-//} \
+			${IMAGE.dir}/${.TARGET:R:S/smp_//} \
+	&& ${TOOL_GZIP} -f ${IMAGE.dir}/${.TARGET:R:S/smp_//} \
+	&& for hash in ${HASHES}; do \
+		(cd ${IMAGE.dir} && ${TOOL_CKSUM} -a $${hash} =
${.TARGET:S/smp_//} > ${.TARGET:S/smp_//}.$${hash}) \
+	done \
+	|| rm ${IMAGE.dir}/${.TARGET:R:S/smp_//}*
+
+__mkcompressed: .USE
+	@echo "=3D=3D=3D> Compressing image =
${IMAGE.dir}/${.TARGET:R:S/smp_//}"
+	${TOOL_GZIP} -f ${GZIP_N_FLAG} -9 =
${IMAGE.dir}/${.TARGET:R:S/smp_//}
+.for hash in ${HASHES}
+	(cd ${IMAGE.dir} && ${TOOL_CKSUM} -a ${hash} ${.TARGET:S/smp_//} =
> ${.TARGET:S/smp_//}.${hash})
.endfor
-	@echo "=3D=3D=3D> Compressing image =
${IMAGE.dir}/${.TARGET:S/smp_//}.img"
-	${TOOL_GZIP} -f ${GZIP_N_FLAG} -9 =
${IMAGE.dir}/${.TARGET:S/smp_//}.img

ARCHES.earmv4=3D		armv4
ARCHES.earmv4eb=3D	armv4
@@ -149,34 +155,75 @@
.endif

.if !empty(MACHINE_ARCH:M*armv7*) && empty(ALL_KERNELS)
-smp_armv7: __mkimage
-MKI_OPTS.smp_armv7=3D	-K ${IMAGE.kern} -x
-SNAP_MD_POST_DEPS+=3D	smp_armv7
+smp_armv7.img: __mkimage
+smp_armv7.img.gz: __mkcompressed
+MKI_OPTS.smp_armv7.img=3D		-K ${IMAGE.kern} -x
+SNAP_MD_POST_DEPS_IMG+=3D		smp_armv7.img
+SNAP_MD_POST_DEPS_IMGZ+=3D	smp_armv7.img.gz
+.for board in ${INSTALLBOOT_BOARDS}
+smp_armv7-${board}.img.gz: __mkbootimage
+SNAP_MD_POST_DEPS_BOOTABLE+=3D	smp_armv7-${board}.img.gz
+.endfor
.endif

.if !empty(MACHINE_ARCH:Maarch64*) && empty(ALL_KERNELS)
-smp_arm64: __mkimage
-MKI_OPTS.smp_arm64=3D	-K ${IMAGE.kern} -x
-SNAP_MD_POST_DEPS+=3D	smp_arm64
-smp_arm64mbr: __mkimage
-MKI_OPTS.smp_arm64mbr=3D	-K ${IMAGE.kern} -x
-SNAP_MD_POST_DEPS+=3D	smp_arm64mbr
+smp_arm64.img: __mkimage
+smp_arm64.img.gz: __mkcompressed
+MKI_OPTS.smp_arm64.img=3D		-K ${IMAGE.kern} -x
+SNAP_MD_POST_DEPS_IMG+=3D		smp_arm64.img
+SNAP_MD_POST_DEPS_IMGZ+=3D	smp_arm64.img.gz
+.for board in ${INSTALLBOOT_BOARDS}
+smp_arm64-${board}.img.gz: __mkbootimage
+SNAP_MD_POST_DEPS_BOOTABLE+=3D	smp_arm64-${board}.img.gz
+.endfor
+smp_arm64mbr.img: __mkimage
+smp_arm64mbr.img.gz: __mkcompressed
+MKI_OPTS.smp_arm64mbr.img=3D	-K ${IMAGE.kern} -x
+SNAP_MD_POST_DEPS_IMG+=3D		smp_arm64mbr.img
+SNAP_MD_POST_DEPS_IMGZ+=3D	smp_arm64mbr.img.gz
+.for board in ${INSTALLBOOT_BOARDS}
+smp_arm64mbr-${board}.img.gz: __mkbootimage
+SNAP_MD_POST_DEPS_BOOTABLE+=3D	smp_arm64mbr-${board}.img.gz
+.endfor
.endif

.if !empty(KERNEL_SETS:MRPI)
-smp_rpi: __mkimage
+smp_rpi.img: __mkimage
+smp_rpi.img.gz: __mkcompressed
.if empty(ALL_KERNELS) || !empty(ALL_KERNELS:MRPI)
-MKI_OPTS.smp_rpi=3D	-K ${IMAGE.kern}/netbsd-RPI.img.gz -x
-SNAP_MD_POST_DEPS+=3D	smp_rpi
+MKI_OPTS.smp_rpi.img=3D		-K =
${IMAGE.kern}/netbsd-RPI.img.gz -x
+SNAP_MD_POST_DEPS_IMG+=3D		smp_rpi.img
+SNAP_MD_POST_DEPS_IMGZ+=3D	smp_rpi.img.gz
+.for board in ${INSTALLBOOT_BOARDS}
+smp_rpi-${board}.img.gz: __mkbootimage
+SNAP_MD_POST_DEPS_BOOTABLE+=3D	smp_rpi-${board}.img.gz
+.endfor
.endif
-smp_rpi_inst: ${IMAGE.instk}/netbsd-RPI_INSTALL.img.gz __mkimage
+smp_rpi_inst.img: ${IMAGE.instk}/netbsd-RPI_INSTALL.img.gz
+smp_rpi_inst.img: __mkimage
+smp_rpi_inst.img.gz: __mkcompressed
.if empty(ALL_KERNELS) || !empty(ALL_KERNELS:MRPI_INSTALL)
-MKI_OPTS.smp_rpi_inst=3D	-b -K =
${IMAGE.instk}/netbsd-RPI_INSTALL.img.gz -x
-SNAP_MD_POST_DEPS+=3D	smp_rpi_inst
+MKI_OPTS.smp_rpi_inst.img=3D	-b -K =
${IMAGE.instk}/netbsd-RPI_INSTALL.img.gz -x
+SNAP_MD_POST_DEPS_IMG+=3D		smp_rpi_inst.img
+SNAP_MD_POST_DEPS_IMGZ+=3D	smp_rpi_inst.img.gz
+.for board in ${INSTALLBOOT_BOARDS}
+smp_rpi_inst-${board}.img.gz: __mkbootimage
+SNAP_MD_POST_DEPS_BOOTABLE+=3D	smp_rpi_inst-${board}.img.gz
+.endfor
.endif
.endif

-snap_md_post: check_DESTDIR check_RELEASEDIR snap_post .WAIT =
${SNAP_MD_POST_DEPS}
+smp_hashes:
+	@echo "=3D=3D=3D> Making hash files: ${HASHES}"
+.for hash in ${HASHES}
+	cat ${IMAGE.dir}/*.${hash} > ${IMAGE.dir}/${hash} \
+	&& rm -f ${IMAGE.dir}/*.${hash}
+.endfor
+SNAP_MD_POST_DEPS_HASHES+=3D	smp_hashes
+
+snap_md_post_images: ${SNAP_MD_POST_DEPS_IMG} .WAIT =
${SNAP_MD_POST_DEPS_BOOTABLE} .WAIT ${SNAP_MD_POST_DEPS_IMGZ} .WAIT =
${SNAP_MD_POST_DEPS_HASHES}
+
+snap_md_post: check_DESTDIR check_RELEASEDIR snap_post .WAIT =
${SNAP_MD_POST_DEPS} snap_md_post_images

.if !empty(MACHINE_ARCH:Maarch64*)
KERNEL_SUFFIXES=3D	img
Index: etc/etc.evbmips/Makefile.inc
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
RCS file: /cvsroot/src/etc/etc.evbmips/Makefile.inc,v
retrieving revision 1.32
diff -u -r1.32 Makefile.inc
--- etc/etc.evbmips/Makefile.inc	22 Mar 2025 09:25:14 -0000	=
1.32
+++ etc/etc.evbmips/Makefile.inc	18 Jul 2026 14:39:37 -0000
@@ -70,11 +70,13 @@
	MACHINE=3D${MACHINE} \
	MKDTB=3D${MKDTB} \
	MKPOSTFIX=3D${MKPOSTFIX} \
-	CKSUM=3D${CKSUM} \
-	IMAGEDIR=3D${IMAGE.dir} \
	${HOST_SH} ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
            -S ${NETBSDSRCDIR} -B ${IMAGEENDIAN} ${MKI_OPTS.${.TARGET}} =
\
            ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
+	@echo "=3D=3D=3D> Making hash files: ${HASHES}"
+	for hash in ${HASHES}; do \
+		(cd ${IMAGE.dir} && ${TOOL_CKSUM} -a $${hash} =
${.TARGET:S/smp_//}.img.gz > $${hash}); \
+	done

.if !empty(MACHINE_ARCH:Mmips*64eb)
smp_octeon: __mkimage
Index: etc/etc.evbppc/Makefile.inc
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
RCS file: /cvsroot/src/etc/etc.evbppc/Makefile.inc,v
retrieving revision 1.22
diff -u -r1.22 Makefile.inc
--- etc/etc.evbppc/Makefile.inc	10 Jan 2026 06:56:58 -0000	1.22
+++ etc/etc.evbppc/Makefile.inc	18 Jul 2026 14:39:37 -0000
@@ -58,11 +58,13 @@
	MACHINE=3D${MACHINE} \
	MKDTB=3D${MKDTB} \
	MKPOSTFIX=3D${MKPOSTFIX} \
-	CKSUM=3D${CKSUM} \
-	IMAGEDIR=3D${IMAGE.dir} \
	${HOST_SH} -x ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
            -S ${NETBSDSRCDIR} -B ${IMAGEENDIAN} ${MKI_OPTS.${.TARGET}} =
\
            ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
+	@echo "=3D=3D=3D> Making hash files: ${HASHES}"
+	for hash in ${HASHES}; do \
+		(cd ${IMAGE.dir} && ${TOOL_CKSUM} -a $${hash} =
${.TARGET:S/smp_//}.img.gz > $${hash}); \
+	done

smp_nintendo: __mkimage
	cd ${KERNSRCDIR}/arch/evbppc/stand/wii && ${MAKE} release
Index: etc/etc.riscv/Makefile.inc
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
RCS file: /cvsroot/src/etc/etc.riscv/Makefile.inc,v
retrieving revision 1.7
diff -u -r1.7 Makefile.inc
--- etc/etc.riscv/Makefile.inc	22 Mar 2025 09:25:14 -0000	1.7
+++ etc/etc.riscv/Makefile.inc	18 Jul 2026 14:39:37 -0000
@@ -15,8 +15,6 @@
IMAGE.dir=3D      ${IMAGE.rel}/binary/gzimg
IMAGE.kern=3D     ${IMAGE.rel}/binary/kernel

-#	TOOL_MKUBOOTIMAGE=3D${TOOL_MKUBOOTIMAGE} \
-
__mkimage: .USE
	TOOL_MAKE=3D${MAKE} \
	TOOL_MAKEFS=3D${TOOL_MAKEFS} \
@@ -34,11 +32,12 @@
	MACHINE=3D${MACHINE} \
	MKDTB=3D${MKDTB} \
	MKPOSTFIX=3D${MKPOSTFIX} \
-	CKSUM=3D${CKSUM} \
-	IMAGEDIR=3D${IMAGE.dir} \
	${HOST_SH} ${MKIMAGE} -h ${.TARGET:S/smp_//} -D ${DESTDIR} \
            -S ${NETBSDSRCDIR} ${MKI_OPTS.${.TARGET}} \
            ${IMAGE.dir}/${.TARGET:S/smp_//}.img.gz
+	for hash in ${HASHES}; do \
+		(cd ${IMAGE.dir} && ${TOOL_CKSUM} -a $${hash} =
${.TARGET:S/smp_//}.img.gz > ${.TARGET:S/smp_//}.img.gz.$${hash}) \
+	done


.if ${MACHINE_ARCH} =3D=3D "riscv64"
@@ -51,18 +50,29 @@
.endif

smp_riscv64: __mkimage
-MKI_OPTS.smp_riscv64=3D	-K ${IMAGE.kern}
-SNAP_MD_POST_DEPS+=3D	smp_riscv64
+MKI_OPTS.smp_riscv64=3D		-K ${IMAGE.kern}
+SNAP_MD_POST_DEPS_IMGZ+=3D	smp_riscv64
.endif


.if ${MACHINE_ARCH} =3D=3D "riscv32"
smp_riscv32: __mkimage
-MKI_OPTS.smp_riscv32=3D	-K ${IMAGE.kern}
-SNAP_MD_POST_DEPS+=3D	smp_riscv32
+MKI_OPTS.smp_riscv32=3D		-K ${IMAGE.kern}
+SNAP_MD_POST_DEPS_IMGZ+=3D	smp_riscv32
.endif

-snap_md_post: check_DESTDIR check_RELEASEDIR snap_post .WAIT =
${SNAP_MD_POST_DEPS}
+
+smp_hashes:
+	@echo "=3D=3D=3D> Making hash files: ${HASHES}"
+.for hash in ${HASHES}
+	cat ${IMAGE.dir}/*.${hash} > ${IMAGE.dir}/${hash} \
+	&& rm -f ${IMAGE.dir}/*.${hash}
+.endfor
+SNAP_MD_POST_DEPS_HASHES+=3D	smp_hashes
+
+snap_md_post_images: ${SNAP_MD_POST_DEPS_IMGZ} .WAIT =
${SNAP_MD_POST_DEPS_HASHES}
+
+snap_md_post: check_DESTDIR check_RELEASEDIR snap_post .WAIT =
${SNAP_MD_POST_DEPS} snap_md_post_images

INSTALLATION_DIRS+=3D	binary/gzimg		\
			installation/instkernel