Re: A better alternative to having builds of main-armv7-default fully disabled and last-built be months out of date

Philip Paeps <[email protected]> Mon, 15 Jul 2024 08:20:55 +0800

Newsgroups	gmane.os.freebsd.devel.arm
Message-ID	<[email protected]>

Sorry for not following up to this thread earlier.
I've been occupied elsewhere in the cluster.

On 2024-07-07 16:25:32 (+0800), Mark Millard wrote:
> On Jul 6, 2024, at 21:35, Michal Meloun <[email protected]> 
> wrote:
>> On 07.07.2024 5:42, Mark Millard wrote:
>>> main's armv7 packages that are distributed are getting to be months
>>> behind because of the build hangups preventing the builds on 
>>> ampere2.

It's worth reinforcing that this only affects main (15-CURRENT).  Our 
stable/13 and stable/14 packages for armv7 are reasonably up to date.  
Reasonably for a tier-2 architecture anyway.  Whatever is causing this, 
it's only in main.

> The only known failures are on ampere2 as far as I know.
> As far as I know there is no known way to configure to
> match the formal build procedures used on ampere2.

According to the current schedule, armv7 builds happen on ampere3, not 
ampere2:

ampere1: - quarterly arm64.aarch64 13.3-RELEASE 133arm64 -a
ampere1: - quarterly arm.armv7 releng/13.3 133releng-armv7 -a
ampere1: - quarterly arm64.aarch64 14.0-RELEASE 140arm64 -a
ampere1: - quarterly arm.armv7 releng/14.0 140releng-armv7 -a
ampere2: - default arm64.aarch64 main main-arm64 -a
ampere3: - default arm64.aarch64 13.3-RELEASE 133arm64 -a
ampere3: - default arm.armv7 releng/13.3 133releng-armv7 -a
ampere3: - default arm64.aarch64 14.0-RELEASE 140arm64 -a
ampere3: - default arm.armv7 releng/14.0 140releng-armv7 -a

I've attached the poudriere.conf from that machine.  It's the same one 
we have on all the builders.

>> I've seen some strange live lockups in arm32 jail, but never managed 
>> to reproduce it.
>
> On what kind(s) of hardware?
> Any kind of relevant context known?

In case it helps: ref15-aarch64.freebsd.org (available to all 
developers) is an identical configuration as ampereX.nyi.freebsd.org.  
The former has a newer BIOS (for some reason) but that hopefully should 
not make a difference.  If we reach the point where we think the BIOS 
version matters, I can try to upgrade the BIOS on the ampereXen.

smbios.bios.reldate="06/25/2020"
smbios.bios.revision="1.14"
smbios.bios.vendor="LENOVO"
smbios.bios.version="hve104q-1.14"

smbios.bios.reldate="05/30/2019"
smbios.bios.revision="1.8"
smbios.bios.vendor="LENOVO"
smbios.bios.version="HVE104J-1.08"

Philip

poudriere.conf (text/plain, 2.5 KB)

# !!!! DO NOT EDIT THIS FILE, IT IS TRACKED IN SVN (https://pm-svn.freebsd.org/cluster) !!!!
# vim: set filetype=sh:
ZPOOL=zroot
ZROOTFS=/poudriere
RESOLV_CONF=/etc/resolv.conf
BASEFS=/usr/local/poudriere
USE_PORTLINT=no
USE_TMPFS=all
DISTFILES_CACHE=/usr/ports/distfiles
SVN_HOST=svn0.us-east.FreeBSD.org
GIT_BASEURL=git.freebsd.org/src.git
GIT_PORTSURL=git.freebsd.org/ports.git
CHECK_CHANGED_OPTIONS=yes
CHECK_CHANGED_DEPS=yes
NO_RESTRICTED=yes
NO_FORCE_PACKAGE=yes
URL_BASE="https://pkg-status.freebsd.org/$(hostname -s)"
#MAX_MEMORY=8
ATOMIC_PACKAGE_REPOSITORY=no
PKG_REPO_FROM_HOST=yes
SIGNING_COMMAND="ssh pkg-sign"
PRIORITY_BOOST="iridium-browser ungoogled-chromium"
PRIORITY_BOOST_VALUE=20
ALLOW_MAKE_JOBS_PACKAGES="pkg chromium electron29 iridium-browser llvm-devel qt5-webengine qt6-webengine ungoogled-chromium"
TMPFS_BLACKLIST="chromium electron29 iridium-browser ungoogled-chromium"
TMPFS_BLACKLIST_TMPDIR=${BASEFS}/data/wrkdirs
HTML_JSON_UPDATE_INTERVAL=10
USE_PTSORT=no
DETERMINE_BUILD_FAILURE_REASON=no
# 48 hours for chromium based browsers
MAX_EXECUTION_TIME=172800
# 2 hours for chromium based browsers 
MAX_EXECUTION_TIME_EXTRACT=7200
MAX_FILES_RStudio=4096
MAX_FILES_RStudio_server=4096
MAX_FILES_atom_ide=4096
MAX_FILES_traefik=2048
MAX_FILES_gatk=2048
MAX_FILES_screego=8192
MAX_FILES_vscode=8192
BUILD_AS_NON_ROOT=no
# Use jail src for native-xtools builds
XDEV_SRC_JAIL=yes
#CCACHE_DIR=/ccache

calc_builders() {
	local GB_PER_BUILDER ncpu arcsize physmem swap vmem emem

	GB_PER_BUILDER=12
	ncpu=$(/sbin/sysctl -n hw.ncpu)
	PREPARE_PARALLEL_JOBS=$((ncpu + 5))

	# Tune BUILD_PARALLEL_JOBS based on amount of VM.
	physmem=$(($(/sbin/sysctl -n hw.physmem) / 1024 / 1024 / 1024))
	arcsize=$(($(/sbin/sysctl -n vfs.zfs.arc_max) / 1024 / 1024 / 1024))
	swap=$(($(/sbin/sysctl -n vm.swap_total) / 1024 / 1024 / 1024))
	vmem=$((physmem + swap))
	# Calculate "effective" memory; Ignore greedy ARC.
	emem=$((vmem - arcsize))
	if [ ${emem} -lt 0 ]; then
		# The ARC is probably too big.
		emem=$((emem + arcsize))
	fi
	# Give each builder GB_PER_BUILDER
	BUILD_PARALLEL_JOBS=$((emem / GB_PER_BUILDER))
	if [ ${BUILD_PARALLEL_JOBS} -gt ${ncpu} ]; then
		BUILD_PARALLEL_JOBS=${ncpu}
	elif [ ${ncpu} -ge $((2 * BUILD_PARALLEL_JOBS)) ]; then
		ALLOW_MAKE_JOBS=yes
	fi
	if [ $# -gt 0 ]; then
		echo "calc_builders(): ncpu=${ncpu} physmem=${physmem} arcsize=${arcsize} JOBS=${BUILD_PARALLEL_JOBS}" >&2
	fi
}
calc_builders "$@"

#case "$(hostname -s)" in
#	foul2)
#		BUILD_PARALLEL_JOBS=20
#		;;
#esac