nimbus-build-system/scripts/build_nim.sh

#!/usr/bin/env bash
# Copyright (c) 2018-2024 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.

set -eo pipefail

# NIM_COMMIT could be a (partial) commit hash, a tag, a branch name, etc. Empty by default.
NIM_COMMIT_HASH="" # full hash for NIM_COMMIT, retrieved in "nim_needs_rebuilding()"

# script arguments
[[ $# -ne 4 ]] && { echo "Usage: $0 nim_dir csources_dir nimble_dir ci_cache_dir"; exit 1; }
NIM_DIR="$1"
CSOURCES_DIR="$2" # can be relative to NIM_DIR; only used when `skipIntegrityCheck` unsupported
NIMBLE_DIR="$3" # can be relative to NIM_DIR; only used when `skipIntegrityCheck` unsupported
CI_CACHE="$4"

## env vars
# verbosity level
[[ -z "$V" ]] && V=0
[[ -z "$CC" ]] && CC="gcc"
# to build csources in parallel, set MAKE="make -jN"
[[ -z "$MAKE" ]] && MAKE="make"
# for 32-bit binaries on a 64-bit host
UCPU=""
[[ "$ARCH_OVERRIDE" == "x86" ]] && UCPU="ucpu=i686"
[[ -z "$NIM_BUILD_MSG" ]] && NIM_BUILD_MSG="Building the Nim compiler"
[[ -z "$QUICK_AND_DIRTY_COMPILER" ]] && QUICK_AND_DIRTY_COMPILER=0
[[ -z "$QUICK_AND_DIRTY_NIMBLE" ]] && QUICK_AND_DIRTY_NIMBLE=0

# Windows detection
if uname | grep -qiE "mingw|msys"; then
	ON_WINDOWS=1
	EXE_SUFFIX=".exe"
	# otherwise it fails in AppVeyor due to https://github.com/git-for-windows/git/issues/2495
	GIT_TIMESTAMP_ARG="--date=unix" # available since Git 2.9.4
else
	ON_WINDOWS=0
	EXE_SUFFIX=""
	GIT_TIMESTAMP_ARG="--date=format-local:%s" # available since Git 2.7.0
fi

NIM_BINARY="${NIM_DIR}/bin/nim${EXE_SUFFIX}"
MAX_NIM_BINARIES="10" # Old ones get deleted.

nim_needs_rebuilding() {
	REBUILD=0
	NO_REBUILD=1

	if [[ ! -e "$NIM_DIR" ]]; then
		# Shallow clone, optimised for the default NIM_COMMIT value.
		git clone -q --depth=1 https://github.com/status-im/Nim.git "$NIM_DIR"
	fi

	pushd "${NIM_DIR}" >/dev/null
	if [[ -n "${NIM_COMMIT}" ]]; then
		# support old Git versions, like the one from Ubuntu-18.04
		git restore . 2>/dev/null || git reset --hard
		if ! git checkout -q ${NIM_COMMIT} 2>/dev/null; then
			# Pay the price for a non-default NIM_COMMIT here, by fetching everything.
			# (This includes upstream branches and tags that might be missing from our fork.)
			git remote add upstream https://github.com/nim-lang/Nim
			git fetch --all --tags --quiet
			git checkout -q ${NIM_COMMIT}
		fi
		# In case the local branch diverged and a fast-forward merge is not possible.
		git fetch || true
		git reset -q --hard origin/${NIM_COMMIT} 2>/dev/null || true
		# In case NIM_COMMIT is a local branch that's behind the remote one it's tracking.
		git pull -q 2>/dev/null || true
		git checkout -q ${NIM_COMMIT}
		# We can't use "rev-parse" here, because it would return the tag object's
		# hash instead of the commit hash, when NIM_COMMIT is a tag.
		NIM_COMMIT_HASH="$(git rev-list -n 1 ${NIM_COMMIT})"
	else
		# NIM_COMMIT is empty, so assume the commit we need is already checked out
		NIM_COMMIT_HASH="$(git rev-list -n 1 HEAD)"
	fi
	popd >/dev/null

	if [[ -n "$CI_CACHE" && -d "$CI_CACHE" ]]; then
		cp -a "$CI_CACHE"/* "$NIM_DIR"/bin/ || true # let this one fail with an empty cache dir
	fi

	# Delete old Nim binaries, to put a limit on how much storage we use.
	for F in "$(ls -t "${NIM_DIR}"/bin/nim_commit_* 2>/dev/null | tail -n +$((MAX_NIM_BINARIES + 1)))"; do
		if [[ -e "${F}" ]]; then
			rm "${F}"
		fi
	done

	# Compare the last built commit to the one requested.
	# Handle the scenario where our symlink is manually deleted by the user.
	if [[ -e "${NIM_DIR}/bin/last_built_commit" && \
	-e "${NIM_DIR}/bin/nim${EXE_SUFFIX}" && \
	"$(cat "${NIM_DIR}/bin/last_built_commit")" == "${NIM_COMMIT_HASH}" ]]; then
		return $NO_REBUILD
	elif [[ -e "${NIM_DIR}/bin/nim_commit_${NIM_COMMIT_HASH}" ]]; then
		# we built the requested commit in the past, so we simply reuse it
		rm -f "${NIM_DIR}/bin/nim${EXE_SUFFIX}"
		ln -s "nim_commit_${NIM_COMMIT_HASH}" "${NIM_DIR}/bin/nim${EXE_SUFFIX}"
		echo ${NIM_COMMIT_HASH} > "${NIM_DIR}/bin/last_built_commit"
		return $NO_REBUILD
	else
		return $REBUILD
	fi
}

build_nim() {
	echo -e "$NIM_BUILD_MSG"
	[[ "$V" == "0" ]] && exec &>/dev/null

	# working directory
	pushd "$NIM_DIR"

	# Otherwise, when updating from pre-v2.0.10 to v2.0.10 or later,
	# https://github.com/nim-lang/Nim/issues/24173 occurs. Simulates
	# https://github.com/nim-lang/Nim/pull/24189 as a workaround.
	#
	# When building Nimbus from a Nix derivation which adds this as part of
	# a preBuild phase, do not use this hack, because it's both unnecessary
	# and prevents Nim from building.
	NIX_BUILD_TOP="${NIX_BUILD_TOP:-}"
	if [[ "${NIX_BUILD_TOP}" != "/build" ]]; then
		rm -rf dist/checksums
	fi

	if grep -q skipIntegrityCheck koch.nim; then
		# Run Nim buildchain, with matching dependency versions
		# - CSOURCES_REPO from Nim/config/build_config.txt (nim_csourcesUrl)
		# - CSOURCES_COMMIT from Nim/config/build_config.txt (nim_csourcesHash)
		# - NIMBLE_REPO from Nim/koch.nim (bundleNimbleExe)
		# - NIMBLE_COMMIT from Nim/koch.nim (NimbleStableCommit)
		. ci/funs.sh
		NIMCORES=1 nimBuildCsourcesIfNeeded $UCPU
		bin/nim c --noNimblePath --skipUserCfg --skipParentCfg --warnings:off --hints:off koch
		./koch --skipIntegrityCheck boot -d:release --skipUserCfg --skipParentCfg --warnings:off --hints:off
		if [[ "${QUICK_AND_DIRTY_COMPILER}" == "0" ]]; then
			# We want tools
			./koch tools -d:release --skipUserCfg --skipParentCfg --warnings:off --hints:off
		elif [[ "${QUICK_AND_DIRTY_NIMBLE}" != "0" ]]; then
			# We just want nimble
			./koch nimble -d:release --skipUserCfg --skipParentCfg --warnings:off --hints:off
		fi
	else
		# Git commits
		: ${CSOURCES_V1_COMMIT:=561b417c65791cd8356b5f73620914ceff845d10}
		: ${CSOURCES_V2_COMMIT:=86742fb02c6606ab01a532a0085784effb2e753e}
		: ${CSOURCES_V1_REPO:=https://github.com/nim-lang/csources_v1.git}
		: ${CSOURCES_V2_REPO:=https://github.com/nim-lang/csources_v2.git}

		# After this Nim commit, use csources v2
		: ${CSOURCES_V2_START_COMMIT:=f7c203fb6c89b5cef83c4f326aeb23ef8c4a2c40}
		: ${NIMBLE_REPO:=https://github.com/nim-lang/nimble.git}
		: ${NIMBLE_COMMIT:=d13f3b8ce288b4dc8c34c219a4e050aaeaf43fc9} # 0.13.1

		# Custom buildchain for older versions
		# TODO Remove this once the default NIM_COMMIT supports `--skipIntegrityCheck`
		# We will still be able to compile older versions by removing the flag,
		# which will just waste a bit of CPU

		# Git repos for csources and Nimble
		if [[ ! -d "$CSOURCES_DIR" ]]; then
			if git merge-base --is-ancestor $CSOURCES_V2_START_COMMIT $NIM_COMMIT_HASH; then
			  CSOURCES_REPO=$CSOURCES_V2_REPO
			  CSOURCES_COMMIT=$CSOURCES_V2_COMMIT
			else
			  CSOURCES_REPO=$CSOURCES_V1_REPO
			  CSOURCES_COMMIT=$CSOURCES_V1_COMMIT
			fi

			mkdir -p "$CSOURCES_DIR"
			pushd "$CSOURCES_DIR"
			git clone $CSOURCES_REPO .
			git checkout $CSOURCES_COMMIT
			popd
		fi
		if [[ "$CSOURCES_DIR" != "csources" ]]; then
			rm -rf csources
			ln -s "$CSOURCES_DIR" csources
		fi

		if [[ ! -d "$NIMBLE_DIR" ]]; then
			mkdir -p "$NIMBLE_DIR"
			pushd "$NIMBLE_DIR"
			git clone $NIMBLE_REPO .
			git checkout $NIMBLE_COMMIT
			# we have to delete .git or koch.nim will checkout a branch tip, overriding our target commit
			rm -rf .git
			popd
		fi
		if [[ "$NIMBLE_DIR" != "dist/nimble" ]]; then
			mkdir -p dist
			rm -rf dist/nimble
			ln -s ../"$NIMBLE_DIR" dist/nimble
		fi

		# bootstrap the Nim compiler and build the tools
		rm -f bin/{nim,nim_csources}
		pushd csources
		if [[ "$ON_WINDOWS" == "0" ]]; then
			$MAKE $UCPU clean
			$MAKE $UCPU LD=$CC
		else
			$MAKE myos=windows $UCPU clean
			$MAKE myos=windows $UCPU CC=gcc LD=gcc
		fi
		popd
		if [[ -e csources/bin ]]; then
			rm -f bin/nim bin/nim_csources
			cp -a csources/bin/nim bin/nim
			cp -a csources/bin/nim bin/nim_csources
			rm -rf csources/bin
		else
			cp -a bin/nim bin/nim_csources
		fi
		if [[ "$QUICK_AND_DIRTY_COMPILER" == "0" ]]; then
			sed \
				-e 's/koch$/--warnings:off --hints:off koch/' \
				-e 's/koch boot/koch boot --warnings:off --hints:off/' \
				-e '/nimBuildCsourcesIfNeeded/d' \
				build_all.sh > build_all_custom.sh
			sh build_all_custom.sh
			rm build_all_custom.sh
		else
			# Don't re-build it multiple times until we get identical
			# binaries, like "build_all.sh" does. Don't build any tools
			# either. This is all about build speed, not developer comfort.
			bin/nim_csources \
				c \
				--compileOnly \
				--nimcache:nimcache \
				-d:release \
				--skipUserCfg \
				--skipParentCfg \
				--warnings:off \
				--hints:off \
				compiler/nim.nim
			bin/nim_csources \
				jsonscript \
				--nimcache:nimcache \
				--skipUserCfg \
				--skipParentCfg \
				compiler/nim.nim
			cp -a compiler/nim bin/nim1
			# If we stop here, we risk ending up with a buggy compiler:
			# https://github.com/status-im/nimbus-eth2/pull/2220
			# https://github.com/status-im/nimbus-eth2/issues/2310
			bin/nim1 \
				c \
				--compileOnly \
				--nimcache:nimcache \
				-d:release \
				--skipUserCfg \
				--skipParentCfg \
				--warnings:off \
				--hints:off \
				compiler/nim.nim
			bin/nim1 \
				jsonscript \
				--nimcache:nimcache \
				--skipUserCfg \
				--skipParentCfg \
				compiler/nim.nim
			rm -f bin/nim
			cp -a compiler/nim bin/nim
			rm bin/nim1

			# Do we want Nimble in this quick build?
			if [[ "${QUICK_AND_DIRTY_NIMBLE}" != "0" ]]; then
				bin/nim c -d:release --noNimblePath --skipUserCfg --skipParentCfg dist/nimble/src/nimble.nim
				mv dist/nimble/src/nimble bin/
			fi
		fi
	fi

	if [[ "$QUICK_AND_DIRTY_COMPILER" == "0" || "${QUICK_AND_DIRTY_NIMBLE}" != "0" ]]; then
		# Nimble needs a CA cert
		rm -f bin/cacert.pem
		curl -LsS -o bin/cacert.pem https://curl.se/ca/cacert.pem || echo "Warning: 'curl' failed to download a CA cert needed by Nimble. Ignoring it."
	fi

	# record the built commit
	echo ${NIM_COMMIT_HASH} > bin/last_built_commit

	# create the symlink
	mv bin/nim bin/nim_commit_${NIM_COMMIT_HASH}
	ln -s nim_commit_${NIM_COMMIT_HASH} bin/nim${EXE_SUFFIX}

	# update the CI cache
	popd # we were in $NIM_DIR
	if [[ -n "$CI_CACHE" ]]; then
		rm -rf "$CI_CACHE"
		mkdir "$CI_CACHE"
		cp "$NIM_DIR"/bin/* "$CI_CACHE"/
	fi
}

if nim_needs_rebuilding; then
	build_nim
fi