new initial submodule update strategy (#904)

* new initial submodule update strategy

* Azure: fix command order

.appveyor.yml

@@ -20,7 +20,6 @@ platform:
   - x86
 
 install:
-  - git submodule update --init --recursive
   # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin
   - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH%
   - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%

Jenkinsfile

@@ -2,7 +2,6 @@ def runStages() {
 	try {
 		stage("Clone") {
 			checkout scm
-			sh "make build-system-checks || true"
 		}
 
 		cache(maxCacheSize: 250, caches: [

Makefile

@@ -10,7 +10,7 @@ SHELL := bash # the shell used internally by "make"
 # used inside the included makefiles
 BUILD_SYSTEM_DIR := vendor/nimbus-build-system
 
-# we don't want an error here, so we can handle things later, in the build-system-checks target
+# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
 -include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
 
 # unconditionally built by the default Make target
@@ -34,7 +34,6 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 
 .PHONY: \
 	all \
-	build-system-checks \
 	deps \
 	update \
 	test \
@@ -49,11 +48,22 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 	libbacktrace
 
 ifeq ($(NIM_PARAMS),)
-# "variables.mk" was not included. We can only execute one target in this state.
-all: | build-system-checks
-else
-all: | build-system-checks $(TOOLS) libnfuzz.so libnfuzz.a
-endif
+# "variables.mk" was not included, so we update the submodules.
+GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+.DEFAULT:
+	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
+		$(GIT_SUBMODULE_UPDATE); \
+		echo
+# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
+# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
+#
+# After restarting, it will execute its original goal, so we don't have to start a child Make here
+# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
+
+else # "variables.mk" was included. Business as usual until the end of this file.
+
+# default target, because it's the first one that doesn't start with '.'
+all: | $(TOOLS) libnfuzz.so libnfuzz.a
 
 # must be included after the default target
 -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
@@ -65,19 +75,6 @@ else
 NIM_PARAMS := $(NIM_PARAMS) -d:release
 endif
 
-#- the Windows build fails on Azure Pipelines if we have Unicode symbols copy/pasted here,
-#  so we encode them in ASCII
-GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
-build-system-checks:
-	@[[ -e "$(BUILD_SYSTEM_DIR)/makefiles" ]] || { \
-		echo -e "'$(BUILD_SYSTEM_DIR)/makefiles' not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
-		$(GIT_SUBMODULE_UPDATE); \
-		CHECKMARK="\xe2\x9c\x94\xef\xb8\x8f"; \
-		echo -e "\n$${CHECKMARK}$${CHECKMARK}$${CHECKMARK} Successfully fetched all required internal dependencies."; \
-		echo -e "        You should now \e[4mre-run '$(MAKE)' to build Nimbus\e[0m\n"; \
-		}; \
-		exit 0
-
 deps: | deps-common beacon_chain.nims
 ifneq ($(USE_LIBBACKTRACE), 0)
 deps: | libbacktrace
@@ -94,7 +91,7 @@ beacon_chain.nims:
 
 # nim-libbacktrace
 libbacktrace:
-	+ $(MAKE) -C vendor/nim-libbacktrace BUILD_CXX_LIB=0 $(HANDLE_OUTPUT)
+	+ $(MAKE) -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
 
 # Windows 10 with WSL enabled, but no distro installed, fails if "../../nimble.sh" is executed directly
 # in a Makefile recipe but works when prefixing it with `bash`. No idea how the PATH is overridden.
@@ -145,3 +142,6 @@ libnfuzz.a: | build deps
 		rm -f build/$@ && \
 		$(ENV_SCRIPT) nim c -d:release --app:staticlib --noMain --nimcache:nimcache/libnfuzz_static -o:build/$@ $(NIM_PARAMS) nfuzz/libnfuzz.nim && \
 		[[ -e "$@" ]] && mv "$@" build/ # workaround for https://github.com/nim-lang/Nim/issues/12745
+
+endif # "variables.mk" was not included
+

README.md

@@ -85,8 +85,8 @@ It also provides a downloading script for prebuilt PCRE.
 ### Android
 
 * Install the [Termux](https://termux.com) app from FDroid or the Google Play store
-* Install a [PRoot](https://wiki.termux.com/wiki/PRoot) of your choice following the instructions for your preferred distribution. 
-Note, the Ubuntu PRoot is known to contain all Nimbus prerequisites compiled on Arm64 architecture (common architecture for Android devices).  
+* Install a [PRoot](https://wiki.termux.com/wiki/PRoot) of your choice following the instructions for your preferred distribution.
+Note, the Ubuntu PRoot is known to contain all Nimbus prerequisites compiled on Arm64 architecture (common architecture for Android devices).
 
 *Assuming Ubuntu PRoot is used*
 
@@ -105,7 +105,6 @@ Once the [prerequisites](#prerequisites) are installed you can connect to testne
 ```bash
 git clone https://github.com/status-im/nim-beacon-chain
 cd nim-beacon-chain
-make                 # This invocation will bootstrap the build system with additional Makefiles
 make testnet0        # This will build Nimbus and all other dependencies
                      # and connect you to testnet0
 ```
@@ -228,9 +227,22 @@ Variables -> Path -> Edit -> New -> C:\mingw-w64\mingw64\bin (it's "C:\mingw-w64
 
 Install [Git for Windows](https://gitforwindows.org/) and use a "Git Bash" shell to clone and build nim-beacon-chain.
 
+Install [CMake](https://cmake.org/) to be able to build libunwind (used for [lightweight stack traces](https://github.com/status-im/nim-libbacktrace)).
+
+When running the tests, you might hit some Windows path length limits. Increase them by editing the Registry in a PowerShell instance with administrator privileges:
+
+```powershell
+Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -Value 1
+```
+
+and run this in a "Git Bash" terminal:
+
+```bash
+git config --global core.longpaths true
+```
+
 If you don't want to compile PCRE separately, you can fetch pre-compiled DLLs with:
 ```bash
-mingw32-make # this first invocation will update the Git submodules
 mingw32-make fetch-dlls # this will place the right DLLs for your architecture in the "build/" directory
 ```
 
@@ -247,9 +259,9 @@ mingw32-make test # run the test suite
 After cloning the repo:
 
 ```bash
-make # The first `make` invocation will update all Git submodules and prompt you to run `make` again.
-     # It's only required once per Git clone. You'll run `make update` after each `git pull`, in the future,
-     # to keep those submodules up to date.
+# The first `make` invocation will update all Git submodules.
+# You'll run `make update` after each `git pull`, in the future, to keep those submodules up to date.
+make
 
 # Run tests
 make test

azure-pipelines.yml

@@ -1,6 +1,8 @@
 jobs:
   - job: Windows
 
+    timeoutInMinutes: 80
+
     pool:
       vmImage: windows-latest
 
@@ -58,11 +60,8 @@ jobs:
           mv "$MINGW_DIR" /c/custom/
           cd ..
           export PATH="/c/custom/${MINGW_DIR}/bin:$PATH"
-          echo "Fetching submodules"
           git config --global core.longpaths true
           git config --global core.autocrlf false
-          git submodule --quiet update --init --recursive
-          scripts/setup_official_tests.sh jsonTestsCache
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} fetch-dlls
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} LOG_LEVEL=TRACE
@@ -70,5 +69,6 @@ jobs:
           file build/beacon_node
           # fail fast
           export NIMTEST_ABORT_ON_ERROR=1
+          scripts/setup_official_tests.sh jsonTestsCache
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} DISABLE_TEST_FIXTURES_SCRIPT=1 test
         displayName: 'build and test'

vendor/nim-libbacktrace

@@ -1 +1 @@
-Subproject commit da216986c635599dccffa2e71eabad03653e5aef
+Subproject commit fc63983c73ba541413f12ee70a2b6f025b03efea

vendor/nimbus-build-system

@@ -1 +1 @@
-Subproject commit d8cde2ad855295fe3539f9beaa2ca95cb991863b
+Subproject commit e8e1f1c2cff5aebf74dff07b1cf119134267922b