From de634963addf1cc66db84475ae65d6b2fa43194b Mon Sep 17 00:00:00 2001 From: andri lim Date: Tue, 19 Sep 2023 07:47:58 +0700 Subject: [PATCH] Selectively download nimbus-eth2 submodules (#1757) --- .github/workflows/ci.yml | 2 +- .github/workflows/fluffy.yml | 4 ++-- .github/workflows/nimbus_verified_proxy.yml | 2 +- .github/workflows/simulators.yml | 6 +++--- Makefile | 18 +++++++++++++++++- docker/dist/entry_point.sh | 12 ++++++------ 6 files changed, 30 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ecfabdc8..8475618fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -241,7 +241,7 @@ jobs: - name: Build Nim and Nimbus-eth1 dependencies run: | - make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinCache update + make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinCache update-from-ci - name: Run nimbus-eth1 tests (Windows) if: runner.os == 'Windows' diff --git a/.github/workflows/fluffy.yml b/.github/workflows/fluffy.yml index 30c84a188..7f014bd8e 100644 --- a/.github/workflows/fluffy.yml +++ b/.github/workflows/fluffy.yml @@ -63,7 +63,7 @@ jobs: - name: Build Nim and Nimbus-eth1 dependencies run: | - make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update + make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update-from-ci - name: build uTP test app container run: | @@ -253,7 +253,7 @@ jobs: run: | # use CC to make sure Nim compiler and subsequent test # using the same glibc version. - env CC=gcc make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update + env CC=gcc make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update-from-ci - name: Run fluffy tests (Windows) if: runner.os == 'Windows' diff --git a/.github/workflows/nimbus_verified_proxy.yml b/.github/workflows/nimbus_verified_proxy.yml index 81c9fed9a..7fdd5434b 100644 --- a/.github/workflows/nimbus_verified_proxy.yml +++ b/.github/workflows/nimbus_verified_proxy.yml @@ -181,7 +181,7 @@ jobs: - name: Build Nim and Nimbus-eth1 dependencies run: | - make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update + make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update-from-ci - name: Run verified proxy tests (Windows) if: runner.os == 'Windows' diff --git a/.github/workflows/simulators.yml b/.github/workflows/simulators.yml index 9927b14f3..a4800febd 100644 --- a/.github/workflows/simulators.yml +++ b/.github/workflows/simulators.yml @@ -33,7 +33,7 @@ jobs: - name: Build Nim and deps run: | ncpu=$(nproc) - make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update + make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update-from-ci make -j${ncpu} deps - name: Run Simulators @@ -74,7 +74,7 @@ jobs: - name: Build Nim and deps run: | ncpu=$(sysctl -n hw.ncpu) - make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update + make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update-from-ci make -j${ncpu} deps - name: Run Simulators @@ -144,7 +144,7 @@ jobs: - name: Build Nim and deps run: | ncpu=${NUMBER_OF_PROCESSORS} - mingw32-make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update + mingw32-make -j${ncpu} ARCH_OVERRIDE=x64 CI_CACHE=NimBinaries update-from-ci mingw32-make -j${ncpu} deps - name: Run Simulators diff --git a/Makefile b/Makefile index f6749140c..8877682cd 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,18 @@ FLUFFY_TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(FLUFFY_TOOLS)) ifeq ($(NIM_PARAMS),) # "variables.mk" was not included, so we update the submodules. -GIT_SUBMODULE_UPDATE := git submodule update --init --recursive +# selectively download nimbus-eth2 submodules because we don't need all of it's modules +# also holesky already exceeds github LFS quota +GIT_SUBMODULE_UPDATE := git -c submodule."vendor/nimbus-eth2".update=none submodule update --init --recursive; \ + git submodule update vendor/nimbus-eth2; \ + cd vendor/nimbus-eth2; \ + git submodule update --init vendor/eth2-networks; \ + git submodule update --init vendor/holesky; \ + git submodule update --init vendor/sepolia; \ + git submodule update --init vendor/gnosis-chain-configs; \ + git submodule update --init --recursive vendor/nim-kzg4844; \ + cd ../.. + .DEFAULT: +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ $(GIT_SUBMODULE_UPDATE); \ @@ -186,6 +197,11 @@ update: | update-common rm -rf nimbus.nims && \ $(MAKE) nimbus.nims $(HANDLE_OUTPUT) +update-from-ci: | sanity-checks update-test + rm -rf nimbus.nims && \ + $(MAKE) nimbus.nims $(HANDLE_OUTPUT) + + "$(MAKE)" --no-print-directory deps-common + # builds the tools, wherever they are $(TOOLS): | build deps for D in $(TOOLS_DIRS); do [ -e "$${D}/$@.nim" ] && TOOL_DIR="$${D}" && break; done && \ diff --git a/docker/dist/entry_point.sh b/docker/dist/entry_point.sh index e5fc63c3e..deb12fd42 100755 --- a/docker/dist/entry_point.sh +++ b/docker/dist/entry_point.sh @@ -85,7 +85,7 @@ if [[ "${PLATFORM}" == "Windows_amd64" ]]; then build_rocksdb TARGET_OS=MINGW CXX="${CXX}" - make -j$(nproc) update + make -j$(nproc) update-from-ci make \ -j$(nproc) \ @@ -133,7 +133,7 @@ elif [[ "${PLATFORM}" == "Linux_arm32v7" ]]; then build_rocksdb TARGET_ARCHITECTURE=arm CXX="${CXX}" - make -j$(nproc) update + make -j$(nproc) update-from-ci env CFLAGS="" make \ -j$(nproc) \ @@ -154,7 +154,7 @@ elif [[ "${PLATFORM}" == "Linux_arm64v8" ]]; then build_rocksdb TARGET_ARCHITECTURE=arm64 CXX="${CXX}" - make -j$(nproc) update + make -j$(nproc) update-from-ci make \ -j$(nproc) \ @@ -183,7 +183,7 @@ elif [[ "${PLATFORM}" == "macOS_amd64" ]]; then build_rocksdb TARGET_OS=Darwin CXX="${CXX}" AR="${AR}" - make -j$(nproc) update + make -j$(nproc) update-from-ci make \ -j$(nproc) \ @@ -225,7 +225,7 @@ elif [[ "${PLATFORM}" == "macOS_arm64" ]]; then build_rocksdb TARGET_OS=Darwin TARGET_ARCHITECTURE=arm64 CXX="${CXX}" AR="${AR}" - make -j$(nproc) update + make -j$(nproc) update-from-ci make \ -j$(nproc) \ @@ -259,7 +259,7 @@ else build_rocksdb - make -j$(nproc) update + make -j$(nproc) update-from-ci make \ -j$(nproc) \