diff --git a/.gitignore b/.gitignore index 8a69d8ee6..58ce586f2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,8 @@ # Executables shall be put in an ignored build/ directory /build -# external packages -/vendor +# Nimble packages +/vendor/.nimble # ntags/ctags output /tags diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..3ddd3789c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,110 @@ +[submodule "vendor/Nim"] + path = vendor/Nim + url = https://github.com/status-im/Nim.git + ignore = dirty + branch = nimbus +[submodule "vendor/nim-chronicles"] + path = vendor/nim-chronicles + url = https://github.com/status-im/nim-chronicles.git + ignore = dirty + branch = master +[submodule "vendor/nimcrypto"] + path = vendor/nimcrypto + url = https://github.com/cheatfate/nimcrypto.git + ignore = dirty + branch = master +[submodule "vendor/nim-ranges"] + path = vendor/nim-ranges + url = https://github.com/status-im/nim-ranges.git + ignore = dirty + branch = master +[submodule "vendor/nim-stint"] + path = vendor/nim-stint + url = https://github.com/status-im/nim-stint.git + ignore = dirty + branch = master +[submodule "vendor/nim-rocksdb"] + path = vendor/nim-rocksdb + url = https://github.com/status-im/nim-rocksdb.git + ignore = dirty + branch = master +[submodule "vendor/nim-eth"] + path = vendor/nim-eth + url = https://github.com/status-im/nim-eth.git + ignore = dirty + branch = master +[submodule "vendor/nim-byteutils"] + path = vendor/nim-byteutils + url = https://github.com/status-im/nim-byteutils.git + ignore = dirty + branch = master +[submodule "vendor/nim-http-utils"] + path = vendor/nim-http-utils + url = https://github.com/status-im/nim-http-utils.git + ignore = dirty + branch = master +[submodule "vendor/nim-asyncdispatch2"] + path = vendor/nim-asyncdispatch2 + url = https://github.com/status-im/nim-asyncdispatch2.git + ignore = dirty + branch = master +[submodule "vendor/nim-json-rpc"] + path = vendor/nim-json-rpc + url = https://github.com/status-im/nim-json-rpc.git + ignore = dirty + branch = master +[submodule "vendor/nim-faststreams"] + path = vendor/nim-faststreams + url = https://github.com/status-im/nim-faststreams.git + ignore = dirty + branch = master +[submodule "vendor/nim-std-shims"] + path = vendor/nim-std-shims + url = https://github.com/status-im/nim-std-shims.git + ignore = dirty + branch = master +[submodule "vendor/nim-serialization"] + path = vendor/nim-serialization + url = https://github.com/status-im/nim-serialization.git + ignore = dirty + branch = master +[submodule "vendor/nim-json-serialization"] + path = vendor/nim-json-serialization + url = https://github.com/status-im/nim-json-serialization.git + ignore = dirty + branch = master +[submodule "vendor/nim-package-visible-types"] + path = vendor/nim-package-visible-types + url = https://github.com/zah/nim-package-visible-types.git + ignore = dirty + branch = master +[submodule "vendor/nim-secp256k1"] + path = vendor/nim-secp256k1 + url = https://github.com/status-im/nim-secp256k1.git + ignore = dirty + branch = master +[submodule "vendor/snappy"] + path = vendor/snappy + url = https://github.com/jangko/snappy.git + ignore = dirty + branch = master +[submodule "vendor/nim-bncurve"] + path = vendor/nim-bncurve + url = https://github.com/status-im/nim-bncurve.git + ignore = dirty + branch = master +[submodule "vendor/nim-confutils"] + path = vendor/nim-confutils + url = https://github.com/status-im/nim-confutils.git + ignore = dirty + branch = master +[submodule "vendor/nim-beacon-chain"] + path = vendor/nim-beacon-chain + url = https://github.com/status-im/nim-beacon-chain.git + ignore = dirty + branch = master +[submodule "vendor/nim-blscurve"] + path = vendor/nim-blscurve + url = https://github.com/status-im/nim-blscurve + ignore = dirty + branch = master diff --git a/Makefile b/Makefile index ff4f1995f..bb62ffa36 100644 --- a/Makefile +++ b/Makefile @@ -13,10 +13,9 @@ GIT_STATUS := git status # version numbers in repo dirs (because those would be in its subdirectories) #- duplicated in "env.sh" for the env var with the same name NIMBLE_DIR := vendor/.nimble -NIMBLE := nimble -y REPOS_DIR := vendor # we want a "recursively expanded" (delayed interpolation) variable here, so we can set CMD in rule recipes -RUN_CMD_IN_ALL_REPOS = for D in . vendor/Nim $(REPOS); do echo -e "\n\e[32m$${D}:\e[39m"; cd "$$D"; $(CMD); cd - >/dev/null; done +RUN_CMD_IN_ALL_REPOS = git submodule foreach --recursive --quiet 'echo -e "\n\e[32m$$name:\e[39m"; $(CMD)'; echo -e "\n\e[32m$$(pwd):\e[39m"; $(CMD) # absolute path, since it will be run at various subdirectory depths ENV_SCRIPT := "$(CURDIR)/env.sh" # duplicated in "env.sh" to prepend NIM_DIR/bin to PATH @@ -28,38 +27,9 @@ BUILD_NIM := cd $(NIM_DIR) && \ sh build_all.sh && \ $(ENV_SCRIPT) nim c -d:release --noNimblePath -p:compiler --nilseqs:on -o:bin/nimble dist/nimble/src/nimble.nim -#- GitHub repositories for those dependencies that a Nimbus developer might want to -# modify and test locally -#- their order ensures that `nimble develop` will run in a certain package's -# repo before Nimble tries to install it as a (direct or indirect) dependency, in -# order to avoid duplicate dirs in ".nimble/pgks/" -#- dependencies not listed here are handled entirely by Nimble with "install -y --depsOnly" -GITHUB_REPOS := \ - status-im/nim-chronicles \ - cheatfate/nimcrypto \ - status-im/nim-ranges \ - status-im/nim-stint \ OpenSystemsLab/tempfile.nim \ - status-im/nim-rocksdb \ - status-im/nim-byteutils \ - status-im/nim-http-utils \ - status-im/nim-chronos \ - status-im/nim-json-rpc \ - status-im/nim-faststreams \ - status-im/nim-std-shims \ - status-im/nim-serialization \ - status-im/nim-json-serialization \ - zah/nim-package-visible-types \ - status-im/nim-secp256k1 \ - jangko/snappy \ status-im/nim-eth \ - status-im/nim-bncurve \ - status-im/nim-confutils \ status-im/nim-blscurve \ - status-im/nim-beacon-chain -# "foo/bar" -> "$(REPOS_DIR)/bar" -REPOS := $(addprefix $(REPOS_DIR)/, $(foreach github_repo,$(GITHUB_REPOS),$(word 2,$(subst /, ,$(github_repo))))) - .PHONY: all premix persist debug dumper hunter deps github-ssh build-nim update status ntags ctags nimbus test clean mrproper fetch-dlls beacon_node validator_keygen clean_eth2_network_simulation_files eth2_network_simulation # default target, because it's the first one that doesn't start with '.' @@ -83,29 +53,19 @@ build: #- runs only the first time and after `make update` actually updates some repo, # or new repos are cloned, so have "normal" (timestamp-checked) prerequisites here -deps: $(REPOS) $(NIMBLE_DIR) +deps: $(NIM_DIR)/bin/nim $(NIMBLE_DIR) #- depends on Git repos being fetched and our Nim and Nimble being built #- runs `nimble develop` in those repos (but not in the Nimbus repo) - not # parallelizable, because package order matters #- installs any remaining Nimbus dependency (those not in $(REPOS)) -$(NIMBLE_DIR): | $(REPOS) $(NIM_DIR) - $(eval CMD := [ "$$$$D" = "." ] && continue; $(ENV_SCRIPT) $(NIMBLE) develop) - $(RUN_CMD_IN_ALL_REPOS) - $(ENV_SCRIPT) $(NIMBLE) install --depsOnly - -#- clones the Git repos -#- can run in parallel with `make -jN` -#- deletes the ".nimble" dir to force Nimble's package db regeneration (useful for newly added repositories to REPOS) -$(REPOS): - $(eval PROJ_NAME := $(subst $(REPOS_DIR)/,,$@)) - $(GIT_CLONE) https://github.com/$(filter %/$(PROJ_NAME),$(GITHUB_REPOS)) $@ && \ - rm -rf $(NIMBLE_DIR) - -# clones and builds the Nim compiler and Nimble -$(NIM_DIR): - $(GIT_CLONE) --depth 1 https://github.com/status-im/Nim $@ - $(BUILD_NIM) +$(NIMBLE_DIR): | $(NIM_DIR)/bin/nim + mkdir -p $(NIMBLE_DIR)/pkgs + git submodule foreach --quiet '\ + [ `ls -1 *.nimble 2>/dev/null | wc -l ` -gt 0 ] && { \ + mkdir -p $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head;\ + echo -e "$$(pwd)\n$$(pwd)" > $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head/$${sm_path#*/}.nimble-link;\ + }' # builds and runs all tests test: | build deps @@ -120,27 +80,26 @@ mrproper: clean rm -rf vendor # for when you have write access to a repo and you want to use SSH keys +# TODO: https://stackoverflow.com/questions/39894103/can-i-override-the-url-of-a-nested-git-submodule-without-forking github-ssh: sed -i 's#https://github.com/#git@github.com:#' .git/config $(NIM_DIR)/.git/config $(REPOS_DIR)/*/.git/config #- re-builds the Nim compiler (not usually needed, because `make update` does it when necessary) -build-nim: | $(NIM_DIR) +build-nim: | deps $(BUILD_NIM) #- runs `git pull` in all Git repos, if there are new commits in the remote branch #- rebuilds the Nim compiler after pulling new commits #- deletes the ".nimble" dir to force the execution of the "deps" target if at least one repo was updated #- ignores non-zero exit codes from [...] tests -update: | $(REPOS) - $(eval CMD := \ - git remote update && \ - [ -n "$$$$(git rev-parse @{u})" -a "$$$$(git rev-parse @)" != "$$$$(git rev-parse @{u})" ] && \ - REPO_UPDATED=1 && \ - $(GIT_PULL) && \ - { [ "$$$$D" = "$(NIM_DIR)" ] && { cd - >/dev/null; $(BUILD_NIM); }; } \ - || true \ - ) - REPO_UPDATED=0; $(RUN_CMD_IN_ALL_REPOS); [ $$REPO_UPDATED = 1 ] && echo -e "\nAt least one repo updated. Deleting '$(NIMBLE_DIR)'." && rm -rf $(NIMBLE_DIR) || true +$(NIM_DIR)/bin/nim update: + git submodule update --init --recursive --rebase + git submodule foreach --recursive 'git checkout $$(git config -f $$toplevel/.gitmodules submodule.$$name.branch || echo master)' + rm -rf $(NIMBLE_DIR) + [ -e $(NIM_DIR)/bin/nim ] || { $(BUILD_NIM); } + +update-remote: + git submodule update --remote --recursive --rebase # runs `git status` in all Git repos status: | $(REPOS) diff --git a/vendor/Nim b/vendor/Nim new file mode 160000 index 000000000..4b2f00350 --- /dev/null +++ b/vendor/Nim @@ -0,0 +1 @@ +Subproject commit 4b2f0035007d4060dd16279c8c5e49c0cfb3b8f7 diff --git a/vendor/nim-asyncdispatch2 b/vendor/nim-asyncdispatch2 new file mode 160000 index 000000000..88933e8f1 --- /dev/null +++ b/vendor/nim-asyncdispatch2 @@ -0,0 +1 @@ +Subproject commit 88933e8f1bd9fce513a9ea885d1859efd14386ee diff --git a/vendor/nim-beacon-chain b/vendor/nim-beacon-chain new file mode 160000 index 000000000..7f1bddb26 --- /dev/null +++ b/vendor/nim-beacon-chain @@ -0,0 +1 @@ +Subproject commit 7f1bddb267dd11d340675204854e2f2d30be2433 diff --git a/vendor/nim-blscurve b/vendor/nim-blscurve new file mode 160000 index 000000000..13a89c9d7 --- /dev/null +++ b/vendor/nim-blscurve @@ -0,0 +1 @@ +Subproject commit 13a89c9d73c7029b53a7a9dd7cededfd51348df7 diff --git a/vendor/nim-bncurve b/vendor/nim-bncurve new file mode 160000 index 000000000..8c74ae19a --- /dev/null +++ b/vendor/nim-bncurve @@ -0,0 +1 @@ +Subproject commit 8c74ae19a9eda65f6e7c29dd4efd88f5a4e977f3 diff --git a/vendor/nim-byteutils b/vendor/nim-byteutils new file mode 160000 index 000000000..9820877e4 --- /dev/null +++ b/vendor/nim-byteutils @@ -0,0 +1 @@ +Subproject commit 9820877e4c1a2a9a7be182d6ab85b5db5104311b diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles new file mode 160000 index 000000000..98e334f63 --- /dev/null +++ b/vendor/nim-chronicles @@ -0,0 +1 @@ +Subproject commit 98e334f63e31671f1cb05dbb3364173a8cd5a8e1 diff --git a/vendor/nim-confutils b/vendor/nim-confutils new file mode 160000 index 000000000..531b40333 --- /dev/null +++ b/vendor/nim-confutils @@ -0,0 +1 @@ +Subproject commit 531b4033378447dc4f196f6fbf040a197f7be307 diff --git a/vendor/nim-eth b/vendor/nim-eth new file mode 160000 index 000000000..0bc80d73c --- /dev/null +++ b/vendor/nim-eth @@ -0,0 +1 @@ +Subproject commit 0bc80d73cdd973c24b5d10c86a74904373a944b9 diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams new file mode 160000 index 000000000..31590a79e --- /dev/null +++ b/vendor/nim-faststreams @@ -0,0 +1 @@ +Subproject commit 31590a79ece0c26bc46a25b500ede396900c93cb diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils new file mode 160000 index 000000000..1616f099a --- /dev/null +++ b/vendor/nim-http-utils @@ -0,0 +1 @@ +Subproject commit 1616f099a4b505abe453adb004696d5a4707e6d5 diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc new file mode 160000 index 000000000..1083b2972 --- /dev/null +++ b/vendor/nim-json-rpc @@ -0,0 +1 @@ +Subproject commit 1083b2972a183c4e1221f0ea3671dc7ad193f92d diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization new file mode 160000 index 000000000..edda2577a --- /dev/null +++ b/vendor/nim-json-serialization @@ -0,0 +1 @@ +Subproject commit edda2577a36e9e0dfda56298d98c40560f521d77 diff --git a/vendor/nim-package-visible-types b/vendor/nim-package-visible-types new file mode 160000 index 000000000..cbdee48fe --- /dev/null +++ b/vendor/nim-package-visible-types @@ -0,0 +1 @@ +Subproject commit cbdee48feb05045c673e7d2cab16ae0f5c957157 diff --git a/vendor/nim-ranges b/vendor/nim-ranges new file mode 160000 index 000000000..0156cbd7f --- /dev/null +++ b/vendor/nim-ranges @@ -0,0 +1 @@ +Subproject commit 0156cbd7f47fb85cb7a2c957ca7a707a50e6ea0e diff --git a/vendor/nim-rocksdb b/vendor/nim-rocksdb new file mode 160000 index 000000000..dce00a6ff --- /dev/null +++ b/vendor/nim-rocksdb @@ -0,0 +1 @@ +Subproject commit dce00a6ff80a7b3cf97c57b44857bd933db4a196 diff --git a/vendor/nim-secp256k1 b/vendor/nim-secp256k1 new file mode 160000 index 000000000..53baf442f --- /dev/null +++ b/vendor/nim-secp256k1 @@ -0,0 +1 @@ +Subproject commit 53baf442f2dd711768b96688f9ac1bfed90e7d61 diff --git a/vendor/nim-serialization b/vendor/nim-serialization new file mode 160000 index 000000000..b4216704f --- /dev/null +++ b/vendor/nim-serialization @@ -0,0 +1 @@ +Subproject commit b4216704f3e5f314ba14766c5d0f7cff194e56f8 diff --git a/vendor/nim-std-shims b/vendor/nim-std-shims new file mode 160000 index 000000000..b18e8877a --- /dev/null +++ b/vendor/nim-std-shims @@ -0,0 +1 @@ +Subproject commit b18e8877a72f8f8c075390a4bae1aa755a749329 diff --git a/vendor/nim-stint b/vendor/nim-stint new file mode 160000 index 000000000..d002fbb15 --- /dev/null +++ b/vendor/nim-stint @@ -0,0 +1 @@ +Subproject commit d002fbb1559989ea75dfadd6cbbe9674c6987675 diff --git a/vendor/nimcrypto b/vendor/nimcrypto new file mode 160000 index 000000000..b06fc1a65 --- /dev/null +++ b/vendor/nimcrypto @@ -0,0 +1 @@ +Subproject commit b06fc1a658e8930eb2ed7423f52c2968104df3df diff --git a/vendor/snappy b/vendor/snappy new file mode 160000 index 000000000..ff70dfbb4 --- /dev/null +++ b/vendor/snappy @@ -0,0 +1 @@ +Subproject commit ff70dfbb47ce782bdbc29598a7beba5267b285f0