diff --git a/.gitmodules b/.gitmodules index 607a141ce..e6c2a4929 100644 --- a/.gitmodules +++ b/.gitmodules @@ -43,11 +43,6 @@ url = https://github.com/status-im/nim-http-utils.git ignore = dirty branch = master -[submodule "vendor/nim-asyncdispatch2"] - path = vendor/nim-asyncdispatch2 - url = https://github.com/status-im/nim-asyncdispatch2.git - ignore = dirty - branch = master [submodule "vendor/nim-json-rpc"] path = vendor/nim-json-rpc url = https://github.com/status-im/nim-json-rpc.git @@ -118,3 +113,8 @@ url = https://github.com/nim-lang/nimble.git ignore = dirty branch = master +[submodule "vendor/nim-chronos"] + path = vendor/nim-chronos + url = https://github.com/status-im/nim-chronos.git + ignore = dirty + branch = master diff --git a/Makefile b/Makefile index 159bf1372..0bf4ab484 100644 --- a/Makefile +++ b/Makefile @@ -14,20 +14,48 @@ GIT_STATUS := git status #- duplicated in "env.sh" for the env var with the same name NIMBLE_DIR := vendor/.nimble REPOS_DIR := vendor +ifeq ($(OS), Windows_NT) + PWD := pwd -W +else + PWD := pwd +endif # we want a "recursively expanded" (delayed interpolation) variable here, so we can set CMD in rule recipes -RUN_CMD_IN_ALL_REPOS = git submodule foreach --recursive --quiet 'echo -e "\n\e[32m$$name:\e[39m"; $(CMD)'; echo -e "\n\e[32m$$(pwd):\e[39m"; $(CMD) +RUN_CMD_IN_ALL_REPOS = git submodule foreach --recursive --quiet 'echo -e "\n\e[32m$$name:\e[39m"; $(CMD)'; echo -e "\n\e[32m$$($(PWD)):\e[39m"; $(CMD) # absolute path, since it will be run at various subdirectory depths ENV_SCRIPT := "$(CURDIR)/env.sh" # duplicated in "env.sh" to prepend NIM_DIR/bin to PATH NIM_DIR := vendor/Nim #- forces a rebuild of csources, Nimble and a complete compiler rebuild, in case we're called after pulling a new Nim version #- uses our Git submodules for csources and Nimble (Git doesn't let us place them in another submodule) +#- build_all.sh looks at the parent dir to decide whether to copy the resulting csources binary there, +# but this is broken when using symlinks, so build csources separately (we get parallel compiling as a bonus) +#- Windows is a special case, as usual #- recompiles Nimble with -d:release until we upgrade to nim-0.20 where koch does it by default # (we don't actually use Nimble in this Makefile, but we need it in submodules to manually run tests: "../../env.sh nimble test") +ifeq ($(OS), Windows_NT) + BUILD_CSOURCES := \ + $(MAKE) myos=windows clean && \ + $(MAKE) myos=windows CC=gcc LD=gcc +else + BUILD_CSOURCES := \ + $(MAKE) clean && \ + $(MAKE) LD=$(CC) +endif BUILD_NIM := cd $(NIM_DIR) && \ rm -rf bin/nim_csources csources dist/nimble && \ ln -sr ../Nim-csources csources && \ + mkdir -p dist && \ ln -sr ../nimble dist/nimble && \ + cd csources && \ + $(BUILD_CSOURCES) && \ + cd - && \ + [ -e csources/bin ] && { \ + cp -a csources/bin/nim bin/nim && \ + cp -a csources/bin/nim bin/nim_csources && \ + rm -rf csources/bin; \ + } || { \ + cp -a bin/nim bin/nim_csources; \ + } && \ sh build_all.sh && \ $(ENV_SCRIPT) nim c -d:release --noNimblePath -p:compiler --nilseqs:on -o:bin/nimble dist/nimble/src/nimble.nim @@ -69,7 +97,7 @@ $(NIMBLE_DIR): | $(NIM_DIR)/bin/nim git submodule foreach --quiet '\ [ `ls -1 *.nimble 2>/dev/null | wc -l ` -gt 0 ] && { \ mkdir -p $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head;\ - echo -e "$$(pwd)\n$$(pwd)" > $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head/$${sm_path#*/}.nimble-link;\ + echo -e "$$($(PWD))\n$$($(PWD))" > $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head/$${sm_path#*/}.nimble-link;\ } || true' # builds and runs all tests @@ -78,7 +106,7 @@ test: | build deps # usual cleaning clean: - rm -rf build/{nimbus,all_tests,beacon_node,validator_keygen,*.exe} $(NIMBLE_DIR) + rm -rf build/{nimbus,all_tests,beacon_node,validator_keygen,*.exe} $(NIMBLE_DIR) $(NIM_DIR)/bin/nim # dangerous cleaning, because you may have not-yet-pushed branches and commits in those vendor repos you're about to delete mrproper: clean @@ -92,22 +120,22 @@ github-ssh: ;done #- re-builds the Nim compiler (not usually needed, because `make update` does it when necessary) +#- allows parallel building with the '+' prefix build-nim: | deps - $(BUILD_NIM) + + $(BUILD_NIM) -#- inits and updates the Git submodules, making sure we're not left on a detached HEAD +#- initialises and updates the Git submodules #- deletes the ".nimble" dir to force the execution of the "deps" target -#- ignores non-zero exit codes from [...] tests +#- allows parallel building with the '+' prefix #- TODO: rebuild the Nim compiler after the corresponding submodule is updated $(NIM_DIR)/bin/nim update: - git submodule update --init --recursive --rebase - git submodule foreach --recursive 'git checkout $$(git config -f $$toplevel/.gitmodules submodule.$$name.branch || echo master)' + git submodule update --init --recursive rm -rf $(NIMBLE_DIR) - [ -e $(NIM_DIR)/bin/nim ] || { $(BUILD_NIM); } + + [ -e $(NIM_DIR)/bin/nim ] || { $(BUILD_NIM); } # don't use this target, or you risk updating dependency repos that are not ready to be used in Nimbus update-remote: - git submodule update --remote --recursive --rebase + git submodule update --remote --rebase # runs `git status` in all Git repos status: | $(REPOS) diff --git a/README.md b/README.md index 156d1ecd0..f96cba735 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ make test To pull the latest changes in all the Git repositories involved: ```bash +git pull make update ``` @@ -100,6 +101,59 @@ You can now follow those instructions in the previous section by replacing `make - the Premix debugging tools are [documented separately](premix/readme.md) +#### Git submodule workflow + +Working on a dependency: + +```bash +cd vendor/nim-chronicles +git checkout -b mybranch +# make some changes +git status +git commit -a +git push origin mybranch +# create a GitHub PR and wait for it to be approved and merged +git checkout master +git pull +git branch -d mybranch +# realise that the merge was done without "--no-ff" +git branch -D mybranch +# update the submodule's commit in the superproject +cd ../.. +git status +git add vendor/nim-chronicles +git commit +``` + +It's important that you only update the submodule commit after it's available upstream. + +You might want to do this on a new branch of the superproject, so you can make +a GitHub PR for it and see the CI test results. + +Don't update all Git submodules at once, just because you found the relevant +Git command or `make` target. You risk updating submodules to other people's +latest commits when they are not ready to be used in the superproject. + +Adding the submodule "https://github.com/status-im/foo" to "vendor/foo": + +```bash +./add_submodule.sh status-im/foo +``` + +Removing the submodule "vendor/bar": + +```bash +git submodule deinit -f -- vendor/bar +git rm -f vendor/bar +``` + +Checking out older commits, either to bisect something or to reproduce an older build: + +```bash +git checkout +make -j8 clean update +``` + ### Troubleshooting Report any errors you encounter, please, if not [already documented](https://github.com/status-im/nimbus/issues)! diff --git a/vendor/nim-asyncdispatch2 b/vendor/nim-asyncdispatch2 deleted file mode 160000 index 88933e8f1..000000000 --- a/vendor/nim-asyncdispatch2 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 88933e8f1bd9fce513a9ea885d1859efd14386ee diff --git a/vendor/nim-beacon-chain b/vendor/nim-beacon-chain index 7f1bddb26..56a48cc60 160000 --- a/vendor/nim-beacon-chain +++ b/vendor/nim-beacon-chain @@ -1 +1 @@ -Subproject commit 7f1bddb267dd11d340675204854e2f2d30be2433 +Subproject commit 56a48cc60f1b1feeaadda4912dae9554ed51ae63 diff --git a/vendor/nim-blscurve b/vendor/nim-blscurve index 13a89c9d7..1d41c83ca 160000 --- a/vendor/nim-blscurve +++ b/vendor/nim-blscurve @@ -1 +1 @@ -Subproject commit 13a89c9d73c7029b53a7a9dd7cededfd51348df7 +Subproject commit 1d41c83ca68e3422d2c54359d38d10fa103d43f9 diff --git a/vendor/nim-chronos b/vendor/nim-chronos new file mode 160000 index 000000000..79376dab2 --- /dev/null +++ b/vendor/nim-chronos @@ -0,0 +1 @@ +Subproject commit 79376dab2044724a4bf527aa26e86afb42bb2ad8 diff --git a/vendor/nim-eth b/vendor/nim-eth index 0bc80d73c..5f6155bd2 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit 0bc80d73cdd973c24b5d10c86a74904373a944b9 +Subproject commit 5f6155bd235654ac25b37bb781ee2a6a671d2b7d diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc index 1083b2972..53ed44dfa 160000 --- a/vendor/nim-json-rpc +++ b/vendor/nim-json-rpc @@ -1 +1 @@ -Subproject commit 1083b2972a183c4e1221f0ea3671dc7ad193f92d +Subproject commit 53ed44dfa59f48ef8e3edfec2c78222458b6454a