add submodules

.gitignore

@@ -3,8 +3,8 @@
 # Executables shall be put in an ignored build/ directory
 /build
 
-# external packages
+# Nimble packages
-/vendor
+/vendor/.nimble
 
 # ntags/ctags output
 /tags

.gitmodules

@@ -0,0 +1,110 @@
+[submodule "vendor/Nim"]
+	path = vendor/Nim
+	url = https://github.com/status-im/Nim.git
+	ignore = dirty
+	branch = nimbus
+[submodule "vendor/nim-chronicles"]
+	path = vendor/nim-chronicles
+	url = https://github.com/status-im/nim-chronicles.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nimcrypto"]
+	path = vendor/nimcrypto
+	url = https://github.com/cheatfate/nimcrypto.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-ranges"]
+	path = vendor/nim-ranges
+	url = https://github.com/status-im/nim-ranges.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-stint"]
+	path = vendor/nim-stint
+	url = https://github.com/status-im/nim-stint.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-rocksdb"]
+	path = vendor/nim-rocksdb
+	url = https://github.com/status-im/nim-rocksdb.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-eth"]
+	path = vendor/nim-eth
+	url = https://github.com/status-im/nim-eth.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-byteutils"]
+	path = vendor/nim-byteutils
+	url = https://github.com/status-im/nim-byteutils.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-http-utils"]
+	path = vendor/nim-http-utils
+	url = https://github.com/status-im/nim-http-utils.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-asyncdispatch2"]
+	path = vendor/nim-asyncdispatch2
+	url = https://github.com/status-im/nim-asyncdispatch2.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-json-rpc"]
+	path = vendor/nim-json-rpc
+	url = https://github.com/status-im/nim-json-rpc.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-faststreams"]
+	path = vendor/nim-faststreams
+	url = https://github.com/status-im/nim-faststreams.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-std-shims"]
+	path = vendor/nim-std-shims
+	url = https://github.com/status-im/nim-std-shims.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-serialization"]
+	path = vendor/nim-serialization
+	url = https://github.com/status-im/nim-serialization.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-json-serialization"]
+	path = vendor/nim-json-serialization
+	url = https://github.com/status-im/nim-json-serialization.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-package-visible-types"]
+	path = vendor/nim-package-visible-types
+	url = https://github.com/zah/nim-package-visible-types.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-secp256k1"]
+	path = vendor/nim-secp256k1
+	url = https://github.com/status-im/nim-secp256k1.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/snappy"]
+	path = vendor/snappy
+	url = https://github.com/jangko/snappy.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-bncurve"]
+	path = vendor/nim-bncurve
+	url = https://github.com/status-im/nim-bncurve.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-confutils"]
+	path = vendor/nim-confutils
+	url = https://github.com/status-im/nim-confutils.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-beacon-chain"]
+	path = vendor/nim-beacon-chain
+	url = https://github.com/status-im/nim-beacon-chain.git
+	ignore = dirty
+	branch = master
+[submodule "vendor/nim-blscurve"]
+	path = vendor/nim-blscurve
+	url = https://github.com/status-im/nim-blscurve
+	ignore = dirty
+	branch = master

Makefile

@@ -13,10 +13,9 @@ GIT_STATUS := git status
 #  version numbers in repo dirs (because those would be in its subdirectories)
 #- duplicated in "env.sh" for the env var with the same name
 NIMBLE_DIR := vendor/.nimble
-NIMBLE := nimble -y
 REPOS_DIR := vendor
 # we want a "recursively expanded" (delayed interpolation) variable here, so we can set CMD in rule recipes
-RUN_CMD_IN_ALL_REPOS = for D in . vendor/Nim $(REPOS); do echo -e "\n\e[32m$${D}:\e[39m"; cd "$$D"; $(CMD); cd - >/dev/null; done
+RUN_CMD_IN_ALL_REPOS = git submodule foreach --recursive --quiet 'echo -e "\n\e[32m$$name:\e[39m"; $(CMD)'; echo -e "\n\e[32m$$(pwd):\e[39m"; $(CMD)
 # absolute path, since it will be run at various subdirectory depths
 ENV_SCRIPT := "$(CURDIR)/env.sh"
 # duplicated in "env.sh" to prepend NIM_DIR/bin to PATH
@@ -28,38 +27,9 @@ BUILD_NIM := cd $(NIM_DIR) && \
 	sh build_all.sh && \
 	$(ENV_SCRIPT) nim c -d:release --noNimblePath -p:compiler --nilseqs:on -o:bin/nimble dist/nimble/src/nimble.nim
 
-#- GitHub repositories for those dependencies that a Nimbus developer might want to
-#  modify and test locally
-#- their order ensures that `nimble develop` will run in a certain package's
-#  repo before Nimble tries to install it as a (direct or indirect) dependency, in
-#  order to avoid duplicate dirs in ".nimble/pgks/"
-#- dependencies not listed here are handled entirely by Nimble with "install -y --depsOnly"
-GITHUB_REPOS := \
-	status-im/nim-chronicles \
-	cheatfate/nimcrypto \
-	status-im/nim-ranges \
-	status-im/nim-stint \
 	OpenSystemsLab/tempfile.nim \
-	status-im/nim-rocksdb \
-	status-im/nim-byteutils \
-	status-im/nim-http-utils \
-	status-im/nim-chronos \
-	status-im/nim-json-rpc \
-	status-im/nim-faststreams \
-	status-im/nim-std-shims \
-	status-im/nim-serialization \
-	status-im/nim-json-serialization \
-	zah/nim-package-visible-types \
-	status-im/nim-secp256k1 \
-	jangko/snappy \
 	status-im/nim-eth \
-	status-im/nim-bncurve \
-	status-im/nim-confutils \
 	status-im/nim-blscurve \
-	status-im/nim-beacon-chain
-# "foo/bar" -> "$(REPOS_DIR)/bar"
-REPOS := $(addprefix $(REPOS_DIR)/, $(foreach github_repo,$(GITHUB_REPOS),$(word 2,$(subst /, ,$(github_repo)))))
-
 .PHONY: all premix persist debug dumper hunter deps github-ssh build-nim update status ntags ctags nimbus test clean mrproper fetch-dlls beacon_node validator_keygen clean_eth2_network_simulation_files eth2_network_simulation
 
 # default target, because it's the first one that doesn't start with '.'
@@ -83,29 +53,19 @@ build:
 
 #- runs only the first time and after `make update` actually updates some repo,
 #  or new repos are cloned, so have "normal" (timestamp-checked) prerequisites here
-deps: $(REPOS) $(NIMBLE_DIR)
+deps: $(NIM_DIR)/bin/nim $(NIMBLE_DIR)
 
 #- depends on Git repos being fetched and our Nim and Nimble being built
 #- runs `nimble develop` in those repos (but not in the Nimbus repo) - not
 #  parallelizable, because package order matters
 #- installs any remaining Nimbus dependency (those not in $(REPOS))
-$(NIMBLE_DIR): | $(REPOS) $(NIM_DIR)
+$(NIMBLE_DIR): | $(NIM_DIR)/bin/nim
-	$(eval CMD := [ "$$$$D" = "." ] && continue; $(ENV_SCRIPT) $(NIMBLE) develop)
+	mkdir -p $(NIMBLE_DIR)/pkgs
-	$(RUN_CMD_IN_ALL_REPOS)
+	git submodule foreach --quiet '\
-	$(ENV_SCRIPT) $(NIMBLE) install --depsOnly
+		[ `ls -1 *.nimble 2>/dev/null | wc -l ` -gt 0 ] && { \
-
+			mkdir -p $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head;\
-#- clones the Git repos
+			echo -e "$$(pwd)\n$$(pwd)" > $$toplevel/$(NIMBLE_DIR)/pkgs/$${sm_path#*/}-#head/$${sm_path#*/}.nimble-link;\
-#- can run in parallel with `make -jN`
+		}'
-#- deletes the ".nimble" dir to force Nimble's package db regeneration (useful for newly added repositories to REPOS)
-$(REPOS):
-	$(eval PROJ_NAME := $(subst $(REPOS_DIR)/,,$@))
-	$(GIT_CLONE) https://github.com/$(filter %/$(PROJ_NAME),$(GITHUB_REPOS)) $@ && \
-		rm -rf $(NIMBLE_DIR)
-
-# clones and builds the Nim compiler and Nimble
-$(NIM_DIR):
-	$(GIT_CLONE) --depth 1 https://github.com/status-im/Nim $@
-	$(BUILD_NIM)
 
 # builds and runs all tests
 test: | build deps
@@ -120,27 +80,26 @@ mrproper: clean
 	rm -rf vendor
 
 # for when you have write access to a repo and you want to use SSH keys
+# TODO: https://stackoverflow.com/questions/39894103/can-i-override-the-url-of-a-nested-git-submodule-without-forking
 github-ssh:
 	sed -i 's#https://github.com/#git@github.com:#' .git/config $(NIM_DIR)/.git/config $(REPOS_DIR)/*/.git/config
 
 #- re-builds the Nim compiler (not usually needed, because `make update` does it when necessary)
-build-nim: | $(NIM_DIR)
+build-nim: | deps
 	$(BUILD_NIM)
 
 #- runs `git pull` in all Git repos, if there are new commits in the remote branch
 #- rebuilds the Nim compiler after pulling new commits
 #- deletes the ".nimble" dir to force the execution of the "deps" target if at least one repo was updated
 #- ignores non-zero exit codes from [...] tests
-update: | $(REPOS)
+$(NIM_DIR)/bin/nim update:
-	$(eval CMD := \
+	git submodule update --init --recursive --rebase
-		git remote update && \
+	git submodule foreach --recursive 'git checkout $$(git config -f $$toplevel/.gitmodules submodule.$$name.branch || echo master)'
-		[ -n "$$$$(git rev-parse @{u})" -a "$$$$(git rev-parse @)" != "$$$$(git rev-parse @{u})" ] && \
+	rm -rf $(NIMBLE_DIR)
-		REPO_UPDATED=1 && \
+	[ -e $(NIM_DIR)/bin/nim ] || { $(BUILD_NIM); }
-		$(GIT_PULL) && \
+
-		{ [ "$$$$D" = "$(NIM_DIR)" ] && { cd - >/dev/null; $(BUILD_NIM); }; } \
+update-remote:
-		|| true \
+	git submodule update --remote --recursive --rebase
-	)
-	REPO_UPDATED=0; $(RUN_CMD_IN_ALL_REPOS); [ $$REPO_UPDATED = 1 ] && echo -e "\nAt least one repo updated. Deleting '$(NIMBLE_DIR)'." && rm -rf $(NIMBLE_DIR) || true
 
 # runs `git status` in all Git repos
 status: | $(REPOS)

vendor/Nim

@@ -0,0 +1 @@
+Subproject commit 4b2f0035007d4060dd16279c8c5e49c0cfb3b8f7

vendor/nim-asyncdispatch2

@@ -0,0 +1 @@
+Subproject commit 88933e8f1bd9fce513a9ea885d1859efd14386ee

vendor/nim-beacon-chain

@@ -0,0 +1 @@
+Subproject commit 7f1bddb267dd11d340675204854e2f2d30be2433

vendor/nim-blscurve

@@ -0,0 +1 @@
+Subproject commit 13a89c9d73c7029b53a7a9dd7cededfd51348df7

vendor/nim-bncurve

@@ -0,0 +1 @@
+Subproject commit 8c74ae19a9eda65f6e7c29dd4efd88f5a4e977f3

vendor/nim-byteutils

@@ -0,0 +1 @@
+Subproject commit 9820877e4c1a2a9a7be182d6ab85b5db5104311b

vendor/nim-chronicles

@@ -0,0 +1 @@
+Subproject commit 98e334f63e31671f1cb05dbb3364173a8cd5a8e1

vendor/nim-confutils

@@ -0,0 +1 @@
+Subproject commit 531b4033378447dc4f196f6fbf040a197f7be307

vendor/nim-eth

@@ -0,0 +1 @@
+Subproject commit 0bc80d73cdd973c24b5d10c86a74904373a944b9

vendor/nim-faststreams

@@ -0,0 +1 @@
+Subproject commit 31590a79ece0c26bc46a25b500ede396900c93cb

vendor/nim-http-utils

@@ -0,0 +1 @@
+Subproject commit 1616f099a4b505abe453adb004696d5a4707e6d5

vendor/nim-json-rpc

@@ -0,0 +1 @@
+Subproject commit 1083b2972a183c4e1221f0ea3671dc7ad193f92d

vendor/nim-json-serialization

@@ -0,0 +1 @@
+Subproject commit edda2577a36e9e0dfda56298d98c40560f521d77

vendor/nim-package-visible-types

@@ -0,0 +1 @@
+Subproject commit cbdee48feb05045c673e7d2cab16ae0f5c957157

vendor/nim-ranges

@@ -0,0 +1 @@
+Subproject commit 0156cbd7f47fb85cb7a2c957ca7a707a50e6ea0e

vendor/nim-rocksdb

@@ -0,0 +1 @@
+Subproject commit dce00a6ff80a7b3cf97c57b44857bd933db4a196

vendor/nim-secp256k1

@@ -0,0 +1 @@
+Subproject commit 53baf442f2dd711768b96688f9ac1bfed90e7d61

vendor/nim-serialization

@@ -0,0 +1 @@
+Subproject commit b4216704f3e5f314ba14766c5d0f7cff194e56f8

vendor/nim-std-shims

@@ -0,0 +1 @@
+Subproject commit b18e8877a72f8f8c075390a4bae1aa755a749329

vendor/nim-stint

@@ -0,0 +1 @@
+Subproject commit d002fbb1559989ea75dfadd6cbbe9674c6987675

vendor/nimcrypto

@@ -0,0 +1 @@
+Subproject commit b06fc1a658e8930eb2ed7423f52c2968104df3df

vendor/snappy

@@ -0,0 +1 @@
+Subproject commit ff70dfbb47ce782bdbc29598a7beba5267b285f0