From 351cac6b3c8c60de77487b6cd33d034c2439dbd9 Mon Sep 17 00:00:00 2001 From: darshankabariya Date: Tue, 6 Jan 2026 13:04:59 +0530 Subject: [PATCH] Initial commit for vendor deprect --- .gitmodules | 187 ++---------------------- Makefile | 116 +++++++++------ apps/wakunode2/wakunode2.nim | 5 +- config.nims | 11 +- vendor/db_connector | 1 - vendor/dnsclient.nim | 1 - vendor/nim-bearssl | 1 - vendor/nim-chronicles | 1 - vendor/nim-chronos | 1 - vendor/nim-confutils | 1 - vendor/nim-dnsdisc | 1 - vendor/nim-eth | 1 - vendor/nim-faststreams | 1 - vendor/nim-http-utils | 1 - vendor/nim-json-rpc | 1 - vendor/nim-json-serialization | 1 - vendor/nim-libbacktrace | 1 - vendor/nim-libp2p | 1 - vendor/nim-metrics | 1 - vendor/nim-minilru | 1 - vendor/nim-nat-traversal | 1 - vendor/nim-presto | 1 - vendor/nim-regex | 1 - vendor/nim-results | 1 - vendor/nim-secp256k1 | 1 - vendor/nim-serialization | 1 - vendor/nim-sqlite3-abi | 1 - vendor/nim-stew | 1 - vendor/nim-stint | 1 - vendor/nim-taskpools | 1 - vendor/nim-testutils | 1 - vendor/nim-toml-serialization | 1 - vendor/nim-unicodedb | 1 - vendor/nim-unittest2 | 1 - vendor/nim-web3 | 1 - vendor/nim-websock | 1 - vendor/nim-zlib | 1 - vendor/nimbus-build-system | 1 - vendor/nimcrypto | 1 - waku.nimble | 122 +++++++++++----- waku/waku_filter_v2/client.nim | 2 +- waku/waku_store_sync/reconciliation.nim | 8 +- 42 files changed, 183 insertions(+), 303 deletions(-) delete mode 160000 vendor/db_connector delete mode 160000 vendor/dnsclient.nim delete mode 160000 vendor/nim-bearssl delete mode 160000 vendor/nim-chronicles delete mode 160000 vendor/nim-chronos delete mode 160000 vendor/nim-confutils delete mode 160000 vendor/nim-dnsdisc delete mode 160000 vendor/nim-eth delete mode 160000 vendor/nim-faststreams delete mode 160000 vendor/nim-http-utils delete mode 160000 vendor/nim-json-rpc delete mode 160000 vendor/nim-json-serialization delete mode 160000 vendor/nim-libbacktrace delete mode 160000 vendor/nim-libp2p delete mode 160000 vendor/nim-metrics delete mode 160000 vendor/nim-minilru delete mode 160000 vendor/nim-nat-traversal delete mode 160000 vendor/nim-presto delete mode 160000 vendor/nim-regex delete mode 160000 vendor/nim-results delete mode 160000 vendor/nim-secp256k1 delete mode 160000 vendor/nim-serialization delete mode 160000 vendor/nim-sqlite3-abi delete mode 160000 vendor/nim-stew delete mode 160000 vendor/nim-stint delete mode 160000 vendor/nim-taskpools delete mode 160000 vendor/nim-testutils delete mode 160000 vendor/nim-toml-serialization delete mode 160000 vendor/nim-unicodedb delete mode 160000 vendor/nim-unittest2 delete mode 160000 vendor/nim-web3 delete mode 160000 vendor/nim-websock delete mode 160000 vendor/nim-zlib delete mode 160000 vendor/nimbus-build-system delete mode 160000 vendor/nimcrypto diff --git a/.gitmodules b/.gitmodules index 4d56c4333..d05c751dd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,189 +1,24 @@ -[submodule "vendor/nim-eth"] - path = vendor/nim-eth - url = https://github.com/status-im/nim-eth.git - ignore = dirty - branch = master -[submodule "vendor/nim-secp256k1"] - path = vendor/nim-secp256k1 - url = https://github.com/status-im/nim-secp256k1.git - ignore = dirty - branch = master -[submodule "vendor/nim-libp2p"] - path = vendor/nim-libp2p - url = https://github.com/vacp2p/nim-libp2p.git - ignore = dirty - branch = master -[submodule "vendor/nim-stew"] - path = vendor/nim-stew - url = https://github.com/status-im/nim-stew.git - ignore = dirty - branch = master -[submodule "vendor/nimbus-build-system"] - path = vendor/nimbus-build-system - url = https://github.com/status-im/nimbus-build-system.git - ignore = dirty - branch = master -[submodule "vendor/nim-nat-traversal"] - path = vendor/nim-nat-traversal - url = https://github.com/status-im/nim-nat-traversal.git - ignore = dirty - branch = master -[submodule "vendor/nim-libbacktrace"] - path = vendor/nim-libbacktrace - url = https://github.com/status-im/nim-libbacktrace.git - ignore = dirty - branch = master -[submodule "vendor/nim-confutils"] - path = vendor/nim-confutils - url = https://github.com/status-im/nim-confutils.git - ignore = dirty - branch = master -[submodule "vendor/nim-chronicles"] - path = vendor/nim-chronicles - url = https://github.com/status-im/nim-chronicles.git - ignore = dirty - branch = master -[submodule "vendor/nim-faststreams"] - path = vendor/nim-faststreams - url = https://github.com/status-im/nim-faststreams.git - ignore = dirty - branch = master -[submodule "vendor/nim-chronos"] - path = vendor/nim-chronos - url = https://github.com/status-im/nim-chronos.git - ignore = dirty - branch = master -[submodule "vendor/nim-json-serialization"] - path = vendor/nim-json-serialization - url = https://github.com/status-im/nim-json-serialization.git - ignore = dirty - branch = master -[submodule "vendor/nim-serialization"] - path = vendor/nim-serialization - url = https://github.com/status-im/nim-serialization.git - ignore = dirty - branch = master -[submodule "vendor/nimcrypto"] - path = vendor/nimcrypto - url = https://github.com/cheatfate/nimcrypto.git - ignore = dirty - branch = master -[submodule "vendor/nim-metrics"] - path = vendor/nim-metrics - url = https://github.com/status-im/nim-metrics.git - ignore = dirty - branch = master -[submodule "vendor/nim-stint"] - path = vendor/nim-stint - url = https://github.com/status-im/nim-stint.git - ignore = dirty - branch = master -[submodule "vendor/nim-json-rpc"] - path = vendor/nim-json-rpc - url = https://github.com/status-im/nim-json-rpc.git - ignore = dirty - branch = master -[submodule "vendor/nim-http-utils"] - path = vendor/nim-http-utils - url = https://github.com/status-im/nim-http-utils.git - ignore = dirty - branch = master -[submodule "vendor/nim-bearssl"] - path = vendor/nim-bearssl - url = https://github.com/status-im/nim-bearssl.git - ignore = dirty - branch = master -[submodule "vendor/nim-sqlite3-abi"] - path = vendor/nim-sqlite3-abi - url = https://github.com/arnetheduck/nim-sqlite3-abi.git - ignore = dirty - branch = master -[submodule "vendor/nim-web3"] - path = vendor/nim-web3 - url = https://github.com/status-im/nim-web3.git -[submodule "vendor/nim-testutils"] - path = vendor/nim-testutils - url = https://github.com/status-im/nim-testutils.git - ignore = untracked - branch = master -[submodule "vendor/nim-unittest2"] - path = vendor/nim-unittest2 - url = https://github.com/status-im/nim-unittest2.git - ignore = untracked - branch = master -[submodule "vendor/nim-websock"] - path = vendor/nim-websock - url = https://github.com/status-im/nim-websock.git - ignore = untracked - branch = main -[submodule "vendor/nim-zlib"] - path = vendor/nim-zlib - url = https://github.com/status-im/nim-zlib.git - ignore = untracked - branch = master -[submodule "vendor/nim-dnsdisc"] - path = vendor/nim-dnsdisc - url = https://github.com/status-im/nim-dnsdisc.git - ignore = untracked - branch = main -[submodule "vendor/dnsclient.nim"] - path = vendor/dnsclient.nim - url = https://github.com/ba0f3/dnsclient.nim.git - ignore = untracked - branch = master -[submodule "vendor/nim-toml-serialization"] - path = vendor/nim-toml-serialization - url = https://github.com/status-im/nim-toml-serialization.git -[submodule "vendor/nim-presto"] - path = vendor/nim-presto - url = https://github.com/status-im/nim-presto.git - ignore = untracked - branch = master +# Native dependencies that require git submodules +# All pure Nim packages are now managed via Nimble with pinned commits + [submodule "vendor/zerokit"] path = vendor/zerokit url = https://github.com/vacp2p/zerokit.git ignore = dirty branch = v0.5.1 -[submodule "vendor/nim-regex"] - path = vendor/nim-regex - url = https://github.com/nitely/nim-regex.git - ignore = untracked - branch = master -[submodule "vendor/nim-unicodedb"] - path = vendor/nim-unicodedb - url = https://github.com/nitely/nim-unicodedb.git - ignore = untracked - branch = master -[submodule "vendor/nim-taskpools"] - path = vendor/nim-taskpools - url = https://github.com/status-im/nim-taskpools.git - ignore = untracked - branch = stable -[submodule "vendor/nim-results"] - ignore = untracked - branch = master - path = vendor/nim-results - url = https://github.com/arnetheduck/nim-results.git -[submodule "vendor/db_connector"] - path = vendor/db_connector - url = https://github.com/nim-lang/db_connector.git - ignore = untracked - branch = devel -[submodule "vendor/nph"] - ignore = untracked - branch = master - path = vendor/nph - url = https://github.com/arnetheduck/nph.git -[submodule "vendor/nim-minilru"] - path = vendor/nim-minilru - url = https://github.com/status-im/nim-minilru.git - ignore = untracked - branch = master + [submodule "vendor/waku-rlnv2-contract"] path = vendor/waku-rlnv2-contract url = https://github.com/logos-messaging/waku-rlnv2-contract.git ignore = untracked branch = master + +[submodule "vendor/nph"] + ignore = untracked + branch = master + path = vendor/nph + url = https://github.com/arnetheduck/nph.git + [submodule "vendor/nim-ffi"] path = vendor/nim-ffi url = https://github.com/logos-messaging/nim-ffi/ diff --git a/Makefile b/Makefile index 87bd7bc74..f331b110b 100644 --- a/Makefile +++ b/Makefile @@ -4,28 +4,33 @@ # - MIT license # at your option. This file may not be copied, modified, or distributed except # according to those terms. -export BUILD_SYSTEM_DIR := vendor/nimbus-build-system -export EXCLUDED_NIM_PACKAGES := vendor/nim-dnsdisc/vendor -LINK_PCRE := 0 -FORMAT_MSG := "\\x1B[95mFormatting:\\x1B[39m" -# we don't want an error here, so we can handle things later, in the ".DEFAULT" target --include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk +# Build system configuration (no longer using nimbus-build-system) +FORMAT_MSG := "\x1B[95mFormatting:\x1B[39m" +BUILD_MSG := "\x1B[92mBuilding:\x1B[39m" +INFO_MSG := "\x1B[94mInfo:\x1B[39m" +WARN_MSG := "\x1B[93mWarning:\x1B[39m" -ifeq ($(NIM_PARAMS),) -# "variables.mk" was not included, so we update the submodules. -GIT_SUBMODULE_UPDATE := git submodule update --init --recursive -.DEFAULT: - +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ - $(GIT_SUBMODULE_UPDATE); \ - echo -# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself: -# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles -# -# After restarting, it will execute its original goal, so we don't have to start a child Make here -# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great? +# Nim binary detection +NIM_BINARY := $(shell which nim 2>/dev/null) +ifeq ($(NIM_BINARY),) + $(error Nim compiler not found. Please install Nim >= 2.2.4) +endif -else # "variables.mk" was included. Business as usual until the end of this file. +# Nimble binary detection +NIMBLE_BINARY := $(shell which nimble 2>/dev/null) +ifeq ($(NIMBLE_BINARY),) + $(error Nimble not found. Please install Nimble) +endif + +# Environment script (simplified - just use current env) +ENV_SCRIPT := + +# Handle output +HANDLE_OUTPUT := +ifneq ($(V), 1) + HANDLE_OUTPUT := &> /dev/null +endif # Determine the OS detected_OS := $(shell uname -s) @@ -72,19 +77,33 @@ endif %: @true +# Build directory +build: + mkdir -p build + waku.nims: ln -s waku.nimble $@ -update: | update-common - rm -rf waku.nims && \ - $(MAKE) waku.nims $(HANDLE_OUTPUT) +# Update dependencies using Nimble +# This is the ONLY target that should install/update Nimble packages +update: | waku.nims + @echo -e $(INFO_MSG) "Updating vendor submodules..." + git submodule update --init --recursive + @echo -e $(INFO_MSG) "Installing/updating Nimble dependencies..." + nimble install --depsOnly -y + @echo -e $(INFO_MSG) "Setting up Nimble paths..." + nimble setup + @echo -e $(INFO_MSG) "Building nph formatter..." $(MAKE) build-nph + @echo -e $(INFO_MSG) "Update complete!" clean: + @echo -e $(INFO_MSG) "Cleaning build artifacts..." rm -rf build - -# must be included after the default target --include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk + @echo -e $(INFO_MSG) "Cleaning Nimble package cache..." + rm -rf ~/.nimble/pkgs2/* + rm -rf ~/.nimble/pkgcache/* + @echo -e $(INFO_MSG) "Clean complete!" ## Possible values: prod; debug TARGET ?= prod @@ -94,6 +113,9 @@ GIT_VERSION ?= $(shell git describe --abbrev=6 --always --tags) ## Compilation parameters. If defined in the CLI the assignments won't be executed NIM_PARAMS := $(NIM_PARAMS) -d:git_version=\"$(GIT_VERSION)\" +# Add vendor/nim-ffi to path (kept as submodule due to broken .nimble file) +NIM_PARAMS := $(NIM_PARAMS) --path:vendor/nim-ffi + ## Heaptracker options HEAPTRACKER ?= 0 HEAPTRACKER_INJECT ?= 0 @@ -134,7 +156,11 @@ endif rln-deps: rustup ./scripts/install_rln_tests_dependencies.sh $(FOUNDRY_VERSION) $(PNPM_VERSION) -deps: | deps-common nat-libs waku.nims +# Check dependencies are ready (does NOT install packages) +# Run 'make update' first to install/update Nimble packages +deps: | waku.nims + @echo -e $(INFO_MSG) "Dependencies check passed" + @echo -e $(INFO_MSG) "If you haven't run 'make update' yet, please do so to install Nimble packages" ### nim-libbacktrace @@ -153,13 +179,21 @@ endif # enable experimental exit is dest feature in libp2p mix NIM_PARAMS := $(NIM_PARAMS) -d:libp2p_mix_experimental_exit_is_dest +# Nimble packages directory +NIMBLE_PKGS_DIR := $(shell nimble path 2>/dev/null | head -1 | xargs dirname 2>/dev/null || echo "$(HOME)/.nimble/pkgs2") + +# libbacktrace is now installed via Nimble, build native parts libbacktrace: - + $(MAKE) -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 + @if [ -d "$(NIMBLE_PKGS_DIR)/nim-libbacktrace-"* ]; then \ + $(MAKE) -C $$(ls -d $(NIMBLE_PKGS_DIR)/nim-libbacktrace-* | head -1) --no-print-directory BUILD_CXX_LIB=0 2>/dev/null || true; \ + fi clean-libbacktrace: - + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) + @echo "libbacktrace cleanup handled by nimble" -# Extend deps and clean targets +# Libbacktrace is optional - disable by default with Nimble +# To enable: make USE_LIBBACKTRACE=1 +USE_LIBBACKTRACE ?= 0 ifneq ($(USE_LIBBACKTRACE), 0) deps: | libbacktrace endif @@ -174,12 +208,6 @@ endif clean: | clean-libbacktrace -### Create nimble links (used when building with Nix) - -nimbus-build-system-nimble-dir: - NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" \ - PWD_CMD="$(PWD)" \ - $(CURDIR)/scripts/generate_nimble_links.sh ################## ## RLN ## @@ -562,6 +590,10 @@ libwaku-ios: $(MAKE) libwaku-ios-device $(MAKE) libwaku-ios-simulator +# Helper to find Nimble package paths +NAT_TRAVERSAL_PKG := $(shell ls -d $(NIMBLE_PKGS_DIR)/nim-nat-traversal-* 2>/dev/null | head -1) +LIBBACKTRACE_PKG := $(shell ls -d $(NIMBLE_PKGS_DIR)/nim-libbacktrace-* 2>/dev/null | head -1) + cwaku_example: | build libwaku echo -e $(BUILD_MSG) "build/$@" && \ cc -o "build/$@" \ @@ -569,10 +601,8 @@ cwaku_example: | build libwaku ./examples/cbindings/base64.c \ -lwaku -Lbuild/ \ -pthread -ldl -lm \ - -lminiupnpc -Lvendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build/ \ - -lnatpmp -Lvendor/nim-nat-traversal/vendor/libnatpmp-upstream/ \ - vendor/nim-libbacktrace/libbacktrace_wrapper.o \ - vendor/nim-libbacktrace/install/usr/lib/libbacktrace.a + $$(if [ -n "$(NAT_TRAVERSAL_PKG)" ]; then echo "-lminiupnpc -L$(NAT_TRAVERSAL_PKG)/vendor/miniupnp/miniupnpc/build/ -lnatpmp -L$(NAT_TRAVERSAL_PKG)/vendor/libnatpmp-upstream/"; fi) \ + $$(if [ -n "$(LIBBACKTRACE_PKG)" ]; then echo "$(LIBBACKTRACE_PKG)/libbacktrace_wrapper.o $(LIBBACKTRACE_PKG)/install/usr/lib/libbacktrace.a"; fi) cppwaku_example: | build libwaku echo -e $(BUILD_MSG) "build/$@" && \ @@ -581,17 +611,13 @@ cppwaku_example: | build libwaku ./examples/cpp/base64.cpp \ -lwaku -Lbuild/ \ -pthread -ldl -lm \ - -lminiupnpc -Lvendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build/ \ - -lnatpmp -Lvendor/nim-nat-traversal/vendor/libnatpmp-upstream/ \ - vendor/nim-libbacktrace/libbacktrace_wrapper.o \ - vendor/nim-libbacktrace/install/usr/lib/libbacktrace.a + $$(if [ -n "$(NAT_TRAVERSAL_PKG)" ]; then echo "-lminiupnpc -L$(NAT_TRAVERSAL_PKG)/vendor/miniupnp/miniupnpc/build/ -lnatpmp -L$(NAT_TRAVERSAL_PKG)/vendor/libnatpmp-upstream/"; fi) \ + $$(if [ -n "$(LIBBACKTRACE_PKG)" ]; then echo "$(LIBBACKTRACE_PKG)/libbacktrace_wrapper.o $(LIBBACKTRACE_PKG)/install/usr/lib/libbacktrace.a"; fi) nodejswaku: | build deps echo -e $(BUILD_MSG) "build/$@" && \ node-gyp build --directory=examples/nodejs/ -endif # "variables.mk" was not included - ################### # Release Targets # ################### diff --git a/apps/wakunode2/wakunode2.nim b/apps/wakunode2/wakunode2.nim index b50c7113b..4b35ee19a 100644 --- a/apps/wakunode2/wakunode2.nim +++ b/apps/wakunode2/wakunode2.nim @@ -5,9 +5,10 @@ import chronicles, chronos, metrics, - libbacktrace, system/ansi_c, libp2p/crypto/crypto +when defined(libbacktrace): + import libbacktrace import ../../tools/[rln_keystore_generator/rln_keystore_generator, confutils/cli_args], waku/[ @@ -87,7 +88,7 @@ when isMainModule: when defined(posix): proc handleSigsegv(signal: cint) {.noconv.} = # Require --debugger:native - fatal "Shutting down after receiving SIGSEGV", stacktrace = getBacktrace() + fatal "Shutting down after receiving SIGSEGV" # Not available in -d:release mode writeStackTrace() diff --git a/config.nims b/config.nims index f74fe183f..6b12280c4 100644 --- a/config.nims +++ b/config.nims @@ -9,11 +9,8 @@ if defined(windows): switch("passL", "rln.lib") switch("define", "postgres=false") - # Automatically add all vendor subdirectories - for dir in walkDir("./vendor"): - if dir.kind == pcDir: - switch("path", dir.path) - switch("path", dir.path / "src") + # Dependencies are now managed via Nimble - no vendor paths needed + # Nimble automatically adds package paths to the compiler # disable timestamps in Windows PE headers - https://wiki.debian.org/ReproducibleBuilds/TimestampsInPEBinaries switch("passL", "-Wl,--no-insert-timestamp") @@ -125,3 +122,7 @@ if defined(android): switch("passC", "--sysroot=" & sysRoot) switch("passL", "--sysroot=" & sysRoot) switch("cincludes", sysRoot & "/usr/include/") +# begin Nimble config (version 2) +when withDir(thisDir(), system.fileExists("nimble.paths")): + include "nimble.paths" +# end Nimble config diff --git a/vendor/db_connector b/vendor/db_connector deleted file mode 160000 index 74aef399e..000000000 --- a/vendor/db_connector +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 74aef399e5c232f95c9fc5c987cebac846f09d62 diff --git a/vendor/dnsclient.nim b/vendor/dnsclient.nim deleted file mode 160000 index 23214235d..000000000 --- a/vendor/dnsclient.nim +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 23214235d4784d24aceed99bbfe153379ea557c8 diff --git a/vendor/nim-bearssl b/vendor/nim-bearssl deleted file mode 160000 index 11e798b62..000000000 --- a/vendor/nim-bearssl +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 11e798b62b8e6beabe958e048e9e24c7e0f9ee63 diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles deleted file mode 160000 index 54f5b7260..000000000 --- a/vendor/nim-chronicles +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 54f5b726025e8c7385e3a6529d3aa27454c6e6ff diff --git a/vendor/nim-chronos b/vendor/nim-chronos deleted file mode 160000 index 0646c444f..000000000 --- a/vendor/nim-chronos +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0646c444fce7c7ed08ef6f2c9a7abfd172ffe655 diff --git a/vendor/nim-confutils b/vendor/nim-confutils deleted file mode 160000 index e214b3992..000000000 --- a/vendor/nim-confutils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e214b3992a31acece6a9aada7d0a1ad37c928f3b diff --git a/vendor/nim-dnsdisc b/vendor/nim-dnsdisc deleted file mode 160000 index b71d029f4..000000000 --- a/vendor/nim-dnsdisc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b71d029f4da4ec56974d54c04518bada00e1b623 diff --git a/vendor/nim-eth b/vendor/nim-eth deleted file mode 160000 index d9135e6c3..000000000 --- a/vendor/nim-eth +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d9135e6c3c5d6d819afdfb566aa8d958756b73a8 diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams deleted file mode 160000 index c3ac3f639..000000000 --- a/vendor/nim-faststreams +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c3ac3f639ed1d62f59d3077d376a29c63ac9750c diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils deleted file mode 160000 index 79cbab146..000000000 --- a/vendor/nim-http-utils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 79cbab1460f4c0cdde2084589d017c43a3d7b4f1 diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc deleted file mode 160000 index 9665c2650..000000000 --- a/vendor/nim-json-rpc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9665c265035f49f5ff94bbffdeadde68e19d6221 diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization deleted file mode 160000 index b65fd6a7e..000000000 --- a/vendor/nim-json-serialization +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b65fd6a7e64c864dabe40e7dfd6c7d07db0014ac diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace deleted file mode 160000 index d8bd4ce5c..000000000 --- a/vendor/nim-libbacktrace +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d8bd4ce5c46bb6d2f984f6b3f3d7380897d95ecb diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p deleted file mode 160000 index e82080f7b..000000000 --- a/vendor/nim-libp2p +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e82080f7b1aa61c6d35fa5311b873f41eff4bb52 diff --git a/vendor/nim-metrics b/vendor/nim-metrics deleted file mode 160000 index ecf64c607..000000000 --- a/vendor/nim-metrics +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ecf64c6078d1276d3b7d9b3d931fbdb70004db11 diff --git a/vendor/nim-minilru b/vendor/nim-minilru deleted file mode 160000 index 0c4b2bce9..000000000 --- a/vendor/nim-minilru +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0c4b2bce959591f0a862e9b541ba43c6d0cf3476 diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal deleted file mode 160000 index 860e18c37..000000000 --- a/vendor/nim-nat-traversal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 860e18c37667b5dd005b94c63264560c35d88004 diff --git a/vendor/nim-presto b/vendor/nim-presto deleted file mode 160000 index 92b1c7ff1..000000000 --- a/vendor/nim-presto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 92b1c7ff141e6920e1f8a98a14c35c1fa098e3be diff --git a/vendor/nim-regex b/vendor/nim-regex deleted file mode 160000 index 4593305ed..000000000 --- a/vendor/nim-regex +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4593305ed1e49731fc75af1dc572dd2559aad19c diff --git a/vendor/nim-results b/vendor/nim-results deleted file mode 160000 index df8113dda..000000000 --- a/vendor/nim-results +++ /dev/null @@ -1 +0,0 @@ -Subproject commit df8113dda4c2d74d460a8fa98252b0b771bf1f27 diff --git a/vendor/nim-secp256k1 b/vendor/nim-secp256k1 deleted file mode 160000 index 9dd3df621..000000000 --- a/vendor/nim-secp256k1 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9dd3df62124aae79d564da636bb22627c53c7676 diff --git a/vendor/nim-serialization b/vendor/nim-serialization deleted file mode 160000 index 6f525d544..000000000 --- a/vendor/nim-serialization +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6f525d5447d97256750ca7856faead03e562ed20 diff --git a/vendor/nim-sqlite3-abi b/vendor/nim-sqlite3-abi deleted file mode 160000 index bdf01cf42..000000000 --- a/vendor/nim-sqlite3-abi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bdf01cf4236fb40788f0733466cdf6708783cbac diff --git a/vendor/nim-stew b/vendor/nim-stew deleted file mode 160000 index e57400149..000000000 --- a/vendor/nim-stew +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e5740014961438610d336cd81706582dbf2c96f0 diff --git a/vendor/nim-stint b/vendor/nim-stint deleted file mode 160000 index 470b78925..000000000 --- a/vendor/nim-stint +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 470b7892561b5179ab20bd389a69217d6213fe58 diff --git a/vendor/nim-taskpools b/vendor/nim-taskpools deleted file mode 160000 index 9e8ccc754..000000000 --- a/vendor/nim-taskpools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9e8ccc754631ac55ac2fd495e167e74e86293edb diff --git a/vendor/nim-testutils b/vendor/nim-testutils deleted file mode 160000 index 94d68e796..000000000 --- a/vendor/nim-testutils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 94d68e796c045d5b37cabc6be32d7bfa168f8857 diff --git a/vendor/nim-toml-serialization b/vendor/nim-toml-serialization deleted file mode 160000 index fea85b27f..000000000 --- a/vendor/nim-toml-serialization +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fea85b27f0badcf617033ca1bc05444b5fd8aa7a diff --git a/vendor/nim-unicodedb b/vendor/nim-unicodedb deleted file mode 160000 index 66f245871..000000000 --- a/vendor/nim-unicodedb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 66f2458710dc641dd4640368f9483c8a0ec70561 diff --git a/vendor/nim-unittest2 b/vendor/nim-unittest2 deleted file mode 160000 index 8b51e99b4..000000000 --- a/vendor/nim-unittest2 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8b51e99b4a57fcfb31689230e75595f024543024 diff --git a/vendor/nim-web3 b/vendor/nim-web3 deleted file mode 160000 index 81ee8ce47..000000000 --- a/vendor/nim-web3 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 81ee8ce479d86acb73be7c4f365328e238d9b4a3 diff --git a/vendor/nim-websock b/vendor/nim-websock deleted file mode 160000 index ebe308a79..000000000 --- a/vendor/nim-websock +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ebe308a79a7b440a11dfbe74f352be86a3883508 diff --git a/vendor/nim-zlib b/vendor/nim-zlib deleted file mode 160000 index daa8723fd..000000000 --- a/vendor/nim-zlib +++ /dev/null @@ -1 +0,0 @@ -Subproject commit daa8723fd32299d4ca621c837430c29a5a11e19a diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system deleted file mode 160000 index e6c2c9da3..000000000 --- a/vendor/nimbus-build-system +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e6c2c9da39c2d368d9cf420ac22692e99715d22c diff --git a/vendor/nimcrypto b/vendor/nimcrypto deleted file mode 160000 index 721fb99ee..000000000 --- a/vendor/nimcrypto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 721fb99ee099b632eb86dfad1f0d96ee87583774 diff --git a/waku.nimble b/waku.nimble index 5c5c09763..26c306fea 100644 --- a/waku.nimble +++ b/waku.nimble @@ -11,27 +11,47 @@ license = "MIT or Apache License 2.0" #bin = @["build/waku"] ### Dependencies -requires "nim >= 2.2.4", - "chronicles", - "confutils", - "chronos", - "dnsdisc", - "eth", - "json_rpc", - "libbacktrace", - "nimcrypto", - "serialization", - "stew", - "stint", - "metrics", - "libp2p >= 1.14.3", - "web3", - "presto", - "regex", - "results", - "db_connector", - "minilru", - "ffi" +requires "nim >= 2.2.4" + +# Pure Nim packages - use flexible version requirements +# Exact versions are locked via nimble.lock for reproducible builds +requires "chronicles" +requires "chronos" +requires "confutils" +requires "dnsdisc" +requires "faststreams" +requires "httputils" +requires "json_rpc" +requires "json_serialization" +requires "serialization" +requires "stew" +requires "stint" +requires "metrics" +requires "presto" +requires "taskpools" +requires "testutils" +requires "unittest2" +requires "websock" +requires "zlib" +requires "toml_serialization" +requires "minilru" +requires "regex" +requires "unicodedb" +requires "results" +requires "nimcrypto" +requires "db_connector" +requires "dnsclient" + +# Packages with native dependencies +requires "eth" +requires "libp2p" +requires "web3" +# nim-libbacktrace removed - optional dependency with build issues in Nimble +# If needed, enable with USE_LIBBACKTRACE=1 and install manually +requires "bearssl" +requires "secp256k1" +requires "nat_traversal" +requires "sqlite3_abi" ### Helper functions proc buildModule(filePath, params = "", lang = "c"): bool = @@ -217,6 +237,22 @@ task libWakuAndroid, "Build the mobile bindings for Android": ### Mobile iOS import std/sequtils +# Helper to find Nimble package path +proc findNimblePkg(pkgName: string): string = + let nimbleDir = getEnv("NIMBLE_DIR", getEnv("HOME") & "/.nimble") + let pkgsDir = nimbleDir & "/pkgs2" + if dirExists(pkgsDir): + for kind, path in walkDir(pkgsDir): + if kind == pcDir and path.contains(pkgName): + return path + # Fallback to pkgs (older nimble) + let pkgsDirOld = nimbleDir & "/pkgs" + if dirExists(pkgsDirOld): + for kind, path in walkDir(pkgsDirOld): + if kind == pcDir and path.contains(pkgName): + return path + return "" + proc buildMobileIOS(srcDir = ".", params = "") = echo "Building iOS libwaku library" @@ -227,6 +263,19 @@ proc buildMobileIOS(srcDir = ".", params = "") = if sdkPath.len == 0: quit "Error: IOS_SDK_PATH not set. Set it to the path of the iOS SDK" + # Find Nimble package paths for native dependencies + let bearSslPkg = findNimblePkg("nim-bearssl") + let secp256k1Pkg = findNimblePkg("nim-secp256k1") + let natTraversalPkg = findNimblePkg("nim-nat-traversal") + let nimLibPath = gorge("nim --version 2>/dev/null | head -1").split(" ")[0] & "/lib" + + if bearSslPkg.len == 0: + quit "Error: nim-bearssl package not found. Run 'nimble install' first." + if secp256k1Pkg.len == 0: + quit "Error: nim-secp256k1 package not found. Run 'nimble install' first." + if natTraversalPkg.len == 0: + quit "Error: nim-nat-traversal package not found. Run 'nimble install' first." + # Use SDK name in path to differentiate device vs simulator let outDir = "build/ios/" & iosSdk & "-" & iosArch if not dirExists outDir: @@ -269,8 +318,8 @@ proc buildMobileIOS(srcDir = ".", params = "") = # --- BearSSL --- echo "Compiling BearSSL for iOS..." - let bearSslSrcDir = "./vendor/nim-bearssl/bearssl/csources/src" - let bearSslIncDir = "./vendor/nim-bearssl/bearssl/csources/inc" + let bearSslSrcDir = bearSslPkg & "/bearssl/csources/src" + let bearSslIncDir = bearSslPkg & "/bearssl/csources/inc" for path in walkDirRec(bearSslSrcDir): if path.endsWith(".c"): let relPath = path.replace(bearSslSrcDir & "/", "").replace("/", "_") @@ -281,7 +330,7 @@ proc buildMobileIOS(srcDir = ".", params = "") = # --- secp256k1 --- echo "Compiling secp256k1 for iOS..." - let secp256k1Dir = "./vendor/nim-secp256k1/vendor/secp256k1" + let secp256k1Dir = secp256k1Pkg & "/vendor/secp256k1" let secp256k1Flags = " -I" & secp256k1Dir & "/include" & " -I" & secp256k1Dir & "/src" & " -I" & secp256k1Dir & @@ -306,9 +355,9 @@ proc buildMobileIOS(srcDir = ".", params = "") = # --- miniupnpc --- echo "Compiling miniupnpc for iOS..." - let miniupnpcSrcDir = "./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/src" - let miniupnpcIncDir = "./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/include" - let miniupnpcBuildDir = "./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build" + let miniupnpcSrcDir = natTraversalPkg & "/vendor/miniupnp/miniupnpc/src" + let miniupnpcIncDir = natTraversalPkg & "/vendor/miniupnp/miniupnpc/include" + let miniupnpcBuildDir = natTraversalPkg & "/vendor/miniupnp/miniupnpc/build" let miniupnpcFiles = @[ "addr_is_reserved.c", "connecthostport.c", "igd_desc_parse.c", "minisoap.c", "minissdpc.c", "miniupnpc.c", "miniwget.c", @@ -329,7 +378,7 @@ proc buildMobileIOS(srcDir = ".", params = "") = # --- libnatpmp --- echo "Compiling libnatpmp for iOS..." - let natpmpSrcDir = "./vendor/nim-nat-traversal/vendor/libnatpmp-upstream" + let natpmpSrcDir = natTraversalPkg & "/vendor/libnatpmp-upstream" # Only compile natpmp.c - getgateway.c uses net/route.h which is not available on iOS let natpmpObj = vendorObjDir / "natpmp_natpmp.o" if not fileExists(natpmpObj): @@ -358,18 +407,21 @@ proc buildMobileIOS(srcDir = ".", params = "") = if kind == pcFile and path.endsWith(".c"): cFiles.add(path) + # Get Nim lib path + let nimPath = gorge("dirname $(which nim)").strip & "/../lib" + for cFile in cFiles: let baseName = extractFilename(cFile).changeFileExt("o") let oFile = objDir / baseName exec clangBase & " -DENABLE_STRNATPMPERR" & - " -I./vendor/nimbus-build-system/vendor/Nim/lib/" & - " -I./vendor/nim-bearssl/bearssl/csources/inc/" & - " -I./vendor/nim-bearssl/bearssl/csources/tools/" & - " -I./vendor/nim-bearssl/bearssl/abi/" & - " -I./vendor/nim-secp256k1/vendor/secp256k1/include/" & - " -I./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/include/" & - " -I./vendor/nim-nat-traversal/vendor/libnatpmp-upstream/" & + " -I" & nimPath & "/" & + " -I" & bearSslPkg & "/bearssl/csources/inc/" & + " -I" & bearSslPkg & "/bearssl/csources/tools/" & + " -I" & bearSslPkg & "/bearssl/abi/" & + " -I" & secp256k1Pkg & "/vendor/secp256k1/include/" & + " -I" & natTraversalPkg & "/vendor/miniupnp/miniupnpc/include/" & + " -I" & natTraversalPkg & "/vendor/libnatpmp-upstream/" & " -I" & nimcacheDir & " -c " & cFile & " -o " & oFile diff --git a/waku/waku_filter_v2/client.nim b/waku/waku_filter_v2/client.nim index c42bca3db..31807eccd 100644 --- a/waku/waku_filter_v2/client.nim +++ b/waku/waku_filter_v2/client.nim @@ -30,7 +30,7 @@ type WakuFilterClient* = ref object of LPProtocol func generateRequestId(rng: ref HmacDrbgContext): string = var bytes: array[10, byte] hmacDrbgGenerate(rng[], bytes) - return toHex(bytes) + return byteutils.toHex(bytes) proc addSubscrObserver*(wfc: WakuFilterClient, obs: SubscriptionObserver) = wfc.subscrObservers.add(obs) diff --git a/waku/waku_store_sync/reconciliation.nim b/waku/waku_store_sync/reconciliation.nim index 0cc15d0df..f18201d0e 100644 --- a/waku/waku_store_sync/reconciliation.nim +++ b/waku/waku_store_sync/reconciliation.nim @@ -79,7 +79,7 @@ proc messageIngress*( let id = SyncID(time: msg.timestamp, hash: msgHash) self.storage.insert(id, pubsubTopic, msg.contentTopic).isOkOr: - error "failed to insert new message", msg_hash = $id.hash.toHex(), error = $error + error "failed to insert new message", msg_hash = byteutils.toHex(id.hash), error = $error proc messageIngress*( self: SyncReconciliation, @@ -87,7 +87,7 @@ proc messageIngress*( pubsubTopic: PubsubTopic, msg: WakuMessage, ) = - trace "message ingress", msg_hash = msgHash.toHex(), msg = msg + trace "message ingress", msg_hash = byteutils.toHex(msgHash), msg = msg if msg.ephemeral: return @@ -95,7 +95,7 @@ proc messageIngress*( let id = SyncID(time: msg.timestamp, hash: msgHash) self.storage.insert(id, pubsubTopic, msg.contentTopic).isOkOr: - error "failed to insert new message", msg_hash = $id.hash.toHex(), error = $error + error "failed to insert new message", msg_hash = byteutils.toHex(id.hash), error = $error proc messageIngress*( self: SyncReconciliation, @@ -104,7 +104,7 @@ proc messageIngress*( contentTopic: ContentTopic, ) = self.storage.insert(id, pubsubTopic, contentTopic).isOkOr: - error "failed to insert new message", msg_hash = $id.hash.toHex(), error = $error + error "failed to insert new message", msg_hash = byteutils.toHex(id.hash), error = $error proc preProcessPayload( self: SyncReconciliation, payload: RangesData