From edc87b702b808b70a12867e7cf0d44fff7f62473 Mon Sep 17 00:00:00 2001
From: Kim De Mey <kim.demey@gmail.com>
Date: Fri, 21 Jul 2023 16:30:22 +0200
Subject: [PATCH] Launch Fluffy builds directly from make to avoid compile
 issue (#1646)

* Launch Fluffy builds directly from make to avoid compile issue

Without this change, builds on latest macos fails when ulimit is
not set to 1024. But it will still cause libbacktrace error to occur
when launching the binaries so it would be still advised to
set it to 1024.

* Fix fluffy local testnet for some macOS systems

And some additional improvements to the script + run the fluffy
nodes at INFO log-level to speed-up the testing time.

* Split up fluffy tests in separate targets

This way the two test binaries can be build and ran
concurrently.
---
 .gitignore                             |  3 +++
 Makefile                               | 24 +++++++++++++++++-------
 fluffy/scripts/launch_local_testnet.sh | 12 ++++++------
 fluffy/scripts/nim.cfg                 |  1 +
 fluffy/scripts/test_portal_testnet.nim |  2 +-
 nimbus.nimble                          |  2 +-
 6 files changed, 29 insertions(+), 15 deletions(-)
 create mode 100644 fluffy/scripts/nim.cfg

diff --git a/.gitignore b/.gitignore
index 1559958c2..59a1b56a7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,6 +32,9 @@ nimcache
 
 /all_test.md
 
+# local testnet files
+/local_testnet_data
+
 # Nimble user files
 nimble.develop
 nimble.paths
diff --git a/Makefile b/Makefile
index 023c08c3b..f3159cb61 100644
--- a/Makefile
+++ b/Makefile
@@ -221,7 +221,7 @@ test-reproducibility:
 # builds the fluffy client
 fluffy: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim fluffy $(NIM_PARAMS) nimbus.nims
+		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/$@ "fluffy/$@.nim"
 
 # primitive reproducibility test
 fluffy-test-reproducibility:
@@ -233,9 +233,18 @@ fluffy-test-reproducibility:
 		[ "$$MD5SUM1" = "$$MD5SUM2" ] && echo -e "\e[92mSuccess: identical binaries.\e[39m" || \
 			{ echo -e "\e[91mFailure: the binary changed between builds.\e[39m"; exit 1; }
 
+# fluffy tests
+all_fluffy_portal_spec_tests: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:chronicles_log_level=ERROR -d:nimbus_db_backend=sqlite -o:build/$@ "fluffy/tests/portal_spec_tests/mainnet/$@.nim"
+
+
+all_fluffy_tests: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:chronicles_log_level=ERROR -d:nimbus_db_backend=sqlite -d:mergeBlockNumber:38130 -o:build/$@ "fluffy/tests/$@.nim"
+
 # builds and runs the fluffy test suite
-fluffy-test: | build deps
-	$(ENV_SCRIPT) nim fluffy_test $(NIM_PARAMS) nimbus.nims
+fluffy-test: | all_fluffy_portal_spec_tests all_fluffy_tests
 
 # builds the fluffy tools, wherever they are
 $(FLUFFY_TOOLS): | build deps
@@ -246,6 +255,11 @@ $(FLUFFY_TOOLS): | build deps
 # builds all the fluffy tools
 fluffy-tools: | $(FLUFFY_TOOLS)
 
+# Build fluffy test_portal_testnet
+test_portal_testnet: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -o:build/$@ "fluffy/scripts/$@.nim"
+
 # builds the uTP test app
 utp-test-app: | build deps
 	$(ENV_SCRIPT) nim utp_test_app $(NIM_PARAMS) nimbus.nims
@@ -254,10 +268,6 @@ utp-test-app: | build deps
 utp-test: | build deps
 	$(ENV_SCRIPT) nim utp_test $(NIM_PARAMS) nimbus.nims
 
-# Build fluffy test_portal_testnet
-fluffy-test-portal-testnet: | build deps
-	$(ENV_SCRIPT) nim fluffy_test_portal_testnet $(NIM_PARAMS) nimbus.nims
-
 # Nimbus Verified Proxy related targets
 
 # Builds the nimbus_verified_proxy
diff --git a/fluffy/scripts/launch_local_testnet.sh b/fluffy/scripts/launch_local_testnet.sh
index d73a6ea6a..582699c6f 100755
--- a/fluffy/scripts/launch_local_testnet.sh
+++ b/fluffy/scripts/launch_local_testnet.sh
@@ -23,7 +23,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")"/../..
 GETOPT_BINARY="getopt"
 if uname | grep -qi darwin; then
   # macOS
-  GETOPT_BINARY="/usr/local/opt/gnu-getopt/bin/getopt"
+  GETOPT_BINARY=$(find /opt/homebrew/opt/gnu-getopt/bin/getopt /usr/local/opt/gnu-getopt/bin/getopt 2> /dev/null | head -n1 || true)
   [[ -f "$GETOPT_BINARY" ]] || { echo "GNU getopt not installed. Please run 'brew install gnu-getopt'. Aborting."; exit 1; }
 fi
 
@@ -40,10 +40,10 @@ LONGOPTS="help,nodes:,data-dir:,enable-htop,log-level:,base-port:,base-rpc-port:
 NUM_NODES="64"
 DATA_DIR="local_testnet_data"
 USE_HTOP="0"
-LOG_LEVEL="TRACE"
+LOG_LEVEL="INFO"
 BASE_PORT="9000"
 BASE_METRICS_PORT="8008"
-BASE_RPC_PORT="7000"
+BASE_RPC_PORT="10000"
 REUSE_EXISTING_DATA_DIR="0"
 TIMEOUT_DURATION="0"
 KILL_OLD_PROCESSES="0"
@@ -183,9 +183,9 @@ fi
 
 # Build the binaries
 BINARIES="fluffy"
-TEST_BINARIES="fluffy-test-portal-testnet"
-$MAKE -j ${NPROC} LOG_LEVEL=TRACE ${BINARIES} NIMFLAGS="-d:chronicles_colors=off -d:chronicles_sinks=textlines"
-$MAKE -j ${NPROC} LOG_LEVEL=INFO ${TEST_BINARIES} NIMFLAGS="-d:chronicles_sinks=textlines"
+TEST_BINARIES="test_portal_testnet"
+$MAKE -j ${NPROC} LOG_LEVEL=TRACE ${BINARIES}
+$MAKE -j ${NPROC} LOG_LEVEL=INFO ${TEST_BINARIES}
 
 # Kill child processes on Ctrl-C/SIGTERM/exit, passing the PID of this shell
 # instance as the parent and the target process name as a pattern to the
diff --git a/fluffy/scripts/nim.cfg b/fluffy/scripts/nim.cfg
new file mode 100644
index 000000000..8dd0048af
--- /dev/null
+++ b/fluffy/scripts/nim.cfg
@@ -0,0 +1 @@
+-d:unittest2DisableParamFiltering
diff --git a/fluffy/scripts/test_portal_testnet.nim b/fluffy/scripts/test_portal_testnet.nim
index 07dc56556..04db3dfcc 100644
--- a/fluffy/scripts/test_portal_testnet.nim
+++ b/fluffy/scripts/test_portal_testnet.nim
@@ -39,7 +39,7 @@ type
       name: "rpc-address" }: string
 
     baseRpcPort* {.
-      defaultValue: 7000
+      defaultValue: 10000
       desc: "Port of the JSON-RPC service of the bootstrap (first) node"
       name: "base-rpc-port" .}: uint16
 
diff --git a/nimbus.nimble b/nimbus.nimble
index 356ee3556..cad9b9838 100644
--- a/nimbus.nimble
+++ b/nimbus.nimble
@@ -88,7 +88,7 @@ task utp_test_app, "Build uTP test app":
 task utp_test, "Run uTP integration tests":
   test "fluffy/tools/utp_testing", "utp_test", "-d:chronicles_log_level=ERROR -d:chronosStrictException"
 
-task fluffy_test_portal_testnet, "Build test_portal_testnet":
+task test_portal_testnet, "Build test_portal_testnet":
   buildBinary "test_portal_testnet", "fluffy/scripts/", "-d:chronicles_log_level=DEBUG -d:unittest2DisableParamFiltering"
 
 ## Nimbus Verified Proxy tasks