Nimbus light client integration with status-go rebased version (#1971)

* Nimbus light client integration with status-go

* Add cleanup code, address review comments

* Disable metrics for libverifproxy only

* Update confutils

* missing import

* build proxy in tests

* more build stuff

* namespace make vars

* export NimMain for windows

* reduce dependency on Nim compiler in header file

* copyright

---------

Co-authored-by: Vitaliy Vlasov <siphiuel@protonmail.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

.github/workflows/nimbus_verified_proxy.yml

@@ -94,24 +94,25 @@ jobs:
           sudo dpkg --add-architecture i386
           sudo apt-fast update -qq
           sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
-            --no-install-recommends -yq gcc-multilib g++-multilib
+            --no-install-recommends -yq gcc-multilib g++-multilib \
+            libz-dev:i386 libbz2-dev:i386 libssl-dev:i386 libpcre3-dev:i386
           mkdir -p external/bin
           cat << EOF > external/bin/gcc
           #!/bin/bash
-          exec $(which gcc) -m32 -mno-adx "\$@"
+          exec $(which gcc) -m32 "\$@"
           EOF
           cat << EOF > external/bin/g++
           #!/bin/bash
-          exec $(which g++) -m32 -mno-adx "\$@"
+          exec $(which g++) -m32 "\$@"
           EOF
           chmod 755 external/bin/gcc external/bin/g++
-          echo "${{ github.workspace }}/external/bin" >> $GITHUB_PATH
+          echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
 
-      # Required for running the local testnet script
-      - name: Install build dependencies (MacOS)
-        if: runner.os == 'macOS'
+      - name: Install build dependencies (Macos)
+        # Some home brew modules were reported missing
+        if: runner.os == 'Macos'
         run: |
-          brew install gnu-getopt
+          HOMEBREW_NO_INSTALL_CLEANUP=1 brew install gnu-getopt
           brew link --force gnu-getopt
 
       - name: Restore llvm-mingw (Windows) from cache
@@ -141,7 +142,7 @@ jobs:
             MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip"
             ARCH=64
           else
-            MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip"
+            MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-i686.zip"
             ARCH=32
           fi
           curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.zip"
@@ -174,23 +175,23 @@ jobs:
           nbsHash=$(getHash status-im/nimbus-build-system)
           echo "nimbus_build_system=$nbsHash" >> $GITHUB_OUTPUT
 
-      - name: Restore prebuilt Nim binaries from cache
+      - name: Restore prebuilt Nim from cache
         id: nim-cache
         uses: actions/cache@v3
         with:
-          path: NimBinaries
+          path: NimBinCache
           key: 'nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.nimbus_build_system }}-verified-proxy'
 
       - name: Build Nim and Nimbus-eth1 dependencies
         run: |
-          make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update-from-ci
+          make V=1 -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinCache update-from-ci
 
       - name: Run verified proxy tests (Windows)
         if: runner.os == 'Windows'
         run: |
           gcc --version
           DEFAULT_MAKE_FLAGS="-j${ncpu}"
-          mingw32-make ${DEFAULT_MAKE_FLAGS} nimbus_verified_proxy
+          mingw32-make ${DEFAULT_MAKE_FLAGS} nimbus_verified_proxy libverifproxy
           build/nimbus_verified_proxy.exe --help
           mingw32-make ${DEFAULT_MAKE_FLAGS} nimbus-verified-proxy-test
 
@@ -199,7 +200,7 @@ jobs:
         run: |
           export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
           DEFAULT_MAKE_FLAGS="-j${ncpu}"
-          env CC=gcc make ${DEFAULT_MAKE_FLAGS} nimbus_verified_proxy
+          env CC=gcc make ${DEFAULT_MAKE_FLAGS} nimbus_verified_proxy libverifproxy
           build/nimbus_verified_proxy --help
           # CC is needed to select correct compiler 32/64 bit
           env CC=gcc CXX=g++ make ${DEFAULT_MAKE_FLAGS} nimbus-verified-proxy-test
@@ -208,7 +209,7 @@ jobs:
         if: runner.os == 'Macos'
         run: |
           DEFAULT_MAKE_FLAGS="-j${ncpu}"
-          make ${DEFAULT_MAKE_FLAGS} nimbus_verified_proxy
+          make ${DEFAULT_MAKE_FLAGS} nimbus_verified_proxy libverifproxy
           build/nimbus_verified_proxy --help
           # "-static" option will not work for osx unless static system libraries are provided
           make ${DEFAULT_MAKE_FLAGS} nimbus-verified-proxy-test

Makefile

@@ -81,6 +81,20 @@ FLUFFY_TOOLS_DIRS := \
 # comma-separated values for the "clean" target
 FLUFFY_TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(FLUFFY_TOOLS))
 
+# Namespaced variables to avoid conflicts with other makefiles
+VERIF_PROXY_OS = $(shell $(CC) -dumpmachine)
+ifneq (, $(findstring darwin, $(VERIF_PROXY_OS)))
+  VERIF_PROXY_SHAREDLIBEXT = dylib
+else
+ifneq (, $(findstring mingw, $(VERIF_PROXY_OS))$(findstring cygwin, $(VERIF_PROXY_OS))$(findstring msys, $(VERIF_PROXY_OS)))
+  VERIF_PROXY_SHAREDLIBEXT = dll
+else
+  VERIF_PROXY_SHAREDLIBEXT = so
+endif
+endif
+
+VERIF_PROXY_OUT_PATH ?= build/libverifproxy/
+
 .PHONY: \
 	all \
 	$(TOOLS) \
@@ -90,6 +104,7 @@ FLUFFY_TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(FLUFFY_TOOLS))
 	nimbus \
 	fluffy \
 	nimbus_verified_proxy \
+	libverifproxy \
 	test \
 	test-reproducibility \
 	clean \
@@ -300,6 +315,15 @@ nimbus_verified_proxy: | build deps
 nimbus-verified-proxy-test: | build deps
 	$(ENV_SCRIPT) nim nimbus_verified_proxy_test $(NIM_PARAMS) nimbus.nims
 
+# Shared library for verified proxy
+
+libverifproxy: | build deps
+	+ echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim --version && \
+		$(ENV_SCRIPT) nim c --app:lib -d:"libp2p_pki_schemes=secp256k1" --noMain:on --threads:on --nimcache:nimcache/libverifproxy -o:$(VERIF_PROXY_OUT_PATH)/$@.$(VERIF_PROXY_SHAREDLIBEXT) $(NIM_PARAMS) nimbus_verified_proxy/libverifproxy/verifproxy.nim
+	cp nimbus_verified_proxy/libverifproxy/verifproxy.h $(VERIF_PROXY_OUT_PATH)/
+	echo -e $(BUILD_END_MSG) "build/$@"
+
 # builds transition tool
 t8n: | build deps
 	$(ENV_SCRIPT) nim c $(NIM_PARAMS) $(T8N_PARAMS) "tools/t8n/$@.nim"
@@ -322,7 +346,7 @@ txparse: | build deps
 
 # usual cleaning
 clean: | clean-common
-	rm -rf build/{nimbus,fluffy,nimbus_verified_proxy,$(TOOLS_CSV),$(FLUFFY_TOOLS_CSV),all_tests,test_kvstore_rocksdb,test_rpc,all_fluffy_tests,all_fluffy_portal_spec_tests,test_portal_testnet,utp_test_app,utp_test,*.dSYM}
+	rm -rf build/{nimbus,fluffy,libverifproxy,nimbus_verified_proxy,$(TOOLS_CSV),$(FLUFFY_TOOLS_CSV),all_tests,test_kvstore_rocksdb,test_rpc,all_fluffy_tests,all_fluffy_portal_spec_tests,test_portal_testnet,utp_test_app,utp_test,*.dSYM}
 	rm -rf tools/t8n/{t8n,t8n_test}
 	rm -rf tools/evmstate/{evmstate,evmstate_test}
 ifneq ($(USE_LIBBACKTRACE), 0)

nimbus/config.nim

@@ -24,7 +24,7 @@ import
     stew/byteutils,
     confutils/std/net
   ],
-  eth/[common, net/nat, p2p/bootnodes, p2p/enode, p2p/discoveryv5/enr],
+  eth/[common, net/utils, net/nat, p2p/bootnodes, p2p/enode, p2p/discoveryv5/enr],
   "."/[db/select_backend,
     constants, vm_compile_info, version
   ],

nimbus_verified_proxy/libverifproxy/nim.cfg

@@ -0,0 +1 @@
+-u:metrics

nimbus_verified_proxy/libverifproxy/verifproxy.h

@@ -0,0 +1,21 @@
+/**
+ * nimbus_verified_proxy
+ * Copyright (c) 2024 Status Research & Development GmbH
+ * Licensed and distributed under either of
+ *   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+ *   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+ * at your option. This file may not be copied, modified, or distributed except according to those terms.
+ */
+
+#ifndef __verifproxy__
+#define __verifproxy__
+
+typedef struct VerifProxyContext VerifProxyContext;
+typedef void (*onHeaderCallback)(const char* s, int t);
+
+void quit(void);
+
+VerifProxyContext* startVerifProxy(const char* configJson, onHeaderCallback onHeader);
+void stopVerifProxy(VerifProxyContext*);
+
+#endif /* __verifproxy__ */

nimbus_verified_proxy/libverifproxy/verifproxy.nim

@@ -0,0 +1,89 @@
+# nimbus_verified_proxy
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  std/[atomics, json, os, strutils],
+  ../nimbus_verified_proxy,
+  ../nimbus_verified_proxy_conf
+
+proc quit*() {.exportc, dynlib.} =
+  echo "Quitting"
+
+proc NimMain() {.importc, exportc, dynlib.}
+
+var initialized: Atomic[bool]
+
+proc initLib() =
+   if not initialized.exchange(true):
+     NimMain() # Every Nim library needs to call `NimMain` once exactly
+   when declared(setupForeignThreadGc): setupForeignThreadGc()
+   when declared(nimGC_setStackBottom):
+     var locals {.volatile, noinit.}: pointer
+     locals = addr(locals)
+     nimGC_setStackBottom(locals)
+
+
+proc runContext(ctx: ptr Context) {.thread.} =
+  let str = $ctx.configJson
+  try:
+    let jsonNode = parseJson(str)
+
+    let rpcAddr = jsonNode["RpcAddress"].getStr()
+    let myConfig = VerifiedProxyConf(
+      rpcAddress: ValidIpAddress.init(rpcAddr),
+      listenAddress: defaultListenAddress,
+      eth2Network: some(jsonNode["Eth2Network"].getStr()),
+      trustedBlockRoot: Eth2Digest.fromHex(jsonNode["TrustedBlockRoot"].getStr()),
+      web3Url: parseCmdArg(Web3Url, jsonNode["Web3Url"].getStr()),
+      rpcPort: Port(jsonNode["RpcPort"].getInt()),
+      logLevel: jsonNode["LogLevel"].getStr(),
+      maxPeers: 160,
+      nat: NatConfig(hasExtIp: false, nat: NatAny),
+      logStdout: StdoutLogKind.Auto,
+      dataDir: OutDir(defaultVerifiedProxyDataDir()),
+      tcpPort: Port(defaultEth2TcpPort),
+      udpPort: Port(defaultEth2TcpPort),
+      agentString: "nimbus",
+      discv5Enabled: true,
+    )
+
+    run(myConfig, ctx)
+  except Exception as err:
+    echo "Exception when running ", getCurrentExceptionMsg(), err.getStackTrace()
+    ctx.onHeader(getCurrentExceptionMsg(), 3)
+    ctx.cleanup()
+
+
+  #[let node = parseConfigAndRun(ctx.configJson)
+
+  while not ctx[].stop: # and node.running:
+    let timeout = sleepAsync(100.millis)
+    waitFor timeout
+
+  # do cleanup
+  node.stop()]#
+
+proc startVerifProxy*(configJson: cstring, onHeader: OnHeaderCallback): ptr Context {.exportc, dynlib.} =
+  initLib()
+
+  let ctx = createShared(Context, 1)
+  ctx.configJson = cast[cstring](allocShared0(len(configJson) + 1))
+  ctx.onHeader = onHeader
+  copyMem(ctx.configJson, configJson, len(configJson))
+
+  try:
+    createThread(ctx.thread, runContext, ctx)
+  except Exception as err:
+    echo "Exception when attempting to invoke createThread ", getCurrentExceptionMsg(), err.getStackTrace()
+    ctx.onHeader(getCurrentExceptionMsg(), 3)
+    ctx.cleanup()
+  return ctx
+
+proc stopVerifProxy*(ctx: ptr Context) {.exportc, dynlib.} =
+  ctx.stop = true
+
+

nimbus_verified_proxy/nimbus_verified_proxy.nim

@@ -8,7 +8,7 @@
 {.push raises: [].}
 
 import
-  std/[os, strutils],
+  std/[atomics, json, os, strutils],
   chronicles, chronos, confutils,
   eth/keys,
   json_rpc/rpcproxy,
@@ -26,6 +26,18 @@ import
 from beacon_chain/gossip_processing/block_processor import newExecutionPayload
 from beacon_chain/gossip_processing/eth2_processor import toValidationResult
 
+type OnHeaderCallback* = proc (s: cstring, t: int) {.cdecl, raises: [], gcsafe.}
+type Context* = object
+  thread*: Thread[ptr Context]
+  configJson*: cstring
+  stop*: bool
+  onHeader*: OnHeaderCallback
+
+proc cleanup*(ctx: ptr Context) =
+  dealloc(ctx.configJson)
+  freeShared(ctx)
+
+
 func getConfiguredChainId(networkMetadata: Eth2NetworkMetadata): Quantity =
   if networkMetadata.eth1Network.isSome():
     let
@@ -39,14 +51,22 @@ func getConfiguredChainId(networkMetadata: Eth2NetworkMetadata): Quantity =
   else:
     return networkMetadata.cfg.DEPOSIT_CHAIN_ID.Quantity
 
-proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
+proc run*(config: VerifiedProxyConf, ctx: ptr Context) {.raises: [CatchableError], gcsafe.} =
+  var headerCallback: OnHeaderCallback
+  if ctx != nil:
+    headerCallback = ctx.onHeader
+
   # Required as both Eth2Node and LightClient requires correct config type
   var lcConfig = config.asLightClientConf()
 
-  setupLogging(config.logLevel, config.logStdout, none(OutFile))
+  {.gcsafe.}:
+    setupLogging(config.logLevel, config.logStdout, none(OutFile))
 
-  notice "Launching Nimbus verified proxy",
-    version = fullVersionStr, cmdParams = commandLineParams(), config
+    try:
+      notice "Launching Nimbus verified proxy",
+        version = fullVersionStr, cmdParams = commandLineParams(), config
+    except Exception:
+      notice "commandLineParams() exception"
 
   let
     metadata = loadEth2Network(config.eth2Network)
@@ -167,6 +187,12 @@ proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
       when lcDataFork > LightClientDataFork.None:
         info "New LC finalized header",
           finalized_header = shortLog(forkyHeader)
+        if headerCallback != nil:
+          try:
+            headerCallback(Json.encode(forkyHeader), 0)
+          except SerializationError as e:
+            notice "finalizedHeaderCallback exception"
+
 
   proc onOptimisticHeader(
       lightClient: LightClient, optimisticHeader: ForkedLightClientHeader) =
@@ -175,6 +201,12 @@ proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
         info "New LC optimistic header",
           optimistic_header = shortLog(forkyHeader)
         optimisticProcessor.setOptimisticHeader(forkyHeader.beacon)
+        if headerCallback != nil:
+          try:
+            headerCallback(Json.encode(forkyHeader), 1)
+          except SerializationError as e:
+              notice "optimisticHeaderCallback exception"
+
 
   lightClient.onFinalizedHeader = onFinalizedHeader
   lightClient.onOptimisticHeader = onOptimisticHeader
@@ -252,11 +284,18 @@ proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
   asyncSpawn runOnSecondLoop()
   while true:
     poll()
+    if ctx != nil and ctx.stop:
+      # Cleanup
+      waitFor network.stop()
+      waitFor rpcProxy.stop()
+      ctx.cleanup()
+      # Notify client that cleanup is finished
+      headerCallback(nil, 2)
+      break
 
 when isMainModule:
   {.pop.}
   var config = makeBannerAndConfig(
     "Nimbus verified proxy " & fullVersionStr, VerifiedProxyConf)
   {.push raises: [].}
-
-  run(config)
+  run(config, nil)

nimbus_verified_proxy/nimbus_verified_proxy_conf.nim

@@ -205,7 +205,7 @@ func asLightClientConf*(pc: VerifiedProxyConf): LightClientConf =
     discv5Enabled: pc.discv5Enabled,
     directPeers: pc.directPeers,
     trustedBlockRoot: pc.trustedBlockRoot,
-    web3Urls: @[],
+    web3Urls: @[EngineApiUrlConfigValue(url: pc.web3url.web3Url)],
     jwtSecret: none(InputFile),
     stopAtEpoch: 0
   )

vendor/nim-confutils

@@ -1 +1 @@
-Subproject commit 7568f1b7c3142d8e87c1f3dd42924238926affbe
+Subproject commit fa6e9b09e2dfe09be5b734e7a7a394a36d953831