Initial Binary for light client proxy (#1202)

* Initial Binary for light client proxy

Makefile

@@ -62,6 +62,7 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 	update \
 	nimbus \
 	fluffy \
+	lc_proxy \
 	test \
 	test-reproducibility \
 	clean \
@@ -192,6 +193,10 @@ fluffy: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim fluffy $(NIM_PARAMS) nimbus.nims
 
+lc-proxy: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim lc_proxy $(NIM_PARAMS) nimbus.nims
+
 # primitive reproducibility test
 fluffy-test-reproducibility:
 	+ [ -e build/fluffy ] || $(MAKE) V=0 fluffy; \

lc_proxy/lc_proxy.nim

@@ -0,0 +1,283 @@
+# light client proxy
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# This implements the pre-release proposal of the libp2p based light client sync
+# protocol. See https://github.com/ethereum/consensus-specs/pull/2802
+
+{.push raises: [Defect].}
+
+import
+  std/[os, strutils],
+  chronicles, chronicles/chronos_tools, chronos,
+  eth/keys,
+  beacon_chain/eth1/eth1_monitor,
+  beacon_chain/gossip_processing/optimistic_processor,
+  beacon_chain/networking/topic_params,
+  beacon_chain/spec/beaconstate,
+  beacon_chain/spec/datatypes/[phase0, altair, bellatrix],
+  beacon_chain/[light_client, nimbus_binary_common, version],
+  ./rpc/rpc_eth_lc_api
+
+from beacon_chain/consensus_object_pools/consensus_manager import runForkchoiceUpdated
+from beacon_chain/gossip_processing/block_processor import newExecutionPayload
+from beacon_chain/gossip_processing/eth2_processor import toValidationResult
+
+# TODO Find what can throw exception
+proc run() {.raises: [Exception, Defect].}=
+  {.pop.}
+  var config = makeBannerAndConfig(
+    "Nimbus light client " & fullVersionStr, LightClientConf)
+  {.push raises: [Defect].}
+
+  setupLogging(config.logLevel, config.logStdout, config.logFile)
+
+  notice "Launching light client",
+    version = fullVersionStr, cmdParams = commandLineParams(), config
+
+  let metadata = loadEth2Network(config.eth2Network)
+  for node in metadata.bootstrapNodes:
+    config.bootstrapNodes.add node
+  template cfg(): auto = metadata.cfg
+
+  let
+    genesisState =
+      try:
+        template genesisData(): auto = metadata.genesisData
+        newClone(readSszForkedHashedBeaconState(
+          cfg, genesisData.toOpenArrayByte(genesisData.low, genesisData.high)))
+      except CatchableError as err:
+        raiseAssert "Invalid baked-in state: " & err.msg
+
+    beaconClock = BeaconClock.init(getStateField(genesisState[], genesis_time))
+
+    getBeaconTime = beaconClock.getBeaconTimeFn()
+
+    genesis_validators_root =
+      getStateField(genesisState[], genesis_validators_root)
+    forkDigests = newClone ForkDigests.init(cfg, genesis_validators_root)
+
+    genesisBlockRoot = get_initial_beacon_block(genesisState[]).root
+
+    rng = keys.newRng()
+
+    netKeys = getRandomNetKeys(rng[])
+
+    network = createEth2Node(
+      rng, config, netKeys, cfg,
+      forkDigests, getBeaconTime, genesis_validators_root)
+
+    eth1Mon =
+      if config.web3Urls.len > 0:
+        let res = Eth1Monitor.init(
+          cfg, db = nil, getBeaconTime, config.web3Urls,
+          none(DepositContractSnapshot), metadata.eth1Network,
+          forcePolling = false,
+          rng[].loadJwtSecret(config, allowCreate = false),
+          true)
+        waitFor res.ensureDataProvider()
+        res
+      else:
+        nil
+
+    rpcServerWithProxy =
+      if config.web3Urls.len > 0:
+        var web3Url = config.web3Urls[0]
+        fixupWeb3Urls web3Url
+
+        let proxyUri = some web3Url
+
+        if proxyUri.isSome:
+          info "Initializing LC eth API proxy", proxyUri = proxyUri.get
+          let
+            ta = initTAddress("127.0.0.1:8545")
+            clientConfig =
+              case parseUri(proxyUri.get).scheme.toLowerAscii():
+              of "http", "https":
+                getHttpClientConfig(proxyUri.get)
+              of "ws", "wss":
+                getWebSocketClientConfig(proxyUri.get)
+              else:
+                fatal "Unsupported scheme", proxyUri = proxyUri.get
+                quit QuitFailure
+          RpcProxy.new([ta], clientConfig)
+        else:
+          warn "Ignoring `rpcEnabled`, no `proxyUri` provided"
+          nil
+      else:
+        nil
+
+    lcProxy =
+      if rpcServerWithProxy != nil:
+        let res = LightClientRpcProxy(proxy: rpcServerWithProxy)
+        res.installEthApiHandlers()
+        res
+      else:
+        nil
+
+    optimisticHandler = proc(signedBlock: ForkedMsgTrustedSignedBeaconBlock):
+        Future[void] {.async.} =
+      notice "New LC optimistic block",
+        opt = signedBlock.toBlockId(),
+        wallSlot = getBeaconTime().slotOrZero
+      withBlck(signedBlock):
+        when stateFork >= BeaconStateFork.Bellatrix:
+          if blck.message.is_execution_block:
+            template payload(): auto = blck.message.body.execution_payload
+
+            if eth1Mon != nil:
+              await eth1Mon.ensureDataProvider()
+
+              # engine_newPayloadV1
+              discard await eth1Mon.newExecutionPayload(payload)
+
+              # engine_forkchoiceUpdatedV1
+              discard await eth1Mon.runForkchoiceUpdated(
+                headBlockRoot = payload.block_hash,
+                safeBlockRoot = payload.block_hash,  # stub value
+                finalizedBlockRoot = ZERO_HASH)
+
+            if lcProxy != nil:
+              lcProxy.executionPayload.ok payload.asEngineExecutionPayload()
+        else: discard
+      return
+
+    optimisticProcessor = initOptimisticProcessor(
+      getBeaconTime, optimisticHandler)
+
+    lightClient = createLightClient(
+      network, rng, config, cfg, forkDigests, getBeaconTime,
+      genesis_validators_root, LightClientFinalizationMode.Optimistic)
+
+  info "Listening to incoming network requests"
+  network.initBeaconSync(cfg, forkDigests, genesisBlockRoot, getBeaconTime)
+  network.addValidator(
+    getBeaconBlocksTopic(forkDigests.phase0),
+    proc (signedBlock: phase0.SignedBeaconBlock): ValidationResult =
+      toValidationResult(
+        optimisticProcessor.processSignedBeaconBlock(signedBlock)))
+  network.addValidator(
+    getBeaconBlocksTopic(forkDigests.altair),
+    proc (signedBlock: altair.SignedBeaconBlock): ValidationResult =
+      toValidationResult(
+        optimisticProcessor.processSignedBeaconBlock(signedBlock)))
+  network.addValidator(
+    getBeaconBlocksTopic(forkDigests.bellatrix),
+    proc (signedBlock: bellatrix.SignedBeaconBlock): ValidationResult =
+      toValidationResult(
+        optimisticProcessor.processSignedBeaconBlock(signedBlock)))
+  lightClient.installMessageValidators()
+  waitFor network.startListening()
+  waitFor network.start()
+
+  if lcProxy != nil:
+    waitFor lcProxy.proxy.start()
+
+  proc onFinalizedHeader(
+      lightClient: LightClient, finalizedHeader: BeaconBlockHeader) =
+    info "New LC finalized header",
+      finalized_header = shortLog(finalizedHeader)
+    optimisticProcessor.setFinalizedHeader(finalizedHeader)
+
+  proc onOptimisticHeader(
+      lightClient: LightClient, optimisticHeader: BeaconBlockHeader) =
+    info "New LC optimistic header",
+      optimistic_header = shortLog(optimisticHeader)
+    optimisticProcessor.setOptimisticHeader(optimisticHeader)
+
+  lightClient.onFinalizedHeader = onFinalizedHeader
+  lightClient.onOptimisticHeader = onOptimisticHeader
+  lightClient.trustedBlockRoot = some config.trustedBlockRoot
+
+  func shouldSyncOptimistically(wallSlot: Slot): bool =
+    # Check whether an EL is connected
+    if eth1Mon == nil and lcProxy == nil:
+      return false
+
+    # Check whether light client is used
+    let optimisticHeader = lightClient.optimisticHeader.valueOr:
+      return false
+
+    # Check whether light client has synced sufficiently close to wall slot
+    const maxAge = 2 * SLOTS_PER_EPOCH
+    if optimisticHeader.slot < max(wallSlot, maxAge.Slot) - maxAge:
+      return false
+
+    true
+
+  var blocksGossipState: GossipState = {}
+  proc updateBlocksGossipStatus(slot: Slot) =
+    let
+      isBehind = not shouldSyncOptimistically(slot)
+
+      targetGossipState = getTargetGossipState(
+        slot.epoch, cfg.ALTAIR_FORK_EPOCH, cfg.BELLATRIX_FORK_EPOCH, isBehind)
+
+    template currentGossipState(): auto = blocksGossipState
+    if currentGossipState == targetGossipState:
+      return
+
+    if currentGossipState.card == 0 and targetGossipState.card > 0:
+      debug "Enabling blocks topic subscriptions",
+        wallSlot = slot, targetGossipState
+    elif currentGossipState.card > 0 and targetGossipState.card == 0:
+      debug "Disabling blocks topic subscriptions",
+        wallSlot = slot
+    else:
+      # Individual forks added / removed
+      discard
+
+    let
+      newGossipForks = targetGossipState - currentGossipState
+      oldGossipForks = currentGossipState - targetGossipState
+
+    for gossipFork in oldGossipForks:
+      let forkDigest = forkDigests[].atStateFork(gossipFork)
+      network.unsubscribe(getBeaconBlocksTopic(forkDigest))
+
+    for gossipFork in newGossipForks:
+      let forkDigest = forkDigests[].atStateFork(gossipFork)
+      network.subscribe(
+        getBeaconBlocksTopic(forkDigest), blocksTopicParams,
+        enableTopicMetrics = true)
+
+    blocksGossipState = targetGossipState
+
+  var nextExchangeTransitionConfTime: Moment
+
+  proc onSecond(time: Moment) =
+    # engine_exchangeTransitionConfigurationV1
+    if time > nextExchangeTransitionConfTime and eth1Mon != nil:
+      nextExchangeTransitionConfTime = time + chronos.minutes(1)
+      traceAsyncErrors eth1Mon.exchangeTransitionConfiguration()
+
+    let wallSlot = getBeaconTime().slotOrZero()
+    if checkIfShouldStopAtEpoch(wallSlot, config.stopAtEpoch):
+      quit(0)
+
+    lightClient.updateGossipStatus(wallSlot + 1)
+
+  proc runOnSecondLoop() {.async.} =
+    let sleepTime = chronos.seconds(1)
+    while true:
+      let start = chronos.now(chronos.Moment)
+      await chronos.sleepAsync(sleepTime)
+      let afterSleep = chronos.now(chronos.Moment)
+      let sleepTime = afterSleep - start
+      onSecond(start)
+      let finished = chronos.now(chronos.Moment)
+      let processingTime = finished - afterSleep
+      trace "onSecond task completed", sleepTime, processingTime
+
+  onSecond(Moment.now())
+  lightClient.start()
+
+  asyncSpawn runOnSecondLoop()
+  while true:
+    poll()
+
+when isMainModule:
+  run()

lc_proxy/rpc/rpc_eth_lc_api.nim

@@ -0,0 +1,51 @@
+# beacon_chain
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [Defect].}
+
+import
+  json_rpc/[rpcproxy, rpcserver],
+  web3/conversions,
+  ../../nimbus/rpc/[hexstrings, rpc_types],
+  beacon_chain/eth1/eth1_monitor,
+  beacon_chain/spec/forks
+
+export rpcproxy, forks
+
+template encodeQuantity(value: UInt256): HexQuantityStr =
+  HexQuantityStr("0x" & value.toHex())
+
+proc encodeQuantity(q: Quantity): hexstrings.HexQuantityStr =
+  return hexstrings.encodeQuantity(distinctBase(q))
+
+type LightClientRpcProxy* = ref object
+  proxy*: RpcProxy
+  executionPayload*: Opt[ExecutionPayloadV1]
+
+proc installEthApiHandlers*(lcProxy: LightClientRpcProxy) =
+  template payload(): Opt[ExecutionPayloadV1] = lcProxy.executionPayload
+
+  lcProxy.proxy.rpc("eth_blockNumber") do() -> HexQuantityStr:
+    ## Returns the number of most recent block.
+    if payload.isNone:
+      raise newException(ValueError, "Syncing")
+
+    return encodeQuantity(payload.get.blockNumber)
+
+  lcProxy.proxy.rpc("eth_getBlockByNumber") do(
+      quantityTag: string, fullTransactions: bool) -> Option[rpc_types.BlockObject]:
+    ## Returns information about a block by number.
+    if payload.isNone:
+      raise newException(ValueError, "Syncing")
+
+    if quantityTag != "latest":
+      raise newException(ValueError, "Only latest block is supported")
+
+    if fullTransactions:
+      raise newException(ValueError, "Transaction bodies not supported")
+
+    return some rpc_types.BlockObject(number: some(encodeQuantity(payload.get.blockNumber)))

nimbus.nimble

@@ -31,6 +31,7 @@ when declared(namedBin):
   namedBin = {
     "nimbus/nimbus": "nimbus",
     "fluffy/fluffy": "fluffy",
+    "lc_proxy/lc_proxy": "lc_proxy",
     "fluffy/tools/portalcli": "portalcli",
   }.toTable()
 
@@ -68,6 +69,9 @@ task test_rocksdb, "Run rocksdb tests":
 task fluffy, "Build fluffy":
   buildBinary "fluffy", "fluffy/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false"
 
+task lc_proxy, "Build light client proxy":
+  buildBinary "lc_proxy", "lc_proxy/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false -d:libp2p_pki_schemes=secp256k1"
+
 task fluffy_tools, "Build fluffy tools":
   buildBinary "portalcli", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false"
   buildBinary "blockwalk", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"

vendor/nim-chronos

@@ -1 +1 @@
-Subproject commit f2e4d447d6aec99b3641d51994650769c5c00d02
+Subproject commit 1334cdfebdc6182ff752e7d20796d9936cc8faa3

vendor/nim-http-utils

@@ -1 +1 @@
-Subproject commit f83fbce4d6ec7927b75be3f85e4fa905fcb69788
+Subproject commit e88e231dfcef4585fe3b2fbd9b664dbd28a88040

vendor/nim-ssz-serialization

@@ -1 +1 @@
-Subproject commit f1b14875792df7b1e76c98c9ee669026d7cfe6bb
+Subproject commit 639758dbd9a5f3e75a15449ddf80d6fd1cfa585e

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit 1e86bd1ef38f78c601b07da7188e65785f2c0ed8
+Subproject commit 018760954a1530b7336aed7133393908875d860f

vendor/nimbus-eth2

@@ -1 +1 @@
-Subproject commit 5c91d29df0eebff2e3ea10d5adf99943545c8b1a
+Subproject commit 64972e3c8a6eccc49054e95a893048916b1806a4