Add initial scaffold for portal lc network (#1277)

* Add initial scaffold for portal lc network

fluffy/network/beacon_light_client/light_client_content.nim

@@ -0,0 +1,108 @@
+# Nimbus - Portal Network
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [Defect].}
+
+import
+  stew/[arrayops, results],
+  beacon_chain/spec/forks,
+  beacon_chain/spec/datatypes/altair,
+  nimcrypto/[sha2, hash],
+  ssz_serialization,
+  ../../common/common_types
+
+export ssz_serialization, common_types, hash
+
+type
+  ContentType* = enum
+    lightClientBootstrap = 0x00
+    lightClientUpdate = 0x01
+    lightClientFinalityUpdate = 0x02
+    lightClientOptimisticUpdate = 0x03
+
+  # TODO Consider how we will gossip bootstraps?
+  # In normal LC operation node trust only one offered bootstrap, therefore offers
+  # of any other bootstraps would be rejected.
+  LightClientBootstrapKey* = object
+    blockHash*: Digest
+
+  #TODO Following types will need revision and improvements
+  LightClientUpdateKey* = object
+
+  LightClientFinalityUpdateKey* = object
+
+  LightClientOptimisticUpdateKey* = object
+
+  ContentKey* = object
+    case contentType*: ContentType
+    of lightClientBootstrap:
+      lightClientBootstrapKey*: LightClientBootstrapKey
+    of lightClientUpdate:
+      lightClientUpdateKey*: LightClientUpdateKey
+    of lightClientFinalityUpdate:
+      lightClientFinalityUpdateKey*: LightClientFinalityUpdateKey
+    of lightClientOptimisticUpdate:
+      lightClientOptimisticUpdateKey*: LightClientOptimisticUpdateKey
+
+  # Object internal to light_client_content module, which represent what will be
+  # published on the wire
+  ForkedLightClientBootstrap = object
+    forkDigest: ForkDigest
+    bootstrap: altair.LightClientBootstrap
+
+func encode*(contentKey: ContentKey): ByteList =
+  ByteList.init(SSZ.encode(contentKey))
+
+func decode*(contentKey: ByteList): Option[ContentKey] =
+  try:
+    some(SSZ.decode(contentKey.asSeq(), ContentKey))
+  except SszError:
+    return none[ContentKey]()
+
+func toContentId*(contentKey: ByteList): ContentId =
+  # TODO: Should we try to parse the content key here for invalid ones?
+  let idHash = sha2.sha256.digest(contentKey.asSeq())
+  readUintBE[256](idHash.data)
+
+func toContentId*(contentKey: ContentKey): ContentId =
+  toContentId(encode(contentKey))
+
+proc decodeBootstrap(
+    data: openArray[byte]): Result[altair.LightClientBootstrap, string] =
+  try:
+    let decoded = SSZ.decode(
+      data,
+      altair.LightClientBootstrap
+    )
+    return ok(decoded)
+  except SszError as exc:
+    return err(exc.msg)
+
+proc encodeBootstrapForked*(
+    fork: ForkDigest,
+    bs: altair.LightClientBootstrap): seq[byte] =
+  SSZ.encode(ForkedLightClientBootstrap(forkDigest: fork, bootstrap: bs))
+
+proc decodeBootstrapForked*(
+    forks: ForkDigests,
+    data: openArray[byte]): Result[altair.LightClientBootstrap, string] =
+
+  if len(data) < 4:
+    return Result[altair.LightClientBootstrap, string].err("Too short data")
+
+  let
+    arr = ForkDigest(array[4, byte].initCopyFrom(data))
+
+    beaconFork = forks.stateForkForDigest(arr).valueOr:
+      return Result[altair.LightClientBootstrap, string].err("Unknown fork")
+
+  if beaconFork >= BeaconStateFork.Altair:
+    return decodeBootstrap(data.toOpenArray(4, len(data) - 1))
+  else:
+    return Result[altair.LightClientBootstrap, string].err(
+      "LighClient data is avaialable only after Altair fork"
+    )

fluffy/network/beacon_light_client/light_client_network.nim

@@ -0,0 +1,167 @@
+# Nimbus - Portal Network
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [Defect].}
+
+import
+  std/[options, tables],
+  stew/[results, arrayops], chronos, chronicles,
+  eth/p2p/discoveryv5/[protocol, enr],
+  beacon_chain/spec/forks,
+  beacon_chain/spec/datatypes/[phase0, altair, bellatrix],
+  ../../content_db,
+  ../../../nimbus/constants,
+  ../wire/[portal_protocol, portal_stream, portal_protocol_config],
+  "."/light_client_content
+
+logScope:
+  topics = "portal_lc"
+
+const
+  lightClientProtocolId* = [byte 0x50, 0x1A]
+
+type
+  LightClientNetwork* = ref object
+    portalProtocol*: PortalProtocol
+    contentDB*: ContentDB
+    contentQueue*: AsyncQueue[(ContentKeysList, seq[seq[byte]])]
+    forkDigests*: ForkDigests
+    processContentLoop: Future[void]
+
+func toContentIdHandler(contentKey: ByteList): Option[ContentId] =
+  some(toContentId(contentKey))
+
+func encodeKey(k: ContentKey): (ByteList, ContentId) =
+  let keyEncoded = encode(k)
+  return (keyEncoded, toContentId(keyEncoded))
+
+proc dbGetHandler(db: ContentDB, contentId: ContentId):
+    Option[seq[byte]] {.raises: [Defect], gcsafe.} =
+  db.get(contentId)
+
+proc getLightClientBootstrap*(
+    l: LightClientNetwork,
+    trustedRoot: Digest): Future[results.Opt[altair.LightClientBootstrap]] {.async.}=
+  let
+    bk = LightClientBootstrapKey(blockHash: trustedRoot)
+    ck = ContentKey(
+      contentType: lightClientbootstrap,
+      lightClientBootstrapKey: bk
+    )
+    keyEncoded = encode(ck)
+    contentID = toContentId(keyEncoded)
+
+  let bootstrapContentLookup =
+      await l.portalProtocol.contentLookup(keyEncoded, contentId)
+
+  if bootstrapContentLookup.isNone():
+      warn "Failed fetching block header from the network", trustedRoot, contentKey = keyEncoded
+      return Opt.none(altair.LightClientBootstrap)
+
+  let
+    bootstrap = bootstrapContentLookup.unsafeGet()
+    decodingResult = decodeBootstrapForked(l.forkDigests, bootstrap.content)
+
+  if decodingResult.isErr:
+    return Opt.none(altair.LightClientBootstrap)
+  else:
+    # TODO Not doing validation for now, as probably it should be done by layer
+    # above
+    return Opt.some(decodingResult.get())
+
+proc new*(
+    T: type LightClientNetwork,
+    baseProtocol: protocol.Protocol,
+    contentDB: ContentDB,
+    streamManager: StreamManager,
+    forkDigests: ForkDigests,
+    bootstrapRecords: openArray[Record] = [],
+    portalConfig: PortalProtocolConfig = defaultPortalProtocolConfig): T =
+  let
+    contentQueue = newAsyncQueue[(ContentKeysList, seq[seq[byte]])](50)
+
+    stream = streamManager.registerNewStream(contentQueue)
+
+    portalProtocol = PortalProtocol.new(
+      baseProtocol, lightClientProtocolId, contentDB,
+      toContentIdHandler, dbGetHandler, stream, bootstrapRecords,
+      config = portalConfig)
+
+  LightClientNetwork(
+    portalProtocol: portalProtocol,
+    contentDB: contentDB,
+    contentQueue: contentQueue,
+    forkDigests: forkDigests
+  )
+
+# TODO this should be probably supplied by upper layer i.e Light client which uses
+# light client network as data provider as only it has all necessary context to
+# validate data
+proc validateContent(
+    n: LightClientNetwork, content: seq[byte], contentKey: ByteList):
+    Future[bool] {.async.} =
+  return true
+
+proc validateContent(
+    n: LightClientNetwork,
+    contentKeys: ContentKeysList,
+    contentItems: seq[seq[byte]]): Future[bool] {.async.} =
+  # content passed here can have less items then contentKeys, but not more.
+  for i, contentItem in contentItems:
+    let contentKey = contentKeys[i]
+    if await n.validateContent(contentItem, contentKey):
+      let contentIdOpt = n.portalProtocol.toContentId(contentKey)
+      if contentIdOpt.isNone():
+        error "Received offered content with invalid content key", contentKey
+        return false
+
+      let contentId = contentIdOpt.get()
+
+      n.portalProtocol.storeContent(contentId, contentItem)
+
+      info "Received offered content validated successfully", contentKey
+
+    else:
+      error "Received offered content failed validation", contentKey
+      return false
+
+  return true
+
+proc neighborhoodGossipDiscardPeers(
+    p: PortalProtocol,
+    contentKeys: ContentKeysList,
+    content: seq[seq[byte]]): Future[void] {.async.} =
+  discard await p.neighborhoodGossip(contentKeys, content)
+
+proc processContentLoop(n: LightClientNetwork) {.async.} =
+  try:
+    while true:
+      let (contentKeys, contentItems) =
+        await n.contentQueue.popFirst()
+
+      # When there is one invalid content item, all other content items are
+      # dropped and not gossiped around.
+      # TODO: Differentiate between failures due to invalid data and failures
+      # due to missing network data for validation.
+      if await n.validateContent(contentKeys, contentItems):
+        asyncSpawn n.portalProtocol.neighborhoodGossipDiscardPeers(
+          contentKeys, contentItems
+        )
+
+  except CancelledError:
+    trace "processContentLoop canceled"
+
+proc start*(n: LightClientNetwork) =
+  info "Starting portal light client network"
+  n.portalProtocol.start()
+  n.processContentLoop = processContentLoop(n)
+
+proc stop*(n: LightClientNetwork) =
+  n.portalProtocol.stop()
+
+  if not n.processContentLoop.isNil:
+    n.processContentLoop.cancel()

fluffy/tests/all_fluffy_tests.nim

@@ -16,4 +16,6 @@ import
   ./test_history_network,
   ./test_content_db,
   ./test_discovery_rpc,
-  ./test_bridge_parser
+  ./test_bridge_parser,
+  ./test_light_client_content,
+  ./test_light_client_network

fluffy/tests/test_light_client_content.nim

@@ -0,0 +1,51 @@
+# Nimbus - Portal Network
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.used.}
+
+import
+  unittest2,
+  stew/byteutils,
+  stint,
+  beacon_chain/spec/forks,
+  beacon_chain/spec/datatypes/altair,
+  ../network/beacon_light_client/light_client_content,
+  ./light_client_data/light_client_test_data
+
+suite "Test light client contentEncodings":
+  var forks: ForkDigests
+  forks.phase0 = ForkDigest([0'u8, 0, 0, 1])
+  forks.altair = ForkDigest([0'u8, 0, 0, 2])
+  forks.bellatrix = ForkDigest([0'u8, 0, 0, 3])
+  forks.capella = ForkDigest([0'u8, 0, 0, 4])
+  forks.sharding = ForkDigest([0'u8, 0, 0, 5])
+
+  test "Light client bootstrap correct":
+    let
+      bootstrap = SSZ.decode(bootStrapBytes, altair.LightClientBootstrap)
+      encodedForked = encodeBootstrapForked(forks.altair, bootstrap)
+      decodedResult = decodeBootstrapForked(forks, encodedForked)
+
+    check:
+      decodedResult.isOk()
+      decodedResult.get() == bootstrap
+
+  test "Light client bootstrap failures":
+    let
+      bootstrap = SSZ.decode(bootStrapBytes, altair.LightClientBootstrap)
+      encodedTooEarlyFork = encodeBootstrapForked(forks.phase0, bootstrap)
+      encodedUnknownFork = encodeBootstrapForked(
+        ForkDigest([0'u8, 0, 0, 6]), bootstrap
+      )
+
+    check:
+      decodeBootstrapForked(forks, @[]).isErr()
+      decodeBootstrapForked(forks, encodedTooEarlyFork).isErr()
+      decodeBootstrapForked(forks, encodedUnknownFork).isErr()
+
+
+

fluffy/tests/test_light_client_network.nim

@@ -0,0 +1,105 @@
+# Nimbus - Portal Network
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  std/os,
+  testutils/unittests, chronos,
+  eth/p2p/discoveryv5/protocol as discv5_protocol, eth/p2p/discoveryv5/routing_table,
+  eth/common/eth_types_rlp,
+  eth/rlp,
+  beacon_chain/spec/forks,
+  beacon_chain/spec/datatypes/altair,
+  ../network/wire/[portal_protocol, portal_stream, portal_protocol_config],
+  ../network/beacon_light_client/[light_client_network, light_client_content],
+  ../../nimbus/constants,
+  ../content_db,
+  ./test_helpers,
+  ./light_client_data/light_client_test_data
+
+type LightClientNode = ref object
+  discoveryProtocol*: discv5_protocol.Protocol
+  lightClientNetwork*: LightClientNetwork
+
+proc getTestForkDigests(): ForkDigests =
+  return ForkDigests(
+    phase0: ForkDigest([0'u8, 0, 0, 1]),
+    altair: ForkDigest([0'u8, 0, 0, 2]),
+    bellatrix: ForkDigest([0'u8, 0, 0, 3]),
+    capella: ForkDigest([0'u8, 0, 0, 4]),
+    sharding: ForkDigest([0'u8, 0, 0, 5])
+  )
+
+proc newLCNode(rng: ref HmacDrbgContext, port: int): LightClientNode =
+  let
+    node = initDiscoveryNode(rng, PrivateKey.random(rng[]), localAddress(port))
+    db = ContentDB.new("", uint32.high, inMemory = true)
+    streamManager = StreamManager.new(node)
+    hn = LightClientNetwork.new(node, db, streamManager, getTestForkDigests())
+
+  return LightClientNode(discoveryProtocol: node, lightClientNetwork: hn)
+
+proc portalProtocol(hn: LightClientNode): PortalProtocol =
+  hn.lightClientNetwork.portalProtocol
+
+proc localNode(hn: LightClientNode): Node =
+  hn.discoveryProtocol.localNode
+
+proc start(hn: LightClientNode) =
+  hn.lightClientNetwork.start()
+
+proc stop(hn: LightClientNode) {.async.} =
+  hn.lightClientNetwork.stop()
+  await hn.discoveryProtocol.closeWait()
+
+proc containsId(hn: LightClientNode, contentId: ContentId): bool =
+  return hn.lightClientNetwork.contentDB.get(contentId).isSome()
+
+procSuite "Light client Content Network":
+  let rng = newRng()
+
+  asyncTest "Get bootstrap by trusted block hash":
+    let
+      lcNode1 = newLCNode(rng, 20302)
+      lcNode2 = newLCNode(rng, 20303)
+      forks = getTestForkDigests()
+
+    check:
+      lcNode1.portalProtocol().addNode(lcNode2.localNode()) == Added
+      lcNode2.portalProtocol().addNode(lcNode1.localNode()) == Added
+
+      (await lcNode1.portalProtocol().ping(lcNode2.localNode())).isOk()
+      (await lcNode2.portalProtocol().ping(lcNode1.localNode())).isOk()
+
+    let
+      bootstrap = SSZ.decode(bootstrapBytes, altair.LightClientBootstrap)
+      bootstrappHeaderHash = hash_tree_root(bootstrap.header)
+      bootstrapKey = LightClientBootstrapKey(
+        blockHash: bootstrappHeaderHash
+      )
+      bootstrapContentKey = ContentKey(
+        contentType: lightClientBootstrap,
+        lightClientBootstrapKey: bootstrapKey
+      )
+
+      bootstrapContentKeyEncoded = encode(bootstrapContentKey)
+      bootstrapContentId = toContentId(bootstrapContentKeyEncoded)
+
+    lcNode2.portalProtocol().storeContent(
+      bootstrapContentId, encodeBootstrapForked(forks.altair, bootstrap)
+    )
+
+    let bootstrapFromNetworkResult =
+      await lcNode1.lightClientNetwork.getLightClientBootstrap(
+        bootstrappHeaderHash
+      )
+
+    check:
+      bootstrapFromNetworkResult.isOk()
+      bootstrapFromNetworkResult.get() == bootstrap
+
+    await lcNode1.stop()
+    await lcNode2.stop()