Extend light client network (#1287)

fluffy/network/beacon_light_client/beacon_light_client_manager.nim

@@ -14,16 +14,13 @@ import
   eth/p2p/discoveryv5/random2,
   beacon_chain/spec/datatypes/altair,
   beacon_chain/beacon_clock,
-  ./light_client_network
+  "."/[light_client_network, light_client_content]
 
 from beacon_chain/consensus_object_pools/block_pools_types import BlockError
 
 logScope:
   topics = "lcman"
 
-# https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/altair/light-client/p2p-interface.md#configuration
-const MAX_REQUEST_LIGHT_CLIENT_UPDATES* = 128
-
 type
   Nothing = object
   NetRes*[T] = Result[T, void]

fluffy/network/beacon_light_client/light_client_content.nim

@@ -8,15 +8,26 @@
 {.push raises: [Defect].}
 
 import
+  std/[sequtils, typetraits],
   stew/[arrayops, results],
   beacon_chain/spec/forks,
   beacon_chain/spec/datatypes/altair,
   nimcrypto/[sha2, hash],
   ssz_serialization,
+  ssz_serialization/codec,
   ../../common/common_types
 
 export ssz_serialization, common_types, hash
 
+# https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/altair/light-client/p2p-interface.md#configuration
+const
+  MAX_REQUEST_LIGHT_CLIENT_UPDATES* = 128
+
+  # Needed to properly encode List[List[byte, XXX], MAX_REQUEST_LIGHT_CLIENT_UPDATES]
+  # based on eth2 MAX_CHUNK_SIZE, light client update should not be bigger than
+  # that
+  MAX_LIGHT_CLIENT_UPDATE_SIZE* = 1 * 1024 * 1024
+
 type
   ContentType* = enum
     lightClientBootstrap = 0x00
@@ -48,11 +59,8 @@ type
     of lightClientOptimisticUpdate:
       lightClientOptimisticUpdateKey*: LightClientOptimisticUpdateKey
 
-  # Object internal to light_client_content module, which represent what will be
-  # published on the wire
-  ForkedLightClientBootstrap = object
-    forkDigest: ForkDigest
-    bootstrap: altair.LightClientBootstrap
+  ForkedLightClientUpdateBytes = List[byte, MAX_LIGHT_CLIENT_UPDATE_SIZE]
+  LightClientUpdateList = List[ForkedLightClientUpdateBytes, MAX_REQUEST_LIGHT_CLIENT_UPDATES]
 
 func encode*(contentKey: ContentKey): ByteList =
   ByteList.init(SSZ.encode(contentKey))
@@ -82,27 +90,125 @@ proc decodeBootstrap(
   except SszError as exc:
     return err(exc.msg)
 
+proc decodeLighClientObject(
+    ObjType: type altair.SomeLightClientObject,
+    data: openArray[byte]): Result[ObjType, string] =
+  try:
+    let decoded = SSZ.decode(
+      data,
+      ObjType
+    )
+    return ok(decoded)
+  except SszError as exc:
+    return err(exc.msg)
+
+proc encodeForked*(
+    ObjType: type altair.SomeLightClientObject,
+    fork: ForkDigest,
+    obj: ObjType): seq[byte] =
+  # TODO probably not super efficient
+  let arr = distinctBase(fork)
+  let enc = SSZ.encode(obj)
+  return concat(@arr, enc)
+
 proc encodeBootstrapForked*(
     fork: ForkDigest,
     bs: altair.LightClientBootstrap): seq[byte] =
-  SSZ.encode(ForkedLightClientBootstrap(forkDigest: fork, bootstrap: bs))
+  return encodeForked(altair.LightClientBootstrap, fork, bs)
 
-proc decodeBootstrapForked*(
+proc encodeFinalityUpdateForked*(
+    fork: ForkDigest,
+    update: altair.LightClientFinalityUpdate): seq[byte] =
+  return encodeForked(altair.LightClientFinalityUpdate, fork, update)
+
+proc encodeOptimisticUpdateForked*(
+    fork: ForkDigest,
+    update: altair.LightClientOptimisticUpdate): seq[byte] =
+  return encodeForked(altair.LightClientOptimisticUpdate, fork, update)
+
+proc decodeForkedLightClientObject(
+    ObjType: type altair.SomeLightClientObject,
     forks: ForkDigests,
-    data: openArray[byte]): Result[altair.LightClientBootstrap, string] =
-
+    data: openArray[byte]): Result[ObjType, string] =
   if len(data) < 4:
-    return Result[altair.LightClientBootstrap, string].err("Too short data")
+    return Result[ObjType, string].err("Too short data")
 
   let
     arr = ForkDigest(array[4, byte].initCopyFrom(data))
 
     beaconFork = forks.stateForkForDigest(arr).valueOr:
-      return Result[altair.LightClientBootstrap, string].err("Unknown fork")
+      return Result[ObjType, string].err("Unknown fork")
 
   if beaconFork >= BeaconStateFork.Altair:
-    return decodeBootstrap(data.toOpenArray(4, len(data) - 1))
+    return decodeLighClientObject(ObjType, data.toOpenArray(4, len(data) - 1))
   else:
-    return Result[altair.LightClientBootstrap, string].err(
+    return Result[ObjType, string].err(
       "LighClient data is avaialable only after Altair fork"
     )
+
+proc decodeBootstrapForked*(
+    forks: ForkDigests,
+    data: openArray[byte]): Result[altair.LightClientBootstrap, string] =
+  return decodeForkedLightClientObject(
+    altair.LightClientBootstrap,
+    forks,
+    data
+  )
+
+proc decodeLightClientUpdateForked*(
+    forks: ForkDigests,
+    data: openArray[byte]): Result[altair.LightClientUpdate, string] =
+  return decodeForkedLightClientObject(
+    altair.LightClientUpdate,
+    forks,
+    data
+  )
+
+proc decodeLightClientFinalityUpdateForked*(
+    forks: ForkDigests,
+    data: openArray[byte]): Result[altair.LightClientFinalityUpdate, string] =
+  return decodeForkedLightClientObject(
+    altair.LightClientFinalityUpdate,
+    forks,
+    data
+  )
+
+proc decodeLightClientOptimisticUpdateForked*(
+    forks: ForkDigests,
+    data: openArray[byte]): Result[altair.LightClientOptimisticUpdate, string] =
+  return decodeForkedLightClientObject(
+    altair.LightClientOptimisticUpdate,
+    forks,
+    data
+  )
+
+proc encodeLightClientUpdatesForked*(
+    fork: ForkDigest,
+    objects: openArray[altair.LightClientUpdate]
+): seq[byte] =
+  var lu: LightClientUpdateList
+  for obj in objects:
+    discard lu.add(
+      ForkedLightClientUpdateBytes(encodeForked(altair.LightClientUpdate, fork, obj))
+    )
+
+  return SSZ.encode(lu)
+
+proc decodeLightClientUpdatesForked*(
+    forks: ForkDigests,
+    data: openArray[byte]): Result[seq[altair.LightClientUpdate], string] =
+  try:
+    let listDecoded = SSZ.decode(
+      data,
+      LightClientUpdateList
+    )
+
+    var updates: seq[altair.LightClientUpdate]
+
+    for enc in listDecoded:
+      let updateDecoded = ? decodeLightClientUpdateForked(forks, enc.asSeq())
+      updates.add(updateDecoded)
+
+    return ok(updates)
+  except SszError as exc:
+    return err(exc.msg)

fluffy/network/beacon_light_client/light_client_network.nim

@@ -80,17 +80,67 @@ proc getLightClientUpdatesByRange*(
   # TODO: Not implemented!
   return Opt.none(seq[altair.LightClientUpdate])
 
+proc getLatestUpdate( l: LightClientNetwork, optimistic: bool):Future[results.Opt[seq[byte]]] {.async.} =
+  let ck =
+    if optimistic:
+      ContentKey(
+        contentType: lightClientOptimisticUpdate,
+        lightClientOptimisticUpdateKey: LightClientOptimisticUpdateKey()
+      )
+    else:
+      ContentKey(
+        contentType: lightClientFinalityUpdate,
+        lightClientFinalityUpdateKey: LightClientFinalityUpdateKey()
+      )
+
+  let
+    keyEncoded = encode(ck)
+    contentID = toContentId(keyEncoded)
+    updateLooukup = await l.portalProtocol.contentLookup(keyEncoded, contentId)
+
+  if updateLooukup.isNone():
+    warn "Failed fetching update from the network", contentKey = keyEncoded
+    return Opt.none(seq[byte])
+
+  return ok(updateLooukup.get().content)
+
+# TODO: Currently both getLightClientFinalityUpdate and getLightClientOptimisticUpdate
+# are implemented in naive way as finding first peer with any of those updates
+# and treating it as latest. This will probably need to get improved.
 proc getLightClientFinalityUpdate*(
     l: LightClientNetwork
   ): Future[results.Opt[altair.LightClientFinalityUpdate]] {.async.} =
-  # TODO: Not implemented!
-  return Opt.none(altair.LightClientFinalityUpdate)
+  let lookupResult = await l.getLatestUpdate(optimistic = false)
+
+  if lookupResult.isErr:
+    return Opt.none(altair.LightClientFinalityUpdate)
+
+  let
+    finalityUpdate = lookupResult.get()
+    decodingResult = decodeLightClientFinalityUpdateForked(l.forkDigests, finalityUpdate)
+
+  if decodingResult.isErr:
+    return Opt.none(altair.LightClientFinalityUpdate)
+  else:
+    return Opt.some(decodingResult.get())
 
 proc getLightClientOptimisticUpdate*(
     l: LightClientNetwork
   ): Future[results.Opt[altair.LightClientOptimisticUpdate]] {.async.} =
-  # TODO: Not implemented!
-  return Opt.none(altair.LightClientOptimisticUpdate)
+
+  let lookupResult = await l.getLatestUpdate(optimistic = true)
+
+  if lookupResult.isErr:
+    return Opt.none(altair.LightClientOptimisticUpdate)
+
+  let
+    optimimsticUpdate = lookupResult.get()
+    decodingResult = decodeLightClientOptimisticUpdateForked(l.forkDigests, optimimsticUpdate)
+
+  if decodingResult.isErr:
+    return Opt.none(altair.LightClientOptimisticUpdate)
+  else:
+    return Opt.some(decodingResult.get())
 
 proc new*(
     T: type LightClientNetwork,

fluffy/tests/beacon_light_client_tests/test_light_client_content.nim

@@ -26,13 +26,54 @@ suite "Test light client contentEncodings":
   test "Light client bootstrap correct":
     let
       bootstrap = SSZ.decode(bootStrapBytes, altair.LightClientBootstrap)
-      encodedForked = encodeBootstrapForked(forks.altair, bootstrap)
+      encodedForked = encodeForked(altair.LightClientBootstrap, forks.altair, bootstrap)
       decodedResult = decodeBootstrapForked(forks, encodedForked)
 
     check:
       decodedResult.isOk()
       decodedResult.get() == bootstrap
 
+  test "Light client update correct":
+    let
+      update = SSZ.decode(lightClientUpdateBytes, altair.LightClientUpdate)
+      encodedForked = encodeForked(altair.LightClientUpdate, forks.altair, update)
+      decodedResult = decodeLightClientUpdateForked(forks, encodedForked)
+
+    check:
+      decodedResult.isOk()
+      decodedResult.get() == update
+
+  test "Light client update list correct":
+    let
+      update = SSZ.decode(lightClientUpdateBytes, altair.LightClientUpdate)
+      updateList = @[update, update]
+      encodedForked = encodeLightClientUpdatesForked(forks.altair, updateList)
+      decodedForked = decodeLightClientUpdatesForked(forks, encodedForked)
+
+    check:
+      decodedForked.isOk()
+      decodedForked.get() == updateList
+
+  test "Light client finality update correct":
+    let
+      update = SSZ.decode(lightClientFinalityUpdateBytes, altair.LightClientFinalityUpdate)
+      encodedForked = encodeForked(altair.LightClientFinalityUpdate, forks.altair, update)
+      decodedResult = decodeLightClientFinalityUpdateForked(forks, encodedForked)
+
+    check:
+      decodedResult.isOk()
+      decodedResult.get() == update
+
+  test "Light client optimistic update correct":
+    let
+      update = SSZ.decode(lightClientOptimisticUpdateBytes, altair.LightClientOptimisticUpdate)
+      encodedForked = encodeForked(altair.LightClientOptimisticUpdate, forks.altair, update)
+      decodedResult = decodeLightClientOptimisticUpdateForked(forks, encodedForked)
+
+    check:
+      decodedResult.isOk()
+      decodedResult.get() == update
+
   test "Light client bootstrap failures":
     let
       bootstrap = SSZ.decode(bootStrapBytes, altair.LightClientBootstrap)
@@ -45,6 +86,3 @@ suite "Test light client contentEncodings":
       decodeBootstrapForked(forks, @[]).isErr()
       decodeBootstrapForked(forks, encodedTooEarlyFork).isErr()
       decodeBootstrapForked(forks, encodedUnknownFork).isErr()
-
-
-

fluffy/tests/beacon_light_client_tests/test_light_client_network.nim

@@ -64,3 +64,57 @@ procSuite "Light client Content Network":
 
     await lcNode1.stop()
     await lcNode2.stop()
+
+  asyncTest "Get latest optimistic and finality updates":
+    let
+      lcNode1 = newLCNode(rng, 20302)
+      lcNode2 = newLCNode(rng, 20303)
+      forks = getTestForkDigests()
+
+    check:
+      lcNode1.portalProtocol().addNode(lcNode2.localNode()) == Added
+      lcNode2.portalProtocol().addNode(lcNode1.localNode()) == Added
+
+      (await lcNode1.portalProtocol().ping(lcNode2.localNode())).isOk()
+      (await lcNode2.portalProtocol().ping(lcNode1.localNode())).isOk()
+
+    let
+      finalityUpdate = SSZ.decode(lightClientFinalityUpdateBytes, altair.LightClientFinalityUpdate)
+      optimisticUpdate = SSZ.decode(lightClientOptimisticUpdateBytes, altair.LightClientOptimisticUpdate)
+
+      finalityUpdateKey = ContentKey(
+        contentType: lightClientFinalityUpdate,
+        lightClientFinalityUpdateKey: LightClientFinalityUpdateKey()
+      )
+      finalityUdpateId = toContentId(encode(finalityUpdateKey))
+
+      optimistUpdateKey = ContentKey(
+        contentType: lightClientOptimisticUpdate,
+        lightClientOptimisticUpdateKey: LightClientOptimisticUpdateKey()
+      )
+
+      optimisticUpdateId = toContentId(encode(optimistUpdateKey))
+
+
+    # This silently assumes that peer stores only one latest update, under
+    # the contentId coresponding to latest update content key
+    lcNode2.portalProtocol().storeContent(
+      finalityUdpateId, encodeFinalityUpdateForked(forks.altair, finalityUpdate)
+    )
+
+    lcNode2.portalProtocol().storeContent(
+      optimisticUpdateId, encodeOptimisticUpdateForked(forks.altair, optimisticUpdate)
+    )
+
+    let
+      finalityResult = await lcNode1.lightClientNetwork.getLightClientFinalityUpdate()
+      optimisticResult = await lcNode1.lightClientNetwork.getLightClientOptimisticUpdate()
+
+    check:
+      finalityResult.isOk()
+      optimisticResult.isOk()
+      finalityResult.get() == finalityUpdate
+      optimisticResult.get() == optimisticUpdate
+
+    await lcNode1.stop()
+    await lcNode2.stop()