Add accumulator content keys for history network (#1123)

fluffy/network/history/history_content.nim

@@ -22,19 +22,39 @@ type
     blockHeader = 0x00
     blockBody = 0x01
     receipts = 0x02
+    epochAccumulator = 0x03
+    masterAccumulator = 0x04
 
-  ContentKeyType* = object
+  BlockKey* = object
     chainId*: uint16
     blockHash*: BlockHash
 
+  EpochAccumulatorKey* = object
+    epochHash*: Digest
+
+  MasterAccumulatorKeyType* = enum
+    latest = 0x00 # An SSZ Union None
+    masterHash = 0x01
+
+  MasterAccumulatorKey* = object
+    case accumulaterKeyType*: MasterAccumulatorKeyType
+    of latest:
+      discard
+    of masterHash:
+      masterHashKey*: Digest
+
   ContentKey* = object
     case contentType*: ContentType
     of blockHeader:
-      blockHeaderKey*: ContentKeyType
+      blockHeaderKey*: BlockKey
     of blockBody:
-      blockBodyKey*: ContentKeyType
+      blockBodyKey*: BlockKey
     of receipts:
-      receiptsKey*: ContentKeyType
+      receiptsKey*: BlockKey
+    of epochAccumulator:
+      epochAccumulatorKey*: EpochAccumulatorKey
+    of masterAccumulator:
+      masterAccumulatorKey*: MasterAccumulatorKey
 
 func encode*(contentKey: ContentKey): ByteList =
   ByteList.init(SSZ.encode(contentKey))
@@ -56,16 +76,30 @@ func toContentId*(contentKey: ContentKey): ContentId =
 func `$`*(x: BlockHash): string =
   "0x" & x.data.toHex()
 
-func `$`*(x: ContentKey): string =
-  let key =
-    case x.contentType:
-    of blockHeader:
-      x.blockHeaderKey
-    of blockBody:
-      x.blockBodyKey
-    of receipts:
-      x.receiptsKey
+func `$`*(x: BlockKey): string =
+  "blockHash: " & $x.blockHash & ", chainId: " & $x.chainId
 
-  "(contentType: " & $x.contentType &
-    ", blockHash: " & $key.blockHash &
-    ", chainId: " & $key.chainId & ")"
+func `$`*(x: ContentKey): string =
+  var res = "(type: " & $x.contentType & ", "
+
+  case x.contentType:
+  of blockHeader:
+    res.add($x.blockHeaderKey)
+  of blockBody:
+    res.add($x.blockBodyKey)
+  of receipts:
+    res.add($x.receiptsKey)
+  of epochAccumulator:
+    let key = x.epochAccumulatorKey
+    res.add("epochHash: " & $key.epochHash)
+  of masterAccumulator:
+    let key = x.masterAccumulatorKey
+    case key.accumulaterKeyType:
+    of latest:
+      res.add($key.accumulaterKeyType)
+    of masterHash:
+      res.add($key.accumulaterKeyType & ": " & $key.masterHashKey)
+
+  res.add(")")
+
+  res

fluffy/network/history/history_network.nim

@@ -44,7 +44,7 @@ func encodeKey(k: ContentKey): (ByteList, ContentId) =
 func getEncodedKeyForContent(
     cType: ContentType, chainId: uint16, hash: BlockHash):
     (ByteList, ContentId) =
-  let contentKeyType = ContentKeyType(chainId: chainId, blockHash: hash)
+  let contentKeyType = BlockKey(chainId: chainId, blockHash: hash)
 
   let contentKey =
     case cType
@@ -54,6 +54,10 @@ func getEncodedKeyForContent(
       ContentKey(contentType: cType, blockBodyKey: contentKeyType)
     of receipts:
       ContentKey(contentType: cType, receiptsKey: contentKeyType)
+    of epochAccumulator:
+      raiseAssert("Not implemented")
+    of masterAccumulator:
+      raiseAssert("Not implemented")
 
   return encodeKey(contentKey)
 
@@ -67,20 +71,20 @@ proc getContentFromBytes(bytes: openArray[byte], T: type): Result[T, string] =
 
 proc validateHeaderBytes*(
     bytes: openArray[byte], hash: BlockHash): Option[BlockHeader] =
-  
+
   let headerResult = getContentFromBytes(bytes, BlockHeader)
-  
+
   if headerResult.isErr():
     error "Failed to decode header ", msg = headerResult.error()
     return none(BlockHeader)
-  
+
   let header = headerResult.unsafeGet()
 
   if not (header.blockHash() == hash):
     # TODO: Header with different hash than expected, maybe we should punish
     # peer which sent us this ?
     return none(BlockHeader)
-  
+
   return some(header)
 
 proc validateExpectedBody(
@@ -99,7 +103,7 @@ proc validateExpectedBody(
       return ok()
   except RlpError as e:
     return err(e.msg)
-    
+
 proc validateBodyBytes*(
     bytes: openArray[byte],
     txRoot: KeccakHash,
@@ -110,20 +114,20 @@ proc validateBodyBytes*(
   if bodyResult.isErr():
     error "Failed to decode block body", msg = bodyResult.error()
     return none(BlockBody)
-  
+
   let blockBody = bodyResult.unsafeGet()
 
   let expectedResult = validateExpectedBody(blockBody, txRoot, ommersHash)
 
   if expectedResult.isErr():
-    error "Failed to validate if block body matches header", 
+    error "Failed to validate if block body matches header",
       msg = expectedResult.error()
 
     # we got block body (bundle of transactions and uncles) which do not match
     # header. For now just ignore it, but maybe we should penalize peer
     # sending us such data?
     return none(BlockBody)
-    
+
   return some(blockBody)
 
 proc getContentFromDb(
@@ -238,7 +242,7 @@ proc validateExpectedReceipts(
       return ok()
   except RlpError as e:
     return err(e.msg)
-    
+
 proc validateReceiptsBytes*(
     bytes: openArray[byte],
     receiptRoot: KeccakHash): Option[seq[Receipt]] =
@@ -248,13 +252,13 @@ proc validateReceiptsBytes*(
   if receiptResult.isErr():
     error "Failed to decode receipts", msg = receiptResult.error()
     return none(seq[Receipt])
-  
+
   let receipts = receiptResult.unsafeGet()
 
   let expectedReceiptsResult = validateExpectedReceipts(receipts, receiptRoot)
 
   if expectedReceiptsResult.isErr():
-    error "Failed to validate if receipts matches header", 
+    error "Failed to validate if receipts matches header",
       msg = expectedReceiptsResult.error()
 
     # we got receipts which do not match
@@ -327,6 +331,10 @@ proc validateContent(content: openArray[byte], contentKey: ByteList): bool =
     # to deal with this?
   of receipts:
     true
+  of epochAccumulator:
+    true
+  of masterAccumulator:
+    true
 
 proc new*(
     T: type HistoryNetwork,

fluffy/populate_db.nim

@@ -82,7 +82,7 @@ func readBlockData(
         $blockData.number & ": " & e.msg)
 
     let contentKeyType =
-      ContentKeyType(chainId: 1'u16, blockHash: blockHash)
+      BlockKey(chainId: 1'u16, blockHash: blockHash)
 
     try:
       # If wanted the hash for the corresponding header can be verified

fluffy/tests/test_history_content.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2021 Status Research & Development GmbH
+# Copyright (c) 2021-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -12,7 +12,8 @@ import
   ../network/history/history_content
 
 # According to test vectors:
-# TODO: Add link once test vectors are merged
+# https://github.com/ethereum/portal-network-specs/blob/master/content-keys-test-vectors.md#history-network-keys
+
 suite "History ContentKey Encodings":
   test "BlockHeader":
     # Input
@@ -32,7 +33,7 @@ suite "History ContentKey Encodings":
 
     let contentKey = ContentKey(
       contentType: blockHeader,
-      blockHeaderKey: ContentKeyType(chainId: 15'u16, blockHash: blockHash))
+      blockHeaderKey: BlockKey(chainId: 15'u16, blockHash: blockHash))
 
     let encoded = encode(contentKey)
     check encoded.asSeq.toHex == contentKeyHex
@@ -66,7 +67,7 @@ suite "History ContentKey Encodings":
 
     let contentKey = ContentKey(
       contentType: blockBody,
-      blockBodyKey: ContentKeyType(chainId: 20'u16, blockHash: blockHash))
+      blockBodyKey: BlockKey(chainId: 20'u16, blockHash: blockHash))
 
     let encoded = encode(contentKey)
     check encoded.asSeq.toHex == contentKeyHex
@@ -99,8 +100,7 @@ suite "History ContentKey Encodings":
 
     let contentKey = ContentKey(
       contentType: receipts,
-      receiptsKey: ContentKeyType(chainId: 4'u16, blockHash: blockHash))
-
+      receiptsKey: BlockKey(chainId: 4'u16, blockHash: blockHash))
 
     let encoded = encode(contentKey)
     check encoded.asSeq.toHex == contentKeyHex
@@ -115,3 +115,104 @@ suite "History ContentKey Encodings":
       toContentId(contentKey) == parse(contentId, Stuint[256], 10)
       # In stint this does BE hex string
       toContentId(contentKey).toHex() == contentIdHexBE
+
+  test "Epoch Accumulator":
+    var epochHash: Digest
+    epochHash.data = hexToByteArray[sizeof(Digest)](
+      "0xe242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491")
+
+    const
+      contentKeyHex =
+        "03e242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491"
+      contentId =
+        "72232402989179419196382321898161638871438419016077939952896528930608027961710"
+      # or
+      contentIdHexBE =
+        "9fb2175e76c6989e0fdac3ee10c40d2a81eb176af32e1c16193e3904fe56896e"
+
+    let contentKey = ContentKey(
+      contentType: epochAccumulator,
+      epochAccumulatorKey: EpochAccumulatorKey(epochHash: epochHash))
+
+    let encoded = encode(contentKey)
+    check encoded.asSeq.toHex == contentKeyHex
+    let decoded = decode(encoded)
+    check decoded.isSome()
+
+    let contentKeyDecoded = decoded.get()
+    check:
+      contentKeyDecoded.contentType == contentKey.contentType
+      contentKeyDecoded.epochAccumulatorKey == contentKey.epochAccumulatorKey
+
+      toContentId(contentKey) == parse(contentId, Stuint[256], 10)
+      # In stint this does BE hex string
+      toContentId(contentKey).toHex() == contentIdHexBE
+
+  test "Master Accumulator - Latest":
+    var accumulatorHash: Digest
+    accumulatorHash.data = hexToByteArray[sizeof(Digest)](
+      "0x88cce8439ebc0c1d007177ffb6831c15c07b4361984cc52235b6fd728434f0c7")
+
+    const
+      contentKeyHex =
+        "0400"
+      contentId =
+        "87173654316145541646904042090629917349369185510102051783618763191692466404071"
+      # or
+      contentIdHexBE =
+        "c0ba8a33ac67f44abff5984dfbb6f56c46b880ac2b86e1f23e7fa9c402c53ae7"
+
+    let contentKey = ContentKey(
+      contentType: masterAccumulator,
+      masterAccumulatorKey: MasterAccumulatorKey(accumulaterKeyType: latest))
+
+    let encoded = encode(contentKey)
+    check encoded.asSeq.toHex == contentKeyHex
+    let decoded = decode(encoded)
+    check decoded.isSome()
+
+    let contentKeyDecoded = decoded.get()
+    check:
+      contentKeyDecoded.contentType == contentKey.contentType
+      contentKeyDecoded.masterAccumulatorKey.accumulaterKeyType ==
+        contentKey.masterAccumulatorKey.accumulaterKeyType
+
+      toContentId(contentKey) == parse(contentId, Stuint[256], 10)
+      # In stint this does BE hex string
+      toContentId(contentKey).toHex() == contentIdHexBE
+
+  test "Master Accumulator - Hash":
+    var accumulatorHash: Digest
+    accumulatorHash.data = hexToByteArray[sizeof(Digest)](
+      "0x88cce8439ebc0c1d007177ffb6831c15c07b4361984cc52235b6fd728434f0c7")
+
+    const
+      contentKeyHex =
+        "040188cce8439ebc0c1d007177ffb6831c15c07b4361984cc52235b6fd728434f0c7"
+      contentId =
+        "79362820890138237094338894474079140563693945795365426184460738681339857347750"
+      # or
+      contentIdHexBE =
+        "af75c3c9d0e89a5083361a3334a9c5583955f0dbe9a413eb79ba26400d1824a6"
+
+    let contentKey = ContentKey(
+      contentType: masterAccumulator,
+      masterAccumulatorKey: MasterAccumulatorKey(
+        accumulaterKeyType: masterHash, masterHashKey: accumulatorHash))
+
+    let encoded = encode(contentKey)
+    check encoded.asSeq.toHex == contentKeyHex
+    let decoded = decode(encoded)
+    check decoded.isSome()
+
+    let contentKeyDecoded = decoded.get()
+    check:
+      contentKeyDecoded.contentType == contentKey.contentType
+      contentKeyDecoded.masterAccumulatorKey.accumulaterKeyType ==
+        contentKey.masterAccumulatorKey.accumulaterKeyType
+      contentKeyDecoded.masterAccumulatorKey.masterHashKey ==
+        contentKey.masterAccumulatorKey.masterHashKey
+
+      toContentId(contentKey) == parse(contentId, Stuint[256], 10)
+      # In stint this does BE hex string
+      toContentId(contentKey).toHex() == contentIdHexBE