Optionally write all epoch accumulators when building accumulator
- Can write epoch accumulators to files now with eth_data_exporter - RPC requests to gossip epoch accumulators now uses these files instead of building on the fly - Other build accumulator calls are adjusted and only used for tests and thus moved to testing folder
This commit is contained in:
parent
09766ef283
commit
e6d8bb4f2f
|
@ -206,14 +206,8 @@ proc readAccumulator*(file: string): Result[FinishedAccumulator, string] =
|
||||||
err("Failed decoding accumulator: " & e.msg)
|
err("Failed decoding accumulator: " & e.msg)
|
||||||
|
|
||||||
|
|
||||||
proc readEpochAccumulator*(dataFile: string): Result[EpochAccumulator, string] =
|
proc readEpochAccumulator*(file: string): Result[EpochAccumulator, string] =
|
||||||
let res = ? readJsonType(dataFile, EpochAccumulatorObject)
|
let encodedAccumulator = ? readAllFile(file).mapErr(toString)
|
||||||
|
|
||||||
let encodedAccumulator =
|
|
||||||
try:
|
|
||||||
res.epochAccumulator.hexToSeqByte()
|
|
||||||
except ValueError as e:
|
|
||||||
return err("Invalid hex data for accumulator: " & e.msg)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
ok(SSZ.decode(encodedAccumulator, EpochAccumulator))
|
ok(SSZ.decode(encodedAccumulator, EpochAccumulator))
|
||||||
|
|
|
@ -8,6 +8,7 @@
|
||||||
{.push raises: [Defect].}
|
{.push raises: [Defect].}
|
||||||
|
|
||||||
import
|
import
|
||||||
|
std/[strformat, os],
|
||||||
stew/results, chronos, chronicles,
|
stew/results, chronos, chronicles,
|
||||||
eth/common/eth_types,
|
eth/common/eth_types,
|
||||||
../network/wire/portal_protocol,
|
../network/wire/portal_protocol,
|
||||||
|
@ -18,38 +19,6 @@ export results
|
||||||
|
|
||||||
### Helper calls to seed the local database and/or the network
|
### Helper calls to seed the local database and/or the network
|
||||||
|
|
||||||
proc buildAccumulator*(dataFile: string): Result[FinishedAccumulator, string] =
|
|
||||||
let blockData = ? readJsonType(dataFile, BlockDataTable)
|
|
||||||
|
|
||||||
var headers: seq[BlockHeader]
|
|
||||||
# Len of headers from blockdata + genesis header
|
|
||||||
headers.setLen(blockData.len() + 1)
|
|
||||||
|
|
||||||
headers[0] = getGenesisHeader()
|
|
||||||
|
|
||||||
for k, v in blockData.pairs:
|
|
||||||
let header = ? v.readBlockHeader()
|
|
||||||
headers[header.blockNumber.truncate(int)] = header
|
|
||||||
|
|
||||||
buildAccumulator(headers)
|
|
||||||
|
|
||||||
proc buildAccumulatorData*(
|
|
||||||
dataFile: string):
|
|
||||||
Result[seq[(ContentKey, EpochAccumulator)], string] =
|
|
||||||
let blockData = ? readJsonType(dataFile, BlockDataTable)
|
|
||||||
|
|
||||||
var headers: seq[BlockHeader]
|
|
||||||
# Len of headers from blockdata + genesis header
|
|
||||||
headers.setLen(blockData.len() + 1)
|
|
||||||
|
|
||||||
headers[0] = getGenesisHeader()
|
|
||||||
|
|
||||||
for k, v in blockData.pairs:
|
|
||||||
let header = ? v.readBlockHeader()
|
|
||||||
headers[header.blockNumber.truncate(int)] = header
|
|
||||||
|
|
||||||
ok(buildAccumulatorData(headers))
|
|
||||||
|
|
||||||
proc historyStore*(
|
proc historyStore*(
|
||||||
p: PortalProtocol, dataFile: string, verify = false):
|
p: PortalProtocol, dataFile: string, verify = false):
|
||||||
Result[void, string] =
|
Result[void, string] =
|
||||||
|
@ -62,32 +31,12 @@ proc historyStore*(
|
||||||
|
|
||||||
ok()
|
ok()
|
||||||
|
|
||||||
proc propagateAccumulatorData*(
|
|
||||||
p: PortalProtocol, dataFile: string):
|
|
||||||
Future[Result[void, string]] {.async.} =
|
|
||||||
## Propagate all epoch accumulators created when building the accumulator
|
|
||||||
## from the block headers.
|
|
||||||
## dataFile holds block data
|
|
||||||
let epochAccumulators = buildAccumulatorData(dataFile)
|
|
||||||
if epochAccumulators.isErr():
|
|
||||||
return err(epochAccumulators.error)
|
|
||||||
else:
|
|
||||||
for (key, epochAccumulator) in epochAccumulators.get():
|
|
||||||
let content = SSZ.encode(epochAccumulator)
|
|
||||||
|
|
||||||
p.storeContent(
|
|
||||||
history_content.toContentId(key), content)
|
|
||||||
discard await p.neighborhoodGossip(
|
|
||||||
ContentKeysList(@[encode(key)]), @[content])
|
|
||||||
|
|
||||||
return ok()
|
|
||||||
|
|
||||||
proc propagateEpochAccumulator*(
|
proc propagateEpochAccumulator*(
|
||||||
p: PortalProtocol, dataFile: string):
|
p: PortalProtocol, file: string):
|
||||||
Future[Result[void, string]] {.async.} =
|
Future[Result[void, string]] {.async.} =
|
||||||
## Propagate a specific epoch accumulator into the network.
|
## Propagate a specific epoch accumulator into the network.
|
||||||
## dataFile holds the SSZ serialized epoch accumulator
|
## file holds the SSZ serialized epoch accumulator.
|
||||||
let epochAccumulatorRes = readEpochAccumulator(dataFile)
|
let epochAccumulatorRes = readEpochAccumulator(file)
|
||||||
if epochAccumulatorRes.isErr():
|
if epochAccumulatorRes.isErr():
|
||||||
return err(epochAccumulatorRes.error)
|
return err(epochAccumulatorRes.error)
|
||||||
else:
|
else:
|
||||||
|
@ -99,10 +48,32 @@ proc propagateEpochAccumulator*(
|
||||||
epochAccumulatorKey: EpochAccumulatorKey(
|
epochAccumulatorKey: EpochAccumulatorKey(
|
||||||
epochHash: rootHash))
|
epochHash: rootHash))
|
||||||
|
|
||||||
|
# Note: The file actually holds the SSZ encoded accumulator, but we need
|
||||||
|
# to decode as we need the root for the content key.
|
||||||
|
encodedAccumulator = SSZ.encode(accumulator)
|
||||||
|
info "Gossiping epoch accumulator", rootHash
|
||||||
|
|
||||||
p.storeContent(
|
p.storeContent(
|
||||||
history_content.toContentId(key), SSZ.encode(accumulator))
|
history_content.toContentId(key), encodedAccumulator)
|
||||||
discard await p.neighborhoodGossip(
|
discard await p.neighborhoodGossip(
|
||||||
ContentKeysList(@[encode(key)]), @[SSZ.encode(accumulator)])
|
ContentKeysList(@[encode(key)]), @[encodedAccumulator])
|
||||||
|
|
||||||
|
return ok()
|
||||||
|
|
||||||
|
proc propagateEpochAccumulators*(
|
||||||
|
p: PortalProtocol, path: string):
|
||||||
|
Future[Result[void, string]] {.async.} =
|
||||||
|
## Propagate all epoch accumulators created when building the accumulator
|
||||||
|
## from the block headers.
|
||||||
|
## path is a directory that holds all SSZ encoded epoch accumulator files.
|
||||||
|
for i in 0..<preMergeEpochs:
|
||||||
|
let file =
|
||||||
|
try: path / &"mainnet-epoch-accumulator-{i.uint64:05}.ssz"
|
||||||
|
except ValueError as e: raiseAssert e.msg
|
||||||
|
|
||||||
|
let res = await p.propagateEpochAccumulator(file)
|
||||||
|
if res.isErr():
|
||||||
|
return err(res.error)
|
||||||
|
|
||||||
return ok()
|
return ok()
|
||||||
|
|
||||||
|
|
|
@ -102,51 +102,6 @@ func finishAccumulator*(a: var Accumulator): FinishedAccumulator =
|
||||||
|
|
||||||
FinishedAccumulator(historicalEpochs: a.historicalEpochs)
|
FinishedAccumulator(historicalEpochs: a.historicalEpochs)
|
||||||
|
|
||||||
func buildAccumulator*(
|
|
||||||
headers: seq[BlockHeader]): Result[FinishedAccumulator, string] =
|
|
||||||
var accumulator: Accumulator
|
|
||||||
for header in headers:
|
|
||||||
updateAccumulator(accumulator, header)
|
|
||||||
|
|
||||||
if header.blockNumber.truncate(uint64) == mergeBlockNumber - 1:
|
|
||||||
return ok(finishAccumulator(accumulator))
|
|
||||||
|
|
||||||
err("Not enough headers provided to finish the accumulator")
|
|
||||||
|
|
||||||
func buildAccumulatorData*(headers: seq[BlockHeader]):
|
|
||||||
seq[(ContentKey, EpochAccumulator)] =
|
|
||||||
var accumulator: Accumulator
|
|
||||||
var epochAccumulators: seq[(ContentKey, EpochAccumulator)]
|
|
||||||
for header in headers:
|
|
||||||
updateAccumulator(accumulator, header)
|
|
||||||
|
|
||||||
# TODO: By allowing updateAccumulator and finishAccumulator to return
|
|
||||||
# optionally the finished epoch accumulators we would avoid double
|
|
||||||
# hash_tree_root computations.
|
|
||||||
if accumulator.currentEpoch.len() == epochSize:
|
|
||||||
let
|
|
||||||
rootHash = accumulator.currentEpoch.hash_tree_root()
|
|
||||||
key = ContentKey(
|
|
||||||
contentType: epochAccumulator,
|
|
||||||
epochAccumulatorKey: EpochAccumulatorKey(
|
|
||||||
epochHash: rootHash))
|
|
||||||
|
|
||||||
epochAccumulators.add((key, accumulator.currentEpoch))
|
|
||||||
|
|
||||||
if header.blockNumber.truncate(uint64) == mergeBlockNumber - 1:
|
|
||||||
let
|
|
||||||
rootHash = accumulator.currentEpoch.hash_tree_root()
|
|
||||||
key = ContentKey(
|
|
||||||
contentType: epochAccumulator,
|
|
||||||
epochAccumulatorKey: EpochAccumulatorKey(
|
|
||||||
epochHash: rootHash))
|
|
||||||
|
|
||||||
epochAccumulators.add((key, accumulator.currentEpoch))
|
|
||||||
|
|
||||||
discard finishAccumulator(accumulator)
|
|
||||||
|
|
||||||
epochAccumulators
|
|
||||||
|
|
||||||
## Calls and helper calls for building header proofs and verifying headers
|
## Calls and helper calls for building header proofs and verifying headers
|
||||||
## against the Accumulator and the header proofs.
|
## against the Accumulator and the header proofs.
|
||||||
|
|
||||||
|
|
|
@ -4,10 +4,8 @@ proc portal_history_storeContent(dataFile: string): bool
|
||||||
proc portal_history_propagate(dataFile: string): bool
|
proc portal_history_propagate(dataFile: string): bool
|
||||||
proc portal_history_propagateHeaders(dataFile: string): bool
|
proc portal_history_propagateHeaders(dataFile: string): bool
|
||||||
proc portal_history_propagateBlock(dataFile: string, blockHash: string): bool
|
proc portal_history_propagateBlock(dataFile: string, blockHash: string): bool
|
||||||
proc portal_history_propagateAccumulatorData(
|
proc portal_history_propagateEpochAccumulator(dataFile: string): bool
|
||||||
dataFile: string): bool
|
proc portal_history_propagateEpochAccumulators(path: string): bool
|
||||||
proc portal_history_propagateEpochAccumulator(
|
|
||||||
dataFile: string): bool
|
|
||||||
proc portal_history_storeContentInNodeRange(
|
proc portal_history_storeContentInNodeRange(
|
||||||
dbPath: string, max: uint32, starting: uint32): bool
|
dbPath: string, max: uint32, starting: uint32): bool
|
||||||
proc portal_history_offerContentInNodeRange(
|
proc portal_history_offerContentInNodeRange(
|
||||||
|
@ -16,4 +14,3 @@ proc portal_history_depthContentPropagate(
|
||||||
dbPath: string, max: uint32): bool
|
dbPath: string, max: uint32): bool
|
||||||
proc portal_history_breadthContentPropagate(
|
proc portal_history_breadthContentPropagate(
|
||||||
dbPath: string): bool
|
dbPath: string): bool
|
||||||
|
|
||||||
|
|
|
@ -64,21 +64,20 @@ proc installPortalDebugApiHandlers*(
|
||||||
else:
|
else:
|
||||||
raise newException(ValueError, $res.error)
|
raise newException(ValueError, $res.error)
|
||||||
|
|
||||||
rpcServer.rpc("portal_" & network & "_propagateAccumulatorData") do(
|
|
||||||
dataFile: string) -> bool:
|
|
||||||
let res = await p.propagateAccumulatorData(dataFile)
|
|
||||||
if res.isOk():
|
|
||||||
return true
|
|
||||||
else:
|
|
||||||
raise newException(ValueError, $res.error)
|
|
||||||
|
|
||||||
rpcServer.rpc("portal_" & network & "_propagateEpochAccumulator") do(
|
rpcServer.rpc("portal_" & network & "_propagateEpochAccumulator") do(
|
||||||
dataFile: string) -> bool:
|
dataFile: string) -> bool:
|
||||||
let res = await p.propagateEpochAccumulator(dataFile)
|
let res = await p.propagateEpochAccumulator(dataFile)
|
||||||
if res.isOk():
|
if res.isOk():
|
||||||
return true
|
return true
|
||||||
else:
|
else:
|
||||||
echo $res.error
|
raise newException(ValueError, $res.error)
|
||||||
|
|
||||||
|
rpcServer.rpc("portal_" & network & "_propagateEpochAccumulators") do(
|
||||||
|
path: string) -> bool:
|
||||||
|
let res = await p.propagateEpochAccumulators(path)
|
||||||
|
if res.isOk():
|
||||||
|
return true
|
||||||
|
else:
|
||||||
raise newException(ValueError, $res.error)
|
raise newException(ValueError, $res.error)
|
||||||
|
|
||||||
rpcServer.rpc("portal_" & network & "_storeContentInNodeRange") do(
|
rpcServer.rpc("portal_" & network & "_storeContentInNodeRange") do(
|
||||||
|
|
|
@ -13,15 +13,15 @@ import
|
||||||
unittest2, stint,
|
unittest2, stint,
|
||||||
eth/common/eth_types_rlp,
|
eth/common/eth_types_rlp,
|
||||||
../data/history_data_parser,
|
../data/history_data_parser,
|
||||||
../network/history/[history_content, accumulator]
|
../network/history/[history_content, accumulator],
|
||||||
|
./test_helpers
|
||||||
|
|
||||||
func buildProof(
|
func buildProof(
|
||||||
epochAccumulators: seq[(ContentKey, EpochAccumulator)],
|
epochAccumulators: seq[EpochAccumulator], header: BlockHeader):
|
||||||
header: BlockHeader):
|
|
||||||
Result[seq[Digest], string] =
|
Result[seq[Digest], string] =
|
||||||
let
|
let
|
||||||
epochIndex = getEpochIndex(header)
|
epochIndex = getEpochIndex(header)
|
||||||
epochAccumulator = epochAccumulators[epochIndex][1]
|
epochAccumulator = epochAccumulators[epochIndex]
|
||||||
|
|
||||||
headerRecordIndex = getHeaderRecordIndex(header, epochIndex)
|
headerRecordIndex = getHeaderRecordIndex(header, epochIndex)
|
||||||
gIndex = GeneralizedIndex(epochSize*2*2 + (headerRecordIndex*2))
|
gIndex = GeneralizedIndex(epochSize*2*2 + (headerRecordIndex*2))
|
||||||
|
@ -54,13 +54,9 @@ suite "Header Accumulator":
|
||||||
headers.add(BlockHeader(
|
headers.add(BlockHeader(
|
||||||
blockNumber: i.stuint(256), difficulty: 1.stuint(256)))
|
blockNumber: i.stuint(256), difficulty: 1.stuint(256)))
|
||||||
|
|
||||||
let
|
let accumulatorRes = buildAccumulatorData(headers)
|
||||||
accumulatorRes = buildAccumulator(headers)
|
|
||||||
epochAccumulators = buildAccumulatorData(headers)
|
|
||||||
|
|
||||||
check accumulatorRes.isOk()
|
check accumulatorRes.isOk()
|
||||||
|
let (accumulator, epochAccumulators) = accumulatorRes.get()
|
||||||
let accumulator = accumulatorRes.get()
|
|
||||||
|
|
||||||
block: # Test valid headers
|
block: # Test valid headers
|
||||||
for i in headersToTest:
|
for i in headersToTest:
|
||||||
|
|
|
@ -9,7 +9,8 @@ import
|
||||||
stew/shims/net,
|
stew/shims/net,
|
||||||
eth/keys,
|
eth/keys,
|
||||||
eth/p2p/discoveryv5/[enr, node, routing_table],
|
eth/p2p/discoveryv5/[enr, node, routing_table],
|
||||||
eth/p2p/discoveryv5/protocol as discv5_protocol
|
eth/p2p/discoveryv5/protocol as discv5_protocol,
|
||||||
|
../network/history/accumulator
|
||||||
|
|
||||||
proc localAddress*(port: int): Address =
|
proc localAddress*(port: int): Address =
|
||||||
Address(ip: ValidIpAddress.init("127.0.0.1"), port: Port(port))
|
Address(ip: ValidIpAddress.init("127.0.0.1"), port: Port(port))
|
||||||
|
@ -43,3 +44,31 @@ proc genByteSeq*(length: int): seq[byte] =
|
||||||
resultSeq[i] = byte(i)
|
resultSeq[i] = byte(i)
|
||||||
inc i
|
inc i
|
||||||
return resultSeq
|
return resultSeq
|
||||||
|
|
||||||
|
func buildAccumulator*(
|
||||||
|
headers: seq[BlockHeader]): Result[FinishedAccumulator, string] =
|
||||||
|
var accumulator: Accumulator
|
||||||
|
for header in headers:
|
||||||
|
updateAccumulator(accumulator, header)
|
||||||
|
|
||||||
|
if header.blockNumber.truncate(uint64) == mergeBlockNumber - 1:
|
||||||
|
return ok(finishAccumulator(accumulator))
|
||||||
|
|
||||||
|
err("Not enough headers provided to finish the accumulator")
|
||||||
|
|
||||||
|
func buildAccumulatorData*(headers: seq[BlockHeader]):
|
||||||
|
Result[(FinishedAccumulator, seq[EpochAccumulator]), string] =
|
||||||
|
var accumulator: Accumulator
|
||||||
|
var epochAccumulators: seq[EpochAccumulator]
|
||||||
|
for header in headers:
|
||||||
|
updateAccumulator(accumulator, header)
|
||||||
|
|
||||||
|
if accumulator.currentEpoch.len() == epochSize:
|
||||||
|
epochAccumulators.add(accumulator.currentEpoch)
|
||||||
|
|
||||||
|
if header.blockNumber.truncate(uint64) == mergeBlockNumber - 1:
|
||||||
|
epochAccumulators.add(accumulator.currentEpoch)
|
||||||
|
|
||||||
|
return ok((finishAccumulator(accumulator), epochAccumulators))
|
||||||
|
|
||||||
|
err("Not enough headers provided to finish the accumulator")
|
||||||
|
|
|
@ -8,7 +8,8 @@
|
||||||
import
|
import
|
||||||
std/os,
|
std/os,
|
||||||
testutils/unittests, chronos,
|
testutils/unittests, chronos,
|
||||||
eth/p2p/discoveryv5/protocol as discv5_protocol, eth/p2p/discoveryv5/routing_table,
|
eth/p2p/discoveryv5/protocol as discv5_protocol,
|
||||||
|
eth/p2p/discoveryv5/routing_table,
|
||||||
eth/common/eth_types_rlp,
|
eth/common/eth_types_rlp,
|
||||||
eth/rlp,
|
eth/rlp,
|
||||||
../network/wire/[portal_protocol, portal_stream, portal_protocol_config],
|
../network/wire/[portal_protocol, portal_stream, portal_protocol_config],
|
||||||
|
@ -89,10 +90,12 @@ procSuite "History Content Network":
|
||||||
epochSize*3 + 1,
|
epochSize*3 + 1,
|
||||||
int(lastBlockNumber)]
|
int(lastBlockNumber)]
|
||||||
|
|
||||||
|
let headers = createEmptyHeaders(0, int(lastBlockNumber))
|
||||||
|
let accumulatorRes = buildAccumulatorData(headers)
|
||||||
|
check accumulatorRes.isOk()
|
||||||
|
|
||||||
let
|
let
|
||||||
headers = createEmptyHeaders(0, int(lastBlockNumber))
|
(masterAccumulator, epochAccumulators) = accumulatorRes.get()
|
||||||
masterAccumulator = buildAccumulator(headers).get()
|
|
||||||
epochAccumulators = buildAccumulatorData(headers)
|
|
||||||
historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
|
historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
|
||||||
historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
|
historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
|
||||||
|
|
||||||
|
@ -107,8 +110,15 @@ procSuite "History Content Network":
|
||||||
headerEncoded = rlp.encode(h)
|
headerEncoded = rlp.encode(h)
|
||||||
historyNode2.portalProtocol().storeContent(contentId, headerEncoded)
|
historyNode2.portalProtocol().storeContent(contentId, headerEncoded)
|
||||||
|
|
||||||
for (contentKey, epochAccumulator) in epochAccumulators:
|
# Need to store the epoch accumulators to be able to do the block to hash
|
||||||
let contentId = toContentId(contentKey)
|
# mapping
|
||||||
|
for epochAccumulator in epochAccumulators:
|
||||||
|
let
|
||||||
|
rootHash = epochAccumulator.hash_tree_root()
|
||||||
|
contentKey = ContentKey(
|
||||||
|
contentType: ContentType.epochAccumulator,
|
||||||
|
epochAccumulatorKey: EpochAccumulatorKey(epochHash: rootHash))
|
||||||
|
contentId = toContentId(contentKey)
|
||||||
historyNode2.portalProtocol().storeContent(
|
historyNode2.portalProtocol().storeContent(
|
||||||
contentId, SSZ.encode(epochAccumulator))
|
contentId, SSZ.encode(epochAccumulator))
|
||||||
|
|
||||||
|
@ -139,11 +149,12 @@ procSuite "History Content Network":
|
||||||
# Need to provide enough headers to have the accumulator "finished".
|
# Need to provide enough headers to have the accumulator "finished".
|
||||||
const lastBlockNumber = int(mergeBlockNumber - 1)
|
const lastBlockNumber = int(mergeBlockNumber - 1)
|
||||||
|
|
||||||
let
|
let headers = createEmptyHeaders(0, lastBlockNumber)
|
||||||
headers = createEmptyHeaders(0, lastBlockNumber)
|
let accumulatorRes = buildAccumulatorData(headers)
|
||||||
masterAccumulator = buildAccumulator(headers).get()
|
check accumulatorRes.isOk()
|
||||||
epochAccumulators = buildAccumulatorData(headers)
|
|
||||||
|
|
||||||
|
let
|
||||||
|
(masterAccumulator, epochAccumulators) = accumulatorRes.get()
|
||||||
historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
|
historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
|
||||||
historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
|
historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
|
||||||
|
|
||||||
|
@ -163,8 +174,13 @@ procSuite "History Content Network":
|
||||||
|
|
||||||
# One of the nodes needs to have the epochAccumulator to build proofs from
|
# One of the nodes needs to have the epochAccumulator to build proofs from
|
||||||
# for the offered headers.
|
# for the offered headers.
|
||||||
for (contentKey, epochAccumulator) in epochAccumulators:
|
for epochAccumulator in epochAccumulators:
|
||||||
let contentId = toContentId(contentKey)
|
let
|
||||||
|
rootHash = epochAccumulator.hash_tree_root()
|
||||||
|
contentKey = ContentKey(
|
||||||
|
contentType: ContentType.epochAccumulator,
|
||||||
|
epochAccumulatorKey: EpochAccumulatorKey(epochHash: rootHash))
|
||||||
|
contentId = toContentId(contentKey)
|
||||||
historyNode2.portalProtocol().storeContent(
|
historyNode2.portalProtocol().storeContent(
|
||||||
contentId, SSZ.encode(epochAccumulator))
|
contentId, SSZ.encode(epochAccumulator))
|
||||||
|
|
||||||
|
@ -222,11 +238,12 @@ procSuite "History Content Network":
|
||||||
lastBlockNumber - 1,
|
lastBlockNumber - 1,
|
||||||
lastBlockNumber]
|
lastBlockNumber]
|
||||||
|
|
||||||
let
|
let headers = createEmptyHeaders(0, lastBlockNumber)
|
||||||
headers = createEmptyHeaders(0, lastBlockNumber)
|
let accumulatorRes = buildAccumulatorData(headers)
|
||||||
masterAccumulator = buildAccumulator(headers).get()
|
check accumulatorRes.isOk()
|
||||||
epochAccumulators = buildAccumulatorData(headers)
|
|
||||||
|
|
||||||
|
let
|
||||||
|
(masterAccumulator, epochAccumulators) = accumulatorRes.get()
|
||||||
historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
|
historyNode1 = newHistoryNode(rng, 20302, masterAccumulator)
|
||||||
historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
|
historyNode2 = newHistoryNode(rng, 20303, masterAccumulator)
|
||||||
|
|
||||||
|
@ -239,8 +256,13 @@ procSuite "History Content Network":
|
||||||
|
|
||||||
# Need to store the epochAccumulators, because else the headers can't be
|
# Need to store the epochAccumulators, because else the headers can't be
|
||||||
# verified if being part of the canonical chain currently
|
# verified if being part of the canonical chain currently
|
||||||
for (contentKey, epochAccumulator) in epochAccumulators:
|
for epochAccumulator in epochAccumulators:
|
||||||
let contentId = toContentId(contentKey)
|
let
|
||||||
|
rootHash = epochAccumulator.hash_tree_root()
|
||||||
|
contentKey = ContentKey(
|
||||||
|
contentType: ContentType.epochAccumulator,
|
||||||
|
epochAccumulatorKey: EpochAccumulatorKey(epochHash: rootHash))
|
||||||
|
contentId = toContentId(contentKey)
|
||||||
historyNode1.portalProtocol.storeContent(
|
historyNode1.portalProtocol.storeContent(
|
||||||
contentId, SSZ.encode(epochAccumulator))
|
contentId, SSZ.encode(epochAccumulator))
|
||||||
|
|
||||||
|
|
|
@ -162,6 +162,10 @@ type
|
||||||
defaultValue: defaultAccumulatorFileName
|
defaultValue: defaultAccumulatorFileName
|
||||||
defaultValueDesc: $defaultAccumulatorFileName
|
defaultValueDesc: $defaultAccumulatorFileName
|
||||||
name: "accumulator-file-name" .}: string
|
name: "accumulator-file-name" .}: string
|
||||||
|
writeEpochAccumulators* {.
|
||||||
|
desc: "Write also the SSZ encoded epoch accumulators to specific files"
|
||||||
|
defaultValue: false
|
||||||
|
name: "write-epoch-accumulators" .}: bool
|
||||||
of printAccumulatorData:
|
of printAccumulatorData:
|
||||||
accumulatorFileNamePrint* {.
|
accumulatorFileNamePrint* {.
|
||||||
desc: "File from which the serialized accumulator is read"
|
desc: "File from which the serialized accumulator is read"
|
||||||
|
@ -492,7 +496,7 @@ when isMainModule:
|
||||||
fatal "Required epoch headers file does not exist", file
|
fatal "Required epoch headers file does not exist", file
|
||||||
quit 1
|
quit 1
|
||||||
|
|
||||||
proc buildAccumulator(dataDir: string):
|
proc buildAccumulator(dataDir: string, writeEpochAccumulators = false):
|
||||||
Result[FinishedAccumulator, string] =
|
Result[FinishedAccumulator, string] =
|
||||||
var accumulator: Accumulator
|
var accumulator: Accumulator
|
||||||
for i in 0..<preMergeEpochs:
|
for i in 0..<preMergeEpochs:
|
||||||
|
@ -525,6 +529,22 @@ when isMainModule:
|
||||||
|
|
||||||
updateAccumulator(accumulator, blockHeader)
|
updateAccumulator(accumulator, blockHeader)
|
||||||
|
|
||||||
|
# Note: writing away of epoch accumulators occurs 1 iteration before
|
||||||
|
# updating the epoch accumulator, as the latter happens when passed
|
||||||
|
# a header for the next epoch (or on finishing the epoch).
|
||||||
|
if writeEpochAccumulators:
|
||||||
|
if accumulator.currentEpoch.len() == epochSize or
|
||||||
|
blockHeader.blockNumber.truncate(uint64) == mergeBlockNumber - 1:
|
||||||
|
let file =
|
||||||
|
try: dataDir / &"mainnet-epoch-accumulator-{i.uint64:05}.ssz"
|
||||||
|
except ValueError as e: raiseAssert e.msg
|
||||||
|
let res = io2.writeFile(file, SSZ.encode(accumulator.currentEpoch))
|
||||||
|
if res.isErr():
|
||||||
|
error "Failed writing epoch accumulator to file",
|
||||||
|
file, error = res.error
|
||||||
|
else:
|
||||||
|
notice "Succesfully wrote epoch accumulator to file", file
|
||||||
|
|
||||||
if count == epochSize - 1:
|
if count == epochSize - 1:
|
||||||
info "Updated an epoch", epoch = i
|
info "Updated an epoch", epoch = i
|
||||||
count.inc()
|
count.inc()
|
||||||
|
@ -539,7 +559,7 @@ when isMainModule:
|
||||||
|
|
||||||
err("Not enough headers provided to finish the accumulator")
|
err("Not enough headers provided to finish the accumulator")
|
||||||
|
|
||||||
let accumulatorRes = buildAccumulator(dataDir)
|
let accumulatorRes = buildAccumulator(dataDir, config.writeEpochAccumulators)
|
||||||
if accumulatorRes.isErr():
|
if accumulatorRes.isErr():
|
||||||
fatal "Could not build accumulator", error = accumulatorRes.error
|
fatal "Could not build accumulator", error = accumulatorRes.error
|
||||||
quit 1
|
quit 1
|
||||||
|
|
Loading…
Reference in New Issue