status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add exporting of headersWithProof and configurable web3 url (#1539)

This commit is contained in:
Kim De Mey 2023-04-11 14:06:45 +02:00 committed by GitHub
parent 9facab91cb
commit 034d0e6305
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 175 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
fluffy/network/history/accumulator.nim
View File

@ -122,7 +122,7 @@ func getEpochIndex*(header: BlockHeader): uint64 =
## Get the index for the historical epochs
getEpochIndex(blockNumber)
func getHeaderRecordIndex(blockNumber: uint64, epochIndex: uint64): uint64 =
func getHeaderRecordIndex*(blockNumber: uint64, epochIndex: uint64): uint64 =
## Get the relative header index for the epoch accumulator
uint64(blockNumber - epochIndex * epochSize)

186
fluffy/tools/eth_data_exporter.nim
View File

@ -37,7 +37,7 @@
{.push raises: [].}
import
std/[json, typetraits, strutils, strformat, os],
std/[json, typetraits, strutils, strformat, os, uri],
confutils,
stew/[byteutils, io2],
json_serialization,
@ -64,10 +64,19 @@ proc defaultDataDir*(): string =
getHomeDir() / dataDir
type
Web3UrlKind* = enum
HttpUrl, WsUrl
Web3Url* = object
kind*: Web3UrlKind
url*: string
const
defaultDataDirDesc = defaultDataDir()
defaultBlockFileName = "eth-block-data"
defaultAccumulatorFileName = "mainnet-master-accumulator.ssz"
defaultWeb3Url = Web3Url(kind: HttpUrl, url: "http://127.0.0.1:8545")
type
ExporterCmd* = enum
@ -85,6 +94,8 @@ type
"Print the root hash of the master accumulator and of all historical epoch accumulators. Requires data generated by exportAccumulatorData command"
exportHeaderRange =
"Export block headers from an Ethereum JSON RPC Execution endpoint to *.e2s files (unlimited amount)"
exportHeadersWithProof =
"Export block headers with proof from *.e2s headers file and epochAccumulator files"
StorageMode* = enum
Json, Db
@ -100,6 +111,10 @@ type
defaultValue: defaultDataDir()
defaultValueDesc: $defaultDataDirDesc
name: "data-dir" .}: OutDir
web3Url* {.
desc: "Execution layer JSON-RPC API URL"
defaultValue: defaultWeb3Url
name: "web3-url" .}: Web3Url
case cmd* {.
command
defaultValue: exportBlockData .}: ExporterCmd
@ -169,6 +184,32 @@ type
endBlockNumber* {.
desc: "Number of the last block header to be exported"
name: "end-block" .}: uint64
of exportHeadersWithProof:
startBlockNumber2* {.
desc: "Number of the first block header to be exported"
name: "start-block" .}: uint64
endBlockNumber2* {.
desc: "Number of the last block header to be exported"
name: "end-block" .}: uint64
proc parseCmdArg*(
T: type Web3Url, p: string): T {.raises: [ConfigurationError].} =
let
url = parseUri(p)
normalizedScheme = url.scheme.toLowerAscii()
if (normalizedScheme == "http" or normalizedScheme == "https"):
Web3Url(kind: HttpUrl, url: p)
elif (normalizedScheme == "ws" or normalizedScheme == "wss"):
Web3Url(kind: WsUrl, url: p)
else:
raise newException(
ConfigurationError,
"The Web3 URL must specify one of following protocols: http/https/ws/wss"
)
proc completeCmdArg*(T: type Web3Url, val: string): seq[string] =
return @[]
proc parseCmdArg*(T: type StorageMode, p: string): T
{.raises: [ConfigurationError].} =
@ -319,6 +360,34 @@ proc exportBlocks(config: ExporterConf, client: RpcClient) =
else:
writeBlocksToDb(config, client)
proc newRpcClient(web3Url: Web3Url): RpcClient =
# TODO: I don't like this API. I think the creation of the RPC clients should
# already include the URL. And then an optional connect may be necessary
# depending on the protocol.
let client: RpcClient =
case web3Url.kind
of HttpUrl:
newRpcHttpClient()
of WsUrl:
newRpcWebSocketClient()
client
proc connectRpcClient(
client: RpcClient, web3Url: Web3Url):
Future[Result[void, string]] {.async.} =
case web3Url.kind
of HttpUrl:
try:
await RpcHttpClient(client).connect(web3Url.url)
except CatchableError as e:
return err(e.msg)
of WsUrl:
try:
await RpcWebSocketClient(client).connect(web3Url.url)
except CatchableError as e:
return err(e.msg)
when isMainModule:
{.pop.}
let config = ExporterConf.load()
@ -334,19 +403,14 @@ when isMainModule:
dir = dataDir, error = ioErrorMsg(res.error)
quit 1
var client: RpcClient
try:
let c = newRpcWebSocketClient()
# TODO: Hardcoded to the default geth ws address. This should become
# a configurable cli option
waitFor c.connect("ws://127.0.0.1:8546")
client = c
except CatchableError as e:
fatal "Error while connecting to data provider", error = e.msg
quit 1
case config.cmd
of ExporterCmd.exportBlockData:
let client = newRpcClient(config.web3Url)
let connectRes = waitFor client.connectRpcClient(config.web3Url)
if connectRes.isErr():
fatal "Failed connecting to JSON-RPC client", error = connectRes.error
quit 1
if (config.endBlock < config.startBlock):
fatal "Initial block number should be smaller than end block number",
startBlock = config.startBlock,
@ -359,6 +423,12 @@ when isMainModule:
waitFor client.close()
of ExporterCmd.exportEpochHeaders:
let client = newRpcClient(config.web3Url)
let connectRes = waitFor client.connectRpcClient(config.web3Url)
if connectRes.isErr():
fatal "Failed connecting to JSON-RPC client", error = connectRes.error
quit 1
proc exportEpochHeaders(file: string, epoch: uint64): Result[void, string] =
# Downloading headers from JSON RPC endpoint
info "Requesting epoch headers", epoch
@ -547,6 +617,12 @@ when isMainModule:
echo &"{i.uint64:05} 0x{root.toHex()}"
of ExporterCmd.exportHeaderRange:
let client = newRpcClient(config.web3Url)
let connectRes = waitFor client.connectRpcClient(config.web3Url)
if connectRes.isErr():
fatal "Failed connecting to JSON-RPC client", error = connectRes.error
quit 1
let
startBlockNumber = config.startBlockNumber
endBlockNumber = config.endBlockNumber
@ -585,3 +661,89 @@ when isMainModule:
if res.isErr():
fatal "Failed exporting headers", error = res.error
quit 1
of ExporterCmd.exportHeadersWithProof:
let
startBlockNumber = config.startBlockNumber2
endBlockNumber = config.endBlockNumber2
if (endBlockNumber < startBlockNumber):
fatal "Start block number should be smaller than end block number",
startBlockNumber, endBlockNumber
quit 1
type
JsonPortalContent = object
content_key*: string
content_value*: string
JsonPortalContentTable = OrderedTable[uint64, JsonPortalContent]
proc writePortalContentToJson(
fh: OutputStreamHandle, content: JsonPortalContentTable) =
try:
var writer = JsonWriter[DefaultFlavor].init(fh.s, pretty = true)
writer.writeValue(content)
except IOError as e:
fatal "Error occured while writing to file", error = e.msg
quit 1
let file = &"mainnet-headersWithProof-{startBlockNumber:05}-{endBlockNumber:05}.json"
let fh = createAndOpenFile(string config.dataDir, file)
var contentTable: JsonPortalContentTable
for blockNumber in startBlockNumber..endBlockNumber:
let
epochIndex = getEpochIndex(blockNumber)
epochHeadersFile =
dataDir / &"mainnet-headers-epoch-{epochIndex:05}.e2s"
epochAccumulatorFile =
dataDir / &"mainnet-epoch-accumulator-{epochIndex:05}.ssz"
let res = readBlockHeaders(epochHeadersFile)
if res.isErr():
error "Could not read headers epoch file", error = res.error
quit 1
let blockHeaders = res.get()
let epochAccumulatorRes = readEpochAccumulatorCached(epochAccumulatorFile)
if epochAccumulatorRes.isErr():
error "Could not read epoch accumulator file", error = res.error
quit 1
let epochAccumulator = epochAccumulatorRes.get()
let headerIndex = getHeaderRecordIndex(blockNumber, epochIndex)
let header = blockHeaders[headerIndex]
if header.isPreMerge():
let headerWithProof = buildHeaderWithProof(header, epochAccumulator)
if headerWithProof.isErr:
error "Error building proof", error = headerWithProof.error
quit 1
let
content = headerWithProof.get()
contentKey = ContentKey(
contentType: blockHeader,
blockHeaderKey: BlockKey(blockHash: header.blockHash()))
encodedContentKey = history_content.encode(contentKey)
encodedContent = SSZ.encode(content)
let portalContent = JsonPortalContent(
content_key: encodedContentKey.asSeq().to0xHex(),
content_value: encodedContent.to0xHex())
contentTable[blockNumber] = portalContent
else:
# TODO: Deal with writing post merge headers
error "Not a pre merge header"
quit 1
writePortalContentToJson(fh, contentTable)
try:
fh.close()
except IOError as e:
fatal "Error occured while closing file", error = e.msg
quit 1