2023-07-05 18:17:03 +00:00
# Nimbus
2024-02-28 17:31:45 +00:00
# Copyright (c) 2023-2024 Status Research & Development GmbH
2023-07-05 18:17:03 +00:00
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{. push raises : [ ] . }
2024-02-28 17:31:45 +00:00
import std / [ os , uri ] , confutils , chronicles , beacon_chain / spec / digest
2023-07-05 18:17:03 +00:00
proc defaultDataDir * ( ) : string =
2024-02-28 17:31:45 +00:00
let dataDir =
when defined ( windows ) :
" AppData " / " Roaming " / " EthData "
elif defined ( macosx ) :
" Library " / " Application Support " / " EthData "
else :
" .cache " / " eth-data "
2023-07-05 18:17:03 +00:00
getHomeDir ( ) / dataDir
type
Web3UrlKind * = enum
2024-02-28 17:31:45 +00:00
HttpUrl
WsUrl
2023-07-05 18:17:03 +00:00
Web3Url * = object
kind * : Web3UrlKind
url * : string
StorageMode * = enum
2024-02-28 17:31:45 +00:00
JsonStorage
DbStorage
2023-07-05 18:17:03 +00:00
const
defaultDataDirDesc * = defaultDataDir ( )
defaultBlockFileName * = " eth-block-data "
defaultAccumulatorFileName * = " mainnet-master-accumulator.ssz "
defaultWeb3Url * = Web3Url ( kind : HttpUrl , url : " http://127.0.0.1:8545 " )
type
ExporterCmd * = enum
2024-02-28 17:31:45 +00:00
history
2023-07-05 18:17:03 +00:00
beacon
HistoryCmd * = enum
# TODO: Multiline strings doesn't work here anymore with 1.6, and concat of
# several lines gives the error: Error: Invalid node kind nnkInfix for macros.`$`
exportBlockData =
" Export block data (headers, bodies and receipts) to a json format or a database. Some of this functionality is likely to get deprecated "
exportEpochHeaders =
" Export block headers from an Ethereum JSON RPC Execution endpoint to *.e2s files arranged per epoch (8192 blocks) "
verifyEpochHeaders =
" Verify *.e2s files containing block headers. Verify currently only means being able to RLP decode the block headers "
exportAccumulatorData =
" Build and export the master accumulator and historical epoch accumulators. Requires *.e2s block header files generated with the exportHeaders command up until the merge block "
printAccumulatorData =
" Print the root hash of the master accumulator and of all historical epoch accumulators. Requires data generated by exportAccumulatorData command "
exportHeaderRange =
" Export block headers from an Ethereum JSON RPC Execution endpoint to *.e2s files (unlimited amount) "
exportHeadersWithProof =
" Export block headers with proof from *.e2s headers file and epochAccumulator files "
2024-02-09 10:13:12 +00:00
exportEra1 = " Export historical data to era1 store "
verifyEra1 = " Read and verify historical data from era1 store "
2023-07-05 18:17:03 +00:00
BeaconCmd * = enum
exportLCBootstrap = " Export Light Client Bootstrap "
exportLCUpdates = " Export Light Client Updates "
exportLCFinalityUpdate = " Export Light Client Finality Update "
exportLCOptimisticUpdate = " Export Light Client Optimistic Update "
2024-03-19 15:45:32 +00:00
exportHistoricalRoots = " Export historical roots from the beacon state (SSZ format) "
2024-05-03 09:50:18 +00:00
exportBeaconBlockProof =
" Export EL beacon block proof from era files (Bellatrix and later) "
2023-07-05 18:17:03 +00:00
ExporterConf * = object
logLevel * {.
2024-02-28 17:31:45 +00:00
defaultValue : LogLevel . INFO ,
defaultValueDesc : $ LogLevel . INFO ,
desc : " Sets the log level " ,
name : " log-level "
. } : LogLevel
2023-07-05 18:17:03 +00:00
dataDir * {.
2024-02-28 17:31:45 +00:00
desc : " The directory where generated data files will be exported to " ,
defaultValue : defaultDataDir ( ) ,
defaultValueDesc : $ defaultDataDirDesc ,
name : " data-dir "
. } : OutDir
2023-07-05 18:17:03 +00:00
case cmd * {. command . } : ExporterCmd
of ExporterCmd . history :
web3Url * {.
2024-02-28 17:31:45 +00:00
desc : " Execution layer JSON-RPC API URL " ,
defaultValue : defaultWeb3Url ,
name : " web3-url "
. } : Web3Url
2023-07-05 18:17:03 +00:00
case historyCmd * {. command . } : HistoryCmd
of exportBlockData :
startBlock * {.
2024-02-28 17:31:45 +00:00
desc : " Number of the first block to be exported " ,
defaultValue : 0 ,
name : " start-block "
. } : uint64
2023-07-05 18:17:03 +00:00
endBlock * {.
2024-02-28 17:31:45 +00:00
desc : " Number of the last block to be exported " ,
defaultValue : 0 ,
name : " end-block "
. } : uint64
2023-07-05 18:17:03 +00:00
fileName * {.
2024-02-28 17:31:45 +00:00
desc : " File name (minus extension) where block data will be exported to " ,
defaultValue : defaultBlockFileName ,
defaultValueDesc : $ defaultBlockFileName ,
name : " file-name "
. } : string
2023-07-05 18:17:03 +00:00
storageMode * {.
2024-02-28 17:31:45 +00:00
desc : " Storage mode of block data export " ,
defaultValue : JsonStorage ,
name : " storage-mode "
. } : StorageMode
2023-07-05 18:17:03 +00:00
headersOnly * {.
2024-02-28 17:31:45 +00:00
desc : " Only export the headers instead of full blocks and receipts " ,
defaultValue : false ,
name : " headers-only "
. } : bool
2023-07-05 18:17:03 +00:00
of exportEpochHeaders :
startEpoch * {.
2024-02-28 17:31:45 +00:00
desc : " Number of the first epoch which should be downloaded " ,
defaultValue : 0 ,
name : " start-epoch "
. } : uint64
2023-07-05 18:17:03 +00:00
endEpoch * {.
2024-02-28 17:31:45 +00:00
desc : " Number of the last epoch which should be downloaded " ,
defaultValue : 1896 ,
name : " end-epoch "
. } : uint64
2023-07-05 18:17:03 +00:00
# TODO:
# Although options are the same as for exportHeaders, we can't drop them
# under the same case of as confutils does not agree with that.
of verifyEpochHeaders :
startEpochVerify * {.
2024-02-28 17:31:45 +00:00
desc : " Number of the first epoch which should be downloaded " ,
defaultValue : 0 ,
name : " start-epoch "
. } : uint64
2023-07-05 18:17:03 +00:00
endEpochVerify * {.
2024-02-28 17:31:45 +00:00
desc : " Number of the last epoch which should be downloaded " ,
defaultValue : 1896 ,
name : " end-epoch "
. } : uint64
2023-07-05 18:17:03 +00:00
of exportAccumulatorData :
accumulatorFileName * {.
2024-02-28 17:31:45 +00:00
desc : " File to which the serialized accumulator is written " ,
defaultValue : defaultAccumulatorFileName ,
defaultValueDesc : $ defaultAccumulatorFileName ,
name : " accumulator-file-name "
. } : string
2023-07-05 18:17:03 +00:00
writeEpochAccumulators * {.
2024-02-28 17:31:45 +00:00
desc : " Write also the SSZ encoded epoch accumulators to specific files " ,
defaultValue : false ,
name : " write-epoch-accumulators "
. } : bool
2023-07-05 18:17:03 +00:00
of printAccumulatorData :
accumulatorFileNamePrint * {.
2024-02-28 17:31:45 +00:00
desc : " File from which the serialized accumulator is read " ,
defaultValue : defaultAccumulatorFileName ,
defaultValueDesc : $ defaultAccumulatorFileName ,
name : " accumulator-file-name "
. } : string
2023-07-05 18:17:03 +00:00
of exportHeaderRange :
startBlockNumber * {.
2024-02-28 17:31:45 +00:00
desc : " Number of the first block header to be exported " , name : " start-block "
. } : uint64
2023-07-05 18:17:03 +00:00
endBlockNumber * {.
2024-02-28 17:31:45 +00:00
desc : " Number of the last block header to be exported " , name : " end-block "
. } : uint64
2023-07-05 18:17:03 +00:00
of exportHeadersWithProof :
startBlockNumber2 * {.
2024-02-28 17:31:45 +00:00
desc : " Number of the first block header to be exported " , name : " start-block "
. } : uint64
2023-07-05 18:17:03 +00:00
endBlockNumber2 * {.
2024-02-28 17:31:45 +00:00
desc : " Number of the last block header to be exported " , name : " end-block "
. } : uint64
2024-02-09 10:13:12 +00:00
of exportEra1 :
2024-02-28 17:31:45 +00:00
era * {. defaultValue : 0 , desc : " The era number to write " . } : uint64
2024-02-09 10:13:12 +00:00
eraCount * {.
2024-02-28 17:31:45 +00:00
defaultValue : 0 , name : " count " , desc : " Number of eras to write (0=all) "
. } : uint64
2024-02-09 10:13:12 +00:00
of verifyEra1 :
2024-02-28 17:31:45 +00:00
era1FileName * {. desc : " Era1 file to read and verify " , name : " era1-file-name " . } :
string
2023-07-05 18:17:03 +00:00
of ExporterCmd . beacon :
restUrl * {.
2024-02-28 17:31:45 +00:00
desc : " URL of the beacon node REST service " ,
defaultValue : " http://127.0.0.1:5052 " ,
name : " rest-url "
. } : string
2023-07-05 18:17:03 +00:00
case beaconCmd * {. command . } : BeaconCmd
of exportLCBootstrap :
trustedBlockRoot * {.
2024-02-28 17:31:45 +00:00
desc : " Trusted finalized block root of the requested bootstrap " ,
name : " trusted-block-root "
. } : Eth2Digest
2023-07-05 18:17:03 +00:00
of exportLCUpdates :
startPeriod * {.
2024-02-28 17:31:45 +00:00
desc : " Period of the first LC update " , defaultValue : 0 , name : " start-period "
. } : uint64
2023-07-05 18:17:03 +00:00
count * {.
2024-02-28 17:31:45 +00:00
desc : " Amount of LC updates to request " , defaultValue : 1 , name : " count "
. } : uint64
2023-07-05 18:17:03 +00:00
of exportLCFinalityUpdate :
discard
of exportLCOptimisticUpdate :
discard
2024-03-19 15:45:32 +00:00
of exportHistoricalRoots :
discard
2024-05-03 09:50:18 +00:00
of exportBeaconBlockProof :
2024-03-21 12:25:32 +00:00
slotNumber * {.
2024-05-03 09:50:18 +00:00
desc : " The slot for which to export the beacon block proof " , name : " slot "
2024-03-21 12:25:32 +00:00
. } : uint64
eraDir * {. desc : " Directory containing era files " , name : " era-dir " . } : InputDir
2023-07-05 18:17:03 +00:00
2024-02-28 17:31:45 +00:00
proc parseCmdArg * ( T : type Web3Url , p : string ) : T {. raises : [ ValueError ] . } =
2023-07-05 18:17:03 +00:00
let
url = parseUri ( p )
normalizedScheme = url . scheme . toLowerAscii ( )
if ( normalizedScheme = = " http " or normalizedScheme = = " https " ) :
Web3Url ( kind : HttpUrl , url : p )
elif ( normalizedScheme = = " ws " or normalizedScheme = = " wss " ) :
Web3Url ( kind : WsUrl , url : p )
else :
raise newException (
2023-08-04 12:43:30 +00:00
ValueError ,
2024-02-28 17:31:45 +00:00
" The Web3 URL must specify one of following protocols: http/https/ws/wss " ,
2023-07-05 18:17:03 +00:00
)
proc completeCmdArg * ( T : type Web3Url , val : string ) : seq [ string ] =
return @ [ ]
2024-02-28 17:31:45 +00:00
proc parseCmdArg * ( T : type StorageMode , p : string ) : T {. raises : [ ValueError ] . } =
2023-07-05 18:17:03 +00:00
if p = = " db " :
return DbStorage
elif p = = " json " :
return JsonStorage
else :
let msg = " Provided mode: " & p & " is not a valid. Should be `json` or `db` "
2023-08-04 12:43:30 +00:00
raise newException ( ValueError , msg )
2023-07-05 18:17:03 +00:00
proc completeCmdArg * ( T : type StorageMode , val : string ) : seq [ string ] =
return @ [ ]
2024-02-28 17:31:45 +00:00
func parseCmdArg * (
T : type Eth2Digest , input : string
) : T {. raises : [ ValueError , Defect ] . } =
2023-07-05 18:17:03 +00:00
Eth2Digest . fromHex ( input )
func completeCmdArg * ( T : type Eth2Digest , input : string ) : seq [ string ] =
return @ [ ]