Safer testnet restarts; Working CLI queries for inspecting the genesis states
When the connect_to_testnet script is invoked it will first verify that the genesis file of the testnet hasn't changed. If it has changed, any previously created database associated with the testnet will be erased. To facilitate this, the genesis file of each network is written to the data folder of the beacon node. The beacon node will refuse to start if it detects a discrepancy between the data folder and any state snapshot specified on the command-line. Since the testnet sharing spec requires us to use SSZ snapshots, the Json support is now phased out. To help with the transition and to preserve the functionality of the multinet scripts, the beacon node now supports a CLI query command that can extract any data from the genesis state. This is based on new developments in the SSZ navigators.
This commit is contained in:
parent
a644839b79
commit
22591deced
|
@ -1,18 +1,18 @@
|
|||
import
|
||||
# Standard library
|
||||
os, net, tables, osproc, random, strutils, times, strformat,
|
||||
os, net, tables, random, strutils, times, strformat, memfiles,
|
||||
|
||||
# Nimble packages
|
||||
stew/[objects, bitseqs, byteutils],
|
||||
stew/[objects, bitseqs, byteutils], stew/ranges/ptr_arith,
|
||||
chronos, chronicles, confutils, metrics,
|
||||
json_serialization/std/[options, sets], serialization/errors,
|
||||
eth/trie/db, eth/trie/backends/rocksdb_backend, eth/async_utils,
|
||||
|
||||
# Local modules
|
||||
spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network],
|
||||
conf, time, state_transition, fork_choice, ssz, beacon_chain_db,
|
||||
conf, time, state_transition, fork_choice, beacon_chain_db,
|
||||
validator_pool, extras, attestation_pool, block_pool, eth2_network,
|
||||
beacon_node_types, mainchain_monitor, version,
|
||||
beacon_node_types, mainchain_monitor, version, ssz, ssz/dynamic_navigator,
|
||||
sync_protocol, request_manager, validator_keygen, interop, statusbar
|
||||
|
||||
const
|
||||
|
@ -62,36 +62,57 @@ proc saveValidatorKey(keyName, key: string, conf: BeaconNodeConf) =
|
|||
writeFile(outputFile, key)
|
||||
info "Imported validator key", file = outputFile
|
||||
|
||||
func stateSnapshotPath*(conf: BeaconNodeConf): string =
|
||||
if conf.stateSnapshot.isSome:
|
||||
conf.stateSnapshot.get.string
|
||||
else:
|
||||
conf.dataDir / genesisFile
|
||||
|
||||
proc getStateFromSnapshot(node: BeaconNode, state: var BeaconState): bool =
|
||||
template conf: untyped = node.config
|
||||
let snapshotFile = conf.stateSnapshotPath
|
||||
|
||||
if fileExists(snapshotFile):
|
||||
template loadSnapshot(Format) =
|
||||
info "Importing snapshot file", path = snapshotFile
|
||||
state = loadFile(Format, snapshotFile, BeaconState)
|
||||
var
|
||||
genesisPath = conf.dataDir/genesisFile
|
||||
snapshotContents: TaintedString
|
||||
writeGenesisFile = false
|
||||
|
||||
let ext = splitFile(snapshotFile).ext
|
||||
try:
|
||||
if cmpIgnoreCase(ext, ".ssz") == 0:
|
||||
loadSnapshot SSZ
|
||||
elif cmpIgnoreCase(ext, ".json") == 0:
|
||||
loadSnapshot Json
|
||||
else:
|
||||
error "The --state-snapshot option expects a json or a ssz file."
|
||||
quit 1
|
||||
except SerializationError as err:
|
||||
stderr.write "Failed to import ", snapshotFile, "\n"
|
||||
stderr.write err.formatMsg(snapshotFile), "\n"
|
||||
if conf.stateSnapshot.isSome:
|
||||
let
|
||||
snapshotPath = conf.stateSnapshot.get.string
|
||||
snapshotExt = splitFile(snapshotPath).ext
|
||||
|
||||
if cmpIgnoreCase(snapshotExt, ".ssz") != 0:
|
||||
error "The supplied state snapshot must be a SSZ file",
|
||||
suppliedPath = snapshotPath
|
||||
quit 1
|
||||
|
||||
result = true
|
||||
snapshotContents = readFile(snapshotPath)
|
||||
if fileExists(genesisPath):
|
||||
let genesisContents = readFile(genesisPath)
|
||||
if snapshotContents != genesisContents:
|
||||
error "Data directory not empty. Existing genesis state differs from supplied snapshot",
|
||||
dataDir = conf.dataDir.string, snapshot = snapshotPath
|
||||
quit 1
|
||||
else:
|
||||
error "missing genesis file"
|
||||
writeGenesisFile = true
|
||||
genesisPath = snapshotPath
|
||||
else:
|
||||
try:
|
||||
snapshotContents = readFile(genesisPath)
|
||||
except CatchableError as err:
|
||||
error "Failed to read genesis file", err = err.msg
|
||||
quit 1
|
||||
|
||||
try:
|
||||
state = SSZ.decode(snapshotContents, BeaconState)
|
||||
except SerializationError as err:
|
||||
error "Failed to import genesis file", path = genesisPath
|
||||
quit 1
|
||||
|
||||
if writeGenesisFile:
|
||||
try:
|
||||
error "writing genesis file", path = conf.dataDir/genesisFile
|
||||
writeFile(conf.dataDir/genesisFile, snapshotContents.string)
|
||||
except CatchableError as err:
|
||||
error "Failed to persist genesis file to data dir", err = err.msg
|
||||
quit 1
|
||||
|
||||
result = true
|
||||
|
||||
proc commitGenesisState(node: BeaconNode, tailState: BeaconState) =
|
||||
info "Got genesis state", hash = hash_tree_root(tailState)
|
||||
|
@ -910,6 +931,10 @@ when hasPrompt:
|
|||
# var t: Thread[ptr Prompt]
|
||||
# createThread(t, processPromptCommands, addr p)
|
||||
|
||||
template bytes(memFile: MemFile): untyped =
|
||||
let f = memFile
|
||||
makeOpenArray(f.mem, byte, f.size)
|
||||
|
||||
when isMainModule:
|
||||
echo "$# ($#)\p" % [clientId, gitRevision]
|
||||
|
||||
|
@ -1031,3 +1056,29 @@ when isMainModule:
|
|||
config.depositWeb3Url,
|
||||
config.depositContractAddress,
|
||||
config.depositPrivateKey)
|
||||
|
||||
of query:
|
||||
var
|
||||
trieDB = trieDB newChainDb(string config.databaseDir)
|
||||
db = BeaconChainDB.init(trieDB)
|
||||
|
||||
case config.queryCmd
|
||||
of QueryCmd.nimQuery:
|
||||
# TODO: This will handle a simple subset of Nim using
|
||||
# dot syntax and `[]` indexing.
|
||||
echo "nim query: ", config.nimQueryExpression
|
||||
|
||||
of QueryCmd.get:
|
||||
let pathFragments = config.getQueryPath.split('/', maxsplit = 1)
|
||||
var navigator: DynamicSszNavigator
|
||||
|
||||
case pathFragments[0]
|
||||
of "genesis_state":
|
||||
var genesisMapFile = memfiles.open(config.dataDir/genesisFile)
|
||||
navigator = DynamicSszNavigator.init(genesisMapFile.bytes, BeaconState)
|
||||
else:
|
||||
stderr.write config.getQueryPath & " is not a valid path"
|
||||
quit 1
|
||||
|
||||
echo navigator.navigatePath(pathFragments[1 .. ^1]).toJson
|
||||
|
||||
|
|
|
@ -221,7 +221,7 @@ else:
|
|||
# name from spec!
|
||||
ValidatorSig(kind: Real, blsValue: key.sign(domain, msg))
|
||||
|
||||
proc fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R =
|
||||
func fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R =
|
||||
# This is a workaround, so that we can deserialize the serialization of a
|
||||
# default-initialized BlsValue without raising an exception
|
||||
when defined(ssz_testing):
|
||||
|
@ -229,20 +229,22 @@ proc fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R =
|
|||
R(kind: OpaqueBlob, blob: toArray(result.blob.len, bytes))
|
||||
else:
|
||||
# Try if valid BLS value
|
||||
let success = init(result.blsValue, bytes)
|
||||
# TODO: address the side-effects in nim-blscurve
|
||||
{.noSideEffect.}:
|
||||
let success = init(result.blsValue, bytes)
|
||||
if not success:
|
||||
# TODO: chronicles trace
|
||||
result = R(kind: OpaqueBlob)
|
||||
assert result.blob.len == bytes.len
|
||||
result.blob[result.blob.low .. result.blob.high] = bytes
|
||||
|
||||
proc fromHex*[T](R: type BlsValue[T], hexStr: string): R =
|
||||
func fromHex*[T](R: type BlsValue[T], hexStr: string): R =
|
||||
fromBytes(R, hexToSeqByte(hexStr))
|
||||
|
||||
proc initFromBytes*[T](val: var BlsValue[T], bytes: openarray[byte]) =
|
||||
func initFromBytes*[T](val: var BlsValue[T], bytes: openarray[byte]) =
|
||||
val = fromBytes(BlsValue[T], bytes)
|
||||
|
||||
proc initFromBytes*(val: var BlsCurveType, bytes: openarray[byte]) =
|
||||
func initFromBytes*(val: var BlsCurveType, bytes: openarray[byte]) =
|
||||
val = init(type(val), bytes)
|
||||
|
||||
proc writeValue*(writer: var JsonWriter, value: ValidatorPubKey) {.inline.} =
|
||||
|
@ -329,3 +331,7 @@ proc toGaugeValue*(hash: Eth2Digest): int64 =
|
|||
# to the ETH2 metrics spec:
|
||||
# https://github.com/ethereum/eth2.0-metrics/blob/6a79914cb31f7d54858c7dd57eee75b6162ec737/metrics.md#interop-metrics
|
||||
cast[int64](uint64.fromBytesLE(hash.data[24..31]))
|
||||
|
||||
template fromSszBytes*(T: type BlsValue, bytes: openarray[byte]): auto =
|
||||
fromBytes(T, bytes)
|
||||
|
||||
|
|
|
@ -264,15 +264,12 @@ func writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) =
|
|||
buf.appendVarint length
|
||||
cursor.writeAndFinalize buf.writtenBytes
|
||||
|
||||
template fromSszBytes*(T: type BlsValue, bytes: openarray[byte]): auto =
|
||||
fromBytes(T, bytes)
|
||||
|
||||
template fromSszBytes*[T; N](_: type TypeWithMaxLen[T, N],
|
||||
bytes: openarray[byte]): auto =
|
||||
mixin fromSszBytes
|
||||
fromSszBytes(T, bytes)
|
||||
|
||||
proc fromSszBytes*(T: type BlsCurveType, bytes: openarray[byte]): auto =
|
||||
func fromSszBytes*(T: type BlsCurveType, bytes: openarray[byte]): auto =
|
||||
init(T, bytes)
|
||||
|
||||
proc readValue*(r: var SszReader, val: var auto) =
|
||||
|
|
|
@ -55,10 +55,10 @@ template fromSszBytes*(T: type enum, bytes: openarray[byte]): auto =
|
|||
template fromSszBytes*(T: type BitSeq, bytes: openarray[byte]): auto =
|
||||
BitSeq @bytes
|
||||
|
||||
proc fromSszBytes*[N](T: type BitList[N], bytes: openarray[byte]): auto =
|
||||
func fromSszBytes*[N](T: type BitList[N], bytes: openarray[byte]): auto =
|
||||
BitList[N] @bytes
|
||||
|
||||
proc readSszValue*(input: openarray[byte], T: type): T =
|
||||
func readSszValue*(input: openarray[byte], T: type): T =
|
||||
mixin fromSszBytes, toSszType
|
||||
|
||||
type T {.used.}= type(result)
|
||||
|
|
|
@ -1,8 +1,12 @@
|
|||
import
|
||||
parseutils,
|
||||
strutils, parseutils,
|
||||
faststreams/output_stream, json_serialization/writer,
|
||||
../spec/datatypes,
|
||||
types, bytes_reader, navigator
|
||||
|
||||
export
|
||||
navigator
|
||||
|
||||
type
|
||||
ObjKind = enum
|
||||
Record
|
||||
|
@ -73,9 +77,11 @@ proc typeInfo*(T: type): TypeInfo =
|
|||
{.gcsafe, noSideEffect.}: res
|
||||
|
||||
func genTypeInfo(T: type): TypeInfo =
|
||||
mixin enumAllSerializedFields
|
||||
|
||||
result = when T is object:
|
||||
mixin toSszType, enumAllSerializedFields
|
||||
type SszType = type(toSszType default(T))
|
||||
result = when type(SszType) isnot T:
|
||||
TypeInfo(kind: LeafValue)
|
||||
elif T is object:
|
||||
var fields: seq[FieldInfo]
|
||||
enumAllSerializedFields(T):
|
||||
fields.add FieldInfo(name: fieldName,
|
||||
|
@ -108,13 +114,32 @@ func navigate*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {.
|
|||
var idx: int
|
||||
let consumed = parseInt(path, idx)
|
||||
if consumed == 0 or idx < 0:
|
||||
raise newException(ValueError, "Indexing should be done with natural numbers")
|
||||
raise newException(KeyError, "Indexing should be done with natural numbers")
|
||||
return n[idx]
|
||||
else:
|
||||
doAssert false, "Navigation should be terminated once you reach a leaf value"
|
||||
|
||||
func init*(T: type DynamicSszNavigator, bytes: openarray[byte], typ: TypeInfo): T =
|
||||
T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len), typ: typ)
|
||||
template navigatePathImpl(nav, iterabalePathFragments: untyped) =
|
||||
result = nav
|
||||
for pathFragment in iterabalePathFragments:
|
||||
if pathFragment.len == 0:
|
||||
continue
|
||||
result = result.navigate(pathFragment)
|
||||
if result.typ.kind == LeafValue:
|
||||
return
|
||||
|
||||
func navigatePath*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {.
|
||||
raises: [Defect, ValueError, MalformedSszError] .} =
|
||||
navigatePathImpl n, split(path, '/')
|
||||
|
||||
func navigatePath*(n: DynamicSszNavigator, path: openarray[string]): DynamicSszNavigator {.
|
||||
raises: [Defect, ValueError, MalformedSszError] .} =
|
||||
navigatePathImpl n, path
|
||||
|
||||
func init*(T: type DynamicSszNavigator,
|
||||
bytes: openarray[byte], Navigated: type): T =
|
||||
T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len),
|
||||
typ: typeInfo(Navigated))
|
||||
|
||||
func writeJson*(n: DynamicSszNavigator, outStream: OutputStreamVar, pretty = true) =
|
||||
n.typ.jsonPrinter(n.m, outStream, pretty)
|
||||
|
|
|
@ -95,10 +95,10 @@ func indexVarSizeList(m: MemRange, idx: int): MemRange =
|
|||
|
||||
MemRange(startAddr: m.startAddr.shift(elemPos), length: endPos - elemPos)
|
||||
|
||||
template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] =
|
||||
template indexList(n, idx, T: untyped): untyped =
|
||||
type R = T
|
||||
mixin toSszType
|
||||
type ElemType = type toSszType(default T)
|
||||
type ElemType = type toSszType(default R)
|
||||
when isFixedSize(ElemType):
|
||||
const elemSize = fixedPortionSize(ElemType)
|
||||
let elemPos = idx * elemSize
|
||||
|
@ -108,8 +108,19 @@ template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] =
|
|||
else:
|
||||
SszNavigator[R](m: indexVarSizeList(n.m, idx))
|
||||
|
||||
template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] =
|
||||
indexList n, idx, T
|
||||
|
||||
template `[]`*[R, T](n: SszNavigator[array[R, T]], idx: int): SszNavigator[T] =
|
||||
indexList(n, idx, T)
|
||||
|
||||
func `[]`*[T](n: SszNavigator[T]): T =
|
||||
readSszValue(toOpenArray(n.m), T)
|
||||
mixin toSszType, fromSszBytes
|
||||
type SszRepr = type(toSszType default(T))
|
||||
when type(SszRepr) is type(T):
|
||||
readSszValue(toOpenArray(n.m), T)
|
||||
else:
|
||||
fromSszBytes(T, toOpenArray(n.m))
|
||||
|
||||
converter derefNavigator*[T](n: SszNavigator[T]): T =
|
||||
n[]
|
||||
|
|
|
@ -10,9 +10,11 @@ const
|
|||
testnetsRepo = "eth2-testnets"
|
||||
|
||||
let
|
||||
testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-testnets")
|
||||
testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-clients")
|
||||
testnetsGitUrl = getEnv("ETH2_TESTNETS_GIT_URL", "https://github.com/" & testnetsOrg & "/" & testnetsRepo)
|
||||
|
||||
mode = Verbose
|
||||
|
||||
proc validateTestnetName(parts: openarray[string]): auto =
|
||||
if parts.len != 2:
|
||||
echo "The testnet name should have the format `client/network-name`"
|
||||
|
@ -61,6 +63,14 @@ cli do (testnetName {.argument.}: string):
|
|||
if fileExists(depositContractFile):
|
||||
depositContractOpt = "--deposit-contract=" & readFile(depositContractFile).strip
|
||||
|
||||
if dirExists(dataDir):
|
||||
if fileExists(dataDir/genesisFile):
|
||||
let localGenesisContent = readFile(dataDir/genesisFile)
|
||||
let testnetGenesisContent = readFile(testnetDir/genesisFile)
|
||||
if localGenesisContent != testnetGenesisContent:
|
||||
echo "Detected testnet restart. Deleting previous database..."
|
||||
rmDir dataDir
|
||||
|
||||
cd rootDir
|
||||
exec &"""nim c {nimFlags} -d:"const_preset={preset}" -o:"{beaconNodeBinary}" beacon_chain/beacon_node.nim"""
|
||||
exec replace(&"""{beaconNodeBinary}
|
||||
|
|
|
@ -102,7 +102,7 @@ suite "SSZ dynamic navigator":
|
|||
var fooOrig = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
|
||||
let fooEncoded = SSZ.encode(fooOrig)
|
||||
|
||||
var navFoo = DynamicSszNavigator.init(fooEncoded, typeInfo(Foo))
|
||||
var navFoo = DynamicSszNavigator.init(fooEncoded, Foo)
|
||||
|
||||
var navBar = navFoo.navigate("bar")
|
||||
check navBar.toJson(pretty = false) == """{"b":"bar","baz":{"i":10}}"""
|
||||
|
|
Loading…
Reference in New Issue