From 22591deced95957070eee868f5898ad1ab4eb0f3 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Sat, 9 Nov 2019 10:46:34 +0000 Subject: [PATCH] Safer testnet restarts; Working CLI queries for inspecting the genesis states When the connect_to_testnet script is invoked it will first verify that the genesis file of the testnet hasn't changed. If it has changed, any previously created database associated with the testnet will be erased. To facilitate this, the genesis file of each network is written to the data folder of the beacon node. The beacon node will refuse to start if it detects a discrepancy between the data folder and any state snapshot specified on the command-line. Since the testnet sharing spec requires us to use SSZ snapshots, the Json support is now phased out. To help with the transition and to preserve the functionality of the multinet scripts, the beacon node now supports a CLI query command that can extract any data from the genesis state. This is based on new developments in the SSZ navigators. --- beacon_chain/beacon_node.nim | 107 ++++++++++++++++++------- beacon_chain/spec/crypto.nim | 16 ++-- beacon_chain/ssz.nim | 5 +- beacon_chain/ssz/bytes_reader.nim | 4 +- beacon_chain/ssz/dynamic_navigator.nim | 39 +++++++-- beacon_chain/ssz/navigator.nim | 17 +++- scripts/connect_to_testnet.nims | 12 ++- tests/test_ssz.nim | 2 +- 8 files changed, 151 insertions(+), 51 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index aef73320c..ec0a8ce65 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -1,18 +1,18 @@ import # Standard library - os, net, tables, osproc, random, strutils, times, strformat, + os, net, tables, random, strutils, times, strformat, memfiles, # Nimble packages - stew/[objects, bitseqs, byteutils], + stew/[objects, bitseqs, byteutils], stew/ranges/ptr_arith, chronos, chronicles, confutils, metrics, json_serialization/std/[options, sets], serialization/errors, eth/trie/db, eth/trie/backends/rocksdb_backend, eth/async_utils, # Local modules spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network], - conf, time, state_transition, fork_choice, ssz, beacon_chain_db, + conf, time, state_transition, fork_choice, beacon_chain_db, validator_pool, extras, attestation_pool, block_pool, eth2_network, - beacon_node_types, mainchain_monitor, version, + beacon_node_types, mainchain_monitor, version, ssz, ssz/dynamic_navigator, sync_protocol, request_manager, validator_keygen, interop, statusbar const @@ -62,36 +62,57 @@ proc saveValidatorKey(keyName, key: string, conf: BeaconNodeConf) = writeFile(outputFile, key) info "Imported validator key", file = outputFile -func stateSnapshotPath*(conf: BeaconNodeConf): string = - if conf.stateSnapshot.isSome: - conf.stateSnapshot.get.string - else: - conf.dataDir / genesisFile - proc getStateFromSnapshot(node: BeaconNode, state: var BeaconState): bool = template conf: untyped = node.config - let snapshotFile = conf.stateSnapshotPath - if fileExists(snapshotFile): - template loadSnapshot(Format) = - info "Importing snapshot file", path = snapshotFile - state = loadFile(Format, snapshotFile, BeaconState) + var + genesisPath = conf.dataDir/genesisFile + snapshotContents: TaintedString + writeGenesisFile = false - let ext = splitFile(snapshotFile).ext - try: - if cmpIgnoreCase(ext, ".ssz") == 0: - loadSnapshot SSZ - elif cmpIgnoreCase(ext, ".json") == 0: - loadSnapshot Json - else: - error "The --state-snapshot option expects a json or a ssz file." - quit 1 - except SerializationError as err: - stderr.write "Failed to import ", snapshotFile, "\n" - stderr.write err.formatMsg(snapshotFile), "\n" + if conf.stateSnapshot.isSome: + let + snapshotPath = conf.stateSnapshot.get.string + snapshotExt = splitFile(snapshotPath).ext + + if cmpIgnoreCase(snapshotExt, ".ssz") != 0: + error "The supplied state snapshot must be a SSZ file", + suppliedPath = snapshotPath quit 1 - result = true + snapshotContents = readFile(snapshotPath) + if fileExists(genesisPath): + let genesisContents = readFile(genesisPath) + if snapshotContents != genesisContents: + error "Data directory not empty. Existing genesis state differs from supplied snapshot", + dataDir = conf.dataDir.string, snapshot = snapshotPath + quit 1 + else: + error "missing genesis file" + writeGenesisFile = true + genesisPath = snapshotPath + else: + try: + snapshotContents = readFile(genesisPath) + except CatchableError as err: + error "Failed to read genesis file", err = err.msg + quit 1 + + try: + state = SSZ.decode(snapshotContents, BeaconState) + except SerializationError as err: + error "Failed to import genesis file", path = genesisPath + quit 1 + + if writeGenesisFile: + try: + error "writing genesis file", path = conf.dataDir/genesisFile + writeFile(conf.dataDir/genesisFile, snapshotContents.string) + except CatchableError as err: + error "Failed to persist genesis file to data dir", err = err.msg + quit 1 + + result = true proc commitGenesisState(node: BeaconNode, tailState: BeaconState) = info "Got genesis state", hash = hash_tree_root(tailState) @@ -910,6 +931,10 @@ when hasPrompt: # var t: Thread[ptr Prompt] # createThread(t, processPromptCommands, addr p) +template bytes(memFile: MemFile): untyped = + let f = memFile + makeOpenArray(f.mem, byte, f.size) + when isMainModule: echo "$# ($#)\p" % [clientId, gitRevision] @@ -1031,3 +1056,29 @@ when isMainModule: config.depositWeb3Url, config.depositContractAddress, config.depositPrivateKey) + + of query: + var + trieDB = trieDB newChainDb(string config.databaseDir) + db = BeaconChainDB.init(trieDB) + + case config.queryCmd + of QueryCmd.nimQuery: + # TODO: This will handle a simple subset of Nim using + # dot syntax and `[]` indexing. + echo "nim query: ", config.nimQueryExpression + + of QueryCmd.get: + let pathFragments = config.getQueryPath.split('/', maxsplit = 1) + var navigator: DynamicSszNavigator + + case pathFragments[0] + of "genesis_state": + var genesisMapFile = memfiles.open(config.dataDir/genesisFile) + navigator = DynamicSszNavigator.init(genesisMapFile.bytes, BeaconState) + else: + stderr.write config.getQueryPath & " is not a valid path" + quit 1 + + echo navigator.navigatePath(pathFragments[1 .. ^1]).toJson + diff --git a/beacon_chain/spec/crypto.nim b/beacon_chain/spec/crypto.nim index ac799457b..d0ed0292d 100644 --- a/beacon_chain/spec/crypto.nim +++ b/beacon_chain/spec/crypto.nim @@ -221,7 +221,7 @@ else: # name from spec! ValidatorSig(kind: Real, blsValue: key.sign(domain, msg)) -proc fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R = +func fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R = # This is a workaround, so that we can deserialize the serialization of a # default-initialized BlsValue without raising an exception when defined(ssz_testing): @@ -229,20 +229,22 @@ proc fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R = R(kind: OpaqueBlob, blob: toArray(result.blob.len, bytes)) else: # Try if valid BLS value - let success = init(result.blsValue, bytes) + # TODO: address the side-effects in nim-blscurve + {.noSideEffect.}: + let success = init(result.blsValue, bytes) if not success: # TODO: chronicles trace result = R(kind: OpaqueBlob) assert result.blob.len == bytes.len result.blob[result.blob.low .. result.blob.high] = bytes -proc fromHex*[T](R: type BlsValue[T], hexStr: string): R = +func fromHex*[T](R: type BlsValue[T], hexStr: string): R = fromBytes(R, hexToSeqByte(hexStr)) -proc initFromBytes*[T](val: var BlsValue[T], bytes: openarray[byte]) = +func initFromBytes*[T](val: var BlsValue[T], bytes: openarray[byte]) = val = fromBytes(BlsValue[T], bytes) -proc initFromBytes*(val: var BlsCurveType, bytes: openarray[byte]) = +func initFromBytes*(val: var BlsCurveType, bytes: openarray[byte]) = val = init(type(val), bytes) proc writeValue*(writer: var JsonWriter, value: ValidatorPubKey) {.inline.} = @@ -329,3 +331,7 @@ proc toGaugeValue*(hash: Eth2Digest): int64 = # to the ETH2 metrics spec: # https://github.com/ethereum/eth2.0-metrics/blob/6a79914cb31f7d54858c7dd57eee75b6162ec737/metrics.md#interop-metrics cast[int64](uint64.fromBytesLE(hash.data[24..31])) + +template fromSszBytes*(T: type BlsValue, bytes: openarray[byte]): auto = + fromBytes(T, bytes) + diff --git a/beacon_chain/ssz.nim b/beacon_chain/ssz.nim index 0f78e2d63..7fb178db5 100644 --- a/beacon_chain/ssz.nim +++ b/beacon_chain/ssz.nim @@ -264,15 +264,12 @@ func writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) = buf.appendVarint length cursor.writeAndFinalize buf.writtenBytes -template fromSszBytes*(T: type BlsValue, bytes: openarray[byte]): auto = - fromBytes(T, bytes) - template fromSszBytes*[T; N](_: type TypeWithMaxLen[T, N], bytes: openarray[byte]): auto = mixin fromSszBytes fromSszBytes(T, bytes) -proc fromSszBytes*(T: type BlsCurveType, bytes: openarray[byte]): auto = +func fromSszBytes*(T: type BlsCurveType, bytes: openarray[byte]): auto = init(T, bytes) proc readValue*(r: var SszReader, val: var auto) = diff --git a/beacon_chain/ssz/bytes_reader.nim b/beacon_chain/ssz/bytes_reader.nim index 4670020ce..761694bd5 100644 --- a/beacon_chain/ssz/bytes_reader.nim +++ b/beacon_chain/ssz/bytes_reader.nim @@ -55,10 +55,10 @@ template fromSszBytes*(T: type enum, bytes: openarray[byte]): auto = template fromSszBytes*(T: type BitSeq, bytes: openarray[byte]): auto = BitSeq @bytes -proc fromSszBytes*[N](T: type BitList[N], bytes: openarray[byte]): auto = +func fromSszBytes*[N](T: type BitList[N], bytes: openarray[byte]): auto = BitList[N] @bytes -proc readSszValue*(input: openarray[byte], T: type): T = +func readSszValue*(input: openarray[byte], T: type): T = mixin fromSszBytes, toSszType type T {.used.}= type(result) diff --git a/beacon_chain/ssz/dynamic_navigator.nim b/beacon_chain/ssz/dynamic_navigator.nim index 7a7b5b19c..497abdb7e 100644 --- a/beacon_chain/ssz/dynamic_navigator.nim +++ b/beacon_chain/ssz/dynamic_navigator.nim @@ -1,8 +1,12 @@ import - parseutils, + strutils, parseutils, faststreams/output_stream, json_serialization/writer, + ../spec/datatypes, types, bytes_reader, navigator +export + navigator + type ObjKind = enum Record @@ -73,9 +77,11 @@ proc typeInfo*(T: type): TypeInfo = {.gcsafe, noSideEffect.}: res func genTypeInfo(T: type): TypeInfo = - mixin enumAllSerializedFields - - result = when T is object: + mixin toSszType, enumAllSerializedFields + type SszType = type(toSszType default(T)) + result = when type(SszType) isnot T: + TypeInfo(kind: LeafValue) + elif T is object: var fields: seq[FieldInfo] enumAllSerializedFields(T): fields.add FieldInfo(name: fieldName, @@ -108,13 +114,32 @@ func navigate*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {. var idx: int let consumed = parseInt(path, idx) if consumed == 0 or idx < 0: - raise newException(ValueError, "Indexing should be done with natural numbers") + raise newException(KeyError, "Indexing should be done with natural numbers") return n[idx] else: doAssert false, "Navigation should be terminated once you reach a leaf value" -func init*(T: type DynamicSszNavigator, bytes: openarray[byte], typ: TypeInfo): T = - T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len), typ: typ) +template navigatePathImpl(nav, iterabalePathFragments: untyped) = + result = nav + for pathFragment in iterabalePathFragments: + if pathFragment.len == 0: + continue + result = result.navigate(pathFragment) + if result.typ.kind == LeafValue: + return + +func navigatePath*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {. + raises: [Defect, ValueError, MalformedSszError] .} = + navigatePathImpl n, split(path, '/') + +func navigatePath*(n: DynamicSszNavigator, path: openarray[string]): DynamicSszNavigator {. + raises: [Defect, ValueError, MalformedSszError] .} = + navigatePathImpl n, path + +func init*(T: type DynamicSszNavigator, + bytes: openarray[byte], Navigated: type): T = + T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len), + typ: typeInfo(Navigated)) func writeJson*(n: DynamicSszNavigator, outStream: OutputStreamVar, pretty = true) = n.typ.jsonPrinter(n.m, outStream, pretty) diff --git a/beacon_chain/ssz/navigator.nim b/beacon_chain/ssz/navigator.nim index 795f20f44..49aa422ab 100644 --- a/beacon_chain/ssz/navigator.nim +++ b/beacon_chain/ssz/navigator.nim @@ -95,10 +95,10 @@ func indexVarSizeList(m: MemRange, idx: int): MemRange = MemRange(startAddr: m.startAddr.shift(elemPos), length: endPos - elemPos) -template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] = +template indexList(n, idx, T: untyped): untyped = type R = T mixin toSszType - type ElemType = type toSszType(default T) + type ElemType = type toSszType(default R) when isFixedSize(ElemType): const elemSize = fixedPortionSize(ElemType) let elemPos = idx * elemSize @@ -108,8 +108,19 @@ template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] = else: SszNavigator[R](m: indexVarSizeList(n.m, idx)) +template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] = + indexList n, idx, T + +template `[]`*[R, T](n: SszNavigator[array[R, T]], idx: int): SszNavigator[T] = + indexList(n, idx, T) + func `[]`*[T](n: SszNavigator[T]): T = - readSszValue(toOpenArray(n.m), T) + mixin toSszType, fromSszBytes + type SszRepr = type(toSszType default(T)) + when type(SszRepr) is type(T): + readSszValue(toOpenArray(n.m), T) + else: + fromSszBytes(T, toOpenArray(n.m)) converter derefNavigator*[T](n: SszNavigator[T]): T = n[] diff --git a/scripts/connect_to_testnet.nims b/scripts/connect_to_testnet.nims index 579698700..41f62e06b 100644 --- a/scripts/connect_to_testnet.nims +++ b/scripts/connect_to_testnet.nims @@ -10,9 +10,11 @@ const testnetsRepo = "eth2-testnets" let - testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-testnets") + testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-clients") testnetsGitUrl = getEnv("ETH2_TESTNETS_GIT_URL", "https://github.com/" & testnetsOrg & "/" & testnetsRepo) +mode = Verbose + proc validateTestnetName(parts: openarray[string]): auto = if parts.len != 2: echo "The testnet name should have the format `client/network-name`" @@ -61,6 +63,14 @@ cli do (testnetName {.argument.}: string): if fileExists(depositContractFile): depositContractOpt = "--deposit-contract=" & readFile(depositContractFile).strip + if dirExists(dataDir): + if fileExists(dataDir/genesisFile): + let localGenesisContent = readFile(dataDir/genesisFile) + let testnetGenesisContent = readFile(testnetDir/genesisFile) + if localGenesisContent != testnetGenesisContent: + echo "Detected testnet restart. Deleting previous database..." + rmDir dataDir + cd rootDir exec &"""nim c {nimFlags} -d:"const_preset={preset}" -o:"{beaconNodeBinary}" beacon_chain/beacon_node.nim""" exec replace(&"""{beaconNodeBinary} diff --git a/tests/test_ssz.nim b/tests/test_ssz.nim index ae16cc6d7..6b71919ce 100644 --- a/tests/test_ssz.nim +++ b/tests/test_ssz.nim @@ -102,7 +102,7 @@ suite "SSZ dynamic navigator": var fooOrig = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64))) let fooEncoded = SSZ.encode(fooOrig) - var navFoo = DynamicSszNavigator.init(fooEncoded, typeInfo(Foo)) + var navFoo = DynamicSszNavigator.init(fooEncoded, Foo) var navBar = navFoo.navigate("bar") check navBar.toJson(pretty = false) == """{"b":"bar","baz":{"i":10}}"""