diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index aef73320c..ec0a8ce65 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -1,18 +1,18 @@ import # Standard library - os, net, tables, osproc, random, strutils, times, strformat, + os, net, tables, random, strutils, times, strformat, memfiles, # Nimble packages - stew/[objects, bitseqs, byteutils], + stew/[objects, bitseqs, byteutils], stew/ranges/ptr_arith, chronos, chronicles, confutils, metrics, json_serialization/std/[options, sets], serialization/errors, eth/trie/db, eth/trie/backends/rocksdb_backend, eth/async_utils, # Local modules spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network], - conf, time, state_transition, fork_choice, ssz, beacon_chain_db, + conf, time, state_transition, fork_choice, beacon_chain_db, validator_pool, extras, attestation_pool, block_pool, eth2_network, - beacon_node_types, mainchain_monitor, version, + beacon_node_types, mainchain_monitor, version, ssz, ssz/dynamic_navigator, sync_protocol, request_manager, validator_keygen, interop, statusbar const @@ -62,36 +62,57 @@ proc saveValidatorKey(keyName, key: string, conf: BeaconNodeConf) = writeFile(outputFile, key) info "Imported validator key", file = outputFile -func stateSnapshotPath*(conf: BeaconNodeConf): string = - if conf.stateSnapshot.isSome: - conf.stateSnapshot.get.string - else: - conf.dataDir / genesisFile - proc getStateFromSnapshot(node: BeaconNode, state: var BeaconState): bool = template conf: untyped = node.config - let snapshotFile = conf.stateSnapshotPath - if fileExists(snapshotFile): - template loadSnapshot(Format) = - info "Importing snapshot file", path = snapshotFile - state = loadFile(Format, snapshotFile, BeaconState) + var + genesisPath = conf.dataDir/genesisFile + snapshotContents: TaintedString + writeGenesisFile = false - let ext = splitFile(snapshotFile).ext - try: - if cmpIgnoreCase(ext, ".ssz") == 0: - loadSnapshot SSZ - elif cmpIgnoreCase(ext, ".json") == 0: - loadSnapshot Json - else: - error "The --state-snapshot option expects a json or a ssz file." - quit 1 - except SerializationError as err: - stderr.write "Failed to import ", snapshotFile, "\n" - stderr.write err.formatMsg(snapshotFile), "\n" + if conf.stateSnapshot.isSome: + let + snapshotPath = conf.stateSnapshot.get.string + snapshotExt = splitFile(snapshotPath).ext + + if cmpIgnoreCase(snapshotExt, ".ssz") != 0: + error "The supplied state snapshot must be a SSZ file", + suppliedPath = snapshotPath quit 1 - result = true + snapshotContents = readFile(snapshotPath) + if fileExists(genesisPath): + let genesisContents = readFile(genesisPath) + if snapshotContents != genesisContents: + error "Data directory not empty. Existing genesis state differs from supplied snapshot", + dataDir = conf.dataDir.string, snapshot = snapshotPath + quit 1 + else: + error "missing genesis file" + writeGenesisFile = true + genesisPath = snapshotPath + else: + try: + snapshotContents = readFile(genesisPath) + except CatchableError as err: + error "Failed to read genesis file", err = err.msg + quit 1 + + try: + state = SSZ.decode(snapshotContents, BeaconState) + except SerializationError as err: + error "Failed to import genesis file", path = genesisPath + quit 1 + + if writeGenesisFile: + try: + error "writing genesis file", path = conf.dataDir/genesisFile + writeFile(conf.dataDir/genesisFile, snapshotContents.string) + except CatchableError as err: + error "Failed to persist genesis file to data dir", err = err.msg + quit 1 + + result = true proc commitGenesisState(node: BeaconNode, tailState: BeaconState) = info "Got genesis state", hash = hash_tree_root(tailState) @@ -910,6 +931,10 @@ when hasPrompt: # var t: Thread[ptr Prompt] # createThread(t, processPromptCommands, addr p) +template bytes(memFile: MemFile): untyped = + let f = memFile + makeOpenArray(f.mem, byte, f.size) + when isMainModule: echo "$# ($#)\p" % [clientId, gitRevision] @@ -1031,3 +1056,29 @@ when isMainModule: config.depositWeb3Url, config.depositContractAddress, config.depositPrivateKey) + + of query: + var + trieDB = trieDB newChainDb(string config.databaseDir) + db = BeaconChainDB.init(trieDB) + + case config.queryCmd + of QueryCmd.nimQuery: + # TODO: This will handle a simple subset of Nim using + # dot syntax and `[]` indexing. + echo "nim query: ", config.nimQueryExpression + + of QueryCmd.get: + let pathFragments = config.getQueryPath.split('/', maxsplit = 1) + var navigator: DynamicSszNavigator + + case pathFragments[0] + of "genesis_state": + var genesisMapFile = memfiles.open(config.dataDir/genesisFile) + navigator = DynamicSszNavigator.init(genesisMapFile.bytes, BeaconState) + else: + stderr.write config.getQueryPath & " is not a valid path" + quit 1 + + echo navigator.navigatePath(pathFragments[1 .. ^1]).toJson + diff --git a/beacon_chain/spec/crypto.nim b/beacon_chain/spec/crypto.nim index ac799457b..d0ed0292d 100644 --- a/beacon_chain/spec/crypto.nim +++ b/beacon_chain/spec/crypto.nim @@ -221,7 +221,7 @@ else: # name from spec! ValidatorSig(kind: Real, blsValue: key.sign(domain, msg)) -proc fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R = +func fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R = # This is a workaround, so that we can deserialize the serialization of a # default-initialized BlsValue without raising an exception when defined(ssz_testing): @@ -229,20 +229,22 @@ proc fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R = R(kind: OpaqueBlob, blob: toArray(result.blob.len, bytes)) else: # Try if valid BLS value - let success = init(result.blsValue, bytes) + # TODO: address the side-effects in nim-blscurve + {.noSideEffect.}: + let success = init(result.blsValue, bytes) if not success: # TODO: chronicles trace result = R(kind: OpaqueBlob) assert result.blob.len == bytes.len result.blob[result.blob.low .. result.blob.high] = bytes -proc fromHex*[T](R: type BlsValue[T], hexStr: string): R = +func fromHex*[T](R: type BlsValue[T], hexStr: string): R = fromBytes(R, hexToSeqByte(hexStr)) -proc initFromBytes*[T](val: var BlsValue[T], bytes: openarray[byte]) = +func initFromBytes*[T](val: var BlsValue[T], bytes: openarray[byte]) = val = fromBytes(BlsValue[T], bytes) -proc initFromBytes*(val: var BlsCurveType, bytes: openarray[byte]) = +func initFromBytes*(val: var BlsCurveType, bytes: openarray[byte]) = val = init(type(val), bytes) proc writeValue*(writer: var JsonWriter, value: ValidatorPubKey) {.inline.} = @@ -329,3 +331,7 @@ proc toGaugeValue*(hash: Eth2Digest): int64 = # to the ETH2 metrics spec: # https://github.com/ethereum/eth2.0-metrics/blob/6a79914cb31f7d54858c7dd57eee75b6162ec737/metrics.md#interop-metrics cast[int64](uint64.fromBytesLE(hash.data[24..31])) + +template fromSszBytes*(T: type BlsValue, bytes: openarray[byte]): auto = + fromBytes(T, bytes) + diff --git a/beacon_chain/ssz.nim b/beacon_chain/ssz.nim index 0f78e2d63..7fb178db5 100644 --- a/beacon_chain/ssz.nim +++ b/beacon_chain/ssz.nim @@ -264,15 +264,12 @@ func writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) = buf.appendVarint length cursor.writeAndFinalize buf.writtenBytes -template fromSszBytes*(T: type BlsValue, bytes: openarray[byte]): auto = - fromBytes(T, bytes) - template fromSszBytes*[T; N](_: type TypeWithMaxLen[T, N], bytes: openarray[byte]): auto = mixin fromSszBytes fromSszBytes(T, bytes) -proc fromSszBytes*(T: type BlsCurveType, bytes: openarray[byte]): auto = +func fromSszBytes*(T: type BlsCurveType, bytes: openarray[byte]): auto = init(T, bytes) proc readValue*(r: var SszReader, val: var auto) = diff --git a/beacon_chain/ssz/bytes_reader.nim b/beacon_chain/ssz/bytes_reader.nim index 4670020ce..761694bd5 100644 --- a/beacon_chain/ssz/bytes_reader.nim +++ b/beacon_chain/ssz/bytes_reader.nim @@ -55,10 +55,10 @@ template fromSszBytes*(T: type enum, bytes: openarray[byte]): auto = template fromSszBytes*(T: type BitSeq, bytes: openarray[byte]): auto = BitSeq @bytes -proc fromSszBytes*[N](T: type BitList[N], bytes: openarray[byte]): auto = +func fromSszBytes*[N](T: type BitList[N], bytes: openarray[byte]): auto = BitList[N] @bytes -proc readSszValue*(input: openarray[byte], T: type): T = +func readSszValue*(input: openarray[byte], T: type): T = mixin fromSszBytes, toSszType type T {.used.}= type(result) diff --git a/beacon_chain/ssz/dynamic_navigator.nim b/beacon_chain/ssz/dynamic_navigator.nim index 7a7b5b19c..497abdb7e 100644 --- a/beacon_chain/ssz/dynamic_navigator.nim +++ b/beacon_chain/ssz/dynamic_navigator.nim @@ -1,8 +1,12 @@ import - parseutils, + strutils, parseutils, faststreams/output_stream, json_serialization/writer, + ../spec/datatypes, types, bytes_reader, navigator +export + navigator + type ObjKind = enum Record @@ -73,9 +77,11 @@ proc typeInfo*(T: type): TypeInfo = {.gcsafe, noSideEffect.}: res func genTypeInfo(T: type): TypeInfo = - mixin enumAllSerializedFields - - result = when T is object: + mixin toSszType, enumAllSerializedFields + type SszType = type(toSszType default(T)) + result = when type(SszType) isnot T: + TypeInfo(kind: LeafValue) + elif T is object: var fields: seq[FieldInfo] enumAllSerializedFields(T): fields.add FieldInfo(name: fieldName, @@ -108,13 +114,32 @@ func navigate*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {. var idx: int let consumed = parseInt(path, idx) if consumed == 0 or idx < 0: - raise newException(ValueError, "Indexing should be done with natural numbers") + raise newException(KeyError, "Indexing should be done with natural numbers") return n[idx] else: doAssert false, "Navigation should be terminated once you reach a leaf value" -func init*(T: type DynamicSszNavigator, bytes: openarray[byte], typ: TypeInfo): T = - T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len), typ: typ) +template navigatePathImpl(nav, iterabalePathFragments: untyped) = + result = nav + for pathFragment in iterabalePathFragments: + if pathFragment.len == 0: + continue + result = result.navigate(pathFragment) + if result.typ.kind == LeafValue: + return + +func navigatePath*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {. + raises: [Defect, ValueError, MalformedSszError] .} = + navigatePathImpl n, split(path, '/') + +func navigatePath*(n: DynamicSszNavigator, path: openarray[string]): DynamicSszNavigator {. + raises: [Defect, ValueError, MalformedSszError] .} = + navigatePathImpl n, path + +func init*(T: type DynamicSszNavigator, + bytes: openarray[byte], Navigated: type): T = + T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len), + typ: typeInfo(Navigated)) func writeJson*(n: DynamicSszNavigator, outStream: OutputStreamVar, pretty = true) = n.typ.jsonPrinter(n.m, outStream, pretty) diff --git a/beacon_chain/ssz/navigator.nim b/beacon_chain/ssz/navigator.nim index 795f20f44..49aa422ab 100644 --- a/beacon_chain/ssz/navigator.nim +++ b/beacon_chain/ssz/navigator.nim @@ -95,10 +95,10 @@ func indexVarSizeList(m: MemRange, idx: int): MemRange = MemRange(startAddr: m.startAddr.shift(elemPos), length: endPos - elemPos) -template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] = +template indexList(n, idx, T: untyped): untyped = type R = T mixin toSszType - type ElemType = type toSszType(default T) + type ElemType = type toSszType(default R) when isFixedSize(ElemType): const elemSize = fixedPortionSize(ElemType) let elemPos = idx * elemSize @@ -108,8 +108,19 @@ template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] = else: SszNavigator[R](m: indexVarSizeList(n.m, idx)) +template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] = + indexList n, idx, T + +template `[]`*[R, T](n: SszNavigator[array[R, T]], idx: int): SszNavigator[T] = + indexList(n, idx, T) + func `[]`*[T](n: SszNavigator[T]): T = - readSszValue(toOpenArray(n.m), T) + mixin toSszType, fromSszBytes + type SszRepr = type(toSszType default(T)) + when type(SszRepr) is type(T): + readSszValue(toOpenArray(n.m), T) + else: + fromSszBytes(T, toOpenArray(n.m)) converter derefNavigator*[T](n: SszNavigator[T]): T = n[] diff --git a/scripts/connect_to_testnet.nims b/scripts/connect_to_testnet.nims index 579698700..41f62e06b 100644 --- a/scripts/connect_to_testnet.nims +++ b/scripts/connect_to_testnet.nims @@ -10,9 +10,11 @@ const testnetsRepo = "eth2-testnets" let - testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-testnets") + testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-clients") testnetsGitUrl = getEnv("ETH2_TESTNETS_GIT_URL", "https://github.com/" & testnetsOrg & "/" & testnetsRepo) +mode = Verbose + proc validateTestnetName(parts: openarray[string]): auto = if parts.len != 2: echo "The testnet name should have the format `client/network-name`" @@ -61,6 +63,14 @@ cli do (testnetName {.argument.}: string): if fileExists(depositContractFile): depositContractOpt = "--deposit-contract=" & readFile(depositContractFile).strip + if dirExists(dataDir): + if fileExists(dataDir/genesisFile): + let localGenesisContent = readFile(dataDir/genesisFile) + let testnetGenesisContent = readFile(testnetDir/genesisFile) + if localGenesisContent != testnetGenesisContent: + echo "Detected testnet restart. Deleting previous database..." + rmDir dataDir + cd rootDir exec &"""nim c {nimFlags} -d:"const_preset={preset}" -o:"{beaconNodeBinary}" beacon_chain/beacon_node.nim""" exec replace(&"""{beaconNodeBinary} diff --git a/tests/test_ssz.nim b/tests/test_ssz.nim index ae16cc6d7..6b71919ce 100644 --- a/tests/test_ssz.nim +++ b/tests/test_ssz.nim @@ -102,7 +102,7 @@ suite "SSZ dynamic navigator": var fooOrig = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64))) let fooEncoded = SSZ.encode(fooOrig) - var navFoo = DynamicSszNavigator.init(fooEncoded, typeInfo(Foo)) + var navFoo = DynamicSszNavigator.init(fooEncoded, Foo) var navBar = navFoo.navigate("bar") check navBar.toJson(pretty = false) == """{"b":"bar","baz":{"i":10}}"""