Safer testnet restarts; Working CLI queries for inspecting the genesis states

When the connect_to_testnet script is invoked it will first verify that
the genesis file of the testnet hasn't changed. If it has changed, any
previously created database associated with the testnet will be erased.

To facilitate this, the genesis file of each network is written to the
data folder of the beacon node. The beacon node will refuse to start if
it detects a discrepancy between the data folder and any state snapshot
specified on the command-line.

Since the testnet sharing spec requires us to use SSZ snapshots, the Json
support is now phased out. To help with the transition and to preserve the
functionality of the multinet scripts, the beacon node now supports a CLI
query command that can extract any data from the genesis state. This is
based on new developments in the SSZ navigators.

beacon_chain/beacon_node.nim

@@ -1,18 +1,18 @@
 import
   # Standard library
-  os, net, tables, osproc, random, strutils, times, strformat,
+  os, net, tables, random, strutils, times, strformat, memfiles,
 
   # Nimble packages
-  stew/[objects, bitseqs, byteutils],
+  stew/[objects, bitseqs, byteutils], stew/ranges/ptr_arith,
   chronos, chronicles, confutils, metrics,
   json_serialization/std/[options, sets], serialization/errors,
   eth/trie/db, eth/trie/backends/rocksdb_backend, eth/async_utils,
 
   # Local modules
   spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network],
-  conf, time, state_transition, fork_choice, ssz, beacon_chain_db,
+  conf, time, state_transition, fork_choice, beacon_chain_db,
   validator_pool, extras, attestation_pool, block_pool, eth2_network,
-  beacon_node_types, mainchain_monitor, version,
+  beacon_node_types, mainchain_monitor, version, ssz, ssz/dynamic_navigator,
   sync_protocol, request_manager, validator_keygen, interop, statusbar
 
 const
@@ -62,36 +62,57 @@ proc saveValidatorKey(keyName, key: string, conf: BeaconNodeConf) =
   writeFile(outputFile, key)
   info "Imported validator key", file = outputFile
 
-func stateSnapshotPath*(conf: BeaconNodeConf): string =
-  if conf.stateSnapshot.isSome:
-    conf.stateSnapshot.get.string
-  else:
-    conf.dataDir / genesisFile
-
 proc getStateFromSnapshot(node: BeaconNode, state: var BeaconState): bool =
   template conf: untyped = node.config
-  let snapshotFile = conf.stateSnapshotPath
 
-  if fileExists(snapshotFile):
-    template loadSnapshot(Format) =
-      info "Importing snapshot file", path = snapshotFile
-      state = loadFile(Format, snapshotFile, BeaconState)
+  var
+    genesisPath = conf.dataDir/genesisFile
+    snapshotContents: TaintedString
+    writeGenesisFile = false
 
-    let ext = splitFile(snapshotFile).ext
-    try:
-      if cmpIgnoreCase(ext, ".ssz") == 0:
-        loadSnapshot SSZ
-      elif cmpIgnoreCase(ext, ".json") == 0:
-        loadSnapshot Json
-      else:
-        error "The --state-snapshot option expects a json or a ssz file."
-        quit 1
-    except SerializationError as err:
-      stderr.write "Failed to import ", snapshotFile, "\n"
-      stderr.write err.formatMsg(snapshotFile), "\n"
+  if conf.stateSnapshot.isSome:
+    let
+      snapshotPath = conf.stateSnapshot.get.string
+      snapshotExt = splitFile(snapshotPath).ext
+
+    if cmpIgnoreCase(snapshotExt, ".ssz") != 0:
+      error "The supplied state snapshot must be a SSZ file",
+            suppliedPath = snapshotPath
       quit 1
 
-    result = true
+    snapshotContents = readFile(snapshotPath)
+    if fileExists(genesisPath):
+      let genesisContents = readFile(genesisPath)
+      if snapshotContents != genesisContents:
+        error "Data directory not empty. Existing genesis state differs from supplied snapshot",
+              dataDir = conf.dataDir.string, snapshot = snapshotPath
+        quit 1
+    else:
+      error "missing genesis file"
+      writeGenesisFile = true
+      genesisPath = snapshotPath
+  else:
+    try:
+      snapshotContents = readFile(genesisPath)
+    except CatchableError as err:
+      error "Failed to read genesis file", err = err.msg
+      quit 1
+
+  try:
+    state = SSZ.decode(snapshotContents, BeaconState)
+  except SerializationError as err:
+    error "Failed to import genesis file", path = genesisPath
+    quit 1
+
+  if writeGenesisFile:
+    try:
+      error "writing genesis file", path = conf.dataDir/genesisFile
+      writeFile(conf.dataDir/genesisFile, snapshotContents.string)
+    except CatchableError as err:
+      error "Failed to persist genesis file to data dir", err = err.msg
+      quit 1
+
+  result = true
 
 proc commitGenesisState(node: BeaconNode, tailState: BeaconState) =
   info "Got genesis state", hash = hash_tree_root(tailState)
@@ -910,6 +931,10 @@ when hasPrompt:
       # var t: Thread[ptr Prompt]
       # createThread(t, processPromptCommands, addr p)
 
+template bytes(memFile: MemFile): untyped =
+  let f = memFile
+  makeOpenArray(f.mem, byte, f.size)
+
 when isMainModule:
   echo "$# ($#)\p" % [clientId, gitRevision]
 
@@ -1031,3 +1056,29 @@ when isMainModule:
         config.depositWeb3Url,
         config.depositContractAddress,
         config.depositPrivateKey)
+
+  of query:
+    var
+      trieDB = trieDB newChainDb(string config.databaseDir)
+      db = BeaconChainDB.init(trieDB)
+
+    case config.queryCmd
+    of QueryCmd.nimQuery:
+      # TODO: This will handle a simple subset of Nim using
+      #       dot syntax and `[]` indexing.
+      echo "nim query: ", config.nimQueryExpression
+
+    of QueryCmd.get:
+      let pathFragments = config.getQueryPath.split('/', maxsplit = 1)
+      var navigator: DynamicSszNavigator
+
+      case pathFragments[0]
+      of "genesis_state":
+        var genesisMapFile = memfiles.open(config.dataDir/genesisFile)
+        navigator = DynamicSszNavigator.init(genesisMapFile.bytes, BeaconState)
+      else:
+        stderr.write config.getQueryPath & " is not a valid path"
+        quit 1
+
+      echo navigator.navigatePath(pathFragments[1 .. ^1]).toJson
+

beacon_chain/spec/crypto.nim

@@ -221,7 +221,7 @@ else:
     # name from spec!
     ValidatorSig(kind: Real, blsValue: key.sign(domain, msg))
 
-proc fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R =
+func fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R =
   # This is a workaround, so that we can deserialize the serialization of a
   # default-initialized BlsValue without raising an exception
   when defined(ssz_testing):
@@ -229,20 +229,22 @@ proc fromBytes*[T](R: type BlsValue[T], bytes: openarray[byte]): R =
     R(kind: OpaqueBlob, blob: toArray(result.blob.len, bytes))
   else:
     # Try if valid BLS value
-    let success = init(result.blsValue, bytes)
+    # TODO: address the side-effects in nim-blscurve
+    {.noSideEffect.}:
+      let success = init(result.blsValue, bytes)
     if not success:
       # TODO: chronicles trace
       result = R(kind: OpaqueBlob)
       assert result.blob.len == bytes.len
       result.blob[result.blob.low .. result.blob.high] = bytes
 
-proc fromHex*[T](R: type BlsValue[T], hexStr: string): R =
+func fromHex*[T](R: type BlsValue[T], hexStr: string): R =
   fromBytes(R, hexToSeqByte(hexStr))
 
-proc initFromBytes*[T](val: var BlsValue[T], bytes: openarray[byte]) =
+func initFromBytes*[T](val: var BlsValue[T], bytes: openarray[byte]) =
   val = fromBytes(BlsValue[T], bytes)
 
-proc initFromBytes*(val: var BlsCurveType, bytes: openarray[byte]) =
+func initFromBytes*(val: var BlsCurveType, bytes: openarray[byte]) =
   val = init(type(val), bytes)
 
 proc writeValue*(writer: var JsonWriter, value: ValidatorPubKey) {.inline.} =
@@ -329,3 +331,7 @@ proc toGaugeValue*(hash: Eth2Digest): int64 =
   # to the ETH2 metrics spec:
   # https://github.com/ethereum/eth2.0-metrics/blob/6a79914cb31f7d54858c7dd57eee75b6162ec737/metrics.md#interop-metrics
   cast[int64](uint64.fromBytesLE(hash.data[24..31]))
+
+template fromSszBytes*(T: type BlsValue, bytes: openarray[byte]): auto =
+  fromBytes(T, bytes)
+

beacon_chain/ssz.nim

@@ -264,15 +264,12 @@ func writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) =
     buf.appendVarint length
     cursor.writeAndFinalize buf.writtenBytes
 
-template fromSszBytes*(T: type BlsValue, bytes: openarray[byte]): auto =
-  fromBytes(T, bytes)
-
 template fromSszBytes*[T; N](_: type TypeWithMaxLen[T, N],
                              bytes: openarray[byte]): auto =
   mixin fromSszBytes
   fromSszBytes(T, bytes)
 
-proc fromSszBytes*(T: type BlsCurveType, bytes: openarray[byte]): auto =
+func fromSszBytes*(T: type BlsCurveType, bytes: openarray[byte]): auto =
   init(T, bytes)
 
 proc readValue*(r: var SszReader, val: var auto) =

beacon_chain/ssz/bytes_reader.nim

@@ -55,10 +55,10 @@ template fromSszBytes*(T: type enum, bytes: openarray[byte]): auto =
 template fromSszBytes*(T: type BitSeq, bytes: openarray[byte]): auto =
   BitSeq @bytes
 
-proc fromSszBytes*[N](T: type BitList[N], bytes: openarray[byte]): auto =
+func fromSszBytes*[N](T: type BitList[N], bytes: openarray[byte]): auto =
   BitList[N] @bytes
 
-proc readSszValue*(input: openarray[byte], T: type): T =
+func readSszValue*(input: openarray[byte], T: type): T =
   mixin fromSszBytes, toSszType
 
   type T {.used.}= type(result)

beacon_chain/ssz/dynamic_navigator.nim

@@ -1,8 +1,12 @@
 import
-  parseutils,
+  strutils, parseutils,
   faststreams/output_stream, json_serialization/writer,
+  ../spec/datatypes,
   types, bytes_reader, navigator
 
+export
+  navigator
+
 type
   ObjKind = enum
     Record
@@ -73,9 +77,11 @@ proc typeInfo*(T: type): TypeInfo =
   {.gcsafe, noSideEffect.}: res
 
 func genTypeInfo(T: type): TypeInfo =
-  mixin enumAllSerializedFields
-
-  result = when T is object:
+  mixin toSszType, enumAllSerializedFields
+  type SszType = type(toSszType default(T))
+  result = when type(SszType) isnot T:
+    TypeInfo(kind: LeafValue)
+  elif T is object:
     var fields: seq[FieldInfo]
     enumAllSerializedFields(T):
       fields.add FieldInfo(name: fieldName,
@@ -108,13 +114,32 @@ func navigate*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {.
     var idx: int
     let consumed = parseInt(path, idx)
     if consumed == 0 or idx < 0:
-      raise newException(ValueError, "Indexing should be done with natural numbers")
+      raise newException(KeyError, "Indexing should be done with natural numbers")
     return n[idx]
   else:
     doAssert false, "Navigation should be terminated once you reach a leaf value"
 
-func init*(T: type DynamicSszNavigator, bytes: openarray[byte], typ: TypeInfo): T =
-  T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len), typ: typ)
+template navigatePathImpl(nav, iterabalePathFragments: untyped) =
+  result = nav
+  for pathFragment in iterabalePathFragments:
+    if pathFragment.len == 0:
+      continue
+    result = result.navigate(pathFragment)
+    if result.typ.kind == LeafValue:
+      return
+
+func navigatePath*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {.
+                   raises: [Defect, ValueError, MalformedSszError] .} =
+  navigatePathImpl n, split(path, '/')
+
+func navigatePath*(n: DynamicSszNavigator, path: openarray[string]): DynamicSszNavigator {.
+                   raises: [Defect, ValueError, MalformedSszError] .} =
+  navigatePathImpl n, path
+
+func init*(T: type DynamicSszNavigator,
+           bytes: openarray[byte], Navigated: type): T =
+  T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len),
+    typ: typeInfo(Navigated))
 
 func writeJson*(n: DynamicSszNavigator, outStream: OutputStreamVar, pretty = true) =
   n.typ.jsonPrinter(n.m, outStream, pretty)

beacon_chain/ssz/navigator.nim

@@ -95,10 +95,10 @@ func indexVarSizeList(m: MemRange, idx: int): MemRange =
 
   MemRange(startAddr: m.startAddr.shift(elemPos), length: endPos - elemPos)
 
-template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] =
+template indexList(n, idx, T: untyped): untyped =
   type R = T
   mixin toSszType
-  type ElemType = type toSszType(default T)
+  type ElemType = type toSszType(default R)
   when isFixedSize(ElemType):
     const elemSize = fixedPortionSize(ElemType)
     let elemPos = idx * elemSize
@@ -108,8 +108,19 @@ template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] =
   else:
     SszNavigator[R](m: indexVarSizeList(n.m, idx))
 
+template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] =
+  indexList n, idx, T
+
+template `[]`*[R, T](n: SszNavigator[array[R, T]], idx: int): SszNavigator[T] =
+  indexList(n, idx, T)
+
 func `[]`*[T](n: SszNavigator[T]): T =
-  readSszValue(toOpenArray(n.m), T)
+  mixin toSszType, fromSszBytes
+  type SszRepr = type(toSszType default(T))
+  when type(SszRepr) is type(T):
+    readSszValue(toOpenArray(n.m), T)
+  else:
+    fromSszBytes(T, toOpenArray(n.m))
 
 converter derefNavigator*[T](n: SszNavigator[T]): T =
   n[]

scripts/connect_to_testnet.nims

@@ -10,9 +10,11 @@ const
   testnetsRepo = "eth2-testnets"
 
 let
-  testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-testnets")
+  testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-clients")
   testnetsGitUrl = getEnv("ETH2_TESTNETS_GIT_URL", "https://github.com/" & testnetsOrg & "/" & testnetsRepo)
 
+mode = Verbose
+
 proc validateTestnetName(parts: openarray[string]): auto =
   if parts.len != 2:
     echo "The testnet name should have the format `client/network-name`"
@@ -61,6 +63,14 @@ cli do (testnetName {.argument.}: string):
   if fileExists(depositContractFile):
     depositContractOpt = "--deposit-contract=" & readFile(depositContractFile).strip
 
+  if dirExists(dataDir):
+    if fileExists(dataDir/genesisFile):
+      let localGenesisContent = readFile(dataDir/genesisFile)
+      let testnetGenesisContent = readFile(testnetDir/genesisFile)
+      if localGenesisContent != testnetGenesisContent:
+        echo "Detected testnet restart. Deleting previous database..."
+        rmDir dataDir
+
   cd rootDir
   exec &"""nim c {nimFlags} -d:"const_preset={preset}" -o:"{beaconNodeBinary}" beacon_chain/beacon_node.nim"""
   exec replace(&"""{beaconNodeBinary}

tests/test_ssz.nim

@@ -102,7 +102,7 @@ suite "SSZ dynamic navigator":
     var fooOrig = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
     let fooEncoded = SSZ.encode(fooOrig)
 
-    var navFoo = DynamicSszNavigator.init(fooEncoded, typeInfo(Foo))
+    var navFoo = DynamicSszNavigator.init(fooEncoded, Foo)
 
     var navBar = navFoo.navigate("bar")
     check navBar.toJson(pretty = false) == """{"b":"bar","baz":{"i":10}}"""