Fixes getting us closer to a valid genesis state

* More +1 fixes for sszLists
* A script for comparing our genesis state against zcli
* Fixes for the interop scripts so they can work on macOS

beacon_chain/beacon_node.nim

@@ -797,6 +797,8 @@ proc start(node: BeaconNode, headState: BeaconState) =
   node.addLocalValidators(headState)
   node.run()
 
+import serialization/testing/tracing
+
 when isMainModule:
   randomize()
   let config = BeaconNodeConf.load(version = fullVersionStr())
@@ -833,14 +835,23 @@ when isMainModule:
       initialState = initialize_beacon_state_from_eth1(
         eth1BlockHash, startTime, deposits, {skipValidation})
 
+    if defined(serialization_tracing):
+      startTime = 1567816020
+
     # https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
     initialState.genesis_time = startTime
 
     doAssert initialState.validators.len > 0
 
-    Json.saveFile(config.outputGenesis.string, initialState, pretty = true)
+    Json.saveFile(config.outputGenesis.string , initialState, pretty = true)
     echo "Wrote ", config.outputGenesis.string
 
+    let sszGenesis = config.outputGenesis.string.changeFileExt "ssz"
+    SSZ.saveFile(sszGenesis, initialState)
+    echo "Wrote ", sszGenesis
+
+    echo "Genesis state eth1_deposit index: ", initialState.eth1_deposit_index
+
     var
       bootstrapAddress = getPersistenBootstrapAddr(
         config, parseIpAddress(config.bootstrapAddress), Port config.bootstrapPort)
@@ -859,6 +870,10 @@ when isMainModule:
     Json.saveFile(config.outputNetwork.string, testnetMetadata, pretty = true)
     echo "Wrote ", config.outputNetwork.string
 
+    when defined(serialization_tracing):
+      tracingEnabled = true
+      echo hash_tree_root(BeaconBlockBody())
+
   of updateTestnet:
     discard waitFor updateTestnetMetadata(config)
 

beacon_chain/spec/beaconstate.nim

@@ -230,18 +230,18 @@ func initialize_beacon_state_from_eth1*(
       Eth1Data(block_hash: eth1_block_hash, deposit_count: uint64(len(deposits))),
     latest_block_header:
       BeaconBlockHeader(
-        body_root: hash_tree_root(BeaconBlockBody()),
-        # TODO - Pure BLSSig cannot be zero: https://github.com/status-im/nim-beacon-chain/issues/374
-        signature: BlsValue[Signature](kind: OpaqueBlob)
-      )
-  )
+        body_root: hash_tree_root(BeaconBlockBody(
+          # TODO: This shouldn't be necessary if OpaqueBlob is the default
+          randao_reveal: ValidatorSig(kind: OpaqueBlob))),
+        # TODO: This shouldn't be necessary if OpaqueBlob is the default
+        signature: BlsValue[Signature](kind: OpaqueBlob)))
 
   # Process deposits
   let leaves = deposits.mapIt(it.data)
   for i, deposit in deposits:
     let deposit_data_list = leaves[0..i]
     state.eth1_data.deposit_root = hash_tree_root(
-      sszList(deposit_data_list, 2'i64^DEPOSIT_CONTRACT_TREE_DEPTH))
+      sszList(deposit_data_list, (2'i64^DEPOSIT_CONTRACT_TREE_DEPTH) + 1))
 
     discard process_deposit(state, deposit, flags)
 
@@ -262,7 +262,7 @@ func initialize_beacon_state_from_eth1*(
   let active_index_root = hash_tree_root(
     sszList(
       get_active_validator_indices(state, GENESIS_EPOCH),
-      VALIDATOR_REGISTRY_LIMIT))
+      VALIDATOR_REGISTRY_LIMIT + 1))
 
   let committee_root = get_compact_committees_root(state, GENESIS_EPOCH)
   for index in 0 ..< EPOCHS_PER_HISTORICAL_VECTOR:

beacon_chain/spec/crypto.nim

@@ -60,6 +60,9 @@ export
 
 type
   BlsValueType* = enum
+    # TODO `OpaqueBlob` should probably be the default, because a default
+    # constructed value is not a valid signature and it must be serialized
+    # as zeros according to the SSZ spec.
     Real
     OpaqueBlob
 
@@ -67,13 +70,13 @@ type
     # TODO This is a temporary type needed until we sort out the
     # issues with invalid BLS values appearing in the SSZ test suites.
     case kind*: BlsValueType
-    of Real:
-      blsValue*: T
     of OpaqueBlob:
       when T is blscurve.Signature:
         blob*: array[96, byte]
       else:
         blob*: array[48, byte]
+    of Real:
+      blsValue*: T
 
   ValidatorPubKey* = BlsValue[blscurve.VerKey]
   # ValidatorPubKey* = blscurve.VerKey

interop/diff_genesis_vs_lighthouse.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Lighthouse genesis state
+curl localhost:5052/beacon/state?slot=0 | python -m json.tool | sed 's/"0x/"/' > /tmp/lighthouse_state.json
+
+# Format nimbus the same
+cat data/state_snapshot.json | python -m json.tool | sed 's/"0x/"/' > /tmp/nimbus_state.json
+
+diff -uw /tmp/nimbus_state.json /tmp/lighthouse_state.json
+

interop/diff_genesis_vs_zcli.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+ETH2_PM=${ETH2_PM_PATH:-"eth2.0-pm"}
+
+set -eu
+
+echo Locating zcli...
+if ! command -v zcli; then
+  go get -tags preset_minimal github.com/protolambda/zcli
+fi
+
+if [[ ! -d "$ETH2_PM" ]]; then
+  git clone https://github.com/ethereum/eth2.0-pm "$ETH2_PM"
+fi
+
+# Fetch genesis time, as set up by start.sh
+if command -v jq; then
+  genesis_time=$(jq '.genesis_time' data/state_snapshot.json)
+else
+  genesis_time=$(grep -oP '(?<=genesis_time": )\w+(?=,)' data/state_snapshot.json)
+fi
+
+echo Genesis time was $genesis_time
+
+zcli genesis mock \
+  --count 16 \
+  --genesis-time $genesis_time \
+  --keys "${ETH2_PM}/interop/mocked_start/keygen_10000_validators.yaml" \
+  --out data/zcli_genesis.ssz
+
+zcli diff state data/zcli_genesis.ssz data/state_snapshot.ssz
+

interop/run_lighthouse.sh

@@ -5,21 +5,58 @@
 
 # https://github.com/sigp/lighthouse/blob/master/docs/interop.md
 
+set -eu
+
+echo Locating protoc...
+if ! command -v protoc; then
+  MSG="protoc (the Google Protobuf compiler) is missing. Please install it manually"
+  if [[ "$OSTYPE" == "linux-gnu" ]]; then
+    MSG+=" with sudo apt install protobuf-compiler"
+  elif [[ "$OSTYPE" == "darwin"* ]]; then
+    MSG+=" with 'brew install protobuf'"
+  elif [[ "$OSTYPE" == "cygwin" ]]; then
+    # POSIX compatibility layer and Linux environment emulation for Windows
+    MSG+=""
+  elif [[ "$OSTYPE" == "msys" ]]; then
+    # Lightweight shell and GNU utilities compiled for Windows (part of MinGW)
+    MSG+=""
+  elif [[ "$OSTYPE" == "win32" ]]; then
+    # I'm not sure this can happen.
+    MSG+=""
+  elif [[ "$OSTYPE" == "freebsd"* ]]; then
+    # ...
+    MSG+=""
+  else
+    # Unknown.
+    MSG+=""
+  fi
+  echo $MSG
+  exit 1
+fi
+
 cargo_path=$(which cargo)
 [[ -x "$cargo_path" ]] || { echo "install rust first (https://rust-lang.org)"; exit 1; }
 
-[[ -d "lighthouse" ]] || {
-  git clone https://github.com/sigp/lighthouse.git
-  cd lighthouse
+LIGHTHOUSE=${LIGHTHOSE_PATH:-"lighthouse"}
+
+[[ -d "$LIGHTHOUSE" ]] || {
+  git clone https://github.com/sigp/lighthouse.git "$LIGHTHOUSE"
+  pushd "$LIGHTHOUSE"
   git checkout interop # temporary interop branch - will get merged soon I expect!
   cargo update
-  cd ..
+  popd
 }
 
 # Fetch genesis time, as set up by start.sh
-genesis_time=$(grep -oP '(?<=genesis_time": )\w+(?=,)' data/state_snapshot.json)
+if command -v jq; then
+  genesis_time=$(jq '.genesis_time' data/state_snapshot.json)
+else
+  genesis_time=$(grep -oP '(?<=genesis_time": )\w+(?=,)' data/state_snapshot.json)
+fi
 
-cd lighthouse
+echo Genesis time was $genesis_time
+
+cd "$LIGHTHOUSE"
 cargo build
 
 cd target/debug