Ssz fixes (#566)

* SSZ signature from EF are always opaque blobs (security issue - https://github.com/status-im/nim-beacon-chain/issues/555)
Enable
- Attestation
- Beaconstate (minimal only)
- Deposit
- DepositData
- ProposerSlashing

Updates #518

* mv debug_ssz to helpers

* Small reorg of the list types

* Fix IndexedAttestation, AttesterSlashing and BeaconBlock

* Deactivate on mainnet: AttesterSlashing, BeaconBlockBody, IndexedAttestation, Attestation, BeaconBlock

* Fix Validators on minimal and mainnet

beacon_chain/spec/datatypes.nim

@@ -352,15 +352,20 @@ macro fieldMaxLen*(x: typed): untyped =
     return newLit(0)
 
   let size = case $x[1]
+             # Obsolete
              of "pubkeys",
                 "compact_validators",
                 "aggregation_bits",
                 "custody_bits": int64(MAX_VALIDATORS_PER_COMMITTEE)
+             # IndexedAttestation
+             of "attesting_indices": MAX_VALIDATORS_PER_COMMITTEE
+             # BeaconBlockBody
              of "proposer_slashings": MAX_PROPOSER_SLASHINGS
              of "attester_slashings": MAX_ATTESTER_SLASHINGS
              of "attestations": MAX_ATTESTATIONS
              of "deposits": MAX_DEPOSITS
              of "voluntary_exits": MAX_VOLUNTARY_EXITS
+             # BeaconState
              of "historical_roots": HISTORICAL_ROOTS_LIMIT
              of "eth1_data_votes": SLOTS_PER_ETH1_VOTING_PERIOD
              of "validators": VALIDATOR_REGISTRY_LIMIT

tests/helpers/debug_ssz.nim

@@ -0,0 +1,82 @@
+# beacon_chain
+# Copyright (c) 2018-Present Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  # Standard library
+  os, unittest, strutils, streams,
+  # Status libraries
+  stint, stew/bitseqs,
+  # Third-party
+  yaml,
+  # Beacon chain internals
+  ../../beacon_chain/spec/[datatypes, digest, crypto],
+  ../../beacon_chain/ssz
+
+# Config
+# ---------------------------------------------
+
+const
+  FixturesDir = currentSourcePath.rsplit(DirSep, 1)[0] / ".." / "official"/"fixtures"
+  SSZDir = FixturesDir/"tests-v0.9.1"/const_preset/"phase0"/"ssz_static"
+  UnitTestDir = SSZDir/"Validator"/"ssz_zero"/"case_0"
+
+type
+  SSZHashTreeRoot = object
+    # The test files have the values at the "root"
+    # so we **must** use "root" as a field name
+    root: string
+    # Some have a signing_root field
+    signing_root: string
+
+# Make signing root optional
+setDefaultValue(SSZHashTreeRoot, signing_root, "")
+
+# Parsing + Test
+# ---------------------------------------------
+
+type Skip = enum
+  SkipNone
+  SkipHashTreeRoot
+  SkipSigningRoot
+
+proc checkSSZ(T: typedesc, dir: string, expectedHash: SSZHashTreeRoot, skip = SkipNone) =
+  # Deserialize into a ref object to not fill Nim stack
+  var deserialized: ref T
+  new deserialized
+  deserialized[] = SSZ.loadFile(dir/"serialized.ssz", T)
+
+  echo "\n\nObject: ", T
+  echo "---------------------------------------"
+  echo deserialized[]
+
+  if not(skip == SkipHashTreeRoot):
+    check: expectedHash.root == "0x" & toLowerASCII($deserialized.hashTreeRoot())
+  if expectedHash.signing_root != "" and not(skip == SkipSigningRoot):
+    check: expectedHash.signing_root == "0x" & toLowerASCII($deserialized[].signingRoot())
+
+proc loadExpectedHashTreeRoot(dir: string): SSZHashTreeRoot =
+  var s = openFileStream(dir/"roots.yaml")
+  yaml.load(s, result)
+  s.close()
+
+# Manual checks
+# ---------------------------------------------
+
+# Compile with -d:ssz_testing for consensus objects
+# as they are always an Opaque Blob even if they might seem like a valid BLS signature
+
+echo "Current preset: ", const_preset
+
+let hash = loadExpectedHashTreeRoot(UnitTestDir)
+checkSSZ(Validator, UnitTestDir, hash)
+
+echo "\n\n"
+var deserialized: ref Validator
+new deserialized
+deserialized[] = SSZ.loadFile(UnitTestDir/"serialized.ssz", Validator)
+
+echo deserialized[].hashTreeRoot()

tests/official/test_fixture_ssz_consensus_objects.nim

@@ -43,28 +43,34 @@ setDefaultValue(SSZHashTreeRoot, signing_root, "")
 # Checking the values against the yaml file is TODO (require more flexible Yaml parser)
 const Unsupported = toHashSet([
     "AggregateAndProof",    # Type for signature aggregation - not implemented
-    "Attestation",          # RangeError on deserialization
+    # "Attestation",        #
-    # "AttestationData",
+    # "AttestationData",    #
-    "AttesterSlashing",     # RangeError on deserialization
+    # "AttesterSlashing",   #
-    "BeaconBlock",          # RangeError on deserialization
+    # "BeaconBlock",        #
-    "BeaconBlockBody",      # RangeError on deserialization
+    # "BeaconBlockBody",    #
-    # "BeaconBlockHeader",  # HashTreeRoot KO - SigningRook OK
+    # "BeaconBlockHeader",  #
-    "BeaconState",          # HashTreeRoot KO
+    # "BeaconState",        #
-    # "Checkpoint",
+    # "Checkpoint",         #
-    "Deposit",              # HashTreeRoot KO
+    # "Deposit",            #
-    "DepositData",          # HashTreeRoot KO - SigningRoot KO
+    # "DepositData",        #
-    # "Eth1Data",
+    # "Eth1Data",           #
-    # "Fork",
+    # "Fork",               #
-    # "HistoricalBatch",    # OK
+    # "HistoricalBatch",    #
-    "IndexedAttestation",   # RangeError on deserialization
+    # "IndexedAttestation", #
-    # "PendingAttestation", # OK
+    # "PendingAttestation", #
-    "ProposerSlashing",     # HashTreeRoot KO
+    # "ProposerSlashing",   #
-    "Validator",            # HashTreeRoot KO
+    # "Validator",            # HashTreeRoot KO
-    # "VoluntaryExit"       # hashTreeRoot KO - SigningRoot OK
+    # "VoluntaryExit"       #
   ])
 
 const UnsupportedMainnet = toHashSet([
     "PendingAttestation",   # HashTreeRoot KO
+    "BeaconState",
+    "AttesterSlashing",
+    "BeaconBlockBody",
+    "IndexedAttestation",
+    "Attestation",
+    "BeaconBlock"
   ])
 
 type Skip = enum
@@ -108,10 +114,6 @@ proc runSSZtests() =
           discard
         continue
 
-    let signingRootOnly = &"                     ↶↶↶ {sszType} - Skipping HashTreeRoot and testing SigningRoot only"
-    if sszType == "BeaconBlockHeader" or sszType == "VoluntaryExit":
-      echo signingRootOnly
-
     test &"  Testing    {sszType:20}   consensus object ✓✓✓":
       let path = SSZDir/sszType
       for pathKind, sszTestKind in walkDir(path, relative = true):
@@ -128,7 +130,7 @@ proc runSSZtests() =
           of "AttesterSlashing": checkSSZ(AttesterSlashing, path, hash)
           of "BeaconBlock": checkSSZ(BeaconBlock, path, hash)
           of "BeaconBlockBody": checkSSZ(BeaconBlockBody, path, hash)
-          of "BeaconBlockHeader": checkSSZ(BeaconBlockHeader, path, hash, SkipHashTreeRoot) # TODO
+          of "BeaconBlockHeader": checkSSZ(BeaconBlockHeader, path, hash)
           of "BeaconState": checkSSZ(BeaconState, path, hash)
           of "Checkpoint": checkSSZ(Checkpoint, path, hash)
           of "Deposit": checkSSZ(Deposit, path, hash)
@@ -139,8 +141,8 @@ proc runSSZtests() =
           of "IndexedAttestation": checkSSZ(IndexedAttestation, path, hash)
           of "PendingAttestation": checkSSZ(PendingAttestation, path, hash)
           of "ProposerSlashing": checkSSZ(ProposerSlashing, path, hash)
-          of "Validator": checkSSZ(VoluntaryExit, path, hash)
+          of "Validator": checkSSZ(Validator, path, hash)
-          of "VoluntaryExit": checkSSZ(VoluntaryExit, path, hash, SkipHashTreeRoot) # TODO
+          of "VoluntaryExit": checkSSZ(VoluntaryExit, path, hash)
           else:
             raise newException(ValueError, "Unsupported test: " & sszType)
 

tests/official/test_fixture_ssz_consensus_objects.nim.cfg

@@ -0,0 +1 @@
+-d:ssz_testing