move spec stuff to spec directory

helps maintain sanity in the face of spec changes

beacon_chain/spec/beaconstate.nim

@@ -0,0 +1,42 @@
+# beacon_chain
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  ../extras,
+  ./datatypes, ./digest, ./helpers, ./validator
+
+func get_shards_and_committees_for_slot*(state: BeaconState,
+                                         slot: uint64
+                                         ): seq[ShardAndCommittee] =
+  let earliest_slot_in_array = state.last_state_recalculation_slot - CYCLE_LENGTH
+  assert earliest_slot_in_array <= slot
+  assert slot < earliest_slot_in_array + CYCLE_LENGTH * 2
+
+  return state.shard_and_committee_for_slots[int slot - earliest_slot_in_array]
+  # TODO, slot is a uint64; will be an issue on int32 arch.
+  #       Clarify with EF if light clients will need the beacon chain
+
+func get_block_hash*(state: BeaconState, current_block: BeaconBlock, slot: int): Eth2Digest =
+  let earliest_slot_in_array = current_block.slot.int - state.recent_block_hashes.len
+  assert earliest_slot_in_array <= slot
+  assert slot < current_block.slot.int
+
+  return state.recent_block_hashes[slot - earliest_slot_in_array]
+
+func get_beacon_proposer*(state: BeaconState, slot: uint64): ValidatorRecord =
+  ## From Casper RPJ mini-spec:
+  ## When slot i begins, validator Vidx is expected
+  ## to create ("propose") a block, which contains a pointer to some parent block
+  ## that they perceive as the "head of the chain",
+  ## and includes all of the **attestations** that they know about
+  ## that have not yet been included into that chain.
+  ##
+  ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee)
+  let
+    first_committee = get_shards_and_committees_for_slot(state, slot)[0].committee
+    index = first_committee[(slot mod len(first_committee).uint64).int]
+  state.validators[index]

beacon_chain/spec/crypto.nim

@@ -0,0 +1,17 @@
+# beacon_chain
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# At the time of writing, the exact definitions of what should be used for
+# cryptography in the spec is in flux, with sizes and test vectors still being
+# hashed out. This layer helps isolate those chagnes.
+
+import
+  milagro_crypto
+
+type
+  Eth2PublicKey* = milagro_crypto.VerKey
+  Eth2Signature* = milagro_crypto.Signature

beacon_chain/spec/datatypes.nim

@@ -5,6 +5,13 @@
 #   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
+# This file contains data types that are part of the spec and thus subject to
+# serialization and spec updates.
+#
+# The spec folder in general contains code that has been hoisted from the
+# specification and that follows the spec as closely as possible, so as to make
+# it easy to keep up-to-date.
+#
 # The latest version can be seen here:
 # https://github.com/ethereum/eth2.0-specs/blob/master/specs/beacon-chain.md
 #
@@ -12,14 +19,8 @@
 # https://github.com/ethereum/eth2.0-specs/compare/126a7abfa86448091a0e037f52966b6a9531a857...master
 
 import
-  intsets, eth_common, math, stint, digest
-
-import milagro_crypto
-  # nimble install https://github.com/status-im/nim-milagro-crypto@#master
-  # Defines
-  #  - SigKey (private/secret key) (48 bytes - 384-bit)
-  #  - Signature                   (48 bytes - 384-bit)
-  #  - VerKey (public key)         (192 bytes)
+  intsets, eth_common, math,
+  ./crypto, ./digest
 
 const
   SHARD_COUNT*                              = 1024 # a constant referring to the number of shards
@@ -55,10 +56,6 @@ const
   INITIAL_FORK_VERSION*                     = 0    #
 
 type
-  # Alias
-  BLSPublicKey* = VerKey
-  BLSsig*       = Signature
-
   Uint24* = range[0'u32 .. 0xFFFFFF'u32] # TODO: wrap-around
 
   BeaconBlock* = object
@@ -72,13 +69,13 @@ type
     state_root*: Eth2Digest                        # State root
     attestations*: seq[AttestationRecord]          # Attestations
     specials*: seq[SpecialRecord]                  # Specials (e.g. logouts, penalties)
-    proposer_signature*: BLSSig                    # Proposer signature
+    proposer_signature*: Eth2Signature             # Proposer signature
 
   AttestationRecord* = object
     data*: AttestationSignedData                   #
     attester_bitfield*: seq[byte]                  # Attester participation bitfield
     poc_bitfield*: seq[byte]                       # Proof of custody bitfield
-    aggregate_sig*: BLSSig                         # BLS aggregate signature
+    aggregate_sig*: Eth2Signature                  # BLS aggregate signature
 
   AttestationSignedData* = object
     slot*: uint64                                 # Slot number
@@ -127,7 +124,7 @@ type
     randao_mix*: Eth2Digest                                # RANDAO state
 
   ValidatorRecord* = object
-    pubkey*: BLSPublicKey                         # BLS public key
+    pubkey*: Eth2PublicKey                        # Public key
     withdrawal_credentials*: Eth2Digest           # Withdrawal credentials
     randao_commitment*: Eth2Digest                # RANDAO commitment
     randao_skips*: uint64                         # Slot the proposer has skipped (ie. layers of RANDAO expected)

beacon_chain/spec/helpers.nim

@@ -5,8 +5,9 @@
 #   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-# Helper functions
-import ../datatypes, ../digest, sequtils, math
+# Uncategorized helper functions from the spec
+
+import ./datatypes, ./digest, sequtils, math
 
 func shuffle*[T](values: seq[T], seed: Eth2Digest): seq[T] =
   ## Returns the shuffled ``values`` with seed as entropy.
@@ -64,24 +65,6 @@ func split*[T](lst: openArray[T], N: Positive): seq[seq[T]] =
   for i in 0 ..< N:
     result[i] = lst[lst.len * i div N ..< lst.len * (i+1) div N] # TODO: avoid alloc via toOpenArray
 
-func get_shards_and_committees_for_slot*(state: BeaconState,
-                                         slot: uint64
-                                         ): seq[ShardAndCommittee] =
-  let earliest_slot_in_array = state.last_state_recalculation_slot - CYCLE_LENGTH
-  assert earliest_slot_in_array <= slot
-  assert slot < earliest_slot_in_array + CYCLE_LENGTH * 2
-
-  return state.shard_and_committee_for_slots[int slot - earliest_slot_in_array]
-  # TODO, slot is a uint64; will be an issue on int32 arch.
-  #       Clarify with EF if light clients will need the beacon chain
-
-func get_block_hash*(state: BeaconState, current_block: BeaconBlock, slot: int): Eth2Digest =
-  let earliest_slot_in_array = current_block.slot.int - state.recent_block_hashes.len
-  assert earliest_slot_in_array <= slot
-  assert slot < current_block.slot.int
-
-  return state.recent_block_hashes[slot - earliest_slot_in_array]
-
 func get_new_recent_block_hashes*(old_block_hashes: seq[Eth2Digest],
                                   parent_slot, current_slot: int64,
                                   parent_hash: Eth2Digest
@@ -93,16 +76,3 @@ func get_new_recent_block_hashes*(old_block_hashes: seq[Eth2Digest],
   for _ in 0 ..< min(d, old_block_hashes.len):
     result.add parent_hash
 
-func get_beacon_proposer*(state: BeaconState, slot: uint64): ValidatorRecord =
-  ## From Casper RPJ mini-spec:
-  ## When slot i begins, validator Vidx is expected
-  ## to create ("propose") a block, which contains a pointer to some parent block
-  ## that they perceive as the "head of the chain",
-  ## and includes all of the **attestations** that they know about
-  ## that have not yet been included into that chain.
-  ##
-  ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee)
-  let
-    first_committee = get_shards_and_committees_for_slot(state, slot)[0].committee
-    index = first_committee[(slot mod len(first_committee).uint64).int]
-  state.validators[index]

beacon_chain/spec/validator.nim

@@ -9,7 +9,7 @@
 import
   options,
   eth_common,
-  ./datatypes, ./digest, ./private/helpers
+  ./crypto, ./datatypes, ./digest, ./helpers
 
 func min_empty_validator(validators: seq[ValidatorRecord], current_slot: uint64): Option[int] =
   for i, v in validators:
@@ -17,8 +17,7 @@ func min_empty_validator(validators: seq[ValidatorRecord], current_slot: uint64)
           return some(i)
 
 func add_validator*(validators: var seq[ValidatorRecord],
-                    fork_data: ForkData,
-                    pubkey: BLSPublicKey,
+                    pubkey: Eth2PublicKey,
                     proof_of_possession: seq[byte],
                     withdrawal_credentials: Eth2Digest,
                     randao_commitment: Eth2Digest,

beacon_chain/ssz.nim

@@ -12,7 +12,7 @@
 import
   endians, typetraits, options, algorithm,
   eth_common, nimcrypto/blake2,
-  ./datatypes, ./digest
+  ./spec/[crypto, datatypes, digest]
 
 from milagro_crypto import getRaw
 
@@ -226,7 +226,7 @@ func hashSSZ*(x: enum): array[32, byte] =
   withHash:
     h.update [uint8 x]
 
-func hashSSZ*(x: BLSsig): array[32, byte] =
+func hashSSZ*(x: Eth2Signature): array[32, byte] =
   ## TODO - Warning ⚠️: not part of the spec
   ## as of https://github.com/ethereum/beacon_chain/pull/133/files
   ## This is a "stub" needed for BeaconBlock hashing

beacon_chain/state_transition.nim

@@ -21,7 +21,7 @@
 {.warning: "The official spec at https://notes.ethereum.org/SCIg8AH5SA-O4C1G1LYZHQ# is not fully defining state transitions.".}
 
 import
-  ./datatypes, ./digest, ./private/helpers,
+  ./spec/[beaconstate, crypto, datatypes, digest, helpers],
   intsets, endians, nimcrypto,
   milagro_crypto # nimble install https://github.com/status-im/nim-milagro-crypto@#master
 
@@ -57,7 +57,7 @@ func process_block*(active_state: BeaconState, crystallized_state: BeaconState,
     doAssert attestation.attester_bitfield.len == attestation_indices.committee.len
 
     # Derive a group public key by adding the public keys of all of the attesters in attestation_indices for whom the corresponding bit in attester_bitfield (the ith bit is (attester_bitfield[i // 8] >> (7 - (i %8))) % 2) equals 1
-    var agg_pubkey: BLSPublicKey
+    var agg_pubkey: Eth2PublicKey
     var empty = true
     for attester_idx in attestation_indices.committee:
       # TODO re-enable, but currently this whole function's a nonfunctional stub

tests/test_block_processing.nim

@@ -7,7 +7,8 @@
 
 import
   unittest,
-  ../beacon_chain/[datatypes, state_transition]
+  ../beacon_chain/spec/datatypes,
+  ../beacon_chain/state_transition
 
 suite "Block processing":
   ## For now just test that we can compile and execute block processing with mock data.

tests/test_ssz.nim

@@ -7,7 +7,7 @@
 
 import
   unittest, nimcrypto, eth_common, sequtils, options,
-  ../beacon_chain/[datatypes, ssz]
+  ../beacon_chain/ssz, ../beacon_chain/spec/datatypes
 
 func filled[N: static[int], T](typ: type array[N, T], value: T): array[N, T] =
   for val in result.mitems:

tests/test_validator.nim

@@ -6,7 +6,7 @@
 
 import
   math,unittest, sequtils,
-  ../beacon_chain/[datatypes, digest, validator]
+  ../beacon_chain/spec/[datatypes, digest, validator]
 
 func sumCommittees(v: openArray[seq[ShardAndCommittee]]): int =
   for x in v: