POC ssz types spec build + update spec defs, typing still needs work

scripts/phase0/build_spec.py

@@ -12,11 +12,11 @@ from typing import (
     NewType,
     Tuple,
 )
-from eth2spec.utils.minimal_ssz import (
+from eth2spec.utils.ssz.ssz_impl import (
-    SSZType,
     hash_tree_root,
     signing_root,
 )
+from eth2spec.utils.ssz.ssz_typing import *
 from eth2spec.utils.bls_stub import (
     bls_aggregate_pubkeys,
     bls_verify,

scripts/phase0/function_puller.py

@@ -2,6 +2,22 @@ import sys
 from typing import List
 
 
+def translate_ssz_type_line(line: str) -> str:
+    if ':' not in line:
+        return line
+    start = line[:line.index(':')]
+    field_type = line[line.index(':')+2:]
+    if field_type.startswith('['):
+        if ',' in line:
+            # TODO: translate [Foobar, SOME_THING] to Vector[Foobar, SSZLen(SOME_THING)] cleanly.
+            # just brute it here
+            field_type = 'Vector[%s, SSLen(%s)]' % (field_type[1:field_type.index(',')], field_type[field_type.index(',')+2:len(field_type)-1])
+        else:
+            field_type = 'List[%s]' % field_type[1:len(field_type)-1]
+    line = start + ': ' + field_type
+    return line
+
+
 def get_spec(file_name: str) -> List[str]:
     code_lines = []
     pulling_from = None
@@ -21,24 +37,23 @@ def get_spec(file_name: str) -> List[str]:
             else:
                 if current_typedef is not None:
                     assert code_lines[-1] == '}'
-                    code_lines[-1] = '})'
+                    code_lines[-1] = ''
-                    current_typedef[-1] = '})'
+                    code_lines.append('')
-                    type_defs.append((current_name, current_typedef))
                 pulling_from = None
                 current_typedef = None
         else:
             if pulling_from == linenum and line == '{':
-                code_lines.append('%s = SSZType({' % current_name)
+                code_lines.append('class %s(SSZContainer):' % current_name)
-                current_typedef = ['global_vars["%s"] = SSZType({' % current_name]
+                current_typedef = current_name
+                type_defs.append(current_name)
             elif pulling_from is not None:
                 # Add some whitespace between functions
                 if line[:3] == 'def':
                     code_lines.append('')
                     code_lines.append('')
-                code_lines.append(line)
-                # Remember type def lines
                 if current_typedef is not None:
-                    current_typedef.append(line)
+                    line = translate_ssz_type_line(line)
+                code_lines.append(line)
             elif pulling_from is None and len(line) > 0 and line[0] == '|':
                 row = line[1:].split('|')
                 if len(row) >= 2:
@@ -56,16 +71,8 @@ def get_spec(file_name: str) -> List[str]:
                         code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890')))
     # Build type-def re-initialization
     code_lines.append('\n')
-    code_lines.append('def init_SSZ_types():')
-    code_lines.append('    global_vars = globals()')
-    for ssz_type_name, ssz_type in type_defs:
-        code_lines.append('')
-        for type_line in ssz_type:
-            if len(type_line) > 0:
-                code_lines.append('    ' + type_line)
-    code_lines.append('\n')
     code_lines.append('ssz_types = [\n')
-    for (ssz_type_name, _) in type_defs:
+    for ssz_type_name in type_defs:
         code_lines.append(f'    {ssz_type_name},\n')
     code_lines.append(']')
     code_lines.append('\n')

specs/core/0_beacon-chain.md

@@ -270,11 +270,11 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Previous fork version
-    'previous_version': 'bytes4',
+    previous_version: bytes4
     # Current fork version
-    'current_version': 'bytes4',
+    current_version: bytes4
     # Fork epoch number
-    'epoch': 'uint64',
+    epoch: uint64
 }
 ```
 
@@ -283,13 +283,13 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Shard number
-    'shard': 'uint64',
+    shard: uint64
     # Epoch number
-    'epoch': 'uint64',
+    epoch: uint64
     # Root of the previous crosslink
-    'parent_root': 'bytes32',
+    parent_root: bytes32
     # Root of the crosslinked shard data since the previous crosslink
-    'data_root': 'bytes32',
+    data_root: bytes32
 }
 ```
 
@@ -298,11 +298,11 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Root of the deposit tree
-    'deposit_root': 'bytes32',
+    deposit_root: bytes32
     # Total number of deposits
-    'deposit_count': 'uint64',
+    deposit_count: uint64
     # Block hash
-    'block_hash': 'bytes32',
+    block_hash: bytes32
 }
 ```
 
@@ -311,16 +311,16 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # LMD GHOST vote
-    'beacon_block_root': 'bytes32',
+    beacon_block_root: bytes32
-
+    
     # FFG vote
-    'source_epoch': 'uint64',
+    source_epoch: uint64
-    'source_root': 'bytes32',
+    source_root: bytes32
-    'target_epoch': 'uint64',
+    target_epoch: uint64
-    'target_root': 'bytes32',
+    target_root: bytes32
-
+    
     # Crosslink vote
-    'crosslink': Crosslink,
+    crosslink: Crosslink
 }
 ```
 
@@ -329,9 +329,9 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Attestation data
-    'data': AttestationData,
+    data: AttestationData
     # Custody bit
-    'custody_bit': 'bool',
+    custody_bit: bool
 }
 ```
 
@@ -340,12 +340,12 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Validator indices
-    'custody_bit_0_indices': ['uint64'],
+    custody_bit_0_indices: [uint64]
-    'custody_bit_1_indices': ['uint64'],
+    custody_bit_1_indices: [uint64]
     # Attestation data
-    'data': AttestationData,
+    data: AttestationData
     # Aggregate signature
-    'signature': 'bytes96',
+    signature: bytes96
 }
 ```
 
@@ -354,13 +354,13 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # BLS pubkey
-    'pubkey': 'bytes48',
+    pubkey: bytes48
     # Withdrawal credentials
-    'withdrawal_credentials': 'bytes32',
+    withdrawal_credentials: bytes32
     # Amount in Gwei
-    'amount': 'uint64',
+    amount: uint64
     # Container self-signature
-    'signature': 'bytes96',
+    signature: bytes96
 }
 ```
 
@@ -368,11 +368,11 @@ The types are defined topologically to aid in facilitating an executable version
 
 ```python
 {
-    'slot': 'uint64',
+    slot: uint64
-    'parent_root': 'bytes32',
+    parent_root: bytes32
-    'state_root': 'bytes32',
+    state_root: bytes32
-    'body_root': 'bytes32',
+    body_root: bytes32
-    'signature': 'bytes96',
+    signature: bytes96
 }
 ```
 #### `Validator`
@@ -380,21 +380,21 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # BLS public key
-    'pubkey': 'bytes48',
+    pubkey: bytes48
     # Withdrawal credentials
-    'withdrawal_credentials': 'bytes32',
+    withdrawal_credentials: bytes32
     # Epoch when became eligible for activation
-    'activation_eligibility_epoch': 'uint64',
+    activation_eligibility_epoch: uint64
     # Epoch when validator activated
-    'activation_epoch': 'uint64',
+    activation_epoch: uint64
     # Epoch when validator exited
-    'exit_epoch': 'uint64',
+    exit_epoch: uint64
     # Epoch when validator is eligible to withdraw
-    'withdrawable_epoch': 'uint64',
+    withdrawable_epoch: uint64
     # Was the validator slashed
-    'slashed': 'bool',
+    slashed: bool
     # Effective balance
-    'effective_balance': 'uint64',
+    effective_balance: uint64
 }
 ```
 
@@ -403,13 +403,13 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Attester aggregation bitfield
-    'aggregation_bitfield': 'bytes',
+    aggregation_bitfield: bytes
     # Attestation data
-    'data': AttestationData,
+    data: AttestationData
     # Inclusion delay
-    'inclusion_delay': 'uint64',
+    inclusion_delay: uint64
     # Proposer index
-    'proposer_index': 'uint64',
+    proposer_index: uint64
 }
 ```
 
@@ -418,9 +418,9 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Block roots
-    'block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT],
+    block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT]
     # State roots
-    'state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT],
+    state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT]
 }
 ```
 
@@ -431,11 +431,11 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Proposer index
-    'proposer_index': 'uint64',
+    proposer_index: uint64
     # First block header
-    'header_1': BeaconBlockHeader,
+    header_1: BeaconBlockHeader
     # Second block header
-    'header_2': BeaconBlockHeader,
+    header_2: BeaconBlockHeader
 }
 ```
 
@@ -444,9 +444,9 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # First attestation
-    'attestation_1': IndexedAttestation,
+    attestation_1: IndexedAttestation
     # Second attestation
-    'attestation_2': IndexedAttestation,
+    attestation_2: IndexedAttestation
 }
 ```
 
@@ -455,13 +455,13 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Attester aggregation bitfield
-    'aggregation_bitfield': 'bytes',
+    aggregation_bitfield: bytes
     # Attestation data
-    'data': AttestationData,
+    data: AttestationData
     # Custody bitfield
-    'custody_bitfield': 'bytes',
+    custody_bitfield: bytes
     # BLS aggregate signature
-    'signature': 'bytes96',
+    signature: bytes96
 }
 ```
 
@@ -470,11 +470,11 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Branch in the deposit tree
-    'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH],
+    proof: [bytes32, DEPOSIT_CONTRACT_TREE_DEPTH]
     # Index in the deposit tree
-    'index': 'uint64',
+    index: uint64
     # Data
-    'data': DepositData,
+    data: DepositData
 }
 ```
 
@@ -483,11 +483,11 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Minimum epoch for processing exit
-    'epoch': 'uint64',
+    epoch: uint64
     # Index of the exiting validator
-    'validator_index': 'uint64',
+    validator_index: uint64
     # Validator signature
-    'signature': 'bytes96',
+    signature: bytes96
 }
 ```
 
@@ -496,19 +496,19 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Sender index
-    'sender': 'uint64',
+    sender: uint64
     # Recipient index
-    'recipient': 'uint64',
+    recipient: uint64
     # Amount in Gwei
-    'amount': 'uint64',
+    amount: uint64
     # Fee in Gwei for block proposer
-    'fee': 'uint64',
+    fee: uint64
     # Inclusion slot
-    'slot': 'uint64',
+    slot: uint64
     # Sender withdrawal pubkey
-    'pubkey': 'bytes48',
+    pubkey: bytes48
     # Sender signature
-    'signature': 'bytes96',
+    signature: bytes96
 }
 ```
 
@@ -518,15 +518,15 @@ The types are defined topologically to aid in facilitating an executable version
 
 ```python
 {
-    'randao_reveal': 'bytes96',
+    randao_reveal: bytes96
-    'eth1_data': Eth1Data,
+    eth1_data: Eth1Data
-    'graffiti': 'bytes32',
+    graffiti: bytes32
-    'proposer_slashings': [ProposerSlashing],
+    proposer_slashings: [ProposerSlashing]
-    'attester_slashings': [AttesterSlashing],
+    attester_slashings: [AttesterSlashing]
-    'attestations': [Attestation],
+    attestations: [Attestation]
-    'deposits': [Deposit],
+    deposits: [Deposit]
-    'voluntary_exits': [VoluntaryExit],
+    voluntary_exits: [VoluntaryExit]
-    'transfers': [Transfer],
+    transfers: [Transfer]
 }
 ```
 
@@ -535,11 +535,11 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Header
-    'slot': 'uint64',
+    slot: uint64
-    'parent_root': 'bytes32',
+    parent_root: bytes32
-    'state_root': 'bytes32',
+    state_root: bytes32
-    'body': BeaconBlockBody,
+    body: BeaconBlockBody
-    'signature': 'bytes96',
+    signature: bytes96
 }
 ```
 
@@ -550,45 +550,45 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Misc
-    'slot': 'uint64',
+    slot: uint64
-    'genesis_time': 'uint64',
+    genesis_time: uint64
-    'fork': Fork,  # For versioning hard forks
+    fork: Fork  # For versioning hard forks
-
+    
     # Validator registry
-    'validator_registry': [Validator],
+    validator_registry: [Validator]
-    'balances': ['uint64'],
+    balances: [uint64]
-
+    
     # Randomness and committees
-    'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH],
+    latest_randao_mixes: [bytes32, LATEST_RANDAO_MIXES_LENGTH]
-    'latest_start_shard': 'uint64',
+    latest_start_shard: uint64
-
+    
     # Finality
-    'previous_epoch_attestations': [PendingAttestation],
+    previous_epoch_attestations: [PendingAttestation]
-    'current_epoch_attestations': [PendingAttestation],
+    current_epoch_attestations: [PendingAttestation]
-    'previous_justified_epoch': 'uint64',
+    previous_justified_epoch: uint64
-    'current_justified_epoch': 'uint64',
+    current_justified_epoch: uint64
-    'previous_justified_root': 'bytes32',
+    previous_justified_root: bytes32
-    'current_justified_root': 'bytes32',
+    current_justified_root: bytes32
-    'justification_bitfield': 'uint64',
+    justification_bitfield: uint64
-    'finalized_epoch': 'uint64',
+    finalized_epoch: uint64
-    'finalized_root': 'bytes32',
+    finalized_root: bytes32
-
+    
     # Recent state
-    'current_crosslinks': [Crosslink, SHARD_COUNT],
+    current_crosslinks: [Crosslink, SHARD_COUNT]
-    'previous_crosslinks': [Crosslink, SHARD_COUNT],
+    previous_crosslinks: [Crosslink, SHARD_COUNT]
-    'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT],
+    latest_block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT]
-    'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT],
+    latest_state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT]
-    'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH],
+    latest_active_index_roots: [bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH]
     # Balances slashed at every withdrawal period
-    'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH],
+    latest_slashed_balances: [uint64, LATEST_SLASHED_EXIT_LENGTH]
     # `latest_block_header.state_root == ZERO_HASH` temporarily
-    'latest_block_header': BeaconBlockHeader,
+    latest_block_header: BeaconBlockHeader
-    'historical_roots': ['bytes32'],
+    historical_roots: [bytes32]
-
+    
     # Ethereum 1.0 chain data
-    'latest_eth1_data': Eth1Data,
+    latest_eth1_data: Eth1Data
-    'eth1_data_votes': [Eth1Data],
+    eth1_data_votes: [Eth1Data]
-    'deposit_index': 'uint64',
+    deposit_index: uint64
 }
 ```