add basic dependencies and build script for phase0 testing

.gitignore

@@ -0,0 +1,5 @@
+*.pyc
+/__pycache__
+/venv
+
+/build

Makefile

@@ -0,0 +1,15 @@
+SPEC_DIR = ./specs
+SCRIPT_DIR = ./scripts
+BUILD_DIR = ./build
+
+.PHONY: clean all
+
+
+clean:
+	rm -rf $(BUILD_DIR)
+
+
+$(BUILD_DIR)/phase0:
+	mkdir -p $@
+	python3 $(SCRIPT_DIR)/phase0/build_spec.py  $(SPEC_DIR)/core/0_beacon-chain.md $(SCRIPT_DIR)/phase0/minimal_ssz.py \
+		$(SCRIPT_DIR)/phase0/bls_stub.py $(SCRIPT_DIR)/phase0/state_transition.py $(SCRIPT_DIR)/phase0/monkey_patches.py > $@/spec.py

requirements.txt

@@ -0,0 +1,6 @@
+eth-utils>=1.3.0,<2
+eth-typing>=2.1.0,<3.0.0
+oyaml==0.7
+pycryptodome==3.7.3
+py_ecc>=1.6.0
+pytest>=3.6,<3.7

scripts/phase0/bls_stub.py

@@ -0,0 +1,12 @@
+
+
+def bls_verify(pubkey, message_hash, signature, domain):
+    return True
+
+
+def bls_verify_multiple(pubkeys, message_hashes, signature, domain):
+    return True
+
+
+def bls_aggregate_pubkeys(pubkeys):
+    return b'\x42'*96

scripts/phase0/build_spec.py

@@ -0,0 +1,43 @@
+import sys
+import function_puller
+
+code_lines = []
+
+for i in (1, 2, 3, 4, 8, 32, 48, 96):
+    code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i))
+code_lines.append("SLOTS_PER_EPOCH = 64")  # stub, will get overwritten by real var
+code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH")
+
+code_lines.append("""
+from typing import (
+    Any,
+    Callable,
+    List,
+    NewType,
+    Tuple,
+)
+
+
+Slot = NewType('Slot', int)  # uint64
+Epoch = NewType('Epoch', int)  # uint64
+Shard = NewType('Shard', int)  # uint64
+ValidatorIndex = NewType('ValidatorIndex', int)  # uint64
+Gwei = NewType('Gwei', int)  # uint64
+Bytes32 = NewType('Bytes32', bytes)  # bytes32
+BLSPubkey = NewType('BLSPubkey', bytes)  # bytes48
+BLSSignature = NewType('BLSSignature', bytes)  # bytes96
+Any = None
+Store = None
+""")
+
+
+code_lines += function_puller.get_lines(sys.argv[1])
+
+print(open(sys.argv[2]).read())
+print(open(sys.argv[3]).read())
+
+for line in code_lines:
+    print(line)
+
+print(open(sys.argv[4]).read())
+print(open(sys.argv[5]).read())

scripts/phase0/function_puller.py

@@ -0,0 +1,46 @@
+import sys
+
+
+def get_lines(file_name):
+    code_lines = []
+    pulling_from = None
+    current_name = None
+    processing_typedef = False
+    for linenum, line in enumerate(open(sys.argv[1]).readlines()):
+        line = line.rstrip()
+        if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`':
+            current_name = line[line[:-1].rfind('`')+1: -1]
+        if line[:9] == '```python':
+            assert pulling_from is None
+            pulling_from = linenum + 1
+        elif line[:3] == '```':
+            if pulling_from is None:
+                pulling_from = linenum
+            else:
+                if processing_typedef:
+                    assert code_lines[-1] == '}'
+                    code_lines[-1] = '})'
+                pulling_from = None
+                processing_typedef = False
+        else:
+            if pulling_from == linenum and line == '{':
+                code_lines.append('%s = SSZType({' % current_name)
+                processing_typedef = True
+            elif pulling_from is not None:
+                code_lines.append(line)
+            elif pulling_from is None and len(line) > 0 and line[0] == '|':
+                row = line[1:].split('|')
+                if len(row) >= 2:
+                    for i in range(2):
+                        row[i] = row[i].strip().strip('`')
+                        if '`' in row[i]:
+                            row[i] = row[i][:row[i].find('`')]
+                    eligible = True
+                    if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_':
+                        eligible = False
+                    for c in row[0]:
+                        if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789':
+                            eligible = False
+                    if eligible:
+                        code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123567890123456789012357890')))
+    return code_lines

scripts/phase0/minimal_ssz.py

@@ -0,0 +1,190 @@
+from utils.hash import hash
+
+
+BYTES_PER_CHUNK = 32
+BYTES_PER_LENGTH_PREFIX = 4
+ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK
+
+def SSZType(fields):
+    class SSZObject():
+        def __init__(self, **kwargs):
+            for f in fields:
+                if f not in kwargs:
+                    raise Exception("Missing constructor argument: %s" % f)
+                setattr(self, f, kwargs[f])
+
+        def __eq__(self, other):
+            return (
+                self.fields == other.fields and
+                self.serialize() == other.serialize()
+            )
+
+        def __hash__(self):
+            return int.from_bytes(self.hash_tree_root(), byteorder="little")
+
+        def __str__(self):
+            output = []
+            for field in self.fields:
+                output.append(f'{field}: {getattr(self, field)}')
+            return "\n".join(output)
+
+        def serialize(self):
+            return serialize_value(self, self.__class__)
+
+        def hash_tree_root(self):
+            return hash_tree_root(self, self.__class__)
+
+    SSZObject.fields = fields
+    return SSZObject
+
+class Vector(list):
+    def __init__(self, x):
+        list.__init__(self, x)
+        self.length = len(x)
+
+    def append(*args):
+        raise Exception("Cannot change the length of a vector")
+
+    remove = clear = extend = pop = insert = append
+
+def is_basic(typ):
+    return isinstance(typ, str) and (typ[:4] in ('uint', 'bool') or typ == 'byte')
+
+def is_constant_sized(typ):
+    if is_basic(typ):
+        return True
+    elif isinstance(typ, list) and len(typ) == 1:
+        return is_constant_sized(typ[0])
+    elif isinstance(typ, list) and len(typ) == 2:
+        return False
+    elif isinstance(typ, str) and typ[:5] == 'bytes':
+        return len(typ) > 5
+    elif hasattr(typ, 'fields'):
+        for subtype in typ.fields.values():
+            if not is_constant_sized(subtype):
+                return False
+        return True
+    else:
+        raise Exception("Type not recognized")
+
+def coerce_to_bytes(x):
+    if isinstance(x, str):
+        o = x.encode('utf-8')
+        assert len(o) == len(x)
+        return o
+    elif isinstance(x, bytes):
+        return x
+    else:
+        raise Exception("Expecting bytes")
+
+def serialize_value(value, typ=None):
+    if typ is None:
+        typ = infer_type(value)
+    if isinstance(typ, str) and typ[:4] == 'uint':
+        length = int(typ[4:])
+        assert length in (8, 16, 32, 64, 128, 256)
+        return value.to_bytes(length // 8, 'little')
+    elif typ == 'bool':
+        assert value in (True, False)
+        return b'\x01' if value is True else b'\x00'
+    elif (isinstance(typ, list) and len(typ) == 1) or typ == 'bytes':
+        serialized_bytes = coerce_to_bytes(value) if typ == 'bytes' else b''.join([serialize_value(element, typ[0]) for element in value])
+        assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX)
+        serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little')
+        return serialized_length + serialized_bytes
+    elif isinstance(typ, list) and len(typ) == 2:
+        assert len(value) == typ[1]
+        return b''.join([serialize_value(element, typ[0]) for element in value])
+    elif isinstance(typ, str) and len(typ) > 5 and typ[:5] == 'bytes':
+        assert len(value) == int(typ[5:]), (value, int(typ[5:]))
+        return coerce_to_bytes(value)
+    elif hasattr(typ, 'fields'):
+        serialized_bytes = b''.join([serialize_value(getattr(value, field), subtype) for field, subtype in typ.fields.items()])
+        if is_constant_sized(typ):
+            return serialized_bytes
+        else:
+            assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX)
+            serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little')
+            return serialized_length + serialized_bytes
+    else:
+        print(value, typ)
+        raise Exception("Type not recognized")
+
+def chunkify(bytez):
+    bytez += b'\x00' * (-len(bytez) % BYTES_PER_CHUNK)
+    return [bytez[i:i+32] for i in range(0, len(bytez), 32)]
+
+def pack(values, subtype):
+    return chunkify(b''.join([serialize_value(value, subtype) for value in values]))
+
+def is_power_of_two(x):
+    return x > 0 and x & (x-1) == 0
+
+def merkleize(chunks):
+    tree = chunks[::]
+    while not is_power_of_two(len(tree)):
+        tree.append(ZERO_CHUNK)
+    tree = [ZERO_CHUNK] * len(tree) + tree
+    for i in range(len(tree)//2-1, 0, -1):
+        tree[i] = hash(tree[i*2] + tree[i*2+1])
+    return tree[1]
+
+def mix_in_length(root, length):
+    return hash(root + length.to_bytes(32, 'little'))
+
+def infer_type(value):
+    if hasattr(value.__class__, 'fields'):
+        return value.__class__
+    elif isinstance(value, Vector):
+        return [infer_type(value[0]) if len(value) > 0 else 'uint64', len(value)]
+    elif isinstance(value, list):
+        return [infer_type(value[0])] if len(value) > 0 else ['uint64']
+    elif isinstance(value, (bytes, str)):
+        return 'bytes'
+    elif isinstance(value, int):
+        return 'uint64'
+    else:
+        raise Exception("Failed to infer type")
+
+def hash_tree_root(value, typ=None):
+    if typ is None:
+        typ = infer_type(value)
+    if is_basic(typ):
+        return merkleize(pack([value], typ))
+    elif isinstance(typ, list) and len(typ) == 1 and is_basic(typ[0]):
+        return mix_in_length(merkleize(pack(value, typ[0])), len(value))
+    elif isinstance(typ, list) and len(typ) == 1 and not is_basic(typ[0]):
+        return mix_in_length(merkleize([hash_tree_root(element, typ[0]) for element in value]), len(value))
+    elif isinstance(typ, list) and len(typ) == 2 and is_basic(typ[0]):
+        assert len(value) == typ[1]
+        return merkleize(pack(value, typ[0]))
+    elif typ == 'bytes':
+        return mix_in_length(merkleize(chunkify(coerce_to_bytes(value))), len(value))
+    elif isinstance(typ, str) and typ[:5] == 'bytes' and len(typ) > 5:
+        assert len(value) == int(typ[5:])
+        return merkleize(chunkify(coerce_to_bytes(value)))
+    elif isinstance(typ, list) and len(typ) == 2 and not is_basic(typ[0]):
+        return merkleize([hash_tree_root(element, typ[0]) for element in value])
+    elif hasattr(typ, 'fields'):
+        return merkleize([hash_tree_root(getattr(value, field), subtype) for field, subtype in typ.fields.items()])
+    else:
+        raise Exception("Type not recognized")
+
+def truncate(container):
+    field_keys = list(container.fields.keys())
+    truncated_fields = {
+        key: container.fields[key]
+        for key in field_keys[:-1]
+    }
+    truncated_class = SSZType(truncated_fields)
+    kwargs = {
+        field: getattr(container, field)
+        for field in field_keys[:-1]
+    }
+    return truncated_class(**kwargs)
+
+def signed_root(container):
+    return hash_tree_root(truncate(container))
+
+def serialize(ssz_object):
+    return getattr(ssz_object, 'serialize')()

scripts/phase0/monkey_patches.py

@@ -0,0 +1,29 @@
+# Monkey patch validator shuffling cache
+_get_shuffling = get_shuffling
+shuffling_cache = {}
+def get_shuffling(seed: Bytes32,
+                  validators: List[Validator],
+                  epoch: Epoch) -> List[List[ValidatorIndex]]:
+
+    param_hash = (seed, hash_tree_root(validators, [Validator]), epoch)
+
+    if param_hash in shuffling_cache:
+        # print("Cache hit, epoch={0}".format(epoch))
+        return shuffling_cache[param_hash]
+    else:
+        # print("Cache miss, epoch={0}".format(epoch))
+        ret = _get_shuffling(seed, validators, epoch)
+        shuffling_cache[param_hash] = ret
+        return ret
+
+
+# Monkey patch hash cache
+_hash = hash
+hash_cache = {}
+def hash(x):
+    if x in hash_cache:
+        return hash_cache[x]
+    else:
+        ret = _hash(x)
+        hash_cache[x] = ret
+        return ret

scripts/phase0/state_transition.py

@@ -0,0 +1,84 @@
+
+
+def process_transaction_type(state: BeaconState,
+                             transactions: List[Any],
+                             max_transactions: int,
+                             tx_fn: Callable[[BeaconState, Any], None]) -> None:
+    assert len(transactions) <= max_transactions
+    for transaction in transactions:
+        tx_fn(state, transaction)
+
+
+def process_transactions(state: BeaconState, block: BeaconBlock) -> None:
+    process_transaction_type(
+        state,
+        block.body.proposer_slashings,
+        MAX_PROPOSER_SLASHINGS,
+        process_proposer_slashing,
+    )
+    process_transaction_type(
+        state,
+        block.body.attester_slashings,
+        MAX_ATTESTER_SLASHINGS,
+        process_attester_slashing,
+    )
+    process_transaction_type(
+        state,
+        block.body.attestations,
+        MAX_ATTESTATIONS,
+        process_attestation,
+    )
+    process_transaction_type(
+        state,
+        block.body.deposits,
+        MAX_DEPOSITS,
+        process_deposit,
+    )
+    process_transaction_type(
+        state,
+        block.body.voluntary_exits,
+        MAX_VOLUNTARY_EXITS,
+        process_voluntary_exit,
+    )
+    assert len(block.body.transfers) == len(set(block.body.transfers))
+    process_transaction_type(
+        state,
+        block.body.transfers,
+        MAX_TRANSFERS,
+        process_transfer,
+    )
+
+
+def process_block(state: BeaconState,
+                  block: BeaconBlock,
+                  verify_state_root: bool=False) -> None:
+    process_block_header(state, block)
+    process_randao(state, block)
+    process_eth1_data(state, block)
+    process_transactions(state, block)
+    if verify_state_root:
+        verify_block_state_root(state, block)
+
+
+def process_epoch_transition(state: BeaconState) -> None:
+    update_justification_and_finalization(state)
+    process_crosslinks(state)
+    maybe_reset_eth1_period(state)
+    apply_rewards(state)
+    process_ejections(state)
+    update_registry_and_shuffling_data(state)
+    process_slashings(state)
+    process_exit_queue(state)
+    finish_epoch_update(state)
+
+
+def state_transition(state: BeaconState,
+                     block: BeaconBlock,
+                     verify_state_root: bool=False) -> BeaconState:
+    while state.slot < block.slot:
+        cache_state(state)
+        if (state.slot + 1) % SLOTS_PER_EPOCH == 0:
+            process_epoch_transition(state)
+        advance_slot(state)
+        if block.slot == state.slot:
+            process_block(state, block)

tests/phase0/conftest.py

@@ -0,0 +1,6 @@
+import pytest
+from build.phase0 import spec
+
+
+# @pytest.fixture(autouse=True)
+# def build_clean():