introduce Eth2Hash, Eth2Digest and friends (#22, fixes #3)

* introduce Eth2Hash, Eth2Digest and friends
This commit is contained in:
Jacek Sieka 2018-11-27 17:10:09 -06:00 committed by GitHub
parent 41d45d4a67
commit 548b6922ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 127 additions and 93 deletions

View File

@ -12,7 +12,7 @@
# https://github.com/ethereum/eth2.0-specs/compare/98312f40b5742de6aa73f24e6225ee68277c4614...master
import
intsets, eth_common, math, stint
intsets, eth_common, math, stint, digest
import milagro_crypto
# nimble install https://github.com/status-im/nim-milagro-crypto@#master
@ -56,7 +56,6 @@ type
BLSPublicKey* = VerKey
BLSsig* = Signature
Blake2_256_Digest* = Hash256 # TODO change to Blake2b-512[0 ..< 32] see https://github.com/status-im/nim-beacon-chain/issues/3
Uint24* = range[0'u32 .. 0xFFFFFF'u32] # TODO: wrap-around
SpecialRecord* = object
@ -65,13 +64,13 @@ type
BeaconBlock* = object
slot*: uint64 # Slot number
randao_reveal*: Blake2_256_Digest # Proposer RANDAO reveal
candidate_pow_receipt_root*: Blake2_256_Digest # Recent PoW chain reference (receipt root)
ancestor_hashes*: seq[Blake2_256_Digest] # Skip list of previous beacon block hashes
randao_reveal*: Eth2Digest # Proposer RANDAO reveal
candidate_pow_receipt_root*: Eth2Digest # Recent PoW chain reference (receipt root)
ancestor_hashes*: seq[Eth2Digest] # Skip list of previous beacon block hashes
# i'th item is most recent ancestor whose
# slot is a multiple of 2**i for
# i == 0, ..., 31
state_root*: Blake2_256_Digest # State root
state_root*: Eth2Digest # State root
attestations*: seq[AttestationRecord] # Attestations
specials*: seq[SpecialRecord] # Specials (e.g. logouts, penalties)
proposer_signature*: BLSSig # Proposer signature
@ -80,16 +79,16 @@ type
fork_version*: uint64 # Fork version
slot*: uint64 # Slot number
shard_id*: uint64 # Shard ID (or `2**64 - 1` for beacon chain)
block_hash*: Blake2_256_Digest # Block hash
block_hash*: Eth2Digest # Block hash
AttestationSignedData* = object
fork_version*: uint64 # Fork version
slot*: uint64 # Slot number
shard*: uint16 # Shard number
parent_hashes*: seq[Blake2_256_Digest] # CYCLE_LENGTH parent hashes
shard_block_hash*: Blake2_256_Digest # Shard block hash
last_crosslink_hash*: Blake2_256_Digest # Last crosslink hash
shard_block_combined_data_root*: Blake2_256_Digest
parent_hashes*: seq[Eth2Digest] # CYCLE_LENGTH parent hashes
shard_block_hash*: Eth2Digest # Shard block hash
last_crosslink_hash*: Eth2Digest # Last crosslink hash
shard_block_combined_data_root*: Eth2Digest
# Root of data between last hash and this one
justified_slot*: uint64 # Slot of last justified beacon block referenced in the attestation
@ -103,21 +102,21 @@ type
slot*: uint64 # When
CrosslinkRecord* = object
slot: uint64 # Slot number
hash: Blake2_256_Digest # Shard chain block hash
slot*: uint64 # Slot number
hash*: Eth2Digest # Shard chain block hash
AttestationRecord* = object
slot*: uint64 # Slot number
shard*: uint16 # Shard number
oblique_parent_hashes*: seq[Blake2_256_Digest]
oblique_parent_hashes*: seq[Eth2Digest]
# Beacon block hashes not part of the current chain, oldest to newest
shard_block_hash*: Blake2_256_Digest # Shard block hash being attested to
last_crosslink_hash*: Blake2_256_Digest # Last crosslink hash
shard_block_combined_data_root*: Blake2_256_Digest
shard_block_hash*: Eth2Digest # Shard block hash being attested to
last_crosslink_hash*: Eth2Digest # Last crosslink hash
shard_block_combined_data_root*: Eth2Digest
# Root of data between last hash and this one
attester_bitfield*: IntSet # Attester participation bitfield (1 bit per attester)
justified_slot*: uint64 # Slot of last justified beacon block
justified_block_hash*: Blake2_256_Digest # Hash of last justified beacon block
justified_block_hash*: Eth2Digest # Hash of last justified beacon block
aggregate_sig*: BLSSig # BLS aggregate signature
BeaconState* = object
@ -133,26 +132,26 @@ type
## worth of assignments
persistent_committees*: seq[seq[ValidatorRecord]] # Persistent shard committees
persistent_committee_reassignments*: seq[ShardReassignmentRecord]
next_shuffling_seed*: Blake2_256_Digest # Randao seed used for next shuffling
next_shuffling_seed*: Eth2Digest # Randao seed used for next shuffling
deposits_penalized_in_period*: uint32 # Total deposits penalized in the given withdrawal period
validator_set_delta_hash_chain*: Blake2_256_Digest # Hash chain of validator set changes (for light clients to easily track deltas)
validator_set_delta_hash_chain*: Eth2Digest # Hash chain of validator set changes (for light clients to easily track deltas)
current_exit_seq*: uint64 # Current sequence number for withdrawals
genesis_time*: uint64 # Genesis time
known_pow_receipt_root*: Blake2_256_Digest # PoW chain reference
candidate_pow_receipt_root*: Blake2_256_Digest
candidate_pow_receipt_root_votes*: Blake2_256_Digest
known_pow_receipt_root*: Eth2Digest # PoW chain reference
candidate_pow_receipt_root*: Eth2Digest
candidate_pow_receipt_root_votes*: Eth2Digest
pre_fork_version*: uint32 # Parameters relevant to hard forks / versioning.
post_fork_version*: uint32 # Should be updated only by hard forks.
fork_slot_number*: uint64
pending_attestations*: seq[AttestationRecord] # Attestations not yet processed
recent_block_hashes*: seq[Blake2_256_Digest] # recent beacon block hashes needed to process attestations, older to newer
randao_mix*: Blake2_256_Digest # RANDAO state
recent_block_hashes*: seq[Eth2Digest] # recent beacon block hashes needed to process attestations, older to newer
randao_mix*: Eth2Digest # RANDAO state
ValidatorRecord* = object
pubkey*: BLSPublicKey # BLS public key
withdrawal_shard*: uint16 # Withdrawal shard number
withdrawal_address*: EthAddress # Withdrawal address
randao_commitment*: Blake2_256_Digest # RANDAO commitment
randao_commitment*: Eth2Digest # RANDAO commitment
randao_last_change*: uint64 # Slot the RANDAO commitment was last changed
balance*: uint64 # Balance in Gwei
status*: ValidatorStatusCodes # Status code
@ -164,7 +163,7 @@ type
proof_of_possession*: seq[byte]
withdrawal_shard*: uint16
withdrawal_address*: EthAddress
randao_commitment*: Blake2_256_Digest
randao_commitment*: Eth2Digest
ValidatorStatusCodes* {.pure.} = enum
PendingActivation = 0

43
beacon_chain/digest.nim Normal file
View File

@ -0,0 +1,43 @@
# beacon_chain
# Copyright (c) 2018 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# Serenity hash function / digest
#
# From spec:
#
# We aim to have a STARK-friendly hash function `hash(x)` for the production
# launch of the beacon chain. While the standardisation process for a
# STARK-friendly hash function takes place—led by STARKware, who will produce a
# detailed report with recommendations—we use `BLAKE2b-512` as a placeholder.
# Specifically, we set `hash(x) := BLAKE2b-512(x)[0:32]` where the `BLAKE2b-512`
# algorithm is defined in [RFC 7693](https://tools.ietf.org/html/rfc7693) and
# the input `x` is of type `bytes`.
#
# In our code base, to enable a smooth transition, we call this function
# `eth2hash`, and it outputs a `Eth2Digest`. Easy to sed :)
import nimcrypto/[blake2, hash]
type
Eth2Digest* = MDigest[32 * 8] ## `hash32` from spec
Eth2Hash* = blake2_512 ## Context for hash function
func eth2hash*(v: openArray[byte]): Eth2Digest =
var tmp = Eth2Hash.digest v
copyMem(result.data.addr, tmp.addr, sizeof(result))
template withEth2Hash*(body: untyped): Eth2Digest =
## This little helper will init the hash function and return the sliced
## hash:
## let hashOfData = withHash: h.update(data)
var h {.inject.}: Eth2Hash
h.init()
body
var res: Eth2Digest
var tmp = h.finish()
copyMem(res.data.addr, tmp.data.addr, sizeof(res))
res

View File

@ -42,8 +42,8 @@ func add_to_multiset[K, V](
v: V or seq[V]) =
multiset.mgetOrPut(k, @[]).add v
func change_head(self: Node, chain: var seq[MDigest[256]], new_head: Block) =
chain.add newSeq[MDigest[256]](new_head.height + 1 - chain.len)
func change_head(self: Node, chain: var seq[Eth2Digest], new_head: Block) =
chain.add newSeq[Eth2Digest](new_head.height + 1 - chain.len)
var (i, c) = (new_head.height, new_head.hash)
while c != chain[i]:
chain[i] = c
@ -55,8 +55,8 @@ func change_head(self: Node, chain: var seq[MDigest[256]], new_head: Block) =
func recalculate_head(self: Node) =
while true:
var
descendant_queue = initDeque[MDigest[256]]()
new_head: MDigest[256]
descendant_queue = initDeque[Eth2Digest]()
new_head: Eth2Digest
max_count = 0
descendant_queue.addFirst self.main_chain[^1]
while descendant_queue.len != 0:
@ -67,18 +67,18 @@ func recalculate_head(self: Node) =
if self.scores.getOrDefault(first, 0) > max_count and first != self.main_chain[^1]:
new_head = first
max_count = self.scores.getOrDefault(first, 0)
if new_head != MDigest[256](): # != default init, a 32-byte array of 0
if new_head != Eth2Digest(): # != default init, a 32-byte array of 0
self.change_head(self.main_chain, self.blocks[new_head])
else:
return
proc process_children(self: Node, h: MDigest[256]) =
proc process_children(self: Node, h: Eth2Digest) =
if h in self.parentqueue:
for b in self.parentqueue[h]:
self.on_receive(b, reprocess = true)
self.parentqueue.del h
func get_common_ancestor(self: Node, hash_a, hash_b: MDigest[256]): Block =
func get_common_ancestor(self: Node, hash_a, hash_b: Eth2Digest): Block =
var (a, b) = (self.blocks[hash_a], self.blocks[hash_b])
while b.height > a.height:
b = self.blocks[b.parent_hash]
@ -89,14 +89,14 @@ func get_common_ancestor(self: Node, hash_a, hash_b: MDigest[256]): Block =
b = self.blocks[b.parent_hash]
return a
func is_descendant(self: Node, hash_a, hash_b: MDigest[256]): bool =
func is_descendant(self: Node, hash_a, hash_b: Eth2Digest): bool =
let a = self.blocks[hash_a]
var b = self.blocks[hash_b]
while b.height > a.height:
b = self.blocks[b.parent_hash]
return a.hash == b.hash
proc have_ancestry(self: Node, h: MDigest[256]): bool =
proc have_ancestry(self: Node, h: Eth2Digest): bool =
let h = BlockHash(raw: h)
while h.raw != Genesis.hash:
if h notin self.processed:
@ -216,7 +216,7 @@ method on_receive(self: Node, sig: Sig, reprocess = false) =
# Rebroadcast
self.network.broadcast(self, sig)
func get_sig_targets(self: Node, start_slot: int32): seq[MDigest[256]] =
func get_sig_targets(self: Node, start_slot: int32): seq[Eth2Digest] =
# Get the portion of the main chain that is within the last EPOCH_LENGTH
# slots, once again duplicating the parent in cases where the parent and
# child's slots are not consecutive

View File

@ -15,7 +15,8 @@ import
tables, deques, strutils, hashes, times,
random,
# Nimble packages
nimcrypto
nimcrypto,
../digest
const
NOTARIES* = 100 # Committee size in Casper v2.1
@ -33,14 +34,14 @@ type
BlockOrSigHash* = ref object of RootObj
BlockHash* = ref object of BlockOrSigHash
raw*: MDigest[256]
raw*: Eth2Digest
SigHash* = ref object of BlockOrSigHash
raw*: MDigest[384]
Block* = ref object of BlockOrSig
contents*: array[32, byte]
parent_hash*: MDigest[256]
hash*: MDigest[256]
parent_hash*: Eth2Digest
hash*: Eth2Digest
height*: int # slot in Casper v2.1 spec
proposer*: int32
slot*: int32
@ -114,26 +115,26 @@ type
Sig* = ref object of BlockOrSig
# TODO: unsure if this is still relevant in Casper v2.1
proposer*: int64 # the validator that creates a block
targets*: seq[MDigest[256]] # the hash of blocks proposed
targets*: seq[Eth2Digest] # the hash of blocks proposed
slot*: int32 # slot number
timestamp*: Duration # ts in the ref implementation
hash*: MDigest[384] # The signature (BLS12-384)
Node* = ref object
blocks*: TableRef[MDigest[256], Block]
blocks*: TableRef[Eth2Digest, Block]
sigs*: TableRef[MDigest[384], Sig]
main_chain*: seq[MDigest[256]]
main_chain*: seq[Eth2Digest]
timequeue*: seq[Block]
parentqueue*: TableRef[MDigest[256], seq[BlockOrSig]]
children*: TableRef[MDigest[256], seq[MDigest[256]]]
scores*: TableRef[MDigest[256], int]
parentqueue*: TableRef[Eth2Digest, seq[BlockOrSig]]
children*: TableRef[Eth2Digest, seq[Eth2Digest]]
scores*: TableRef[Eth2Digest, int]
scores_at_height*: TableRef[array[36, byte], int] # Should be slot not height in v2.1
justified*: TableRef[MDigest[256], bool]
finalized*: TableRef[MDigest[256], bool]
justified*: TableRef[Eth2Digest, bool]
finalized*: TableRef[Eth2Digest, bool]
timestamp*: Duration
id*: int32
network*: NetworkSimulator
used_parents*: TableRef[MDigest[256], Node]
used_parents*: TableRef[Eth2Digest, Node]
processed*: TableRef[BlockOrSigHash, BlockOrSig]
sleepy*: bool
careless*: bool
@ -143,7 +144,7 @@ type
proc newSig*(
proposer: int32,
targets: seq[MDigest[256]],
targets: seq[Eth2Digest],
slot: int32,
ts: Duration): Sig =
new result
@ -171,12 +172,12 @@ proc newNode*(
# Boilerplate empty initialization
result.processed = newTable[BlockOrSigHash, BlockOrSig]()
result.children = newTable[MDigest[256], seq[MDigest[256]]]()
result.parentqueue = newTable[MDigest[256], seq[BlockOrSig]]()
result.scores = newTable[MDigest[256], int]()
result.children = newTable[Eth2Digest, seq[Eth2Digest]]()
result.parentqueue = newTable[Eth2Digest, seq[BlockOrSig]]()
result.scores = newTable[Eth2Digest, int]()
result.scores_at_height = newTable[array[36, byte], int]()
result.sigs = newTable[MDigest[384], Sig]()
result.justified = newTable[MDigest[256], bool]()
result.justified = newTable[Eth2Digest, bool]()
###########################################################
# Forward declarations

View File

@ -6,9 +6,9 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# Helper functions
import ../datatypes, sequtils, nimcrypto, math
import ../datatypes, ../digest, sequtils, math
func shuffle*[T](values: seq[T], seed: Blake2_256_Digest): seq[T] =
func shuffle*[T](values: seq[T], seed: Eth2Digest): seq[T] =
## Returns the shuffled ``values`` with seed as entropy.
## TODO: this calls out for tests, but I odn't particularly trust spec
## right now.
@ -31,7 +31,7 @@ func shuffle*[T](values: seq[T], seed: Blake2_256_Digest): seq[T] =
index = 0
while index < values_count - 1:
# Re-hash the `source` to obtain a new pattern of bytes.
source = blake2_256.digest source.data
source = eth2hash source.data
# Iterate through the `source` bytes in 3-byte chunks.
for pos in countup(0, 29, 3):
@ -75,17 +75,17 @@ func get_shards_and_committees_for_slot*(state: BeaconState,
# TODO, slot is a uint64; will be an issue on int32 arch.
# Clarify with EF if light clients will need the beacon chain
func get_block_hash*(state: BeaconState, current_block: BeaconBlock, slot: int): Blake2_256_Digest =
func get_block_hash*(state: BeaconState, current_block: BeaconBlock, slot: int): Eth2Digest =
let earliest_slot_in_array = current_block.slot.int - state.recent_block_hashes.len
assert earliest_slot_in_array <= slot
assert slot < current_block.slot.int
return state.recent_block_hashes[slot - earliest_slot_in_array]
func get_new_recent_block_hashes*(old_block_hashes: seq[Blake2_256_Digest],
func get_new_recent_block_hashes*(old_block_hashes: seq[Eth2Digest],
parent_slot, current_slot: int64,
parent_hash: Blake2_256_Digest
): seq[Blake2_256_Digest] =
parent_hash: Eth2Digest
): seq[Eth2Digest] =
# Should throw for `current_slot - CYCLE_LENGTH * 2 - 1` according to spec comment
let d = current_slot - parent_slot

View File

@ -11,8 +11,8 @@
import
endians, typetraits, options, algorithm,
eth_common, nimcrypto,
./datatypes
eth_common, nimcrypto/blake2,
./datatypes, ./digest
from milagro_crypto import getRaw
@ -38,7 +38,7 @@ func toBytesSSZ(x: Uint24): array[3, byte] =
result[0] = byte((v shr 16) and 0xff)
func toBytesSSZ(x: EthAddress): array[sizeof(x), byte] = x
func toBytesSSZ(x: MDigest[32*8]): array[32, byte] = x.data
func toBytesSSZ(x: Eth2Digest): array[32, byte] = x.data
func fromBytesSSZUnsafe(T: typedesc[SomeInteger], data: ptr byte): T =
## Convert directly to bytes the size of the int. (e.g. ``uint16 = 2 bytes``)
@ -102,7 +102,7 @@ func deserialize(data: ptr byte, pos: var int, len: int, typ: typedesc[object]):
var t: typ
for field in t.fields:
when field is EthAddress | MDigest:
when field is EthAddress | Eth2Digest:
if not eat(field, data, pos, len): return
elif field is (SomeInteger or byte):
if not eatInt(field, data, pos, len): return
@ -140,20 +140,11 @@ const CHUNK_SIZE = 128
# ################### Hashing helpers ###################################
template withHash(body: untyped): untyped =
## Spec defines hash as BLAKE2b-512(x)[0:32]
## This little helper will init the hash function and return the sliced
## hash:
## let hashOfData = withHash: h.update(data)
var h {.inject.}: blake2_512
h.init()
body
var res: array[32, byte]
var tmp = h.finish().data
copyMem(res.addr, tmp.addr, 32)
res
# XXX varargs openarray, anyone?
template withHash(body: untyped): array[32, byte] =
let tmp = withEth2Hash: body
toBytesSSZ tmp
func hash(a: openArray[byte]): array[32, byte] =
withHash:
h.update(a)
@ -185,7 +176,7 @@ func hashSSZ*(x: EthAddress): array[sizeof(x), byte] =
## Addresses copied as-is
toBytesSSZ(x)
func hashSSZ*(x: MDigest[32*8]): array[32, byte] =
func hashSSZ*(x: Eth2Digest): array[32, byte] =
## Hash32 copied as-is
toBytesSSZ(x)
@ -207,14 +198,14 @@ func hashSSZ*(x: ValidatorRecord): array[32, byte] =
h.update hashSSZ(x.exit_slot)
func hashSSZ*(x: ShardAndCommittee): array[32, byte] =
return withHash:
withHash:
h.update hashSSZ(x.shard_id)
h.update merkleHash(x.committee)
func hashSSZ*[T: not enum](x: T): array[32, byte] =
when T is seq:
## Sequences are tree-hashed
return merkleHash(x)
merkleHash(x)
else:
## Containers have their fields recursively hashed, concatenated and hashed
# XXX could probaby compile-time-macro-sort fields...
@ -222,7 +213,7 @@ func hashSSZ*[T: not enum](x: T): array[32, byte] =
for name, field in x.fieldPairs:
fields.add (name, @(hashSSZ(field)))
return withHash:
withHash:
for name, value in fields.sortedByIt(it.name):
h.update hashSSZ(value.value)

View File

@ -21,7 +21,7 @@
{.warning: "The official spec at https://notes.ethereum.org/SCIg8AH5SA-O4C1G1LYZHQ# is not fully defining state transitions.".}
import
./datatypes, ./private/helpers,
./datatypes, ./digest, ./private/helpers,
intsets, endians, nimcrypto,
milagro_crypto # nimble install https://github.com/status-im/nim-milagro-crypto@#master
@ -38,7 +38,7 @@ func process_block*(active_state: BeaconState, crystallized_state: BeaconState,
# Compute parent_hashes = [get_block_hash(active_state, block, slot - CYCLE_LENGTH + i)
# for i in range(1, CYCLE_LENGTH - len(oblique_parent_hashes) + 1)] + oblique_parent_hashes
# TODO - don't allocate in tight loop
var parent_hashes = newSeq[Blake2_256_Digest](CYCLE_LENGTH - attestation.oblique_parent_hashes.len)
var parent_hashes = newSeq[Eth2Digest](CYCLE_LENGTH - attestation.oblique_parent_hashes.len)
for idx, val in parent_hashes.mpairs:
val = get_block_hash(active_state, blck, cast[int](slot - CYCLE_LENGTH + cast[uint64](idx) + 1))
parent_hashes.add attestation.oblique_parent_hashes
@ -80,7 +80,7 @@ func process_block*(active_state: BeaconState, crystallized_state: BeaconState,
bigEndian64(be_slot[0].addr, attestation.slot.unsafeAddr)
ctx.update be_slot
let size_p_hashes = uint parent_hashes.len * sizeof(Blake2_256_Digest)
let size_p_hashes = uint parent_hashes.len * sizeof(Eth2Digest)
ctx.update(cast[ptr byte](parent_hashes[0].addr), size_p_hashes)
var be_shard_id: array[2, byte] # Unsure, spec doesn't mention big-endian representation

View File

@ -8,8 +8,8 @@
import
options,
eth_common, nimcrypto/blake2,
./datatypes, ./private/helpers
eth_common,
./datatypes, ./digest, ./private/helpers
func min_empty_validator(validators: seq[ValidatorRecord], current_slot: uint64): Option[int] =
for i, v in validators:
@ -21,7 +21,7 @@ func add_validator*(validators: var seq[ValidatorRecord],
proof_of_possession: seq[byte],
withdrawal_shard: uint16,
withdrawal_address: EthAddress,
randao_commitment: Blake2_256_Digest,
randao_commitment: Eth2Digest,
status: ValidatorStatusCodes,
current_slot: uint64
): int =
@ -61,7 +61,7 @@ func get_active_validator_indices(validators: openArray[ValidatorRecord]): seq[U
if val.status == ACTIVE:
result.add idx.Uint24
func get_new_shuffling*(seed: Blake2_256_Digest,
func get_new_shuffling*(seed: Eth2Digest,
validators: openArray[ValidatorRecord],
crosslinking_start_shard: int
): array[CYCLE_LENGTH, seq[ShardAndCommittee]] =

View File

@ -5,8 +5,8 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
math, nimcrypto, unittest, sequtils,
../beacon_chain/[datatypes, validator]
math,unittest, sequtils,
../beacon_chain/[datatypes, digest, validator]
func sumCommittees(v: openArray[seq[ShardAndCommittee]]): int =
for x in v:
@ -25,7 +25,7 @@ suite "Validators":
), 1024)
# XXX the shuffling looks really odd, probably buggy
let s = get_new_shuffling(Blake2_256_Digest(), validators, 0)
let s = get_new_shuffling(Eth2Digest(), validators, 0)
check:
s.len == CYCLE_LENGTH
sumCommittees(s) == validators.len() # all validators accounted for