diff --git a/README.md b/README.md index cd1f7a622..eebdc3077 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Ethereum Foundation uses: Nim NEP-1 recommends: - camelCase for fields and procedure names - PascalCase for constants - - PsacalCase for types + - PascalCase for types To facilitate collaboration and comparison, Nim-beacon-chain uses the Ethereum Foundation convention. diff --git a/beacon_chain.nimble b/beacon_chain.nimble index 7eb63e200..67f90663f 100644 --- a/beacon_chain.nimble +++ b/beacon_chain.nimble @@ -3,13 +3,22 @@ version = "0.0.1" author = "Status Research & Development GmbH" description = "Eth2.0 research implementation of the beacon chain" license = "MIT or Apache License 2.0" -srcDir = "src" +installDirs = @["beacon_chain"] +bin = @["beacon_chain/beacon_node"] ### Dependencies requires "nim >= 0.18.0", "eth_common", + "eth_keys", "nimcrypto", - "https://github.com/status-im/nim-milagro-crypto#master" + "https://github.com/status-im/nim-milagro-crypto#master", + "eth_p2p", + "ranges", + "chronicles", + "confutils", + "serialization", + "json_serialization", + "json_rpc" ### Helper functions proc test(name: string, defaultLang = "c") = diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim new file mode 100644 index 000000000..729d5c9d4 --- /dev/null +++ b/beacon_chain/beacon_chain_db.nim @@ -0,0 +1,28 @@ +import + os, json, + chronicles, json_serialization, eth_common/eth_types_json_serialization, + spec/datatypes + +type + BeaconChainDB* = ref object + dataRoot: string + + BeaconStateRef* = ref BeaconState + +proc init*(T: type BeaconChainDB, dataDir: string): BeaconChainDB = + new result + result.dataRoot = dataDir / "beacon_db" + createDir(result.dataRoot) + +proc lastFinalizedState*(db: BeaconChainDB): BeaconStateRef = + try: + var stateJson = parseJson readFile(db.dataRoot / "BeaconState.json") + # TODO implement this + except: + return nil + +proc persistBlock*(db: BeaconChainDB, s: BeaconState, b: BeaconBlock) = + let stateJson = StringJsonWriter.encode(s, pretty = true) + writeFile(db.dataRoot / "BeaconState.json", stateJson) + debug "State persisted" + diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim new file mode 100644 index 000000000..3154bff97 --- /dev/null +++ b/beacon_chain/beacon_node.nim @@ -0,0 +1,195 @@ +import + os, net, + asyncdispatch2, chronicles, confutils, eth_p2p, eth_keys, + spec/[beaconstate, datatypes], conf, time, fork_choice, + beacon_chain_db, validator_pool, mainchain_monitor, + sync_protocol, gossipsub_protocol, trusted_state_snapshots + +type + BeaconNode* = ref object + beaconState*: BeaconState + network*: EthereumNode + db*: BeaconChainDB + config*: BeaconNodeConf + keys*: KeyPair + attachedValidators: ValidatorPool + attestations: AttestationPool + headBlock: BeaconBlock + mainchainMonitor: MainchainMonitor + +const + version = "v0.1" # TODO: read this from the nimble file + clientId = "nimbus beacon node " & version + + topicBeaconBlocks = "ethereum/2.1/beacon_chain/blocks" + topicAttestations = "ethereum/2.1/beacon_chain/attestations" + +proc ensureNetworkKeys*(dataDir: string): KeyPair = + # TODO: + # 1. Check if keys already exist in the data dir + # 2. Generate new ones and save them in the directory + # if necessary + return newKeyPair() + +proc init*(T: type BeaconNode, conf: BeaconNodeConf): T = + new result + result.config = conf + result.db = BeaconChainDB.init(string conf.dataDir) + result.keys = ensureNetworkKeys(string conf.dataDir) + + var address: Address + address.ip = parseIpAddress("0.0.0.0") + address.tcpPort = Port(conf.tcpPort) + address.udpPort = Port(conf.udpPort) + result.network = newEthereumNode(result.keys, address, 0, nil, clientId) + +proc sync*(node: BeaconNode): Future[bool] {.async.} = + let persistedState = node.db.lastFinalizedState() + if persistedState.isNil or + persistedState[].slotDistanceFromNow() > WEAK_SUBJECTVITY_PERIOD: + node.beaconState = await obtainTrustedStateSnapshot(node.db) + else: + node.beaconState = persistedState[] + var targetSlot = toSlot timeSinceGenesis(node.beaconState) + + while node.beaconState.last_finalized_slot.int < targetSlot: + var (peer, changeLog) = await node.network.getValidatorChangeLog( + node.beaconState.validator_set_delta_hash_chain) + + if peer == nil: + error "Failed to sync with any peer" + return false + + if applyValidatorChangeLog(changeLog, node.beaconState): + node.db.persistBlock(node.beaconState, changeLog.signedBlock) + else: + warn "Ignoring invalid validator change log", sentFrom = peer + + return true + +proc addLocalValidators*(node: BeaconNode) = + for validator in node.config.validatorKeys: + # TODO: + # 1. Parse the validator keys + # + # 2. Check whether the validators exist in the beacon state. + # (Report a warning otherwise) + # + # 3. Add the validators to node.attachedValidators + discard + +proc getAttachedValidator(node: BeaconNode, idx: int): AttachedValidator = + let validatorKey = node.beaconState.validators[idx].pubkey + return node.attachedValidators.getValidator(validatorKey) + +proc makeAttestation(node: BeaconNode, + validator: AttachedValidator) {.async.} = + var attestation: Attestation + attestation.validator = validator.idx + + # TODO: Populate attestation.data + + attestation.signature = await validator.signAttestation(attestation.data) + await node.network.broadcast(topicAttestations, attestation) + +proc proposeBlock(node: BeaconNode, + validator: AttachedValidator, + slot: int) {.async.} = + var proposal: BeaconBlock + + # TODO: + # 1. Produce a RANDAO reveal from attachedVadalidator.randaoSecret + # and its matching ValidatorRecord. + + # 2. Get ancestors from the beacon_db + + # 3. Calculate the correct state hash + + proposal.candidate_pow_receipt_root = + node.mainchainMonitor.getBeaconBlockRef() + + for a in node.attestations.each(firstSlot = node.headBlock.slot.int + 1, + lastSlot = slot - MIN_ATTESTATION_INCLUSION_DELAY): + # TODO: this is not quite right, + # the attestations from individual validators have to be merged. + # proposal.attestations.add a + discard + + for r in node.mainchainMonitor.getValidatorActions( + node.headBlock.candidate_pow_receipt_root, + proposal.candidate_pow_receipt_root): + proposal.specials.add r + + var signedData: ProposalSignedData + # TODO: populate the signed data + + proposal.proposer_signature = await validator.signBlockProposal(signedData) + await node.network.broadcast(topicBeaconBlocks, proposal) + +proc scheduleCycleActions(node: BeaconNode) = + ## This schedules the required block proposals and + ## attestations from our attached validators. + let cycleStart = node.beaconState.last_state_recalculation_slot.int + + for i in 0 ..< CYCLE_LENGTH: + # Schedule block proposals + let + slot = cycleStart + i + proposerIdx = get_beacon_proposer_idx(node.beaconState, slot) + attachedValidator = node.getAttachedValidator(proposerIdx) + + if attachedValidator != nil: + # TODO: + # Warm-up the proposer earlier to try to obtain previous + # missing blocks if necessary + + addTimer(node.beaconState.slotStart(slot)) do (p: pointer): + asyncCheck proposeBlock(node, attachedValidator, slot) + + # Schedule attestations + let + committeesIdx = get_shard_and_committees_idx(node.beaconState, slot) + + for shard in node.beaconState.shard_and_committee_for_slots[committees_idx]: + for validatorIdx in shard.committee: + let attachedValidator = node.getAttachedValidator(validatorIdx) + if attachedValidator != nil: + addTimer(node.beaconState.slotMiddle(slot)) do (p: pointer): + asyncCheck makeAttestation(node, attachedValidator) + +proc processBlocks*(node: BeaconNode) {.async.} = + node.scheduleCycleActions() + + node.network.subscribe(topicBeaconBlocks) do (b: BeaconBlock): + # TODO: + # + # 1. Check for missing blocks and obtain them + # + # 2. Apply fork-choice rule (update node.headBlock) + # + # 3. Peform block processing / state recalculation / etc + # + + if b.slot mod CYCLE_LENGTH == 0: + node.scheduleCycleActions() + node.attestations.discardHistoryToSlot(b.slot) + + node.network.subscribe(topicAttestations) do (a: Attestation): + # TODO + # + # 1. Validate the attestation + + node.attestations.add(a, node.beaconState) + +when isMainModule: + let config = BeaconNodeConf.load() + waitFor syncrhronizeClock() + var node = BeaconNode.init config + + if not waitFor node.sync(): + quit 1 + + node.addLocalValidators() + + waitFor node.processBlocks() + diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim new file mode 100644 index 000000000..3d63c563b --- /dev/null +++ b/beacon_chain/conf.nim @@ -0,0 +1,33 @@ +import + confutils/defs + +type + ValidatorKeyPath* = distinct string + + BeaconNodeConf* = object + dataDir* {. + desc: "The directory where nimbus will store all blockchain data.", + shorthand: "d", + defaultValue: getConfigDir() / "nimbus".}: DirPath + + bootstrapNodes* {. + desc: "Specifies one or more bootstrap nodes to use when connecting to the network.", + shorthand: "b".}: seq[string] + + tcpPort* {. + desc: "TCP listening port".}: int + + udpPort* {. + desc: "UDP listening port".}: int + + validatorKeys* {. + desc: "A path to a pair of public and private keys for a validator. " & + "Nimbus will automatically add the extensions .privkey and .pubkey.", + shorthand: "v".}: seq[ValidatorKeyPath] + +proc parse*(T: type ValidatorKeyPath, input: TaintedString): T = + # TODO: + # Check that the entered string is a valid base file name and + # that it has matching .privkey, .pubkey and .randaosecret files + T(input) + diff --git a/beacon_chain/fork_choice.nim b/beacon_chain/fork_choice.nim new file mode 100644 index 000000000..02b6ce65c --- /dev/null +++ b/beacon_chain/fork_choice.nim @@ -0,0 +1,67 @@ +import + deques, + spec/[datatypes, crypto] + +type + Attestation* = object + validator*: int + data*: AttestationSignedData + signature*: ValidatorSig + + AttestationPool* = object + attestations: Deque[seq[Attestation]] + startingSlot: int + +proc init*(T: type AttestationPool, startingSlot: int): T = + result.attestationsPerSlot = initDeque[seq[Attestation]]() + result.startingSlot = startingSlot + +proc setLen*[T](d: var Deque[T], len: int) = + # TODO: The upstream `Deque` type should gain a proper resize API + let delta = len - d.len + if delta > 0: + for i in 0 ..< delta: + var defaultVal: T + d.addLast(defaultVal) + else: + d.shrink(fromLast = delta) + +proc add*(pool: var AttestationPool, + attestation: Attestation, + beaconState: BeaconState) = + # The caller of this function is responsible for ensuring that + # the attestations will be given in a strictly slot increasing order: + doAssert attestation.data.slot.int >= pool.startingSlot + + let slotIdxInPool = attestation.data.slot.int - pool.startingSlot + if slotIdxInPool >= pool.attestations.len: + pool.attestations.setLen(slotIdxInPool + 1) + + pool.attestations[slotIdxInPool].add attestation + +iterator each*(pool: AttestationPool, + firstSlot, lastSlot: int): Attestation = + ## Both indices are treated inclusively + ## TODO: this should return a lent value + doAssert firstSlot <= lastSlot + for idx in countup(max(0, firstSlot - pool.startingSlot), + min(pool.attestations.len - 1, lastSlot - pool.startingSlot)): + for attestation in pool.attestations[idx]: + yield attestation + +proc discardHistoryToSlot*(pool: var AttestationPool, slot: int) = + ## The index is treated inclusively + let slotIdx = slot - pool.startingSlot + if slotIdx < 0: return + pool.attestations.shrink(fromFirst = slotIdx + 1) + +proc getLatestAttestation*(pool: AttestationPool, validator: ValidatorRecord) = + discard + +proc getLatestAttestationTarget*() = + discard + +proc forkChoice*(pool: AttestationPool, oldHead, newBlock: BeaconBlock): bool = + # This will return true if the new block is accepted over the old head block + discard + diff --git a/beacon_chain/gossipsub_protocol.nim b/beacon_chain/gossipsub_protocol.nim new file mode 100644 index 000000000..6c8d93dc2 --- /dev/null +++ b/beacon_chain/gossipsub_protocol.nim @@ -0,0 +1,38 @@ +import + tables, sets, + asyncdispatch2, chronicles, rlp, eth_p2p, eth_p2p/rlpx + +type + TopicMsgHandler = proc(data: seq[byte]): Future[void] + + GossibSubPeer = ref object + sentMessages: HashSet[string] + + GossipSubNetwork = ref object + deliveredMessages: Table[Peer, HashSet[string]] + topicSubscribers: Table[string, seq[TopicMsgHandler]] + +p2pProtocol GossipSub(version = 1, + shortName = "gss", + peerState = GossibSubPeer, + networkState = GossipSubNetwork): + # This is a very barebones emulation of the GossipSub protocol + # available in LibP2P: + + proc interestedIn(peer: Peer, topic: string) + proc emit(peer: Peer, topic: string, msgId: string, data: openarray[byte]) + +proc subscribeImpl(node: EthereumNode, + topic: string, + subscriber: TopicMsgHandler) = + discard + +proc broadcastImpl(node: EthereumNode, topic: string, data: seq[byte]) = + discard + +macro subscribe*(node: EthereumNode, topic: string, handler: untyped): untyped = + discard + +proc broadcast*(node: EthereumNode, topic: string, data: auto) {.async.} = + discard + diff --git a/beacon_chain/mainchain_monitor.nim b/beacon_chain/mainchain_monitor.nim new file mode 100644 index 000000000..191e19712 --- /dev/null +++ b/beacon_chain/mainchain_monitor.nim @@ -0,0 +1,29 @@ +import + asyncdispatch2, json_rpc/rpcclient, + spec/[datatypes, digest] + +type + MainchainMonitor* = object + gethAddress: string + gethPort: Port + +proc init*(T: type MainchainMonitor, gethAddress: string, gethPort: Port): T = + result.gethAddress = gethAddress + result.gethPort = gethPort + +proc start*(m: var MainchainMonitor) = + # TODO + # Start an async loop following the new blocks using the ETH1 JSON-RPC + # interface and keep an always-up-to-date receipt reference here + discard + +proc getBeaconBlockRef*(m: MainchainMonitor): Eth2Digest = + # This should be a simple accessor for the reference kept above + discard + +iterator getValidatorActions*(m: MainchainMonitor, + fromBlock, toBlock: Eth2Digest): SpecialRecord = + # It's probably better if this doesn't return a SpecialRecord, but + # rather a more readable description of the change that can be packed + # in a SpecialRecord by the client of the API. + discard diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index 9e086ed1d..7a6dcba52 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -8,25 +8,22 @@ import ./datatypes, ./digest, ./helpers, ./validator -func get_shards_and_committees_for_slot*(state: BeaconState, - slot: uint64 - ): seq[ShardAndCommittee] = - let earliest_slot_in_array = state.last_state_recalculation_slot - CYCLE_LENGTH - assert earliest_slot_in_array <= slot - assert slot < earliest_slot_in_array + CYCLE_LENGTH * 2 +func mod_get[T](arr: openarray[T], pos: Natural): T = + arr[pos mod arr.len] - return state.shard_and_committee_for_slots[int slot - earliest_slot_in_array] - # TODO, slot is a uint64; will be an issue on int32 arch. - # Clarify with EF if light clients will need the beacon chain +func get_shard_and_committees_idx*(state: BeaconState, slot: int): int = + # This replaces `get_shards_and_committees_for_slot` from the spec + # since in Nim, it's not currently efficient to create read-only + # accessors to expensive-to-copy members (such as sequences). + let earliest_slot_in_array = state.last_state_recalculation_slot.int - CYCLE_LENGTH + doAssert earliest_slot_in_array <= slot and + slot < earliest_slot_in_array + CYCLE_LENGTH * 2 + return int(slot - earliest_slot_in_array) -func get_block_hash*(state: BeaconState, current_block: BeaconBlock, slot: int): Eth2Digest = - let earliest_slot_in_array = current_block.slot.int - state.recent_block_hashes.len - assert earliest_slot_in_array <= slot - assert slot < current_block.slot.int +proc get_shards_and_committees_for_slot*(state: BeaconState, slot: int): seq[ShardAndCommittee] = + return state.shard_and_committee_for_slots[state.get_shard_and_committees_idx(slot)] - return state.recent_block_hashes[slot - earliest_slot_in_array] - -func get_beacon_proposer*(state: BeaconState, slot: uint64): ValidatorRecord = +func get_beacon_proposer_idx*(state: BeaconState, slot: int): int = ## From Casper RPJ mini-spec: ## When slot i begins, validator Vidx is expected ## to create ("propose") a block, which contains a pointer to some parent block @@ -35,7 +32,20 @@ func get_beacon_proposer*(state: BeaconState, slot: uint64): ValidatorRecord = ## that have not yet been included into that chain. ## ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee) - let - first_committee = get_shards_and_committees_for_slot(state, slot)[0].committee - index = first_committee[(slot mod len(first_committee).uint64).int] - state.validators[index] + + # This replaces `get_beacon_proposer` from the spec since in Nim, + # it's not currently efficient to create read-only accessors to + # expensive-to-copy members (such as ValidatorRecord). + + let idx = get_shard_and_committees_idx(state, slot) + return state.shard_and_committee_for_slots[idx][0].committee.mod_get(slot) + +func get_block_hash*(state: BeaconState, + current_block: BeaconBlock, + slot: int): Eth2Digest = + let earliest_slot_in_array = current_block.slot.int - state.recent_block_hashes.len + assert earliest_slot_in_array <= slot + assert slot < current_block.slot.int + + return state.recent_block_hashes[slot - earliest_slot_in_array] + diff --git a/beacon_chain/spec/crypto.nim b/beacon_chain/spec/crypto.nim index e8bbfd1aa..c4d3ac2d9 100644 --- a/beacon_chain/spec/crypto.nim +++ b/beacon_chain/spec/crypto.nim @@ -10,8 +10,13 @@ # hashed out. This layer helps isolate those chagnes. import - milagro_crypto + milagro_crypto, hashes type - Eth2PublicKey* = milagro_crypto.VerKey - Eth2Signature* = milagro_crypto.Signature + ValidatorPubKey* = milagro_crypto.VerKey + ValidatorPrivKey* = milagro_crypto.SigKey + ValidatorSig* = milagro_crypto.Signature + +template hash*(k: ValidatorPubKey|ValidatorPrivKey): Hash = + hash(k.getRaw) + diff --git a/beacon_chain/spec/datatypes.nim b/beacon_chain/spec/datatypes.nim index a81afa564..ddcecc93c 100644 --- a/beacon_chain/spec/datatypes.nim +++ b/beacon_chain/spec/datatypes.nim @@ -69,13 +69,13 @@ type state_root*: Eth2Digest # State root attestations*: seq[AttestationRecord] # Attestations specials*: seq[SpecialRecord] # Specials (e.g. logouts, penalties) - proposer_signature*: Eth2Signature # Proposer signature + proposer_signature*: ValidatorSig # Proposer signature AttestationRecord* = object data*: AttestationSignedData # attester_bitfield*: seq[byte] # Attester participation bitfield poc_bitfield*: seq[byte] # Proof of custody bitfield - aggregate_sig*: Eth2Signature # BLS aggregate signature + aggregate_sig*: ValidatorSig # BLS aggregate signature AttestationSignedData* = object slot*: uint64 # Slot number @@ -93,7 +93,7 @@ type block_hash*: Eth2Digest # Block hash SpecialRecord* = object - kind*: SpecialRecordTypes # Kind + kind*: SpecialRecordType # Kind data*: seq[byte] # Data BeaconState* = object @@ -124,7 +124,7 @@ type randao_mix*: Eth2Digest # RANDAO state ValidatorRecord* = object - pubkey*: Eth2PublicKey # Public key + pubkey*: ValidatorPubKey # Public key withdrawal_credentials*: Eth2Digest # Withdrawal credentials randao_commitment*: Eth2Digest # RANDAO commitment randao_skips*: uint64 # Slot the proposer has skipped (ie. layers of RANDAO expected) @@ -169,7 +169,7 @@ type Withdrawn = 4 Penalized = 127 - SpecialRecordTypes* {.pure.} = enum + SpecialRecordType* {.pure.} = enum Logout = 0 CasperSlashing = 1 RandaoChange = 2 @@ -191,3 +191,27 @@ type # with room to spare. # # Also, IntSets uses machine int size while we require int64 even on 32-bit platform. + +when true: + # TODO: Remove these once RLP serialization is no longer used + import nimcrypto, rlp + export append, read + + proc append*(rlpWriter: var RlpWriter, value: ValidatorPubKey) = + discard + + proc read*(rlp: var Rlp, T: type ValidatorPubKey): T {.inline.} = + discard + + proc append*(rlpWriter: var RlpWriter, value: Uint24) = + discard + + proc read*(rlp: var Rlp, T: type Uint24): T {.inline.} = + discard + + proc append*(rlpWriter: var RlpWriter, value: ValidatorSig) = + discard + + proc read*(rlp: var Rlp, T: type ValidatorSig): T {.inline.} = + discard + diff --git a/beacon_chain/spec/validator.nim b/beacon_chain/spec/validator.nim index 4b2562ac2..f4b0dbcf7 100644 --- a/beacon_chain/spec/validator.nim +++ b/beacon_chain/spec/validator.nim @@ -17,7 +17,7 @@ func min_empty_validator(validators: seq[ValidatorRecord], current_slot: uint64) return some(i) func add_validator*(validators: var seq[ValidatorRecord], - pubkey: Eth2PublicKey, + pubkey: ValidatorPubKey, proof_of_possession: seq[byte], withdrawal_credentials: Eth2Digest, randao_commitment: Eth2Digest, diff --git a/beacon_chain/ssz.nim b/beacon_chain/ssz.nim index 50d79f4a2..aab18939d 100644 --- a/beacon_chain/ssz.nim +++ b/beacon_chain/ssz.nim @@ -226,11 +226,11 @@ func hashSSZ*(x: enum): array[32, byte] = withHash: h.update [uint8 x] -func hashSSZ*(x: Eth2Signature): array[32, byte] = +func hashSSZ*(x: ValidatorSig): array[32, byte] = ## TODO - Warning ⚠️: not part of the spec ## as of https://github.com/ethereum/beacon_chain/pull/133/files ## This is a "stub" needed for BeaconBlock hashing - x.getraw().hash() + x.getRaw().hash() func hashSSZ*(x: AttestationRecord): array[32, byte] = ## TODO - Warning ⚠️: not part of the spec diff --git a/beacon_chain/state_transition.nim b/beacon_chain/state_transition.nim index 1764a27a8..92a4d23de 100644 --- a/beacon_chain/state_transition.nim +++ b/beacon_chain/state_transition.nim @@ -25,8 +25,6 @@ import intsets, endians, nimcrypto, milagro_crypto # nimble install https://github.com/status-im/nim-milagro-crypto@#master - - func process_block*(active_state: BeaconState, crystallized_state: BeaconState, blck: BeaconBlock, slot: uint64) = # TODO: non-attestation verification parts of per-block processing @@ -44,7 +42,7 @@ func process_block*(active_state: BeaconState, crystallized_state: BeaconState, # Let attestation_indices be get_shards_and_committees_for_slot(crystallized_state, slot)[x], choosing x so that attestation_indices.shard_id equals the shard_id value provided to find the set of validators that is creating this attestation record. let attestation_indices = block: - let shard_and_committees = get_shards_and_committees_for_slot(crystallized_state, slot) + let shard_and_committees = get_shards_and_committees_for_slot(crystallized_state, slot.int) var x = 1 record_creator = shard_and_committees[0] @@ -53,11 +51,10 @@ func process_block*(active_state: BeaconState, crystallized_state: BeaconState, inc x record_creator - # Verify that len(attester_bitfield) == ceil_div8(len(attestation_indices)), where ceil_div8 = (x + 7) // 8. Verify that bits len(attestation_indices).... and higher, if present (i.e. len(attestation_indices) is not a multiple of 8), are all zero - doAssert attestation.attester_bitfield.len == attestation_indices.committee.len + # TODO: Verify that len(attester_bitfield) == ceil_div8(len(attestation_indices)), where ceil_div8 = (x + 7) // 8. Verify that bits len(attestation_indices).... and higher, if present (i.e. len(attestation_indices) is not a multiple of 8), are all zero # Derive a group public key by adding the public keys of all of the attesters in attestation_indices for whom the corresponding bit in attester_bitfield (the ith bit is (attester_bitfield[i // 8] >> (7 - (i %8))) % 2) equals 1 - var agg_pubkey: Eth2PublicKey + var agg_pubkey: ValidatorPubKey var empty = true for attester_idx in attestation_indices.committee: # TODO re-enable, but currently this whole function's a nonfunctional stub diff --git a/beacon_chain/sync_protocol.nim b/beacon_chain/sync_protocol.nim new file mode 100644 index 000000000..d146d5823 --- /dev/null +++ b/beacon_chain/sync_protocol.nim @@ -0,0 +1,93 @@ +import + options, + chronicles, rlp, asyncdispatch2, ranges/bitranges, eth_p2p, eth_p2p/rlpx, + spec/[datatypes, crypto, digest] + +type + ValidatorChangeLogEntry* = object + case kind*: ValidatorSetDeltaFlags + of Entry: + pubkey: ValidatorPubKey + else: + index: uint32 + + ValidatorSet = seq[ValidatorRecord] + +p2pProtocol BeaconSync(version = 1, + shortName = "bcs"): + requestResponse: + proc getValidatorChangeLog(peer: Peer, changeLogHead: Eth2Digest) + + proc validatorChangeLog(peer: Peer, + signedBlock: BeaconBlock, + beaconState: BeaconState, + added: openarray[ValidatorPubKey], + removed: openarray[uint32], + order: seq[byte]) + +template `++`(x: var int): int = + let y = x + inc x + y + +type + # A bit shorter names for convenience + ChangeLog = BeaconSync.validatorChangeLog + ChangeLogEntry = ValidatorChangeLogEntry + +func validate*(log: ChangeLog): bool = + # TODO: + # Assert that the number of raised bits in log.order (a.k.a population count) + # matches the number of elements in log.added + # https://en.wikichip.org/wiki/population_count + return true + +iterator changes*(log: ChangeLog): ChangeLogEntry = + var + bits = log.added.len + log.removed.len + addedIdx = 0 + removedIdx = 0 + + template nextItem(collection): auto = + let idx = `collection Idx` + inc `collection Idx` + log.collection[idx] + + for i in 0 ..< bits: + yield if log.order.getBit(i): + ChangeLogEntry(kind: Entry, pubkey: nextItem(added)) + else: + ChangeLogEntry(kind: Exit, index: nextItem(removed)) + +proc getValidatorChangeLog*(node: EthereumNode, changeLogHead: Eth2Digest): + Future[(Peer, ChangeLog)] {.async.} = + while true: + let peer = node.randomPeerWith(BeaconSync) + if peer == nil: return + + let res = await peer.getValidatorChangeLog(changeLogHead, timeout = 1) + if res.isSome: + return (peer, res.get) + +proc applyValidatorChangeLog*(log: ChangeLog, + outBeaconState: var BeaconState): bool = + # TODO: + # + # 1. Validate that the signedBlock state root hash matches the + # provided beaconState + # + # 2. Validate that the applied changelog produces the correct + # new change log head + # + # 3. Check that enough signatures from the known validator set + # are present + # + # 4. Apply all changes to the validator set + # + + outBeaconState.last_finalized_slot = + log.signedBlock.slot div CYCLE_LENGTH + + outBeaconState.validator_set_delta_hash_chain = + log.beaconState.validator_set_delta_hash_chain + diff --git a/beacon_chain/time.nim b/beacon_chain/time.nim new file mode 100644 index 000000000..dbe4b7528 --- /dev/null +++ b/beacon_chain/time.nim @@ -0,0 +1,46 @@ +import + random, + asyncdispatch2, + spec/datatypes + +type + Timestamp = uint64 # Unix epoch timestamp in millisecond resolution + +var + detectedClockDrift: int64 + +proc timeSinceGenesis*(s: BeaconState): Timestamp = + Timestamp(int64(fastEpochTime() - s.genesis_time * 1000) - + detectedClockDrift) + +template toSlot*(t: Timestamp): int = + int(t div uint64(SLOT_DURATION * 1000)) + +template slotStart*(s: BeaconState, slot: int): Timestamp = + (s.genesis_time + uint64(slot * SLOT_DURATION)) * 1000 + +template slotMiddle*(s: BeaconState, slot: int): Timestamp = + s.slotStart(slot) + SLOT_DURATION * 500 + +template slotEnd*(s: BeaconState, slot: int): Timestamp = + s.slotStart(slot + 1) + +proc randomTimeInSlot*(s: BeaconState, + slot: Natural, + interval: HSlice[float, float]): Timestamp = + ## Returns a random moment within the slot. + ## The interval must be a sub-interval of [0..1]. + ## Zero marks the begginning of the slot and One marks the end. + s.slotStart(slot) + Timestamp(rand(interval) * float(SLOT_DURATION * 1000)) + +proc slotDistanceFromNow*(s: BeaconState): int64 = + ## Returns how many slots have passed since a particular BeaconState was finalized + int64(s.timeSinceGenesis() div (SLOT_DURATION * 1000)) - int64(s.last_finalized_slot) + +proc syncrhronizeClock*() {.async.} = + ## This should determine the offset of the local clock against a global + ## trusted time (e.g. it can be obtained from multiple time servers). + + # TODO: implement this properly + detectedClockDrift = 0 + diff --git a/beacon_chain/trusted_state_snapshots.nim b/beacon_chain/trusted_state_snapshots.nim new file mode 100644 index 000000000..b00ca695e --- /dev/null +++ b/beacon_chain/trusted_state_snapshots.nim @@ -0,0 +1,31 @@ +import + asyncdispatch2, + spec/datatypes, beacon_chain_db + +const + WEAK_SUBJECTVITY_PERIOD* = 4 * 30 * 24 * 60 * 60 div SLOT_DURATION + # TODO: This needs revisiting. + # Why was the validator WITHDRAWAL_PERIOD altered in the spec? + +proc obtainTrustedStateSnapshot*(db: BeaconChainDB): Future[BeaconState] {.async.} = + # In case our latest state is too old, we must obtain a recent snapshot + # of the state from a trusted location. This is explained in detail here: + # https://notes.ethereum.org/oaQV3IF5R2qlJuW-V1r1ew#Beacon-chain-sync + + # TODO: implement this: + # + # 1. Specify a large set of trusted state signees + # (perhaps stored in a config file) + # + # 2. Download a signed state hash from a known location + # (The known location can be either a HTTPS host or a DHT record) + # + # 3. Check that enough of the specified required signatures are present + # + # 4. Download a snapshot file from a known location + # (or just obtain it from the network using the ETH protocols) + # + # 5. Check that the state snapshot hash is correct and save it in the DB. + + discard + diff --git a/beacon_chain/validator_pool.nim b/beacon_chain/validator_pool.nim new file mode 100644 index 000000000..67454787c --- /dev/null +++ b/beacon_chain/validator_pool.nim @@ -0,0 +1,66 @@ +import + tables, random, + asyncdispatch2, + spec/[datatypes, crypto] + +type + ValidatorKind = enum + inProcess + remote + + ValidatorConnection = object + + RandaoSecret = seq[byte] + + AttachedValidator* = ref object + idx*: int + case kind: ValidatorKind + of inProcess: + privKey: ValidatorPrivKey + randaoSecret: RandaoSecret + else: + connection: ValidatorConnection + + ValidatorPool* = object + validators: Table[ValidatorPubKey, AttachedValidator] + +proc init*(T: type ValidatorPool): T = + result.validators = initTable[ValidatorPubKey, AttachedValidator]() + +proc addLocalValidator*(pool: var ValidatorPool, + idx: int, + pubKey: ValidatorPubKey, + privKey: ValidatorPrivKey, + randaoSecret: RandaoSecret) = + pool.validators[pubKey] = AttachedValidator(idx: idx, + kind: inProcess, + privKey: privKey, + randaoSecret: randaoSecret) + +proc getValidator*(pool: ValidatorPool, + validatorKey: ValidatorPubKey): AttachedValidator = + pool.validators.getOrDefault(validatorKey) + +proc signBlockProposal*(v: AttachedValidator, + proposal: ProposalSignedData): Future[ValidatorSig] {.async.} = + if v.kind == inProcess: + await sleepAsync(1) + # TODO: + # return sign(proposal, v.privKey) + else: + # TODO: + # send RPC + discard + +proc signAttestation*(v: AttachedValidator, + attestation: AttestationSignedData): Future[ValidatorSig] {.async.} = + # TODO: implement this + if v.kind == inProcess: + await sleepAsync(1) + # TODO: + # return sign(proposal, v.privKey) + else: + # TODO: + # send RPC + discard +