diff --git a/beacon_chain/attestation_pool.nim b/beacon_chain/attestation_pool.nim index dd198516d..5f3710cf2 100644 --- a/beacon_chain/attestation_pool.nim +++ b/beacon_chain/attestation_pool.nim @@ -2,66 +2,9 @@ import deques, options, sequtils, tables, chronicles, ./spec/[beaconstate, datatypes, crypto, digest, helpers, validator], extras, - ./beacon_chain_db, ./ssz, ./block_pool + ./beacon_chain_db, ./ssz, ./block_pool, + beacon_node_types -type - Validation* = object - aggregation_bitfield*: seq[byte] - custody_bitfield*: seq[byte] ##\ - ## Phase 1 - the handling of this field is probably broken.. - aggregate_signature*: ValidatorSig - - # Per Danny as of 2018-12-21: - # Yeah, you can do any linear combination of signatures. but you have to - # remember the linear combination of pubkeys that constructed - # if you have two instances of a signature from pubkey p, then you need 2*p - # in the group pubkey because the attestation bitfield is only 1 bit per - # pubkey right now, attestations do not support this it could be extended to - # support N overlaps up to N times per pubkey if we had N bits per validator - # instead of 1 - # We are shying away from this for the time being. If there end up being - # substantial difficulties in network layer aggregation, then adding bits to - # aid in supporting overlaps is one potential solution - - AttestationEntry* = object - data*: AttestationData - blck: BlockRef - validations*: seq[Validation] ## \ - ## Instead of aggregating the signatures eagerly, we simply dump them in - ## this seq and aggregate only when needed - ## TODO there are obvious caching opportunities here.. - - SlotData* = object - attestations*: seq[AttestationEntry] ## \ - ## Depending on the world view of the various validators, they may have - ## voted on different states - here we collect all the different - ## combinations that validators have come up with so that later, we can - ## count how popular each world view is (fork choice) - ## TODO this could be a Table[AttestationData, seq[Validation] or something - ## less naive - - UnresolvedAttestation* = object - attestation: Attestation - tries: int - - AttestationPool* = object - ## The attestation pool keeps all attestations that are known to the - ## client - each attestation counts as votes towards the fork choice - ## rule that determines which block we consider to be the head. The pool - ## contains both votes that have been included in the chain and those that - ## have not. - - slots*: Deque[SlotData] ## \ - ## We keep one item per slot such that indexing matches slot number - ## together with startingSlot - - startingSlot*: Slot ## \ - ## Generally, we keep attestations only until a slot has been finalized - - ## after that, they may no longer affect fork choice. - - blockPool: BlockPool - - unresolved: Table[Eth2Digest, UnresolvedAttestation] proc init*(T: type AttestationPool, blockPool: BlockPool): T = T( diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index f0f6237e4..67f37ff5d 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -5,28 +5,21 @@ import state_transition, fork_choice, ssz, beacon_chain_db, validator_pool, extras, attestation_pool, block_pool, eth2_network, mainchain_monitor, trusted_state_snapshots, - eth/trie/db, eth/trie/backends/rocksdb_backend - -type - BeaconNode* = ref object - network*: Eth2Node - db*: BeaconChainDB - config*: BeaconNodeConf - attachedValidators: ValidatorPool - blockPool*: BlockPool - state*: StateData - attestationPool: AttestationPool - mainchainMonitor: MainchainMonitor - potentialHeads: seq[Eth2Digest] + eth/trie/db, eth/trie/backends/rocksdb_backend, + beacon_node_types const topicBeaconBlocks = "ethereum/2.1/beacon_chain/blocks" topicAttestations = "ethereum/2.1/beacon_chain/attestations" topicfetchBlocks = "ethereum/2.1/beacon_chain/fetch" +# ################################################# +# Careful handling of beacon_node <-> sync_protocol +# to avoid recursive dependencies proc onBeaconBlock*(node: BeaconNode, blck: BeaconBlock) {.gcsafe.} - + # Forward decl for sync_protocol import sync_protocol +# ################################################# func shortValidatorKey(node: BeaconNode, validatorIdx: int): string = ($node.state.data.validator_registry[validatorIdx].pubkey)[0..7] diff --git a/beacon_chain/beacon_node_types.nim b/beacon_chain/beacon_node_types.nim new file mode 100644 index 000000000..faed7fcac --- /dev/null +++ b/beacon_chain/beacon_node_types.nim @@ -0,0 +1,207 @@ +import # Beacon Node + eth/[p2p, keys], + spec/digest, + beacon_chain_db, conf, mainchain_monitor + +import # Attestation Pool + spec/[datatypes, crypto, digest], + deques, tables + # block_pool + +import # Block Pool + spec/[datatypes, digest], + beacon_chain_db, + tables + +import # Validator Pool + spec/crypto, tables + +type + + # ############################################# + # + # Beacon Node + # + # ############################################# + BeaconNode* = ref object + network*: EthereumNode + db*: BeaconChainDB + config*: BeaconNodeConf + keys*: KeyPair + attachedValidators*: ValidatorPool + blockPool*: BlockPool + state*: StateData + attestationPool*: AttestationPool + mainchainMonitor*: MainchainMonitor + potentialHeads*: seq[Eth2Digest] + + # ############################################# + # + # Attestation Pool + # + # ############################################# + Validation* = object + aggregation_bitfield*: seq[byte] + custody_bitfield*: seq[byte] ##\ + ## Phase 1 - the handling of this field is probably broken.. + aggregate_signature*: ValidatorSig + + # Per Danny as of 2018-12-21: + # Yeah, you can do any linear combination of signatures. but you have to + # remember the linear combination of pubkeys that constructed + # if you have two instances of a signature from pubkey p, then you need 2*p + # in the group pubkey because the attestation bitfield is only 1 bit per + # pubkey right now, attestations do not support this it could be extended to + # support N overlaps up to N times per pubkey if we had N bits per validator + # instead of 1 + # We are shying away from this for the time being. If there end up being + # substantial difficulties in network layer aggregation, then adding bits to + # aid in supporting overlaps is one potential solution + + AttestationEntry* = object + data*: AttestationData + blck*: BlockRef + validations*: seq[Validation] ## \ + ## Instead of aggregating the signatures eagerly, we simply dump them in + ## this seq and aggregate only when needed + ## TODO there are obvious caching opportunities here.. + + SlotData* = object + attestations*: seq[AttestationEntry] ## \ + ## Depending on the world view of the various validators, they may have + ## voted on different states - here we collect all the different + ## combinations that validators have come up with so that later, we can + ## count how popular each world view is (fork choice) + ## TODO this could be a Table[AttestationData, seq[Validation] or something + ## less naive + + UnresolvedAttestation* = object + attestation*: Attestation + tries*: int + + AttestationPool* = object + ## The attestation pool keeps all attestations that are known to the + ## client - each attestation counts as votes towards the fork choice + ## rule that determines which block we consider to be the head. The pool + ## contains both votes that have been included in the chain and those that + ## have not. + + slots*: Deque[SlotData] ## \ + ## We keep one item per slot such that indexing matches slot number + ## together with startingSlot + + startingSlot*: Slot ## \ + ## Generally, we keep attestations only until a slot has been finalized - + ## after that, they may no longer affect fork choice. + + blockPool*: BlockPool + + unresolved*: Table[Eth2Digest, UnresolvedAttestation] + + # ############################################# + # + # Block Pool + # + # ############################################# + BlockPool* = ref object + ## Pool of blocks responsible for keeping a graph of resolved blocks as well + ## as candidates that may yet become part of that graph. + ## Currently, this type works as a facade to the BeaconChainDB, making + ## assumptions about the block composition therein. + ## + ## The general idea here is that blocks known to us are divided into two + ## camps - unresolved and resolved. When we start the chain, we have a + ## genesis state that serves as the root of the graph we're interested in. + ## Every block that belongs to that chain will have a path to that block - + ## conversely, blocks that do not are not interesting to us. + ## + ## As the chain progresses, some states become finalized as part of the + ## consensus process. One way to think of that is that the blocks that + ## come before them are no longer relevant, and the finalized state + ## is the new genesis from which we build. Thus, instead of tracing a path + ## to genesis, we can trace a path to any finalized block that follows - we + ## call the oldest such block a tail block. + ## + ## It's important to note that blocks may arrive in any order due to + ## chainging network conditions - we counter this by buffering unresolved + ## blocks for some time while trying to establish a path. + ## + ## Once a path is established, the block becomes resolved. We store the + ## graph in memory, in the form of BlockRef objects. This is also when + ## we forward the block for storage in the database + ## + ## TODO evaluate the split of responsibilities between the two + ## TODO prune the graph as tail moves + + pending*: Table[Eth2Digest, BeaconBlock] ##\ + ## Blocks that have passed validation but that we lack a link back to tail + ## for - when we receive a "missing link", we can use this data to build + ## an entire branch + + unresolved*: Table[Eth2Digest, UnresolvedBlock] ##\ + ## Roots of blocks that we would like to have (either parent_root of + ## unresolved blocks or block roots of attestations) + + blocks*: Table[Eth2Digest, BlockRef] ##\ + ## Tree of blocks pointing back to a finalized block on the chain we're + ## interested in - we call that block the tail + + blocksBySlot*: Table[uint64, seq[BlockRef]] + + tail*: BlockData ##\ + ## The earliest finalized block we know about + + db*: BeaconChainDB + + UnresolvedBlock* = object + tries*: int + + BlockRef* = ref object {.acyclic.} + ## Node in object graph guaranteed to lead back to tail block, and to have + ## a corresponding entry in database. + ## Block graph should form a tree - in particular, there are no cycles. + + root*: Eth2Digest ##\ + ## Root that can be used to retrieve block data from database + + parent*: BlockRef ##\ + ## Not nil, except for the tail + + children*: seq[BlockRef] + + BlockData* = object + ## Body and graph in one + + data*: BeaconBlock + refs*: BlockRef + + StateData* = object + data*: BeaconState + root*: Eth2Digest ##\ + ## Root of above data (cache) + + blck*: BlockRef ##\ + ## The block associated with the state found in data - in particular, + ## blck.state_root == root + + # ############################################# + # + # Validator Pool + # + # ############################################# + ValidatorKind* = enum + inProcess + remote + + ValidatorConnection* = object + + AttachedValidator* = ref object + idx*: int # index in the registry + case kind*: ValidatorKind + of inProcess: + privKey*: ValidatorPrivKey + else: + connection*: ValidatorConnection + + ValidatorPool* = object + validators*: Table[ValidatorPubKey, AttachedValidator] \ No newline at end of file diff --git a/beacon_chain/beacon_node_utils.nim b/beacon_chain/beacon_node_utils.nim new file mode 100644 index 000000000..15912acfd --- /dev/null +++ b/beacon_chain/beacon_node_utils.nim @@ -0,0 +1,4 @@ +import + spec/datatypes, beacon_node_types + +proc onBeaconBlock*(node: BeaconNode, blck: BeaconBlock) {.gcsafe.} \ No newline at end of file diff --git a/beacon_chain/block_pool.nim b/beacon_chain/block_pool.nim index 792d93dc7..fffc83dc3 100644 --- a/beacon_chain/block_pool.nim +++ b/beacon_chain/block_pool.nim @@ -1,89 +1,8 @@ import - bitops, chronicles, options, sequtils, sets, tables, + bitops, chronicles, options, tables, ssz, beacon_chain_db, state_transition, extras, - spec/[crypto, datatypes, digest, helpers] - -type - BlockPool* = ref object - ## Pool of blocks responsible for keeping a graph of resolved blocks as well - ## as candidates that may yet become part of that graph. - ## Currently, this type works as a facade to the BeaconChainDB, making - ## assumptions about the block composition therein. - ## - ## The general idea here is that blocks known to us are divided into two - ## camps - unresolved and resolved. When we start the chain, we have a - ## genesis state that serves as the root of the graph we're interested in. - ## Every block that belongs to that chain will have a path to that block - - ## conversely, blocks that do not are not interesting to us. - ## - ## As the chain progresses, some states become finalized as part of the - ## consensus process. One way to think of that is that the blocks that - ## come before them are no longer relevant, and the finalized state - ## is the new genesis from which we build. Thus, instead of tracing a path - ## to genesis, we can trace a path to any finalized block that follows - we - ## call the oldest such block a tail block. - ## - ## It's important to note that blocks may arrive in any order due to - ## chainging network conditions - we counter this by buffering unresolved - ## blocks for some time while trying to establish a path. - ## - ## Once a path is established, the block becomes resolved. We store the - ## graph in memory, in the form of BlockRef objects. This is also when - ## we forward the block for storage in the database - ## - ## TODO evaluate the split of responsibilities between the two - ## TODO prune the graph as tail moves - - pending*: Table[Eth2Digest, BeaconBlock] ##\ - ## Blocks that have passed validation but that we lack a link back to tail - ## for - when we receive a "missing link", we can use this data to build - ## an entire branch - - unresolved*: Table[Eth2Digest, UnresolvedBlock] ##\ - ## Roots of blocks that we would like to have (either parent_root of - ## unresolved blocks or block roots of attestations) - - blocks*: Table[Eth2Digest, BlockRef] ##\ - ## Tree of blocks pointing back to a finalized block on the chain we're - ## interested in - we call that block the tail - - blocksBySlot: Table[uint64, seq[BlockRef]] - - tail*: BlockData ##\ - ## The earliest finalized block we know about - - db*: BeaconChainDB - - UnresolvedBlock = object - tries*: int - - BlockRef* = ref object {.acyclic.} - ## Node in object graph guaranteed to lead back to tail block, and to have - ## a corresponding entry in database. - ## Block graph should form a tree - in particular, there are no cycles. - - root*: Eth2Digest ##\ - ## Root that can be used to retrieve block data from database - - parent*: BlockRef ##\ - ## Not nil, except for the tail - - children*: seq[BlockRef] - - BlockData* = object - ## Body and graph in one - - data*: BeaconBlock - refs*: BlockRef - - StateData* = object - data*: BeaconState - root*: Eth2Digest ##\ - ## Root of above data (cache) - - blck*: BlockRef ##\ - ## The block associated with the state found in data - in particular, - ## blck.state_root == root + spec/[crypto, datatypes, digest], + beacon_node_types proc link(parent, child: BlockRef) = doAssert (not (parent.root == Eth2Digest() or child.root == Eth2Digest())), diff --git a/beacon_chain/fork_choice.nim b/beacon_chain/fork_choice.nim index c3b1df29a..ae4623e6b 100644 --- a/beacon_chain/fork_choice.nim +++ b/beacon_chain/fork_choice.nim @@ -2,7 +2,7 @@ import deques, options, sequtils, tables, chronicles, ./spec/[beaconstate, datatypes, crypto, digest, helpers, validator], extras, - ./attestation_pool, ./beacon_chain_db, ./ssz + ./beacon_node_types, ./beacon_chain_db, ./ssz # ################################################################## # Specs diff --git a/beacon_chain/sync_protocol.nim b/beacon_chain/sync_protocol.nim index 2e5287f55..026e41106 100644 --- a/beacon_chain/sync_protocol.nim +++ b/beacon_chain/sync_protocol.nim @@ -2,7 +2,11 @@ import options, tables, chronicles, chronos, ranges/bitranges, spec/[datatypes, crypto, digest], - beacon_node, eth2_network, beacon_chain_db, block_pool, time, ssz + beacon_node_types, eth2_network, beacon_chain_db, block_pool, time, ssz + +from beacon_node import onBeaconBlock + # Careful handling of beacon_node <-> sync_protocol + # to avoid recursive dependencies type ValidatorChangeLogEntry* = object diff --git a/beacon_chain/validator_pool.nim b/beacon_chain/validator_pool.nim index 7e8cb4baf..e8e1079a9 100644 --- a/beacon_chain/validator_pool.nim +++ b/beacon_chain/validator_pool.nim @@ -1,25 +1,9 @@ import - tables, random, + tables, chronos, - spec/[datatypes, crypto, digest, helpers], ssz + spec/[datatypes, crypto, helpers], ssz, + beacon_node_types -type - ValidatorKind = enum - inProcess - remote - - ValidatorConnection = object - - AttachedValidator* = ref object - idx*: int # index in the registry - case kind: ValidatorKind - of inProcess: - privKey: ValidatorPrivKey - else: - connection: ValidatorConnection - - ValidatorPool* = object - validators: Table[ValidatorPubKey, AttachedValidator] proc init*(T: type ValidatorPool): T = result.validators = initTable[ValidatorPubKey, AttachedValidator]() diff --git a/tests/test_attestation_pool.nim b/tests/test_attestation_pool.nim index b49785622..929861f32 100644 --- a/tests/test_attestation_pool.nim +++ b/tests/test_attestation_pool.nim @@ -9,7 +9,7 @@ import options, sequtils, unittest, ./testutil, ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator], - ../beacon_chain/[attestation_pool, block_pool, extras, state_transition, ssz] + ../beacon_chain/[beacon_node_types, attestation_pool, block_pool, extras, state_transition, ssz] suite "Attestation pool processing": ## For now just test that we can compile and execute block processing with diff --git a/tests/test_block_pool.nim b/tests/test_block_pool.nim index e392aa66a..eeba20edb 100644 --- a/tests/test_block_pool.nim +++ b/tests/test_block_pool.nim @@ -9,7 +9,7 @@ import options, sequtils, unittest, ./testutil, ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator], - ../beacon_chain/[block_pool, beacon_chain_db, extras, state_transition, ssz] + ../beacon_chain/[beacon_node_types, block_pool, beacon_chain_db, extras, state_transition, ssz] suite "Block pool processing": let