Consensus object pools [reorg 4/5] (#2374)

* Add documentation

* make test doesn't try to build the beacon node :/

beacon_chain/attestation_aggregation.nim

@@ -12,8 +12,11 @@ import
   chronos, chronicles,
   ./spec/[
     beaconstate, datatypes, crypto, digest, helpers, network, signatures],
-  ./block_pools/[spec_cache, chain_dag, quarantine, spec_cache],
-  ./attestation_pool, ./beacon_node_types, ./ssz, ./time
+  ./consensus_object_pools/[
+    spec_cache, blockchain_dag, block_quarantine, spec_cache,
+    attestation_pool
+  ],
+  ./beacon_node_types, ./ssz, ./time
 
 logScope:
   topics = "att_aggr"

beacon_chain/beacon_node_common.nim

@@ -14,18 +14,18 @@ import
   chronos, json_rpc/rpcserver,
 
   # Local modules
-  ./conf, ./time, ./beacon_chain_db, ./attestation_pool, ./eth2_network,
+  ./conf, ./time, ./beacon_chain_db, ./eth2_network,
   ./beacon_node_types,
   ./eth2_processor,
   ./eth1/eth1_monitor,
-  ./block_pools/[chain_dag, quarantine],
+  ./consensus_object_pools/[blockchain_dag, block_quarantine, attestation_pool],
   ./spec/datatypes,
   ./sync/[sync_manager, request_manager]
 
 export
   osproc, chronos, rpcserver, conf, time, beacon_chain_db,
   attestation_pool, eth2_network, beacon_node_types, eth1_monitor,
-  request_manager, sync_manager, eth2_processor, chain_dag, quarantine,
+  request_manager, sync_manager, eth2_processor, blockchain_dag, block_quarantine,
   datatypes
 
 type

beacon_chain/beacon_node_types.nim

@@ -4,7 +4,7 @@ import
   std/[deques, intsets, streams, tables],
   stew/endians2,
   spec/[datatypes, digest, crypto],
-  block_pools/block_pools_types,
+  consensus_object_pools/block_pools_types,
   fork_choice/fork_choice_types,
   validators/slashing_protection
 

beacon_chain/consensus_object_pools/README.md

@@ -0,0 +1,26 @@
+# Consensus object pools
+
+This folder holds the various consensus object pools needed for a blockchain client.
+
+Object in those pools have passed the "gossip validation" filter according
+to specs:
+- blocks: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#beacon_block
+- aggregate attestations: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#beacon_aggregate_and_proof
+- unaggregate attestation: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#beacon_attestation_subnet_id
+- voluntary exits https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#voluntary_exit
+- Attester slashings https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#attester_slashing
+- Proposer slashins https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#proposer_slashing
+
+After "gossip validation" the consensus objects can be rebroadcasted as they are optimistically good, however for internal processing further verification is needed.
+For blocks, this means verifying state transition and all contained cryptographic signatures (instead of just the proposer signature).
+For other consensus objects, it is possible that gossip validation is a superset of consensus verification (TODO).
+
+The pools presenet in this folder are:
+- block_pools:
+  - block_quarantine: for seemingly valid blocks that are on a fork unknown to us.
+  - block_clearance: to verify (state_transition + cryptography) candidate blocks.
+  - blockchain_dag: an in-memory direct-acyclic graph of fully validated and verified blockchain candidates with the tail being the last finalized epoch. A block in the DAG MUST be in the fork choice and a block in the fork choice MUST be in the DAG (except for orphans following finalization). On finalization non-empty epoch blocks are stored in the beacon_chain_db.
+- attestation_pool:
+  Handles the attestation received from gossip and collect them for fork choice.
+- exit_pool:
+  Handle voluntary exits and forced exits (attester slashings and proposer slashings)

beacon_chain/consensus_object_pools/attestation_pool.nim

@@ -13,11 +13,11 @@ import
   # Status libraries
   chronicles, stew/[byteutils], json_serialization/std/sets as jsonSets,
   # Internal
-  ./spec/[beaconstate, datatypes, crypto, digest, helpers],
-  ssz/merkleization,
-  ./block_pools/[spec_cache, chain_dag, clearance, quarantine],
-  ./beacon_node_types,
-  ./fork_choice/fork_choice
+  ../spec/[beaconstate, datatypes, crypto, digest, helpers],
+  ../ssz/merkleization,
+  "."/[spec_cache, blockchain_dag, block_clearance, block_quarantine],
+  ../beacon_node_types,
+  ../fork_choice/fork_choice
 
 export beacon_node_types
 

beacon_chain/consensus_object_pools/block_clearance.nim

@@ -14,7 +14,7 @@ import
   eth/keys,
   ../extras, ../time,
   ../spec/[crypto, datatypes, digest, helpers, signatures, signatures_batch, state_transition],
-  ./block_pools_types, ./chain_dag, ./quarantine
+  ./block_pools_types, ./blockchain_dag, ./block_quarantine
 
 export results
 

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -16,7 +16,7 @@ import
     crypto, datatypes, digest, helpers, validator, state_transition,
     beaconstate],
   ../time,
-  "."/[block_pools_types, quarantine]
+  "."/[block_pools_types, block_quarantine]
 
 export block_pools_types, helpers
 

beacon_chain/consensus_object_pools/exit_pool.nim

@@ -13,9 +13,9 @@ import
   # Status libraries
   chronicles, json_serialization/std/sets as jsonSets,
   # Internal
-  ./spec/[crypto, datatypes, helpers, state_transition_block],
-  ./block_pools/[chain_dag, clearance, quarantine],
-  ./beacon_node_types
+  ../spec/[crypto, datatypes, helpers, state_transition_block],
+  "."/[blockchain_dag, block_clearance, block_quarantine],
+  ../beacon_node_types
 
 export beacon_node_types, intsets
 

beacon_chain/consensus_object_pools/spec_cache.nim

@@ -14,7 +14,7 @@ import
     crypto, datatypes, digest, helpers, presets, signatures,
     validator],
   ../extras,
-  ./block_pools_types, ./chain_dag
+  ./block_pools_types, ./blockchain_dag
 
 # Spec functions implemented based on cached values instead of the full state
 func count_active_validators*(epochInfo: EpochRef): uint64 =

beacon_chain/eth2_processor.nim

@@ -12,10 +12,10 @@ import
   stew/results,
   chronicles, chronos, metrics,
   ./spec/[crypto, datatypes, digest],
-  ./block_pools/[clearance, chain_dag],
-  ./attestation_aggregation, ./exit_pool,
+  ./consensus_object_pools/[block_clearance, blockchain_dag, exit_pool, attestation_pool],
+  ./attestation_aggregation,
   ./validators/validator_pool,
-  ./beacon_node_types, ./attestation_pool,
+  ./beacon_node_types,
   ./time, ./conf, ./ssz/sszdump
 
 # Metrics for tracking attestation and beacon block loss

beacon_chain/fork_choice/fork_choice.nim

@@ -16,7 +16,7 @@ import
   ../spec/[beaconstate, datatypes, digest, helpers],
   # Fork choice
   ./fork_choice_types, ./proto_array,
-  ../block_pools/[spec_cache, chain_dag]
+  ../consensus_object_pools/[spec_cache, blockchain_dag]
 
 export results, fork_choice_types
 export proto_array.len

beacon_chain/fork_choice/fork_choice_types.nim

@@ -16,7 +16,7 @@ import
   chronicles,
   # Internal
   ../spec/[datatypes, digest],
-  ../block_pools/block_pools_types
+  ../consensus_object_pools/block_pools_types
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/fork-choice.md
 # This is a port of https://github.com/sigp/lighthouse/pull/804

beacon_chain/nimbus_beacon_node.nim

@@ -23,9 +23,9 @@ import
 
   # Local modules
   "."/[
-    attestation_aggregation, attestation_pool, beacon_chain_db,
+    attestation_aggregation, beacon_chain_db,
     beacon_node_common, beacon_node_status, beacon_node_types, conf,
-    eth2_discovery, eth2_network, eth2_processor, exit_pool,
+    eth2_discovery, eth2_network, eth2_processor,
     extras, filepath, interop, network_metadata,
     nimbus_binary_common, ssz/merkleization, statusbar,
     time, version],
@@ -36,7 +36,7 @@ import
   ./spec/[
     datatypes, digest, crypto, beaconstate, eth2_apis/beacon_rpc_client,
     helpers, network, presets, validator, weak_subjectivity, signatures],
-  ./block_pools/[chain_dag, quarantine, clearance, block_pools_types],
+  ./consensus_object_pools/[blockchain_dag, block_quarantine, block_clearance, block_pools_types, attestation_pool, exit_pool],
   ./eth1/eth1_monitor
 
 from eth/common/eth_types import BlockHashOrNumber

beacon_chain/rpc/beacon_api.nim

@@ -12,7 +12,7 @@ import
   nimcrypto/utils as ncrutils,
   ../beacon_node_common, ../eth2_json_rpc_serialization, ../eth2_network,
   ../validators/validator_duties,
-  ../block_pools/chain_dag, ../exit_pool,
+  ../consensus_object_pools/[blockchain_dag, exit_pool],
   ../spec/[crypto, digest, datatypes, validator, network],
   ../spec/eth2_apis/callsigs_types,
   ../ssz/merkleization,

beacon_chain/rpc/rpc_utils.nim

@@ -2,10 +2,10 @@ import
   std/[strutils, parseutils],
   stew/byteutils,
   ../beacon_node_common, ../validators/validator_duties,
-  ../block_pools/[block_pools_types, chain_dag],
+  ../consensus_object_pools/[block_pools_types, blockchain_dag],
   ../spec/[datatypes, digest, helpers]
 
-export chain_dag
+export blockchain_dag
 
 template withStateForStateId*(stateId: string, body: untyped): untyped =
   # TODO this can be optimized for the "head" case since that should be most common

beacon_chain/rpc/validator_api.nim

@@ -17,8 +17,8 @@ import
   # Local modules
   ../spec/[datatypes, digest, crypto, helpers, network, signatures],
   ../spec/eth2_apis/callsigs_types,
-  ../block_pools/[chain_dag, spec_cache], ../ssz/merkleization,
-  ../beacon_node_common, ../beacon_node_types, ../attestation_pool,
+  ../consensus_object_pools/[blockchain_dag, spec_cache, attestation_pool], ../ssz/merkleization,
+  ../beacon_node_common, ../beacon_node_types,
   ../validators/validator_duties, ../eth2_network,
   ../eth2_json_rpc_serialization,
   ./rpc_utils

beacon_chain/sync/sync_manager.nim

@@ -5,7 +5,7 @@ import ../spec/[datatypes, digest, helpers, eth2_apis/callsigs_types],
        ../peer_pool, ../eth2_network
 
 import ../eth2_processor
-import ../block_pools/block_pools_types
+import ../consensus_object_pools/block_pools_types
 export datatypes, digest, chronos, chronicles, results, block_pools_types
 
 logScope:

beacon_chain/sync/sync_protocol.nim

@@ -3,7 +3,7 @@ import
   chronicles, chronos, stew/ranges/bitranges, libp2p/switch,
   ../spec/[datatypes, network, crypto, digest],
   ".."/[beacon_node_types, eth2_network,
-      block_pools/chain_dag]
+      consensus_object_pools/blockchain_dag]
 
 logScope:
   topics = "sync"

beacon_chain/validators/validator_duties.nim

@@ -22,8 +22,9 @@ import
     datatypes, digest, crypto, helpers, network, signatures, state_transition,
     validator],
   ../conf, ../time,
-  ../attestation_pool, ../exit_pool,
-  ../block_pools/[spec_cache, chain_dag, clearance],
+  ../consensus_object_pools/[
+    spec_cache, blockchain_dag, block_clearance,
+    attestation_pool, exit_pool],
   ../eth1/eth1_monitor,
   ../eth2_network, ../beacon_node_common,
   ../beacon_node_types, ../nimbus_binary_common, ../version,

docs/nbc_audit_2020/eth2_spec_core/fork_choice_logic.md

@@ -7,11 +7,11 @@ repositories: "nim-beacon-chain"
 ---
 
 Fork choice backend:
-- [https://github.com/status-im/nim-beacon-chain/tree/devel/beacon_chain/fork_choice](https://github.com/status-im/nim-beacon-chain/tree/devel/beacon_chain/fork_choice)
+- [https://github.com/status-im/nim-beacon-chain/tree/unstable/beacon_chain/fork_choice](https://github.com/status-im/nim-beacon-chain/tree/unstable/beacon_chain/fork_choice)
 
 Fork choice is provided by the "attestation_pool" when "select_head" is called
 
-- [https://github.com/status-im/nim-beacon-chain/blob/devel/beacon_chain/attestation_pool.nim](https://github.com/status-im/nim-beacon-chain/blob/devel/beacon_chain/attestation_pool.nim)
+- [https://github.com/status-im/nim-beacon-chain/blob/unstable/beacon_chain/consensus_object_pools/attestation_pool.nim](https://github.com/status-im/nim-beacon-chain/blob/unstable/beacon_chain/consensus_object_pools/attestation_pool.nim)
 
 Tests:
 - [https://github.com/status-im/nim-beacon-chain/tree/master/tests/fork_choice](https://github.com/status-im/nim-beacon-chain/tree/master/tests/fork_choice)

ncli/ncli_db.nim

@@ -3,7 +3,7 @@ import
   chronicles, confutils, stew/byteutils, eth/db/kvstore_sqlite3,
   ../beacon_chain/network_metadata,
   ../beacon_chain/[beacon_chain_db, extras],
-  ../beacon_chain/block_pools/chain_dag,
+  ../beacon_chain/consensus_object_pools/blockchain_dag,
   ../beacon_chain/spec/[crypto, datatypes, digest, helpers,
                         state_transition, presets],
   ../beacon_chain/ssz, ../beacon_chain/ssz/sszdump,

research/block_sim.nim

@@ -22,11 +22,10 @@ import
   ../tests/[testblockutil],
   ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, presets,
                         helpers, validator, signatures, state_transition],
-  ../beacon_chain/[
-    attestation_pool, beacon_node_types, beacon_chain_db, extras],
+  ../beacon_chain/[beacon_node_types, beacon_chain_db, extras],
   ../beacon_chain/eth1/eth1_monitor,
   ../beacon_chain/validators/validator_pool,
-  ../beacon_chain/block_pools/[chain_dag, quarantine, clearance],
+  ../beacon_chain/consensus_object_pools/[blockchain_dag, block_quarantine, block_clearance, attestation_pool],
   ../beacon_chain/ssz/[merkleization, ssz_serialization],
   ./simutils
 

tests/test_attestation_pool.nim

@@ -17,10 +17,9 @@ import
   # Internal
   ../beacon_chain/spec/[crypto, datatypes, digest, validator, state_transition,
                         helpers, beaconstate, presets, network],
-  ../beacon_chain/[
-    beacon_node_types, attestation_pool, attestation_aggregation, extras, time],
+  ../beacon_chain/[beacon_node_types, attestation_aggregation, extras, time],
   ../beacon_chain/fork_choice/[fork_choice_types, fork_choice],
-  ../beacon_chain/block_pools/[quarantine, chain_dag, clearance],
+  ../beacon_chain/consensus_object_pools/[block_quarantine, blockchain_dag, block_clearance, attestation_pool],
   # Test utilities
   ./testutil, ./testblockutil
 

tests/test_block_pool.nim

@@ -15,7 +15,7 @@ import
   ../beacon_chain/spec/[datatypes, digest, helpers, state_transition, presets],
   ../beacon_chain/beacon_node_types,
   ../beacon_chain/ssz,
-  ../beacon_chain/block_pools/[chain_dag, quarantine, clearance]
+  ../beacon_chain/consensus_object_pools/[blockchain_dag, block_quarantine, block_clearance]
 
 when isMainModule:
   import chronicles # or some random compile error happens...

tests/test_exit_pool.nim

@@ -11,8 +11,7 @@ import std/unittest
 import chronicles, chronos, testutil
 import eth/keys
 import ../beacon_chain/spec/[datatypes, presets]
-import ../beacon_chain/exit_pool
-import ../beacon_chain/block_pools/[quarantine, chain_dag]
+import ../beacon_chain/consensus_object_pools/[block_quarantine, blockchain_dag, exit_pool]
 
 proc getExitPool(): auto =
   let chainDag =

tests/test_statediff.nim

@@ -16,7 +16,7 @@ import
     state_transition, presets],
   ../beacon_chain/[beacon_node_types, statediff],
   ../beacon_chain/ssz,
-  ../beacon_chain/block_pools/[chain_dag, quarantine, clearance]
+  ../beacon_chain/consensus_object_pools/[blockchain_dag, block_quarantine, block_clearance]
 
 when isMainModule:
   import chronicles # or some random compile error happens...

tests/testutil.nim

@@ -11,7 +11,7 @@ import
   ../beacon_chain/[beacon_chain_db, extras],
   ../beacon_chain/ssz,
   ../beacon_chain/spec/[digest, beaconstate, datatypes, presets],
-  ../beacon_chain/block_pools/chain_dag,
+  ../beacon_chain/consensus_object_pools/blockchain_dag,
   eth/db/[kvstore, kvstore_sqlite3],
   testblockutil