Reorg (5/5) (#2377)

* Reorg things left into networking and gossip_processing

* time -> beacon_clock

* fix builds

beacon_chain/beacon_chain_db.nim

@@ -5,11 +5,11 @@ import
   stew/[endians2, io2, objects, results],
   serialization, chronicles, snappy,
   eth/db/[kvstore, kvstore_sqlite3],
-  ./network_metadata,
+  ./networking/network_metadata,
   ./spec/[crypto, datatypes, digest, state_transition],
   ./ssz/[ssz_serialization, merkleization],
   ./eth1/merkle_minimal,
-  filepath
+  ./filepath
 
 type
   DbSeq*[T] = object

beacon_chain/beacon_clock.nim

@@ -2,7 +2,7 @@
 
 import
   chronos, chronicles,
-  spec/datatypes
+  ./spec/datatypes
 
 from times import Time, getTime, fromUnix, `<`, `-`, inNanoseconds
 
@@ -115,4 +115,3 @@ func shortLog*(v: BeaconTime): string = $(Duration v)
 
 chronicles.formatIt Duration: $it
 chronicles.formatIt BeaconTime: $(Duration it)
-

beacon_chain/beacon_node_common.nim

@@ -14,16 +14,17 @@ import
   chronos, json_rpc/rpcserver,
 
   # Local modules
-  ./conf, ./time, ./beacon_chain_db, ./eth2_network,
+  ./conf, ./beacon_clock, ./beacon_chain_db,
   ./beacon_node_types,
-  ./eth2_processor,
+  ./gossip_processing/eth2_processor,
+  ./networking/eth2_network,
   ./eth1/eth1_monitor,
   ./consensus_object_pools/[blockchain_dag, block_quarantine, attestation_pool],
   ./spec/datatypes,
   ./sync/[sync_manager, request_manager]
 
 export
-  osproc, chronos, rpcserver, conf, time, beacon_chain_db,
+  osproc, chronos, rpcserver, conf, beacon_clock, beacon_chain_db,
   attestation_pool, eth2_network, beacon_node_types, eth1_monitor,
   request_manager, sync_manager, eth2_processor, blockchain_dag, block_quarantine,
   datatypes

beacon_chain/conf.nim

@@ -9,7 +9,8 @@ import
   eth/p2p/discoveryv5/enr,
   json_serialization, web3/[ethtypes, confutils_defs],
   spec/[crypto, keystore, digest, datatypes, network],
-  network_metadata, filepath
+  ./networking/network_metadata,
+  filepath
 
 export
   uri,

beacon_chain/consensus_object_pools/block_clearance.nim

@@ -12,7 +12,7 @@ import
   chronicles,
   stew/[assign2, results],
   eth/keys,
-  ../extras, ../time,
+  ../extras, ../beacon_clock,
   ../spec/[crypto, datatypes, digest, helpers, signatures, signatures_batch, state_transition],
   ./block_pools_types, ./blockchain_dag, ./block_quarantine
 

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -15,7 +15,7 @@ import
   ../spec/[
     crypto, datatypes, digest, helpers, validator, state_transition,
     beaconstate],
-  ../time,
+  ../beacon_clock,
   "."/[block_pools_types, block_quarantine]
 
 export block_pools_types, helpers

beacon_chain/eth1/deposit_contract.nim

@@ -1,7 +1,8 @@
 import
   os, sequtils, strutils, options, json, terminal, random,
   chronos, chronicles, confutils, stint, json_serialization,
-  ".."/[network_metadata, filepath],
+  ../filepath,
+  ../networking/network_metadata,
   web3, web3/confutils_defs, eth/keys, stew/io2,
   ../spec/[datatypes, crypto, presets], ../ssz/merkleization,
   ../validators/keystore_management

beacon_chain/eth1/eth1_monitor.nim

@@ -6,8 +6,9 @@ import
   web3, web3/ethtypes as web3Types, eth/common/eth_types, eth/async_utils,
   # Local modules:
   ../spec/[datatypes, digest, crypto, helpers],
+  ../networking/network_metadata,
   ../ssz,
-  ".."/[beacon_chain_db, network_metadata, beacon_node_status],
+  ".."/[beacon_chain_db, beacon_node_status],
   ./merkle_minimal
 
 export

beacon_chain/gossip_processing/README.md

@@ -0,0 +1,34 @@
+# Gossip Processing
+
+This folders hold a collection of modules to:
+- validate raw gossip data before
+  - rebroadcasting them (potentially aggregated)
+  - sending it to one of the consensus object pool
+
+## Validation
+
+Gossip Validation is different from consensus verification in particular for blocks.
+
+- Blocks: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#beacon_block
+- Attestations (aggregate): https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#beacon_aggregate_and_proof
+- Attestations (single): https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#attestation-subnets
+- Exits: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#voluntary_exit
+- Proposer slashings: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#proposer_slashing
+- Attester slashing: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#attester_slashing
+
+There are 2 consumers of validated consensus objects:
+- a `ValidationResult.Accept` output triggers rebroadcasting in libp2p
+  - method `validate(PubSub, message)` in libp2p/protocols/pubsub/pubsub.nim in the
+  - which was called by `rpcHandler(GossipSub, PubSubPeer, RPCMsg)`
+- a `xyzValidator` message enqueues the validated object in one of the processing queue in eth2_processor
+  - `blocksQueue: AsyncQueue[BlockEntry]`, (shared with request_manager and sync_manager)
+  - `attestationsQueue: AsyncQueue[AttestationEntry]`
+  - `aggregatesQueue: AsyncQueue[AggregateEntry]`
+
+Those queues are then regularly processed to be made available to the consensus object pools.
+
+## Security concerns
+
+As the first line of defense in Nimbus, modules must be able to handle burst of data that may come:
+- from malicious nodes trying to DOS us
+- from long periods of non-finality, creating lots of forks, attestations, forks

beacon_chain/gossip_processing/attestation_aggregation.nim

@@ -10,13 +10,13 @@
 import
   std/[options, sequtils],
   chronos, chronicles,
-  ./spec/[
+  ../spec/[
     beaconstate, datatypes, crypto, digest, helpers, network, signatures],
-  ./consensus_object_pools/[
+  ../consensus_object_pools/[
     spec_cache, blockchain_dag, block_quarantine, spec_cache,
     attestation_pool
   ],
-  ./beacon_node_types, ./ssz, ./time
+  ".."/[beacon_node_types, ssz, beacon_clock]
 
 logScope:
   topics = "att_aggr"

beacon_chain/gossip_processing/eth2_processor.nim

@@ -11,12 +11,12 @@ import
   std/[math, tables],
   stew/results,
   chronicles, chronos, metrics,
-  ./spec/[crypto, datatypes, digest],
+  ../spec/[crypto, datatypes, digest],
-  ./consensus_object_pools/[block_clearance, blockchain_dag, exit_pool, attestation_pool],
+  ../consensus_object_pools/[block_clearance, blockchain_dag, exit_pool, attestation_pool],
   ./attestation_aggregation,
-  ./validators/validator_pool,
+  ../validators/validator_pool,
-  ./beacon_node_types,
+  ../beacon_node_types,
-  ./time, ./conf, ./ssz/sszdump
+  ../beacon_clock, ../conf, ../ssz/sszdump
 
 # Metrics for tracking attestation and beacon block loss
 declareCounter beacon_attestations_received,

beacon_chain/networking/README.md

@@ -0,0 +1,14 @@
+# Networking
+
+This folders hold a collection of modules to:
+- configure the Eth2 P2P network
+- discover, connect, and maintain quality Eth2 peers
+
+Data received is handed other to the `../gossip_processing` modules for validation.
+
+## Security concerns
+
+- Collusion: part of the peer selection must be kept random. This avoids peers bringing all their friends and colluding against a beacon node.
+- Denial-of-service: The beacon node must provide ways to handle burst of data that may come:
+  - from malicious nodes trying to DOS us
+  - from long periods of non-finality, creating lots of forks, attestations, forks

beacon_chain/networking/eth2_discovery.nim

@@ -4,7 +4,7 @@ import
   std/[os, strutils],
   chronicles, stew/shims/net, stew/results, bearssl,
   eth/keys, eth/p2p/discoveryv5/[enr, protocol, node],
-  ./conf
+  ../conf
 
 export protocol, keys
 

beacon_chain/networking/eth2_network.nim

@@ -21,11 +21,12 @@ import
   libp2p/stream/connection,
   eth/[keys, async_utils], eth/p2p/p2p_protocol_dsl,
   eth/net/nat, eth/p2p/discoveryv5/[enr, node],
-  "."/[
+  ".."/[
-    version, conf, eth2_discovery, libp2p_json_serialization,
+    version, conf,
-    ssz/ssz_serialization, peer_pool, time],
+    ssz/ssz_serialization, beacon_clock],
-  ./spec/[datatypes, digest, helpers, network],
+  ../spec/[datatypes, digest, helpers, network],
-  ./validators/keystore_management
+  ../validators/keystore_management,
+  ./eth2_discovery, ./peer_pool, ./libp2p_json_serialization
 
 import libp2p/protocols/pubsub/gossipsub
 

beacon_chain/networking/network_metadata.nim

@@ -12,8 +12,8 @@ import
   chronicles,
   json_serialization,
   json_serialization/std/[options, sets, net], serialization/errors,
-  ssz/navigator,
+  ../ssz/navigator,
-  spec/[presets, datatypes, digest]
+  ../spec/[presets, datatypes, digest]
 
 # ATTENTION! This file will produce a large C file, because we are inlining
 # genesis states as C literals in the generated code (and blobs in the final
@@ -74,7 +74,7 @@ type
       incompatibilityDesc*: string
 
 const
-  eth2testnetsDir = currentSourcePath.parentDir.replace('\\', '/') & "/../vendor/eth2-testnets"
+  eth2testnetsDir = currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor/eth2-testnets"
 
 const presetValueLoaders = genExpr(nnkBracket):
   for constName in PresetValue:
@@ -228,4 +228,3 @@ proc getRuntimePresetForNetwork*(eth2Network: Option[string]): RuntimePreset =
 
 proc extractGenesisValidatorRootFromSnapshop*(snapshot: string): Eth2Digest =
   sszMount(snapshot, BeaconState).genesis_validators_root[]
-

beacon_chain/nimbus_beacon_node.nim

@@ -23,12 +23,13 @@ import
 
   # Local modules
   "."/[
-    attestation_aggregation, beacon_chain_db,
+    beacon_chain_db,
     beacon_node_common, beacon_node_status, beacon_node_types, conf,
-    eth2_discovery, eth2_network, eth2_processor,
+    extras, filepath, interop,
-    extras, filepath, interop, network_metadata,
     nimbus_binary_common, ssz/merkleization, statusbar,
-    time, version],
+    beacon_clock, version],
+  ./networking/[eth2_discovery, eth2_network, network_metadata],
+  ./gossip_processing/[eth2_processor, attestation_aggregation],
   ./validators/[validator_duties, validator_pool, slashing_protection, keystore_management],
   ./sync/[sync_manager, sync_protocol, request_manager],
   ./rpc/[beacon_api, config_api, debug_api, event_api, nimbus_api, node_api,

beacon_chain/nimbus_binary_common.nim

@@ -17,7 +17,8 @@ import
   stew/io2,
 
   # Local modules
-  spec/[datatypes, crypto, helpers], eth2_network, time, filepath
+  ./spec/[datatypes, crypto, helpers], beacon_clock, filepath,
+  ./networking/eth2_network
 
 proc setupStdoutLogging*(logLevel: string) =
   when compiles(defaultChroniclesStream.output.writer):
@@ -80,7 +81,7 @@ template makeBannerAndConfig*(clientId: string, ConfType: type): untyped =
   {.pop.}
   config
 
-# TODO not sure if this belongs here but it doesn't belong in `time.nim` either
+# TODO not sure if this belongs here but it doesn't belong in `beacon_clock.nim` either
 proc sleepToSlotOffset*(clock: BeaconClock, extra: chronos.Duration,
                         slot: Slot, msg: static string): Future[bool] {.async.} =
   let

beacon_chain/nimbus_signing_process.nim

@@ -10,8 +10,8 @@ import
   os, strutils, tables,
 
   # Local modules
-  spec/[digest, crypto],
+  ./spec/[digest, crypto],
-  validators/keystore_management
+  ./validators/keystore_management
 
 programMain:
   var validators: Table[ValidatorPubKey, ValidatorPrivKey]

beacon_chain/nimbus_validator_client.nim

@@ -16,18 +16,19 @@ import
   json_serialization/std/[options, net],
 
   # Local modules
-  spec/[datatypes, digest, crypto, helpers, network, signatures],
+  ./spec/[datatypes, digest, crypto, helpers, network, signatures],
-  spec/eth2_apis/beacon_rpc_client,
+  ./spec/eth2_apis/beacon_rpc_client,
-  sync/sync_manager,
+  ./sync/sync_manager,
-  conf, time, version,
+  "."/[conf, beacon_clock, version],
-  eth2_network, eth2_discovery, beacon_node_types,
+  ./networking/[eth2_network, eth2_discovery],
-  attestation_aggregation,
+  ./rpc/eth2_json_rpc_serialization,
-  nimbus_binary_common,
+  ./beacon_node_types,
-  version, ssz/merkleization,
+  ./gossip_processing/attestation_aggregation,
-  spec/eth2_apis/callsigs_types,
+  ./nimbus_binary_common,
-  eth2_json_rpc_serialization,
+  ./ssz/merkleization,
-  validators/[keystore_management, validator_pool, slashing_protection],
+  ./spec/eth2_apis/callsigs_types,
-  eth/db/[kvstore, kvstore_sqlite3]
+  ./validators/[keystore_management, validator_pool, slashing_protection],
+  ./eth/db/[kvstore, kvstore_sqlite3]
 
 logScope: topics = "vc"
 

beacon_chain/rpc/beacon_api.nim

@@ -10,13 +10,14 @@ import
   json_rpc/[rpcserver, jsonmarshal],
   chronicles,
   nimcrypto/utils as ncrutils,
-  ../beacon_node_common, ../eth2_json_rpc_serialization, ../eth2_network,
+  ../beacon_node_common,
+  ../networking/eth2_network,
   ../validators/validator_duties,
   ../consensus_object_pools/[blockchain_dag, exit_pool],
   ../spec/[crypto, digest, datatypes, validator, network],
   ../spec/eth2_apis/callsigs_types,
   ../ssz/merkleization,
-  ./rpc_utils
+  ./rpc_utils, ./eth2_json_rpc_serialization
 
 logScope: topics = "beaconapi"
 

beacon_chain/rpc/debug_api.nim

@@ -2,10 +2,10 @@ import
   std/sequtils,
   json_rpc/[rpcserver, jsonmarshal],
   chronicles,
-  ../version, ../beacon_node_common, ../eth2_json_rpc_serialization,
+  ../version, ../beacon_node_common,
-  ../eth2_network, ../peer_pool,
+  ../networking/[eth2_network, peer_pool],
   ../spec/[datatypes, digest, presets],
-  ./rpc_utils
+  ./rpc_utils, ./eth2_json_rpc_serialization
 
 logScope: topics = "debugapi"
 
@@ -20,4 +20,3 @@ proc installDebugApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
 
   rpcServer.rpc("get_v1_debug_beacon_heads") do () -> seq[tuple[root: Eth2Digest, slot: Slot]]:
     return node.chainDag.heads.mapIt((it.root, it.slot))
-

beacon_chain/rpc/eth2_json_rpc_serialization.nim

@@ -3,11 +3,12 @@ import
   std/[tables, json, typetraits],
 
   # Nimble packages
-  stew/byteutils, ssz/types,
+  stew/byteutils,
   json_rpc/jsonmarshal,
 
   # Local modules
-  spec/[datatypes, crypto, digest]
+  ../ssz/types,
+  ../spec/[datatypes, crypto, digest]
 
 proc toJsonHex(data: openArray[byte]): string =
   # Per the eth2 API spec, hex arrays are printed with leading 0x

beacon_chain/rpc/nimbus_api.nim

@@ -12,7 +12,8 @@ import
   json_rpc/[rpcserver, jsonmarshal],
 
   rpc_utils,
-  ../beacon_node_common, ../nimbus_binary_common, ../eth2_network,
+  ../beacon_node_common, ../nimbus_binary_common,
+  ../networking/eth2_network,
   ../eth1/eth1_monitor,
   ../validators/validator_duties,
   ../spec/[digest, datatypes, presets],

beacon_chain/rpc/node_api.nim

@@ -4,8 +4,9 @@ import std/options,
   eth/p2p/discoveryv5/enr,
   libp2p/[multiaddress, multicodec],
   nimcrypto/utils as ncrutils,
-  ../beacon_node_common, ../eth2_network, ../sync/sync_manager,
+  ../beacon_node_common, ../version,
-  ../peer_pool, ../version,
+  ../networking/[eth2_network, peer_pool],
+  ../sync/sync_manager,
   ../spec/[datatypes, digest, presets],
   ../spec/eth2_apis/callsigs_types
 

beacon_chain/rpc/validator_api.nim

@@ -19,8 +19,9 @@ import
   ../spec/eth2_apis/callsigs_types,
   ../consensus_object_pools/[blockchain_dag, spec_cache, attestation_pool], ../ssz/merkleization,
   ../beacon_node_common, ../beacon_node_types,
-  ../validators/validator_duties, ../eth2_network,
+  ../validators/validator_duties,
-  ../eth2_json_rpc_serialization,
+  ../networking/eth2_network,
+  ./eth2_json_rpc_serialization,
   ./rpc_utils
 
 logScope: topics = "valapi"

beacon_chain/spec/eth2_apis/beacon_rpc_client.nim

@@ -1,7 +1,7 @@
 import
   std/[os, json],
   json_rpc/[rpcclient, jsonmarshal],
-  ../../eth2_json_rpc_serialization,
+  ../../rpc/eth2_json_rpc_serialization,
   ../crypto, ../digest, ../datatypes,
   callsigs_types
 

beacon_chain/statediff.nim

@@ -8,8 +8,8 @@
 {.push raises: [Defect].}
 
 import
-  ssz/types,
+  ./ssz/types,
-  spec/[datatypes, digest, helpers]
+  ./spec/[datatypes, digest, helpers]
 
 func diffModIncEpoch[T, U](hl: HashArray[U, T], startSlot: uint64):
     array[SLOTS_PER_EPOCH, T] =

beacon_chain/sync/request_manager.nim

@@ -1,8 +1,10 @@
 import options, sequtils, strutils
 import chronos, chronicles
 import ../spec/[datatypes, digest],
-       ".."/[eth2_network, beacon_node_types,
+       ../networking/eth2_network,
-           ssz/merkleization, eth2_processor],
+       ../beacon_node_types,
+       ../ssz/merkleization,
+       ../gossip_processing/eth2_processor,
        ./sync_protocol, ./sync_manager
 export sync_manager
 

beacon_chain/sync/sync_manager.nim

@@ -2,9 +2,9 @@ import chronicles
 import options, deques, heapqueue, tables, strutils, sequtils, math, algorithm
 import stew/results, chronos, chronicles
 import ../spec/[datatypes, digest, helpers, eth2_apis/callsigs_types],
-       ../peer_pool, ../eth2_network
+       ../networking/[peer_pool, eth2_network]
 
-import ../eth2_processor
+import ../gossip_processing/eth2_processor
 import ../consensus_object_pools/block_pools_types
 export datatypes, digest, chronos, chronicles, results, block_pools_types
 

beacon_chain/sync/sync_protocol.nim

@@ -2,8 +2,9 @@ import
   options, tables, sets, macros,
   chronicles, chronos, stew/ranges/bitranges, libp2p/switch,
   ../spec/[datatypes, network, crypto, digest],
-  ".."/[beacon_node_types, eth2_network,
+  ../beacon_node_types,
-      consensus_object_pools/blockchain_dag]
+  ../networking/eth2_network,
+  ../consensus_object_pools/blockchain_dag
 
 logScope:
   topics = "sync"

beacon_chain/validators/keystore_management.nim

@@ -5,7 +5,8 @@ import
   ../spec/[datatypes, digest, crypto, keystore],
   stew/io2, libp2p/crypto/crypto as lcrypto,
   nimcrypto/utils as ncrutils,
-  ".."/[conf, ssz/merkleization, network_metadata, filepath]
+  ".."/[conf, ssz/merkleization, filepath],
+  ../networking/network_metadata
 
 export
   keystore

beacon_chain/validators/validator_duties.nim

@@ -21,14 +21,15 @@ import
   ../spec/[
     datatypes, digest, crypto, helpers, network, signatures, state_transition,
     validator],
-  ../conf, ../time,
+  ../conf, ../beacon_clock,
   ../consensus_object_pools/[
     spec_cache, blockchain_dag, block_clearance,
     attestation_pool, exit_pool],
   ../eth1/eth1_monitor,
-  ../eth2_network, ../beacon_node_common,
+  ../networking/eth2_network,
-  ../beacon_node_types, ../nimbus_binary_common, ../version,
+  ".."/[beacon_node_common, beacon_node_types, nimbus_binary_common, version],
-  ../ssz, ../ssz/sszdump, ../attestation_aggregation, ../sync/sync_manager,
+  ../gossip_processing/attestation_aggregation,
+  ../ssz, ../ssz/sszdump, ../sync/sync_manager,
   ./slashing_protection,
   ./validator_pool, ./keystore_management
 

ncli/ncli.nim

@@ -5,7 +5,7 @@ import
   ../research/simutils,
   ../beacon_chain/spec/[crypto, datatypes, digest, helpers, state_transition],
   ../beacon_chain/extras,
-  ../beacon_chain/network_metadata,
+  ../beacon_chain/networking/network_metadata,
   ../beacon_chain/ssz/[merkleization, ssz_serialization]
 
 type

ncli/ncli_db.nim

@@ -1,7 +1,7 @@
 import
   os, stats, strformat, tables,
   chronicles, confutils, stew/byteutils, eth/db/kvstore_sqlite3,
-  ../beacon_chain/network_metadata,
+  ../beacon_chain/networking/network_metadata,
   ../beacon_chain/[beacon_chain_db, extras],
   ../beacon_chain/consensus_object_pools/blockchain_dag,
   ../beacon_chain/spec/[crypto, datatypes, digest, helpers,

tests/test_attestation_pool.nim

@@ -17,7 +17,8 @@ import
   # Internal
   ../beacon_chain/spec/[crypto, datatypes, digest, validator, state_transition,
                         helpers, beaconstate, presets, network],
-  ../beacon_chain/[beacon_node_types, attestation_aggregation, extras, time],
+  ../beacon_chain/[beacon_node_types, extras, beacon_clock],
+  ../beacon_chain/gossip_processing/attestation_aggregation,
   ../beacon_chain/fork_choice/[fork_choice_types, fork_choice],
   ../beacon_chain/consensus_object_pools/[block_quarantine, blockchain_dag, block_clearance, attestation_pool],
   # Test utilities

tests/test_honest_validator.nim

@@ -3,7 +3,7 @@
 import
   unittest, ./testutil,
   ../beacon_chain/spec/[crypto, datatypes, network],
-  ../beacon_chain/attestation_aggregation
+  ../beacon_chain/gossip_processing/attestation_aggregation
 
 suiteReport "Honest validator":
   var forkDigest: ForkDigest

tests/test_peer_pool.nim

@@ -9,7 +9,7 @@
 
 import std/[unittest, random, heapqueue, tables]
 import chronos
-import ../beacon_chain/peer_pool
+import ../beacon_chain/networking/peer_pool
 import testutil
 
 type

tests/test_sync_manager.nim

@@ -2,7 +2,7 @@
 
 import unittest
 import chronos
-import ../beacon_chain/eth2_processor,
+import ../beacon_chain/gossip_processing/eth2_processor,
        ../beacon_chain/sync/sync_manager
 
 type