From d47f53cd9d9d42d738b0e7381b2eb7b380c9642a Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Fri, 5 Mar 2021 14:12:00 +0100 Subject: [PATCH] Reorg (5/5) (#2377) * Reorg things left into networking and gossip_processing * time -> beacon_clock * fix builds --- beacon_chain/beacon_chain_db.nim | 4 +-- beacon_chain/{time.nim => beacon_clock.nim} | 3 +- beacon_chain/beacon_node_common.nim | 7 ++-- beacon_chain/conf.nim | 3 +- .../block_clearance.nim | 2 +- .../consensus_object_pools/blockchain_dag.nim | 2 +- beacon_chain/eth1/deposit_contract.nim | 3 +- beacon_chain/eth1/eth1_monitor.nim | 3 +- beacon_chain/gossip_processing/README.md | 34 +++++++++++++++++++ .../attestation_aggregation.nim | 6 ++-- .../eth2_processor.nim | 10 +++--- beacon_chain/networking/README.md | 14 ++++++++ .../{ => networking}/eth2_discovery.nim | 2 +- .../{ => networking}/eth2_network.nim | 11 +++--- .../{ => networking}/faststreams_backend.nim | 0 .../libp2p_json_serialization.nim | 0 .../libp2p_streams_backend.nim | 0 .../{ => networking}/network_metadata.nim | 7 ++-- beacon_chain/{ => networking}/peer_pool.nim | 0 beacon_chain/nimbus_beacon_node.nim | 9 ++--- beacon_chain/nimbus_binary_common.nim | 5 +-- beacon_chain/nimbus_signing_process.nim | 4 +-- beacon_chain/nimbus_validator_client.nim | 25 +++++++------- beacon_chain/rpc/beacon_api.nim | 5 +-- beacon_chain/rpc/debug_api.nim | 7 ++-- .../{ => rpc}/eth2_json_rpc_serialization.nim | 5 +-- beacon_chain/rpc/nimbus_api.nim | 3 +- beacon_chain/rpc/node_api.nim | 5 +-- beacon_chain/rpc/validator_api.nim | 5 +-- .../spec/eth2_apis/beacon_rpc_client.nim | 2 +- beacon_chain/statediff.nim | 4 +-- beacon_chain/sync/request_manager.nim | 6 ++-- beacon_chain/sync/sync_manager.nim | 4 +-- beacon_chain/sync/sync_protocol.nim | 5 +-- .../validators/keystore_management.nim | 3 +- beacon_chain/validators/validator_duties.nim | 9 ++--- ncli/ncli.nim | 2 +- ncli/ncli_db.nim | 2 +- tests/test_attestation_pool.nim | 3 +- tests/test_honest_validator.nim | 2 +- tests/test_peer_pool.nim | 2 +- tests/test_sync_manager.nim | 2 +- 42 files changed, 147 insertions(+), 83 deletions(-) rename beacon_chain/{time.nim => beacon_clock.nim} (99%) create mode 100644 beacon_chain/gossip_processing/README.md rename beacon_chain/{ => gossip_processing}/attestation_aggregation.nim (99%) rename beacon_chain/{ => gossip_processing}/eth2_processor.nim (98%) create mode 100644 beacon_chain/networking/README.md rename beacon_chain/{ => networking}/eth2_discovery.nim (99%) rename beacon_chain/{ => networking}/eth2_network.nim (99%) rename beacon_chain/{ => networking}/faststreams_backend.nim (100%) rename beacon_chain/{ => networking}/libp2p_json_serialization.nim (100%) rename beacon_chain/{ => networking}/libp2p_streams_backend.nim (100%) rename beacon_chain/{ => networking}/network_metadata.nim (98%) rename beacon_chain/{ => networking}/peer_pool.nim (100%) rename beacon_chain/{ => rpc}/eth2_json_rpc_serialization.nim (98%) diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim index b80d1a2c1..12d0756df 100644 --- a/beacon_chain/beacon_chain_db.nim +++ b/beacon_chain/beacon_chain_db.nim @@ -5,11 +5,11 @@ import stew/[endians2, io2, objects, results], serialization, chronicles, snappy, eth/db/[kvstore, kvstore_sqlite3], - ./network_metadata, + ./networking/network_metadata, ./spec/[crypto, datatypes, digest, state_transition], ./ssz/[ssz_serialization, merkleization], ./eth1/merkle_minimal, - filepath + ./filepath type DbSeq*[T] = object diff --git a/beacon_chain/time.nim b/beacon_chain/beacon_clock.nim similarity index 99% rename from beacon_chain/time.nim rename to beacon_chain/beacon_clock.nim index 9b2c8f6c6..0875860c5 100644 --- a/beacon_chain/time.nim +++ b/beacon_chain/beacon_clock.nim @@ -2,7 +2,7 @@ import chronos, chronicles, - spec/datatypes + ./spec/datatypes from times import Time, getTime, fromUnix, `<`, `-`, inNanoseconds @@ -115,4 +115,3 @@ func shortLog*(v: BeaconTime): string = $(Duration v) chronicles.formatIt Duration: $it chronicles.formatIt BeaconTime: $(Duration it) - diff --git a/beacon_chain/beacon_node_common.nim b/beacon_chain/beacon_node_common.nim index 437d83f0f..95ccc668e 100644 --- a/beacon_chain/beacon_node_common.nim +++ b/beacon_chain/beacon_node_common.nim @@ -14,16 +14,17 @@ import chronos, json_rpc/rpcserver, # Local modules - ./conf, ./time, ./beacon_chain_db, ./eth2_network, + ./conf, ./beacon_clock, ./beacon_chain_db, ./beacon_node_types, - ./eth2_processor, + ./gossip_processing/eth2_processor, + ./networking/eth2_network, ./eth1/eth1_monitor, ./consensus_object_pools/[blockchain_dag, block_quarantine, attestation_pool], ./spec/datatypes, ./sync/[sync_manager, request_manager] export - osproc, chronos, rpcserver, conf, time, beacon_chain_db, + osproc, chronos, rpcserver, conf, beacon_clock, beacon_chain_db, attestation_pool, eth2_network, beacon_node_types, eth1_monitor, request_manager, sync_manager, eth2_processor, blockchain_dag, block_quarantine, datatypes diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index e410473fc..2306dfd72 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -9,7 +9,8 @@ import eth/p2p/discoveryv5/enr, json_serialization, web3/[ethtypes, confutils_defs], spec/[crypto, keystore, digest, datatypes, network], - network_metadata, filepath + ./networking/network_metadata, + filepath export uri, diff --git a/beacon_chain/consensus_object_pools/block_clearance.nim b/beacon_chain/consensus_object_pools/block_clearance.nim index 01590a2ed..8dbb9317d 100644 --- a/beacon_chain/consensus_object_pools/block_clearance.nim +++ b/beacon_chain/consensus_object_pools/block_clearance.nim @@ -12,7 +12,7 @@ import chronicles, stew/[assign2, results], eth/keys, - ../extras, ../time, + ../extras, ../beacon_clock, ../spec/[crypto, datatypes, digest, helpers, signatures, signatures_batch, state_transition], ./block_pools_types, ./blockchain_dag, ./block_quarantine diff --git a/beacon_chain/consensus_object_pools/blockchain_dag.nim b/beacon_chain/consensus_object_pools/blockchain_dag.nim index 725a7ac2a..297912278 100644 --- a/beacon_chain/consensus_object_pools/blockchain_dag.nim +++ b/beacon_chain/consensus_object_pools/blockchain_dag.nim @@ -15,7 +15,7 @@ import ../spec/[ crypto, datatypes, digest, helpers, validator, state_transition, beaconstate], - ../time, + ../beacon_clock, "."/[block_pools_types, block_quarantine] export block_pools_types, helpers diff --git a/beacon_chain/eth1/deposit_contract.nim b/beacon_chain/eth1/deposit_contract.nim index abdfb6148..52f7b2f3d 100644 --- a/beacon_chain/eth1/deposit_contract.nim +++ b/beacon_chain/eth1/deposit_contract.nim @@ -1,7 +1,8 @@ import os, sequtils, strutils, options, json, terminal, random, chronos, chronicles, confutils, stint, json_serialization, - ".."/[network_metadata, filepath], + ../filepath, + ../networking/network_metadata, web3, web3/confutils_defs, eth/keys, stew/io2, ../spec/[datatypes, crypto, presets], ../ssz/merkleization, ../validators/keystore_management diff --git a/beacon_chain/eth1/eth1_monitor.nim b/beacon_chain/eth1/eth1_monitor.nim index 3ba4b9962..1351dfffe 100644 --- a/beacon_chain/eth1/eth1_monitor.nim +++ b/beacon_chain/eth1/eth1_monitor.nim @@ -6,8 +6,9 @@ import web3, web3/ethtypes as web3Types, eth/common/eth_types, eth/async_utils, # Local modules: ../spec/[datatypes, digest, crypto, helpers], + ../networking/network_metadata, ../ssz, - ".."/[beacon_chain_db, network_metadata, beacon_node_status], + ".."/[beacon_chain_db, beacon_node_status], ./merkle_minimal export diff --git a/beacon_chain/gossip_processing/README.md b/beacon_chain/gossip_processing/README.md new file mode 100644 index 000000000..efa325bc9 --- /dev/null +++ b/beacon_chain/gossip_processing/README.md @@ -0,0 +1,34 @@ +# Gossip Processing + +This folders hold a collection of modules to: +- validate raw gossip data before + - rebroadcasting them (potentially aggregated) + - sending it to one of the consensus object pool + +## Validation + +Gossip Validation is different from consensus verification in particular for blocks. + +- Blocks: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#beacon_block +- Attestations (aggregate): https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#beacon_aggregate_and_proof +- Attestations (single): https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#attestation-subnets +- Exits: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#voluntary_exit +- Proposer slashings: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#proposer_slashing +- Attester slashing: https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#attester_slashing + +There are 2 consumers of validated consensus objects: +- a `ValidationResult.Accept` output triggers rebroadcasting in libp2p + - method `validate(PubSub, message)` in libp2p/protocols/pubsub/pubsub.nim in the + - which was called by `rpcHandler(GossipSub, PubSubPeer, RPCMsg)` +- a `xyzValidator` message enqueues the validated object in one of the processing queue in eth2_processor + - `blocksQueue: AsyncQueue[BlockEntry]`, (shared with request_manager and sync_manager) + - `attestationsQueue: AsyncQueue[AttestationEntry]` + - `aggregatesQueue: AsyncQueue[AggregateEntry]` + +Those queues are then regularly processed to be made available to the consensus object pools. + +## Security concerns + +As the first line of defense in Nimbus, modules must be able to handle burst of data that may come: +- from malicious nodes trying to DOS us +- from long periods of non-finality, creating lots of forks, attestations, forks diff --git a/beacon_chain/attestation_aggregation.nim b/beacon_chain/gossip_processing/attestation_aggregation.nim similarity index 99% rename from beacon_chain/attestation_aggregation.nim rename to beacon_chain/gossip_processing/attestation_aggregation.nim index 308e74971..06a0e1e06 100644 --- a/beacon_chain/attestation_aggregation.nim +++ b/beacon_chain/gossip_processing/attestation_aggregation.nim @@ -10,13 +10,13 @@ import std/[options, sequtils], chronos, chronicles, - ./spec/[ + ../spec/[ beaconstate, datatypes, crypto, digest, helpers, network, signatures], - ./consensus_object_pools/[ + ../consensus_object_pools/[ spec_cache, blockchain_dag, block_quarantine, spec_cache, attestation_pool ], - ./beacon_node_types, ./ssz, ./time + ".."/[beacon_node_types, ssz, beacon_clock] logScope: topics = "att_aggr" diff --git a/beacon_chain/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim similarity index 98% rename from beacon_chain/eth2_processor.nim rename to beacon_chain/gossip_processing/eth2_processor.nim index c735109cc..d3066ee71 100644 --- a/beacon_chain/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -11,12 +11,12 @@ import std/[math, tables], stew/results, chronicles, chronos, metrics, - ./spec/[crypto, datatypes, digest], - ./consensus_object_pools/[block_clearance, blockchain_dag, exit_pool, attestation_pool], + ../spec/[crypto, datatypes, digest], + ../consensus_object_pools/[block_clearance, blockchain_dag, exit_pool, attestation_pool], ./attestation_aggregation, - ./validators/validator_pool, - ./beacon_node_types, - ./time, ./conf, ./ssz/sszdump + ../validators/validator_pool, + ../beacon_node_types, + ../beacon_clock, ../conf, ../ssz/sszdump # Metrics for tracking attestation and beacon block loss declareCounter beacon_attestations_received, diff --git a/beacon_chain/networking/README.md b/beacon_chain/networking/README.md new file mode 100644 index 000000000..8c04a02ea --- /dev/null +++ b/beacon_chain/networking/README.md @@ -0,0 +1,14 @@ +# Networking + +This folders hold a collection of modules to: +- configure the Eth2 P2P network +- discover, connect, and maintain quality Eth2 peers + +Data received is handed other to the `../gossip_processing` modules for validation. + +## Security concerns + +- Collusion: part of the peer selection must be kept random. This avoids peers bringing all their friends and colluding against a beacon node. +- Denial-of-service: The beacon node must provide ways to handle burst of data that may come: + - from malicious nodes trying to DOS us + - from long periods of non-finality, creating lots of forks, attestations, forks diff --git a/beacon_chain/eth2_discovery.nim b/beacon_chain/networking/eth2_discovery.nim similarity index 99% rename from beacon_chain/eth2_discovery.nim rename to beacon_chain/networking/eth2_discovery.nim index 916f009bb..06b27870e 100644 --- a/beacon_chain/eth2_discovery.nim +++ b/beacon_chain/networking/eth2_discovery.nim @@ -4,7 +4,7 @@ import std/[os, strutils], chronicles, stew/shims/net, stew/results, bearssl, eth/keys, eth/p2p/discoveryv5/[enr, protocol, node], - ./conf + ../conf export protocol, keys diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/networking/eth2_network.nim similarity index 99% rename from beacon_chain/eth2_network.nim rename to beacon_chain/networking/eth2_network.nim index a6f00e424..0a59c0058 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/networking/eth2_network.nim @@ -21,11 +21,12 @@ import libp2p/stream/connection, eth/[keys, async_utils], eth/p2p/p2p_protocol_dsl, eth/net/nat, eth/p2p/discoveryv5/[enr, node], - "."/[ - version, conf, eth2_discovery, libp2p_json_serialization, - ssz/ssz_serialization, peer_pool, time], - ./spec/[datatypes, digest, helpers, network], - ./validators/keystore_management + ".."/[ + version, conf, + ssz/ssz_serialization, beacon_clock], + ../spec/[datatypes, digest, helpers, network], + ../validators/keystore_management, + ./eth2_discovery, ./peer_pool, ./libp2p_json_serialization import libp2p/protocols/pubsub/gossipsub diff --git a/beacon_chain/faststreams_backend.nim b/beacon_chain/networking/faststreams_backend.nim similarity index 100% rename from beacon_chain/faststreams_backend.nim rename to beacon_chain/networking/faststreams_backend.nim diff --git a/beacon_chain/libp2p_json_serialization.nim b/beacon_chain/networking/libp2p_json_serialization.nim similarity index 100% rename from beacon_chain/libp2p_json_serialization.nim rename to beacon_chain/networking/libp2p_json_serialization.nim diff --git a/beacon_chain/libp2p_streams_backend.nim b/beacon_chain/networking/libp2p_streams_backend.nim similarity index 100% rename from beacon_chain/libp2p_streams_backend.nim rename to beacon_chain/networking/libp2p_streams_backend.nim diff --git a/beacon_chain/network_metadata.nim b/beacon_chain/networking/network_metadata.nim similarity index 98% rename from beacon_chain/network_metadata.nim rename to beacon_chain/networking/network_metadata.nim index ad2bdf53a..7f88ce0d3 100644 --- a/beacon_chain/network_metadata.nim +++ b/beacon_chain/networking/network_metadata.nim @@ -12,8 +12,8 @@ import chronicles, json_serialization, json_serialization/std/[options, sets, net], serialization/errors, - ssz/navigator, - spec/[presets, datatypes, digest] + ../ssz/navigator, + ../spec/[presets, datatypes, digest] # ATTENTION! This file will produce a large C file, because we are inlining # genesis states as C literals in the generated code (and blobs in the final @@ -74,7 +74,7 @@ type incompatibilityDesc*: string const - eth2testnetsDir = currentSourcePath.parentDir.replace('\\', '/') & "/../vendor/eth2-testnets" + eth2testnetsDir = currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor/eth2-testnets" const presetValueLoaders = genExpr(nnkBracket): for constName in PresetValue: @@ -228,4 +228,3 @@ proc getRuntimePresetForNetwork*(eth2Network: Option[string]): RuntimePreset = proc extractGenesisValidatorRootFromSnapshop*(snapshot: string): Eth2Digest = sszMount(snapshot, BeaconState).genesis_validators_root[] - diff --git a/beacon_chain/peer_pool.nim b/beacon_chain/networking/peer_pool.nim similarity index 100% rename from beacon_chain/peer_pool.nim rename to beacon_chain/networking/peer_pool.nim diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index d62a343c6..3266192cf 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -23,12 +23,13 @@ import # Local modules "."/[ - attestation_aggregation, beacon_chain_db, + beacon_chain_db, beacon_node_common, beacon_node_status, beacon_node_types, conf, - eth2_discovery, eth2_network, eth2_processor, - extras, filepath, interop, network_metadata, + extras, filepath, interop, nimbus_binary_common, ssz/merkleization, statusbar, - time, version], + beacon_clock, version], + ./networking/[eth2_discovery, eth2_network, network_metadata], + ./gossip_processing/[eth2_processor, attestation_aggregation], ./validators/[validator_duties, validator_pool, slashing_protection, keystore_management], ./sync/[sync_manager, sync_protocol, request_manager], ./rpc/[beacon_api, config_api, debug_api, event_api, nimbus_api, node_api, diff --git a/beacon_chain/nimbus_binary_common.nim b/beacon_chain/nimbus_binary_common.nim index 8ba70cbe0..492d50e44 100644 --- a/beacon_chain/nimbus_binary_common.nim +++ b/beacon_chain/nimbus_binary_common.nim @@ -17,7 +17,8 @@ import stew/io2, # Local modules - spec/[datatypes, crypto, helpers], eth2_network, time, filepath + ./spec/[datatypes, crypto, helpers], beacon_clock, filepath, + ./networking/eth2_network proc setupStdoutLogging*(logLevel: string) = when compiles(defaultChroniclesStream.output.writer): @@ -80,7 +81,7 @@ template makeBannerAndConfig*(clientId: string, ConfType: type): untyped = {.pop.} config -# TODO not sure if this belongs here but it doesn't belong in `time.nim` either +# TODO not sure if this belongs here but it doesn't belong in `beacon_clock.nim` either proc sleepToSlotOffset*(clock: BeaconClock, extra: chronos.Duration, slot: Slot, msg: static string): Future[bool] {.async.} = let diff --git a/beacon_chain/nimbus_signing_process.nim b/beacon_chain/nimbus_signing_process.nim index 288412da1..824bd467a 100644 --- a/beacon_chain/nimbus_signing_process.nim +++ b/beacon_chain/nimbus_signing_process.nim @@ -10,8 +10,8 @@ import os, strutils, tables, # Local modules - spec/[digest, crypto], - validators/keystore_management + ./spec/[digest, crypto], + ./validators/keystore_management programMain: var validators: Table[ValidatorPubKey, ValidatorPrivKey] diff --git a/beacon_chain/nimbus_validator_client.nim b/beacon_chain/nimbus_validator_client.nim index 31369917f..80215bd8b 100644 --- a/beacon_chain/nimbus_validator_client.nim +++ b/beacon_chain/nimbus_validator_client.nim @@ -16,18 +16,19 @@ import json_serialization/std/[options, net], # Local modules - spec/[datatypes, digest, crypto, helpers, network, signatures], - spec/eth2_apis/beacon_rpc_client, - sync/sync_manager, - conf, time, version, - eth2_network, eth2_discovery, beacon_node_types, - attestation_aggregation, - nimbus_binary_common, - version, ssz/merkleization, - spec/eth2_apis/callsigs_types, - eth2_json_rpc_serialization, - validators/[keystore_management, validator_pool, slashing_protection], - eth/db/[kvstore, kvstore_sqlite3] + ./spec/[datatypes, digest, crypto, helpers, network, signatures], + ./spec/eth2_apis/beacon_rpc_client, + ./sync/sync_manager, + "."/[conf, beacon_clock, version], + ./networking/[eth2_network, eth2_discovery], + ./rpc/eth2_json_rpc_serialization, + ./beacon_node_types, + ./gossip_processing/attestation_aggregation, + ./nimbus_binary_common, + ./ssz/merkleization, + ./spec/eth2_apis/callsigs_types, + ./validators/[keystore_management, validator_pool, slashing_protection], + ./eth/db/[kvstore, kvstore_sqlite3] logScope: topics = "vc" diff --git a/beacon_chain/rpc/beacon_api.nim b/beacon_chain/rpc/beacon_api.nim index abcb6d0be..11d4f19b5 100644 --- a/beacon_chain/rpc/beacon_api.nim +++ b/beacon_chain/rpc/beacon_api.nim @@ -10,13 +10,14 @@ import json_rpc/[rpcserver, jsonmarshal], chronicles, nimcrypto/utils as ncrutils, - ../beacon_node_common, ../eth2_json_rpc_serialization, ../eth2_network, + ../beacon_node_common, + ../networking/eth2_network, ../validators/validator_duties, ../consensus_object_pools/[blockchain_dag, exit_pool], ../spec/[crypto, digest, datatypes, validator, network], ../spec/eth2_apis/callsigs_types, ../ssz/merkleization, - ./rpc_utils + ./rpc_utils, ./eth2_json_rpc_serialization logScope: topics = "beaconapi" diff --git a/beacon_chain/rpc/debug_api.nim b/beacon_chain/rpc/debug_api.nim index ce6c5fbb4..4ed402eed 100644 --- a/beacon_chain/rpc/debug_api.nim +++ b/beacon_chain/rpc/debug_api.nim @@ -2,10 +2,10 @@ import std/sequtils, json_rpc/[rpcserver, jsonmarshal], chronicles, - ../version, ../beacon_node_common, ../eth2_json_rpc_serialization, - ../eth2_network, ../peer_pool, + ../version, ../beacon_node_common, + ../networking/[eth2_network, peer_pool], ../spec/[datatypes, digest, presets], - ./rpc_utils + ./rpc_utils, ./eth2_json_rpc_serialization logScope: topics = "debugapi" @@ -20,4 +20,3 @@ proc installDebugApiHandlers*(rpcServer: RpcServer, node: BeaconNode) = rpcServer.rpc("get_v1_debug_beacon_heads") do () -> seq[tuple[root: Eth2Digest, slot: Slot]]: return node.chainDag.heads.mapIt((it.root, it.slot)) - diff --git a/beacon_chain/eth2_json_rpc_serialization.nim b/beacon_chain/rpc/eth2_json_rpc_serialization.nim similarity index 98% rename from beacon_chain/eth2_json_rpc_serialization.nim rename to beacon_chain/rpc/eth2_json_rpc_serialization.nim index d7610869a..d87da45f2 100644 --- a/beacon_chain/eth2_json_rpc_serialization.nim +++ b/beacon_chain/rpc/eth2_json_rpc_serialization.nim @@ -3,11 +3,12 @@ import std/[tables, json, typetraits], # Nimble packages - stew/byteutils, ssz/types, + stew/byteutils, json_rpc/jsonmarshal, # Local modules - spec/[datatypes, crypto, digest] + ../ssz/types, + ../spec/[datatypes, crypto, digest] proc toJsonHex(data: openArray[byte]): string = # Per the eth2 API spec, hex arrays are printed with leading 0x diff --git a/beacon_chain/rpc/nimbus_api.nim b/beacon_chain/rpc/nimbus_api.nim index 4cac508e0..882e6f916 100644 --- a/beacon_chain/rpc/nimbus_api.nim +++ b/beacon_chain/rpc/nimbus_api.nim @@ -12,7 +12,8 @@ import json_rpc/[rpcserver, jsonmarshal], rpc_utils, - ../beacon_node_common, ../nimbus_binary_common, ../eth2_network, + ../beacon_node_common, ../nimbus_binary_common, + ../networking/eth2_network, ../eth1/eth1_monitor, ../validators/validator_duties, ../spec/[digest, datatypes, presets], diff --git a/beacon_chain/rpc/node_api.nim b/beacon_chain/rpc/node_api.nim index 625ba1c1e..adfd82513 100644 --- a/beacon_chain/rpc/node_api.nim +++ b/beacon_chain/rpc/node_api.nim @@ -4,8 +4,9 @@ import std/options, eth/p2p/discoveryv5/enr, libp2p/[multiaddress, multicodec], nimcrypto/utils as ncrutils, - ../beacon_node_common, ../eth2_network, ../sync/sync_manager, - ../peer_pool, ../version, + ../beacon_node_common, ../version, + ../networking/[eth2_network, peer_pool], + ../sync/sync_manager, ../spec/[datatypes, digest, presets], ../spec/eth2_apis/callsigs_types diff --git a/beacon_chain/rpc/validator_api.nim b/beacon_chain/rpc/validator_api.nim index 337d9dc26..b3b17b70c 100644 --- a/beacon_chain/rpc/validator_api.nim +++ b/beacon_chain/rpc/validator_api.nim @@ -19,8 +19,9 @@ import ../spec/eth2_apis/callsigs_types, ../consensus_object_pools/[blockchain_dag, spec_cache, attestation_pool], ../ssz/merkleization, ../beacon_node_common, ../beacon_node_types, - ../validators/validator_duties, ../eth2_network, - ../eth2_json_rpc_serialization, + ../validators/validator_duties, + ../networking/eth2_network, + ./eth2_json_rpc_serialization, ./rpc_utils logScope: topics = "valapi" diff --git a/beacon_chain/spec/eth2_apis/beacon_rpc_client.nim b/beacon_chain/spec/eth2_apis/beacon_rpc_client.nim index 0ba41da75..63abd65ac 100644 --- a/beacon_chain/spec/eth2_apis/beacon_rpc_client.nim +++ b/beacon_chain/spec/eth2_apis/beacon_rpc_client.nim @@ -1,7 +1,7 @@ import std/[os, json], json_rpc/[rpcclient, jsonmarshal], - ../../eth2_json_rpc_serialization, + ../../rpc/eth2_json_rpc_serialization, ../crypto, ../digest, ../datatypes, callsigs_types diff --git a/beacon_chain/statediff.nim b/beacon_chain/statediff.nim index 39e8baf32..02c578601 100644 --- a/beacon_chain/statediff.nim +++ b/beacon_chain/statediff.nim @@ -8,8 +8,8 @@ {.push raises: [Defect].} import - ssz/types, - spec/[datatypes, digest, helpers] + ./ssz/types, + ./spec/[datatypes, digest, helpers] func diffModIncEpoch[T, U](hl: HashArray[U, T], startSlot: uint64): array[SLOTS_PER_EPOCH, T] = diff --git a/beacon_chain/sync/request_manager.nim b/beacon_chain/sync/request_manager.nim index f6ca156a6..eaa55cbbb 100644 --- a/beacon_chain/sync/request_manager.nim +++ b/beacon_chain/sync/request_manager.nim @@ -1,8 +1,10 @@ import options, sequtils, strutils import chronos, chronicles import ../spec/[datatypes, digest], - ".."/[eth2_network, beacon_node_types, - ssz/merkleization, eth2_processor], + ../networking/eth2_network, + ../beacon_node_types, + ../ssz/merkleization, + ../gossip_processing/eth2_processor, ./sync_protocol, ./sync_manager export sync_manager diff --git a/beacon_chain/sync/sync_manager.nim b/beacon_chain/sync/sync_manager.nim index e9671ac73..f36bb3d8e 100644 --- a/beacon_chain/sync/sync_manager.nim +++ b/beacon_chain/sync/sync_manager.nim @@ -2,9 +2,9 @@ import chronicles import options, deques, heapqueue, tables, strutils, sequtils, math, algorithm import stew/results, chronos, chronicles import ../spec/[datatypes, digest, helpers, eth2_apis/callsigs_types], - ../peer_pool, ../eth2_network + ../networking/[peer_pool, eth2_network] -import ../eth2_processor +import ../gossip_processing/eth2_processor import ../consensus_object_pools/block_pools_types export datatypes, digest, chronos, chronicles, results, block_pools_types diff --git a/beacon_chain/sync/sync_protocol.nim b/beacon_chain/sync/sync_protocol.nim index 96dc51613..20a626ce6 100644 --- a/beacon_chain/sync/sync_protocol.nim +++ b/beacon_chain/sync/sync_protocol.nim @@ -2,8 +2,9 @@ import options, tables, sets, macros, chronicles, chronos, stew/ranges/bitranges, libp2p/switch, ../spec/[datatypes, network, crypto, digest], - ".."/[beacon_node_types, eth2_network, - consensus_object_pools/blockchain_dag] + ../beacon_node_types, + ../networking/eth2_network, + ../consensus_object_pools/blockchain_dag logScope: topics = "sync" diff --git a/beacon_chain/validators/keystore_management.nim b/beacon_chain/validators/keystore_management.nim index b283d19db..9edc3fbeb 100644 --- a/beacon_chain/validators/keystore_management.nim +++ b/beacon_chain/validators/keystore_management.nim @@ -5,7 +5,8 @@ import ../spec/[datatypes, digest, crypto, keystore], stew/io2, libp2p/crypto/crypto as lcrypto, nimcrypto/utils as ncrutils, - ".."/[conf, ssz/merkleization, network_metadata, filepath] + ".."/[conf, ssz/merkleization, filepath], + ../networking/network_metadata export keystore diff --git a/beacon_chain/validators/validator_duties.nim b/beacon_chain/validators/validator_duties.nim index 5ae489f40..06fd27733 100644 --- a/beacon_chain/validators/validator_duties.nim +++ b/beacon_chain/validators/validator_duties.nim @@ -21,14 +21,15 @@ import ../spec/[ datatypes, digest, crypto, helpers, network, signatures, state_transition, validator], - ../conf, ../time, + ../conf, ../beacon_clock, ../consensus_object_pools/[ spec_cache, blockchain_dag, block_clearance, attestation_pool, exit_pool], ../eth1/eth1_monitor, - ../eth2_network, ../beacon_node_common, - ../beacon_node_types, ../nimbus_binary_common, ../version, - ../ssz, ../ssz/sszdump, ../attestation_aggregation, ../sync/sync_manager, + ../networking/eth2_network, + ".."/[beacon_node_common, beacon_node_types, nimbus_binary_common, version], + ../gossip_processing/attestation_aggregation, + ../ssz, ../ssz/sszdump, ../sync/sync_manager, ./slashing_protection, ./validator_pool, ./keystore_management diff --git a/ncli/ncli.nim b/ncli/ncli.nim index 05cd2c35b..1c8082ad9 100644 --- a/ncli/ncli.nim +++ b/ncli/ncli.nim @@ -5,7 +5,7 @@ import ../research/simutils, ../beacon_chain/spec/[crypto, datatypes, digest, helpers, state_transition], ../beacon_chain/extras, - ../beacon_chain/network_metadata, + ../beacon_chain/networking/network_metadata, ../beacon_chain/ssz/[merkleization, ssz_serialization] type diff --git a/ncli/ncli_db.nim b/ncli/ncli_db.nim index 6d29b77fa..adaf660f8 100644 --- a/ncli/ncli_db.nim +++ b/ncli/ncli_db.nim @@ -1,7 +1,7 @@ import os, stats, strformat, tables, chronicles, confutils, stew/byteutils, eth/db/kvstore_sqlite3, - ../beacon_chain/network_metadata, + ../beacon_chain/networking/network_metadata, ../beacon_chain/[beacon_chain_db, extras], ../beacon_chain/consensus_object_pools/blockchain_dag, ../beacon_chain/spec/[crypto, datatypes, digest, helpers, diff --git a/tests/test_attestation_pool.nim b/tests/test_attestation_pool.nim index a2f8eb605..42212e581 100644 --- a/tests/test_attestation_pool.nim +++ b/tests/test_attestation_pool.nim @@ -17,7 +17,8 @@ import # Internal ../beacon_chain/spec/[crypto, datatypes, digest, validator, state_transition, helpers, beaconstate, presets, network], - ../beacon_chain/[beacon_node_types, attestation_aggregation, extras, time], + ../beacon_chain/[beacon_node_types, extras, beacon_clock], + ../beacon_chain/gossip_processing/attestation_aggregation, ../beacon_chain/fork_choice/[fork_choice_types, fork_choice], ../beacon_chain/consensus_object_pools/[block_quarantine, blockchain_dag, block_clearance, attestation_pool], # Test utilities diff --git a/tests/test_honest_validator.nim b/tests/test_honest_validator.nim index c3933a47d..e6cf1b83d 100644 --- a/tests/test_honest_validator.nim +++ b/tests/test_honest_validator.nim @@ -3,7 +3,7 @@ import unittest, ./testutil, ../beacon_chain/spec/[crypto, datatypes, network], - ../beacon_chain/attestation_aggregation + ../beacon_chain/gossip_processing/attestation_aggregation suiteReport "Honest validator": var forkDigest: ForkDigest diff --git a/tests/test_peer_pool.nim b/tests/test_peer_pool.nim index 32ee3d7b6..4d03ecd70 100644 --- a/tests/test_peer_pool.nim +++ b/tests/test_peer_pool.nim @@ -9,7 +9,7 @@ import std/[unittest, random, heapqueue, tables] import chronos -import ../beacon_chain/peer_pool +import ../beacon_chain/networking/peer_pool import testutil type diff --git a/tests/test_sync_manager.nim b/tests/test_sync_manager.nim index f24336cc0..01603e15c 100644 --- a/tests/test_sync_manager.nim +++ b/tests/test_sync_manager.nim @@ -2,7 +2,7 @@ import unittest import chronos -import ../beacon_chain/eth2_processor, +import ../beacon_chain/gossip_processing/eth2_processor, ../beacon_chain/sync/sync_manager type