From e3d14bd9210683cf84af4119886560b852614154 Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:32:57 +0800 Subject: [PATCH] Fluffy portal testnet support (#2383) * Bump portal-mainnet repo. * Update command line arguments and parsing on startup. * Read in angelfood bootstrap nodes and update Fluffy guide. * Configure subnetwork protocol ids. --- fluffy/conf.nim | 55 ++++++++++--------- .../the_fluffy_book/docs/connect-to-portal.md | 12 ++-- fluffy/fluffy.nim | 48 ++++++++++------ fluffy/network/beacon/beacon_network.nim | 5 +- fluffy/network/history/history_network.nim | 5 +- fluffy/network/network_seed.nim | 3 +- fluffy/network/state/state_network.nim | 5 +- fluffy/network/wire/portal_protocol.nim | 35 ++++++++++++ .../network/wire/portal_protocol_config.nim | 14 +++++ fluffy/network_metadata.nim | 2 + fluffy/rpc/rpc_eth_api.nim | 4 -- .../beacon_test_helpers.nim | 5 +- .../state_test_helpers.nim | 5 +- fluffy/tests/test_history_network.nim | 5 +- fluffy/tools/portalcli.nim | 2 +- vendor/portal-mainnet | 2 +- 16 files changed, 134 insertions(+), 73 deletions(-) diff --git a/fluffy/conf.nim b/fluffy/conf.nim index 8a5176045..45d3e76e6 100644 --- a/fluffy/conf.nim +++ b/fluffy/conf.nim @@ -56,17 +56,6 @@ type PortalCmd* = enum noCommand - PortalNetwork* = enum - none - mainnet - testnet - - # The networks alias Portal sub-protocols - Network* = enum - beacon - history - state - PortalConf* = object logLevel* {. desc: @@ -94,27 +83,37 @@ type name: "listen-address" .}: IpAddress - portalNetwork* {. + portalNetworkDeprecated* {. + hidden, + desc: + "DEPRECATED: The --portal-network flag will be removed in the future, " & + "please use the drop in replacement --network flag instead", + defaultValue: none(PortalNetwork), + name: "portal-network" + .}: Option[PortalNetwork] + + network* {. desc: "Select which Portal network to join. This will set the " & "Portal network specific bootstrap nodes automatically", defaultValue: PortalNetwork.mainnet, - name: "portal-network" + name: "network" .}: PortalNetwork - portalNetworkDeprecated* {. + networksDeprecated* {. hidden, desc: - "DEPRECATED: The --network flag will be removed in the future, please use the drop in replacement --portal-network flag instead", - defaultValue: none(PortalNetwork), - name: "network" - .}: Option[PortalNetwork.mainnet] - - networks* {. - desc: "Select which networks (Portal sub-protocols) to enable", - defaultValue: {Network.history}, + "DEPRECATED: The --networks flag will be removed in the future, " & + "please use the drop in replacement --portal-subnetworks flag instead", + defaultValue: {}, name: "networks" - .}: set[Network] + .}: set[PortalSubnetwork] + + portalSubnetworks* {. + desc: "Select which networks (Portal sub-protocols) to enable", + defaultValue: {PortalSubnetwork.history}, + name: "portal-subnetworks" + .}: set[PortalSubnetwork] # Note: This will add bootstrap nodes for both Discovery v5 network and each # enabled Portal network. No distinction is made on bootstrap nodes per @@ -363,21 +362,23 @@ proc parseCmdArg*(T: type ClientConfig, p: string): T {.raises: [ValueError].} = proc completeCmdArg*(T: type ClientConfig, val: string): seq[string] = return @[] -proc parseCmdArg*(T: type set[Network], p: string): T {.raises: [ValueError].} = - var res: set[Network] = {} +proc parseCmdArg*( + T: type set[PortalSubnetwork], p: string +): T {.raises: [ValueError].} = + var res: set[PortalSubnetwork] = {} let values = p.split({' ', ','}) for value in values: let stripped = value.strip() let network = try: - parseEnum[Network](stripped) + parseEnum[PortalSubnetwork](stripped) except ValueError: raise newException(ValueError, "Invalid network: " & stripped) res.incl(network) res -proc completeCmdArg*(T: type set[Network], val: string): seq[string] = +proc completeCmdArg*(T: type set[PortalSubnetwork], val: string): seq[string] = return @[] chronicles.formatIt(InputDir): diff --git a/fluffy/docs/the_fluffy_book/docs/connect-to-portal.md b/fluffy/docs/the_fluffy_book/docs/connect-to-portal.md index 473b8bd4a..cdb10d9bb 100644 --- a/fluffy/docs/the_fluffy_book/docs/connect-to-portal.md +++ b/fluffy/docs/the_fluffy_book/docs/connect-to-portal.md @@ -6,15 +6,14 @@ Connecting to the current Portal network is as easy as running following command ./build/fluffy --rpc ``` -This will connect to the public [Portal testnet](https://github.com/ethereum/portal-network-specs/blob/master/testnet.md#portal-network-testnet) -which contains nodes of the different clients. +This will connect to the public Portal mainnet which contains nodes of the different clients. !!! note - Default the Fluffy node will connect to the - [bootstrap nodes](https://github.com/ethereum/portal-network-specs/blob/master/testnet.md#bootnodes) of the public testnet. + By default the Fluffy node will connect to the + [bootstrap nodes](https://github.com/ethereum/portal-network-specs/blob/master/bootnodes.md#bootnodes-mainnet) of the public mainnet. - When testing locally the `--portal-network:none` option can be provided to avoid - connecting to any of the testnet bootstrap nodes. + When testing locally the `--network:none` option can be provided to avoid + connecting to any of the default bootstrap nodes. The `--rpc` option will also enable the different JSON-RPC interfaces through which you can access the Portal Network. @@ -32,4 +31,3 @@ Fluffy also supports a small subset of the [Execution JSON-RPC API](https://ethe - diff --git a/fluffy/fluffy.nim b/fluffy/fluffy.nim index 2d0d90e79..944450763 100644 --- a/fluffy/fluffy.nim +++ b/fluffy/fluffy.nim @@ -35,7 +35,7 @@ import ./network/state/[state_network, state_content], ./network/history/[history_network, history_content], ./network/beacon/[beacon_init_loader, beacon_light_client], - ./network/wire/[portal_stream, portal_protocol_config], + ./network/wire/[portal_stream, portal_protocol_config, portal_protocol], ./eth_data/history_data_ssz_e2s, ./database/content_db, ./version, @@ -101,24 +101,35 @@ proc run(config: PortalConf) {.raises: [CatchableError].} = loadBootstrapFile(string config.bootstrapNodesFile, bootstrapRecords) bootstrapRecords.add(config.bootstrapNodes) - var portalNetwork: PortalNetwork - if config.portalNetworkDeprecated.isSome(): - warn "DEPRECATED: The --network flag will be removed in the future, please use the drop in replacement --portal-network flag instead" - portalNetwork = config.portalNetworkDeprecated.get() - else: - portalNetwork = config.portalNetwork + let portalNetwork = + if config.portalNetworkDeprecated.isNone(): + config.network + else: + warn "DEPRECATED: The --portal-network flag will be removed in the future, " & + "please use the drop in replacement --network flag instead" + config.portalNetworkDeprecated.get() + + let portalSubnetworks = + if config.networksDeprecated == {}: + config.portalSubnetworks + else: + warn "DEPRECATED: The --networks flag will be removed in the future, " & + "please use the drop in replacement --portal-subnetworks flag instead" + config.networksDeprecated case portalNetwork - of mainnet: + of PortalNetwork.none: + discard # don't connect to any network bootstrap nodes + of PortalNetwork.mainnet: for enrURI in mainnetBootstrapNodes: var record: Record if fromURI(record, enrURI): bootstrapRecords.add(record) - of testnet: - # TODO: add testnet repo with bootstrap file. - discard - else: - discard + of PortalNetwork.angelfood: + for enrURI in angelfoodBootstrapNodes: + var record: Record + if fromURI(record, enrURI): + bootstrapRecords.add(record) let discoveryConfig = @@ -210,9 +221,10 @@ proc run(config: PortalConf) {.raises: [CatchableError].} = loadAccumulator() historyNetwork = - if Network.history in config.networks: + if PortalSubnetwork.history in portalSubnetworks: Opt.some( HistoryNetwork.new( + portalNetwork, d, db, streamManager, @@ -225,9 +237,10 @@ proc run(config: PortalConf) {.raises: [CatchableError].} = Opt.none(HistoryNetwork) stateNetwork = - if Network.state in config.networks: + if PortalSubnetwork.state in portalSubnetworks: Opt.some( StateNetwork.new( + portalNetwork, d, db, streamManager, @@ -243,12 +256,15 @@ proc run(config: PortalConf) {.raises: [CatchableError].} = beaconLightClient = # TODO: Currently disabled by default as it is not sufficiently polished. # Eventually this should be always-on functionality. - if Network.beacon in config.networks and config.trustedBlockRoot.isSome(): + if PortalSubnetwork.beacon in portalSubnetworks and + config.trustedBlockRoot.isSome(): let # Portal works only over mainnet data currently + # TODO: investigate this load network data function networkData = loadNetworkData("mainnet") beaconDb = BeaconDb.new(networkData, config.dataDir / "db" / "beacon_db") beaconNetwork = BeaconNetwork.new( + portalNetwork, d, beaconDb, streamManager, diff --git a/fluffy/network/beacon/beacon_network.nim b/fluffy/network/beacon/beacon_network.nim index 18d7aab41..8959513bf 100644 --- a/fluffy/network/beacon/beacon_network.nim +++ b/fluffy/network/beacon/beacon_network.nim @@ -23,8 +23,6 @@ export beacon_content, beacon_db logScope: topics = "beacon_network" -const lightClientProtocolId* = [byte 0x50, 0x1A] - type BeaconNetwork* = ref object portalProtocol*: PortalProtocol beaconDb*: BeaconDb @@ -183,6 +181,7 @@ proc getHistoricalSummaries*( proc new*( T: type BeaconNetwork, + portalNetwork: PortalNetwork, baseProtocol: protocol.Protocol, beaconDb: BeaconDb, streamManager: StreamManager, @@ -206,7 +205,7 @@ proc new*( portalProtocol = PortalProtocol.new( baseProtocol, - lightClientProtocolId, + getProtocolId(portalNetwork, PortalSubnetwork.beacon), toContentIdHandler, createGetHandler(beaconDb), stream, diff --git a/fluffy/network/history/history_network.nim b/fluffy/network/history/history_network.nim index 0df904b6b..753f7d475 100644 --- a/fluffy/network/history/history_network.nim +++ b/fluffy/network/history/history_network.nim @@ -47,8 +47,6 @@ export accumulator proc `$`(x: BlockHeader): string = $x -const historyProtocolId* = [byte 0x50, 0x0B] - type HistoryNetwork* = ref object portalProtocol*: PortalProtocol @@ -703,6 +701,7 @@ proc validateContent( proc new*( T: type HistoryNetwork, + portalNetwork: PortalNetwork, baseProtocol: protocol.Protocol, contentDB: ContentDB, streamManager: StreamManager, @@ -718,7 +717,7 @@ proc new*( portalProtocol = PortalProtocol.new( baseProtocol, - historyProtocolId, + getProtocolId(portalNetwork, PortalSubnetwork.history), toContentIdHandler, createGetHandler(contentDB), stream, diff --git a/fluffy/network/network_seed.nim b/fluffy/network/network_seed.nim index 85b45bce7..c2f180d16 100644 --- a/fluffy/network/network_seed.nim +++ b/fluffy/network/network_seed.nim @@ -33,8 +33,7 @@ const #TODO currently we are using value for history network, but this should be #caluculated per netowork basis maxItemsPerOfferBySize = getMaxOfferedContentKeys( - uint32(len(history_network.historyProtocolId)), - uint32(history_content.maxContentKeySize), + uint32(len(PortalProtocolId)), uint32(history_content.maxContentKeySize) ) # Offering is restricted to max 64 items diff --git a/fluffy/network/state/state_network.nim b/fluffy/network/state/state_network.nim index b512455b1..f033ac691 100644 --- a/fluffy/network/state/state_network.nim +++ b/fluffy/network/state/state_network.nim @@ -26,8 +26,6 @@ export results, state_content logScope: topics = "portal_state" -const stateProtocolId* = [byte 0x50, 0x0A] - type StateNetwork* = ref object portalProtocol*: PortalProtocol contentDB*: ContentDB @@ -41,6 +39,7 @@ func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] = proc new*( T: type StateNetwork, + portalNetwork: PortalNetwork, baseProtocol: protocol.Protocol, contentDB: ContentDB, streamManager: StreamManager, @@ -55,7 +54,7 @@ proc new*( let portalProtocol = PortalProtocol.new( baseProtocol, - stateProtocolId, + getProtocolId(portalNetwork, PortalSubnetwork.state), toContentIdHandler, createGetHandler(contentDB), s, diff --git a/fluffy/network/wire/portal_protocol.nim b/fluffy/network/wire/portal_protocol.nim index 5ec134403..10dcbe5ed 100644 --- a/fluffy/network/wire/portal_protocol.nim +++ b/fluffy/network/wire/portal_protocol.nim @@ -254,6 +254,41 @@ func init*( nodesInterestedInContent: nodesInterestedInContent, ) +func getProtocolId*( + network: PortalNetwork, subnetwork: PortalSubnetwork +): PortalProtocolId = + const portalPrefix = byte(0x50) + + case network + of PortalNetwork.none, PortalNetwork.mainnet: + case subnetwork + of PortalSubnetwork.state: + [portalPrefix, 0x0A] + of PortalSubnetwork.history: + [portalPrefix, 0x0B] + of PortalSubnetwork.beacon: + [portalPrefix, 0x0C] + of PortalSubnetwork.transactionIndex: + [portalPrefix, 0x0D] + of PortalSubnetwork.verkleState: + [portalPrefix, 0x0E] + of PortalSubnetwork.transactionGossip: + [portalPrefix, 0x0F] + of PortalNetwork.angelfood: + case subnetwork + of PortalSubnetwork.state: + [portalPrefix, 0x4A] + of PortalSubnetwork.history: + [portalPrefix, 0x4B] + of PortalSubnetwork.beacon: + [portalPrefix, 0x4C] + of PortalSubnetwork.transactionIndex: + [portalPrefix, 0x4D] + of PortalSubnetwork.verkleState: + [portalPrefix, 0x4E] + of PortalSubnetwork.transactionGossip: + [portalPrefix, 0x4F] + func `$`(id: PortalProtocolId): string = id.toHex() diff --git a/fluffy/network/wire/portal_protocol_config.nim b/fluffy/network/wire/portal_protocol_config.nim index 3e8c08814..44763e779 100644 --- a/fluffy/network/wire/portal_protocol_config.nim +++ b/fluffy/network/wire/portal_protocol_config.nim @@ -10,6 +10,20 @@ import std/strutils, confutils, chronos, stint, eth/p2p/discoveryv5/routing_table type + PortalNetwork* = enum + none + mainnet + angelfood + + # The Portal sub-protocols + PortalSubnetwork* = enum + state + history + beacon + transactionIndex + verkleState + transactionGossip + RadiusConfigKind* = enum Static Dynamic diff --git a/fluffy/network_metadata.nim b/fluffy/network_metadata.nim index 535edd71f..2961ed4f1 100644 --- a/fluffy/network_metadata.nim +++ b/fluffy/network_metadata.nim @@ -44,6 +44,8 @@ const # rlp.rawData() in the enr code. mainnetBootstrapNodes* = loadCompileTimeBootstrapNodes(portalConfigDir / "bootstrap_nodes.txt") + angelfoodBootstrapNodes* = + loadCompileTimeBootstrapNodes(portalConfigDir / "bootstrap_nodes_angelfood.txt") finishedAccumulatorSSZ* = slurp(portalConfigDir / "finished_accumulator.ssz") diff --git a/fluffy/rpc/rpc_eth_api.nim b/fluffy/rpc/rpc_eth_api.nim index 99ce4df79..152865faf 100644 --- a/fluffy/rpc/rpc_eth_api.nim +++ b/fluffy/rpc/rpc_eth_api.nim @@ -369,7 +369,6 @@ proc installEthApiHandlers*( raise newException(ValueError, error) balance = (await stateNetwork.get().getBalance(blockHash, data.EthAddress)).valueOr: - # Should we return 0 here or throw a more detailed error? raise newException(ValueError, "Unable to get balance") return balance @@ -397,7 +396,6 @@ proc installEthApiHandlers*( nonce = ( await stateNetwork.get().getTransactionCount(blockHash, data.EthAddress) ).valueOr: - # Should we return 0 here or throw a more detailed error? raise newException(ValueError, "Unable to get transaction count") return nonce.Quantity @@ -425,7 +423,6 @@ proc installEthApiHandlers*( slotValue = ( await stateNetwork.get().getStorageAt(blockHash, data.EthAddress, slot) ).valueOr: - # Should we return 0 here or throw a more detailed error? raise newException(ValueError, "Unable to get storage slot") return FixedBytes[32](slotValue.toBytesBE()) @@ -450,7 +447,6 @@ proc installEthApiHandlers*( raise newException(ValueError, error) bytecode = (await stateNetwork.get().getCode(blockHash, data.EthAddress)).valueOr: - # Should we return empty sequence here or throw a more detailed error? raise newException(ValueError, "Unable to get code") return bytecode.asSeq() diff --git a/fluffy/tests/beacon_network_tests/beacon_test_helpers.nim b/fluffy/tests/beacon_network_tests/beacon_test_helpers.nim index 6f226acad..1015b889c 100644 --- a/fluffy/tests/beacon_network_tests/beacon_test_helpers.nim +++ b/fluffy/tests/beacon_network_tests/beacon_test_helpers.nim @@ -9,7 +9,7 @@ import chronos, eth/p2p/discoveryv5/protocol as discv5_protocol, beacon_chain/spec/forks, - ../../network/wire/[portal_protocol, portal_stream], + ../../network/wire/[portal_protocol, portal_protocol_config, portal_stream], ../../network/beacon/[beacon_init_loader, beacon_network], ../test_helpers @@ -24,7 +24,8 @@ proc newLCNode*( node = initDiscoveryNode(rng, PrivateKey.random(rng[]), localAddress(port)) db = BeaconDb.new(networkData, "", inMemory = true) streamManager = StreamManager.new(node) - network = BeaconNetwork.new(node, db, streamManager, networkData.forks) + network = + BeaconNetwork.new(PortalNetwork.none, node, db, streamManager, networkData.forks) return BeaconNode(discoveryProtocol: node, beaconNetwork: network) diff --git a/fluffy/tests/state_network_tests/state_test_helpers.nim b/fluffy/tests/state_network_tests/state_test_helpers.nim index b63a2bf04..2e032a080 100644 --- a/fluffy/tests/state_network_tests/state_test_helpers.nim +++ b/fluffy/tests/state_network_tests/state_test_helpers.nim @@ -148,8 +148,9 @@ proc newStateNode*( node = initDiscoveryNode(rng, PrivateKey.random(rng[]), localAddress(port)) db = ContentDB.new("", uint32.high, inMemory = true) sm = StreamManager.new(node) - hn = HistoryNetwork.new(node, db, sm, FinishedAccumulator()) - sn = StateNetwork.new(node, db, sm, historyNetwork = Opt.some(hn)) + hn = HistoryNetwork.new(PortalNetwork.none, node, db, sm, FinishedAccumulator()) + sn = + StateNetwork.new(PortalNetwork.none, node, db, sm, historyNetwork = Opt.some(hn)) return StateNode(discoveryProtocol: node, stateNetwork: sn) diff --git a/fluffy/tests/test_history_network.nim b/fluffy/tests/test_history_network.nim index 99e303ae7..b336b9719 100644 --- a/fluffy/tests/test_history_network.nim +++ b/fluffy/tests/test_history_network.nim @@ -28,7 +28,8 @@ proc newHistoryNode( node = initDiscoveryNode(rng, PrivateKey.random(rng[]), localAddress(port)) db = ContentDB.new("", uint32.high, inMemory = true) streamManager = StreamManager.new(node) - historyNetwork = HistoryNetwork.new(node, db, streamManager, accumulator) + historyNetwork = + HistoryNetwork.new(PortalNetwork.none, node, db, streamManager, accumulator) return HistoryNode(discoveryProtocol: node, historyNetwork: historyNetwork) @@ -187,7 +188,7 @@ procSuite "History Content Network": historyNode2.start() let maxOfferedHistoryContent = - getMaxOfferedContentKeys(uint32(len(historyProtocolId)), maxContentKeySize) + getMaxOfferedContentKeys(uint32(len(PortalProtocolId)), maxContentKeySize) let headersWithProof = buildHeadersWithProof(headers[0 .. maxOfferedHistoryContent], epochAccumulators) diff --git a/fluffy/tools/portalcli.nim b/fluffy/tools/portalcli.nim index 6291402e3..9c13d5159 100644 --- a/fluffy/tools/portalcli.nim +++ b/fluffy/tools/portalcli.nim @@ -119,7 +119,7 @@ type .}: Port protocolId* {. - defaultValue: historyProtocolId, + defaultValue: getProtocolId(PortalNetwork.mainnet, PortalSubnetwork.history), desc: "Portal wire protocol id for the network to connect to", name: "protocol-id" .}: PortalProtocolId diff --git a/vendor/portal-mainnet b/vendor/portal-mainnet index 0f18d18c1..52d206cda 160000 --- a/vendor/portal-mainnet +++ b/vendor/portal-mainnet @@ -1 +1 @@ -Subproject commit 0f18d18c1a1ce75d751dee3a440a5a5fcc0ac89e +Subproject commit 52d206cdadae04c4faa204d3deb63cf1beb215ea