diff --git a/Makefile b/Makefile index 98fa648c3..877585358 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,6 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) FLUFFY_TOOLS := \ portal_bridge \ beacon_lc_bridge \ - state_bridge \ eth_data_exporter \ content_verifier \ blockwalk \ diff --git a/fluffy/network/state/state_content.nim b/fluffy/network/state/state_content.nim index 7519cfc73..31222aae2 100644 --- a/fluffy/network/state/state_content.nim +++ b/fluffy/network/state/state_content.nim @@ -11,33 +11,17 @@ {.push raises: [].} import - nimcrypto/[hash, sha2, keccak], stew/[objects, results], stint, + nimcrypto/[hash, sha2, keccak], stew/results, stint, + eth/common/eth_types, ssz_serialization, ../../common/common_types export ssz_serialization, common_types, hash, results -type JsonAccount* = object - nonce*: int - balance*: string - storage_hash*: string - code_hash*: string - -type JsonProof* = object - address*: string - state*: JsonAccount - proof*: seq[string] - -type JsonProofVector* = object - `block`*: int - block_hash*: string - state_root*: string - proofs*: seq[JsonProof] - type - NodeHash* = MDigest[32 * 8] # keccak256 - CodeHash* = MDigest[32 * 8] # keccak256 - Address* = array[20, byte] + NodeHash* = KeccakHash + CodeHash* = KeccakHash + Address* = EthAddress ContentType* = enum # Note: Need to add this unused value as a case object with an enum without @@ -49,52 +33,71 @@ type # the SSZ spec is not explicit about disallowing this. unused = 0x00 accountTrieNode = 0x20 - contractStorageTrieNode = 0x21 - accountTrieProof = 0x22 - contractStorageTrieProof = 0x23 - contractBytecode = 0x24 + contractTrieNode = 0x21 + contractCode = 0x22 + + NibblePair* = byte + Nibbles* = object + isOddLength*: bool + packedNibbles*: List[NibblePair, 32] + + WitnessNode* = List[byte, 1024] + Witness* = List[WitnessNode, 1024] + + StateWitness* = object + key*: Nibbles + proof*: Witness + + StorageWitness* = object + key*: Nibbles + proof*: Witness + stateWitness*: StateWitness AccountTrieNodeKey* = object - path*: ByteList + path*: Nibbles nodeHash*: NodeHash - stateRoot*: Bytes32 - ContractStorageTrieNodeKey* = object + ContractTrieNodeKey* = object address*: Address - path*: ByteList + path*: Nibbles nodeHash*: NodeHash - stateRoot*: Bytes32 - AccountTrieProofKey* = object - address*: Address - stateRoot*: Bytes32 - - ContractStorageTrieProofKey* = object - address*: Address - slot*: UInt256 - stateRoot*: Bytes32 - - ContractBytecodeKey* = object + ContractCodeKey* = object address*: Address codeHash*: CodeHash - WitnessNode* = ByteList - AccountTrieProof* = List[WitnessNode, 32] - ContentKey* = object case contentType*: ContentType of unused: discard of accountTrieNode: accountTrieNodeKey*: AccountTrieNodeKey - of contractStorageTrieNode: - contractStorageTrieNodeKey*: ContractStorageTrieNodeKey - of accountTrieProof: - accountTrieProofKey*: AccountTrieProofKey - of contractStorageTrieProof: - contractStorageTrieProofKey*: ContractStorageTrieProofKey - of contractBytecode: - contractBytecodeKey*: ContractBytecodeKey + of contractTrieNode: + contractTrieNodeKey*: ContractTrieNodeKey + of contractCode: + contractCodeKey*: ContractCodeKey + + AccountTrieNodeOffer* = object + proof*: StateWitness + blockHash*: BlockHash + + AccountTrieNodeRetrieval* = object + node*: WitnessNode + + ContractTrieNodeOffer* = object + proof*: StorageWitness + blockHash*: BlockHash + + ContractTrieNodeRetrieval* = object + node*: WitnessNode + + ContractCodeOffer* = object + code*: ByteList + accountProof*: StateWitness + blockHash*: BlockHash + + ContractCodeRetrieval* = object + code*: ByteList func encode*(contentKey: ContentKey): ByteList = doAssert(contentKey.contentType != unused) @@ -115,53 +118,10 @@ func decode*(contentKey: ByteList): Opt[ContentKey] = except SerializationError: return Opt.none(ContentKey) -template computeContentId*(digestCtxType: type, body: untyped): ContentId = - var h {.inject.}: digestCtxType - init(h) - body - let idHash = finish(h) +func toContentId*(contentKey: ByteList): ContentId = + # TODO: Should we try to parse the content key here for invalid ones? + let idHash = sha2.sha256.digest(contentKey.asSeq()) readUintBE[256](idHash.data) func toContentId*(contentKey: ContentKey): ContentId = - case contentKey.contentType: - of unused: - raiseAssert "Should not be used and fail at decoding" - of accountTrieNode: # sha256(path | node_hash) - let key = contentKey.accountTrieNodeKey - computeContentId sha256: - h.update(key.path.asSeq()) - h.update(key.nodeHash.data) - of contractStorageTrieNode: # sha256(address | path | node_hash) - let key = contentKey.contractStorageTrieNodeKey - computeContentId sha256: - h.update(key.address) - h.update(key.path.asSeq()) - h.update(key.nodeHash.data) - of accountTrieProof: # keccak(address) - let key = contentKey.accountTrieProofKey - computeContentId keccak256: - h.update(key.address) - of contractStorageTrieProof: # (keccak(address) + keccak(slot)) % 2**256 - # TODO: Why is keccak run on slot, when it can be used directly? - # Also, value to LE or BE? Not mentioned in specification. - let key = contentKey.contractStorageTrieProofKey - let n1 = - block: computeContentId keccak256: - h.update(key.address) - let n2 = - block: computeContentId keccak256: - h.update(toBytesBE(key.slot)) - - n1 + n2 # uint256 will wrap arround, practically applying the modulo 256 - of contractBytecode: # sha256(address | code_hash) - let key = contentKey.contractBytecodeKey - computeContentId sha256: - h.update(key.address) - h.update(key.codeHash.data) - -func toContentId*(contentKey: ByteList): results.Opt[ContentId] = - let key = decode(contentKey) - if key.isSome(): - ok(key.get().toContentId()) - else: - Opt.none(ContentId) + toContentId(encode(contentKey)) diff --git a/fluffy/network/state/state_distance.nim b/fluffy/network/state/state_distance.nim deleted file mode 100644 index 27e46f391..000000000 --- a/fluffy/network/state/state_distance.nim +++ /dev/null @@ -1,80 +0,0 @@ -# Nimbus -# Copyright (c) 2021-2022 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -import - eth/p2p/discoveryv5/routing_table, - stint - -const - MID* = u256(2).pow(u256(255)) - MAX* = high(UInt256) - -# Custom distance function described in: https://notes.ethereum.org/h58LZcqqRRuarxx4etOnGQ#Storage-Layout -# The implementation looks different than in spec, due to the fact that in practice -# we are operating on unsigned 256bit integers instead of signed big ints. -# Thanks to this we do not need to use: -# - modulo operations -# - abs operation -# and the results are eqivalent to function described in spec. -# -# The way it works is as follows. Let say we have integers modulo 8: -# [0, 1, 2, 3, 4, 5, 6, 7] -# and we want to calculate minimal distance between 0 and 5. -# Raw difference is: 5 - 0 = 5, which is larger than mid point which is equal to 4. -# From this we know that the shorter distance is the one wraping around 0, which -# is equal to 3 -func stateDistance*(node_id: UInt256, content_id: UInt256): UInt256 = - let rawDiff = - if node_id > content_id: - node_id - content_id - else: - content_id - node_id - - if rawDiff > MID: - # If rawDiff is larger than mid this means that distance between node_id and - # content_id is smaller when going from max side. - MAX - rawDiff + UInt256.one - else: - rawDiff - -# TODO we do not have Uint256 log2 implementation. It would be nice to implement -# it in stint library in some more performant way. This version has O(n) complexity. -func log2DistanceImpl(value: UInt256): uint16 = - # Logarithm is not defined for zero values. Implementation in stew for builtin - # types return -1 in that case, but here it is just internal function so just make sure - # 0 is never provided. - doAssert(not value.isZero()) - - if value == UInt256.one: - return 0'u16 - - var comp = value - var ret = 0'u16 - while (comp > 1): - comp = comp shr 1 - ret = ret + 1 - return ret - -func stateIdAtDistance*(id: UInt256, dist: uint16): UInt256 = - # TODO With current distance function there are always two ids at given distance - # so we might as well do: id - u256(dist), maybe it is worth discussing if every client - # should use the same id in this case. - id + u256(2).pow(dist) - -func stateLogDistance*(a, b: UInt256): uint16 = - let distance = stateDistance(a, b) - if distance.isZero(): - return 0 - else: - return log2DistanceImpl(distance) - -const stateDistanceCalculator* = - DistanceCalculator( - calculateDistance: stateDistance, - calculateLogDistance: stateLogDistance, - calculateIdAtDistance: stateIdAtDistance - ) diff --git a/fluffy/network/state/state_network.nim b/fluffy/network/state/state_network.nim index 9c7cea65c..f56c115c7 100644 --- a/fluffy/network/state/state_network.nim +++ b/fluffy/network/state/state_network.nim @@ -13,8 +13,7 @@ import eth/p2p/discoveryv5/[protocol, enr], ../../database/content_db, ../wire/[portal_protocol, portal_stream, portal_protocol_config], - ./state_content, - ./state_distance + ./state_content logScope: topics = "portal_state" @@ -29,7 +28,7 @@ type StateNetwork* = ref object processContentLoop: Future[void] func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] = - toContentId(contentKey) + ok(toContentId(contentKey)) proc getContent*(n: StateNetwork, key: ContentKey): Future[Opt[seq[byte]]] {.async.} = @@ -75,27 +74,9 @@ proc validateContent( false of accountTrieNode: true - of contractStorageTrieNode: + of contractTrieNode: true - of accountTrieProof: - let decodedProof = decodeSsz(contentValue, AccountTrieProof).valueOr: - warn "Received invalid account trie proof", error - return false - let - proof = decodedProof.asSeq().map((p: ByteList) => p.toSeq()) - trieKey = keccakHash(key.accountTrieProofKey.address).data.toSeq() - value = proof[^1].decode(seq[seq[byte]])[^1] - stateRoot = MDigest[256](data: key.accountTrieProofKey.stateRoot) - verificationResult = verifyMptProof(proof, stateRoot, trieKey, value) - case verificationResult.kind: - of ValidProof: - true - else: - warn "Received invalid account trie proof" - false - of contractStorageTrieProof: - true - of contractBytecode: + of contractCode: true proc validateContent( @@ -131,8 +112,7 @@ proc new*( let portalProtocol = PortalProtocol.new( baseProtocol, stateProtocolId, toContentIdHandler, createGetHandler(contentDB), s, - bootstrapRecords, stateDistanceCalculator, - config = portalConfig) + bootstrapRecords, config = portalConfig) portalProtocol.dbPut = createStoreHandler(contentDB, portalConfig.radiusConfig, portalProtocol) diff --git a/fluffy/tests/all_fluffy_tests.nim b/fluffy/tests/all_fluffy_tests.nim index 5024028b6..4b8534e01 100644 --- a/fluffy/tests/all_fluffy_tests.nim +++ b/fluffy/tests/all_fluffy_tests.nim @@ -9,8 +9,6 @@ import ./test_portal_wire_protocol, - ./state_network_tests/test_state_distance, - ./state_network_tests/test_state_network, ./state_network_tests/test_state_content, ./test_state_proof_verification, ./test_accumulator, diff --git a/fluffy/tests/portal_spec_tests/mainnet/all_fluffy_portal_spec_tests.nim b/fluffy/tests/portal_spec_tests/mainnet/all_fluffy_portal_spec_tests.nim index 6aa861b86..665bae68e 100644 --- a/fluffy/tests/portal_spec_tests/mainnet/all_fluffy_portal_spec_tests.nim +++ b/fluffy/tests/portal_spec_tests/mainnet/all_fluffy_portal_spec_tests.nim @@ -13,5 +13,4 @@ import ./test_history_content, ./test_history_content_validation, ./test_header_content, - ./test_state_content, ./test_accumulator_root diff --git a/fluffy/tests/portal_spec_tests/mainnet/test_state_content.nim b/fluffy/tests/portal_spec_tests/mainnet/test_state_content.nim deleted file mode 100644 index a14999958..000000000 --- a/fluffy/tests/portal_spec_tests/mainnet/test_state_content.nim +++ /dev/null @@ -1,225 +0,0 @@ -# Fluffy -# Copyright (c) 2021-2023 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.used.} - -import - unittest2, stew/byteutils, - ../../../network/state/state_content - -# According to test vectors: -# https://github.com/ethereum/portal-network-specs/blob/master/content-keys-test-vectors.md#state-network-keys - -suite "State ContentKey Encodings": - # Common input - const - stateRoot = hexToByteArray[sizeof(Bytes32)]( - "0xd1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d") - address = hexToByteArray[sizeof(Address)]( - "0x829bd824b016326a401d083b33d092293333a830") - - test "AccountTrieNode": - # Input - const - nodeHash = NodeHash.fromHex( - "0xb8be7903aee73b8f6a59cd44a1f52c62148e1f376c0dfa1f5f773a98666efc2b") - path = ByteList.init(@[byte 1, 2, 0, 1]) - - # Output - contentKeyHex = - "2044000000b8be7903aee73b8f6a59cd44a1f52c62148e1f376c0dfa1f5f773a98666efc2bd1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d01020001" - contentId = - "41237096982860596884042712109427867048220765019203857308279863638242761605893" - # or - contentIdHexBE = - "5b2b5ea9a7384491010c1aa459a0f967dcf8b69988adbfe7e0bed513e9bb8305" - - let - accountTrieNodeKey = AccountTrieNodeKey( - path: path, nodeHash: nodeHash, stateRoot: stateRoot) - contentKey = ContentKey( - contentType: accountTrieNode, accountTrieNodeKey: accountTrieNodeKey) - - let encoded = encode(contentKey) - check encoded.asSeq.toHex == contentKeyHex - let decoded = decode(encoded) - check decoded.isSome() - - let contentKeyDecoded = decoded.get() - check: - contentKeyDecoded.contentType == contentKey.contentType - contentKeyDecoded.accountTrieNodeKey == contentKey.accountTrieNodeKey - - toContentId(contentKey) == parse(contentId, StUint[256], 10) - # In stint this does BE hex string - toContentId(contentKey).toHex() == contentIdHexBE - - test "ContractStorageTrieNode": - # Input - const - nodeHash = NodeHash.fromHex( - "0x3e190b68719aecbcb28ed2271014dd25f2aa633184988eb414189ce0899cade5") - path = ByteList.init(@[byte 1, 0, 15, 14, 12, 0]) - - # Output - contentKeyHex = - "21829bd824b016326a401d083b33d092293333a830580000003e190b68719aecbcb28ed2271014dd25f2aa633184988eb414189ce0899cade5d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d01000f0e0c00" - contentId = - "43529358882110548041037387588279806363134301284609868141745095118932570363585" - # or - contentIdHexBE = - "603cbe7902925ce359822378a4cb1b4b53e1bf19d003de2c26e55812d76956c1" - - let - contractStorageTrieNodeKey = ContractStorageTrieNodeKey( - address: address, path: path, nodeHash: nodeHash, stateRoot: stateRoot) - contentKey = ContentKey( - contentType: contractStorageTrieNode, - contractStorageTrieNodeKey: contractStorageTrieNodeKey) - - let encoded = encode(contentKey) - check encoded.asSeq.toHex == contentKeyHex - let decoded = decode(encoded) - check decoded.isSome() - - let contentKeyDecoded = decoded.get() - check: - contentKeyDecoded.contentType == contentKey.contentType - contentKeyDecoded.contractStorageTrieNodeKey == - contentKey.contractStorageTrieNodeKey - - toContentId(contentKey) == parse(contentId, StUint[256], 10) - # In stint this does BE hex string - toContentId(contentKey).toHex() == contentIdHexBE - - test "AccountTrieProof": - # Output - const - contentKeyHex = - "22829bd824b016326a401d083b33d092293333a830d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d" - contentId = - "45301550050471302973396879294932122279426162994178563319590607565171451545101" - # or - contentIdHexBE = - "6427c4c8d42db15c2aca8dfc7dff7ce2c8c835441b566424fa3377dd031cc60d" - - let - accountTrieProofKey = AccountTrieProofKey( - address: address, stateRoot: stateRoot) - contentKey = ContentKey( - contentType: accountTrieProof, - accountTrieProofKey: accountTrieProofKey) - - let encoded = encode(contentKey) - check encoded.asSeq.toHex == contentKeyHex - let decoded = decode(encoded) - check decoded.isSome() - - let contentKeyDecoded = decoded.get() - check: - contentKeyDecoded.contentType == contentKey.contentType - contentKeyDecoded.accountTrieProofKey == contentKey.accountTrieProofKey - - toContentId(contentKey) == parse(contentId, StUint[256], 10) - # In stint this does BE hex string - toContentId(contentKey).toHex() == contentIdHexBE - - test "ContractStorageTrieProof": - # Input - const - slot = 239304.stuint(256) - - # Output - contentKeyHex = - "23829bd824b016326a401d083b33d092293333a830c8a6030000000000000000000000000000000000000000000000000000000000d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d" - contentId = - "80413803151602881485894828440259195604313253842905231566803078625935967002376" - # or - contentIdHexBE = - "b1c89984803cebd325303ba035f9c4ca0d0d91b2cbfef84d455e7a847ade1f08" - - let - contractStorageTrieProofKey = ContractStorageTrieProofKey( - address: address, slot: slot, stateRoot: stateRoot) - contentKey = ContentKey( - contentType: contractStorageTrieProof, - contractStorageTrieProofKey: contractStorageTrieProofKey) - - let encoded = encode(contentKey) - check encoded.asSeq.toHex == contentKeyHex - let decoded = decode(encoded) - check decoded.isSome() - - let contentKeyDecoded = decoded.get() - check: - contentKeyDecoded.contentType == contentKey.contentType - contentKeyDecoded.contractStorageTrieProofKey == - contentKey.contractStorageTrieProofKey - - toContentId(contentKey) == parse(contentId, StUint[256], 10) - # In stint this does BE hex string - toContentId(contentKey).toHex() == contentIdHexBE - - test "ContractBytecode": - # Input - const codeHash = CodeHash.fromHex( - "0xd1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d") - - # Output - const - contentKeyHex = - "24829bd824b016326a401d083b33d092293333a830d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d" - contentId = - "9243655320250466575533858917172702581481192615849913473767356296630272634800" - # or - contentIdHexBE = - "146fb937afe42bcf11d25ad57d67734b9a7138677d59eeec3f402908f54dafb0" - - let - contractBytecodeKey = ContractBytecodeKey( - address: address, codeHash: codeHash) - contentKey = ContentKey( - contentType: contractBytecode, - contractBytecodeKey: contractBytecodeKey) - - let encoded = encode(contentKey) - check encoded.asSeq.toHex == contentKeyHex - let decoded = decode(encoded) - check decoded.isSome() - - let contentKeyDecoded = decoded.get() - check: - contentKeyDecoded.contentType == contentKey.contentType - contentKeyDecoded.contractBytecodeKey == contentKey.contractBytecodeKey - - toContentId(contentKey) == parse(contentId, StUint[256], 10) - # In stint this does BE hex string - toContentId(contentKey).toHex() == contentIdHexBE - - test "Invalid prefix - 0 value": - let encoded = ByteList.init(@[byte 0x00]) - let decoded = decode(encoded) - - check decoded.isNone() - - test "Invalid prefix - before valid range": - let encoded = ByteList.init(@[byte 0x01]) - let decoded = decode(encoded) - - check decoded.isNone() - - test "Invalid prefix - after valid range": - let encoded = ByteList.init(@[byte 0x25]) - let decoded = decode(encoded) - - check decoded.isNone() - - test "Invalid key - empty input": - let encoded = ByteList.init(@[]) - let decoded = decode(encoded) - - check decoded.isNone() diff --git a/fluffy/tests/state_network_tests/test_state_content.nim b/fluffy/tests/state_network_tests/test_state_content.nim index c3c98917f..09a6403be 100644 --- a/fluffy/tests/state_network_tests/test_state_content.nim +++ b/fluffy/tests/state_network_tests/test_state_content.nim @@ -12,33 +12,229 @@ import eth/keys, ../../network/state/state_content -const testVectorDir = - "./vendor/portal-spec-tests/tests/mainnet/state/" - -procSuite "State Content": - let rng = newRng() - - test "Encode/decode accountTrieProof": - let file = testVectorDir & "/proofs.full.block.0.json" - let content = readAllFile(file).valueOr: - quit(1) - - let decoded = - try: - Json.decode(content, state_content.JsonProofVector) - except SerializationError: - quit(1) - - let proof = decoded.proofs[0].proof.map(hexToSeqByte) - - var accountTrieProof = AccountTrieProof(@[]) - for witness in proof: - let witnessNode = ByteList(witness) - discard accountTrieProof.add(witnessNode) +suite "State Content Keys": + const evenNibles = "0005000000123456789abc" + test "Encode/decode even nibbles": + const + packedNibbles = Nibbles.packedNibbles.init(@[NibblePair 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC]) + isOddLength = false let - encodedProof = SSZ.encode(accountTrieProof) - decodedProof = decodeSsz(encodedProof, AccountTrieProof).get() + nibbles = Nibbles(packedNibbles: packedNibbles, isOddLength: isOddLength) + encoded = SSZ.encode(nibbles) + check encoded.toHex() == evenNibles + # echo ">>>", encoded.toHex() + + const oddNibbles = "0105000000123456789abc0d" + test "Encode/decode odd nibbles": + const + packedNibbles = Nibbles.packedNibbles.init(@[NibblePair 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0x0D]) + isOddLength = true - check decodedProof == accountTrieProof + let + nibbles = Nibbles(packedNibbles: packedNibbles, isOddLength: isOddLength) + encoded = SSZ.encode(nibbles) + check encoded.toHex() == oddNibbles + # echo ">>>", encoded.toHex() + const accountTrieNodeKeyEncoded = "2024000000c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a4700005000000123456789abc" + test "Encode/decode AccountTrieNodeKey": + const + packedNibbles = Nibbles.packedNibbles.init(@[NibblePair 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC]) + isOddLength = false + nodeHash = NodeHash.fromHex("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") + + let + nibbles = Nibbles(packedNibbles: packedNibbles, isOddLength: isOddLength) + accountTrieNodeKey = AccountTrieNodeKey(path: nibbles, nodeHash: nodeHash) + contentKey = ContentKey(contentType: accountTrieNode, accountTrieNodeKey: accountTrieNodeKey) + encoded = contentKey.encode() + # echo ">>>", $encoded + check $encoded == accountTrieNodeKeyEncoded + + let decoded = encoded.decode().valueOr: + raiseAssert "Cannot decode AccountTrieNodeKey" + check: + decoded.contentType == accountTrieNode + decoded.accountTrieNodeKey == AccountTrieNodeKey(path: nibbles, nodeHash: nodeHash) + + const contractTrieNodeKeyEncoded = "21000d836201318ec6899a6754069038278074328038000000c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a4700005000000123456789abc" + test "Encode/decode ContractTrieNodeKey": + const + address = Address.fromHex("000d836201318ec6899a67540690382780743280") + packedNibbles = Nibbles.packedNibbles.init(@[NibblePair 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC]) + isOddLength = false + nodeHash = NodeHash.fromHex("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") + + let + nibbles = Nibbles(packedNibbles: (packedNibbles), isOddLength: isOddLength) + contractTrieNodeKey = ContractTrieNodeKey(address: address, path: nibbles, nodeHash: nodeHash) + contentKey = ContentKey(contentType: contractTrieNode, contractTrieNodeKey: contractTrieNodeKey) + encoded = contentKey.encode() + # echo ">>>", $encoded + check $encoded == contractTrieNodeKeyEncoded + + let decoded = encoded.decode().valueOr: + raiseAssert "Cannot decode ContractTrieNodeKey" + check: + decoded.contentType == contractTrieNode + decoded.contractTrieNodeKey == ContractTrieNodeKey(address: address, path: nibbles, nodeHash: nodeHash) + + const contractCodeKeyEncoded = "22000d836201318ec6899a67540690382780743280c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + test "Encode/decode ContractCodeKey": + const + address = Address.fromHex("000d836201318ec6899a67540690382780743280") + codeHash = CodeHash.fromHex("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") + + let + contractCodeKey = ContractCodeKey(address: address, codeHash: codeHash) + contentKey = ContentKey(contentType: contractCode, contractCodeKey: contractCodeKey) + encoded = contentKey.encode() + # echo ">>>", $encoded + check $encoded == contractCodeKeyEncoded + + let decoded = encoded.decode().valueOr: + raiseAssert "Cannot decode ContractCodeKey" + check: + decoded.contentType == contractCode + decoded.contractCodeKey.address == address + decoded.contractCodeKey.codeHash == codeHash + + test "Invalid prefix - 0 value": + let encoded = ByteList.init(@[byte 0x00]) + let decoded = decode(encoded) + + check decoded.isNone() + + test "Invalid prefix - before valid range": + let encoded = ByteList.init(@[byte 0x01]) + let decoded = decode(encoded) + + check decoded.isNone() + + test "Invalid prefix - after valid range": + let encoded = ByteList.init(@[byte 0x25]) + let decoded = decode(encoded) + + check decoded.isNone() + + test "Invalid key - empty input": + let encoded = ByteList.init(@[]) + let decoded = decode(encoded) + + check decoded.isNone() + +suite "State Content Values": + const accountTrieNodeOfferEncoded = "24000000d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa308000000130000000005000000123456789abc04000000010203040506" + test "Encode/decode AccountTrieNodeOffer": + let + blockHash = "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" + packedNibbles = @[NibblePair 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC] + isOddLength = false + key = Nibbles(packedNibbles: Nibbles.packedNibbles.init(packedNibbles), isOddLength: isOddLength) + witnessNode = WitnessNode(@[byte 0x01, 0x02, 0x03, 0x04, 0x05, 0x06]) + proof = Witness(@[witnessNode]) + + accountTrieNodeOffer = AccountTrieNodeOffer(blockHash: BlockHash.fromHex(blockHash), proof: StateWitness(key: key, proof: proof)) + encoded = SSZ.encode(accountTrieNodeOffer) + # echo ">>>", encoded.toHex() + check encoded.toHex() == accountTrieNodeOfferEncoded + + let decoded = SSZ.decode(encoded, AccountTrieNodeOffer) + check: + decoded.blockHash == BlockHash.fromHex(blockHash) + decoded.proof == StateWitness(key: key, proof: proof) + + const accountTrieNodeRetrievalEncoded = "04000000010203040506" + test "Encode/decode AccountTrieNodeRetrieval": + let + witnessNode = WitnessNode(@[byte 0x01, 0x02, 0x03, 0x04, 0x05, 0x06]) + + accountTrieNodeRetrieval = AccountTrieNodeRetrieval(node: witnessNode) + encoded = SSZ.encode(accountTrieNodeRetrieval) + # echo ">>>", encoded.toHex() + check encoded.toHex() == accountTrieNodeRetrievalEncoded + + let decoded = SSZ.decode(encoded, AccountTrieNodeRetrieval) + check decoded.node == witnessNode + + const contractTrieNodeOfferEncoded = "24000000d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa30c00000017000000220000000005000000123456789abc040000000102030405060708000000150000000005000000123456789abcdef104000000010203040506070809" + test "Encode/decode ContractTrieNodeOffer": + let + blockHash = "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" + contractWitnessKeyPackedNibbles = @[NibblePair 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC] + contractWitnessProof = Witness(@[WitnessNode(@[byte 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07])]) + stateWitnessKeyPackedNibbles = @[NibblePair 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF1] + stateWitnessProof = Witness(@[WitnessNode(@[byte 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09])]) + + contractTrieNodeOffer = ContractTrieNodeOffer( + blockHash: BlockHash.fromHex(blockHash), + proof: StorageWitness( + key: Nibbles(packedNibbles: Nibbles.packedNibbles.init(contractWitnessKeyPackedNibbles), isOddLength: false), + proof: contractWitnessProof, + stateWitness: StateWitness( + key: Nibbles(packedNibbles: Nibbles.packedNibbles.init(stateWitnessKeyPackedNibbles), isOddLength: false), + proof: stateWitnessProof + ))) + encoded = SSZ.encode(contractTrieNodeOffer) + # echo ">>>", encoded.toHex() + check encoded.toHex() == contractTrieNodeOfferEncoded + + let decoded = SSZ.decode(encoded, ContractTrieNodeOffer) + check: + decoded.blockHash == BlockHash.fromHex(blockHash) + decoded.proof == StorageWitness( + key: Nibbles(packedNibbles: Nibbles.packedNibbles.init(contractWitnessKeyPackedNibbles), isOddLength: false), + proof: contractWitnessProof, + stateWitness: StateWitness( + key: Nibbles(packedNibbles: Nibbles.packedNibbles.init(stateWitnessKeyPackedNibbles), isOddLength: false), + proof: stateWitnessProof + )) + + const contractTrieNodeRetrievalEncoded = "04000000010203040506" + test "Encode/decode ContractTrieNodeRetrieval": + let + witnessNode = WitnessNode(@[byte 0x01, 0x02, 0x03, 0x04, 0x05, 0x06]) + + contractTrieNodeRetrieval = ContractTrieNodeRetrieval(node: witnessNode) + encoded = SSZ.encode(contractTrieNodeRetrieval) + # echo ">>>", encoded.toHex() + check encoded.toHex() == contractTrieNodeRetrievalEncoded + + let decoded = SSZ.decode(encoded, ContractTrieNodeRetrieval) + check decoded.node == witnessNode + + const contractCodeOfferEncoded = "2800000036000000d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3010203040506070809a0a1a2a3a408000000130000000005000000123456789abc04000000010203040506" + test "Encode/decode ContractCodeOffer": + let + code = @[byte 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4] + blockHash = "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" + accountProofKey = Nibbles(packedNibbles: Nibbles.packedNibbles.init(@[NibblePair 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC]), isOddLength: false) + accountProofWitness = Witness(@[WitnessNode(@[byte 0x01, 0x02, 0x03, 0x04, 0x05, 0x06])]) + + contractCodeOffer = ContractCodeOffer( + code: ByteList.init(code), + blockHash: BlockHash.fromHex(blockHash), + accountProof: StateWitness(key: accountProofKey, proof: accountProofWitness)) + encoded = SSZ.encode(contractCodeOffer) + # echo ">>>", encoded.toHex() + check encoded.toHex() == contractCodeOfferEncoded + + let decoded = SSZ.decode(encoded, ContractCodeOffer) + check: + decoded.code == ByteList.init(code) + decoded.blockHash == BlockHash.fromHex(blockHash) + decoded.accountProof == StateWitness(key: accountProofKey, proof: accountProofWitness) + + const contractCodeRetrievalEncoded = "04000000010203040506070809a0a1a2a3a4" + test "Encode/decode ContractCodeRetrieval": + let + code = @[byte 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4] + + contractCodeRetrieval = ContractCodeRetrieval(code: ByteList.init(code)) + encoded = SSZ.encode(contractCodeRetrieval) + # echo ">>>", encoded.toHex() + check encoded.toHex() == contractCodeRetrievalEncoded + + let decoded = SSZ.decode(encoded, ContractCodeRetrieval) + check decoded.code == ByteList.init(code) diff --git a/fluffy/tests/state_network_tests/test_state_distance.nim b/fluffy/tests/state_network_tests/test_state_distance.nim deleted file mode 100644 index 4c85e37fb..000000000 --- a/fluffy/tests/state_network_tests/test_state_distance.nim +++ /dev/null @@ -1,57 +0,0 @@ -# Nimbus - Portal Network -# Copyright (c) 2021 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.used.} - -import - std/sequtils, - stint, unittest2, - ../../network/state/state_distance - -suite "State network custom distance function": - test "Calculate distance according to spec": - check: - # Test cases from spec - stateDistance(u256(10), u256(10)) == 0 - stateDistance(u256(5), high(UInt256)) == 6 - stateDistance(high(UInt256), u256(6)) == 7 - stateDistance(u256(5), u256(1)) == 4 - stateDistance(u256(1), u256(5)) == 4 - stateDistance(UInt256.zero, MID) == MID - stateDistance(UInt256.zero, MID + UInt256.one) == MID - UInt256.one - - # Additional test cases to check some basic properties - stateDistance(UInt256.zero, MID + MID) == UInt256.zero - stateDistance(UInt256.zero, UInt256.one) == stateDistance(UInt256.zero, high(UInt256)) - - test "Calculate logarithimic distance": - check: - stateLogDistance(u256(0), u256(0)) == 0 - stateLogDistance(u256(0), u256(1)) == 0 - stateLogDistance(u256(0), u256(2)) == 1 - stateLogDistance(u256(0), u256(4)) == 2 - stateLogDistance(u256(0), u256(8)) == 3 - stateLogDistance(u256(8), u256(0)) == 3 - stateLogDistance(UInt256.zero, MID) == 255 - stateLogDistance(UInt256.zero, MID + UInt256.one) == 254 - - test "Calculate id at log distance": - let logDistances = @[ - 0'u16, 1, 2, 3, 4, 5, 6, 7, 8 - ] - - # for each log distance, calulate node-id at given distance from node zero, and then - # log distance from calculate node-id to node zero. The results should equal - # starting log distances - let logCalculated = logDistances.map( - proc (x: uint16): uint16 = - let nodeAtDist = stateIdAtDistance(UInt256.zero, x) - return stateLogDistance(UInt256.zero, nodeAtDist) - ) - - check: - logDistances == logCalculated diff --git a/fluffy/tests/state_network_tests/test_state_network.nim b/fluffy/tests/state_network_tests/test_state_network.nim index 741b19a67..065801548 100644 --- a/fluffy/tests/state_network_tests/test_state_network.nim +++ b/fluffy/tests/state_network_tests/test_state_network.nim @@ -40,81 +40,6 @@ proc genesisToTrie(filePath: string): CoreDbMptRef = procSuite "State Network": let rng = newRng() - test "Test account state proof": - let file = testVectorDir & "/proofs.full.block.0.json" - let content = readAllFile(file).valueOr: - quit(1) - - let decoded = - try: - Json.decode(content, state_content.JsonProofVector) - except SerializationError: - quit(1) - let - proof = decoded.proofs[0].proof.map(hexToSeqByte) - stateRoot = MDigest[256].fromHex(decoded.state_root) - address = hexToByteArray[20](decoded.proofs[0].address) - key = keccakHash(address).data.toSeq() - value = proof[^1].decode(seq[seq[byte]])[^1] - proofResult = verifyMptProof(proof, stateRoot, key, value) - check proofResult.kind == ValidProof - - asyncTest "Decode and use proofs": - let file = testVectorDir & "/proofs.full.block.0.json" - let content = readAllFile(file).valueOr: - quit(1) - - let decoded = - try: - Json.decode(content, state_content.JsonProofVector) - except SerializationError: - quit(1) - - let - node1 = initDiscoveryNode( - rng, PrivateKey.random(rng[]), localAddress(20302)) - sm1 = StreamManager.new(node1) - node2 = initDiscoveryNode( - rng, PrivateKey.random(rng[]), localAddress(20303)) - sm2 = StreamManager.new(node2) - - proto1 = StateNetwork.new(node1, ContentDB.new("", uint32.high, inMemory = true), sm1) - proto2 = StateNetwork.new(node2, ContentDB.new("", uint32.high, inMemory = true), sm2) - - state_root = hexToByteArray[sizeof(state_content.AccountTrieProofKey.stateRoot)](decoded.state_root) - - check proto2.portalProtocol.addNode(node1.localNode) == Added - - - for proof in decoded.proofs: - let - address = hexToByteArray[sizeof(state_content.Address)](proof.address) - key = AccountTrieProofKey( - address: address, - stateRoot: state_root) - contentKey = ContentKey( - contentType: state_content.ContentType.accountTrieProof, - accountTrieProofKey: key) - - var accountTrieProof = AccountTrieProof(@[]) - for witness in proof.proof: - let witnessNode = ByteList(hexToSeqByte(witness)) - discard accountTrieProof.add(witnessNode) - - let encodedValue = SSZ.encode(accountTrieProof) - - discard proto1.contentDB.put(contentKey.toContentId(), encodedValue, proto1.portalProtocol.localNode.id) - - let foundContent = await proto2.getContent(contentKey) - - check foundContent.isSome() - - check decodeSsz(foundContent.get(), AccountTrieProof).isOk() - - await node1.closeWait() - await node2.closeWait() - - asyncTest "Test Share Full State": let trie = genesisToTrie("fluffy" / "tests" / "custom_genesis" / "chainid7.json") @@ -235,36 +160,3 @@ procSuite "State Network": await node1.closeWait() await node2.closeWait() await node3.closeWait() - - asyncTest "Find other nodes in state network with correct custom distance": - let - node1 = initDiscoveryNode( - rng, PrivateKey.random(rng[]), localAddress(20302)) - sm1 = StreamManager.new(node1) - node2 = initDiscoveryNode( - rng, PrivateKey.random(rng[]), localAddress(20303)) - sm2 = StreamManager.new(node2) - - proto1 = StateNetwork.new(node1, ContentDB.new("", uint32.high, inMemory = true), sm1) - proto2 = StateNetwork.new(node2, ContentDB.new("", uint32.high, inMemory = true), sm2) - - check (await node1.ping(node2.localNode)).isOk() - check (await node2.ping(node1.localNode)).isOk() - - proto2.portalProtocol.seedTable() - - let distance = proto1.portalProtocol.routingTable.logDistance( - node1.localNode.id, node2.localNode.id) - - let nodes = await proto1.portalProtocol.findNodes( - proto2.portalProtocol.localNode, @[distance]) - - # TODO: This gives an error because of the custom distances issues that - # need to be resolved first. - skip() - # check: - # nodes.isOk() - # nodes.get().len() == 1 - - await node1.closeWait() - await node2.closeWait() diff --git a/fluffy/tools/state_bridge/nim.cfg b/fluffy/tools/state_bridge/nim.cfg deleted file mode 100644 index 0745d2ac5..000000000 --- a/fluffy/tools/state_bridge/nim.cfg +++ /dev/null @@ -1 +0,0 @@ --d:"chronicles_sinks=textlines[dynamic],json[dynamic]" diff --git a/fluffy/tools/state_bridge/state_bridge.nim b/fluffy/tools/state_bridge/state_bridge.nim deleted file mode 100644 index 97efb0c73..000000000 --- a/fluffy/tools/state_bridge/state_bridge.nim +++ /dev/null @@ -1,82 +0,0 @@ -# Nimbus -# Copyright (c) 2023-2024 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -# This is a fake bridge that reads state from a directory and backfills it to the portal state network. - -{.push raises: [].} - -import - std/[os, sugar], - confutils, confutils/std/net, chronicles, chronicles/topics_registry, - json_rpc/clients/httpclient, - chronos, - stew/[byteutils, io2], - eth/async_utils, - eth/common/eth_types, - ../../network/state/state_content, - ../../rpc/portal_rpc_client, - ../../logging, - ../eth_data_exporter/cl_data_exporter, - ./state_bridge_conf - -proc run(config: StateBridgeConf) {.raises: [CatchableError].} = - setupLogging(config.logLevel, config.logStdout) - - notice "Launching Fluffy fake state bridge", - cmdParams = commandLineParams() - - let portalRpcClient = newRpcHttpClient() - - proc backfill(rpcAddress: string, rpcPort: Port) {.async raises: [OSError].} = - # info "Backfilling...", config.rpcAddress, ":", config.rpcPort - await portalRpcClient.connect(config.rpcAddress, Port(config.rpcPort), false) - let files = collect(for f in walkDir(config.dataDir): f.path) - for file in files: - let - content = readAllFile(file).valueOr: - warn "Skipping file because of error \n", file=file, error=($error) - continue - decoded = - try: - Json.decode(content, JsonProofVector) - except SerializationError as e: - warn "Skipping file because of error \n", file=file, error = e.msg - continue - state_root = hexToByteArray[sizeof(Bytes32)](decoded.state_root) - - for proof in decoded.proofs: - let - address = hexToByteArray[sizeof(state_content.Address)](proof.address) - key = AccountTrieProofKey( - address: address, - stateRoot: state_root) - contentKey = ContentKey( - contentType: ContentType.accountTrieProof, - accountTrieProofKey: key) - encodedKey = encode(contentKey) - - var accountTrieProof = AccountTrieProof(@[]) - for witness in proof.proof: - let witnessNode = ByteList(hexToSeqByte(witness)) - discard accountTrieProof.add(witnessNode) - discard await portalRpcClient.portal_stateGossip(encodedKey.asSeq().toHex(), SSZ.encode(accountTrieProof).toHex()) - await portalRpcClient.close() - notice "Backfill done..." - - waitFor backfill(config.rpcAddress, Port(config.rpcPort)) - - while true: - poll() - -when isMainModule: - {.pop.} - let config = StateBridgeConf.load() - {.push raises: [].} - - case config.cmd - of StateBridgeCmd.noCommand: - run(config) diff --git a/fluffy/tools/state_bridge/state_bridge_conf.nim b/fluffy/tools/state_bridge/state_bridge_conf.nim deleted file mode 100644 index f9268a97f..000000000 --- a/fluffy/tools/state_bridge/state_bridge_conf.nim +++ /dev/null @@ -1,54 +0,0 @@ -# Nimbus -# Copyright (c) 2023-2024 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.push raises: [].} - -import - confutils, confutils/std/net, - nimcrypto/hash, - ../../logging - -export net - -type - StateBridgeCmd* = enum - noCommand - - StateBridgeConf* = object - # Logging - logLevel* {. - desc: "Sets the log level" - defaultValue: "INFO" - name: "log-level" .}: string - - logStdout* {. - hidden - desc: "Specifies what kind of logs should be written to stdout (auto, colors, nocolors, json)" - defaultValueDesc: "auto" - defaultValue: StdoutLogKind.Auto - name: "log-format" .}: StdoutLogKind - - # Portal JSON-RPC API server to connect to - rpcAddress* {. - desc: "Listening address of the Portal JSON-RPC server" - defaultValue: "127.0.0.1" - name: "rpc-address" .}: string - - rpcPort* {. - desc: "Listening port of the Portal JSON-RPC server" - defaultValue: 8545 - name: "rpc-port" .}: Port - - dataDir* {. - desc: "Data directory to lookup state data. Should point to the directory with json files generated by https://github.com/morph-dev/young-ethereum e.g. ./vendor/portal-spec-tests/tests/mainnet/state/" - name: "data-dir".}: string - - case cmd* {. - command - defaultValue: noCommand .}: StateBridgeCmd - of noCommand: - discard