status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

addressing io, json and formatting

This commit is contained in:
Daniil Sobol 2023-12-21 23:49:15 +03:00
parent c545be676f
commit 987917de64
No known key found for this signature in database
GPG Key ID: 5121E3B3F2304E22
5 changed files with 142508 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
fluffy/network/state/state_content.nim
View File

@ -12,6 +12,7 @@
import
nimcrypto/[hash, sha2, keccak], stew/[objects, results, endians2], stint,
eth/common/eth_types,
ssz_serialization,
../../common/common_types
@ -20,7 +21,6 @@ export ssz_serialization, common_types, hash, results
type
NodeHash* = MDigest[32 * 8] # keccak256
CodeHash* = MDigest[32 * 8] # keccak256
StorageHash* = MDigest[32 * 8] # keccak256
StateRoot* = MDigest[32 * 8] # keccak256
Address* = array[20, byte]
@ -67,11 +67,7 @@ type
AccountTrieProof* = List[WitnessNode, 32]
AccountState* = object
nonce*: uint64
balance*: UInt256
storageHash*: StorageHash # is it NodeHash?
codeHash*: CodeHash
stateRoot*: StateRoot
account*: Account
proof*: AccountTrieProof
ContentKey* = object

26
fluffy/network/state/state_network.nim
View File

@ -6,8 +6,10 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
stew/results, chronos, chronicles,
eth/rlp,
std/sequtils,
stew/[results, byteutils], chronos, chronicles,
eth/[rlp, common],
eth/trie/hexary_proof_verification,
eth/p2p/discoveryv5/[protocol, enr],
../../database/content_db,
../wire/[portal_protocol, portal_stream, portal_protocol_config],
@ -61,9 +63,9 @@ proc getContent*(n: StateNetwork, key: ContentKey):
return Opt.some(contentResult.content)
proc validateContent(
n: StateNetwork,
contentKey: ByteList,
conetentValue: seq[byte]): Future[bool] {.async.} =
n: StateNetwork,
contentKey: ByteList,
contentValue: seq[byte]): Future[bool] {.async.} =
let key = contentKey.decode().valueOr:
return false
@ -73,9 +75,15 @@ proc validateContent(
of contractStorageTrieNode:
return true
of accountTrieProof:
var decodedValue = decodeSsz(conetentValue, AccountState).valueOr:
var decodedValue = decodeSsz(contentValue, AccountState).valueOr:
warn "Received invalid account trie proof", error
return false
# var decodedProof = seq[seq[byte]](@[])
# for path in decodedValue.proof:
# decodedProof.add(path.asSeq())
# let value = rlp.encode(decodedValue.account)
# var result = verifyMptProof(decodedProof, keccakHash(key.accountTrieProofKey.stateRoot), toSeq(key.accountTrieProofKey.address), value)
# echo ">>> verifyMptProof result", result
return true
of contractStorageTrieProof:
return true
@ -83,9 +91,9 @@ proc validateContent(
return true
proc validateContent(
n: StateNetwork,
contentKeys: ContentKeysList,
contentValues: seq[seq[byte]]): Future[bool] {.async.} =
n: StateNetwork,
contentKeys: ContentKeysList,
contentValues: seq[seq[byte]]): Future[bool] {.async.} =
for i, contentValue in contentValues:
let contentKey = contentKeys[i]
if await n.validateContent(contentKey, contentValue):

142409
fluffy/tests/state_data/proofs.full.block.0.json
View File

File diff suppressed because it is too large Load Diff

52
fluffy/tests/test_state_network.nim
View File

@ -7,11 +7,13 @@
import
std/[os, json, sequtils, strutils],
stew/byteutils,
stew/[byteutils, io2],
nimcrypto/hash,
testutils/unittests, chronos,
eth/[common/eth_hash, keys],
eth/keys,
eth/common/[eth_types, eth_hash],
eth/p2p/discoveryv5/protocol as discv5_protocol, eth/p2p/discoveryv5/routing_table,
../tools/state_bridge/state_bridge,
../../nimbus/[config, db/core_db, db/state_db],
../../nimbus/common/[chain_config, genesis],
../network/wire/[portal_protocol, portal_stream],
@ -37,13 +39,15 @@ procSuite "State Content Network":
asyncTest "Decode and use proofs":
let file = "./fluffy/tests/state_data/proofs.full.block.0.json"
var content: JsonNode
try:
content = io.readFile(file).parseJson()
except Exception as e:
echo "Skipping file ", file, " because of error"
let content = readAllFile(file).valueOr:
quit(1)
let decoded =
try:
Json.decode(content, state_bridge.JsonProofVector)
except SerializationError:
quit(1)
let
node1 = initDiscoveryNode(
rng, PrivateKey.random(rng[]), localAddress(20302))
@ -55,14 +59,14 @@ procSuite "State Content Network":
proto1 = StateNetwork.new(node1, ContentDB.new("", uint32.high, inMemory = true), sm1)
proto2 = StateNetwork.new(node2, ContentDB.new("", uint32.high, inMemory = true), sm2)
state_root = hexToByteArray[sizeof(Bytes32)](content["state_root"].getStr())
state_root = hexToByteArray[sizeof(state_content.StateRoot)](decoded.state_root)
check proto2.portalProtocol.addNode(node1.localNode) == Added
for proof in content["proofs"]:
for proof in decoded.proofs:
let
address = hexToByteArray[sizeof(state_content.Address)](proof["address"].getStr())
address = hexToByteArray[sizeof(state_content.Address)](proof.address)
key = AccountTrieProofKey(
address: address,
stateRoot: state_root)
@ -71,17 +75,18 @@ procSuite "State Content Network":
accountTrieProofKey: key)
var accountTrieProof = AccountTrieProof(@[])
for witness in proof["proof"]:
let witnessNode = ByteList(hexToSeqByte(witness.getStr()))
for witness in proof.proof:
let witnessNode = ByteList(hexToSeqByte(witness))
discard accountTrieProof.add(witnessNode)
let
state = proof["state"]
state = proof.state
account = Account(
nonce: state.nonce.uint64(),
balance: UInt256.fromHex(state.balance),
storageRoot: MDigest[256].fromHex(state.storage_hash),
codeHash: MDigest[256].fromHex(state.code_hash))
accountState = AccountState(
nonce: state["nonce"].getInt().uint64(),
balance: UInt256.fromHex(state["balance"].getStr()),
storageHash: MDigest[256].fromHex(state["storage_hash"].getStr()),
codeHash: MDigest[256].fromHex(state["code_hash"].getStr()),
stateRoot: MDigest[256].fromHex(content["state_root"].getStr()),
account: account,
proof: accountTrieProof)
encodedValue = SSZ.encode(accountState)
@ -92,13 +97,10 @@ procSuite "State Content Network":
check foundContent.isSome()
var decoded = decodeSsz(foundContent.get(), AccountState).get()
check decoded.nonce == 0
check decoded.balance.toHex() == "ad78ebc5ac6200000"
check toLowerAscii($decoded.storageHash) == "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
check toLowerAscii($decoded.codeHash) == "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
check toLowerAscii($decoded.stateRoot) == "d7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544"
# echo ">>> decoded: ", decoded
check decoded.account.nonce == 0
check decoded.account.balance.toHex() == "ad78ebc5ac6200000"
check toLowerAscii($decoded.account.storageRoot) == "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
check toLowerAscii($decoded.account.codeHash) == "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
await node1.closeWait()
await node2.closeWait()

83
fluffy/tools/state_bridge/state_bridge.nim
View File

@ -28,8 +28,9 @@ import
confutils, confutils/std/net, chronicles, chronicles/topics_registry,
json_rpc/clients/httpclient,
chronos,
stew/byteutils,
stew/[byteutils, io2],
eth/async_utils,
eth/common/eth_types,
../../network/state/state_content,
../../rpc/portal_rpc_client,
../../logging,
@ -38,6 +39,23 @@ import
const restRequestsTimeout = 30.seconds
type JsonAccount* = object
nonce*: int
balance*: string
storage_hash*: string
code_hash*: string
type JsonProof* = object
address*: string
state*: JsonAccount
proof*: seq[string]
type JsonProofVector* = object
`block`*: int
block_hash*: string
state_root*: string
proofs*: seq[JsonProof]
# TODO: From nimbus_binary_common, but we don't want to import that.
proc sleepAsync(t: TimeDiff): Future[void] =
sleepAsync(nanoseconds(
@ -56,36 +74,41 @@ proc run(config: BeaconBridgeConf) {.raises: [CatchableError].} =
await portalRpcClient.connect(config.rpcAddress, Port(config.rpcPort), false)
let files = collect(for f in walkDir(config.dataDir): f.path)
for file in files:
try:
let content = io.readFile(file).parseJson()
let state_root = hexToByteArray[sizeof(Bytes32)](content["state_root"].getStr())
for proof in content["proofs"]:
let address = hexToByteArray[sizeof(state_content.Address)](proof["address"].getStr())
let key = AccountTrieProofKey(
address: address,
stateRoot: state_root)
let contentKey = ContentKey(
contentType: ContentType.accountTrieProof,
accountTrieProofKey: key)
let encodedKey = encode(contentKey)
var accountTrieProof = AccountTrieProof(@[])
for witness in proof["proof"]:
let witnessNode = ByteList(hexToSeqByte(witness.getStr()))
discard accountTrieProof.add(witnessNode)
let state = proof["state"]
let accountState = AccountState(
nonce: state["nonce"].getInt().uint64(),
balance: UInt256.fromHex(state["balance"].getStr()),
storageHash: MDigest[256].fromHex(state["storage_hash"].getStr()),
codeHash: MDigest[256].fromHex(state["code_hash"].getStr()),
stateRoot: MDigest[256].fromHex(content["state_root"].getStr()),
proof: accountTrieProof)
let encodedValue = SSZ.encode(accountState)
discard await portalRpcClient.portal_stateGossip(encodedKey.asSeq().toHex(), encodedValue.toHex())
except Exception as e:
echo "Skipping file ", file, " because of error \n", e.msg
let content = readAllFile(file).valueOr:
echo "Skipping file ", file, " because of error \n", error
continue
let decoded =
try:
Json.decode(content, state_bridge.JsonProofVector)
except SerializationError as e:
echo "Skipping file ", file, " because of error \n", e.msg
continue
let state_root = hexToByteArray[sizeof(Bytes32)](decoded.state_root)
for proof in decoded.proofs:
let address = hexToByteArray[sizeof(state_content.Address)](proof.address)
let key = AccountTrieProofKey(
address: address,
stateRoot: state_root)
let contentKey = ContentKey(
contentType: ContentType.accountTrieProof,
accountTrieProofKey: key)
let encodedKey = encode(contentKey)
var accountTrieProof = AccountTrieProof(@[])
for witness in proof.proof:
let witnessNode = ByteList(hexToSeqByte(witness))
discard accountTrieProof.add(witnessNode)
let state = proof.state
let account = Account(
nonce: state.nonce.uint64(),
balance: UInt256.fromHex(state.balance),
storageRoot: MDigest[256].fromHex(state.storage_hash),
codeHash: MDigest[256].fromHex(state.code_hash))
let accountState = AccountState(
account: account,
proof: accountTrieProof)
let encodedValue = SSZ.encode(accountState)
discard await portalRpcClient.portal_stateGossip(encodedKey.asSeq().toHex(), encodedValue.toHex())
await portalRpcClient.close()
notice "Backfill done..."