persist discovered ENRs from the discv5 into CSV (#560)

* persist ENRs (parsed) from the discovery5 protocol

* filter fork_digest from eth2 data

* Update eth/p2p/discoveryv5/dcli.nim

Co-authored-by: Kim De Mey <kim.demey@gmail.com>

* apply reviewer's suggestions

Co-authored-by: Kim De Mey <kim.demey@gmail.com>
This commit is contained in:
Mikel Cortes 2022-11-23 23:48:44 +01:00 committed by GitHub
parent 9ba1eb99e3
commit f4ef9181c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 42 additions and 6 deletions

View File

@ -1,7 +1,7 @@
import import
std/[options, strutils, tables], std/[options, strutils, tables, sets],
confutils, confutils/std/net, chronicles, chronicles/topics_registry, confutils, confutils/std/net, chronicles, chronicles/topics_registry,
chronos, metrics, metrics/chronos_httpserver, stew/byteutils, chronos, metrics, metrics/chronos_httpserver, stew/byteutils, stew/bitops2,
../../keys, ../../net/nat, ../../keys, ../../net/nat,
"."/[enr, node], "."/[enr, node],
"."/protocol as discv5_protocol "."/protocol as discv5_protocol
@ -29,6 +29,11 @@ type
desc: "Listening address for the Discovery v5 traffic" desc: "Listening address for the Discovery v5 traffic"
name: "listen-address" }: ValidIpAddress name: "listen-address" }: ValidIpAddress
persistingFile* {.
defaultValue: "peerstore.csv",
desc: "File where the tool will keep the discovered records"
name: "persisting-file" .}: string
bootnodes* {. bootnodes* {.
desc: "ENR URI of node to bootstrap discovery with. Argument may be repeated" desc: "ENR URI of node to bootstrap discovery with. Argument may be repeated"
name: "bootnode" .}: seq[enr.Record] name: "bootnode" .}: seq[enr.Record]
@ -132,11 +137,42 @@ proc parseCmdArg*(T: type PrivateKey, p: string): T =
proc completeCmdArg*(T: type PrivateKey, val: string): seq[string] = proc completeCmdArg*(T: type PrivateKey, val: string): seq[string] =
return @[] return @[]
proc discover(d: discv5_protocol.Protocol) {.async.} = proc discover(d: discv5_protocol.Protocol, psFile: string) {.async.} =
info "Starting peer-discovery in Ethereum - persisting peers at: ", psFile
var ethNodes: HashSet[seq[byte]]
let ps = open(psFile, fmWrite)
defer: ps.close()
ps.write("pubkey,node_id,fork_digest,ip:port,attnets,attnets_number\n")
while true: while true:
let iTime = now(chronos.Moment)
let discovered = await d.queryRandom() let discovered = await d.queryRandom()
info "Lookup finished", nodes = discovered.len let qDuration = now(chronos.Moment) - iTime
await sleepAsync(30.seconds) info "Lookup finished", query_time = qDuration.secs, new_nodes = discovered.len, tot_peers=len(ethNodes)
for dNode in discovered:
let eth2 = dNode.record.tryGet("eth2", seq[byte])
let pubkey = dNode.record.tryGet("secp256k1", seq[byte])
let attnets = dNode.record.tryGet("attnets", seq[byte])
if eth2.isNone or attnets.isNone or pubkey.isNone: continue
if pubkey.get() in ethNodes: continue
ethNodes.incl(pubkey.get())
let forkDigest = eth2.get()
var bits = 0
for byt in attnets.get():
bits.inc(countOnes(byt.uint))
let str = "$#,$#,$#,$#,$#,$#\n"
let newLine = str % [pubkey.get().toHex, dNode.id.toHex, forkDigest[0..3].toHex, $dNode.address.get(), attnets.get().toHex, $bits]
ps.write(newLine)
await sleepAsync(1000) # 1 sec of delay
proc run(config: DiscoveryConf) = proc run(config: DiscoveryConf) =
let let
@ -188,7 +224,7 @@ proc run(config: DiscoveryConf) =
echo "No Talk Response message returned" echo "No Talk Response message returned"
of noCommand: of noCommand:
d.start() d.start()
waitFor(discover(d)) waitFor(discover(d, config.persistingFile))
when isMainModule: when isMainModule:
let config = DiscoveryConf.load() let config = DiscoveryConf.load()