status-im
/
nim-eth
mirror of https://github.com/status-im/nim-eth.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #310 from status-im/remove-discv5.0 

Remove support for discv5.0
This commit is contained in:
Kim De Mey 2020-11-13 15:08:34 +01:00 committed by GitHub
parent 5dff021cbc 18430ef127
commit bc8a8947fc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 1446 additions and 3225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
eth.nimble
View File

@ -52,21 +52,13 @@ proc runP2pTests() =
"test_hkdf",
"test_lru",
"test_discoveryv5",
"test_discv5_encoding",
"test_discv51_encoding",
"test_discoveryv5_encoding",
"test_routing_table"
]:
runTest("tests/p2p/" & filename)
proc runDiscv51Test() =
let path = "tests/p2p/test_discoveryv5"
echo "\nRunning: ", path
exec "nim c -r -d:UseDiscv51=true -d:release -d:chronicles_log_level=ERROR --verbosity:0 --hints:off " & path
rmFile path
task test_p2p, "run p2p tests":
runP2pTests()
runDiscv51Test()
proc runRlpTests() =
runTest("tests/rlp/all_tests")
@ -105,7 +97,7 @@ proc runDiscv5Tests() =
"test_hkdf",
"test_lru",
"test_discoveryv5",
"test_discv5_encoding",
"test_discoveryv5_encoding",
"test_routing_table"
]:
runTest("tests/p2p/" & filename)

18
eth/p2p/discoveryv5/dcli.nim
View File

@ -9,6 +9,7 @@ type
noCommand
ping
findnode
talkreq
DiscoveryConf* = object
logLevel* {.
@ -71,6 +72,11 @@ type
argument
desc: "ENR URI of the node to send a findNode message"
name: "node" .}: Node
of talkreq:
talkreqTarget* {.
argument
desc: "ENR URI of the node to send a talkreq message"
name: "node" .}: Node
proc parseCmdArg*(T: type enr.Record, p: TaintedString): T =
if not fromURI(result, p):
@ -166,17 +172,19 @@ proc run(config: DiscoveryConf) =
else:
echo "No Pong message returned"
of findnode:
# Discv5.1 and Discv5.0 have a different findnode API
when UseDiscv51:
let nodes = waitFor d.findNode(config.findNodeTarget, @[config.distance])
else:
let nodes = waitFor d.findNode(config.findNodeTarget, config.distance)
let nodes = waitFor d.findNode(config.findNodeTarget, @[config.distance])
if nodes.isOk():
echo "Received valid records:"
for node in nodes[]:
echo $node.record & " - " & shortLog(node)
else:
echo "No Nodes message returned"
of talkreq:
let talkresp = waitFor d.talkreq(config.talkreqTarget, @[], @[])
if talkresp.isOk():
echo talkresp[]
else:
echo "No Talk Response message returned"
of noCommand:
d.start()
runForever()

725
eth/p2p/discoveryv5/encoding.nim
View File

@ -1,75 +1,103 @@
import
std/[tables, options],
nimcrypto, stint, chronicles, stew/results, bearssl,
nimcrypto, stint, chronicles, bearssl, stew/[results, byteutils],
eth/[rlp, keys], types, node, enr, hkdf, sessions
from stew/objects import checkedEnumAssign
export keys
{.push raises: [Defect].}
logScope:
topics = "discv5"
const
idNoncePrefix = "discovery-id-nonce"
version: uint16 = 1
idSignatureText = "discovery v5 identity proof"
keyAgreementPrefix = "discovery v5 key agreement"
authSchemeName* = "gcm"
protocolIdStr = "discv5"
protocolId = toBytes(protocolIdStr)
gcmNonceSize* = 12
idNonceSize* = 16
gcmTagSize* = 16
tagSize* = 32 ## size of the tag where each message (except whoareyou) starts
## with
ivSize* = 16
staticHeaderSize = protocolId.len + 2 + 2 + 1 + gcmNonceSize
authdataHeadSize = sizeof(NodeId) + 1 + 1
whoareyouSize = ivSize + staticHeaderSize + idNonceSize + 8
type
PacketTag* = array[tagSize, byte]
AESGCMNonce* = array[gcmNonceSize, byte]
IdNonce* = array[idNonceSize, byte]
AuthResponse* = object
version*: int
signature*: array[64, byte]
record*: Option[enr.Record]
WhoareyouData* = object
requestNonce*: AESGCMNonce
idNonce*: IdNonce # TODO: This data is also available in challengeData
recordSeq*: uint64
challengeData*: seq[byte]
Challenge* = object
whoareyouData*: WhoareyouData
pubkey*: Option[PublicKey]
StaticHeader* = object
flag: Flag
nonce: AESGCMNonce
authdataSize: uint16
HandshakeSecrets* = object
initiatorKey*: AesKey
recipientKey*: AesKey
Flag* = enum
OrdinaryMessage = 0x00
Whoareyou = 0x01
HandshakeMessage = 0x02
Packet* = object
case flag*: Flag
of OrdinaryMessage:
messageOpt*: Option[Message]
requestNonce*: AESGCMNonce
srcId*: NodeId
of Whoareyou:
whoareyou*: WhoareyouData
of HandshakeMessage:
message*: Message # In a handshake we expect to always be able to decrypt
# TODO record or node immediately?
node*: Option[Node]
srcIdHs*: NodeId
Codec* = object
localNode*: Node
privKey*: PrivateKey
handshakes*: Table[HandShakeKey, Whoareyou]
handshakes*: Table[HandShakeKey, Challenge]
sessions*: Sessions
HandshakeSecrets = object
writeKey: AesKey
readKey: AesKey
authRespKey: AesKey
DecodeResult*[T] = Result[T, cstring]
AuthHeader* = object
auth*: AuthTag
idNonce*: IdNonce
scheme*: string
ephemeralKey*: array[64, byte]
response*: seq[byte]
DecodeError* = enum
HandshakeError = "discv5: handshake failed"
PacketError = "discv5: invalid packet"
DecryptError = "discv5: decryption failed"
UnsupportedMessage = "discv5: unsupported message"
DecodeResult*[T] = Result[T, DecodeError]
EncodeResult*[T] = Result[T, cstring]
proc mapErrTo[T, E](r: Result[T, E], v: static DecodeError):
DecodeResult[T] =
r.mapErr(proc (e: E): DecodeError = v)
proc idNonceHash(nonce, ephkey: openarray[byte]): MDigest[256] =
proc idHash(challengeData, ephkey: openarray[byte], nodeId: NodeId):
MDigest[256] =
var ctx: sha256
ctx.init()
ctx.update(idNoncePrefix)
ctx.update(nonce)
ctx.update(idSignatureText)
ctx.update(challengeData)
ctx.update(ephkey)
ctx.update(nodeId.toByteArrayBE())
result = ctx.finish()
ctx.clear()
proc signIDNonce*(privKey: PrivateKey, idNonce, ephKey: openarray[byte]):
SignatureNR =
signNR(privKey, SkMessage(idNonceHash(idNonce, ephKey).data))
proc createIdSignature*(privKey: PrivateKey, challengeData,
ephKey: openarray[byte], nodeId: NodeId): SignatureNR =
signNR(privKey, SkMessage(idHash(challengeData, ephKey, nodeId).data))
proc deriveKeys(n1, n2: NodeID, priv: PrivateKey, pub: PublicKey,
idNonce: openarray[byte]): HandshakeSecrets =
proc verifyIdSignature*(sig: SignatureNR, challengeData, ephKey: openarray[byte],
nodeId: NodeId, pubKey: PublicKey): bool =
let h = idHash(challengeData, ephKey, nodeId)
verify(sig, SkMessage(h.data), pubKey)
proc deriveKeys*(n1, n2: NodeID, priv: PrivateKey, pub: PublicKey,
challengeData: openarray[byte]): HandshakeSecrets =
let eph = ecdhRawFull(priv, pub)
var info = newSeqOfCap[byte](keyAgreementPrefix.len + 32 * 2)
@ -78,9 +106,11 @@ proc deriveKeys(n1, n2: NodeID, priv: PrivateKey, pub: PublicKey,
info.add(n2.toByteArrayBE())
var secrets: HandshakeSecrets
static: assert(sizeof(secrets) == aesKeySize * 3)
static: assert(sizeof(secrets) == aesKeySize * 2)
var res = cast[ptr UncheckedArray[byte]](addr secrets)
hkdf(sha256, eph.data, idNonce, info, toOpenArray(res, 0, sizeof(secrets) - 1))
hkdf(sha256, eph.data, challengeData, info,
toOpenArray(res, 0, sizeof(secrets) - 1))
secrets
proc encryptGCM*(key, nonce, pt, authData: openarray[byte]): seq[byte] =
@ -91,101 +121,6 @@ proc encryptGCM*(key, nonce, pt, authData: openarray[byte]): seq[byte] =
ectx.getTag(result.toOpenArray(pt.len, result.high))
ectx.clear()
proc encodeAuthHeader*(rng: var BrHmacDrbgContext,
c: Codec,
toId: NodeID,
nonce: array[gcmNonceSize, byte],
challenge: Whoareyou):
(seq[byte], HandshakeSecrets) =
## Encodes the auth-header, which is required for the packet in response to a
## WHOAREYOU packet. Requires the id-nonce and the enr-seq that were in the
## WHOAREYOU packet, and the public key of the node sending it.
var resp = AuthResponse(version: 5)
let ln = c.localNode
if challenge.recordSeq < ln.record.seqNum:
resp.record = some(ln.record)
else:
resp.record = none(enr.Record)
let ephKeys = KeyPair.random(rng)
let signature = signIDNonce(c.privKey, challenge.idNonce,
ephKeys.pubkey.toRaw)
resp.signature = signature.toRaw
# Calling `encodePacket` for handshake should always be with a challenge
# with the pubkey of the node we are targetting.
doAssert(challenge.pubKey.isSome())
let secrets = deriveKeys(ln.id, toId, ephKeys.seckey, challenge.pubKey.get(),
challenge.idNonce)
let respRlp = rlp.encode(resp)
var zeroNonce: array[gcmNonceSize, byte]
let respEnc = encryptGCM(secrets.authRespKey, zeroNonce, respRlp, [])
let header = AuthHeader(auth: nonce, idNonce: challenge.idNonce,
scheme: authSchemeName, ephemeralKey: ephKeys.pubkey.toRaw,
response: respEnc)
(rlp.encode(header), secrets)
proc `xor`[N: static[int], T](a, b: array[N, T]): array[N, T] =
for i in 0 .. a.high:
result[i] = a[i] xor b[i]
proc packetTag(destNode, srcNode: NodeID): PacketTag =
let
destId = destNode.toByteArrayBE()
srcId = srcNode.toByteArrayBE()
destidHash = sha256.digest(destId)
result = srcId xor destidHash.data
proc encodePacket*(
rng: var BrHmacDrbgContext,
c: var Codec,
toId: NodeID,
toAddr: Address,
message: openarray[byte],
challenge: Whoareyou):
(seq[byte], array[gcmNonceSize, byte]) =
## Encode a packet. This can be a regular packet or a packet in response to a
## WHOAREYOU packet. The latter is the case when the `challenge` parameter is
## provided.
var nonce: array[gcmNonceSize, byte]
brHmacDrbgGenerate(rng, nonce)
let tag = packetTag(toId, c.localNode.id)
var packet: seq[byte]
packet.add(tag)
if challenge.isNil:
# Message packet or random packet
let headEnc = rlp.encode(nonce)
packet.add(headEnc)
# TODO: Should we change API to get just the key we need?
var writeKey, readKey: AesKey
if c.sessions.load(toId, toAddr, readKey, writeKey):
packet.add(encryptGCM(writeKey, nonce, message, tag))
else:
# We might not have the node's keys if the handshake hasn't been performed
# yet. That's fine, we send a random-packet and we will be responded with
# a WHOAREYOU packet.
var randomData: array[44, byte]
brHmacDrbgGenerate(rng, randomData)
packet.add(randomData)
else:
# Handshake
let (headEnc, secrets) = encodeAuthHeader(rng, c, toId, nonce, challenge)
packet.add(headEnc)
c.sessions.store(toId, toAddr, secrets.readKey, secrets.writeKey)
packet.add(encryptGCM(secrets.writeKey, nonce, message, tag))
(packet, nonce)
proc decryptGCM*(key: AesKey, nonce, ct, authData: openarray[byte]):
Option[seq[byte]] =
if ct.len <= gcmTagSize:
@ -205,24 +140,218 @@ proc decryptGCM*(key: AesKey, nonce, ct, authData: openarray[byte]):
return some(res)
proc encryptHeader*(id: NodeId, iv, header: openarray[byte]): seq[byte] =
var ectx: CTR[aes128]
ectx.init(id.toByteArrayBE().toOpenArray(0, 15), iv)
result = newSeq[byte](header.len)
ectx.encrypt(header, result)
ectx.clear()
proc hasHandshake*(c: Codec, key: HandShakeKey): bool =
c.handshakes.hasKey(key)
proc encodeStaticHeader*(flag: Flag, nonce: AESGCMNonce, authSize: int):
seq[byte] =
result.add(protocolId)
result.add(version.toBytesBE())
result.add(byte(flag))
result.add(nonce)
# TODO: assert on authSize of > 2^16?
result.add((uint16(authSize)).toBytesBE())
proc encodeMessagePacket*(rng: var BrHmacDrbgContext, c: var Codec,
toId: NodeID, toAddr: Address, message: openarray[byte]):
(seq[byte], AESGCMNonce) =
var nonce: AESGCMNonce
brHmacDrbgGenerate(rng, nonce) # Random AESGCM nonce
var iv: array[ivSize, byte]
brHmacDrbgGenerate(rng, iv) # Random IV
# static-header
let authdata = c.localNode.id.toByteArrayBE()
let staticHeader = encodeStaticHeader(Flag.OrdinaryMessage, nonce,
authdata.len())
# header = static-header || authdata
var header: seq[byte]
header.add(staticHeader)
header.add(authdata)
# message
var messageEncrypted: seq[byte]
var initiatorKey, recipientKey: AesKey
if c.sessions.load(toId, toAddr, recipientKey, initiatorKey):
messageEncrypted = encryptGCM(initiatorKey, nonce, message, @iv & header)
else:
# We might not have the node's keys if the handshake hasn't been performed
# yet. That's fine, we send a random-packet and we will be responded with
# a WHOAREYOU packet.
# Select 20 bytes of random data, which is the smallest possible ping
# message. 16 bytes for the gcm tag and 4 bytes for ping with requestId of
# 1 byte (e.g "01c20101"). Could increase to 27 for 8 bytes requestId in
# case this must not look like a random packet.
var randomData: array[gcmTagSize + 4, byte]
brHmacDrbgGenerate(rng, randomData)
messageEncrypted.add(randomData)
let maskedHeader = encryptHeader(toId, iv, header)
var packet: seq[byte]
packet.add(iv)
packet.add(maskedHeader)
packet.add(messageEncrypted)
return (packet, nonce)
proc encodeWhoareyouPacket*(rng: var BrHmacDrbgContext, c: var Codec,
toId: NodeID, toAddr: Address, requestNonce: AESGCMNonce, recordSeq: uint64,
pubkey: Option[PublicKey]): seq[byte] =
var idNonce: IdNonce
brHmacDrbgGenerate(rng, idNonce)
# authdata
var authdata: seq[byte]
authdata.add(idNonce)
authdata.add(recordSeq.tobytesBE)
# static-header
let staticHeader = encodeStaticHeader(Flag.Whoareyou, requestNonce,
authdata.len())
# header = static-header || authdata
var header: seq[byte]
header.add(staticHeader)
header.add(authdata)
var iv: array[ivSize, byte]
brHmacDrbgGenerate(rng, iv) # Random IV
let maskedHeader = encryptHeader(toId, iv, header)
var packet: seq[byte]
packet.add(iv)
packet.add(maskedHeader)
let
whoareyouData = WhoareyouData(
requestNonce: requestNonce,
idNonce: idNonce,
recordSeq: recordSeq,
challengeData: @iv & header)
challenge = Challenge(whoareyouData: whoareyouData, pubkey: pubkey)
key = HandShakeKey(nodeId: toId, address: $toAddr)
c.handshakes[key] = challenge
return packet
proc encodeHandshakePacket*(rng: var BrHmacDrbgContext, c: var Codec,
toId: NodeID, toAddr: Address, message: openarray[byte],
whoareyouData: WhoareyouData, pubkey: PublicKey): seq[byte] =
var header: seq[byte]
var nonce: AESGCMNonce
brHmacDrbgGenerate(rng, nonce)
var iv: array[ivSize, byte]
brHmacDrbgGenerate(rng, iv) # Random IV
var authdata: seq[byte]
var authdataHead: seq[byte]
authdataHead.add(c.localNode.id.toByteArrayBE())
authdataHead.add(64'u8) # sig-size: 64
authdataHead.add(33'u8) # eph-key-size: 33
authdata.add(authdataHead)
let ephKeys = KeyPair.random(rng)
let signature = createIdSignature(c.privKey, whoareyouData.challengeData,
ephKeys.pubkey.toRawCompressed(), toId)
authdata.add(signature.toRaw())
# compressed pub key format (33 bytes)
authdata.add(ephKeys.pubkey.toRawCompressed())
# Add ENR of sequence number is newer
if whoareyouData.recordSeq < c.localNode.record.seqNum:
authdata.add(encode(c.localNode.record))
let secrets = deriveKeys(c.localNode.id, toId, ephKeys.seckey, pubkey,
whoareyouData.challengeData)
# Header
let staticHeader = encodeStaticHeader(Flag.HandshakeMessage, nonce,
authdata.len())
header.add(staticHeader)
header.add(authdata)
c.sessions.store(toId, toAddr, secrets.recipientKey, secrets.initiatorKey)
let messageEncrypted = encryptGCM(secrets.initiatorKey, nonce, message,
@iv & header)
let maskedHeader = encryptHeader(toId, iv, header)
var packet: seq[byte]
packet.add(iv)
packet.add(maskedHeader)
packet.add(messageEncrypted)
return packet
proc decodeHeader*(id: NodeId, iv, maskedHeader: openarray[byte]):
DecodeResult[(StaticHeader, seq[byte])] =
# No need to check staticHeader size as that is included in minimum packet
# size check in decodePacket
var ectx: CTR[aes128]
ectx.init(id.toByteArrayBE().toOpenArray(0, ivSize - 1), iv)
# Decrypt static-header part of the header
var staticHeader = newSeq[byte](staticHeaderSize)
ectx.decrypt(maskedHeader.toOpenArray(0, staticHeaderSize - 1), staticHeader)
# Check fields of the static-header
if staticHeader.toOpenArray(0, protocolId.len - 1) != protocolId:
return err("Invalid protocol id")
if uint16.fromBytesBE(staticHeader.toOpenArray(6, 7)) != version:
return err("Invalid protocol version")
var flag: Flag
if not checkedEnumAssign(flag, staticHeader[8]):
return err("Invalid packet flag")
var nonce: AESGCMNonce
copyMem(addr nonce[0], unsafeAddr staticHeader[9], gcmNonceSize)
let authdataSize = uint16.fromBytesBE(staticHeader.toOpenArray(21,
staticHeader.high))
# Input should have minimum size of staticHeader + provided authdata size
# Can be larger as there can come a message after.
if maskedHeader.len < staticHeaderSize + int(authdataSize):
return err("Authdata is smaller than authdata-size indicates")
var authdata = newSeq[byte](int(authdataSize))
ectx.decrypt(maskedHeader.toOpenArray(staticHeaderSize,
staticHeaderSize + int(authdataSize) - 1), authdata)
ectx.clear()
ok((StaticHeader(authdataSize: authdataSize, flag: flag, nonce: nonce),
staticHeader & authdata))
proc decodeMessage*(body: openarray[byte]): DecodeResult[Message] =
## Decodes to the specific `Message` type.
if body.len < 1:
return err(PacketError)
return err("No message data")
if body[0] < MessageKind.low.byte or body[0] > MessageKind.high.byte:
return err(PacketError)
var kind: MessageKind
if not checkedEnumAssign(kind, body[0]):
return err("Invalid message type")
# This cast is covered by the above check (else we could get enum with invalid
# data!). However, can't we do this in a cleaner way?
let kind = cast[MessageKind](body[0])
var message = Message(kind: kind)
var rlp = rlpFromBytes(body.toOpenArray(1, body.high))
if rlp.enterList:
try:
message.reqId = rlp.read(RequestId)
except RlpError:
return err(PacketError)
except RlpError, ValueError:
return err("Invalid request-id")
proc decode[T](rlp: var Rlp, v: var T)
{.inline, nimcall, raises:[RlpError, ValueError, Defect].} =
@ -231,160 +360,214 @@ proc decodeMessage*(body: openarray[byte]): DecodeResult[Message] =
try:
case kind
of unused: return err(PacketError)
of unused: return err("Invalid message type")
of ping: rlp.decode(message.ping)
of pong: rlp.decode(message.pong)
of findNode: rlp.decode(message.findNode)
of nodes: rlp.decode(message.nodes)
of talkreq: rlp.decode(message.talkreq)
of talkresp: rlp.decode(message.talkresp)
of regtopic, ticket, regconfirmation, topicquery:
# TODO: Implement support for topic advertisement
return err(UnsupportedMessage)
# We just pass the empty type of this message without attempting to
# decode, so that the protocol knows what was received.
# But we ignore the message as per specification as "the content and
# semantics of this message are not final".
discard
except RlpError, ValueError:
return err(PacketError)
return err("Invalid message encoding")
ok(message)
else:
err(PacketError)
err("Invalid message encoding: no rlp list")
proc decodeAuthResp*(c: Codec, fromId: NodeId, head: AuthHeader,
challenge: Whoareyou, newNode: var Node): DecodeResult[HandshakeSecrets] =
## Decrypts and decodes the auth-response, which is part of the auth-header.
## Requires the id-nonce from the WHOAREYOU packet that was send.
## newNode can be nil in case node was already known (no was ENR send).
if head.scheme != authSchemeName:
warn "Unknown auth scheme"
return err(HandshakeError)
proc decodeMessagePacket(c: var Codec, fromAddr: Address, nonce: AESGCMNonce,
iv, header, ct: openArray[byte]): DecodeResult[Packet] =
# We now know the exact size that the header should be
if header.len != staticHeaderSize + sizeof(NodeId):
return err("Invalid header length for ordinary message packet")
let ephKey = ? PublicKey.fromRaw(head.ephemeralKey).mapErrTo(HandshakeError)
# Need to have at minimum the gcm tag size for the message.
if ct.len < gcmTagSize:
return err("Invalid message length for ordinary message packet")
let secrets =
deriveKeys(fromId, c.localNode.id, c.privKey, ephKey, challenge.idNonce)
let srcId = NodeId.fromBytesBE(header.toOpenArray(staticHeaderSize,
header.high))
var zeroNonce: array[gcmNonceSize, byte]
let respData = decryptGCM(secrets.authRespKey, zeroNonce, head.response, [])
if respData.isNone():
return err(HandshakeError)
var initiatorKey, recipientKey: AesKey
if not c.sessions.load(srcId, fromAddr, recipientKey, initiatorKey):
# Don't consider this an error, simply haven't done a handshake yet or
# the session got removed.
trace "Decrypting failed (no keys)"
return ok(Packet(flag: Flag.OrdinaryMessage, requestNonce: nonce,
srcId: srcId))
var authResp: AuthResponse
try:
# Signature check of record happens in decode.
authResp = rlp.decode(respData.get(), AuthResponse)
except RlpError, ValueError:
return err(HandshakeError)
let pt = decryptGCM(recipientKey, nonce, ct, @iv & @header)
if pt.isNone():
# Don't consider this an error, the session got probably removed at the
# peer's side and a random message is send.
trace "Decrypting failed (invalid keys)"
c.sessions.del(srcId, fromAddr)
return ok(Packet(flag: Flag.OrdinaryMessage, requestNonce: nonce,
srcId: srcId))
let message = ? decodeMessage(pt.get())
return ok(Packet(flag: Flag.OrdinaryMessage,
messageOpt: some(message), requestNonce: nonce, srcId: srcId))
proc decodeWhoareyouPacket(c: var Codec, nonce: AESGCMNonce,
iv, header: openArray[byte]): DecodeResult[Packet] =
# TODO improve this
let authdata = header[staticHeaderSize..header.high()]
# We now know the exact size that the authdata should be
if authdata.len != idNonceSize + sizeof(uint64):
return err("Invalid header length for whoareyou packet")
var idNonce: IdNonce
copyMem(addr idNonce[0], unsafeAddr authdata[0], idNonceSize)
let whoareyou = WhoareyouData(requestNonce: nonce, idNonce: idNonce,
recordSeq: uint64.fromBytesBE(
authdata.toOpenArray(idNonceSize, authdata.high)),
challengeData: @iv & @header)
return ok(Packet(flag: Flag.Whoareyou, whoareyou: whoareyou))
proc decodeHandshakePacket(c: var Codec, fromAddr: Address, nonce: AESGCMNonce,
iv, header, ct: openArray[byte]): DecodeResult[Packet] =
# Checking if there is enough data to decode authdata-head
if header.len <= staticHeaderSize + authdataHeadSize:
return err("Invalid header for handshake message packet: no authdata-head")
# Need to have at minimum the gcm tag size for the message.
# TODO: And actually, as we should be able to decrypt it, it should also be
# a valid message and thus we could increase here to the size of the smallest
# message possible.
if ct.len < gcmTagSize:
return err("Invalid message length for ordinary message packet")
let
authdata = header[staticHeaderSize..header.high()]
srcId = NodeId.fromBytesBE(authdata.toOpenArray(0, 31))
sigSize = uint8(authdata[32])
ephKeySize = uint8(authdata[33])
# If smaller, as it can be equal and bigger (in case it holds an enr)
if header.len < staticHeaderSize + authdataHeadSize + int(sigSize) + int(ephKeySize):
return err("Invalid header for handshake message packet")
let key = HandShakeKey(nodeId: srcId, address: $fromAddr)
var challenge: Challenge
if not c.handshakes.pop(key, challenge):
return err("No challenge found: timed out or unsolicited packet")
# This should be the compressed public key. But as we use the provided
# ephKeySize, it should also work with full sized key. However, the idNonce
# signature verification will fail.
let
ephKeyPos = authdataHeadSize + int(sigSize)
ephKeyRaw = authdata[ephKeyPos..<ephKeyPos + int(ephKeySize)]
ephKey = ? PublicKey.fromRaw(ephKeyRaw)
var record: Option[enr.Record]
let recordPos = ephKeyPos + int(ephKeySize)
if authdata.len() > recordPos:
# There is possibly an ENR still
try:
# Signature check of record happens in decode.
record = some(rlp.decode(authdata.toOpenArray(recordPos, authdata.high),
enr.Record))
except RlpError, ValueError:
return err("Invalid encoded ENR")
var pubKey: PublicKey
if authResp.record.isSome():
var newNode: Option[Node]
# TODO: Shall we return Node or Record? Record makes more sense, but we do
# need the pubkey and the nodeid
if record.isSome():
# Node returned might not have an address or not a valid address.
newNode = ? newNode(authResp.record.get()).mapErrTo(HandshakeError)
if newNode.id != fromId:
return err(HandshakeError)
let node = ? newNode(record.get())
if node.id != srcId:
return err("Invalid node id: does not match node id of ENR")
pubKey = newNode.pubKey
# Note: Not checking if the record seqNum is higher than the one we might
# have stored as it comes from this node directly.
pubKey = node.pubKey
newNode = some(node)
else:
if challenge.pubKey.isSome():
pubKey = challenge.pubKey.get()
# TODO: Hmm, should we still verify node id of the ENR of this node?
if challenge.pubkey.isSome():
pubKey = challenge.pubkey.get()
else:
# We should have received a Record in this case.
return err(HandshakeError)
return err("Missing ENR in handshake packet")
# Verify the id-nonce-sig
let sig = ? SignatureNR.fromRaw(authResp.signature).mapErrTo(HandshakeError)
let h = idNonceHash(head.idNonce, head.ephemeralKey)
if verify(sig, SkMessage(h.data), pubkey):
ok(secrets)
else:
err(HandshakeError)
# Verify the id-signature
let sig = ? SignatureNR.fromRaw(
authdata.toOpenArray(authdataHeadSize, authdataHeadSize + int(sigSize) - 1))
if not verifyIdSignature(sig, challenge.whoareyouData.challengeData,
ephKeyRaw, c.localNode.id, pubkey):
return err("Invalid id-signature")
proc decodePacket*(c: var Codec,
fromId: NodeID,
fromAddr: Address,
input: openArray[byte],
authTag: var AuthTag,
newNode: var Node): DecodeResult[Message] =
# Do the key derivation step only after id-signature is verified as this is
# costly.
var secrets = deriveKeys(srcId, c.localNode.id, c.privKey,
ephKey, challenge.whoareyouData.challengeData)
swap(secrets.recipientKey, secrets.initiatorKey)
let pt = decryptGCM(secrets.recipientKey, nonce, ct, @iv & @header)
if pt.isNone():
c.sessions.del(srcId, fromAddr)
# Differently from an ordinary message, this is seen as an error as the
# secrets just got negotiated in the handshake and thus decryption should
# always work. We do not send a new Whoareyou on these as it probably means
# there is a compatiblity issue and we might loop forever in failed
# handshakes with this peer.
return err("Decryption of message failed in handshake packet")
let message = ? decodeMessage(pt.get())
# Only store the session secrets in case decryption was successful and also
# in case the message can get decoded.
c.sessions.store(srcId, fromAddr, secrets.recipientKey, secrets.initiatorKey)
return ok(Packet(flag: Flag.HandshakeMessage, message: message,
srcIdHs: srcId, node: newNode))
proc decodePacket*(c: var Codec, fromAddr: Address, input: openArray[byte]):
DecodeResult[Packet] =
## Decode a packet. This can be a regular packet or a packet in response to a
## WHOAREYOU packet. In case of the latter a `newNode` might be provided.
var r = rlpFromBytes(input.toOpenArray(tagSize, input.high))
var auth: AuthHeader
# Smallest packet is Whoareyou packet so that is the minimum size
if input.len() < whoareyouSize:
return err("Packet size too short")
var readKey: AesKey
logScope: sender = $fromAddr
# TODO: Just pass in the full input? Makes more sense perhaps.
let (staticHeader, header) = ? decodeHeader(c.localNode.id,
input.toOpenArray(0, ivSize - 1), # IV
# Don't know the size yet of the full header, so we pass all.
input.toOpenArray(ivSize, input.high))
if r.isList:
# Handshake - rlp list indicates auth-header
try:
auth = r.read(AuthHeader)
except RlpError:
return err(PacketError)
authTag = auth.auth
case staticHeader.flag
of OrdinaryMessage:
return decodeMessagePacket(c, fromAddr, staticHeader.nonce,
input.toOpenArray(0, ivSize - 1), header,
input.toOpenArray(ivSize + header.len, input.high))
let key = HandShakeKey(nodeId: fromId, address: $fromAddr)
var challenge: Whoareyou
# Note: We remove (pop) the stored handshake data here on failure on purpose
# as mitigation for a DoS attack where an invalid handshake is send
# repeatedly, which causes the signature verification to be done until
# handshake timeout, in case the stored data is not removed at first fail.
# See also more info here: https://github.com/prysmaticlabs/prysm/issues/7346
#
# It should be noted though that this means that now it might be possible to
# drop a handshake on purpose by a malicious party. But only if that
# attacker manages to spoof the IP-address of a peer A, and manages to
# listen to traffic between peer A and B that are starting a handshake, and
# next manages to be faster in sending out the (invalid) handshake. And this
# for each attempt in order to deny the peers setting up a session.
# However, this looks like a much more difficult scenario to pull off than
# the more convenient DoS attack. The DoS attack might have less heavy
# consequences though.
if not c.handshakes.pop(key, challenge):
debug "Decoding failed (no previous stored handshake challenge)"
return err(HandshakeError)
of Whoareyou:
# Header size got checked in decode header
return decodeWhoareyouPacket(c, staticHeader.nonce,
input.toOpenArray(0, ivSize - 1), header)
if auth.idNonce != challenge.idNonce:
trace "Decoding failed (different nonce)"
return err(HandshakeError)
let secrets = c.decodeAuthResp(fromId, auth, challenge, newNode)
if secrets.isErr:
trace "Decoding failed (invalid auth response)"
return err(HandshakeError)
var sec = secrets[]
c.handshakes.del(key)
# Swap keys to match remote
swap(sec.readKey, sec.writeKey)
c.sessions.store(fromId, fromAddr, sec.readKey, sec.writeKey)
readKey = sec.readKey
else:
# Message packet or random packet - rlp bytes (size 12) indicates auth-tag
try:
authTag = r.read(AuthTag)
except RlpError:
return err(PacketError)
auth.auth = authTag
# TODO: Should we change API to get just the key we need?
var writeKey: AesKey
if not c.sessions.load(fromId, fromAddr, readKey, writeKey):
trace "Decoding failed (no keys)"
return err(DecryptError)
let headSize = tagSize + r.position
let message = decryptGCM(
readKey, auth.auth,
input.toOpenArray(headSize, input.high),
input.toOpenArray(0, tagSize - 1))
if message.isNone():
c.sessions.del(fromId, fromAddr)
return err(DecryptError)
decodeMessage(message.get())
of HandshakeMessage:
return decodeHandshakePacket(c, fromAddr, staticHeader.nonce,
input.toOpenArray(0, ivSize - 1), header,
input.toOpenArray(ivSize + header.len, input.high))
proc init*(T: type RequestId, rng: var BrHmacDrbgContext): T =
var buf: array[sizeof(T), byte]
brHmacDrbgGenerate(rng, buf)
var id: T
copyMem(addr id, addr buf[0], sizeof(id))
id
var reqId = RequestId(id: newSeq[byte](8)) # RequestId must be <= 8 bytes
brHmacDrbgGenerate(rng, reqId.id)
reqId
proc numFields(T: typedesc): int =
for k, v in fieldPairs(default(T)): inc result

584
eth/p2p/discoveryv5/encodingv1.nim
View File

@ -1,584 +0,0 @@
import
std/[tables, options],
nimcrypto, stint, chronicles, bearssl, stew/[results, byteutils],
eth/[rlp, keys], typesv1, node, enr, hkdf, sessions
from stew/objects import checkedEnumAssign
export keys
{.push raises: [Defect].}
logScope:
topics = "discv5"
const
version: uint16 = 1
idSignatureText = "discovery v5 identity proof"
keyAgreementPrefix = "discovery v5 key agreement"
protocolIdStr = "discv5"
protocolId = toBytes(protocolIdStr)
gcmNonceSize* = 12
idNonceSize* = 16
gcmTagSize* = 16
ivSize* = 16
staticHeaderSize = protocolId.len + 2 + 2 + 1 + gcmNonceSize
authdataHeadSize = sizeof(NodeId) + 1 + 1
whoareyouSize = ivSize + staticHeaderSize + idNonceSize + 8
type
AESGCMNonce* = array[gcmNonceSize, byte]
IdNonce* = array[idNonceSize, byte]
WhoareyouData* = object
requestNonce*: AESGCMNonce
idNonce*: IdNonce # TODO: This data is also available in challengeData
recordSeq*: uint64
challengeData*: seq[byte]
Challenge* = object
whoareyouData*: WhoareyouData
pubkey*: Option[PublicKey]
StaticHeader* = object
flag: Flag
nonce: AESGCMNonce
authdataSize: uint16
HandshakeSecrets* = object
initiatorKey*: AesKey
recipientKey*: AesKey
Flag* = enum
OrdinaryMessage = 0x00
Whoareyou = 0x01
HandshakeMessage = 0x02
Packet* = object
case flag*: Flag
of OrdinaryMessage:
messageOpt*: Option[Message]
requestNonce*: AESGCMNonce
srcId*: NodeId
of Whoareyou:
whoareyou*: WhoareyouData
of HandshakeMessage:
message*: Message # In a handshake we expect to always be able to decrypt
# TODO record or node immediately?
node*: Option[Node]
srcIdHs*: NodeId
Codec* = object
localNode*: Node
privKey*: PrivateKey
handshakes*: Table[HandShakeKey, Challenge]
sessions*: Sessions
DecodeResult*[T] = Result[T, cstring]
proc idHash(challengeData, ephkey: openarray[byte], nodeId: NodeId):
MDigest[256] =
var ctx: sha256
ctx.init()
ctx.update(idSignatureText)
ctx.update(challengeData)
ctx.update(ephkey)
ctx.update(nodeId.toByteArrayBE())
result = ctx.finish()
ctx.clear()
proc createIdSignature*(privKey: PrivateKey, challengeData,
ephKey: openarray[byte], nodeId: NodeId): SignatureNR =
signNR(privKey, SkMessage(idHash(challengeData, ephKey, nodeId).data))
proc verifyIdSignature*(sig: SignatureNR, challengeData, ephKey: openarray[byte],
nodeId: NodeId, pubKey: PublicKey): bool =
let h = idHash(challengeData, ephKey, nodeId)
verify(sig, SkMessage(h.data), pubKey)
proc deriveKeys*(n1, n2: NodeID, priv: PrivateKey, pub: PublicKey,
challengeData: openarray[byte]): HandshakeSecrets =
let eph = ecdhRawFull(priv, pub)
var info = newSeqOfCap[byte](keyAgreementPrefix.len + 32 * 2)
for i, c in keyAgreementPrefix: info.add(byte(c))
info.add(n1.toByteArrayBE())
info.add(n2.toByteArrayBE())
var secrets: HandshakeSecrets
static: assert(sizeof(secrets) == aesKeySize * 2)
var res = cast[ptr UncheckedArray[byte]](addr secrets)
hkdf(sha256, eph.data, challengeData, info,
toOpenArray(res, 0, sizeof(secrets) - 1))
secrets
proc encryptGCM*(key, nonce, pt, authData: openarray[byte]): seq[byte] =
var ectx: GCM[aes128]
ectx.init(key, nonce, authData)
result = newSeq[byte](pt.len + gcmTagSize)
ectx.encrypt(pt, result)
ectx.getTag(result.toOpenArray(pt.len, result.high))
ectx.clear()
proc decryptGCM*(key: AesKey, nonce, ct, authData: openarray[byte]):
Option[seq[byte]] =
if ct.len <= gcmTagSize:
debug "cipher is missing tag", len = ct.len
return
var dctx: GCM[aes128]
dctx.init(key, nonce, authData)
var res = newSeq[byte](ct.len - gcmTagSize)
var tag: array[gcmTagSize, byte]
dctx.decrypt(ct.toOpenArray(0, ct.high - gcmTagSize), res)
dctx.getTag(tag)
dctx.clear()
if tag != ct.toOpenArray(ct.len - gcmTagSize, ct.high):
return
return some(res)
proc encryptHeader*(id: NodeId, iv, header: openarray[byte]): seq[byte] =
var ectx: CTR[aes128]
ectx.init(id.toByteArrayBE().toOpenArray(0, 15), iv)
result = newSeq[byte](header.len)
ectx.encrypt(header, result)
ectx.clear()
proc hasHandshake*(c: Codec, key: HandShakeKey): bool =
c.handshakes.hasKey(key)
proc encodeStaticHeader*(flag: Flag, nonce: AESGCMNonce, authSize: int):
seq[byte] =
result.add(protocolId)
result.add(version.toBytesBE())
result.add(byte(flag))
result.add(nonce)
# TODO: assert on authSize of > 2^16?
result.add((uint16(authSize)).toBytesBE())
proc encodeMessagePacket*(rng: var BrHmacDrbgContext, c: var Codec,
toId: NodeID, toAddr: Address, message: openarray[byte]):
(seq[byte], AESGCMNonce) =
var nonce: AESGCMNonce
brHmacDrbgGenerate(rng, nonce) # Random AESGCM nonce
var iv: array[ivSize, byte]
brHmacDrbgGenerate(rng, iv) # Random IV
# static-header
let authdata = c.localNode.id.toByteArrayBE()
let staticHeader = encodeStaticHeader(Flag.OrdinaryMessage, nonce,
authdata.len())
# header = static-header || authdata
var header: seq[byte]
header.add(staticHeader)
header.add(authdata)
# message
var messageEncrypted: seq[byte]
var initiatorKey, recipientKey: AesKey
if c.sessions.load(toId, toAddr, recipientKey, initiatorKey):
messageEncrypted = encryptGCM(initiatorKey, nonce, message, @iv & header)
else:
# We might not have the node's keys if the handshake hasn't been performed
# yet. That's fine, we send a random-packet and we will be responded with
# a WHOAREYOU packet.
# Select 20 bytes of random data, which is the smallest possible ping
# message. 16 bytes for the gcm tag and 4 bytes for ping with requestId of
# 1 byte (e.g "01c20101"). Could increase to 27 for 8 bytes requestId in
# case this must not look like a random packet.
var randomData: array[gcmTagSize + 4, byte]
brHmacDrbgGenerate(rng, randomData)
messageEncrypted.add(randomData)
let maskedHeader = encryptHeader(toId, iv, header)
var packet: seq[byte]
packet.add(iv)
packet.add(maskedHeader)
packet.add(messageEncrypted)
return (packet, nonce)
proc encodeWhoareyouPacket*(rng: var BrHmacDrbgContext, c: var Codec,
toId: NodeID, toAddr: Address, requestNonce: AESGCMNonce, recordSeq: uint64,
pubkey: Option[PublicKey]): seq[byte] =
var idNonce: IdNonce
brHmacDrbgGenerate(rng, idNonce)
# authdata
var authdata: seq[byte]
authdata.add(idNonce)
authdata.add(recordSeq.tobytesBE)
# static-header
let staticHeader = encodeStaticHeader(Flag.Whoareyou, requestNonce,
authdata.len())
# header = static-header || authdata
var header: seq[byte]
header.add(staticHeader)
header.add(authdata)
var iv: array[ivSize, byte]
brHmacDrbgGenerate(rng, iv) # Random IV
let maskedHeader = encryptHeader(toId, iv, header)
var packet: seq[byte]
packet.add(iv)
packet.add(maskedHeader)
let
whoareyouData = WhoareyouData(
requestNonce: requestNonce,
idNonce: idNonce,
recordSeq: recordSeq,
challengeData: @iv & header)
challenge = Challenge(whoareyouData: whoareyouData, pubkey: pubkey)
key = HandShakeKey(nodeId: toId, address: $toAddr)
c.handshakes[key] = challenge
return packet
proc encodeHandshakePacket*(rng: var BrHmacDrbgContext, c: var Codec,
toId: NodeID, toAddr: Address, message: openarray[byte],
whoareyouData: WhoareyouData, pubkey: PublicKey): seq[byte] =
var header: seq[byte]
var nonce: AESGCMNonce
brHmacDrbgGenerate(rng, nonce)
var iv: array[ivSize, byte]
brHmacDrbgGenerate(rng, iv) # Random IV
var authdata: seq[byte]
var authdataHead: seq[byte]
authdataHead.add(c.localNode.id.toByteArrayBE())
authdataHead.add(64'u8) # sig-size: 64
authdataHead.add(33'u8) # eph-key-size: 33
authdata.add(authdataHead)
let ephKeys = KeyPair.random(rng)
let signature = createIdSignature(c.privKey, whoareyouData.challengeData,
ephKeys.pubkey.toRawCompressed(), toId)
authdata.add(signature.toRaw())
# compressed pub key format (33 bytes)
authdata.add(ephKeys.pubkey.toRawCompressed())
# Add ENR of sequence number is newer
if whoareyouData.recordSeq < c.localNode.record.seqNum:
authdata.add(encode(c.localNode.record))
let secrets = deriveKeys(c.localNode.id, toId, ephKeys.seckey, pubkey,
whoareyouData.challengeData)
# Header
let staticHeader = encodeStaticHeader(Flag.HandshakeMessage, nonce,
authdata.len())
header.add(staticHeader)
header.add(authdata)
c.sessions.store(toId, toAddr, secrets.recipientKey, secrets.initiatorKey)
let messageEncrypted = encryptGCM(secrets.initiatorKey, nonce, message,
@iv & header)
let maskedHeader = encryptHeader(toId, iv, header)
var packet: seq[byte]
packet.add(iv)
packet.add(maskedHeader)
packet.add(messageEncrypted)
return packet
proc decodeHeader*(id: NodeId, iv, maskedHeader: openarray[byte]):
DecodeResult[(StaticHeader, seq[byte])] =
# No need to check staticHeader size as that is included in minimum packet
# size check in decodePacket
var ectx: CTR[aes128]
ectx.init(id.toByteArrayBE().toOpenArray(0, ivSize - 1), iv)
# Decrypt static-header part of the header
var staticHeader = newSeq[byte](staticHeaderSize)
ectx.decrypt(maskedHeader.toOpenArray(0, staticHeaderSize - 1), staticHeader)
# Check fields of the static-header
if staticHeader.toOpenArray(0, protocolId.len - 1) != protocolId:
return err("Invalid protocol id")
if uint16.fromBytesBE(staticHeader.toOpenArray(6, 7)) != version:
return err("Invalid protocol version")
var flag: Flag
if not checkedEnumAssign(flag, staticHeader[8]):
return err("Invalid packet flag")
var nonce: AESGCMNonce
copyMem(addr nonce[0], unsafeAddr staticHeader[9], gcmNonceSize)
let authdataSize = uint16.fromBytesBE(staticHeader.toOpenArray(21,
staticHeader.high))
# Input should have minimum size of staticHeader + provided authdata size
# Can be larger as there can come a message after.
if maskedHeader.len < staticHeaderSize + int(authdataSize):
return err("Authdata is smaller than authdata-size indicates")
var authdata = newSeq[byte](int(authdataSize))
ectx.decrypt(maskedHeader.toOpenArray(staticHeaderSize,
staticHeaderSize + int(authdataSize) - 1), authdata)
ectx.clear()
ok((StaticHeader(authdataSize: authdataSize, flag: flag, nonce: nonce),
staticHeader & authdata))
proc decodeMessage*(body: openarray[byte]): DecodeResult[Message] =
## Decodes to the specific `Message` type.
if body.len < 1:
return err("No message data")
var kind: MessageKind
if not checkedEnumAssign(kind, body[0]):
return err("Invalid message type")
var message = Message(kind: kind)
var rlp = rlpFromBytes(body.toOpenArray(1, body.high))
if rlp.enterList:
try:
message.reqId = rlp.read(RequestId)
except RlpError, ValueError:
return err("Invalid request-id")
proc decode[T](rlp: var Rlp, v: var T)
{.inline, nimcall, raises:[RlpError, ValueError, Defect].} =
for k, v in v.fieldPairs:
v = rlp.read(typeof(v))
try:
case kind
of unused: return err("Invalid message type")
of ping: rlp.decode(message.ping)
of pong: rlp.decode(message.pong)
of findNode: rlp.decode(message.findNode)
of nodes: rlp.decode(message.nodes)
of talkreq: rlp.decode(message.talkreq)
of talkresp: rlp.decode(message.talkresp)
of regtopic, ticket, regconfirmation, topicquery:
# We just pass the empty type of this message without attempting to
# decode, so that the protocol knows what was received.
# But we ignore the message as per specification as "the content and
# semantics of this message are not final".
discard
except RlpError, ValueError:
return err("Invalid message encoding")
ok(message)
else:
err("Invalid message encoding: no rlp list")
proc decodeMessagePacket(c: var Codec, fromAddr: Address, nonce: AESGCMNonce,
iv, header, ct: openArray[byte]): DecodeResult[Packet] =
# We now know the exact size that the header should be
if header.len != staticHeaderSize + sizeof(NodeId):
return err("Invalid header length for ordinary message packet")
# Need to have at minimum the gcm tag size for the message.
if ct.len < gcmTagSize:
return err("Invalid message length for ordinary message packet")
let srcId = NodeId.fromBytesBE(header.toOpenArray(staticHeaderSize,
header.high))
var initiatorKey, recipientKey: AesKey
if not c.sessions.load(srcId, fromAddr, recipientKey, initiatorKey):
# Don't consider this an error, simply haven't done a handshake yet or
# the session got removed.
trace "Decrypting failed (no keys)"
return ok(Packet(flag: Flag.OrdinaryMessage, requestNonce: nonce,
srcId: srcId))
let pt = decryptGCM(recipientKey, nonce, ct, @iv & @header)
if pt.isNone():
# Don't consider this an error, the session got probably removed at the
# peer's side and a random message is send.
trace "Decrypting failed (invalid keys)"
c.sessions.del(srcId, fromAddr)
return ok(Packet(flag: Flag.OrdinaryMessage, requestNonce: nonce,
srcId: srcId))
let message = ? decodeMessage(pt.get())
return ok(Packet(flag: Flag.OrdinaryMessage,
messageOpt: some(message), requestNonce: nonce, srcId: srcId))
proc decodeWhoareyouPacket(c: var Codec, nonce: AESGCMNonce,
iv, header: openArray[byte]): DecodeResult[Packet] =
# TODO improve this
let authdata = header[staticHeaderSize..header.high()]
# We now know the exact size that the authdata should be
if authdata.len != idNonceSize + sizeof(uint64):
return err("Invalid header length for whoareyou packet")
var idNonce: IdNonce
copyMem(addr idNonce[0], unsafeAddr authdata[0], idNonceSize)
let whoareyou = WhoareyouData(requestNonce: nonce, idNonce: idNonce,
recordSeq: uint64.fromBytesBE(
authdata.toOpenArray(idNonceSize, authdata.high)),
challengeData: @iv & @header)
return ok(Packet(flag: Flag.Whoareyou, whoareyou: whoareyou))
proc decodeHandshakePacket(c: var Codec, fromAddr: Address, nonce: AESGCMNonce,
iv, header, ct: openArray[byte]): DecodeResult[Packet] =
# Checking if there is enough data to decode authdata-head
if header.len <= staticHeaderSize + authdataHeadSize:
return err("Invalid header for handshake message packet: no authdata-head")
# Need to have at minimum the gcm tag size for the message.
# TODO: And actually, as we should be able to decrypt it, it should also be
# a valid message and thus we could increase here to the size of the smallest
# message possible.
if ct.len < gcmTagSize:
return err("Invalid message length for ordinary message packet")
let
authdata = header[staticHeaderSize..header.high()]
srcId = NodeId.fromBytesBE(authdata.toOpenArray(0, 31))
sigSize = uint8(authdata[32])
ephKeySize = uint8(authdata[33])
# If smaller, as it can be equal and bigger (in case it holds an enr)
if header.len < staticHeaderSize + authdataHeadSize + int(sigSize) + int(ephKeySize):
return err("Invalid header for handshake message packet")
let key = HandShakeKey(nodeId: srcId, address: $fromAddr)
var challenge: Challenge
if not c.handshakes.pop(key, challenge):
return err("No challenge found: timed out or unsolicited packet")
# This should be the compressed public key. But as we use the provided
# ephKeySize, it should also work with full sized key. However, the idNonce
# signature verification will fail.
let
ephKeyPos = authdataHeadSize + int(sigSize)
ephKeyRaw = authdata[ephKeyPos..<ephKeyPos + int(ephKeySize)]
ephKey = ? PublicKey.fromRaw(ephKeyRaw)
var record: Option[enr.Record]
let recordPos = ephKeyPos + int(ephKeySize)
if authdata.len() > recordPos:
# There is possibly an ENR still
try:
# Signature check of record happens in decode.
record = some(rlp.decode(authdata.toOpenArray(recordPos, authdata.high),
enr.Record))
except RlpError, ValueError:
return err("Invalid encoded ENR")
var pubKey: PublicKey
var newNode: Option[Node]
# TODO: Shall we return Node or Record? Record makes more sense, but we do
# need the pubkey and the nodeid
if record.isSome():
# Node returned might not have an address or not a valid address.
let node = ? newNode(record.get())
if node.id != srcId:
return err("Invalid node id: does not match node id of ENR")
# Note: Not checking if the record seqNum is higher than the one we might
# have stored as it comes from this node directly.
pubKey = node.pubKey
newNode = some(node)
else:
# TODO: Hmm, should we still verify node id of the ENR of this node?
if challenge.pubkey.isSome():
pubKey = challenge.pubkey.get()
else:
# We should have received a Record in this case.
return err("Missing ENR in handshake packet")
# Verify the id-signature
let sig = ? SignatureNR.fromRaw(
authdata.toOpenArray(authdataHeadSize, authdataHeadSize + int(sigSize) - 1))
if not verifyIdSignature(sig, challenge.whoareyouData.challengeData,
ephKeyRaw, c.localNode.id, pubkey):
return err("Invalid id-signature")
# Do the key derivation step only after id-signature is verified as this is
# costly.
var secrets = deriveKeys(srcId, c.localNode.id, c.privKey,
ephKey, challenge.whoareyouData.challengeData)
swap(secrets.recipientKey, secrets.initiatorKey)
let pt = decryptGCM(secrets.recipientKey, nonce, ct, @iv & @header)
if pt.isNone():
c.sessions.del(srcId, fromAddr)
# Differently from an ordinary message, this is seen as an error as the
# secrets just got negotiated in the handshake and thus decryption should
# always work. We do not send a new Whoareyou on these as it probably means
# there is a compatiblity issue and we might loop forever in failed
# handshakes with this peer.
return err("Decryption of message failed in handshake packet")
let message = ? decodeMessage(pt.get())
# Only store the session secrets in case decryption was successful and also
# in case the message can get decoded.
c.sessions.store(srcId, fromAddr, secrets.recipientKey, secrets.initiatorKey)
return ok(Packet(flag: Flag.HandshakeMessage, message: message,
srcIdHs: srcId, node: newNode))
proc decodePacket*(c: var Codec, fromAddr: Address, input: openArray[byte]):
DecodeResult[Packet] =
## Decode a packet. This can be a regular packet or a packet in response to a
## WHOAREYOU packet. In case of the latter a `newNode` might be provided.
# Smallest packet is Whoareyou packet so that is the minimum size
if input.len() < whoareyouSize:
return err("Packet size too short")
# TODO: Just pass in the full input? Makes more sense perhaps.
let (staticHeader, header) = ? decodeHeader(c.localNode.id,
input.toOpenArray(0, ivSize - 1), # IV
# Don't know the size yet of the full header, so we pass all.
input.toOpenArray(ivSize, input.high))
case staticHeader.flag
of OrdinaryMessage:
return decodeMessagePacket(c, fromAddr, staticHeader.nonce,
input.toOpenArray(0, ivSize - 1), header,
input.toOpenArray(ivSize + header.len, input.high))
of Whoareyou:
# Header size got checked in decode header
return decodeWhoareyouPacket(c, staticHeader.nonce,
input.toOpenArray(0, ivSize - 1), header)
of HandshakeMessage:
return decodeHandshakePacket(c, fromAddr, staticHeader.nonce,
input.toOpenArray(0, ivSize - 1), header,
input.toOpenArray(ivSize + header.len, input.high))
proc init*(T: type RequestId, rng: var BrHmacDrbgContext): T =
var reqId = RequestId(id: newSeq[byte](8)) # RequestId must be <= 8 bytes
brHmacDrbgGenerate(rng, reqId.id)
reqId
proc numFields(T: typedesc): int =
for k, v in fieldPairs(default(T)): inc result
proc encodeMessage*[T: SomeMessage](p: T, reqId: RequestId): seq[byte] =
result = newSeqOfCap[byte](64)
result.add(messageKind(T).ord)
const sz = numFields(T)
var writer = initRlpList(sz + 1)
writer.append(reqId)
for k, v in fieldPairs(p):
writer.append(v)
result.add(writer.finish())

842
eth/p2p/discoveryv5/protocol.nim
View File

@ -1,9 +1,835 @@
### This is all just temporary to support both versions
const UseDiscv51* {.booldefine.} = false
# nim-eth - Node Discovery Protocol v5
# Copyright (c) 2020 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, (LICENSE-APACHEv2)
# * MIT license (LICENSE-MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
when UseDiscv51:
import protocolv1
export protocolv1
else:
import protocolv0
export protocolv0
## Node Discovery Protocol v5
##
## Node discovery protocol implementation as per specification:
## https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md
##
## This node discovery protocol implementation uses the same underlying
## implementation of routing table as is also used for the discovery v4
## implementation, which is the same or similar as the one described in the
## original Kademlia paper:
## https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf
##
## This might not be the most optimal implementation for the node discovery
## protocol v5. Why?
##
## The Kademlia paper describes an implementation that starts off from one
## k-bucket, and keeps splitting the bucket as more nodes are discovered and
## added. The bucket splits only on the part of the binary tree where our own
## node its id belongs too (same prefix). Resulting eventually in a k-bucket per
## logarithmic distance (log base2 distance). Well, not really, as nodes with
## ids in the closer distance ranges will never be found. And because of this an
## optimisation is done where buckets will also split sometimes even if the
## nodes own id does not have the same prefix (this is to avoid creating highly
## unbalanced branches which would require longer lookups).
##
## Now, some implementations take a more simplified approach. They just create
## directly a bucket for each possible logarithmic distance (e.g. here 1->256).
## Some implementations also don't create buckets with logarithmic distance
## lower than a certain value (e.g. only 1/15th of the highest buckets),
## because the closer to the node (the lower the distance), the less chance
## there is to still find nodes.
##
## The discovery protocol v4 its `FindNode` call will request the k closest
## nodes. As does original Kademlia. This effectively puts the work at the node
## that gets the request. This node will have to check its buckets and gather
## the closest. Some implementations go over all the nodes in all the buckets
## for this (e.g. go-ethereum discovery v4). However, in our bucket splitting
## approach, this search is improved.
##
## In the discovery protocol v5 the `FindNode` call is changed and now the
## logarithmic distance is passed as parameter instead of the NodeId. And only
## nodes that match that logarithmic distance are allowed to be returned.
## This change was made to not put the trust at the requested node for selecting
## the closest nodes. To counter a possible (mistaken) difference in
## implementation, but more importantly for security reasons. See also:
## https://github.com/ethereum/devp2p/blob/master/discv5/discv5-rationale.md#115-guard-against-kademlia-implementation-flaws
##
## The result is that in an implementation which just stores buckets per
## logarithmic distance, it simply needs to return the right bucket. In our
## split-bucket implementation, this cannot be done as such and thus the closest
## neighbours search is still done. And to do this, a reverse calculation of an
## id at given logarithmic distance is needed (which is why there is the
## `idAtDistance` proc). Next, nodes with invalid distances need to be filtered
## out to be compliant to the specification. This can most likely get further
## optimised, but it sounds likely better to switch away from the split-bucket
## approach. I believe that the main benefit it has is improved lookups
## (due to no unbalanced branches), and it looks like this will be negated by
## limiting the returned nodes to only the ones of the requested logarithmic
## distance for the `FindNode` call.
## This `FindNode` change in discovery v5 will also have an effect on the
## efficiency of the network. Work will be moved from the receiver of
## `FindNodes` to the requester. But this also means more network traffic,
## as less nodes will potentially be passed around per `FindNode` call, and thus
## more requests will be needed for a lookup (adding bandwidth and latency).
## This might be a concern for mobile devices.
import
std/[tables, sets, options, math, sequtils],
stew/shims/net as stewNet, json_serialization/std/net,
stew/[byteutils, endians2], chronicles, chronos, stint, bearssl,
eth/[rlp, keys, async_utils],
types, encoding, node, routing_table, enr, random2, sessions
import nimcrypto except toHex
export options
{.push raises: [Defect].}
logScope:
topics = "discv5"
const
alpha = 3 ## Kademlia concurrency factor
lookupRequestLimit = 3
findNodeResultLimit = 15 # applies in FINDNODE handler
maxNodesPerMessage = 3
lookupInterval = 60.seconds ## Interval of launching a random lookup to
## populate the routing table. go-ethereum seems to do 3 runs every 30
## minutes. Trinity starts one every minute.
revalidateMax = 10000 ## Revalidation of a peer is done between 0 and this
## value in milliseconds
handshakeTimeout* = 2.seconds ## timeout for the reply on the
## whoareyou message
responseTimeout* = 4.seconds ## timeout for the response of a request-response
## call
type
Protocol* = ref object
transp: DatagramTransport
localNode*: Node
privateKey: PrivateKey
bindAddress: Address ## UDP binding address
pendingRequests: Table[AESGCMNonce, PendingRequest]
routingTable: RoutingTable
codec*: Codec
awaitedMessages: Table[(NodeId, RequestId), Future[Option[Message]]]
lookupLoop: Future[void]
revalidateLoop: Future[void]
bootstrapRecords*: seq[Record]
rng*: ref BrHmacDrbgContext
PendingRequest = object
node: Node
message: seq[byte]
DiscResult*[T] = Result[T, cstring]
proc addNode*(d: Protocol, node: Node): bool =
## Add `Node` to discovery routing table.
##
## Returns false only if `Node` is not eligable for adding (no Address).
if node.address.isSome():
# Only add nodes with an address to the routing table
discard d.routingTable.addNode(node)
return true
proc addNode*(d: Protocol, r: Record): bool =
## Add `Node` from a `Record` to discovery routing table.
##
## Returns false only if no valid `Node` can be created from the `Record` or
## on the conditions of `addNode` from a `Node`.
let node = newNode(r)
if node.isOk():
return d.addNode(node[])
proc addNode*(d: Protocol, enr: EnrUri): bool =
## Add `Node` from a ENR URI to discovery routing table.
##
## Returns false if no valid ENR URI, or on the conditions of `addNode` from
## an `Record`.
var r: Record
let res = r.fromUri(enr)
if res:
return d.addNode(r)
proc getNode*(d: Protocol, id: NodeId): Option[Node] =
## Get the node with id from the routing table.
d.routingTable.getNode(id)
proc randomNodes*(d: Protocol, maxAmount: int): seq[Node] =
## Get a `maxAmount` of random nodes from the local routing table.
d.routingTable.randomNodes(maxAmount)
proc randomNodes*(d: Protocol, maxAmount: int,
pred: proc(x: Node): bool {.gcsafe, noSideEffect.}): seq[Node] =
## Get a `maxAmount` of random nodes from the local routing table with the
## `pred` predicate function applied as filter on the nodes selected.
d.routingTable.randomNodes(maxAmount, pred)
proc randomNodes*(d: Protocol, maxAmount: int,
enrField: (string, seq[byte])): seq[Node] =
## Get a `maxAmount` of random nodes from the local routing table. The
## the nodes selected are filtered by provided `enrField`.
d.randomNodes(maxAmount, proc(x: Node): bool = x.record.contains(enrField))
proc neighbours*(d: Protocol, id: NodeId, k: int = BUCKET_SIZE): seq[Node] =
## Return up to k neighbours (closest node ids) of the given node id.
d.routingTable.neighbours(id, k)
proc nodesDiscovered*(d: Protocol): int {.inline.} = d.routingTable.len
func privKey*(d: Protocol): lent PrivateKey =
d.privateKey
func getRecord*(d: Protocol): Record =
## Get the ENR of the local node.
d.localNode.record
proc updateRecord*(
d: Protocol, enrFields: openarray[(string, seq[byte])]): DiscResult[void] =
## Update the ENR of the local node with provided `enrFields` k:v pairs.
let fields = mapIt(enrFields, toFieldPair(it[0], it[1]))
d.localNode.record.update(d.privateKey, fields)
# TODO: Would it make sense to actively ping ("broadcast") to all the peers
# we stored a handshake with in order to get that ENR updated?
proc send(d: Protocol, a: Address, data: seq[byte]) =
let ta = initTAddress(a.ip, a.port)
try:
let f = d.transp.sendTo(ta, data)
f.callback = proc(data: pointer) {.gcsafe.} =
if f.failed:
# Could be `TransportUseClosedError` in case the transport is already
# closed, or could be `TransportOsError` in case of a socket error.
# In the latter case this would probably mostly occur if the network
# interface underneath gets disconnected or similar.
# TODO: Should this kind of error be propagated upwards? Probably, but
# it should not stop the process as that would reset the discovery
# progress in case there is even a small window of no connection.
# One case that needs this error available upwards is when revalidating
# nodes. Else the revalidation might end up clearing the routing tabl
# because of ping failures due to own network connection failure.
warn "Discovery send failed", msg = f.readError.msg
except Exception as e:
# TODO: General exception still being raised from Chronos, but in practice
# all CatchableErrors should be grabbed by the above `f.failed`.
if e of Defect:
raise (ref Defect)(e)
else: doAssert(false)
proc send(d: Protocol, n: Node, data: seq[byte]) =
doAssert(n.address.isSome())
d.send(n.address.get(), data)
proc sendNodes(d: Protocol, toId: NodeId, toAddr: Address, reqId: RequestId,
nodes: openarray[Node]) =
proc sendNodes(d: Protocol, toId: NodeId, toAddr: Address,
message: NodesMessage, reqId: RequestId) {.nimcall.} =
let (data, _) = encodeMessagePacket(d.rng[], d.codec, toId, toAddr,
encodeMessage(message, reqId))
trace "Respond message packet", dstId = toId, address = toAddr,
kind = MessageKind.nodes
d.send(toAddr, data)
if nodes.len == 0:
# In case of 0 nodes, a reply is still needed
d.sendNodes(toId, toAddr, NodesMessage(total: 1, enrs: @[]), reqId)
return
var message: NodesMessage
# TODO: Do the total calculation based on the max UDP packet size we want to
# send and the ENR size of all (max 16) nodes.
# Which UDP packet size to take? 1280? 576?
message.total = ceil(nodes.len / maxNodesPerMessage).uint32
for i in 0 ..< nodes.len:
message.enrs.add(nodes[i].record)
if message.enrs.len == maxNodesPerMessage:
d.sendNodes(toId, toAddr, message, reqId)
message.enrs.setLen(0)
if message.enrs.len != 0:
d.sendNodes(toId, toAddr, message, reqId)
proc handlePing(d: Protocol, fromId: NodeId, fromAddr: Address,
ping: PingMessage, reqId: RequestId) =
let a = fromAddr
var pong: PongMessage
pong.enrSeq = d.localNode.record.seqNum
pong.ip = case a.ip.family
of IpAddressFamily.IPv4: @(a.ip.address_v4)
of IpAddressFamily.IPv6: @(a.ip.address_v6)
pong.port = a.port.uint16
let (data, _) = encodeMessagePacket(d.rng[], d.codec, fromId, fromAddr,
encodeMessage(pong, reqId))
trace "Respond message packet", dstId = fromId, address = fromAddr,
kind = MessageKind.pong
d.send(fromAddr, data)
proc handleFindNode(d: Protocol, fromId: NodeId, fromAddr: Address,
fn: FindNodeMessage, reqId: RequestId) =
if fn.distances.len == 0:
d.sendNodes(fromId, fromAddr, reqId, [])
elif fn.distances.contains(0):
# A request for our own record.
# It would be a weird request if there are more distances next to 0
# requested, so in this case lets just pass only our own. TODO: OK?
d.sendNodes(fromId, fromAddr, reqId, [d.localNode])
else:
# TODO: Still deduplicate also?
if fn.distances.all(proc (x: uint32): bool = return x <= 256):
d.sendNodes(fromId, fromAddr, reqId,
d.routingTable.neighboursAtDistances(fn.distances, seenOnly = true))
else:
# At least one invalid distance, but the polite node we are, still respond
# with empty nodes.
d.sendNodes(fromId, fromAddr, reqId, [])
proc handleTalkReq(d: Protocol, fromId: NodeId, fromAddr: Address,
talkreq: TalkReqMessage, reqId: RequestId) =
# No support for any protocol yet so an empty response is send as per
# specification.
let talkresp = TalkRespMessage(response: @[])
let (data, _) = encodeMessagePacket(d.rng[], d.codec, fromId, fromAddr,
encodeMessage(talkresp, reqId))
trace "Respond message packet", dstId = fromId, address = fromAddr,
kind = MessageKind.talkresp
d.send(fromAddr, data)
proc handleMessage(d: Protocol, srcId: NodeId, fromAddr: Address,
message: Message) {.raises:[Exception].} =
case message.kind
of ping:
d.handlePing(srcId, fromAddr, message.ping, message.reqId)
of findNode:
d.handleFindNode(srcId, fromAddr, message.findNode, message.reqId)
of talkreq:
d.handleTalkReq(srcId, fromAddr, message.talkreq, message.reqId)
of regtopic, topicquery:
trace "Received unimplemented message kind", kind = message.kind,
origin = fromAddr
else:
var waiter: Future[Option[Message]]
if d.awaitedMessages.take((srcId, message.reqId), waiter):
waiter.complete(some(message)) # TODO: raises: [Exception]
else:
trace "Timed out or unrequested message", kind = message.kind,
origin = fromAddr
proc sendWhoareyou(d: Protocol, toId: NodeId, a: Address,
requestNonce: AESGCMNonce, node: Option[Node]) {.raises: [Exception].} =
let key = HandShakeKey(nodeId: toId, address: $a)
if not d.codec.hasHandshake(key):
let
recordSeq = if node.isSome(): node.get().record.seqNum
else: 0
pubkey = if node.isSome(): some(node.get().pubkey)
else: none(PublicKey)
let data = encodeWhoareyouPacket(d.rng[], d.codec, toId, a, requestNonce,
recordSeq, pubkey)
sleepAsync(handshakeTimeout).addCallback() do(data: pointer):
# TODO: should we still provide cancellation in case handshake completes
# correctly?
d.codec.handshakes.del(key)
trace "Send whoareyou", dstId = toId, address = a
d.send(a, data)
else:
debug "Node with this id already has ongoing handshake, ignoring packet"
proc receive*(d: Protocol, a: Address, packet: openArray[byte]) {.gcsafe,
raises: [
Defect,
# This just comes now from a future.complete() and `sendWhoareyou` which
# has it because of `sleepAsync` with `addCallback`, but practically, no
# CatchableError should be raised here, we just can't enforce it for now.
Exception
].} =
let decoded = d.codec.decodePacket(a, packet)
if decoded.isOk:
let packet = decoded[]
case packet.flag
of OrdinaryMessage:
if packet.messageOpt.isSome():
let message = packet.messageOpt.get()
trace "Received message packet", srcId = packet.srcId, address = a,
kind = message.kind
d.handleMessage(packet.srcId, a, message)
else:
trace "Not decryptable message packet received",
srcId = packet.srcId, address = a
d.sendWhoareyou(packet.srcId, a, packet.requestNonce,
d.getNode(packet.srcId))
of Flag.Whoareyou:
trace "Received whoareyou packet", address = a
var pr: PendingRequest
if d.pendingRequests.take(packet.whoareyou.requestNonce, pr):
let toNode = pr.node
# This is a node we previously contacted and thus must have an address.
doAssert(toNode.address.isSome())
let address = toNode.address.get()
let data = encodeHandshakePacket(d.rng[], d.codec, toNode.id,
address, pr.message, packet.whoareyou, toNode.pubkey)
trace "Send handshake message packet", dstId = toNode.id, address
d.send(toNode, data)
else:
debug "Timed out or unrequested whoareyou packet", address = a
of HandshakeMessage:
trace "Received handshake message packet", srcId = packet.srcIdHs,
address = a, kind = packet.message.kind
d.handleMessage(packet.srcIdHs, a, packet.message)
# For a handshake message it is possible that we received an newer ENR.
# In that case we can add/update it to the routing table.
if packet.node.isSome():
let node = packet.node.get()
# Not filling table with nodes without correct IP in the ENR
# TODO: Should we care about this???
if node.address.isSome() and a == node.address.get():
debug "Adding new node to routing table", node
discard d.addNode(node)
else:
debug "Packet decoding error", error = decoded.error, address = a
# TODO: Not sure why but need to pop the raises here as it is apparently not
# enough to put it in the raises pragma of `processClient` and other async procs.
{.pop.}
# Next, below there is no more effort done in catching the general `Exception`
# as async procs always require `Exception` in the raises pragma, see also:
# https://github.com/status-im/nim-chronos/issues/98
# So I don't bother for now and just add them in the raises pragma until this
# gets fixed. It does not mean that we expect these calls to be raising
# CatchableErrors, in fact, we really don't, but hey, they might, considering we
# can't enforce it.
proc processClient(transp: DatagramTransport, raddr: TransportAddress):
Future[void] {.async, gcsafe, raises: [Exception, Defect].} =
let proto = getUserData[Protocol](transp)
# TODO: should we use `peekMessage()` to avoid allocation?
# TODO: This can still raise general `Exception` while it probably should
# only give TransportOsError.
let buf = try: transp.getMessage()
except TransportOsError as e:
# This is likely to be local network connection issues.
warn "Transport getMessage", exception = e.name, msg = e.msg
return
except Exception as e:
if e of Defect:
raise (ref Defect)(e)
else: doAssert(false)
return # Make compiler happy
let ip = try: raddr.address()
except ValueError as e:
error "Not a valid IpAddress", exception = e.name, msg = e.msg
return
let a = Address(ip: ValidIpAddress.init(ip), port: raddr.port)
try:
proto.receive(a, buf)
except Exception as e:
if e of Defect:
raise (ref Defect)(e)
else: doAssert(false)
proc validIp(sender, address: IpAddress): bool {.raises: [Defect].} =
let
s = initTAddress(sender, Port(0))
a = initTAddress(address, Port(0))
if a.isAnyLocal():
return false
if a.isMulticast():
return false
if a.isLoopback() and not s.isLoopback():
return false
if a.isSiteLocal() and not s.isSiteLocal():
return false
# TODO: Also check for special reserved ip addresses:
# https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
# https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
return true
proc replaceNode(d: Protocol, n: Node) =
if n.record notin d.bootstrapRecords:
d.routingTable.replaceNode(n)
else:
# For now we never remove bootstrap nodes. It might make sense to actually
# do so and to retry them only in case we drop to a really low amount of
# peers in the routing table.
debug "Message request to bootstrap node failed", enr = toURI(n.record)
# TODO: This could be improved to do the clean-up immediatily in case a non
# whoareyou response does arrive, but we would need to store the AuthTag
# somewhere
proc registerRequest(d: Protocol, n: Node, message: seq[byte],
nonce: AESGCMNonce) {.raises: [Exception, Defect].} =
let request = PendingRequest(node: n, message: message)
if not d.pendingRequests.hasKeyOrPut(nonce, request):
# TODO: raises: [Exception]
sleepAsync(responseTimeout).addCallback() do(data: pointer):
d.pendingRequests.del(nonce)
proc waitMessage(d: Protocol, fromNode: Node, reqId: RequestId):
Future[Option[Message]] {.raises: [Exception, Defect].} =
result = newFuture[Option[Message]]("waitMessage")
let res = result
let key = (fromNode.id, reqId)
# TODO: raises: [Exception]
sleepAsync(responseTimeout).addCallback() do(data: pointer):
d.awaitedMessages.del(key)
if not res.finished:
res.complete(none(Message)) # TODO: raises: [Exception]
d.awaitedMessages[key] = result
proc verifyNodesRecords*(enrs: openarray[Record], fromNode: Node,
distances: varargs[uint32]): seq[Node] {.raises: [Defect].} =
## Verify and convert ENRs to a sequence of nodes. Only ENRs that pass
## verification will be added. ENRs are verified for duplicates, invalid
## addresses and invalid distances.
# TODO:
# - Should we fail and ignore values on first invalid Node?
# - Should we limit the amount of nodes? The discovery v5 specification holds
# no limit on the amount that can be returned.
var seen: HashSet[Node]
for r in enrs:
let node = newNode(r)
if node.isOk():
let n = node.get()
# Check for duplicates in the nodes reply. Duplicates are checked based
# on node id.
if n in seen:
trace "Nodes reply contained records with duplicate node ids",
record = n.record.toURI, id = n.id, sender = fromNode.record.toURI
continue
# Check if the node has an address and if the address is public or from
# the same local network or lo network as the sender. The latter allows
# for local testing.
if not n.address.isSome() or not
validIp(fromNode.address.get().ip, n.address.get().ip):
trace "Nodes reply contained record with invalid ip-address",
record = n.record.toURI, node = n, sender = fromNode.record.toURI
continue
# Check if returned node has one of the requested distances.
if not distances.contains(logDist(n.id, fromNode.id)):
warn "Nodes reply contained record with incorrect distance",
record = n.record.toURI, sender = fromNode.record.toURI
continue
# No check on UDP port and thus any port is allowed, also the so called
# "well-known" ports.
seen.incl(n)
result.add(n)
proc waitNodes(d: Protocol, fromNode: Node, reqId: RequestId):
Future[DiscResult[seq[Record]]] {.async, raises: [Exception, Defect].} =
## Wait for one or more nodes replies.
##
## The first reply will hold the total number of replies expected, and based
## on that, more replies will be awaited.
## If one reply is lost here (timed out), others are ignored too.
## Same counts for out of order receival.
var op = await d.waitMessage(fromNode, reqId)
if op.isSome and op.get.kind == nodes:
var res = op.get.nodes.enrs
let total = op.get.nodes.total
for i in 1 ..< total:
op = await d.waitMessage(fromNode, reqId)
if op.isSome and op.get.kind == nodes:
res.add(op.get.nodes.enrs)
else:
# No error on this as we received some nodes.
break
return ok(res)
else:
return err("Nodes message not received in time")
proc sendMessage*[T: SomeMessage](d: Protocol, toNode: Node, m: T):
RequestId {.raises: [Exception, Defect].} =
doAssert(toNode.address.isSome())
let
address = toNode.address.get()
reqId = RequestId.init(d.rng[])
message = encodeMessage(m, reqId)
let (data, nonce) = encodeMessagePacket(d.rng[], d.codec, toNode.id,
address, message)
d.registerRequest(toNode, message, nonce)
trace "Send message packet", dstId = toNode.id, address, kind = messageKind(T)
d.send(toNode, data)
return reqId
proc ping*(d: Protocol, toNode: Node):
Future[DiscResult[PongMessage]] {.async, raises: [Exception, Defect].} =
## Send a discovery ping message.
##
## Returns the received pong message or an error.
let reqId = d.sendMessage(toNode,
PingMessage(enrSeq: d.localNode.record.seqNum))
let resp = await d.waitMessage(toNode, reqId)
if resp.isSome() and resp.get().kind == pong:
d.routingTable.setJustSeen(toNode)
return ok(resp.get().pong)
else:
d.replaceNode(toNode)
return err("Pong message not received in time")
proc findNode*(d: Protocol, toNode: Node, distances: seq[uint32]):
Future[DiscResult[seq[Node]]] {.async, raises: [Exception, Defect].} =
## Send a discovery findNode message.
##
## Returns the received nodes or an error.
## Received ENRs are already validated and converted to `Node`.
let reqId = d.sendMessage(toNode, FindNodeMessage(distances: distances))
let nodes = await d.waitNodes(toNode, reqId)
if nodes.isOk:
let res = verifyNodesRecords(nodes.get(), toNode, distances)
d.routingTable.setJustSeen(toNode)
return ok(res)
else:
d.replaceNode(toNode)
return err(nodes.error)
proc talkreq*(d: Protocol, toNode: Node, protocol, request: seq[byte]):
Future[DiscResult[TalkRespMessage]] {.async, raises: [Exception, Defect].} =
## Send a discovery talkreq message.
##
## Returns the received talkresp message or an error.
let reqId = d.sendMessage(toNode,
TalkReqMessage(protocol: protocol, request: request))
let resp = await d.waitMessage(toNode, reqId)
if resp.isSome() and resp.get().kind == talkresp:
d.routingTable.setJustSeen(toNode)
return ok(resp.get().talkresp)
else:
d.replaceNode(toNode)
return err("Talk response message not received in time")
proc lookupDistances(target, dest: NodeId): seq[uint32] {.raises: [Defect].} =
let td = logDist(target, dest)
result.add(td)
var i = 1'u32
while result.len < lookupRequestLimit:
if td + i < 256:
result.add(td + i)
if td - i > 0'u32:
result.add(td - i)
inc i
proc lookupWorker(d: Protocol, destNode: Node, target: NodeId):
Future[seq[Node]] {.async, raises: [Exception, Defect].} =
let dists = lookupDistances(target, destNode.id)
var i = 0
# TODO: We can make use of the multiple distances here now.
while i < lookupRequestLimit and result.len < findNodeResultLimit:
let r = await d.findNode(destNode, @[dists[i]])
# TODO: Handle failures better. E.g. stop on different failures than timeout
if r.isOk:
# TODO: I guess it makes sense to limit here also to `findNodeResultLimit`?
result.add(r[])
inc i
for n in result:
discard d.routingTable.addNode(n)
proc lookup*(d: Protocol, target: NodeId): Future[seq[Node]]
{.async, raises: [Exception, Defect].} =
## Perform a lookup for the given target, return the closest n nodes to the
## target. Maximum value for n is `BUCKET_SIZE`.
# TODO: Sort the returned nodes on distance
# Also use unseen nodes as a form of validation.
result = d.routingTable.neighbours(target, BUCKET_SIZE, seenOnly = false)
var asked = initHashSet[NodeId]()
asked.incl(d.localNode.id)
var seen = asked
for node in result:
seen.incl(node.id)
var pendingQueries = newSeqOfCap[Future[seq[Node]]](alpha)
while true:
var i = 0
while i < result.len and pendingQueries.len < alpha:
let n = result[i]
if not asked.containsOrIncl(n.id):
pendingQueries.add(d.lookupWorker(n, target))
inc i
trace "discv5 pending queries", total = pendingQueries.len
if pendingQueries.len == 0:
break
let idx = await oneIndex(pendingQueries)
trace "Got discv5 lookup response", idx
let nodes = pendingQueries[idx].read
pendingQueries.del(idx)
for n in nodes:
if not seen.containsOrIncl(n.id):
if result.len < BUCKET_SIZE:
result.add(n)
proc lookupRandom*(d: Protocol): Future[seq[Node]]
{.async, raises:[Exception, Defect].} =
## Perform a lookup for a random target, return the closest n nodes to the
## target. Maximum value for n is `BUCKET_SIZE`.
var id: NodeId
var buf: array[sizeof(id), byte]
brHmacDrbgGenerate(d.rng[], buf)
copyMem(addr id, addr buf[0], sizeof(id))
return await d.lookup(id)
proc resolve*(d: Protocol, id: NodeId): Future[Option[Node]]
{.async, raises: [Exception, Defect].} =
## Resolve a `Node` based on provided `NodeId`.
##
## This will first look in the own routing table. If the node is known, it
## will try to contact if for newer information. If node is not known or it
## does not reply, a lookup is done to see if it can find a (newer) record of
## the node on the network.
let node = d.getNode(id)
if node.isSome():
let request = await d.findNode(node.get(), @[0'u32])
# TODO: Handle failures better. E.g. stop on different failures than timeout
if request.isOk() and request[].len > 0:
return some(request[][0])
let discovered = await d.lookup(id)
for n in discovered:
if n.id == id:
if node.isSome() and node.get().record.seqNum >= n.record.seqNum:
return node
else:
return some(n)
return node
proc revalidateNode*(d: Protocol, n: Node)
{.async, raises: [Exception, Defect].} = # TODO: Exception
let pong = await d.ping(n)
if pong.isOK():
if pong.get().enrSeq > n.record.seqNum:
# Request new ENR
let nodes = await d.findNode(n, @[0'u32])
if nodes.isOk() and nodes[].len > 0:
discard d.addNode(nodes[][0])
proc revalidateLoop(d: Protocol) {.async, raises: [Exception, Defect].} =
# TODO: General Exception raised.
try:
while true:
await sleepAsync(milliseconds(d.rng[].rand(revalidateMax)))
let n = d.routingTable.nodeToRevalidate()
if not n.isNil:
traceAsyncErrors d.revalidateNode(n)
except CancelledError:
trace "revalidateLoop canceled"
proc lookupLoop(d: Protocol) {.async, raises: [Exception, Defect].} =
# TODO: General Exception raised.
try:
# lookup self (neighbour nodes)
let selfLookup = await d.lookup(d.localNode.id)
trace "Discovered nodes in self lookup", nodes = selfLookup
while true:
let randomLookup = await d.lookupRandom()
trace "Discovered nodes in random lookup", nodes = randomLookup
debug "Total nodes in discv5 routing table", total = d.routingTable.len()
await sleepAsync(lookupInterval)
except CancelledError:
trace "lookupLoop canceled"
proc newProtocol*(privKey: PrivateKey,
externalIp: Option[ValidIpAddress], tcpPort, udpPort: Port,
localEnrFields: openarray[(string, seq[byte])] = [],
bootstrapRecords: openarray[Record] = [],
previousRecord = none[enr.Record](),
bindIp = IPv4_any(), rng = newRng()):
Protocol {.raises: [Defect].} =
# TODO: Tried adding bindPort = udpPort as parameter but that gave
# "Error: internal error: environment misses: udpPort" in nim-beacon-chain.
# Anyhow, nim-beacon-chain would also require some changes to support port
# remapping through NAT and this API is also subject to change once we
# introduce support for ipv4 + ipv6 binding/listening.
let extraFields = mapIt(localEnrFields, toFieldPair(it[0], it[1]))
# TODO:
# - Defect as is now or return a result for enr errors?
# - In case incorrect key, allow for new enr based on new key (new node id)?
var record: Record
if previousRecord.isSome():
record = previousRecord.get()
record.update(privKey, externalIp, tcpPort, udpPort,
extraFields).expect("Record within size limits and correct key")
else:
record = enr.Record.init(1, privKey, externalIp, tcpPort, udpPort,
extraFields).expect("Record within size limits")
let node = newNode(record).expect("Properly initialized record")
# TODO Consider whether this should be a Defect
doAssert rng != nil, "RNG initialization failed"
result = Protocol(
privateKey: privKey,
localNode: node,
bindAddress: Address(ip: ValidIpAddress.init(bindIp), port: udpPort),
codec: Codec(localNode: node, privKey: privKey,
sessions: Sessions.init(256)),
bootstrapRecords: @bootstrapRecords,
rng: rng)
result.routingTable.init(node, 5, rng)
proc open*(d: Protocol) {.raises: [Exception, Defect].} =
info "Starting discovery node", node = d.localNode,
bindAddress = d.bindAddress, uri = toURI(d.localNode.record)
# TODO allow binding to specific IP / IPv6 / etc
let ta = initTAddress(d.bindAddress.ip, d.bindAddress.port)
# TODO: raises `OSError` and `IOSelectorsException`, the latter which is
# object of Exception. In Nim devel this got changed to CatchableError.
d.transp = newDatagramTransport(processClient, udata = d, local = ta)
for record in d.bootstrapRecords:
debug "Adding bootstrap node", uri = toURI(record)
discard d.addNode(record)
proc start*(d: Protocol) {.raises: [Exception, Defect].} =
d.lookupLoop = lookupLoop(d)
d.revalidateLoop = revalidateLoop(d)
proc close*(d: Protocol) {.raises: [Exception, Defect].} =
doAssert(not d.transp.closed)
debug "Closing discovery node", node = d.localNode
if not d.revalidateLoop.isNil:
d.revalidateLoop.cancel()
if not d.lookupLoop.isNil:
d.lookupLoop.cancel()
d.transp.close()
proc closeWait*(d: Protocol) {.async, raises: [Exception, Defect].} =
doAssert(not d.transp.closed)
debug "Closing discovery node", node = d.localNode
if not d.revalidateLoop.isNil:
await d.revalidateLoop.cancelAndWait()
if not d.lookupLoop.isNil:
await d.lookupLoop.cancelAndWait()
await d.transp.closeWait()

835
eth/p2p/discoveryv5/protocolv0.nim
View File

@ -1,835 +0,0 @@
# nim-eth - Node Discovery Protocol v5
# Copyright (c) 2020 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, (LICENSE-APACHEv2)
# * MIT license (LICENSE-MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
## Node Discovery Protocol v5
##
## Node discovery protocol implementation as per specification:
## https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md
##
## This node discovery protocol implementation uses the same underlying
## implementation of routing table as is also used for the discovery v4
## implementation, which is the same or similar as the one described in the
## original Kademlia paper:
## https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf
##
## This might not be the most optimal implementation for the node discovery
## protocol v5. Why?
##
## The Kademlia paper describes an implementation that starts off from one
## k-bucket, and keeps splitting the bucket as more nodes are discovered and
## added. The bucket splits only on the part of the binary tree where our own
## node its id belongs too (same prefix). Resulting eventually in a k-bucket per
## logarithmic distance (log base2 distance). Well, not really, as nodes with
## ids in the closer distance ranges will never be found. And because of this an
## optimisation is done where buckets will also split sometimes even if the
## nodes own id does not have the same prefix (this is to avoid creating highly
## unbalanced branches which would require longer lookups).
##
## Now, some implementations take a more simplified approach. They just create
## directly a bucket for each possible logarithmic distance (e.g. here 1->256).
## Some implementations also don't create buckets with logarithmic distance
## lower than a certain value (e.g. only 1/15th of the highest buckets),
## because the closer to the node (the lower the distance), the less chance
## there is to still find nodes.
##
## The discovery protocol v4 its `FindNode` call will request the k closest
## nodes. As does original Kademlia. This effectively puts the work at the node
## that gets the request. This node will have to check its buckets and gather
## the closest. Some implementations go over all the nodes in all the buckets
## for this (e.g. go-ethereum discovery v4). However, in our bucket splitting
## approach, this search is improved.
##
## In the discovery protocol v5 the `FindNode` call is changed and now the
## logarithmic distance is passed as parameter instead of the NodeId. And only
## nodes that match that logarithmic distance are allowed to be returned.
## This change was made to not put the trust at the requested node for selecting
## the closest nodes. To counter a possible (mistaken) difference in
## implementation, but more importantly for security reasons. See also:
## https://github.com/ethereum/devp2p/blob/master/discv5/discv5-rationale.md#115-guard-against-kademlia-implementation-flaws
##
## The result is that in an implementation which just stores buckets per
## logarithmic distance, it simply needs to return the right bucket. In our
## split-bucket implementation, this cannot be done as such and thus the closest
## neighbours search is still done. And to do this, a reverse calculation of an
## id at given logarithmic distance is needed (which is why there is the
## `idAtDistance` proc). Next, nodes with invalid distances need to be filtered
## out to be compliant to the specification. This can most likely get further
## optimised, but it sounds likely better to switch away from the split-bucket
## approach. I believe that the main benefit it has is improved lookups
## (due to no unbalanced branches), and it looks like this will be negated by
## limiting the returned nodes to only the ones of the requested logarithmic
## distance for the `FindNode` call.
## This `FindNode` change in discovery v5 will also have an effect on the
## efficiency of the network. Work will be moved from the receiver of
## `FindNodes` to the requester. But this also means more network traffic,
## as less nodes will potentially be passed around per `FindNode` call, and thus
## more requests will be needed for a lookup (adding bandwidth and latency).
## This might be a concern for mobile devices.
import
std/[tables, sets, options, math, sequtils],
stew/shims/net as stewNet, json_serialization/std/net,
stew/[byteutils, endians2], chronicles, chronos, stint, bearssl,
eth/[rlp, keys, async_utils],
types, encoding, node, routing_table, enr, random2, sessions
import nimcrypto except toHex
export options
{.push raises: [Defect].}
logScope:
topics = "discv5"
const
alpha = 3 ## Kademlia concurrency factor
lookupRequestLimit = 3
findNodeResultLimit = 15 # applies in FINDNODE handler
maxNodesPerMessage = 3
lookupInterval = 60.seconds ## Interval of launching a random lookup to
## populate the routing table. go-ethereum seems to do 3 runs every 30
## minutes. Trinity starts one every minute.
revalidateMax = 1000 ## Revalidation of a peer is done between 0 and this
## value in milliseconds
handshakeTimeout* = 2.seconds ## timeout for the reply on the
## whoareyou message
responseTimeout* = 4.seconds ## timeout for the response of a request-response
## call
magicSize = 32 ## size of the magic which is the start of the whoareyou
## message
type
Protocol* = ref object
transp: DatagramTransport
localNode*: Node
privateKey: PrivateKey
bindAddress: Address ## UDP binding address
whoareyouMagic: array[magicSize, byte]
idHash: array[32, byte]
pendingRequests: Table[AuthTag, PendingRequest]
routingTable: RoutingTable
codec*: Codec
awaitedMessages: Table[(NodeId, RequestId), Future[Option[Message]]]
lookupLoop: Future[void]
revalidateLoop: Future[void]
bootstrapRecords*: seq[Record]
rng*: ref BrHmacDrbgContext
PendingRequest = object
node: Node
message: seq[byte]
DiscResult*[T] = Result[T, cstring]
proc addNode*(d: Protocol, node: Node): bool =
## Add `Node` to discovery routing table.
##
## Returns false only if `Node` is not eligable for adding (no Address).
if node.address.isSome():
# Only add nodes with an address to the routing table
discard d.routingTable.addNode(node)
return true
proc addNode*(d: Protocol, r: Record): bool =
## Add `Node` from a `Record` to discovery routing table.
##
## Returns false only if no valid `Node` can be created from the `Record` or
## on the conditions of `addNode` from a `Node`.
let node = newNode(r)
if node.isOk():
return d.addNode(node[])
proc addNode*(d: Protocol, enr: EnrUri): bool =
## Add `Node` from a ENR URI to discovery routing table.
##
## Returns false if no valid ENR URI, or on the conditions of `addNode` from
## an `Record`.
var r: Record
let res = r.fromUri(enr)
if res:
return d.addNode(r)
proc getNode*(d: Protocol, id: NodeId): Option[Node] =
## Get the node with id from the routing table.
d.routingTable.getNode(id)
proc randomNodes*(d: Protocol, maxAmount: int): seq[Node] =
## Get a `maxAmount` of random nodes from the local routing table.
d.routingTable.randomNodes(maxAmount)
proc randomNodes*(d: Protocol, maxAmount: int,
pred: proc(x: Node): bool {.gcsafe, noSideEffect.}): seq[Node] =
## Get a `maxAmount` of random nodes from the local routing table with the
## `pred` predicate function applied as filter on the nodes selected.
d.routingTable.randomNodes(maxAmount, pred)
proc randomNodes*(d: Protocol, maxAmount: int,
enrField: (string, seq[byte])): seq[Node] =
## Get a `maxAmount` of random nodes from the local routing table. The
## the nodes selected are filtered by provided `enrField`.
d.randomNodes(maxAmount, proc(x: Node): bool = x.record.contains(enrField))
proc neighbours*(d: Protocol, id: NodeId, k: int = BUCKET_SIZE): seq[Node] =
## Return up to k neighbours (closest node ids) of the given node id.
d.routingTable.neighbours(id, k)
proc nodesDiscovered*(d: Protocol): int {.inline.} = d.routingTable.len
func privKey*(d: Protocol): lent PrivateKey =
d.privateKey
func getRecord*(d: Protocol): Record =
## Get the ENR of the local node.
d.localNode.record
proc updateRecord*(
d: Protocol, enrFields: openarray[(string, seq[byte])]): DiscResult[void] =
## Update the ENR of the local node with provided `enrFields` k:v pairs.
let fields = mapIt(enrFields, toFieldPair(it[0], it[1]))
d.localNode.record.update(d.privateKey, fields)
# TODO: Would it make sense to actively ping ("broadcast") to all the peers
# we stored a handshake with in order to get that ENR updated?
proc send(d: Protocol, a: Address, data: seq[byte]) =
let ta = initTAddress(a.ip, a.port)
try:
let f = d.transp.sendTo(ta, data)
f.callback = proc(data: pointer) {.gcsafe.} =
if f.failed:
# Could be `TransportUseClosedError` in case the transport is already
# closed, or could be `TransportOsError` in case of a socket error.
# In the latter case this would probably mostly occur if the network
# interface underneath gets disconnected or similar.
# TODO: Should this kind of error be propagated upwards? Probably, but
# it should not stop the process as that would reset the discovery
# progress in case there is even a small window of no connection.
# One case that needs this error available upwards is when revalidating
# nodes. Else the revalidation might end up clearing the routing tabl
# because of ping failures due to own network connection failure.
debug "Discovery send failed", msg = f.readError.msg
except Exception as e:
# TODO: General exception still being raised from Chronos, but in practice
# all CatchableErrors should be grabbed by the above `f.failed`.
if e of Defect:
raise (ref Defect)(e)
else: doAssert(false)
proc send(d: Protocol, n: Node, data: seq[byte]) =
doAssert(n.address.isSome())
d.send(n.address.get(), data)
proc `xor`[N: static[int], T](a, b: array[N, T]): array[N, T] =
for i in 0 .. a.high:
result[i] = a[i] xor b[i]
proc whoareyouMagic*(toNode: NodeId): array[magicSize, byte] =
const prefix = "WHOAREYOU"
var data: array[prefix.len + sizeof(toNode), byte]
data[0 .. sizeof(toNode) - 1] = toNode.toByteArrayBE()
for i, c in prefix: data[sizeof(toNode) + i] = byte(c)
sha256.digest(data).data
proc isWhoAreYou(d: Protocol, packet: openArray[byte]): bool =
if packet.len > d.whoareyouMagic.len:
result = d.whoareyouMagic == packet.toOpenArray(0, magicSize - 1)
proc decodeWhoAreYou(d: Protocol, packet: openArray[byte]):
Whoareyou {.raises: [RlpError].} =
result = Whoareyou()
result[] = rlp.decode(packet.toOpenArray(magicSize, packet.high), WhoareyouObj)
proc sendWhoareyou(d: Protocol, address: Address, toNode: NodeId,
authTag: AuthTag): DiscResult[void] {.raises: [Exception, Defect].} =
trace "sending who are you", to = $toNode, toAddress = $address
let n = d.getNode(toNode)
let challenge = if n.isSome():
Whoareyou(authTag: authTag, recordSeq: n.get().record.seqNum,
pubKey: some(n.get().pubkey))
else:
Whoareyou(authTag: authTag, recordSeq: 0)
brHmacDrbgGenerate(d.rng[], challenge.idNonce)
# If there is already a handshake going on for this nodeid then we drop this
# new one. Handshake will get cleaned up after `handshakeTimeout`.
# If instead overwriting the handshake would be allowed, the handshake timeout
# will need to be canceled each time.
# TODO: could also clean up handshakes in a seperate call, e.g. triggered in
# a loop.
# Use toNode + address to make it more difficult for an attacker to occupy
# the handshake of another node.
let key = HandShakeKey(nodeId: toNode, address: $address)
if not d.codec.handshakes.hasKeyOrPut(key, challenge):
# TODO: raises: [Exception], but it shouldn't.
sleepAsync(handshakeTimeout).addCallback() do(data: pointer):
# TODO: should we still provide cancellation in case handshake completes
# correctly?
d.codec.handshakes.del(key)
var data = @(whoareyouMagic(toNode))
data.add(rlp.encode(challenge[]))
d.send(address, data)
ok()
else:
err("NodeId already has ongoing handshake")
proc sendNodes(d: Protocol, toId: NodeId, toAddr: Address, reqId: RequestId,
nodes: openarray[Node]) =
proc sendNodes(d: Protocol, toId: NodeId, toAddr: Address,
message: NodesMessage, reqId: RequestId) {.nimcall.} =
let (data, _) = encodePacket(
d.rng[], d.codec, toId, toAddr,
encodeMessage(message, reqId), challenge = nil)
d.send(toAddr, data)
if nodes.len == 0:
# In case of 0 nodes, a reply is still needed
d.sendNodes(toId, toAddr, NodesMessage(total: 1, enrs: @[]), reqId)
return
var message: NodesMessage
# TODO: Do the total calculation based on the max UDP packet size we want to
# send and the ENR size of all (max 16) nodes.
# Which UDP packet size to take? 1280? 576?
message.total = ceil(nodes.len / maxNodesPerMessage).uint32
for i in 0 ..< nodes.len:
message.enrs.add(nodes[i].record)
if message.enrs.len == maxNodesPerMessage:
d.sendNodes(toId, toAddr, message, reqId)
message.enrs.setLen(0)
if message.enrs.len != 0:
d.sendNodes(toId, toAddr, message, reqId)
proc handlePing(d: Protocol, fromId: NodeId, fromAddr: Address,
ping: PingMessage, reqId: RequestId) =
let a = fromAddr
var pong: PongMessage
pong.enrSeq = d.localNode.record.seqNum
pong.ip = case a.ip.family
of IpAddressFamily.IPv4: @(a.ip.address_v4)
of IpAddressFamily.IPv6: @(a.ip.address_v6)
pong.port = a.port.uint16
let (data, _) = encodePacket(d.rng[], d.codec, fromId, fromAddr,
encodeMessage(pong, reqId), challenge = nil)
d.send(fromAddr, data)
proc handleFindNode(d: Protocol, fromId: NodeId, fromAddr: Address,
fn: FindNodeMessage, reqId: RequestId) =
if fn.distance == 0:
d.sendNodes(fromId, fromAddr, reqId, [d.localNode])
else:
if fn.distance <= 256:
d.sendNodes(fromId, fromAddr, reqId,
d.routingTable.neighboursAtDistance(fn.distance, seenOnly = true))
else:
# The polite node we are, still respond with empty nodes.
d.sendNodes(fromId, fromAddr, reqId, [])
proc receive*(d: Protocol, a: Address, packet: openArray[byte]) {.gcsafe,
raises: [
Defect,
# This just comes now from a future.complete() and `sendWhoareyou` which
# has it because of `sleepAsync` with `addCallback`, but practically, no
# CatchableError should be raised here, we just can't enforce it for now.
Exception
].} =
if packet.len < tagSize: # or magicSize, can be either
return # Invalid packet
# debug "Packet received: ", length = packet.len
if d.isWhoAreYou(packet):
trace "Received whoareyou", localNode = d.localNode, address = a
var whoareyou: WhoAreYou
try:
whoareyou = d.decodeWhoAreYou(packet)
except RlpError:
debug "Invalid WhoAreYou packet, decoding failed"
return
var pr: PendingRequest
if d.pendingRequests.take(whoareyou.authTag, pr):
let toNode = pr.node
whoareyou.pubKey = some(toNode.pubkey) # TODO: Yeah, rather ugly this.
doAssert(toNode.address.isSome())
let (data, _) = encodePacket(d.rng[], d.codec, toNode.id, toNode.address.get(),
pr.message, challenge = whoareyou)
d.send(toNode, data)
else:
debug "Timed out or unrequested WhoAreYou packet"
else:
var tag: array[tagSize, byte]
tag[0 .. ^1] = packet.toOpenArray(0, tagSize - 1)
let senderData = tag xor d.idHash
let sender = readUintBE[256](senderData)
var authTag: AuthTag
var node: Node
let decoded = d.codec.decodePacket(sender, a, packet, authTag, node)
if decoded.isOk:
let message = decoded[]
if not node.isNil:
# Not filling table with nodes without correct IP in the ENR
# TODO: Should we care about this???
if node.address.isSome() and a == node.address.get():
debug "Adding new node to routing table", node = node,
localNode = d.localNode
discard d.addNode(node)
case message.kind
of ping:
d.handlePing(sender, a, message.ping, message.reqId)
of findNode:
d.handleFindNode(sender, a, message.findNode, message.reqId)
else:
var waiter: Future[Option[Message]]
if d.awaitedMessages.take((sender, message.reqId), waiter):
waiter.complete(some(message)) # TODO: raises: [Exception]
else:
trace "Timed out or unrequested message", message = message.kind,
origin = a
elif decoded.error == DecodeError.DecryptError:
trace "Could not decrypt packet, respond with whoareyou",
localNode = d.localNode, address = a
# only sendingWhoareyou in case it is a decryption failure
let res = d.sendWhoareyou(a, sender, authTag)
if res.isErr():
trace "Sending WhoAreYou packet failed", err = res.error
elif decoded.error == DecodeError.UnsupportedMessage:
# Still adding the node in case failure is because of unsupported message.
if not node.isNil:
# Not filling table with nodes without correct IP in the ENR
# TODO: Should we care about this???s
if node.address.isSome() and a == node.address.get():
debug "Adding new node to routing table", node = node,
localNode = d.localNode
discard d.addNode(node)
# elif decoded.error == DecodeError.PacketError:
# Not adding this node as from our perspective it is sending rubbish.
# TODO: Not sure why but need to pop the raises here as it is apparently not
# enough to put it in the raises pragma of `processClient` and other async procs.
{.pop.}
# Next, below there is no more effort done in catching the general `Exception`
# as async procs always require `Exception` in the raises pragma, see also:
# https://github.com/status-im/nim-chronos/issues/98
# So I don't bother for now and just add them in the raises pragma until this
# gets fixed. It does not mean that we expect these calls to be raising
# CatchableErrors, in fact, we really don't, but hey, they might, considering we
# can't enforce it.
proc processClient(transp: DatagramTransport, raddr: TransportAddress):
Future[void] {.async, gcsafe, raises: [Exception, Defect].} =
let proto = getUserData[Protocol](transp)
# TODO: should we use `peekMessage()` to avoid allocation?
# TODO: This can still raise general `Exception` while it probably should
# only give TransportOsError.
let buf = try: transp.getMessage()
except TransportOsError as e:
# This is likely to be local network connection issues.
error "Transport getMessage", exception = e.name, msg = e.msg
return
except Exception as e:
if e of Defect:
raise (ref Defect)(e)
else: doAssert(false)
return # Make compiler happy
let ip = try: raddr.address()
except ValueError as e:
error "Not a valid IpAddress", exception = e.name, msg = e.msg
return
let a = Address(ip: ValidIpAddress.init(ip), port: raddr.port)
try:
proto.receive(a, buf)
except Exception as e:
if e of Defect:
raise (ref Defect)(e)
else: doAssert(false)
proc validIp(sender, address: IpAddress): bool {.raises: [Defect].} =
let
s = initTAddress(sender, Port(0))
a = initTAddress(address, Port(0))
if a.isAnyLocal():
return false
if a.isMulticast():
return false
if a.isLoopback() and not s.isLoopback():
return false
if a.isSiteLocal() and not s.isSiteLocal():
return false
# TODO: Also check for special reserved ip addresses:
# https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
# https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
return true
proc replaceNode(d: Protocol, n: Node) =
if n.record notin d.bootstrapRecords:
d.routingTable.replaceNode(n)
else:
# For now we never remove bootstrap nodes. It might make sense to actually
# do so and to retry them only in case we drop to a really low amount of
# peers in the routing table.
debug "Message request to bootstrap node failed", enr = toURI(n.record)
# TODO: This could be improved to do the clean-up immediatily in case a non
# whoareyou response does arrive, but we would need to store the AuthTag
# somewhere
proc registerRequest(d: Protocol, n: Node, message: seq[byte], nonce: AuthTag)
{.raises: [Exception, Defect].} =
let request = PendingRequest(node: n, message: message)
if not d.pendingRequests.hasKeyOrPut(nonce, request):
# TODO: raises: [Exception]
sleepAsync(responseTimeout).addCallback() do(data: pointer):
d.pendingRequests.del(nonce)
proc waitMessage(d: Protocol, fromNode: Node, reqId: RequestId):
Future[Option[Message]] {.raises: [Exception, Defect].} =
result = newFuture[Option[Message]]("waitMessage")
let res = result
let key = (fromNode.id, reqId)
# TODO: raises: [Exception]
sleepAsync(responseTimeout).addCallback() do(data: pointer):
d.awaitedMessages.del(key)
if not res.finished:
res.complete(none(Message)) # TODO: raises: [Exception]
d.awaitedMessages[key] = result
proc verifyNodesRecords*(enrs: openarray[Record], fromNode: Node,
distance: uint32): seq[Node] {.raises: [Defect].} =
## Verify and convert ENRs to a sequence of nodes. Only ENRs that pass
## verification will be added. ENRs are verified for duplicates, invalid
## addresses and invalid distances.
# TODO:
# - Should we fail and ignore values on first invalid Node?
# - Should we limit the amount of nodes? The discovery v5 specification holds
# no limit on the amount that can be returned.
var seen: HashSet[Node]
for r in enrs:
let node = newNode(r)
if node.isOk():
let n = node.get()
# Check for duplicates in the nodes reply. Duplicates are checked based
# on node id.
if n in seen:
trace "Nodes reply contained records with duplicate node ids",
record = n.record.toURI, sender = fromNode.record.toURI, id = n.id
continue
# Check if the node has an address and if the address is public or from
# the same local network or lo network as the sender. The latter allows
# for local testing.
if not n.address.isSome() or not
validIp(fromNode.address.get().ip, n.address.get().ip):
trace "Nodes reply contained record with invalid ip-address",
record = n.record.toURI, sender = fromNode.record.toURI, node = n
continue
# Check if returned node has exactly the requested distance.
if logDist(n.id, fromNode.id) != distance:
warn "Nodes reply contained record with incorrect distance",
record = n.record.toURI, sender = fromNode.record.toURI
continue
# No check on UDP port and thus any port is allowed, also the so called
# "well-known" ports.
seen.incl(n)
result.add(n)
proc waitNodes(d: Protocol, fromNode: Node, reqId: RequestId):
Future[DiscResult[seq[Record]]] {.async, raises: [Exception, Defect].} =
## Wait for one or more nodes replies.
##
## The first reply will hold the total number of replies expected, and based
## on that, more replies will be awaited.
## If one reply is lost here (timed out), others are ignored too.
## Same counts for out of order receival.
var op = await d.waitMessage(fromNode, reqId)
if op.isSome and op.get.kind == nodes:
var res = op.get.nodes.enrs
let total = op.get.nodes.total
for i in 1 ..< total:
op = await d.waitMessage(fromNode, reqId)
if op.isSome and op.get.kind == nodes:
res.add(op.get.nodes.enrs)
else:
# No error on this as we received some nodes.
break
return ok(res)
else:
return err("Nodes message not received in time")
proc sendMessage*[T: SomeMessage](d: Protocol, toNode: Node, m: T):
RequestId {.raises: [Exception, Defect].} =
doAssert(toNode.address.isSome())
let
reqId = RequestId.init(d.rng[])
message = encodeMessage(m, reqId)
(data, nonce) = encodePacket(d.rng[], d.codec, toNode.id, toNode.address.get(),
message, challenge = nil)
d.registerRequest(toNode, message, nonce)
d.send(toNode, data)
return reqId
proc ping*(d: Protocol, toNode: Node):
Future[DiscResult[PongMessage]] {.async, raises: [Exception, Defect].} =
## Send a discovery ping message.
##
## Returns the received pong message or an error.
let reqId = d.sendMessage(toNode,
PingMessage(enrSeq: d.localNode.record.seqNum))
let resp = await d.waitMessage(toNode, reqId)
if resp.isSome() and resp.get().kind == pong:
d.routingTable.setJustSeen(toNode)
return ok(resp.get().pong)
else:
d.replaceNode(toNode)
return err("Pong message not received in time")
proc findNode*(d: Protocol, toNode: Node, distance: uint32):
Future[DiscResult[seq[Node]]] {.async, raises: [Exception, Defect].} =
## Send a discovery findNode message.
##
## Returns the received nodes or an error.
## Received ENRs are already validated and converted to `Node`.
let reqId = d.sendMessage(toNode, FindNodeMessage(distance: distance))
let nodes = await d.waitNodes(toNode, reqId)
if nodes.isOk:
let res = verifyNodesRecords(nodes.get(), toNode, distance)
d.routingTable.setJustSeen(toNode)
return ok(res)
else:
d.replaceNode(toNode)
return err(nodes.error)
proc lookupDistances(target, dest: NodeId): seq[uint32] {.raises: [Defect].} =
let td = logDist(target, dest)
result.add(td)
var i = 1'u32
while result.len < lookupRequestLimit:
if td + i < 256:
result.add(td + i)
if td - i > 0'u32:
result.add(td - i)
inc i
proc lookupWorker(d: Protocol, destNode: Node, target: NodeId):
Future[seq[Node]] {.async, raises: [Exception, Defect].} =
let dists = lookupDistances(target, destNode.id)
var i = 0
while i < lookupRequestLimit and result.len < findNodeResultLimit:
let r = await d.findNode(destNode, dists[i])
# TODO: Handle failures better. E.g. stop on different failures than timeout
if r.isOk:
# TODO: I guess it makes sense to limit here also to `findNodeResultLimit`?
result.add(r[])
inc i
for n in result:
discard d.routingTable.addNode(n)
proc lookup*(d: Protocol, target: NodeId): Future[seq[Node]]
{.async, raises: [Exception, Defect].} =
## Perform a lookup for the given target, return the closest n nodes to the
## target. Maximum value for n is `BUCKET_SIZE`.
# TODO: Sort the returned nodes on distance
# Also use unseen nodes as a form of validation.
result = d.routingTable.neighbours(target, BUCKET_SIZE, seenOnly = false)
var asked = initHashSet[NodeId]()
asked.incl(d.localNode.id)
var seen = asked
for node in result:
seen.incl(node.id)
var pendingQueries = newSeqOfCap[Future[seq[Node]]](alpha)
while true:
var i = 0
while i < result.len and pendingQueries.len < alpha:
let n = result[i]
if not asked.containsOrIncl(n.id):
pendingQueries.add(d.lookupWorker(n, target))
inc i
trace "discv5 pending queries", total = pendingQueries.len
if pendingQueries.len == 0:
break
let idx = await oneIndex(pendingQueries)
trace "Got discv5 lookup response", idx
let nodes = pendingQueries[idx].read
pendingQueries.del(idx)
for n in nodes:
if not seen.containsOrIncl(n.id):
if result.len < BUCKET_SIZE:
result.add(n)
proc lookupRandom*(d: Protocol): Future[seq[Node]]
{.async, raises:[Exception, Defect].} =
## Perform a lookup for a random target, return the closest n nodes to the
## target. Maximum value for n is `BUCKET_SIZE`.
var id: NodeId
var buf: array[sizeof(id), byte]
brHmacDrbgGenerate(d.rng[], buf)
copyMem(addr id, addr buf[0], sizeof(id))
return await d.lookup(id)
proc resolve*(d: Protocol, id: NodeId): Future[Option[Node]]
{.async, raises: [Exception, Defect].} =
## Resolve a `Node` based on provided `NodeId`.
##
## This will first look in the own routing table. If the node is known, it
## will try to contact if for newer information. If node is not known or it
## does not reply, a lookup is done to see if it can find a (newer) record of
## the node on the network.
let node = d.getNode(id)
if node.isSome():
let request = await d.findNode(node.get(), 0)
# TODO: Handle failures better. E.g. stop on different failures than timeout
if request.isOk() and request[].len > 0:
return some(request[][0])
let discovered = await d.lookup(id)
for n in discovered:
if n.id == id:
if node.isSome() and node.get().record.seqNum >= n.record.seqNum:
return node
else:
return some(n)
return node
proc revalidateNode*(d: Protocol, n: Node)
{.async, raises: [Exception, Defect].} = # TODO: Exception
let pong = await d.ping(n)
if pong.isOK():
if pong.get().enrSeq > n.record.seqNum:
# Request new ENR
let nodes = await d.findNode(n, 0)
if nodes.isOk() and nodes[].len > 0:
discard d.addNode(nodes[][0])
proc revalidateLoop(d: Protocol) {.async, raises: [Exception, Defect].} =
# TODO: General Exception raised.
try:
while true:
await sleepAsync(d.rng[].rand(revalidateMax).milliseconds)
let n = d.routingTable.nodeToRevalidate()
if not n.isNil:
traceAsyncErrors d.revalidateNode(n)
except CancelledError:
trace "revalidateLoop canceled"
proc lookupLoop(d: Protocol) {.async, raises: [Exception, Defect].} =
# TODO: General Exception raised.
try:
# lookup self (neighbour nodes)
let selfLookup = await d.lookup(d.localNode.id)
trace "Discovered nodes in self lookup", nodes = selfLookup
while true:
let randomLookup = await d.lookupRandom()
trace "Discovered nodes in random lookup", nodes = randomLookup
trace "Total nodes in routing table", total = d.routingTable.len()
await sleepAsync(lookupInterval)
except CancelledError:
trace "lookupLoop canceled"
proc newProtocol*(privKey: PrivateKey,
externalIp: Option[ValidIpAddress], tcpPort, udpPort: Port,
localEnrFields: openarray[(string, seq[byte])] = [],
bootstrapRecords: openarray[Record] = [],
previousRecord = none[enr.Record](),
bindIp = IPv4_any(), rng = newRng()):
Protocol {.raises: [Defect].} =
# TODO: Tried adding bindPort = udpPort as parameter but that gave
# "Error: internal error: environment misses: udpPort" in nim-beacon-chain.
# Anyhow, nim-beacon-chain would also require some changes to support port
# remapping through NAT and this API is also subject to change once we
# introduce support for ipv4 + ipv6 binding/listening.
let extraFields = mapIt(localEnrFields, toFieldPair(it[0], it[1]))
# TODO:
# - Defect as is now or return a result for enr errors?
# - In case incorrect key, allow for new enr based on new key (new node id)?
var record: Record
if previousRecord.isSome():
record = previousRecord.get()
record.update(privKey, externalIp, tcpPort, udpPort,
extraFields).expect("Record within size limits and correct key")
else:
record = enr.Record.init(1, privKey, externalIp, tcpPort, udpPort,
extraFields).expect("Record within size limits")
let node = newNode(record).expect("Properly initialized record")
# TODO Consider whether this should be a Defect
doAssert rng != nil, "RNG initialization failed"
result = Protocol(
privateKey: privKey,
localNode: node,
bindAddress: Address(ip: ValidIpAddress.init(bindIp), port: udpPort),
whoareyouMagic: whoareyouMagic(node.id),
idHash: sha256.digest(node.id.toByteArrayBE).data,
codec: Codec(localNode: node, privKey: privKey,
sessions: Sessions.init(256)),
bootstrapRecords: @bootstrapRecords,
rng: rng)
result.routingTable.init(node, 5, rng)
proc open*(d: Protocol) {.raises: [Exception, Defect].} =
info "Starting discovery node", node = d.localNode,
uri = toURI(d.localNode.record), bindAddress = d.bindAddress
# TODO allow binding to specific IP / IPv6 / etc
let ta = initTAddress(d.bindAddress.ip, d.bindAddress.port)
# TODO: raises `OSError` and `IOSelectorsException`, the latter which is
# object of Exception. In Nim devel this got changed to CatchableError.
d.transp = newDatagramTransport(processClient, udata = d, local = ta)
for record in d.bootstrapRecords:
debug "Adding bootstrap node", uri = toURI(record)
discard d.addNode(record)
proc start*(d: Protocol) {.raises: [Exception, Defect].} =
d.lookupLoop = lookupLoop(d)
d.revalidateLoop = revalidateLoop(d)
proc close*(d: Protocol) {.raises: [Exception, Defect].} =
doAssert(not d.transp.closed)
debug "Closing discovery node", node = d.localNode
if not d.revalidateLoop.isNil:
d.revalidateLoop.cancel()
if not d.lookupLoop.isNil:
d.lookupLoop.cancel()
d.transp.close()
proc closeWait*(d: Protocol) {.async, raises: [Exception, Defect].} =
doAssert(not d.transp.closed)
debug "Closing discovery node", node = d.localNode
if not d.revalidateLoop.isNil:
await d.revalidateLoop.cancelAndWait()
if not d.lookupLoop.isNil:
await d.lookupLoop.cancelAndWait()
await d.transp.closeWait()

835
eth/p2p/discoveryv5/protocolv1.nim
View File

@ -1,835 +0,0 @@
# nim-eth - Node Discovery Protocol v5
# Copyright (c) 2020 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, (LICENSE-APACHEv2)
# * MIT license (LICENSE-MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
## Node Discovery Protocol v5
##
## Node discovery protocol implementation as per specification:
## https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md
##
## This node discovery protocol implementation uses the same underlying
## implementation of routing table as is also used for the discovery v4
## implementation, which is the same or similar as the one described in the
## original Kademlia paper:
## https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf
##
## This might not be the most optimal implementation for the node discovery
## protocol v5. Why?
##
## The Kademlia paper describes an implementation that starts off from one
## k-bucket, and keeps splitting the bucket as more nodes are discovered and
## added. The bucket splits only on the part of the binary tree where our own
## node its id belongs too (same prefix). Resulting eventually in a k-bucket per
## logarithmic distance (log base2 distance). Well, not really, as nodes with
## ids in the closer distance ranges will never be found. And because of this an
## optimisation is done where buckets will also split sometimes even if the
## nodes own id does not have the same prefix (this is to avoid creating highly
## unbalanced branches which would require longer lookups).
##
## Now, some implementations take a more simplified approach. They just create
## directly a bucket for each possible logarithmic distance (e.g. here 1->256).
## Some implementations also don't create buckets with logarithmic distance
## lower than a certain value (e.g. only 1/15th of the highest buckets),
## because the closer to the node (the lower the distance), the less chance
## there is to still find nodes.
##
## The discovery protocol v4 its `FindNode` call will request the k closest
## nodes. As does original Kademlia. This effectively puts the work at the node
## that gets the request. This node will have to check its buckets and gather
## the closest. Some implementations go over all the nodes in all the buckets
## for this (e.g. go-ethereum discovery v4). However, in our bucket splitting
## approach, this search is improved.
##
## In the discovery protocol v5 the `FindNode` call is changed and now the
## logarithmic distance is passed as parameter instead of the NodeId. And only
## nodes that match that logarithmic distance are allowed to be returned.
## This change was made to not put the trust at the requested node for selecting
## the closest nodes. To counter a possible (mistaken) difference in
## implementation, but more importantly for security reasons. See also:
## https://github.com/ethereum/devp2p/blob/master/discv5/discv5-rationale.md#115-guard-against-kademlia-implementation-flaws
##
## The result is that in an implementation which just stores buckets per
## logarithmic distance, it simply needs to return the right bucket. In our
## split-bucket implementation, this cannot be done as such and thus the closest
## neighbours search is still done. And to do this, a reverse calculation of an
## id at given logarithmic distance is needed (which is why there is the
## `idAtDistance` proc). Next, nodes with invalid distances need to be filtered
## out to be compliant to the specification. This can most likely get further
## optimised, but it sounds likely better to switch away from the split-bucket
## approach. I believe that the main benefit it has is improved lookups
## (due to no unbalanced branches), and it looks like this will be negated by
## limiting the returned nodes to only the ones of the requested logarithmic
## distance for the `FindNode` call.
## This `FindNode` change in discovery v5 will also have an effect on the
## efficiency of the network. Work will be moved from the receiver of
## `FindNodes` to the requester. But this also means more network traffic,
## as less nodes will potentially be passed around per `FindNode` call, and thus
## more requests will be needed for a lookup (adding bandwidth and latency).
## This might be a concern for mobile devices.
import
std/[tables, sets, options, math, sequtils],
stew/shims/net as stewNet, json_serialization/std/net,
stew/[byteutils, endians2], chronicles, chronos, stint, bearssl,
eth/[rlp, keys, async_utils],
typesv1, encodingv1, node, routing_table, enr, random2, sessions
import nimcrypto except toHex
export options
{.push raises: [Defect].}
logScope:
topics = "discv5"
const
alpha = 3 ## Kademlia concurrency factor
lookupRequestLimit = 3
findNodeResultLimit = 15 # applies in FINDNODE handler
maxNodesPerMessage = 3
lookupInterval = 60.seconds ## Interval of launching a random lookup to
## populate the routing table. go-ethereum seems to do 3 runs every 30
## minutes. Trinity starts one every minute.
revalidateMax = 10000 ## Revalidation of a peer is done between 0 and this
## value in milliseconds
handshakeTimeout* = 2.seconds ## timeout for the reply on the
## whoareyou message
responseTimeout* = 4.seconds ## timeout for the response of a request-response
## call
type
Protocol* = ref object
transp: DatagramTransport
localNode*: Node
privateKey: PrivateKey
bindAddress: Address ## UDP binding address
pendingRequests: Table[AESGCMNonce, PendingRequest]
routingTable: RoutingTable
codec*: Codec
awaitedMessages: Table[(NodeId, RequestId), Future[Option[Message]]]
lookupLoop: Future[void]
revalidateLoop: Future[void]
bootstrapRecords*: seq[Record]
rng*: ref BrHmacDrbgContext
PendingRequest = object
node: Node
message: seq[byte]
DiscResult*[T] = Result[T, cstring]
proc addNode*(d: Protocol, node: Node): bool =
## Add `Node` to discovery routing table.
##
## Returns false only if `Node` is not eligable for adding (no Address).
if node.address.isSome():
# Only add nodes with an address to the routing table
discard d.routingTable.addNode(node)
return true
proc addNode*(d: Protocol, r: Record): bool =
## Add `Node` from a `Record` to discovery routing table.
##
## Returns false only if no valid `Node` can be created from the `Record` or
## on the conditions of `addNode` from a `Node`.
let node = newNode(r)
if node.isOk():
return d.addNode(node[])
proc addNode*(d: Protocol, enr: EnrUri): bool =
## Add `Node` from a ENR URI to discovery routing table.
##
## Returns false if no valid ENR URI, or on the conditions of `addNode` from
## an `Record`.
var r: Record
let res = r.fromUri(enr)
if res:
return d.addNode(r)
proc getNode*(d: Protocol, id: NodeId): Option[Node] =
## Get the node with id from the routing table.
d.routingTable.getNode(id)
proc randomNodes*(d: Protocol, maxAmount: int): seq[Node] =
## Get a `maxAmount` of random nodes from the local routing table.
d.routingTable.randomNodes(maxAmount)
proc randomNodes*(d: Protocol, maxAmount: int,
pred: proc(x: Node): bool {.gcsafe, noSideEffect.}): seq[Node] =
## Get a `maxAmount` of random nodes from the local routing table with the
## `pred` predicate function applied as filter on the nodes selected.
d.routingTable.randomNodes(maxAmount, pred)
proc randomNodes*(d: Protocol, maxAmount: int,
enrField: (string, seq[byte])): seq[Node] =
## Get a `maxAmount` of random nodes from the local routing table. The
## the nodes selected are filtered by provided `enrField`.
d.randomNodes(maxAmount, proc(x: Node): bool = x.record.contains(enrField))
proc neighbours*(d: Protocol, id: NodeId, k: int = BUCKET_SIZE): seq[Node] =
## Return up to k neighbours (closest node ids) of the given node id.
d.routingTable.neighbours(id, k)
proc nodesDiscovered*(d: Protocol): int {.inline.} = d.routingTable.len
func privKey*(d: Protocol): lent PrivateKey =
d.privateKey
func getRecord*(d: Protocol): Record =
## Get the ENR of the local node.
d.localNode.record
proc updateRecord*(
d: Protocol, enrFields: openarray[(string, seq[byte])]): DiscResult[void] =
## Update the ENR of the local node with provided `enrFields` k:v pairs.
let fields = mapIt(enrFields, toFieldPair(it[0], it[1]))
d.localNode.record.update(d.privateKey, fields)
# TODO: Would it make sense to actively ping ("broadcast") to all the peers
# we stored a handshake with in order to get that ENR updated?
proc send(d: Protocol, a: Address, data: seq[byte]) =
let ta = initTAddress(a.ip, a.port)
try:
let f = d.transp.sendTo(ta, data)
f.callback = proc(data: pointer) {.gcsafe.} =
if f.failed:
# Could be `TransportUseClosedError` in case the transport is already
# closed, or could be `TransportOsError` in case of a socket error.
# In the latter case this would probably mostly occur if the network
# interface underneath gets disconnected or similar.
# TODO: Should this kind of error be propagated upwards? Probably, but
# it should not stop the process as that would reset the discovery
# progress in case there is even a small window of no connection.
# One case that needs this error available upwards is when revalidating
# nodes. Else the revalidation might end up clearing the routing tabl
# because of ping failures due to own network connection failure.
warn "Discovery send failed", msg = f.readError.msg
except Exception as e:
# TODO: General exception still being raised from Chronos, but in practice
# all CatchableErrors should be grabbed by the above `f.failed`.
if e of Defect:
raise (ref Defect)(e)
else: doAssert(false)
proc send(d: Protocol, n: Node, data: seq[byte]) =
doAssert(n.address.isSome())
d.send(n.address.get(), data)
proc sendNodes(d: Protocol, toId: NodeId, toAddr: Address, reqId: RequestId,
nodes: openarray[Node]) =
proc sendNodes(d: Protocol, toId: NodeId, toAddr: Address,
message: NodesMessage, reqId: RequestId) {.nimcall.} =
let (data, _) = encodeMessagePacket(d.rng[], d.codec, toId, toAddr,
encodeMessage(message, reqId))
trace "Respond message packet", dstId = toId, address = toAddr,
kind = MessageKind.nodes
d.send(toAddr, data)
if nodes.len == 0:
# In case of 0 nodes, a reply is still needed
d.sendNodes(toId, toAddr, NodesMessage(total: 1, enrs: @[]), reqId)
return
var message: NodesMessage
# TODO: Do the total calculation based on the max UDP packet size we want to
# send and the ENR size of all (max 16) nodes.
# Which UDP packet size to take? 1280? 576?
message.total = ceil(nodes.len / maxNodesPerMessage).uint32
for i in 0 ..< nodes.len:
message.enrs.add(nodes[i].record)
if message.enrs.len == maxNodesPerMessage:
d.sendNodes(toId, toAddr, message, reqId)
message.enrs.setLen(0)
if message.enrs.len != 0:
d.sendNodes(toId, toAddr, message, reqId)
proc handlePing(d: Protocol, fromId: NodeId, fromAddr: Address,
ping: PingMessage, reqId: RequestId) =
let a = fromAddr
var pong: PongMessage
pong.enrSeq = d.localNode.record.seqNum
pong.ip = case a.ip.family
of IpAddressFamily.IPv4: @(a.ip.address_v4)
of IpAddressFamily.IPv6: @(a.ip.address_v6)
pong.port = a.port.uint16
let (data, _) = encodeMessagePacket(d.rng[], d.codec, fromId, fromAddr,
encodeMessage(pong, reqId))
trace "Respond message packet", dstId = fromId, address = fromAddr,
kind = MessageKind.pong
d.send(fromAddr, data)
proc handleFindNode(d: Protocol, fromId: NodeId, fromAddr: Address,
fn: FindNodeMessage, reqId: RequestId) =
if fn.distances.len == 0:
d.sendNodes(fromId, fromAddr, reqId, [])
elif fn.distances.contains(0):
# A request for our own record.
# It would be a weird request if there are more distances next to 0
# requested, so in this case lets just pass only our own. TODO: OK?
d.sendNodes(fromId, fromAddr, reqId, [d.localNode])
else:
# TODO: Still deduplicate also?
if fn.distances.all(proc (x: uint32): bool = return x <= 256):
d.sendNodes(fromId, fromAddr, reqId,
d.routingTable.neighboursAtDistances(fn.distances, seenOnly = true))
else:
# At least one invalid distance, but the polite node we are, still respond
# with empty nodes.
d.sendNodes(fromId, fromAddr, reqId, [])
proc handleTalkReq(d: Protocol, fromId: NodeId, fromAddr: Address,
talkreq: TalkReqMessage, reqId: RequestId) =
# No support for any protocol yet so an empty response is send as per
# specification.
let talkresp = TalkRespMessage(response: @[])
let (data, _) = encodeMessagePacket(d.rng[], d.codec, fromId, fromAddr,
encodeMessage(talkresp, reqId))
trace "Respond message packet", dstId = fromId, address = fromAddr,
kind = MessageKind.talkresp
d.send(fromAddr, data)
proc handleMessage(d: Protocol, srcId: NodeId, fromAddr: Address,
message: Message) {.raises:[Exception].} =
case message.kind
of ping:
d.handlePing(srcId, fromAddr, message.ping, message.reqId)
of findNode:
d.handleFindNode(srcId, fromAddr, message.findNode, message.reqId)
of talkreq:
d.handleTalkReq(srcId, fromAddr, message.talkreq, message.reqId)
of regtopic, topicquery:
trace "Received unimplemented message kind", kind = message.kind,
origin = fromAddr
else:
var waiter: Future[Option[Message]]
if d.awaitedMessages.take((srcId, message.reqId), waiter):
waiter.complete(some(message)) # TODO: raises: [Exception]
else:
trace "Timed out or unrequested message", kind = message.kind,
origin = fromAddr
proc sendWhoareyou(d: Protocol, toId: NodeId, a: Address,
requestNonce: AESGCMNonce, node: Option[Node]) {.raises: [Exception].} =
let key = HandShakeKey(nodeId: toId, address: $a)
if not d.codec.hasHandshake(key):
let
recordSeq = if node.isSome(): node.get().record.seqNum
else: 0
pubkey = if node.isSome(): some(node.get().pubkey)
else: none(PublicKey)
let data = encodeWhoareyouPacket(d.rng[], d.codec, toId, a, requestNonce,
recordSeq, pubkey)
sleepAsync(handshakeTimeout).addCallback() do(data: pointer):
# TODO: should we still provide cancellation in case handshake completes
# correctly?
d.codec.handshakes.del(key)
trace "Send whoareyou", dstId = toId, address = a
d.send(a, data)
else:
debug "Node with this id already has ongoing handshake, ignoring packet"
proc receive*(d: Protocol, a: Address, packet: openArray[byte]) {.gcsafe,
raises: [
Defect,
# This just comes now from a future.complete() and `sendWhoareyou` which
# has it because of `sleepAsync` with `addCallback`, but practically, no
# CatchableError should be raised here, we just can't enforce it for now.
Exception
].} =
let decoded = d.codec.decodePacket(a, packet)
if decoded.isOk:
let packet = decoded[]
case packet.flag
of OrdinaryMessage:
if packet.messageOpt.isSome():
let message = packet.messageOpt.get()
trace "Received message packet", srcId = packet.srcId, address = a,
kind = message.kind
d.handleMessage(packet.srcId, a, message)
else:
trace "Not decryptable message packet received",
srcId = packet.srcId, address = a
d.sendWhoareyou(packet.srcId, a, packet.requestNonce,
d.getNode(packet.srcId))
of Flag.Whoareyou:
trace "Received whoareyou packet", address = a
var pr: PendingRequest
if d.pendingRequests.take(packet.whoareyou.requestNonce, pr):
let toNode = pr.node
# This is a node we previously contacted and thus must have an address.
doAssert(toNode.address.isSome())
let address = toNode.address.get()
let data = encodeHandshakePacket(d.rng[], d.codec, toNode.id,
address, pr.message, packet.whoareyou, toNode.pubkey)
trace "Send handshake message packet", dstId = toNode.id, address
d.send(toNode, data)
else:
debug "Timed out or unrequested whoareyou packet", address = a
of HandshakeMessage:
trace "Received handshake message packet", srcId = packet.srcIdHs,
address = a, kind = packet.message.kind
d.handleMessage(packet.srcIdHs, a, packet.message)
# For a handshake message it is possible that we received an newer ENR.
# In that case we can add/update it to the routing table.
if packet.node.isSome():
let node = packet.node.get()
# Not filling table with nodes without correct IP in the ENR
# TODO: Should we care about this???
if node.address.isSome() and a == node.address.get():
debug "Adding new node to routing table", node
discard d.addNode(node)
else:
debug "Packet decoding error", error = decoded.error, address = a
# TODO: Not sure why but need to pop the raises here as it is apparently not
# enough to put it in the raises pragma of `processClient` and other async procs.
{.pop.}
# Next, below there is no more effort done in catching the general `Exception`
# as async procs always require `Exception` in the raises pragma, see also:
# https://github.com/status-im/nim-chronos/issues/98
# So I don't bother for now and just add them in the raises pragma until this
# gets fixed. It does not mean that we expect these calls to be raising
# CatchableErrors, in fact, we really don't, but hey, they might, considering we
# can't enforce it.
proc processClient(transp: DatagramTransport, raddr: TransportAddress):
Future[void] {.async, gcsafe, raises: [Exception, Defect].} =
let proto = getUserData[Protocol](transp)
# TODO: should we use `peekMessage()` to avoid allocation?
# TODO: This can still raise general `Exception` while it probably should
# only give TransportOsError.
let buf = try: transp.getMessage()
except TransportOsError as e:
# This is likely to be local network connection issues.
warn "Transport getMessage", exception = e.name, msg = e.msg
return
except Exception as e:
if e of Defect:
raise (ref Defect)(e)
else: doAssert(false)
return # Make compiler happy
let ip = try: raddr.address()
except ValueError as e:
error "Not a valid IpAddress", exception = e.name, msg = e.msg
return
let a = Address(ip: ValidIpAddress.init(ip), port: raddr.port)
try:
proto.receive(a, buf)
except Exception as e:
if e of Defect:
raise (ref Defect)(e)
else: doAssert(false)
proc validIp(sender, address: IpAddress): bool {.raises: [Defect].} =
let
s = initTAddress(sender, Port(0))
a = initTAddress(address, Port(0))
if a.isAnyLocal():
return false
if a.isMulticast():
return false
if a.isLoopback() and not s.isLoopback():
return false
if a.isSiteLocal() and not s.isSiteLocal():
return false
# TODO: Also check for special reserved ip addresses:
# https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
# https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
return true
proc replaceNode(d: Protocol, n: Node) =
if n.record notin d.bootstrapRecords:
d.routingTable.replaceNode(n)
else:
# For now we never remove bootstrap nodes. It might make sense to actually
# do so and to retry them only in case we drop to a really low amount of
# peers in the routing table.
debug "Message request to bootstrap node failed", enr = toURI(n.record)
# TODO: This could be improved to do the clean-up immediatily in case a non
# whoareyou response does arrive, but we would need to store the AuthTag
# somewhere
proc registerRequest(d: Protocol, n: Node, message: seq[byte],
nonce: AESGCMNonce) {.raises: [Exception, Defect].} =
let request = PendingRequest(node: n, message: message)
if not d.pendingRequests.hasKeyOrPut(nonce, request):
# TODO: raises: [Exception]
sleepAsync(responseTimeout).addCallback() do(data: pointer):
d.pendingRequests.del(nonce)
proc waitMessage(d: Protocol, fromNode: Node, reqId: RequestId):
Future[Option[Message]] {.raises: [Exception, Defect].} =
result = newFuture[Option[Message]]("waitMessage")
let res = result
let key = (fromNode.id, reqId)
# TODO: raises: [Exception]
sleepAsync(responseTimeout).addCallback() do(data: pointer):
d.awaitedMessages.del(key)
if not res.finished:
res.complete(none(Message)) # TODO: raises: [Exception]
d.awaitedMessages[key] = result
proc verifyNodesRecords*(enrs: openarray[Record], fromNode: Node,
distances: varargs[uint32]): seq[Node] {.raises: [Defect].} =
## Verify and convert ENRs to a sequence of nodes. Only ENRs that pass
## verification will be added. ENRs are verified for duplicates, invalid
## addresses and invalid distances.
# TODO:
# - Should we fail and ignore values on first invalid Node?
# - Should we limit the amount of nodes? The discovery v5 specification holds
# no limit on the amount that can be returned.
var seen: HashSet[Node]
for r in enrs:
let node = newNode(r)
if node.isOk():
let n = node.get()
# Check for duplicates in the nodes reply. Duplicates are checked based
# on node id.
if n in seen:
trace "Nodes reply contained records with duplicate node ids",
record = n.record.toURI, id = n.id, sender = fromNode.record.toURI
continue
# Check if the node has an address and if the address is public or from
# the same local network or lo network as the sender. The latter allows
# for local testing.
if not n.address.isSome() or not
validIp(fromNode.address.get().ip, n.address.get().ip):
trace "Nodes reply contained record with invalid ip-address",
record = n.record.toURI, node = n, sender = fromNode.record.toURI
continue
# Check if returned node has one of the requested distances.
if not distances.contains(logDist(n.id, fromNode.id)):
warn "Nodes reply contained record with incorrect distance",
record = n.record.toURI, sender = fromNode.record.toURI
continue
# No check on UDP port and thus any port is allowed, also the so called
# "well-known" ports.
seen.incl(n)
result.add(n)
proc waitNodes(d: Protocol, fromNode: Node, reqId: RequestId):
Future[DiscResult[seq[Record]]] {.async, raises: [Exception, Defect].} =
## Wait for one or more nodes replies.
##
## The first reply will hold the total number of replies expected, and based
## on that, more replies will be awaited.
## If one reply is lost here (timed out), others are ignored too.
## Same counts for out of order receival.
var op = await d.waitMessage(fromNode, reqId)
if op.isSome and op.get.kind == nodes:
var res = op.get.nodes.enrs
let total = op.get.nodes.total
for i in 1 ..< total:
op = await d.waitMessage(fromNode, reqId)
if op.isSome and op.get.kind == nodes:
res.add(op.get.nodes.enrs)
else:
# No error on this as we received some nodes.
break
return ok(res)
else:
return err("Nodes message not received in time")
proc sendMessage*[T: SomeMessage](d: Protocol, toNode: Node, m: T):
RequestId {.raises: [Exception, Defect].} =
doAssert(toNode.address.isSome())
let
address = toNode.address.get()
reqId = RequestId.init(d.rng[])
message = encodeMessage(m, reqId)
let (data, nonce) = encodeMessagePacket(d.rng[], d.codec, toNode.id,
address, message)
d.registerRequest(toNode, message, nonce)
trace "Send message packet", dstId = toNode.id, address, kind = messageKind(T)
d.send(toNode, data)
return reqId
proc ping*(d: Protocol, toNode: Node):
Future[DiscResult[PongMessage]] {.async, raises: [Exception, Defect].} =
## Send a discovery ping message.
##
## Returns the received pong message or an error.
let reqId = d.sendMessage(toNode,
PingMessage(enrSeq: d.localNode.record.seqNum))
let resp = await d.waitMessage(toNode, reqId)
if resp.isSome() and resp.get().kind == pong:
d.routingTable.setJustSeen(toNode)
return ok(resp.get().pong)
else:
d.replaceNode(toNode)
return err("Pong message not received in time")
proc findNode*(d: Protocol, toNode: Node, distances: seq[uint32]):
Future[DiscResult[seq[Node]]] {.async, raises: [Exception, Defect].} =
## Send a discovery findNode message.
##
## Returns the received nodes or an error.
## Received ENRs are already validated and converted to `Node`.
let reqId = d.sendMessage(toNode, FindNodeMessage(distances: distances))
let nodes = await d.waitNodes(toNode, reqId)
if nodes.isOk:
let res = verifyNodesRecords(nodes.get(), toNode, distances)
d.routingTable.setJustSeen(toNode)
return ok(res)
else:
d.replaceNode(toNode)
return err(nodes.error)
proc talkreq*(d: Protocol, toNode: Node, protocol, request: seq[byte]):
Future[DiscResult[TalkRespMessage]] {.async, raises: [Exception, Defect].} =
## Send a discovery talkreq message.
##
## Returns the received talkresp message or an error.
let reqId = d.sendMessage(toNode,
TalkReqMessage(protocol: protocol, request: request))
let resp = await d.waitMessage(toNode, reqId)
if resp.isSome() and resp.get().kind == talkresp:
d.routingTable.setJustSeen(toNode)
return ok(resp.get().talkresp)
else:
d.replaceNode(toNode)
return err("Talk response message not received in time")
proc lookupDistances(target, dest: NodeId): seq[uint32] {.raises: [Defect].} =
let td = logDist(target, dest)
result.add(td)
var i = 1'u32
while result.len < lookupRequestLimit:
if td + i < 256:
result.add(td + i)
if td - i > 0'u32:
result.add(td - i)
inc i
proc lookupWorker(d: Protocol, destNode: Node, target: NodeId):
Future[seq[Node]] {.async, raises: [Exception, Defect].} =
let dists = lookupDistances(target, destNode.id)
var i = 0
# TODO: We can make use of the multiple distances here now.
while i < lookupRequestLimit and result.len < findNodeResultLimit:
let r = await d.findNode(destNode, @[dists[i]])
# TODO: Handle failures better. E.g. stop on different failures than timeout
if r.isOk:
# TODO: I guess it makes sense to limit here also to `findNodeResultLimit`?
result.add(r[])
inc i
for n in result:
discard d.routingTable.addNode(n)
proc lookup*(d: Protocol, target: NodeId): Future[seq[Node]]
{.async, raises: [Exception, Defect].} =
## Perform a lookup for the given target, return the closest n nodes to the
## target. Maximum value for n is `BUCKET_SIZE`.
# TODO: Sort the returned nodes on distance
# Also use unseen nodes as a form of validation.
result = d.routingTable.neighbours(target, BUCKET_SIZE, seenOnly = false)
var asked = initHashSet[NodeId]()
asked.incl(d.localNode.id)
var seen = asked
for node in result:
seen.incl(node.id)
var pendingQueries = newSeqOfCap[Future[seq[Node]]](alpha)
while true:
var i = 0
while i < result.len and pendingQueries.len < alpha:
let n = result[i]
if not asked.containsOrIncl(n.id):
pendingQueries.add(d.lookupWorker(n, target))
inc i
trace "discv5 pending queries", total = pendingQueries.len
if pendingQueries.len == 0:
break
let idx = await oneIndex(pendingQueries)
trace "Got discv5 lookup response", idx
let nodes = pendingQueries[idx].read
pendingQueries.del(idx)
for n in nodes:
if not seen.containsOrIncl(n.id):
if result.len < BUCKET_SIZE:
result.add(n)
proc lookupRandom*(d: Protocol): Future[seq[Node]]
{.async, raises:[Exception, Defect].} =
## Perform a lookup for a random target, return the closest n nodes to the
## target. Maximum value for n is `BUCKET_SIZE`.
var id: NodeId
var buf: array[sizeof(id), byte]
brHmacDrbgGenerate(d.rng[], buf)
copyMem(addr id, addr buf[0], sizeof(id))
return await d.lookup(id)
proc resolve*(d: Protocol, id: NodeId): Future[Option[Node]]
{.async, raises: [Exception, Defect].} =
## Resolve a `Node` based on provided `NodeId`.
##
## This will first look in the own routing table. If the node is known, it
## will try to contact if for newer information. If node is not known or it
## does not reply, a lookup is done to see if it can find a (newer) record of
## the node on the network.
let node = d.getNode(id)
if node.isSome():
let request = await d.findNode(node.get(), @[0'u32])
# TODO: Handle failures better. E.g. stop on different failures than timeout
if request.isOk() and request[].len > 0:
return some(request[][0])
let discovered = await d.lookup(id)
for n in discovered:
if n.id == id:
if node.isSome() and node.get().record.seqNum >= n.record.seqNum:
return node
else:
return some(n)
return node
proc revalidateNode*(d: Protocol, n: Node)
{.async, raises: [Exception, Defect].} = # TODO: Exception
let pong = await d.ping(n)
if pong.isOK():
if pong.get().enrSeq > n.record.seqNum:
# Request new ENR
let nodes = await d.findNode(n, @[0'u32])
if nodes.isOk() and nodes[].len > 0:
discard d.addNode(nodes[][0])
proc revalidateLoop(d: Protocol) {.async, raises: [Exception, Defect].} =
# TODO: General Exception raised.
try:
while true:
await sleepAsync(milliseconds(d.rng[].rand(revalidateMax)))
let n = d.routingTable.nodeToRevalidate()
if not n.isNil:
traceAsyncErrors d.revalidateNode(n)
except CancelledError:
trace "revalidateLoop canceled"
proc lookupLoop(d: Protocol) {.async, raises: [Exception, Defect].} =
# TODO: General Exception raised.
try:
# lookup self (neighbour nodes)
let selfLookup = await d.lookup(d.localNode.id)
trace "Discovered nodes in self lookup", nodes = selfLookup
while true:
let randomLookup = await d.lookupRandom()
trace "Discovered nodes in random lookup", nodes = randomLookup
debug "Total nodes in discv5 routing table", total = d.routingTable.len()
await sleepAsync(lookupInterval)
except CancelledError:
trace "lookupLoop canceled"
proc newProtocol*(privKey: PrivateKey,
externalIp: Option[ValidIpAddress], tcpPort, udpPort: Port,
localEnrFields: openarray[(string, seq[byte])] = [],
bootstrapRecords: openarray[Record] = [],
previousRecord = none[enr.Record](),
bindIp = IPv4_any(), rng = newRng()):
Protocol {.raises: [Defect].} =
# TODO: Tried adding bindPort = udpPort as parameter but that gave
# "Error: internal error: environment misses: udpPort" in nim-beacon-chain.
# Anyhow, nim-beacon-chain would also require some changes to support port
# remapping through NAT and this API is also subject to change once we
# introduce support for ipv4 + ipv6 binding/listening.
let extraFields = mapIt(localEnrFields, toFieldPair(it[0], it[1]))
# TODO:
# - Defect as is now or return a result for enr errors?
# - In case incorrect key, allow for new enr based on new key (new node id)?
var record: Record
if previousRecord.isSome():
record = previousRecord.get()
record.update(privKey, externalIp, tcpPort, udpPort,
extraFields).expect("Record within size limits and correct key")
else:
record = enr.Record.init(1, privKey, externalIp, tcpPort, udpPort,
extraFields).expect("Record within size limits")
let node = newNode(record).expect("Properly initialized record")
# TODO Consider whether this should be a Defect
doAssert rng != nil, "RNG initialization failed"
result = Protocol(
privateKey: privKey,
localNode: node,
bindAddress: Address(ip: ValidIpAddress.init(bindIp), port: udpPort),
codec: Codec(localNode: node, privKey: privKey,
sessions: Sessions.init(256)),
bootstrapRecords: @bootstrapRecords,
rng: rng)
result.routingTable.init(node, 5, rng)
proc open*(d: Protocol) {.raises: [Exception, Defect].} =
info "Starting discovery node", node = d.localNode,
bindAddress = d.bindAddress, uri = toURI(d.localNode.record)
# TODO allow binding to specific IP / IPv6 / etc
let ta = initTAddress(d.bindAddress.ip, d.bindAddress.port)
# TODO: raises `OSError` and `IOSelectorsException`, the latter which is
# object of Exception. In Nim devel this got changed to CatchableError.
d.transp = newDatagramTransport(processClient, udata = d, local = ta)
for record in d.bootstrapRecords:
debug "Adding bootstrap node", uri = toURI(record)
discard d.addNode(record)
proc start*(d: Protocol) {.raises: [Exception, Defect].} =
d.lookupLoop = lookupLoop(d)
d.revalidateLoop = revalidateLoop(d)
proc close*(d: Protocol) {.raises: [Exception, Defect].} =
doAssert(not d.transp.closed)
debug "Closing discovery node", node = d.localNode
if not d.revalidateLoop.isNil:
d.revalidateLoop.cancel()
if not d.lookupLoop.isNil:
d.lookupLoop.cancel()
d.transp.close()
proc closeWait*(d: Protocol) {.async, raises: [Exception, Defect].} =
doAssert(not d.transp.closed)
debug "Closing discovery node", node = d.localNode
if not d.revalidateLoop.isNil:
await d.revalidateLoop.cancelAndWait()
if not d.lookupLoop.isNil:
await d.lookupLoop.cancelAndWait()
await d.transp.closeWait()

2
eth/p2p/discoveryv5/sessions.nim
View File

@ -1,7 +1,7 @@
import
std/options,
stint, stew/endians2, stew/shims/net,
typesv1, node, lru
types, node, lru
export lru

101
eth/p2p/discoveryv5/types.nim
View File

@ -1,34 +1,22 @@
import
std/hashes,
stint, chronos,
eth/[keys, rlp], enr, node
stint,
eth/rlp, enr, node
{.push raises: [Defect].}
const
authTagSize* = 12
idNonceSize* = 32
aesKeySize* = 128 div 8
type
AuthTag* = array[authTagSize, byte]
IdNonce* = array[idNonceSize, byte]
AesKey* = array[aesKeySize, byte]
HandshakeKey* = object
nodeId*: NodeId
address*: string # TODO: Replace with Address, need hash
WhoareyouObj* = object
authTag*: AuthTag
idNonce*: IdNonce
recordSeq*: uint64
pubKey* {.rlpIgnore.}: Option[PublicKey]
Whoareyou* = ref WhoareyouObj
MessageKind* = enum
# TODO This is needed only to make Nim 1.0.4 happy
# TODO This is needed only to make Nim 1.2.6 happy
# Without it, the `MessageKind` type cannot be used as
# a discriminator in case objects.
unused = 0x00
@ -37,12 +25,15 @@ type
pong = 0x02
findnode = 0x03
nodes = 0x04
regtopic = 0x05
ticket = 0x06
regconfirmation = 0x07
topicquery = 0x08
talkreq = 0x05
talkresp = 0x06
regtopic = 0x07
ticket = 0x08
regconfirmation = 0x09
topicquery = 0x0A
RequestId* = uint64
RequestId* = object
id*: seq[byte]
PingMessage* = object
enrSeq*: uint64
@ -53,13 +44,27 @@ type
port*: uint16
FindNodeMessage* = object
distance*: uint32
distances*: seq[uint32]
NodesMessage* = object
total*: uint32
enrs*: seq[Record]
SomeMessage* = PingMessage or PongMessage or FindNodeMessage or NodesMessage
TalkReqMessage* = object
protocol*: seq[byte]
request*: seq[byte]
TalkRespMessage* = object
response*: seq[byte]
# Not implemented, specification is not final here.
RegTopicMessage* = object
TicketMessage* = object
RegConfirmationMessage* = object
TopicQueryMessage* = object
SomeMessage* = PingMessage or PongMessage or FindNodeMessage or NodesMessage or
TalkReqMessage or TalkRespMessage
Message* = object
reqId*: RequestId
@ -72,8 +77,19 @@ type
findNode*: FindNodeMessage
of nodes:
nodes*: NodesMessage
of talkreq:
talkreq*: TalkReqMessage
of talkresp:
talkresp*: TalkRespMessage
of regtopic:
regtopic*: RegTopicMessage
of ticket:
ticket*: TicketMessage
of regconfirmation:
regconfirmation*: RegConfirmationMessage
of topicquery:
topicquery*: TopicQueryMessage
else:
# TODO: Define the rest
discard
template messageKind*(T: typedesc[SomeMessage]): MessageKind =
@ -81,6 +97,25 @@ template messageKind*(T: typedesc[SomeMessage]): MessageKind =
elif T is PongMessage: pong
elif T is FindNodeMessage: findNode
elif T is NodesMessage: nodes
elif T is TalkReqMessage: talkreq
elif T is TalkRespMessage: talkresp
proc read*(rlp: var Rlp, T: type RequestId): T
{.raises: [ValueError, RlpError, Defect].} =
mixin read
var reqId: RequestId
reqId.id = rlp.toBytes()
if reqId.id.len > 8:
raise newException(ValueError, "RequestId is > 8 bytes")
rlp.skipElem()
reqId
proc append*(writer: var RlpWriter, value: RequestId) =
writer.append(value.id)
proc hash*(reqId: RequestId): Hash =
hash(reqId.id)
proc toBytes*(id: NodeId): array[32, byte] {.inline.} =
id.toByteArrayBE()
@ -96,23 +131,3 @@ proc hash*(address: Address): Hash {.inline.} =
proc hash*(key: HandshakeKey): Hash =
result = key.nodeId.hash !& key.address.hash
result = !$result
proc read*(rlp: var Rlp, O: type Option[Record]): O
{.raises: [ValueError, RlpError, Defect].} =
mixin read
if not rlp.isList:
raise newException(
ValueError, "Could not deserialize optional ENR, expected list")
# The discovery specification states that in case no ENR is send in the
# handshake, an empty rlp list instead should be send.
if rlp.listLen == 0:
none(Record)
else:
some(read(rlp, Record))
proc append*(writer: var RlpWriter, value: Option[Record]) =
if value.isSome:
writer.append value.get
else:
writer.startList(0)

133
eth/p2p/discoveryv5/typesv1.nim
View File

@ -1,133 +0,0 @@
import
std/hashes,
stint,
eth/rlp, enr, node
{.push raises: [Defect].}
const
aesKeySize* = 128 div 8
type
AesKey* = array[aesKeySize, byte]
HandshakeKey* = object
nodeId*: NodeId
address*: string # TODO: Replace with Address, need hash
MessageKind* = enum
# TODO This is needed only to make Nim 1.2.6 happy
# Without it, the `MessageKind` type cannot be used as
# a discriminator in case objects.
unused = 0x00
ping = 0x01
pong = 0x02
findnode = 0x03
nodes = 0x04
talkreq = 0x05
talkresp = 0x06
regtopic = 0x07
ticket = 0x08
regconfirmation = 0x09
topicquery = 0x0A
RequestId* = object
id*: seq[byte]
PingMessage* = object
enrSeq*: uint64
PongMessage* = object
enrSeq*: uint64
ip*: seq[byte]
port*: uint16
FindNodeMessage* = object
distances*: seq[uint32]
NodesMessage* = object
total*: uint32
enrs*: seq[Record]
TalkReqMessage* = object
protocol*: seq[byte]
request*: seq[byte]
TalkRespMessage* = object
response*: seq[byte]
# Not implemented, specification is not final here.
RegTopicMessage* = object
TicketMessage* = object
RegConfirmationMessage* = object
TopicQueryMessage* = object
SomeMessage* = PingMessage or PongMessage or FindNodeMessage or NodesMessage or
TalkReqMessage or TalkRespMessage
Message* = object
reqId*: RequestId
case kind*: MessageKind
of ping:
ping*: PingMessage
of pong:
pong*: PongMessage
of findnode:
findNode*: FindNodeMessage
of nodes:
nodes*: NodesMessage
of talkreq:
talkreq*: TalkReqMessage
of talkresp:
talkresp*: TalkRespMessage
of regtopic:
regtopic*: RegTopicMessage
of ticket:
ticket*: TicketMessage
of regconfirmation:
regconfirmation*: RegConfirmationMessage
of topicquery:
topicquery*: TopicQueryMessage
else:
discard
template messageKind*(T: typedesc[SomeMessage]): MessageKind =
when T is PingMessage: ping
elif T is PongMessage: pong
elif T is FindNodeMessage: findNode
elif T is NodesMessage: nodes
elif T is TalkReqMessage: talkreq
elif T is TalkRespMessage: talkresp
proc read*(rlp: var Rlp, T: type RequestId): T
{.raises: [ValueError, RlpError, Defect].} =
mixin read
var reqId: RequestId
reqId.id = rlp.toBytes()
if reqId.id.len > 8:
raise newException(ValueError, "RequestId is > 8 bytes")
rlp.skipElem()
reqId
proc append*(writer: var RlpWriter, value: RequestId) =
writer.append(value.id)
proc hash*(reqId: RequestId): Hash =
hash(reqId.id)
proc toBytes*(id: NodeId): array[32, byte] {.inline.} =
id.toByteArrayBE()
proc hash*(id: NodeId): Hash {.inline.} =
result = hashData(unsafeAddr id, sizeof(id))
# TODO: To make this work I think we also need to implement `==` due to case
# fields in object
proc hash*(address: Address): Hash {.inline.} =
hashData(unsafeAddr address, sizeof(address))
proc hash*(key: HandshakeKey): Hash =
result = key.nodeId.hash !& key.address.hash
result = !$result

29
tests/fuzzing/discoveryv5.1/fuzz_decode_message.nim
View File

@ -1,29 +0,0 @@
import
testutils/fuzzing, stew/byteutils,
eth/rlp, eth/p2p/discoveryv5/[encodingv1, typesv1]
test:
block:
let decoded = decodeMessage(payload)
if decoded.isOK():
let message = decoded.get()
var encoded: seq[byte]
case message.kind
of unused: break
of ping: encoded = encodeMessage(message.ping, message.reqId)
of pong: encoded = encodeMessage(message.pong, message.reqId)
of findNode: encoded = encodeMessage(message.findNode, message.reqId)
of nodes: encoded = encodeMessage(message.nodes, message.reqId)
of talkreq: encoded = encodeMessage(message.talkreq, message.reqId)
of talkresp: encoded = encodeMessage(message.talkresp, message.reqId)
of regtopic, ticket, regconfirmation, topicquery:
break
# This will hit assert because of issue:
# https://github.com/status-im/nim-eth/issues/255
# if encoded != payload:
# echo "payload: ", toHex(payload)
# echo "encoded: ", toHex(encoded)
# doAssert(false, "re-encoded result does not equal original payload")

29
tests/fuzzing/discoveryv5/fuzz_decode_authresp.nim
View File

@ -1,29 +0,0 @@
import
testutils/fuzzing, chronicles, stew/byteutils,
eth/rlp, eth/p2p/discoveryv5/encoding
test:
block:
# This test also includes the decoding of the ENR, so it kinda overlaps with
# the fuzz_enr test. And it will fail to decode most of the time for the
# same reasons.
let decoded = try: rlp.decode(payload, AuthResponse)
except RlpError as e:
debug "decode failed", err = e.msg
break
except ValueError as e:
debug "decode failed", err = e.msg
break
let encoded = try: rlp.encode(decoded)
except RlpError as e:
debug "decode failed", err = e.msg
doAssert(false, "decoding worked but encoding failed")
break
# This will hit assert because of issue:
# https://github.com/status-im/nim-eth/issues/255
# if encoded != payload.toOpenArray(0, encoded.len - 1):
# echo "payload: ", toHex(payload.toOpenArray(0, encoded.len - 1))
# echo "encoded: ", toHex(encoded)
# doAssert(false, "re-encoded result does not equal original payload")

2
tests/fuzzing/discoveryv5/fuzz_decode_message.nim
View File

@ -15,6 +15,8 @@ test:
of pong: encoded = encodeMessage(message.pong, message.reqId)
of findNode: encoded = encodeMessage(message.findNode, message.reqId)
of nodes: encoded = encodeMessage(message.nodes, message.reqId)
of talkreq: encoded = encodeMessage(message.talkreq, message.reqId)
of talkresp: encoded = encodeMessage(message.talkresp, message.reqId)
of regtopic, ticket, regconfirmation, topicquery:
break

0
tests/fuzzing/discoveryv5.1/fuzz_decode_packet.nim → tests/fuzzing/discoveryv5/fuzz_decode_packet.nim
View File

29
tests/fuzzing/discoveryv5/fuzz_receive.nim
View File

@ -1,29 +0,0 @@
import
testutils/fuzzing, bearssl, stew/shims/net,
eth/[keys, trie/db], eth/p2p/discoveryv5/[protocol, discovery_db],
../p2p/discv5_test_helper
var targetNode: protocol.Protocol
init:
let
rng = newRng()
privKey = PrivateKey.fromHex(
"5d2908f3f09ea1ff2e327c3f623159639b00af406e9009de5fd4b910fc34049d")[]
ip = some(ValidIpAddress.init("127.0.0.1"))
port = Port(20301)
dbb = DiscoveryDB.init(newMemoryDB())
targetNode = newProtocol(privKey, dbb, ip, port, port, rng = rng)
# Need to open socket else the response part will fail, would be nice if we
# could skip that part during fuzzing.
targetNode.open()
test:
# Some dummy address
let address = localAddress(20302)
# This is a quick and easy, high level fuzzing test and considering that the
# auth-response and the message gets encrypted, and that a handshake needs to
# be done, it will not be able to reach into testing those depths. However, it
# should still be of use hitting the more "simple" code paths (random-packet,
# whoareyou-packet, and the beginnings of other packets).
targetNode.receive(address, payload)

32
tests/fuzzing/discoveryv5/generate_auth_response.nim
View File

@ -1,32 +0,0 @@
import
std/[os, strutils],
stew/shims/net,
eth/[rlp, keys], eth/p2p/discoveryv5/[encoding, enr, types],
../fuzzing_helpers
template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
const inputsDir = sourceDir / "corpus" & DirSep
proc generate() =
let
rng = keys.newRng()
privKey = PrivateKey.random(rng[])
pubKey = PrivateKey.random(rng[]).toPublicKey()
var idNonce: IdNonce
brHmacDrbgGenerate(rng[], idNonce)
let
ephKeys = KeyPair.random(rng[])
signature = signIDNonce(privKey, idNonce, ephKeys.pubkey.toRaw)
record = enr.Record.init(1, privKey, none(ValidIpAddress), Port(9000),
Port(9000))[]
authResponse =
AuthResponse(version: 5, signature: signature.toRaw, record: some(record))
authResponseNoRecord =
AuthResponse(version: 5, signature: signature.toRaw, record: none(enr.Record))
rlp.encode(authResponse).toFile(inputsDir & "auth-response")
rlp.encode(authResponseNoRecord).toFile(inputsDir & "auth-response-no-enr")
discard existsOrCreateDir(inputsDir)
generate()

51
tests/fuzzing/discoveryv5/generate_packet.nim
View File

@ -1,51 +0,0 @@
import
std/[os, strutils],
stew/shims/net,
eth/[keys, rlp, trie/db],
eth/p2p/discoveryv5/[protocol, discovery_db, enr, node, types, encoding],
../fuzzing_helpers
template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
const inputsDir = sourceDir / "corpus" & DirSep
proc generate() =
let
rng = keys.newRng()
privKey = PrivateKey.random(rng[])
ip = some(ValidIpAddress.init("127.0.0.1"))
port = Port(20301)
dbb = DiscoveryDB.init(newMemoryDB())
d = newProtocol(privKey, dbb, ip, port, port, rng = rng)
# Same as the on in the fuzz test to have at least one working packet for
# the whoareyou-packet.
toPrivKey = PrivateKey.fromHex(
"5d2908f3f09ea1ff2e327c3f623159639b00af406e9009de5fd4b910fc34049d")[]
toRecord = enr.Record.init(1, toPrivKey,
some(ValidIpAddress.init("127.0.0.1")), Port(9000), Port(9000))[]
toNode = newNode(toRecord)[]
block: # random packet
# No handshake done obviously so a new packet will be a random packet.
let
reqId = RequestId.init(d.rng[])
message = encodeMessage(PingMessage(enrSeq: d.localNode.record.seqNum), reqId)
(data, _) = encodePacket(d.rng[], d.codec, toNode.id, toNode.address.get(),
message, challenge = nil)
data.toFile(inputsDir & "random-packet")
block: # whoareyou packet
var authTag: AuthTag
var idNonce: IdNonce
brHmacDrbgGenerate(d.rng[], authTag)
brHmacDrbgGenerate(d.rng[], idNonce)
let challenge = Whoareyou(authTag: authTag, idNonce: idNonce, recordSeq: 0)
var data = @(whoareyouMagic(toNode.id))
data.add(rlp.encode(challenge[]))
data.toFile(inputsDir & "whoareyou-packet")
discard existsOrCreateDir(inputsDir)
generate()

141
tests/p2p/test_discoveryv5.nim
View File

@ -2,21 +2,10 @@ import
std/tables,
chronos, chronicles, stint, testutils/unittests,
stew/shims/net, eth/[keys, rlp], bearssl,
eth/p2p/discoveryv5/[enr, node, routing_table],
eth/p2p/discoveryv5/[enr, node, routing_table, encoding, sessions, types],
eth/p2p/discoveryv5/protocol as discv5_protocol,
./discv5_test_helper
### This is all just temporary to support both versions
when not UseDiscv51:
import
eth/p2p/discoveryv5/[types, encoding]
proc findNode*(d: discv5_protocol.Protocol, toNode: Node, distances: seq[uint32]):
Future[DiscResult[seq[Node]]] =
if distances.len > 0:
return d.findNode(toNode, distances[0])
###
procSuite "Discovery v5 Tests":
let rng = newRng()
@ -530,64 +519,96 @@ procSuite "Discovery v5 Tests":
test = verifyNodesRecords(records, fromNode, 0'u32)
check test.len == 0
when not UseDiscv51:
proc randomPacket(rng: var BrHmacDrbgContext, tag: PacketTag): seq[byte] =
var
authTag: AuthTag
msg: array[44, byte]
asyncTest "Handshake cleanup: different ids":
# Node to test the handshakes on.
let receiveNode = initDiscoveryNode(
rng, PrivateKey.random(rng[]), localAddress(20302))
brHmacDrbgGenerate(rng, authTag)
brHmacDrbgGenerate(rng, msg)
result.add(tag)
result.add(rlp.encode(authTag))
result.add(msg)
# Create random packets with same ip but different node ids
# and "receive" them on receiveNode
let a = localAddress(20303)
for i in 0 ..< 5:
let
privKey = PrivateKey.random(rng[])
enrRec = enr.Record.init(1, privKey,
some(ValidIpAddress.init("127.0.0.1")), Port(9000),
Port(9000)).expect("Properly intialized private key")
sendNode = newNode(enrRec).expect("Properly initialized record")
var codec = Codec(localNode: sendNode, privKey: privKey, sessions: Sessions.init(5))
asyncTest "Handshake cleanup":
let node = initDiscoveryNode(
rng, PrivateKey.random(rng[]), localAddress(20302))
var tag: PacketTag
let a = localAddress(20303)
let (packet, _) = encodeMessagePacket(rng[], codec,
receiveNode.localNode.id, receiveNode.localNode.address.get(), @[])
receiveNode.receive(a, packet)
for i in 0 ..< 5:
brHmacDrbgGenerate(rng[], tag)
node.receive(a, randomPacket(rng[], tag))
# Checking different nodeIds but same address
check receiveNode.codec.handshakes.len == 5
# TODO: Could get rid of the sleep by storing the timeout future of the
# handshake
await sleepAsync(handshakeTimeout)
# Checking handshake cleanup
check receiveNode.codec.handshakes.len == 0
# Checking different nodeIds but same address
check node.codec.handshakes.len == 5
# TODO: Could get rid of the sleep by storing the timeout future of the
# handshake
await sleepAsync(handshakeTimeout)
# Checking handshake cleanup
check node.codec.handshakes.len == 0
await receiveNode.closeWait()
await node.closeWait()
asyncTest "Handshake cleanup: different ips":
# Node to test the handshakes on.
let receiveNode = initDiscoveryNode(
rng, PrivateKey.random(rng[]), localAddress(20302))
asyncTest "Handshake different address":
let node = initDiscoveryNode(
rng, PrivateKey.random(rng[]), localAddress(20302))
var tag: PacketTag
# Create random packets with same node ids but different ips
# and "receive" them on receiveNode
let
privKey = PrivateKey.random(rng[])
enrRec = enr.Record.init(1, privKey,
some(ValidIpAddress.init("127.0.0.1")), Port(9000),
Port(9000)).expect("Properly intialized private key")
sendNode = newNode(enrRec).expect("Properly initialized record")
var codec = Codec(localNode: sendNode, privKey: privKey, sessions: Sessions.init(5))
for i in 0 ..< 5:
let a = localAddress(20303 + i)
let (packet, _) = encodeMessagePacket(rng[], codec,
receiveNode.localNode.id, receiveNode.localNode.address.get(), @[])
receiveNode.receive(a, packet)
for i in 0 ..< 5:
let a = localAddress(20303 + i)
node.receive(a, randomPacket(rng[], tag))
# Checking different nodeIds but same address
check receiveNode.codec.handshakes.len == 5
# TODO: Could get rid of the sleep by storing the timeout future of the
# handshake
await sleepAsync(handshakeTimeout)
# Checking handshake cleanup
check receiveNode.codec.handshakes.len == 0
check node.codec.handshakes.len == 5
await receiveNode.closeWait()
await node.closeWait()
asyncTest "Handshake duplicates":
# Node to test the handshakes on.
let receiveNode = initDiscoveryNode(
rng, PrivateKey.random(rng[]), localAddress(20302))
asyncTest "Handshake duplicates":
let node = initDiscoveryNode(
rng, PrivateKey.random(rng[]), localAddress(20302))
var tag: PacketTag
let a = localAddress(20303)
# Create random packets with same node ids and same ips
# and "receive" them on receiveNode
let
a = localAddress(20303)
privKey = PrivateKey.random(rng[])
enrRec = enr.Record.init(1, privKey,
some(ValidIpAddress.init("127.0.0.1")), Port(9000),
Port(9000)).expect("Properly intialized private key")
sendNode = newNode(enrRec).expect("Properly initialized record")
var codec = Codec(localNode: sendNode, privKey: privKey, sessions: Sessions.init(5))
for i in 0 ..< 5:
node.receive(a, randomPacket(rng[], tag))
var firstRequestNonce: AESGCMNonce
for i in 0 ..< 5:
let (packet, requestNonce) = encodeMessagePacket(rng[], codec,
receiveNode.localNode.id, receiveNode.localNode.address.get(), @[])
receiveNode.receive(a, packet)
if i == 0:
firstRequestNonce = requestNonce
# Checking handshake duplicates
check node.codec.handshakes.len == 1
# Check handshake duplicates
check receiveNode.codec.handshakes.len == 1
# Check if it is for the first packet that a handshake is stored
let key = HandShakeKey(nodeId: sendNode.id, address: $a)
check receiveNode.codec.handshakes[key].whoareyouData.requestNonce ==
firstRequestNonce
# TODO: add check that gets the Whoareyou value and checks if its authTag
# is that of the first packet.
await node.closeWait()
await receiveNode.closeWait()

2
tests/p2p/test_discv51_encoding.nim → tests/p2p/test_discoveryv5_encoding.nim
View File

@ -2,7 +2,7 @@ import
std/[unittest, options, sequtils, tables],
stint, stew/byteutils, stew/shims/net,
eth/[rlp, keys],
eth/p2p/discoveryv5/[typesv1, encodingv1, enr, node, sessions]
eth/p2p/discoveryv5/[types, encoding, enr, node, sessions]
let rng = newRng()

269
tests/p2p/test_discv5_encoding.nim
View File

@ -1,269 +0,0 @@
import
std/[unittest, options, sequtils],
stint, stew/byteutils, stew/shims/net,
eth/[rlp, keys] , eth/p2p/discoveryv5/[types, encoding, enr, node]
# According to test vectors:
# https://github.com/ethereum/devp2p/blob/master/discv5/discv5-wire-test-vectors.md
let rng = newRng()
suite "Discovery v5 Packet Encodings":
# TODO: These tests are currently not completely representative for the code
# and thus will not necessarily notice failures. Refactor/restructure code
# where possible to make this more useful.
test "Random Packet":
const
# input
tag = "0x0101010101010101010101010101010101010101010101010101010101010101"
authTag = "0x020202020202020202020202"
randomData = "0x0404040404040404040404040404040404040404040404040404040404040404040404040404040404040404"
# expected output
randomPacketRlp = "0x01010101010101010101010101010101010101010101010101010101010101018c0202020202020202020202020404040404040404040404040404040404040404040404040404040404040404040404040404040404040404"
var data: seq[byte]
data.add(hexToByteArray[tagSize](tag))
data.add(rlp.encode(hexToByteArray[authTagSize](authTag)))
data.add(hexToSeqByte(randomData))
check data == hexToSeqByte(randomPacketRlp)
test "WHOAREYOU Packet":
const
# input
magic = "0x0101010101010101010101010101010101010101010101010101010101010101"
token = "0x020202020202020202020202"
idNonce = "0x0303030303030303030303030303030303030303030303030303030303030303"
enrSeq = 0x01'u64
# expected output
whoareyouPacketRlp = "0x0101010101010101010101010101010101010101010101010101010101010101ef8c020202020202020202020202a0030303030303030303030303030303030303030303030303030303030303030301"
let challenge = Whoareyou(authTag: hexToByteArray[authTagSize](token),
idNonce: hexToByteArray[idNonceSize](idNonce),
recordSeq: enrSeq)
var data = hexToSeqByte(magic)
data.add(rlp.encode(challenge[]))
check data == hexToSeqByte(whoareyouPacketRlp)
test "Authenticated Message Packet":
const
# input
tag = "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903"
authTag = "0x27b5af763c446acd2749fe8e"
idNonce = "0xe551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c65"
ephemeralPubkey = "0xb35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81"
authRespCiphertext = "0x570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852"
messageCiphertext = "0xa5d12a2d94b8ccb3ba55558229867dc13bfa3648"
# expected output
authMessageRlp = "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903f8cc8c27b5af763c446acd2749fe8ea0e551b1c44264ab92bc0b3c9b26293e1ba4fed9128f3c3645301e8e119f179c658367636db840b35608c01ee67edff2cffa424b219940a81cf2fb9b66068b1cf96862a17d353e22524fbdcdebc609f85cbd58ebe7a872b01e24a3829b97dd5875e8ffbc4eea81b856570fbf23885c674867ab00320294a41732891457969a0f14d11c995668858b2ad731aa7836888020e2ccc6e0e5776d0d4bc4439161798565a4159aa8620992fb51dcb275c4f755c8b8030c82918898f1ac387f606852a5d12a2d94b8ccb3ba55558229867dc13bfa3648"
let authHeader = AuthHeader(auth: hexToByteArray[authTagSize](authTag),
idNonce: hexToByteArray[idNonceSize](idNonce),
scheme: authSchemeName,
ephemeralKey: hexToByteArray[64](ephemeralPubkey),
response: hexToSeqByte(authRespCiphertext))
var data: seq[byte]
data.add(hexToSeqByte(tag))
data.add(rlp.encode(authHeader))
data.add(hexToSeqByte(messageCiphertext))
check data == hexToSeqByte(authMessageRlp)
test "Message Packet":
const
# input
tag = "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903"
authTag = "0x27b5af763c446acd2749fe8e"
randomData = "0xa5d12a2d94b8ccb3ba55558229867dc13bfa3648"
# expected output
messageRlp = "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f421079038c27b5af763c446acd2749fe8ea5d12a2d94b8ccb3ba55558229867dc13bfa3648"
var data: seq[byte]
data.add(hexToByteArray[tagSize](tag))
data.add(rlp.encode(hexToByteArray[authTagSize](authTag)))
data.add(hexToSeqByte(randomData))
check data == hexToSeqByte(messageRlp)
suite "Discovery v5 Protocol Message Encodings":
test "Ping Request":
var p: PingMessage
p.enrSeq = 1
var reqId: RequestId = 1
check encodeMessage(p, reqId).toHex == "01c20101"
test "Pong Response":
var p: PongMessage
p.enrSeq = 1
p.port = 5000
p.ip = @[127.byte, 0, 0, 1]
var reqId: RequestId = 1
check encodeMessage(p, reqId).toHex == "02ca0101847f000001821388"
test "FindNode Request":
var p: FindNodeMessage
p.distance = 0x0100
var reqId: RequestId = 1
check encodeMessage(p, reqId).toHex == "03c401820100"
test "Nodes Response (empty)":
var p: NodesMessage
p.total = 0x1
var reqId: RequestId = 1
check encodeMessage(p, reqId).toHex == "04c30101c0"
test "Nodes Response (multiple)":
var p: NodesMessage
p.total = 0x1
var e1, e2: Record
check e1.fromURI("enr:-HW4QBzimRxkmT18hMKaAL3IcZF1UcfTMPyi3Q1pxwZZbcZVRI8DC5infUAB_UauARLOJtYTxaagKoGmIjzQxO2qUygBgmlkgnY0iXNlY3AyNTZrMaEDymNMrg1JrLQB2KTGtv6MVbcNEVv0AHacwUAPMljNMTg")
check e2.fromURI("enr:-HW4QNfxw543Ypf4HXKXdYxkyzfcxcO-6p9X986WldfVpnVTQX1xlTnWrktEWUbeTZnmgOuAY_KUhbVV1Ft98WoYUBMBgmlkgnY0iXNlY3AyNTZrMaEDDiy3QkHAxPyOgWbxp5oF1bDdlYE6dLCUUp8xfVw50jU")
p.enrs = @[e1, e2]
var reqId: RequestId = 1
check encodeMessage(p, reqId).toHex == "04f8f20101f8eef875b8401ce2991c64993d7c84c29a00bdc871917551c7d330fca2dd0d69c706596dc655448f030b98a77d4001fd46ae0112ce26d613c5a6a02a81a6223cd0c4edaa53280182696482763489736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138f875b840d7f1c39e376297f81d7297758c64cb37dcc5c3beea9f57f7ce9695d7d5a67553417d719539d6ae4b445946de4d99e680eb8063f29485b555d45b7df16a1850130182696482763489736563703235366b31a1030e2cb74241c0c4fc8e8166f1a79a05d5b0dd95813a74b094529f317d5c39d235"
suite "Discovery v5 Cryptographic Primitives":
test "ECDH":
const
# input
publicKey = "0x9961e4c2356d61bedb83052c115d311acb3a96f5777296dcf297351130266231503061ac4aaee666073d7e5bc2c80c3f5c5b500c1cb5fd0a76abbb6b675ad157"
secretKey = "0xfb757dc581730490a1d7a00deea65e9b1936924caaea8f44d476014856b68736"
# expected output
sharedSecret = "0x033b11a2a1f214567e1537ce5e509ffd9b21373247f2a3ff6841f4976f53165e7e"
let
pub = PublicKey.fromHex(publicKey)[]
priv = PrivateKey.fromHex(secretKey)[]
let eph = ecdhRawFull(priv, pub)
check:
eph.data == hexToSeqByte(sharedSecret)
test "Key Derivation":
# const
# # input
# secretKey = "0x02a77e3aa0c144ae7c0a3af73692b7d6e5b7a2fdc0eda16e8d5e6cb0d08e88dd04"
# nodeIdA = "0xa448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"
# nodeIdB = "0x885bba8dfeddd49855459df852ad5b63d13a3fae593f3f9fa7e317fd43651409"
# idNonce = "0x0101010101010101010101010101010101010101010101010101010101010101"
# # expected output
# initiatorKey = "0x238d8b50e4363cf603a48c6cc3542967"
# recipientKey = "0xbebc0183484f7e7ca2ac32e3d72c8891"
# authRespKey = "0xe987ad9e414d5b4f9bfe4ff1e52f2fae"
# Code doesn't allow to start from shared `secretKey`, but only from the
# public and private key. Would require pulling `ecdhAgree` out of
# `deriveKeys`
skip()
test "Nonce Signing":
const
# input
idNonce = "0xa77e3aa0c144ae7c0a3af73692b7d6e5b7a2fdc0eda16e8d5e6cb0d08e88dd04"
ephemeralKey = "0x9961e4c2356d61bedb83052c115d311acb3a96f5777296dcf297351130266231503061ac4aaee666073d7e5bc2c80c3f5c5b500c1cb5fd0a76abbb6b675ad157"
localSecretKey = "0xfb757dc581730490a1d7a00deea65e9b1936924caaea8f44d476014856b68736"
# expected output
idNonceSig = "0xc5036e702a79902ad8aa147dabfe3958b523fd6fa36cc78e2889b912d682d8d35fdea142e141f690736d86f50b39746ba2d2fc510b46f82ee08f08fd55d133a4"
let
privKey = PrivateKey.fromHex(localSecretKey)[]
signature = signIDNonce(privKey, hexToByteArray[idNonceSize](idNonce),
hexToByteArray[64](ephemeralKey))
check signature.toRaw() == hexToByteArray[64](idNonceSig)
test "Encryption/Decryption":
const
# input
encryptionKey = "0x9f2d77db7004bf8a1a85107ac686990b"
nonce = "0x27b5af763c446acd2749fe8e"
pt = "0x01c20101"
ad = "0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903"
# expected output
messageCiphertext = "0xa5d12a2d94b8ccb3ba55558229867dc13bfa3648"
let encrypted = encryptGCM(hexToByteArray[aesKeySize](encryptionKey),
hexToByteArray[authTagSize](nonce),
hexToSeqByte(pt),
hexToByteArray[tagSize](ad))
check encrypted == hexToSeqByte(messageCiphertext)
test "Authentication Header and Encrypted Message Generation":
# Can't work directly with the provided shared secret as keys are derived
# inside makeAuthHeader, and passed on one call up.
# The encryption of the auth-resp-pt uses one of these keys, as does the
# encryption of the message itself. So the whole test depends on this.
skip()
suite "Discovery v5 Additional":
test "Encryption/Decryption":
let
encryptionKey = hexToByteArray[aesKeySize]("0x9f2d77db7004bf8a1a85107ac686990b")
nonce = hexToByteArray[authTagSize]("0x27b5af763c446acd2749fe8e")
ad = hexToByteArray[tagSize]("0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903")
pt = hexToSeqByte("0xa1")
let ct = encryptGCM(encryptionKey, nonce, pt, ad)
let decrypted = decryptGCM(encryptionKey, nonce, ct, ad)
check decrypted.get() == pt
test "Decryption":
let
encryptionKey = hexToByteArray[aesKeySize]("0x9f2d77db7004bf8a1a85107ac686990b")
nonce = hexToByteArray[authTagSize]("0x27b5af763c446acd2749fe8e")
ad = hexToByteArray[tagSize]("0x93a7400fa0d6a694ebc24d5cf570f65d04215b6ac00757875e3f3a5f42107903")
pt = hexToSeqByte("0x01c20101")
ct = hexToSeqByte("0xa5d12a2d94b8ccb3ba55558229867dc13bfa3648")
# valid case
check decryptGCM(encryptionKey, nonce, ct, ad).get() == pt
# invalid tag/data sizes
var invalidCipher: seq[byte] = @[]
check decryptGCM(encryptionKey, nonce, invalidCipher, ad).isNone()
invalidCipher = repeat(byte(4), gcmTagSize)
check decryptGCM(encryptionKey, nonce, invalidCipher, ad).isNone()
# invalid tag/data itself
invalidCipher = repeat(byte(4), gcmTagSize + 1)
check decryptGCM(encryptionKey, nonce, invalidCipher, ad).isNone()
test "AuthHeader encode/decode":
let
privKey = PrivateKey.random(rng[])
enrRec = enr.Record.init(1, privKey, none(ValidIpAddress), Port(9000),
Port(9000)).expect("Properly intialized private key")
node = newNode(enrRec).expect("Properly initialized record")
nonce = hexToByteArray[authTagSize]("0x27b5af763c446acd2749fe8e")
pubKey = PrivateKey.random(rng[]).toPublicKey()
nodeId = pubKey.toNodeId()
idNonce = hexToByteArray[idNonceSize](
"0xa77e3aa0c144ae7c0a3af73692b7d6e5b7a2fdc0eda16e8d5e6cb0d08e88dd04")
c = Codec(localNode: node, privKey: privKey)
block: # With ENR
let
whoareyou = Whoareyou(idNonce: idNonce, recordSeq: 0, pubKey: some(pubKey))
(auth, _) = encodeAuthHeader(rng[], c, nodeId, nonce, whoareyou)
var rlp = rlpFromBytes(auth)
let authHeader = rlp.read(AuthHeader)
var newNode: Node
let secrets = c.decodeAuthResp(privKey.toPublicKey().toNodeId(),
authHeader, whoareyou, newNode)
block: # Without ENR
let
whoareyou = Whoareyou(idNonce: idNonce, recordSeq: 1, pubKey: some(pubKey))
(auth, _) = encodeAuthHeader(rng[], c, nodeId, nonce, whoareyou)
var rlp = rlpFromBytes(auth)
let authHeader = rlp.read(AuthHeader)
var newNode: Node
let secrets = c.decodeAuthResp(privKey.toPublicKey().toNodeId(),
authHeader, whoareyou, newNode)
# TODO: Test cases with invalid nodeId and invalid signature, the latter
# is in the current code structure rather difficult and would need some
# helper proc.