status-im
/
nim-eth
mirror of https://github.com/status-im/nim-eth.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Rlp experimental (#227) 

* rlp: remove experimental features

* avoid range library

* trie: avoid reference-unsafe bitrange type
This commit is contained in:
Jacek Sieka 2020-04-20 20:14:39 +02:00 committed by GitHub
parent 1646d78d83
commit fd6caa0fdc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
56 changed files with 1032 additions and 1025 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
doc/rlp.md
View File

@ -10,15 +10,9 @@ and [Wiki](https://github.com/ethereum/wiki/wiki/RLP).
### Reading RLP data
The `Rlp` type provided by this library represents a cursor over an RLP-encoded
byte stream. Before instantiating such a cursor, you must convert your
input data a `BytesRange` value provided by the [nim-ranges][RNG] library,
which represents an immutable and thus cheap-to-copy sub-range view over an
underlying `seq[byte]` instance:
[RNG]: https://github.com/status-im/nim-ranges
byte stream.
``` nim
proc rlpFromBytes*(data: BytesRange): Rlp
proc rlpFromBytes*(data: openArray[byte]): Rlp
```
### Streaming API
@ -67,7 +61,7 @@ type
RlpNode* = object
case kind*: RlpNodeType
of rlpBlob:
bytes*: BytesRange
bytes*: seq[byte]
of rlpList:
elems*: seq[RlpNode]
```
@ -86,7 +80,7 @@ proc inspect*(self: Rlp, indent = 0): string
The `RlpWriter` type can be used to encode RLP data. Instances are created
with the `initRlpWriter` proc. This should be followed by one or more calls
to `append` which is overloaded to accept arbitrary values. Finally, you can
call `finish` to obtain the final `BytesRange`.
call `finish` to obtain the final `seq[byte]`.
If the end result should be a RLP list of particular length, you can replace
the initial call to `initRlpWriter` with `initRlpList(n)`. Calling `finish`

33
doc/trie.md
View File

@ -62,12 +62,9 @@ The primary API for Binary-trie is `set` and `get`.
* set(key, value) --- _store a value associated with a key_
* get(key): value --- _get a value using a key_
Both `key` and `value` are of `BytesRange` type. And they cannot have zero length.
You can also use convenience API `get` and `set` which accepts
`Bytes` or `string` (a `string` is conceptually wrong in this context
and may costlier than a `BytesRange`, but it is good for testing purpose).
Both `key` and `value` are of `seq[byte]` type. And they cannot have zero length.
Getting a non-existent key will return zero length BytesRange.
Getting a non-existent key will return zero length seq[byte].
Binary-trie also provide dictionary syntax API for `set` and `get`.
* trie[key] = value -- same as `set`
@ -81,11 +78,11 @@ Additional APIs are:
that starts with the same key prefix
* rootNode() -- get root node
* rootNode(node) -- replace the root node
* getRootHash(): `KeccakHash` with `BytesRange` type
* getRootHash(): `KeccakHash` with `seq[byte]` type
* getDB(): `DB` -- get flat-db pointer
Constructor API:
* initBinaryTrie(DB, rootHash[optional]) -- rootHash has `BytesRange` or KeccakHash type
* initBinaryTrie(DB, rootHash[optional]) -- rootHash has `seq[byte]` or KeccakHash type
* init(BinaryTrie, DB, rootHash[optional])
Normally you would not set the rootHash when constructing an empty Binary-trie.
@ -103,17 +100,17 @@ var db = newMemoryDB()
var trie = initBinaryTrie(db)
trie.set("key1", "value1")
trie.set("key2", "value2")
doAssert trie.get("key1") == "value1".toRange
doAssert trie.get("key2") == "value2".toRange
doAssert trie.get("key1") == "value1".toBytes
doAssert trie.get("key2") == "value2".toBytes
# delete all subtrie with key prefixes "key"
trie.deleteSubtrie("key")
doAssert trie.get("key1") == zeroBytesRange
doAssert trie.get("key2") == zeroBytesRange
doAssert trie.get("key1") == []
doAssert trie.get("key2") == []]
trie["moon"] = "sun"
doAssert "moon" in trie
doAssert trie["moon"] == "sun".toRange
doAssert trie["moon"] == "sun".toBytes
```
Remember, `set` and `get` are trie operations. A single `set` operation may invoke
@ -142,12 +139,12 @@ The branch utils consist of these API:
* getTrieNodes(DB; nodeHash): branch
`keyPrefix`, `key`, and `value` are bytes container with length greater than zero.
They can be BytesRange, Bytes, and string(again, for convenience and testing purpose).
They can be openArray[byte].
`rootHash` and `nodeHash` also bytes container,
but they have constraint: must be 32 bytes in length, and it must be a keccak_256 hash value.
`branch` is a list of nodes, or in this case a seq[BytesRange].
`branch` is a list of nodes, or in this case a `seq[seq[byte]]`.
A list? yes, the structure is stored along with the encoded node.
Therefore a list is enough to reconstruct the entire trie/branch.
@ -303,14 +300,14 @@ let
trie.set(key1, "value1")
trie.set(key2, "value2")
doAssert trie.get(key1) == "value1".toRange
doAssert trie.get(key2) == "value2".toRange
doAssert trie.get(key1) == "value1".toBytes
doAssert trie.get(key2) == "value2".toBytes
trie.delete(key1)
doAssert trie.get(key1) == zeroBytesRange
doAssert trie.get(key1) == []
trie.delete(key2)
doAssert trie[key2] == zeroBytesRange
doAssert trie[key2] == []
```
Remember, `set` and `get` are trie operations. A single `set` operation may invoke

4
eth/common/eth_types.nim
View File

@ -244,7 +244,7 @@ proc read*(rlp: var Rlp, T: typedesc[StUint]): T {.inline.} =
if bytes.len > 0:
# be sure the amount of bytes matches the size of the stint
if bytes.len <= sizeof(result):
result.initFromBytesBE(bytes.toOpenArray)
result.initFromBytesBE(bytes)
else:
raise newException(RlpTypeMismatch, "Unsigned integer expected, but the source RLP has the wrong length")
else:
@ -375,7 +375,7 @@ method getTrieDB*(db: AbstractChainDB): TrieDatabaseRef {.base, gcsafe.} =
method getCodeByHash*(db: AbstractChainDB, hash: KeccakHash): Blob {.base, gcsafe.} =
notImplemented()
method getSetting*(db: AbstractChainDB, key: string): Bytes {.base, gcsafe.} =
method getSetting*(db: AbstractChainDB, key: string): seq[byte] {.base, gcsafe.} =
notImplemented()
method setSetting*(db: AbstractChainDB, key: string, val: openarray[byte]) {.base, gcsafe.} =

5
eth/common/state_accessors.nim
View File

@ -6,7 +6,7 @@ proc getAccount*(db: TrieDatabaseRef,
rootHash: KeccakHash,
account: EthAddress): Account =
let trie = initSecureHexaryTrie(db, rootHash)
let data = trie.get(unnecessary_OpenArrayToRange account)
let data = trie.get(account)
if data.len > 0:
result = rlp.decode(data, Account)
else:
@ -21,6 +21,3 @@ proc getContractCode*(chain: AbstractChainDB, req: ContractCodeRequest): Blob {.
proc getStorageNode*(chain: AbstractChainDB, hash: KeccakHash): Blob =
let db = chain.getTrieDB
return db.get(hash.data)
# let trie = initSecureHexaryTrie(db, emptyRlpHash) # TODO emptyRlpHash is not correct here
# return trie.get(unnecessary_OpenArrayToRange hash.data)

44
eth/p2p/auth.nim
View File

@ -14,7 +14,7 @@
import eth/[keys, rlp], nimcrypto
import ecies
import stew/[byteutils, endians2, results]
import stew/[byteutils, endians2, objects, results]
export results
@ -362,9 +362,6 @@ proc decodeAuthMessageV4(h: var Handshake, m: openarray[byte]): AuthResult[void]
proc decodeAuthMessageEip8(h: var Handshake, m: openarray[byte]): AuthResult[void] =
## Decodes EIP-8 AuthMessage.
var
nonce: Nonce
let size = uint16.fromBytesBE(m)
h.expectedLength = int(size) + 2
if h.expectedLength > len(m):
@ -374,7 +371,7 @@ proc decodeAuthMessageEip8(h: var Handshake, m: openarray[byte]): AuthResult[voi
toa(m, 0, 2)).isErr:
return err(EciesError)
try:
var reader = rlpFromBytes(buffer.toRange())
var reader = rlpFromBytes(buffer)
if not reader.isList() or reader.listLen() < 4:
return err(InvalidAuth)
if reader.listElem(0).blobLen != RawSignatureSize:
@ -385,29 +382,28 @@ proc decodeAuthMessageEip8(h: var Handshake, m: openarray[byte]): AuthResult[voi
return err(InvalidAuth)
if reader.listElem(3).blobLen != 1:
return err(InvalidAuth)
var signatureBr = reader.listElem(0).toBytes()
var pubkeyBr = reader.listElem(1).toBytes()
var nonceBr = reader.listElem(2).toBytes()
var versionBr = reader.listElem(3).toBytes()
let
signatureBr = reader.listElem(0).toBytes()
pubkeyBr = reader.listElem(1).toBytes()
nonceBr = reader.listElem(2).toBytes()
versionBr = reader.listElem(3).toBytes()
let pubkey =
? PublicKey.fromRaw(pubkeyBr.toOpenArray()).mapErrTo(InvalidPubKey)
copyMem(addr nonce[0], nonceBr.baseAddr, KeyLength)
let
signature = ? Signature.fromRaw(signatureBr).mapErrTo(SignatureError)
pubkey = ? PublicKey.fromRaw(pubkeyBr).mapErrTo(InvalidPubKey)
nonce = toArray(KeyLength, nonceBr)
var secret = ? ecdhRaw(h.host.seckey, pubkey).mapErrTo(EcdhError)
let xornonce = nonce xor secret.data
secret.clear()
let signature =
? Signature.fromRaw(signatureBr.toOpenArray()).mapErrTo(SignatureError)
h.remoteEPubkey =
? recover(signature, SkMessage(data: xornonce)).mapErrTo(SignatureError)
h.initiatorNonce = nonce
h.remoteHPubkey = pubkey
h.version = cast[ptr byte](versionBr.baseAddr)[]
h.version = versionBr[0]
ok()
except CatchableError:
err(RlpError)
@ -424,7 +420,7 @@ proc decodeAckMessageEip8*(h: var Handshake, m: openarray[byte]): AuthResult[voi
toa(m, 0, 2)).isErr:
return err(EciesError)
try:
var reader = rlpFromBytes(buffer.toRange())
var reader = rlpFromBytes(buffer)
if not reader.isList() or reader.listLen() < 3:
return err(InvalidAck)
if reader.listElem(0).blobLen != RawPublicKeySize:
@ -433,14 +429,14 @@ proc decodeAckMessageEip8*(h: var Handshake, m: openarray[byte]): AuthResult[voi
return err(InvalidAck)
if reader.listElem(2).blobLen != 1:
return err(InvalidAck)
let pubkeyBr = reader.listElem(0).toBytes()
let nonceBr = reader.listElem(1).toBytes()
let versionBr = reader.listElem(2).toBytes()
h.remoteEPubkey =
? PublicKey.fromRaw(pubkeyBr.toOpenArray()).mapErrTo(InvalidPubKey)
let
pubkeyBr = reader.listElem(0).toBytes()
nonceBr = reader.listElem(1).toBytes()
versionBr = reader.listElem(2).toBytes()
copyMem(addr h.responderNonce[0], nonceBr.baseAddr, KeyLength)
h.version = cast[ptr byte](versionBr.baseAddr)[]
h.remoteEPubkey = ? PublicKey.fromRaw(pubkeyBr).mapErrTo(InvalidPubKey)
h.responderNonce = toArray(KeyLength, nonceBr)
h.version = versionBr[0]
ok()
except CatchableError:

54
eth/p2p/discovery.nim
View File

@ -13,7 +13,7 @@ import
chronos, stint, nimcrypto, chronicles,
eth/[keys, rlp],
kademlia, enode,
stew/results
stew/[objects, results]
export
Node, results
@ -53,27 +53,27 @@ const MinListLen: array[CommandId, int] = [4, 3, 2, 2]
proc append*(w: var RlpWriter, a: IpAddress) =
case a.family
of IpAddressFamily.IPv6:
w.append(a.address_v6.toMemRange)
w.append(a.address_v6)
of IpAddressFamily.IPv4:
w.append(a.address_v4.toMemRange)
w.append(a.address_v4)
proc append(w: var RlpWriter, p: Port) {.inline.} = w.append(p.int)
proc append(w: var RlpWriter, pk: PublicKey) {.inline.} = w.append(pk.toRaw())
proc append(w: var RlpWriter, h: MDigest[256]) {.inline.} = w.append(h.data)
proc pack(cmdId: CommandId, payload: BytesRange, pk: PrivateKey): Bytes =
proc pack(cmdId: CommandId, payload: openArray[byte], pk: PrivateKey): seq[byte] =
## Create and sign a UDP message to be sent to a remote node.
##
## See https://github.com/ethereum/devp2p/blob/master/rlpx.md#node-discovery for information on
## how UDP packets are structured.
# TODO: There is a lot of unneeded allocations here
let encodedData = @[cmdId.byte] & payload.toSeq()
let encodedData = @[cmdId.byte] & @payload
let signature = @(pk.sign(encodedData).tryGet().toRaw())
let msgHash = keccak256.digest(signature & encodedData)
result = @(msgHash.data) & signature & encodedData
proc validateMsgHash(msg: Bytes): DiscResult[MDigest[256]] =
proc validateMsgHash(msg: openArray[byte]): DiscResult[MDigest[256]] =
if msg.len > HEAD_SIZE:
var ret: MDigest[256]
ret.data[0 .. ^1] = msg.toOpenArray(0, ret.data.high)
@ -90,7 +90,7 @@ proc recoverMsgPublicKey(msg: openArray[byte]): DiscResult[PublicKey] =
let sig = ? Signature.fromRaw(msg.toOpenArray(MAC_SIZE, HEAD_SIZE))
recover(sig, msg.toOpenArray(HEAD_SIZE, msg.high))
proc unpack(msg: Bytes): tuple[cmdId: CommandId, payload: Bytes] =
proc unpack(msg: openArray[byte]): tuple[cmdId: CommandId, payload: seq[byte]] =
# Check against possible RangeError
if msg[HEAD_SIZE].int < CommandId.low.ord or
msg[HEAD_SIZE].int > CommandId.high.ord:
@ -112,14 +112,14 @@ proc send(d: DiscoveryProtocol, n: Node, data: seq[byte]) =
proc sendPing*(d: DiscoveryProtocol, n: Node): seq[byte] =
let payload = rlp.encode((PROTO_VERSION, d.address, n.node.address,
expiration())).toRange
expiration()))
let msg = pack(cmdPing, payload, d.privKey)
result = msg[0 ..< MAC_SIZE]
trace ">>> ping ", n
d.send(n, msg)
proc sendPong*(d: DiscoveryProtocol, n: Node, token: MDigest[256]) =
let payload = rlp.encode((n.node.address, token, expiration())).toRange
let payload = rlp.encode((n.node.address, token, expiration()))
let msg = pack(cmdPong, payload, d.privKey)
trace ">>> pong ", n
d.send(n, msg)
@ -127,7 +127,7 @@ proc sendPong*(d: DiscoveryProtocol, n: Node, token: MDigest[256]) =
proc sendFindNode*(d: DiscoveryProtocol, n: Node, targetNodeId: NodeId) =
var data: array[64, byte]
data[32 .. ^1] = targetNodeId.toByteArrayBE()
let payload = rlp.encode((data, expiration())).toRange
let payload = rlp.encode((data, expiration()))
let msg = pack(cmdFindNode, payload, d.privKey)
trace ">>> find_node to ", n#, ": ", msg.toHex()
d.send(n, msg)
@ -140,7 +140,7 @@ proc sendNeighbours*(d: DiscoveryProtocol, node: Node, neighbours: seq[Node]) =
template flush() =
block:
let payload = rlp.encode((nodes, expiration())).toRange
let payload = rlp.encode((nodes, expiration()))
let msg = pack(cmdNeighbours, payload, d.privkey)
trace "Neighbours to", node, nodes
d.send(node, msg)
@ -169,14 +169,14 @@ proc recvPing(d: DiscoveryProtocol, node: Node,
msgHash: MDigest[256]) {.inline.} =
d.kademlia.recvPing(node, msgHash)
proc recvPong(d: DiscoveryProtocol, node: Node, payload: Bytes) {.inline.} =
let rlp = rlpFromBytes(payload.toRange)
let tok = rlp.listElem(1).toBytes().toSeq()
proc recvPong(d: DiscoveryProtocol, node: Node, payload: seq[byte]) {.inline.} =
let rlp = rlpFromBytes(payload)
let tok = rlp.listElem(1).toBytes()
d.kademlia.recvPong(node, tok)
proc recvNeighbours(d: DiscoveryProtocol, node: Node,
payload: Bytes) {.inline.} =
let rlp = rlpFromBytes(payload.toRange)
payload: seq[byte]) {.inline.} =
let rlp = rlpFromBytes(payload)
let neighboursList = rlp.listElem(0)
let sz = neighboursList.listLen()
@ -187,18 +187,18 @@ proc recvNeighbours(d: DiscoveryProtocol, node: Node,
var ip: IpAddress
case ipBlob.len
of 4:
ip = IpAddress(family: IpAddressFamily.IPv4)
copyMem(addr ip.address_v4[0], baseAddr ipBlob, 4)
ip = IpAddress(
family: IpAddressFamily.IPv4, address_v4: toArray(4, ipBlob))
of 16:
ip = IpAddress(family: IpAddressFamily.IPv6)
copyMem(addr ip.address_v6[0], baseAddr ipBlob, 16)
ip = IpAddress(
family: IpAddressFamily.IPv6, address_v6: toArray(16, ipBlob))
else:
error "Wrong ip address length!"
continue
let udpPort = n.listElem(1).toInt(uint16).Port
let tcpPort = n.listElem(2).toInt(uint16).Port
let pk = PublicKey.fromRaw(n.listElem(3).toBytes.toOpenArray())
let pk = PublicKey.fromRaw(n.listElem(3).toBytes)
if pk.isErr:
warn "Could not parse public key"
continue
@ -206,24 +206,24 @@ proc recvNeighbours(d: DiscoveryProtocol, node: Node,
neighbours.add(newNode(pk[], Address(ip: ip, udpPort: udpPort, tcpPort: tcpPort)))
d.kademlia.recvNeighbours(node, neighbours)
proc recvFindNode(d: DiscoveryProtocol, node: Node, payload: Bytes) {.inline, gcsafe.} =
let rlp = rlpFromBytes(payload.toRange)
proc recvFindNode(d: DiscoveryProtocol, node: Node, payload: openArray[byte]) {.inline, gcsafe.} =
let rlp = rlpFromBytes(payload)
trace "<<< find_node from ", node
let rng = rlp.listElem(0).toBytes
# Check for pubkey len
if rng.len == 64:
let nodeId = readUIntBE[256](rng[32 .. ^1].toOpenArray())
let nodeId = readUIntBE[256](rng[32 .. ^1])
d.kademlia.recvFindNode(node, nodeId)
else:
trace "Invalid target public key received"
proc expirationValid(cmdId: CommandId, rlpEncodedPayload: seq[byte]):
proc expirationValid(cmdId: CommandId, rlpEncodedPayload: openArray[byte]):
bool {.inline, raises:[DiscProtocolError, RlpError].} =
## Can only raise `DiscProtocolError` and all of `RlpError`
# Check if there is a payload
if rlpEncodedPayload.len <= 0:
raise newException(DiscProtocolError, "RLP stream is empty")
let rlp = rlpFromBytes(rlpEncodedPayload.toRange)
let rlp = rlpFromBytes(rlpEncodedPayload)
# Check payload is an RLP list and if the list has the minimum items required
# for this packet type
if rlp.isList and rlp.listLen >= MinListLen[cmdId]:
@ -233,7 +233,7 @@ proc expirationValid(cmdId: CommandId, rlpEncodedPayload: seq[byte]):
else:
raise newException(DiscProtocolError, "Invalid RLP list for this packet id")
proc receive*(d: DiscoveryProtocol, a: Address, msg: Bytes) {.gcsafe.} =
proc receive*(d: DiscoveryProtocol, a: Address, msg: openArray[byte]) {.gcsafe.} =
## Can raise `DiscProtocolError` and all of `RlpError`
# Note: export only needed for testing
let msgHash = validateMsgHash(msg)

14
eth/p2p/discoveryv5/encoding.nim
View File

@ -202,7 +202,7 @@ proc decodePacketBody(typ: byte,
let kind = cast[PacketKind](typ)
res = Packet(kind: kind)
var rlp = rlpFromBytes(@body.toRange)
var rlp = rlpFromBytes(body)
if rlp.enterList:
res.reqId = rlp.read(RequestId)
@ -258,12 +258,11 @@ proc decodeAuthResp(c: Codec, fromId: NodeId, head: AuthHeader,
proc decodeEncrypted*(c: var Codec,
fromId: NodeID,
fromAddr: Address,
input: seq[byte],
input: openArray[byte],
authTag: var AuthTag,
newNode: var Node,
packet: var Packet): DecodeStatus =
let input = input.toRange
var r = rlpFromBytes(input[tagSize .. ^1])
var r = rlpFromBytes(input.toOpenArray(tagSize, input.high))
var auth: AuthHeader
var readKey: AesKey
@ -312,10 +311,11 @@ proc decodeEncrypted*(c: var Codec,
# doAssert(false, "TODO: HANDLE ME!")
let headSize = tagSize + r.position
let bodyEnc = input[headSize .. ^1]
let body = decryptGCM(readKey, auth.auth, bodyEnc.toOpenArray,
input[0 .. tagSize - 1].toOpenArray)
let body = decryptGCM(
readKey, auth.auth,
input.toOpenArray(headSize, input.high),
input.toOpenArray(0, tagSize - 1))
if body.isNone():
discard c.db.deleteKeys(fromId, fromAddr)
return DecryptError

12
eth/p2p/discoveryv5/enr.nim
View File

@ -214,11 +214,11 @@ proc verifySignatureV4(r: Record, sigData: openarray[byte], content: seq[byte]):
return verify(sig[], h, publicKey.get)
proc verifySignature(r: Record): bool =
var rlp = rlpFromBytes(r.raw.toRange)
var rlp = rlpFromBytes(r.raw)
let sz = rlp.listLen
if not rlp.enterList:
return false
let sigData = rlp.read(Bytes)
let sigData = rlp.read(seq[byte])
let content = block:
var writer = initRlpList(sz - 1)
var reader = rlp
@ -240,7 +240,7 @@ proc fromBytesAux(r: var Record): bool =
if r.raw.len > maxEnrSize:
return false
var rlp = rlpFromBytes(r.raw.toRange)
var rlp = rlpFromBytes(r.raw)
if not rlp.isList:
return false
@ -270,7 +270,7 @@ proc fromBytesAux(r: var Record): bool =
let v = rlp.read(uint16)
r.pairs.add((k, Field(kind: kNum, num: v)))
else:
r.pairs.add((k, Field(kind: kBytes, bytes: rlp.read(Bytes))))
r.pairs.add((k, Field(kind: kBytes, bytes: rlp.read(seq[byte]))))
verifySignature(r)
@ -329,9 +329,9 @@ proc `$`*(r: Record): string =
proc `==`*(a, b: Record): bool = a.raw == b.raw
proc read*(rlp: var Rlp, T: typedesc[Record]): T {.inline.} =
if not result.fromBytes(rlp.rawData.toOpenArray):
if not result.fromBytes(rlp.rawData):
raise newException(ValueError, "Could not deserialize")
rlp.skipElem()
proc append*(rlpWriter: var RlpWriter, value: Record) =
rlpWriter.appendRawBytes(value.raw.toRange)
rlpWriter.appendRawBytes(value.raw)

8
eth/p2p/discoveryv5/protocol.nim
View File

@ -97,13 +97,13 @@ proc whoareyouMagic(toNode: NodeId): array[magicSize, byte] =
for i, c in prefix: data[sizeof(toNode) + i] = byte(c)
sha256.digest(data).data
proc isWhoAreYou(d: Protocol, msg: Bytes): bool =
proc isWhoAreYou(d: Protocol, msg: openArray[byte]): bool =
if msg.len > d.whoareyouMagic.len:
result = d.whoareyouMagic == msg.toOpenArray(0, magicSize - 1)
proc decodeWhoAreYou(d: Protocol, msg: Bytes): Whoareyou =
proc decodeWhoAreYou(d: Protocol, msg: openArray[byte]): Whoareyou =
result = Whoareyou()
result[] = rlp.decode(msg.toRange[magicSize .. ^1], WhoareyouObj)
result[] = rlp.decode(msg.toOpenArray(magicSize, msg.high), WhoareyouObj)
proc sendWhoareyou(d: Protocol, address: Address, toNode: NodeId, authTag: AuthTag) =
trace "sending who are you", to = $toNode, toAddress = $address
@ -172,7 +172,7 @@ proc handleFindNode(d: Protocol, fromId: NodeId, fromAddr: Address,
d.sendNodes(fromId, fromAddr, reqId,
d.routingTable.neighboursAtDistance(distance))
proc receive*(d: Protocol, a: Address, msg: Bytes) {.gcsafe,
proc receive*(d: Protocol, a: Address, msg: openArray[byte]) {.gcsafe,
raises: [
Defect,
# TODO This is now coming from Chronos's callSoon

7
eth/p2p/rlpx.nim
View File

@ -255,7 +255,7 @@ proc invokeThunk*(peer: Peer, msgId: int, msgData: var Rlp): Future[void] =
return thunk(peer, msgId, msgData)
template compressMsg(peer: Peer, data: Bytes): Bytes =
template compressMsg(peer: Peer, data: seq[byte]): seq[byte] =
when useSnappy:
if peer.snappyEnabled:
snappy.compress(data)
@ -263,7 +263,7 @@ template compressMsg(peer: Peer, data: Bytes): Bytes =
else:
data
proc sendMsg*(peer: Peer, data: Bytes) {.gcsafe, async.} =
proc sendMsg*(peer: Peer, data: seq[byte]) {.gcsafe, async.} =
try:
var cipherText = encryptMsg(peer.compressMsg(data), peer.secretsState)
var res = await peer.transport.write(cipherText)
@ -426,7 +426,7 @@ proc recvMsg*(peer: Peer): Future[tuple[msgId: int, msgData: Rlp]] {.async.} =
if decryptedBytes.len == 0:
await peer.disconnectAndRaise(BreachOfProtocol,
"Snappy uncompress encountered malformed data")
var rlp = rlpFromBytes(decryptedBytes.toRange)
var rlp = rlpFromBytes(decryptedBytes)
try:
# int32 as this seems more than big enough for the amount of msgIds
@ -561,7 +561,6 @@ proc p2pProtocolBackendImpl*(protocol: P2PProtocol): Backend =
EthereumNode = bindSym "EthereumNode"
initRlpWriter = bindSym "initRlpWriter"
rlpFromBytes = bindSym "rlpFromBytes"
append = bindSym("append", brForceOpen)
read = bindSym("read", brForceOpen)
checkedRlpRead = bindSym "checkedRlpRead"

2
eth/p2p/rlpx_protocols/les/flow_control.nim
View File

@ -103,7 +103,7 @@ proc loadMessageStats*(network: LesNetwork,
break readFromDB
try:
var statsRlp = rlpFromBytes(stats.toRange)
var statsRlp = rlpFromBytes(stats)
if not statsRlp.enterList:
notice "Found a corrupted LES stats record"
break readFromDB

4
eth/p2p/rlpx_protocols/waku_mail.nim
View File

@ -15,12 +15,12 @@ const
requestCompleteTimeout = chronos.seconds(5)
type
Cursor = Bytes
Cursor = seq[byte]
MailRequest* = object
lower*: uint32 ## Unix timestamp; oldest requested envelope's creation time
upper*: uint32 ## Unix timestamp; newest requested envelope's creation time
bloom*: Bytes ## Bloom filter to apply on the envelopes
bloom*: seq[byte] ## Bloom filter to apply on the envelopes
limit*: uint32 ## Maximum amount of envelopes to return
cursor*: Cursor ## Optional cursor

8
eth/p2p/rlpx_protocols/waku_protocol.nim
View File

@ -144,7 +144,7 @@ proc append*(rlpWriter: var RlpWriter, value: StatusOptions) =
let bytes = list.finish()
rlpWriter.append(rlpFromBytes(bytes.toRange))
rlpWriter.append(rlpFromBytes(bytes))
proc read*(rlp: var Rlp, T: typedesc[StatusOptions]): T =
if not rlp.isList():
@ -379,7 +379,7 @@ p2pProtocol Waku(version = wakuVersion,
proc p2pRequestComplete(peer: Peer, requestId: Hash, lastEnvelopeHash: Hash,
cursor: Bytes) = discard
cursor: seq[byte]) = discard
# TODO:
# In the current specification the parameters are not wrapped in a regular
# envelope as is done for the P2P Request packet. If we could alter this in
@ -488,8 +488,8 @@ proc queueMessage(node: EthereumNode, msg: Message): bool =
proc postMessage*(node: EthereumNode, pubKey = none[PublicKey](),
symKey = none[SymKey](), src = none[PrivateKey](),
ttl: uint32, topic: Topic, payload: Bytes,
padding = none[Bytes](), powTime = 1'f,
ttl: uint32, topic: Topic, payload: seq[byte],
padding = none[seq[byte]](), powTime = 1'f,
powTarget = defaultMinPow,
targetPeer = none[NodeId]()): bool =
## Post a message on the message queue which will be processed at the

26
eth/p2p/rlpx_protocols/whisper/whisper_types.nim
View File

@ -9,7 +9,7 @@
#
import
algorithm, bitops, math, options, strutils, tables, times, chronicles, hashes,
algorithm, bitops, math, options, tables, times, chronicles, hashes, strutils,
stew/[byteutils, endians2], metrics,
nimcrypto/[bcmode, hash, keccak, rijndael, sysrand],
eth/[keys, rlp, p2p], eth/p2p/ecies
@ -56,16 +56,16 @@ type
src*: Option[PrivateKey] ## Optional key used for signing message
dst*: Option[PublicKey] ## Optional key used for asymmetric encryption
symKey*: Option[SymKey] ## Optional key used for symmetric encryption
payload*: Bytes ## Application data / message contents
padding*: Option[Bytes] ## Padding - if unset, will automatically pad up to
payload*: seq[byte] ## Application data / message contents
padding*: Option[seq[byte]] ## Padding - if unset, will automatically pad up to
## nearest maxPadLen-byte boundary
DecodedPayload* = object
## The decoded payload of a received message.
src*: Option[PublicKey] ## If the message was signed, this is the public key
## of the source
payload*: Bytes ## Application data / message contents
padding*: Option[Bytes] ## Message padding
payload*: seq[byte] ## Application data / message contents
padding*: Option[seq[byte]] ## Message padding
Envelope* = object
## What goes on the wire in the whisper protocol - a payload and some
@ -74,7 +74,7 @@ type
expiry*: uint32 ## Unix timestamp when message expires
ttl*: uint32 ## Time-to-live, seconds - message was created at (expiry - ttl)
topic*: Topic
data*: Bytes ## Payload, as given by user
data*: seq[byte] ## Payload, as given by user
nonce*: uint64 ## Nonce used for proof-of-work calculation
Message* = object
@ -177,7 +177,7 @@ proc `or`(a, b: Bloom): Bloom =
for i in 0..<a.len:
result[i] = a[i] or b[i]
proc bytesCopy*(bloom: var Bloom, b: Bytes) =
proc bytesCopy*(bloom: var Bloom, b: openArray[byte]) =
doAssert b.len == bloomSize
copyMem(addr bloom[0], unsafeAddr b[0], bloomSize)
@ -198,7 +198,7 @@ proc fullBloom*(): Bloom =
result[i] = 0xFF
proc encryptAesGcm(plain: openarray[byte], key: SymKey,
iv: array[gcmIVLen, byte]): Bytes =
iv: array[gcmIVLen, byte]): seq[byte] =
## Encrypt using AES-GCM, making sure to append tag and iv, in that order
var gcm: GCM[aes256]
result = newSeqOfCap[byte](plain.len + gcmTagLen + iv.len)
@ -210,7 +210,7 @@ proc encryptAesGcm(plain: openarray[byte], key: SymKey,
result.add tag
result.add iv
proc decryptAesGcm(cipher: openarray[byte], key: SymKey): Option[Bytes] =
proc decryptAesGcm(cipher: openarray[byte], key: SymKey): Option[seq[byte]] =
## Decrypt AES-GCM ciphertext and validate authenticity - assumes
## cipher-tag-iv format of the buffer
if cipher.len < gcmTagLen + gcmIVLen:
@ -237,7 +237,7 @@ proc decryptAesGcm(cipher: openarray[byte], key: SymKey): Option[Bytes] =
# simply because that makes it closer to EIP 627 - see also:
# https://github.com/paritytech/parity-ethereum/issues/9652
proc encode*(self: Payload): Option[Bytes] =
proc encode*(self: Payload): Option[seq[byte]] =
## Encode a payload according so as to make it suitable to put in an Envelope
## The format follows EIP 627 - https://eips.ethereum.org/EIPS/eip-627
@ -333,7 +333,7 @@ proc decode*(data: openarray[byte], dst = none[PrivateKey](),
var res: DecodedPayload
var plain: Bytes
var plain: seq[byte]
if dst.isSome():
# XXX: eciesDecryptedLength is pretty fragile, API-wise.. is this really the
# way to check for errors / sufficient length?
@ -426,11 +426,11 @@ proc valid*(self: Envelope, now = epochTime()): bool =
proc len(self: Envelope): int = 20 + self.data.len
proc toShortRlp*(self: Envelope): Bytes =
proc toShortRlp*(self: Envelope): seq[byte] =
## RLP-encoded message without nonce is used during proof-of-work calculations
rlp.encodeList(self.expiry, self.ttl, self.topic, self.data)
proc toRlp(self: Envelope): Bytes =
proc toRlp(self: Envelope): seq[byte] =
## What gets sent out over the wire includes the nonce
rlp.encode(self)

8
eth/p2p/rlpx_protocols/whisper_protocol.nim
View File

@ -167,7 +167,7 @@ p2pProtocol Whisper(version = whisperVersion,
proc status(peer: Peer,
protocolVersion: uint,
powConverted: uint64,
bloom: Bytes,
bloom: seq[byte],
isLightNode: bool)
proc messages(peer: Peer, envelopes: openarray[Envelope]) =
@ -220,7 +220,7 @@ p2pProtocol Whisper(version = whisperVersion,
peer.state.powRequirement = cast[float64](value)
proc bloomFilterExchange(peer: Peer, bloom: Bytes) =
proc bloomFilterExchange(peer: Peer, bloom: openArray[byte]) =
if not peer.state.initialized:
warn "Handshake not completed yet, discarding bloomFilterExchange"
return
@ -343,8 +343,8 @@ proc queueMessage(node: EthereumNode, msg: Message): bool =
proc postMessage*(node: EthereumNode, pubKey = none[PublicKey](),
symKey = none[SymKey](), src = none[PrivateKey](),
ttl: uint32, topic: Topic, payload: Bytes,
padding = none[Bytes](), powTime = 1'f,
ttl: uint32, topic: Topic, payload: seq[byte],
padding = none[seq[byte]](), powTime = 1'f,
powTarget = defaultMinPow,
targetPeer = none[NodeId]()): bool =
## Post a message on the message queue which will be processed at the

2
eth/p2p/rlpxcrypt.nim
View File

@ -12,7 +12,7 @@
{.push raises: [Defect].}
import stew/ranges/stackarrays, eth/rlp/types, nimcrypto, stew/results
import stew/ranges/stackarrays, nimcrypto, stew/results
from auth import ConnectionSecret
export results

197
eth/rlp.nim
View File

@ -3,16 +3,16 @@
## https://ethereum.github.io/yellowpaper/paper.pdf
import
macros, strutils, parseutils,
rlp/[types, writer, object_serialization],
macros, strutils, stew/byteutils,
rlp/[writer, object_serialization],
rlp/priv/defs
export
types, writer, object_serialization
writer, object_serialization
type
Rlp* = object
bytes: BytesRange
bytes: seq[byte]
position*: int
RlpNodeType* = enum
@ -22,7 +22,7 @@ type
RlpNode* = object
case kind*: RlpNodeType
of rlpBlob:
bytes*: BytesRange
bytes*: seq[byte]
of rlpList:
elems*: seq[RlpNode]
@ -31,101 +31,86 @@ type
UnsupportedRlpError* = object of RlpError
RlpTypeMismatch* = object of RlpError
proc rlpFromBytes*(data: BytesRange): Rlp =
result.bytes = data
result.position = 0
proc rlpFromBytes*(data: seq[byte]): Rlp =
Rlp(bytes: data, position: 0)
proc rlpFromBytes*(data: openArray[byte]): Rlp =
rlpFromBytes(@data)
const zeroBytesRlp* = Rlp()
proc rlpFromHex*(input: string): Rlp =
doAssert input.len mod 2 == 0,
"rlpFromHex expects a string with even number of characters (assuming two characters per byte)"
var startByte = if input.len >= 2 and input[0] == '0' and input[1] == 'x': 2
else: 0
let totalBytes = (input.len - startByte) div 2
var backingStore = newSeq[byte](totalBytes)
for i in 0 ..< totalBytes:
var nextByte: int
if parseHex(input, nextByte, startByte + i*2, 2) == 2:
backingStore[i] = byte(nextByte)
else:
doAssert false, "rlpFromHex expects a hexademical string, but the input contains non hexademical characters"
result.bytes = backingStore.toRange()
{.this: self.}
rlpFromBytes(hexToSeqByte(input))
proc hasData*(self: Rlp): bool =
position < bytes.len
self.position < self.bytes.len
proc currentElemEnd*(self: Rlp): int {.gcsafe.}
proc rawData*(self: Rlp): BytesRange =
return self.bytes[position ..< self.currentElemEnd]
template rawData*(self: Rlp): openArray[byte] =
self.bytes.toOpenArray(self.position, self.currentElemEnd - 1)
proc isBlob*(self: Rlp): bool =
hasData() and bytes[position] < LIST_START_MARKER
self.hasData() and self.bytes[self.position] < LIST_START_MARKER
proc isEmpty*(self: Rlp): bool =
### Contains a blob or a list of zero length
hasData() and (bytes[position] == BLOB_START_MARKER or
bytes[position] == LIST_START_MARKER)
self.hasData() and (self.bytes[self.position] == BLOB_START_MARKER or
self.bytes[self.position] == LIST_START_MARKER)
proc isList*(self: Rlp): bool =
hasData() and bytes[position] >= LIST_START_MARKER
self.hasData() and self.bytes[self.position] >= LIST_START_MARKER
template eosError =
raise newException(MalformedRlpError, "Read past the end of the RLP stream")
template requireData {.dirty.} =
if not hasData():
if not self.hasData():
raise newException(MalformedRlpError, "Illegal operation over an empty RLP stream")
proc getType*(self: Rlp): RlpNodeType =
requireData()
return if isBlob(): rlpBlob else: rlpList
return if self.isBlob(): rlpBlob else: rlpList
proc lengthBytesCount(self: Rlp): int =
var marker = bytes[position]
if isBlob() and marker > LEN_PREFIXED_BLOB_MARKER:
var marker = self.bytes[self.position]
if self.isBlob() and marker > LEN_PREFIXED_BLOB_MARKER:
return int(marker - LEN_PREFIXED_BLOB_MARKER)
if isList() and marker > LEN_PREFIXED_LIST_MARKER:
if self.isList() and marker > LEN_PREFIXED_LIST_MARKER:
return int(marker - LEN_PREFIXED_LIST_MARKER)
return 0
proc isSingleByte*(self: Rlp): bool =
hasData() and bytes[position] < BLOB_START_MARKER
self.hasData() and self.bytes[self.position] < BLOB_START_MARKER
proc getByteValue*(self: Rlp): byte =
doAssert self.isSingleByte()
return bytes[position]
return self.bytes[self.position]
proc payloadOffset(self: Rlp): int =
if isSingleByte(): 0 else: 1 + lengthBytesCount()
if self.isSingleByte(): 0 else: 1 + self.lengthBytesCount()
template readAheadCheck(numberOfBytes) =
template readAheadCheck(numberOfBytes: int) =
# important to add nothing to the left side of the equation as `numberOfBytes`
# can in theory be at max size of its type already
if numberOfBytes > bytes.len - position - payloadOffset(): eosError()
if numberOfBytes > self.bytes.len - self.position - self.payloadOffset():
eosError()
template nonCanonicalNumberError =
raise newException(MalformedRlpError, "Small number encoded in a non-canonical way")
proc payloadBytesCount(self: Rlp): int =
if not hasData():
if not self.hasData():
return 0
var marker = bytes[position]
var marker = self.bytes[self.position]
if marker < BLOB_START_MARKER:
return 1
if marker <= LEN_PREFIXED_BLOB_MARKER:
result = int(marker - BLOB_START_MARKER)
readAheadCheck(result)
if result == 1:
if bytes[position + 1] < BLOB_START_MARKER:
if self.bytes[self.position + 1] < BLOB_START_MARKER:
nonCanonicalNumberError()
return
@ -162,22 +147,22 @@ proc payloadBytesCount(self: Rlp): int =
readAheadCheck(result)
proc blobLen*(self: Rlp): int =
if isBlob(): payloadBytesCount() else: 0
if self.isBlob(): self.payloadBytesCount() else: 0
proc isInt*(self: Rlp): bool =
if not hasData():
if not self.hasData():
return false
var marker = bytes[position]
var marker = self.bytes[self.position]
if marker < BLOB_START_MARKER:
return marker != 0
if marker == BLOB_START_MARKER:
return true
if marker <= LEN_PREFIXED_BLOB_MARKER:
return bytes[position + 1] != 0
return self.bytes[self.position + 1] != 0
if marker < LIST_START_MARKER:
let offset = position + int(marker + 1 - LEN_PREFIXED_BLOB_MARKER)
if offset >= bytes.len: eosError()
return bytes[offset] != 0
let offset = self.position + int(marker + 1 - LEN_PREFIXED_BLOB_MARKER)
if offset >= self.bytes.len: eosError()
return self.bytes[offset] != 0
return false
template maxBytes*(o: type[Ordinal | uint64 | uint]): int = sizeof(o)
@ -206,83 +191,83 @@ proc toInt*(self: Rlp, IntType: type): IntType =
result = (result shl 8) or OutputType(self.bytes[self.position + i])
proc toString*(self: Rlp): string =
if not isBlob():
if not self.isBlob():
raise newException(RlpTypeMismatch, "String expected, but the source RLP is not a blob")
let
payloadOffset = payloadOffset()
payloadLen = payloadBytesCount()
payloadOffset = self.payloadOffset()
payloadLen = self.payloadBytesCount()
result = newString(payloadLen)
for i in 0 ..< payloadLen:
# XXX: switch to copyMem here
result[i] = char(bytes[position + payloadOffset + i])
result[i] = char(self.bytes[self.position + payloadOffset + i])
proc toBytes*(self: Rlp): BytesRange =
if not isBlob():
proc toBytes*(self: Rlp): seq[byte] =
if not self.isBlob():
raise newException(RlpTypeMismatch,
"Bytes expected, but the source RLP in not a blob")
let payloadLen = payloadBytesCount()
let payloadLen = self.payloadBytesCount()
if payloadLen > 0:
let
payloadOffset = payloadOffset()
ibegin = position + payloadOffset
payloadOffset = self.payloadOffset()
ibegin = self.position + payloadOffset
iend = ibegin + payloadLen - 1
result = bytes.slice(ibegin, iend)
result = self.bytes[ibegin..iend]
proc currentElemEnd*(self: Rlp): int =
doAssert hasData()
result = position
doAssert self.hasData()
result = self.position
if isSingleByte():
if self.isSingleByte():
result += 1
elif isBlob() or isList():
result += payloadOffset() + payloadBytesCount()
elif self.isBlob() or self.isList():
result += self.payloadOffset() + self.payloadBytesCount()
proc enterList*(self: var Rlp): bool =
if not isList():
if not self.isList():
return false
position += payloadOffset()
self.position += self.payloadOffset()
return true
proc tryEnterList*(self: var Rlp) =
if not enterList():
if not self.enterList():
raise newException(RlpTypeMismatch, "List expected, but source RLP is not a list")
proc skipElem*(rlp: var Rlp) =
rlp.position = rlp.currentElemEnd
iterator items*(self: var Rlp): var Rlp =
doAssert isList()
doAssert self.isList()
var
payloadOffset = payloadOffset()
payloadEnd = position + payloadOffset + payloadBytesCount()
payloadOffset = self.payloadOffset()
payloadEnd = self.position + payloadOffset + self.payloadBytesCount()
if payloadEnd > bytes.len:
if payloadEnd > self.bytes.len:
raise newException(MalformedRlpError, "List length extends past the end of the stream")
position += payloadOffset
self.position += payloadOffset
while position < payloadEnd:
let elemEnd = currentElemEnd()
while self.position < payloadEnd:
let elemEnd = self.currentElemEnd()
yield self
position = elemEnd
self.position = elemEnd
proc listElem*(self: Rlp, i: int): Rlp =
doAssert isList()
doAssert self.isList()
let
payloadOffset = payloadOffset()
payloadOffset = self.payloadOffset()
# This will only check if there is some data, not if it is correct according
# to list length. Could also run here payloadBytesCount() instead.
if position + payloadOffset + 1 > bytes.len: eosError()
if self.position + payloadOffset + 1 > self.bytes.len: eosError()
let payload = bytes.slice(position + payloadOffset)
let payload = self.bytes[self.position + payloadOffset..^1]
result = rlpFromBytes payload
var pos = 0
while pos < i and result.hasData:
@ -290,7 +275,7 @@ proc listElem*(self: Rlp, i: int): Rlp =
inc pos
proc listLen*(self: Rlp): int =
if not isList():
if not self.isList():
return 0
var rlp = self
@ -336,7 +321,7 @@ proc readImpl[R, E](rlp: var Rlp, T: type array[R, E]): T =
if result.len != bytes.len:
raise newException(RlpTypeMismatch, "Fixed-size array expected, but the source RLP contains a blob of different length")
copyMem(addr result[0], bytes.baseAddr, bytes.len)
copyMem(addr result[0], unsafeAddr bytes[0], bytes.len)
rlp.skipElem
@ -356,10 +341,7 @@ proc readImpl[E](rlp: var Rlp, T: type seq[E]): T =
mixin read
when E is (byte or char):
var bytes = rlp.toBytes
if bytes.len != 0:
result = newSeq[byte](bytes.len)
copyMem(addr result[0], bytes.baseAddr, bytes.len)
result = rlp.toBytes
rlp.skipElem
else:
if not rlp.isList:
@ -383,7 +365,9 @@ proc readImpl(rlp: var Rlp, T: type[object|tuple],
"List expected, but the source RLP is not a list.")
var
payloadOffset = rlp.payloadOffset()
payloadEnd = rlp.position + payloadOffset + rlp.payloadBytesCount()
# there's an exception-raising side effect in there *sigh*
discard rlp.payloadBytesCount()
rlp.position += payloadOffset
@ -398,16 +382,16 @@ proc readImpl(rlp: var Rlp, T: type[object|tuple],
proc toNodes*(self: var Rlp): RlpNode =
requireData()
if isList():
if self.isList():
result.kind = rlpList
newSeq result.elems, 0
for e in self:
result.elems.add e.toNodes
else:
doAssert isBlob()
doAssert self.isBlob()
result.kind = rlpBlob
result.bytes = toBytes()
position = currentElemEnd()
result.bytes = self.toBytes()
self.position = self.currentElemEnd()
# We define a single `read` template with a pretty low specifity
# score in order to facilitate easier overloading with user types:
@ -422,22 +406,18 @@ template readRecordType*(rlp: var Rlp, T: type, wrappedInList: bool): auto =
readImpl(rlp, T, wrappedInList)
proc decode*(bytes: openarray[byte]): RlpNode =
var
bytesCopy = @bytes
rlp = rlpFromBytes(bytesCopy.toRange())
return rlp.toNodes
var rlp = rlpFromBytes(bytes)
rlp.toNodes
template decode*(bytes: BytesRange, T: type): untyped =
template decode*(bytes: openArray[byte], T: type): untyped =
mixin read
var rlp = rlpFromBytes(bytes)
rlp.read(T)
template decode*(bytes: openarray[byte], T: type): T =
var bytesCopy = @bytes
decode(bytesCopy.toRange, T)
template decode*(bytes: seq[byte], T: type): untyped =
decode(bytes.toRange, T)
mixin read
var rlp = rlpFromBytes(bytes)
rlp.read(T)
proc append*(writer: var RlpWriter; rlp: Rlp) =
appendRawBytes(writer, rlp.rawData)
@ -450,7 +430,7 @@ proc isPrintable(s: string): bool =
return true
proc inspectAux(self: var Rlp, depth: int, hexOutput: bool, output: var string) =
if not hasData():
if not self.hasData():
return
template indent =
@ -461,7 +441,7 @@ proc inspectAux(self: var Rlp, depth: int, hexOutput: bool, output: var string)
if self.isSingleByte:
output.add "byte "
output.add $bytes[position]
output.add $self.bytes[self.position]
elif self.isBlob:
let str = self.toString
if str.isPrintable:
@ -491,6 +471,5 @@ proc inspectAux(self: var Rlp, depth: int, hexOutput: bool, output: var string)
proc inspect*(self: Rlp, indent = 0, hexOutput = true): string =
var rlpCopy = self
result = newStringOfCap(bytes.len)
result = newStringOfCap(self.bytes.len)
inspectAux(rlpCopy, indent, hexOutput, result)

6
eth/rlp/types.nim
View File

@ -1,6 +0,0 @@
import stew/ranges
export ranges
type
Bytes* = seq[byte]
BytesRange* = Range[byte]

143
eth/rlp/writer.nim
View File

@ -1,15 +1,11 @@
import
macros, types,
stew/ranges/[memranges, ptr_arith],
macros,
object_serialization, priv/defs
export
memranges
type
RlpWriter* = object
pendingLists: seq[tuple[remainingItems, outBytes: int]]
output: Bytes
output: seq[byte]
IntLike* = concept x, y
type T = type(x)
@ -39,7 +35,7 @@ proc bytesNeeded(num: Integer): int =
inc result
n = n shr 8
proc writeBigEndian(outStream: var Bytes, number: Integer,
proc writeBigEndian(outStream: var seq[byte], number: Integer,
lastByteIdx: int, numberOfBytes: int) =
mixin `and`, `shr`
@ -48,12 +44,12 @@ proc writeBigEndian(outStream: var Bytes, number: Integer,
outStream[i] = byte(n and 0xff)
n = n shr 8
proc writeBigEndian(outStream: var Bytes, number: Integer,
proc writeBigEndian(outStream: var seq[byte], number: Integer,
numberOfBytes: int) {.inline.} =
outStream.setLen(outStream.len + numberOfBytes)
outStream.writeBigEndian(number, outStream.len - 1, numberOfBytes)
proc writeCount(bytes: var Bytes, count: int, baseMarker: byte) =
proc writeCount(bytes: var seq[byte], count: int, baseMarker: byte) =
if count < THRESHOLD_LIST_LEN:
bytes.add(baseMarker + byte(count))
else:
@ -65,19 +61,6 @@ proc writeCount(bytes: var Bytes, count: int, baseMarker: byte) =
bytes[origLen] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes)
bytes.writeBigEndian(count, bytes.len - 1, lenPrefixBytes)
proc add(outStream: var Bytes, newChunk: BytesRange) =
let prevLen = outStream.len
outStream.setLen(prevLen + newChunk.len)
# XXX: Use copyMem here
for i in 0 ..< newChunk.len:
outStream[prevLen + i] = newChunk[i]
{.this: self.}
{.experimental.}
using
self: var RlpWriter
proc initRlpWriter*: RlpWriter =
newSeq(result.pendingLists, 0)
newSeq(result.output, 0)
@ -86,88 +69,74 @@ proc decRet(n: var int, delta: int): int =
n -= delta
return n
proc maybeClosePendingLists(self) =
while pendingLists.len > 0:
let lastListIdx = pendingLists.len - 1
doAssert pendingLists[lastListIdx].remainingItems >= 1
if decRet(pendingLists[lastListIdx].remainingItems, 1) == 0:
proc maybeClosePendingLists(self: var RlpWriter) =
while self.pendingLists.len > 0:
let lastListIdx = self.pendingLists.len - 1
doAssert self.pendingLists[lastListIdx].remainingItems >= 1
if decRet(self.pendingLists[lastListIdx].remainingItems, 1) == 0:
# A list have been just finished. It was started in `startList`.
let listStartPos = pendingLists[lastListIdx].outBytes
pendingLists.setLen lastListIdx
let listStartPos = self.pendingLists[lastListIdx].outBytes
self.pendingLists.setLen lastListIdx
# How many bytes were written since the start?
let listLen = output.len - listStartPos
let listLen = self.output.len - listStartPos
# Compute the number of bytes required to write down the list length
let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1
else: int(listLen.bytesNeeded) + 1
# Shift the written data to make room for the prefix length
output.setLen(output.len + totalPrefixBytes)
let outputBaseAddr = output.baseAddr
self.output.setLen(self.output.len + totalPrefixBytes)
moveMem(outputBaseAddr.shift(listStartPos + totalPrefixBytes),
outputBaseAddr.shift(listStartPos),
moveMem(addr self.output[listStartPos + totalPrefixBytes],
unsafeAddr self.output[listStartPos],
listLen)
# Write out the prefix length
if listLen < THRESHOLD_LIST_LEN:
output[listStartPos] = LIST_START_MARKER + byte(listLen)
self.output[listStartPos] = LIST_START_MARKER + byte(listLen)
else:
let listLenBytes = totalPrefixBytes - 1
output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes)
output.writeBigEndian(listLen, listStartPos + listLenBytes, listLenBytes)
self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes)
self.output.writeBigEndian(listLen, listStartPos + listLenBytes, listLenBytes)
else:
# The currently open list is not finished yet. Nothing to do.
return
proc appendRawList(self; bytes: BytesRange) =
output.writeCount(bytes.len, LIST_START_MARKER)
output.add(bytes)
maybeClosePendingLists()
proc appendRawList(self: var RlpWriter, bytes: openArray[byte]) =
self.output.writeCount(bytes.len, LIST_START_MARKER)
self.output.add(bytes)
self.maybeClosePendingLists()
proc appendRawBytes*(self; bytes: BytesRange) =
output.add(bytes)
maybeClosePendingLists()
proc appendRawBytes*(self: var RlpWriter, bytes: openArray[byte]) =
self.output.add(bytes)
self.maybeClosePendingLists()
proc startList*(self; listSize: int) =
proc startList*(self: var RlpWriter, listSize: int) =
if listSize == 0:
appendRawList(BytesRange())
self.appendRawList([])
else:
pendingLists.add((listSize, output.len))
template appendBlob(self; data, startMarker) =
mixin baseAddr
self.pendingLists.add((listSize, self.output.len))
proc appendBlob(self: var RlpWriter, data: openArray[byte], startMarker: byte) =
if data.len == 1 and byte(data[0]) < BLOB_START_MARKER:
self.output.add byte(data[0])
else:
self.output.writeCount(data.len, startMarker)
self.output.add data
let startPos = output.len
self.output.setLen(startPos + data.len)
copyMem(shift(baseAddr(self.output), startPos),
baseAddr(data),
data.len)
self.maybeClosePendingLists()
maybeClosePendingLists()
proc appendImpl(self: var RlpWriter, data: string) =
appendBlob(self, data.toOpenArrayByte(0, data.high), BLOB_START_MARKER)
proc appendImpl(self; data: string) =
proc appendBlob(self: var RlpWriter, data: openarray[byte]) =
appendBlob(self, data, BLOB_START_MARKER)
proc appendBlob(self; data: openarray[byte]) =
appendBlob(self, data, BLOB_START_MARKER)
proc appendBlob(self: var RlpWriter, data: openarray[char]) =
appendBlob(self, data.toOpenArrayByte(0, data.high), BLOB_START_MARKER)
proc appendBlob(self; data: openarray[char]) =
appendBlob(self, data, BLOB_START_MARKER)
proc appendBytesRange(self; data: BytesRange) =
appendBlob(self, data, BLOB_START_MARKER)
proc appendImpl(self; data: MemRange) =
appendBlob(self, data, BLOB_START_MARKER)
proc appendInt(self; i: Integer) =
proc appendInt(self: var RlpWriter, i: Integer) =
# this is created as a separate proc as an extra precaution against
# any overloading resolution problems when matching the IntLike concept.
type IntType = type(i)
@ -183,7 +152,7 @@ proc appendInt(self; i: Integer) =
self.maybeClosePendingLists()
proc appendFloat(self; data: float64) =
proc appendFloat(self: var RlpWriter, data: float64) =
# This is not covered in the RLP spec, but Geth uses Go's
# `math.Float64bits`, which is defined here:
# https://github.com/gopherjs/gopherjs/blob/master/compiler/natives/src/math/math.go
@ -191,16 +160,16 @@ proc appendFloat(self; data: float64) =
let uint64bits = (uint64(uintWords[1]) shl 32) or uint64(uintWords[0])
self.appendInt(uint64bits)
template appendImpl(self; i: Integer) =
template appendImpl(self: var RlpWriter, i: Integer) =
appendInt(self, i)
template appendImpl(self; e: enum) =
template appendImpl(self: var RlpWriter, e: enum) =
appendImpl(self, int(e))
template appendImpl(self; b: bool) =
template appendImpl(self: var RlpWriter, b: bool) =
appendImpl(self, int(b))
proc appendImpl[T](self; listOrBlob: openarray[T]) =
proc appendImpl[T](self: var RlpWriter, listOrBlob: openarray[T]) =
mixin append
# TODO: This append proc should be overloaded by `openarray[byte]` after
@ -212,7 +181,7 @@ proc appendImpl[T](self; listOrBlob: openarray[T]) =
for i in 0 ..< listOrBlob.len:
self.append listOrBlob[i]
proc appendRecordType*(self; obj: object|tuple, wrapInList = wrapObjsInList) =
proc appendRecordType*(self: var RlpWriter, obj: object|tuple, wrapInList = wrapObjsInList) =
mixin enumerateRlpFields, append
if wrapInList:
@ -226,15 +195,10 @@ proc appendRecordType*(self; obj: object|tuple, wrapInList = wrapObjsInList) =
enumerateRlpFields(obj, op)
proc appendImpl(self; data: object) {.inline.} =
# TODO: This append proc should be overloaded by `BytesRange` after
# nim bug #7416 is fixed.
when data is BytesRange:
self.appendBytesRange(data)
else:
proc appendImpl(self: var RlpWriter, data: object) {.inline.} =
self.appendRecordType(data)
proc appendImpl(self; data: tuple) {.inline.} =
proc appendImpl(self: var RlpWriter, data: tuple) {.inline.} =
self.appendRecordType(data)
# We define a single `append` template with a pretty low specifity
@ -253,22 +217,22 @@ proc initRlpList*(listSize: int): RlpWriter =
startList(result, listSize)
# TODO: This should return a lent value
proc finish*(self): Bytes =
doAssert pendingLists.len == 0, "Insufficient number of elements written to a started list"
result = output
template finish*(self: RlpWriter): seq[byte] =
doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list"
self.output
proc encode*[T](v: T): Bytes =
proc encode*[T](v: T): seq[byte] =
mixin append
var writer = initRlpWriter()
writer.append(v)
return writer.finish
proc encodeInt*(i: Integer): Bytes =
proc encodeInt*(i: Integer): seq[byte] =
var writer = initRlpWriter()
writer.appendInt(i)
return writer.finish
macro encodeList*(args: varargs[untyped]): Bytes =
macro encodeList*(args: varargs[untyped]): seq[byte] =
var
listLen = args.len
writer = genSym(nskVar, "rlpWriter")
@ -286,10 +250,9 @@ macro encodeList*(args: varargs[untyped]): Bytes =
when false:
# XXX: Currently fails with a malformed AST error on the args.len expression
template encodeList*(args: varargs[untyped]): BytesRange =
template encodeList*(args: varargs[untyped]): seq[byte] =
mixin append
var writer = initRlpList(args.len)
for arg in args:
writer.append(arg)
writer.finish

2
eth/trie/backends/caching_backend.nim
View File

@ -1,5 +1,5 @@
import
stew/ranges, tables, sets,
tables, sets,
eth/trie/db
type

2
eth/trie/backends/lmdb_backend.nim
View File

@ -1,4 +1,4 @@
import os, stew/ranges, eth/trie/[trie_defs, db_tracing]
import os, eth/trie/[trie_defs, db_tracing]
import backend_defs
when defined(windows):

2
eth/trie/backends/rocksdb_backend.nim
View File

@ -1,4 +1,4 @@
import os, rocksdb, stew/ranges, eth/trie/[trie_defs, db_tracing]
import os, rocksdb, eth/trie/[trie_defs, db_tracing]
import backend_defs
type

2
eth/trie/backends/sqlite_backend.nim
View File

@ -1,5 +1,5 @@
import
os, sqlite3, stew/ranges, stew/ranges/ptr_arith, eth/trie/[db_tracing, trie_defs],
os, sqlite3, stew/ranges/ptr_arith, eth/trie/[db_tracing, trie_defs],
backend_defs
type

36
eth/trie/binaries.nim
View File

@ -1,6 +1,7 @@
import
sequtils,
stew/ranges/[ptr_arith, bitranges], eth/rlp/types, trie_defs
stew/ranges/ptr_arith, trie_defs,
./trie_bitseq
type
TrieNodeKind* = enum
@ -8,31 +9,30 @@ type
BRANCH_TYPE = 1
LEAF_TYPE = 2
TrieNodeKey* = BytesRange
TrieBitRange* = BitRange
TrieNodeKey* = seq[byte]
TrieNode* = object
case kind*: TrieNodeKind
of KV_TYPE:
keyPath*: TrieBitRange
keyPath*: TrieBitSeq
child*: TrieNodeKey
of BRANCH_TYPE:
leftChild*: TrieNodeKey
rightChild*: TrieNodeKey
of LEAF_TYPE:
value*: BytesRange
value*: seq[byte]
InvalidNode* = object of CorruptedTrieDatabase
ValidationError* = object of CorruptedTrieDatabase
# ----------------------------------------------
template sliceToEnd*(r: TrieBitRange, index: int): TrieBitRange =
if r.len <= index: TrieBitRange() else: r[index .. ^1]
template sliceToEnd*(r: TrieBitSeq, index: int): TrieBitSeq =
if r.len <= index: TrieBitSeq() else: r[index .. ^1]
proc decodeToBinKeypath*(path: BytesRange): TrieBitRange =
proc decodeToBinKeypath*(path: seq[byte]): TrieBitSeq =
## Decodes bytes into a sequence of 0s and 1s
## Used in decoding key path of a KV-NODE
var path = MutByteRange(path).bits
var path = path.bits
if path[0]:
path = path[4..^1]
@ -42,11 +42,11 @@ proc decodeToBinKeypath*(path: BytesRange): TrieBitRange =
bits = bits or path[3].int
if path.len > 4:
result = path[4+((4 - bits) mod 4)..^1]
path[4+((4 - bits) mod 4)..^1]
else:
result = BitRange()
TrieBitSeq()
proc parseNode*(node: BytesRange): TrieNode =
proc parseNode*(node: openArray[byte]): TrieNode =
# Input: a serialized node
if node.len == 0:
@ -76,7 +76,7 @@ proc parseNode*(node: BytesRange): TrieNode =
# Output: node type, value
return TrieNode(kind: LEAF_TYPE, value: node[1..^1])
proc encodeKVNode*(keyPath: TrieBitRange, childHash: TrieNodeKey): Bytes =
proc encodeKVNode*(keyPath: TrieBitSeq, childHash: TrieNodeKey): seq[byte] =
## Serializes a key/value node
if keyPath.len == 0:
raise newException(ValidationError, "Key path can not be empty")
@ -110,13 +110,13 @@ proc encodeKVNode*(keyPath: TrieBitRange, childHash: TrieNodeKey): Bytes =
inc(nbits, 8)
copyMem(result[^32].addr, childHash.baseAddr, 32)
proc encodeKVNode*(keyPath: bool, childHash: TrieNodeKey): Bytes =
proc encodeKVNode*(keyPath: bool, childHash: TrieNodeKey): seq[byte] =
result = newSeq[byte](34)
result[0] = KV_TYPE.byte
result[1] = byte(16) or byte(keyPath)
copyMem(result[^32].addr, childHash.baseAddr, 32)
proc encodeBranchNode*(leftChildHash, rightChildHash: TrieNodeKey): Bytes =
proc encodeBranchNode*(leftChildHash, rightChildHash: TrieNodeKey): seq[byte] =
## Serializes a branch node
const
BRANCH_TYPE_PREFIX = @[BRANCH_TYPE.byte]
@ -126,7 +126,7 @@ proc encodeBranchNode*(leftChildHash, rightChildHash: TrieNodeKey): Bytes =
result = BRANCH_TYPE_PREFIX.concat(leftChildHash, rightChildHash)
proc encodeLeafNode*(value: BytesRange | Bytes): Bytes =
proc encodeLeafNode*(value: openArray[byte]): seq[byte] =
## Serializes a leaf node
const
LEAF_TYPE_PREFIX = @[LEAF_TYPE.byte]
@ -134,9 +134,9 @@ proc encodeLeafNode*(value: BytesRange | Bytes): Bytes =
if value.len == 0:
raise newException(ValidationError, "Value of leaf node can not be empty")
result = LEAF_TYPE_PREFIX.concat(value)
result = LEAF_TYPE_PREFIX.concat(@value)
proc getCommonPrefixLength*(a, b: TrieBitRange): int =
proc getCommonPrefixLength*(a, b: TrieBitSeq): int =
let len = min(a.len, b.len)
for i in 0..<len:
if a[i] != b[i]: return i

102
eth/trie/binary.nim
View File

@ -1,9 +1,9 @@
import
stew/ranges/[typedranges, bitranges], eth/rlp/types,
trie_defs, db, binaries, trie_utils
./trie_bitseq,
./trie_defs, ./db, ./binaries, ./trie_utils
export
types, trie_utils
trie_utils
type
DB = TrieDatabaseRef
@ -14,18 +14,18 @@ type
NodeOverrideError* = object of CatchableError
let
zeroHash* = zeroBytesRange
const
zeroHash* = default(seq[byte])
proc init*(x: typedesc[BinaryTrie], db: DB,
rootHash: BytesContainer | KeccakHash = zeroHash): BinaryTrie =
rootHash: openArray[byte]): BinaryTrie =
checkValidHashZ(rootHash)
result.db = db
result.rootHash = toRange(rootHash)
result.rootHash = @(rootHash)
proc getDB*(t: BinaryTrie): auto = t.db
proc initBinaryTrie*(db: DB, rootHash: BytesContainer | KeccakHash): BinaryTrie =
proc initBinaryTrie*(db: DB, rootHash: openArray[byte]): BinaryTrie =
init(BinaryTrie, db, rootHash)
proc initBinaryTrie*(db: DB): BinaryTrie =
@ -36,64 +36,64 @@ proc getRootHash*(self: BinaryTrie): TrieNodeKey {.inline.} =
template fetchNode(self: BinaryTrie, nodeHash: TrieNodeKey): TrieNode =
doAssert(nodeHash.len == 32)
parseNode self.db.get(nodeHash.toOpenArray).toRange
parseNode self.db.get(nodeHash)
proc getAux(self: BinaryTrie, nodeHash: TrieNodeKey, keyPath: TrieBitRange): BytesRange =
proc getAux(self: BinaryTrie, nodeHash: TrieNodeKey, keyPath: TrieBitSeq): seq[byte] =
# Empty trie
if isZeroHash(nodeHash):
return zeroBytesRange
return
let node = self.fetchNode(nodeHash)
# Key-value node descend
if node.kind == LEAF_TYPE:
if keyPath.len != 0: return zeroBytesRange
if keyPath.len != 0: return
return node.value
elif node.kind == KV_TYPE:
# keyPath too short
if keyPath.len == 0: return zeroBytesRange
if keyPath.len == 0: return
let sliceLen = min(node.keyPath.len, keyPath.len)
if keyPath[0..<sliceLen] == node.keyPath:
return self.getAux(node.child, keyPath.sliceToEnd(node.keyPath.len))
else:
return zeroBytesRange
return
# Branch node descend
elif node.kind == BRANCH_TYPE:
# keyPath too short
if keyPath.len == 0: return zeroBytesRange
if keyPath.len == 0: return
if keyPath[0]: # first bit == 1
return self.getAux(node.rightChild, keyPath.sliceToEnd(1))
else:
return self.getAux(node.leftChild, keyPath.sliceToEnd(1))
proc get*(self: BinaryTrie, key: BytesContainer): BytesRange {.inline.} =
var keyBits = MutByteRange(key.toRange).bits
proc get*(self: BinaryTrie, key: openArray[byte]): seq[byte] {.inline.} =
var keyBits = key.bits
return self.getAux(self.rootHash, keyBits)
proc hashAndSave*(self: BinaryTrie, node: BytesRange | Bytes): TrieNodeKey =
result = keccakHash(node)
self.db.put(result.toOpenArray, node.toRange.toOpenArray)
proc hashAndSave*(self: BinaryTrie, node: openArray[byte]): TrieNodeKey =
result = @(keccakHash(node).data)
self.db.put(result, node)
template saveKV(self: BinaryTrie, keyPath: TrieBitRange | bool, child: BytesRange): untyped =
template saveKV(self: BinaryTrie, keyPath: TrieBitSeq | bool, child: openArray[byte]): untyped =
self.hashAndsave(encodeKVNode(keyPath, child))
template saveLeaf(self: BinaryTrie, value: BytesRange): untyped =
template saveLeaf(self: BinaryTrie, value: openArray[byte]): untyped =
self.hashAndsave(encodeLeafNode(value))
template saveBranch(self: BinaryTrie, L, R: BytesRange): untyped =
template saveBranch(self: BinaryTrie, L, R: openArray[byte]): untyped =
self.hashAndsave(encodeBranchNode(L, R))
proc setBranchNode(self: BinaryTrie, keyPath: TrieBitRange, node: TrieNode,
value: BytesRange, deleteSubtrie = false): TrieNodeKey
proc setKVNode(self: BinaryTrie, keyPath: TrieBitRange, nodeHash: TrieNodeKey,
node: TrieNode, value: BytesRange, deleteSubtrie = false): TrieNodeKey
proc setBranchNode(self: BinaryTrie, keyPath: TrieBitSeq, node: TrieNode,
value: openArray[byte], deleteSubtrie = false): TrieNodeKey
proc setKVNode(self: BinaryTrie, keyPath: TrieBitSeq, nodeHash: TrieNodeKey,
node: TrieNode, value: openArray[byte], deleteSubtrie = false): TrieNodeKey
const
overrideErrorMsg =
"Fail to set the value because the prefix of it's key is the same as existing key"
proc setAux(self: BinaryTrie, nodeHash: TrieNodeKey, keyPath: TrieBitRange,
value: BytesRange, deleteSubtrie = false): TrieNodeKey =
proc setAux(self: BinaryTrie, nodeHash: TrieNodeKey, keyPath: TrieBitSeq,
value: openArray[byte], deleteSubtrie = false): TrieNodeKey =
## If deleteSubtrie is set to True, what it will do is that it take in a keyPath
## and traverse til the end of keyPath, then delete the whole subtrie of that node.
## Note: keyPath should be in binary array format, i.e., encoded by encode_to_bin()
@ -131,15 +131,15 @@ proc setAux(self: BinaryTrie, nodeHash: TrieNodeKey, keyPath: TrieBitRange,
checkBadKeyPath()
return self.setBranchNode(keyPath, node, value, deleteSubtrie)
proc set*(self: var BinaryTrie, key, value: distinct BytesContainer) {.inline.} =
proc set*(self: var BinaryTrie, key, value: openArray[byte]) {.inline.} =
## Sets the value at the given keyPath from the given node
## Key will be encoded into binary array format first.
var keyBits = bits MutByteRange(key.toRange)
self.rootHash = self.setAux(self.rootHash, keyBits, toRange(value))
var keyBits = key.bits
self.rootHash = self.setAux(self.rootHash, keyBits, value)
proc setBranchNode(self: BinaryTrie, keyPath: TrieBitRange, node: TrieNode,
value: BytesRange, deleteSubtrie = false): TrieNodeKey =
proc setBranchNode(self: BinaryTrie, keyPath: TrieBitSeq, node: TrieNode,
value: openArray[byte], deleteSubtrie = false): TrieNodeKey =
# Which child node to update? Depends on first bit in keyPath
var newLeftChild, newRightChild: TrieNodeKey
@ -169,8 +169,8 @@ proc setBranchNode(self: BinaryTrie, keyPath: TrieBitRange, node: TrieNode,
else:
result = self.saveBranch(newLeftChild, newRightChild)
proc setKVNode(self: BinaryTrie, keyPath: TrieBitRange, nodeHash: TrieNodeKey,
node: TrieNode, value: BytesRange, deleteSubtrie = false): TrieNodeKey =
proc setKVNode(self: BinaryTrie, keyPath: TrieBitSeq, nodeHash: TrieNodeKey,
node: TrieNode, value: openArray[byte], deleteSubtrie = false): TrieNodeKey =
# keyPath prefixes match
if deleteSubtrie:
if keyPath.len < node.keyPath.len and keyPath == node.keyPath[0..<keyPath.len]:
@ -251,35 +251,35 @@ proc setKVNode(self: BinaryTrie, keyPath: TrieBitRange, nodeHash: TrieNodeKey,
else:
return newSub
template exists*(self: BinaryTrie, key: BytesContainer): bool =
self.get(toRange(key)) != zeroBytesRange
template exists*(self: BinaryTrie, key: openArray[byte]): bool =
self.get(key) != []
proc delete*(self: var BinaryTrie, key: BytesContainer) {.inline.} =
proc delete*(self: var BinaryTrie, key: openArray[byte]) {.inline.} =
## Equals to setting the value to zeroBytesRange
var keyBits = bits MutByteRange(key.toRange)
self.rootHash = self.setAux(self.rootHash, keyBits, zeroBytesRange)
var keyBits = key.bits
self.rootHash = self.setAux(self.rootHash, keyBits, [])
proc deleteSubtrie*(self: var BinaryTrie, key: BytesContainer) {.inline.} =
proc deleteSubtrie*(self: var BinaryTrie, key: openArray[byte]) {.inline.} =
## Given a key prefix, delete the whole subtrie that starts with the key prefix.
## Key will be encoded into binary array format first.
## It will call `setAux` with `deleteSubtrie` set to true.
var keyBits = bits MutByteRange(key.toRange)
self.rootHash = self.setAux(self.rootHash, keyBits, zeroBytesRange, true)
var keyBits = key.bits
self.rootHash = self.setAux(self.rootHash, keyBits, [], true)
# Convenience
proc rootNode*(self: BinaryTrie): BytesRange {.inline.} =
self.db.get(self.rootHash.toOpenArray).toRange
proc rootNode*(self: BinaryTrie): seq[byte] {.inline.} =
self.db.get(self.rootHash)
proc rootNode*(self: var BinaryTrie, node: BytesContainer) {.inline.} =
self.rootHash = self.hashAndSave(toRange(node))
proc rootNode*(self: var BinaryTrie, node: openArray[byte]) {.inline.} =
self.rootHash = self.hashAndSave(node)
# Dictionary API
template `[]`*(self: BinaryTrie, key: BytesContainer): BytesRange =
template `[]`*(self: BinaryTrie, key: seq[byte]): seq[byte] =
self.get(key)
template `[]=`*(self: var BinaryTrie, key, value: distinct BytesContainer) =
template `[]=`*(self: var BinaryTrie, key, value: seq[byte]) =
self.set(key, value)
template contains*(self: BinaryTrie, key: BytesContainer): bool =
template contains*(self: BinaryTrie, key: seq[byte]): bool =
self.exists(key)

51
eth/trie/branches.nim
View File

@ -1,6 +1,5 @@
import
eth/rlp/types, stew/ranges/bitranges,
trie_defs, binary, binaries, db, trie_utils
./trie_defs, ./binary, ./binaries, ./db, ./trie_utils, ./trie_bitseq
type
DB = TrieDatabaseRef
@ -8,10 +7,10 @@ type
# TODO: replace the usages of this with regular asserts
InvalidKeyError* = object of Defect
template query(db: DB, nodeHash: TrieNodeKey): BytesRange =
db.get(nodeHash.toOpenArray).toRange
template query(db: DB, nodeHash: TrieNodeKey): seq[byte] =
db.get(nodeHash)
proc checkIfBranchExistImpl(db: DB; nodeHash: TrieNodeKey; keyPrefix: TrieBitRange): bool =
proc checkIfBranchExistImpl(db: DB; nodeHash: TrieNodeKey; keyPrefix: TrieBitSeq): bool =
if nodeHash == zeroHash:
return false
@ -37,14 +36,14 @@ proc checkIfBranchExistImpl(db: DB; nodeHash: TrieNodeKey; keyPrefix: TrieBitRan
else:
return checkIfBranchExistImpl(db, node.rightChild, keyPrefix.sliceToEnd(1))
proc checkIfBranchExist*(db: DB; rootHash: BytesContainer | KeccakHash, keyPrefix: BytesContainer): bool =
proc checkIfBranchExist*(db: DB; rootHash: TrieNodeKey, keyPrefix: openArray[byte]): bool =
## Given a key prefix, return whether this prefix is
## the prefix of an existing key in the trie.
checkValidHashZ(rootHash)
var keyPrefixBits = bits MutByteRange(keyPrefix.toRange)
checkIfBranchExistImpl(db, toRange(rootHash), keyPrefixBits)
var keyPrefixBits = bits keyPrefix
checkIfBranchExistImpl(db, rootHash, keyPrefixBits)
proc getBranchImpl(db: DB; nodeHash: TrieNodeKey, keyPath: TrieBitRange, output: var seq[BytesRange]) =
proc getBranchImpl(db: DB; nodeHash: TrieNodeKey, keyPath: TrieBitSeq, output: var seq[seq[byte]]) =
if nodeHash == zeroHash: return
let nodeVal = db.query(nodeHash)
@ -76,14 +75,14 @@ proc getBranchImpl(db: DB; nodeHash: TrieNodeKey, keyPath: TrieBitRange, output:
else:
getBranchImpl(db, node.rightChild, keyPath.sliceToEnd(1), output)
proc getBranch*(db: DB; rootHash: BytesContainer | KeccakHash; key: BytesContainer): seq[BytesRange] =
proc getBranch*(db: DB; rootHash: seq[byte]; key: openArray[byte]): seq[seq[byte]] =
## Get a long-format Merkle branch
checkValidHashZ(rootHash)
result = @[]
var keyBits = bits MutByteRange(key.toRange)
getBranchImpl(db, toRange(rootHash), keyBits, result)
var keyBits = bits key
getBranchImpl(db, rootHash, keyBits, result)
proc isValidBranch*(branch: seq[BytesRange], rootHash: BytesContainer | KeccakHash, key, value: BytesContainer): bool =
proc isValidBranch*(branch: seq[seq[byte]], rootHash: seq[byte], key, value: openArray[byte]): bool =
checkValidHashZ(rootHash)
# branch must not be empty
doAssert(branch.len != 0)
@ -92,18 +91,18 @@ proc isValidBranch*(branch: seq[BytesRange], rootHash: BytesContainer | KeccakHa
for node in branch:
doAssert(node.len != 0)
let nodeHash = keccakHash(node)
db.put(nodeHash.toOpenArray, node.toOpenArray)
db.put(nodeHash.data, node)
var trie = initBinaryTrie(db, rootHash)
result = trie.get(key) == toRange(value)
result = trie.get(key) == value
proc getTrieNodesImpl(db: DB; nodeHash: TrieNodeKey, output: var seq[BytesRange]): bool =
proc getTrieNodesImpl(db: DB; nodeHash: TrieNodeKey, output: var seq[seq[byte]]): bool =
## Get full trie of a given root node
if nodeHash.isZeroHash(): return false
var nodeVal: BytesRange
if nodeHash.toOpenArray in db:
var nodeVal: seq[byte]
if nodeHash in db:
nodeVal = db.query(nodeHash)
else:
return false
@ -121,19 +120,19 @@ proc getTrieNodesImpl(db: DB; nodeHash: TrieNodeKey, output: var seq[BytesRange]
of LEAF_TYPE:
output.add nodeVal
proc getTrieNodes*(db: DB; nodeHash: BytesContainer | KeccakHash): seq[BytesRange] =
proc getTrieNodes*(db: DB; nodeHash: TrieNodeKey): seq[seq[byte]] =
checkValidHashZ(nodeHash)
result = @[]
discard getTrieNodesImpl(db, toRange(nodeHash), result)
discard getTrieNodesImpl(db, nodeHash, result)
proc getWitnessImpl*(db: DB; nodeHash: TrieNodeKey; keyPath: TrieBitRange; output: var seq[BytesRange]) =
proc getWitnessImpl*(db: DB; nodeHash: TrieNodeKey; keyPath: TrieBitSeq; output: var seq[seq[byte]]) =
if keyPath.len == 0:
if not getTrieNodesImpl(db, nodeHash, output): return
if nodeHash.isZeroHash(): return
var nodeVal: BytesRange
if nodeHash.toOpenArray in db:
var nodeVal: seq[byte]
if nodeHash in db:
nodeVal = db.query(nodeHash)
else:
return
@ -157,7 +156,7 @@ proc getWitnessImpl*(db: DB; nodeHash: TrieNodeKey; keyPath: TrieBitRange; outpu
else:
getWitnessImpl(db, node.rightChild, keyPath.sliceToEnd(1), output)
proc getWitness*(db: DB; nodeHash: BytesContainer | KeccakHash; key: BytesContainer): seq[BytesRange] =
proc getWitness*(db: DB; nodeHash: TrieNodeKey; key: openArray[byte]): seq[seq[byte]] =
## Get all witness given a keyPath prefix.
## Include
##
@ -165,5 +164,5 @@ proc getWitness*(db: DB; nodeHash: BytesContainer | KeccakHash; key: BytesContai
## 2. witness in the subtrie of the last node in keyPath
checkValidHashZ(nodeHash)
result = @[]
var keyBits = bits MutByteRange(key.toRange)
getWitnessImpl(db, toRange(nodeHash), keyBits, result)
var keyBits = bits key
getWitnessImpl(db, nodeHash, keyBits, result)

36
eth/trie/db.nim
View File

@ -1,30 +1,22 @@
import
tables, hashes, sets,
nimcrypto/[hash, keccak], eth/rlp,
nimcrypto/[hash, keccak],
trie_defs, db_tracing
type
MemDBRec = object
refCount: int
value: Bytes
value: seq[byte]
MemoryLayer* = ref object of RootObj
records: Table[Bytes, MemDBRec]
deleted: HashSet[Bytes]
TrieDatabaseConcept* = concept DB
mixin put, del, get
put(var DB, KeccakHash, BytesRange)
del(var DB, KeccakHash)
get(DB, KeccakHash) is Bytes
contains(DB, KeccakHash) is bool
records: Table[seq[byte], MemDBRec]
deleted: HashSet[seq[byte]]
# XXX: poor's man vtref types
PutProc = proc (db: RootRef, key, val: openarray[byte]) {.
gcsafe, raises: [Defect, CatchableError] .}
GetProc = proc (db: RootRef, key: openarray[byte]): Bytes {.
GetProc = proc (db: RootRef, key: openarray[byte]): seq[byte] {.
gcsafe, raises: [Defect, CatchableError] .}
## The result will be empty seq if not found
@ -56,14 +48,14 @@ type
TransactionID* = distinct DbTransaction
proc put*(db: TrieDatabaseRef, key, val: openarray[byte]) {.gcsafe.}
proc get*(db: TrieDatabaseRef, key: openarray[byte]): Bytes {.gcsafe.}
proc get*(db: TrieDatabaseRef, key: openarray[byte]): seq[byte] {.gcsafe.}
proc del*(db: TrieDatabaseRef, key: openarray[byte]) {.gcsafe.}
proc beginTransaction*(db: TrieDatabaseRef): DbTransaction {.gcsafe.}
proc keccak*(r: BytesRange): KeccakHash =
keccak256.digest r.toOpenArray
proc keccak*(r: openArray[byte]): KeccakHash =
keccak256.digest r
proc get*(db: MemoryLayer, key: openarray[byte]): Bytes =
proc get*(db: MemoryLayer, key: openarray[byte]): seq[byte] =
result = db.records.getOrDefault(@key).value
traceGet key, result
@ -107,8 +99,8 @@ proc put*(db: MemoryLayer, key, val: openarray[byte]) =
proc newMemoryLayer: MemoryLayer =
result.new
result.records = initTable[Bytes, MemDBRec]()
result.deleted = initHashSet[Bytes]()
result.records = initTable[seq[byte], MemDBRec]()
result.deleted = initHashSet[seq[byte]]()
proc commit(memDb: MemoryLayer, db: TrieDatabaseRef, applyDeletes: bool = true) =
if applyDeletes:
@ -136,7 +128,7 @@ proc totalRecordsInMemoryDB*(db: TrieDatabaseRef): int =
doAssert isMemoryDB(db)
return db.mostInnerTransaction.modifications.records.len
iterator pairsInMemoryDB*(db: TrieDatabaseRef): (Bytes, Bytes) =
iterator pairsInMemoryDB*(db: TrieDatabaseRef): (seq[byte], seq[byte]) =
doAssert isMemoryDB(db)
for k, v in db.mostInnerTransaction.modifications.records:
yield (k, v.value)
@ -178,7 +170,7 @@ proc putImpl[T](db: RootRef, key, val: openarray[byte]) =
mixin put
put(T(db), key, val)
proc getImpl[T](db: RootRef, key: openarray[byte]): Bytes =
proc getImpl[T](db: RootRef, key: openarray[byte]): seq[byte] =
mixin get
return get(T(db), key)
@ -207,7 +199,7 @@ proc put*(db: TrieDatabaseRef, key, val: openarray[byte]) =
else:
db.putProc(db.obj, key, val)
proc get*(db: TrieDatabaseRef, key: openarray[byte]): Bytes =
proc get*(db: TrieDatabaseRef, key: openarray[byte]): seq[byte] =
# TODO: This is quite inefficient and it won't be necessary once
# https://github.com/nim-lang/Nim/issues/7457 is developed.
let key = @key

4
eth/trie/db_tracing.nim
View File

@ -9,11 +9,11 @@ when db_tracing in ["on", "1"]:
template traceGet*(k, v) =
if dbTracingEnabled:
echo "GET ", toHex(k), " = ", toHex(v) # rlpFromBytes(@v.toRange).inspect
echo "GET ", toHex(k), " = ", toHex(v) # rlpFromBytes(v).inspect
template tracePut*(k, v) =
if dbTracingEnabled:
echo "PUT ", toHex(k), " = ", toHex(v) # rlpFromBytes(@v.toRange).inspect
echo "PUT ", toHex(k), " = ", toHex(v) # rlpFromBytes(v).inspect
template traceDel*(k) =
if dbTracingEnabled:

201
eth/trie/hexary.nim
View File

@ -1,7 +1,7 @@
import
tables,
nimcrypto/[keccak, hash, utils], stew/ranges/ptr_arith, eth/rlp,
trie_defs, nibbles, trie_utils as trieUtils, db
nimcrypto/[keccak, hash], eth/rlp,
trie_defs, nibbles, db
type
TrieNodeKey = object
@ -17,28 +17,26 @@ type
SecureHexaryTrie* = distinct HexaryTrie
TrieNode = Rlp
template len(key: TrieNodeKey): int =
key.usedBytes.int
proc keccak*(r: BytesRange): KeccakHash =
keccak256.digest r.toOpenArray
proc keccak*(r: openArray[byte]): KeccakHash =
keccak256.digest r
template asDbKey(k: TrieNodeKey): untyped =
doAssert k.usedBytes == 32
k.hash.data
proc expectHash(r: Rlp): BytesRange =
proc expectHash(r: Rlp): seq[byte] =
result = r.toBytes
if result.len != 32:
raise newException(RlpTypeMismatch,
"RLP expected to be a Keccak hash value, but has an incorrect length")
proc dbPut(db: DB, data: BytesRange): TrieNodeKey {.gcsafe.}
proc dbPut(db: DB, data: openArray[byte]): TrieNodeKey {.gcsafe.}
template get(db: DB, key: Rlp): BytesRange =
db.get(key.expectHash.toOpenArray).toRange
template get(db: DB, key: Rlp): seq[byte] =
db.get(key.expectHash)
converter toTrieNodeKey(hash: KeccakHash): TrieNodeKey =
result.hash = hash
@ -54,7 +52,7 @@ template initSecureHexaryTrie*(db: DB, rootHash: KeccakHash, isPruning = true):
proc initHexaryTrie*(db: DB, isPruning = true): HexaryTrie =
result.db = db
result.root = result.db.dbPut(emptyRlp.toRange)
result.root = result.db.dbPut(emptyRlp)
result.isPruning = isPruning
template initSecureHexaryTrie*(db: DB, isPruning = true): SecureHexaryTrie =
@ -72,38 +70,32 @@ template prune(t: HexaryTrie, x: openArray[byte]) =
proc isPruning*(t: HexaryTrie): bool =
t.isPruning
proc getLocalBytes(x: TrieNodeKey): BytesRange =
proc getLocalBytes(x: TrieNodeKey): seq[byte] =
## This proc should be used on nodes using the optimization
## of short values within the key.
doAssert x.usedBytes < 32
x.hash.data[0..<x.usedBytes]
when defined(rangesEnableUnsafeAPI):
result = unsafeRangeConstruction(x.data, x.usedBytes)
else:
var dataCopy = newSeq[byte](x.usedBytes)
copyMem(dataCopy.baseAddr, x.hash.data.baseAddr, x.usedBytes)
return dataCopy.toRange
template keyToLocalBytes(db: DB, k: TrieNodeKey): BytesRange =
template keyToLocalBytes(db: DB, k: TrieNodeKey): seq[byte] =
if k.len < 32: k.getLocalBytes
else: db.get(k.asDbKey).toRange
else: db.get(k.asDbKey)
template extensionNodeKey(r: Rlp): auto =
hexPrefixDecode r.listElem(0).toBytes
proc getAux(db: DB, nodeRlp: Rlp, path: NibblesRange): BytesRange {.gcsafe.}
proc getAux(db: DB, nodeRlp: Rlp, path: NibblesSeq): seq[byte] {.gcsafe.}
proc getAuxByHash(db: DB, node: TrieNodeKey, path: NibblesRange): BytesRange =
proc getAuxByHash(db: DB, node: TrieNodeKey, path: NibblesSeq): seq[byte] =
var nodeRlp = rlpFromBytes keyToLocalBytes(db, node)
return getAux(db, nodeRlp, path)
template getLookup(elem: untyped): untyped =
if elem.isList: elem
else: rlpFromBytes(get(db, toOpenArray(elem.expectHash)).toRange)
else: rlpFromBytes(get(db, elem.expectHash))
proc getAux(db: DB, nodeRlp: Rlp, path: NibblesRange): BytesRange =
proc getAux(db: DB, nodeRlp: Rlp, path: NibblesSeq): seq[byte] =
if not nodeRlp.hasData or nodeRlp.isEmpty:
return zeroBytesRange
return
case nodeRlp.listLen
of 2:
@ -118,13 +110,13 @@ proc getAux(db: DB, nodeRlp: Rlp, path: NibblesRange): BytesRange =
let nextLookup = value.getLookup
return getAux(db, nextLookup, path.slice(sharedNibbles))
return zeroBytesRange
return
of 17:
if path.len == 0:
return nodeRlp.listElem(16).toBytes
var branch = nodeRlp.listElem(path[0].int)
if branch.isEmpty:
return zeroBytesRange
return
else:
let nextLookup = branch.getLookup
return getAux(db, nextLookup, path.slice(1))
@ -132,10 +124,10 @@ proc getAux(db: DB, nodeRlp: Rlp, path: NibblesRange): BytesRange =
raise newException(CorruptedTrieDatabase,
"HexaryTrie node with an unexpected number of children")
proc get*(self: HexaryTrie; key: BytesRange): BytesRange =
proc get*(self: HexaryTrie; key: openArray[byte]): seq[byte] =
return getAuxByHash(self.db, self.root, initNibbleRange(key))
proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesRange]]): BytesRange =
proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]): seq[byte] =
while stack.len > 0:
let (nodeRlp, path) = stack.pop()
if not nodeRlp.hasData or nodeRlp.isEmpty:
@ -172,15 +164,14 @@ proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesRange]])
raise newException(CorruptedTrieDatabase,
"HexaryTrie node with an unexpected number of children")
iterator keys*(self: HexaryTrie): BytesRange =
iterator keys*(self: HexaryTrie): seq[byte] =
var
nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root)
path = newRange[byte](0)
stack = @[(nodeRlp, initNibbleRange(path))]
stack = @[(nodeRlp, initNibbleRange([]))]
while stack.len > 0:
yield getKeysAux(self.db, stack)
proc getValuesAux(db: DB, stack: var seq[Rlp]): BytesRange =
proc getValuesAux(db: DB, stack: var seq[Rlp]): seq[byte] =
while stack.len > 0:
let nodeRlp = stack.pop()
if not nodeRlp.hasData or nodeRlp.isEmpty:
@ -211,14 +202,14 @@ proc getValuesAux(db: DB, stack: var seq[Rlp]): BytesRange =
raise newException(CorruptedTrieDatabase,
"HexaryTrie node with an unexpected number of children")
iterator values*(self: HexaryTrie): BytesRange =
iterator values*(self: HexaryTrie): seq[byte] =
var
nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root)
stack = @[nodeRlp]
while stack.len > 0:
yield getValuesAux(self.db, stack)
proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesRange]]): (BytesRange, BytesRange) =
proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]): (seq[byte], seq[byte]) =
while stack.len > 0:
let (nodeRlp, path) = stack.pop()
if not nodeRlp.hasData or nodeRlp.isEmpty:
@ -254,11 +245,10 @@ proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesRange]]
raise newException(CorruptedTrieDatabase,
"HexaryTrie node with an unexpected number of children")
iterator pairs*(self: HexaryTrie): (BytesRange, BytesRange) =
iterator pairs*(self: HexaryTrie): (seq[byte], seq[byte]) =
var
nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root)
path = newRange[byte](0)
stack = @[(nodeRlp, initNibbleRange(path))]
stack = @[(nodeRlp, initNibbleRange([]))]
while stack.len > 0:
# perhaps a Nim bug #9778
# cannot yield the helper proc directly
@ -266,7 +256,7 @@ iterator pairs*(self: HexaryTrie): (BytesRange, BytesRange) =
let res = getPairsAux(self.db, stack)
yield res
iterator replicate*(self: HexaryTrie): (BytesRange, BytesRange) =
iterator replicate*(self: HexaryTrie): (seq[byte], seq[byte]) =
# this iterator helps 'rebuild' the entire trie without
# going through a trie algorithm, but it will pull the entire
# low level KV pairs. Thus the target db will only use put operations
@ -274,19 +264,18 @@ iterator replicate*(self: HexaryTrie): (BytesRange, BytesRange) =
var
localBytes = keyToLocalBytes(self.db, self.root)
nodeRlp = rlpFromBytes localBytes
path = newRange[byte](0)
stack = @[(nodeRlp, initNibbleRange(path))]
stack = @[(nodeRlp, initNibbleRange([]))]
template pushOrYield(elem: untyped) =
if elem.isList:
stack.add((elem, key))
else:
let rlpBytes = get(self.db, toOpenArray(elem.expectHash)).toRange
let rlpBytes = get(self.db, elem.expectHash)
let nextLookup = rlpFromBytes(rlpBytes)
stack.add((nextLookup, key))
yield (elem.toBytes, rlpBytes)
yield (self.rootHash.toRange, localBytes)
yield (@(self.rootHash.data), localBytes)
while stack.len > 0:
let (nodeRlp, path) = stack.pop()
if not nodeRlp.hasData or nodeRlp.isEmpty:
@ -310,21 +299,21 @@ iterator replicate*(self: HexaryTrie): (BytesRange, BytesRange) =
raise newException(CorruptedTrieDatabase,
"HexaryTrie node with an unexpected number of children")
proc getValues*(self: HexaryTrie): seq[BytesRange] =
proc getValues*(self: HexaryTrie): seq[seq[byte]] =
result = @[]
for v in self.values:
result.add v
proc getKeys*(self: HexaryTrie): seq[BytesRange] =
proc getKeys*(self: HexaryTrie): seq[seq[byte]] =
result = @[]
for k in self.keys:
result.add k
template getNode(elem: untyped): untyped =
if elem.isList: elem.rawData
else: get(db, toOpenArray(elem.expectHash)).toRange
if elem.isList: @(elem.rawData)
else: get(db, elem.expectHash)
proc getBranchAux(db: DB, node: BytesRange, path: NibblesRange, output: var seq[BytesRange]) =
proc getBranchAux(db: DB, node: openArray[byte], path: NibblesSeq, output: var seq[seq[byte]]) =
var nodeRlp = rlpFromBytes node
if not nodeRlp.hasData or nodeRlp.isEmpty: return
@ -349,21 +338,21 @@ proc getBranchAux(db: DB, node: BytesRange, path: NibblesRange, output: var seq[
raise newException(CorruptedTrieDatabase,
"HexaryTrie node with an unexpected number of children")
proc getBranch*(self: HexaryTrie; key: BytesRange): seq[BytesRange] =
proc getBranch*(self: HexaryTrie; key: openArray[byte]): seq[seq[byte]] =
result = @[]
var node = keyToLocalBytes(self.db, self.root)
result.add node
getBranchAux(self.db, node, initNibbleRange(key), result)
proc dbDel(t: var HexaryTrie, data: BytesRange) =
proc dbDel(t: var HexaryTrie, data: openArray[byte]) =
if data.len >= 32: t.prune(data.keccak.data)
proc dbPut(db: DB, data: BytesRange): TrieNodeKey =
proc dbPut(db: DB, data: openArray[byte]): TrieNodeKey =
result.hash = data.keccak
result.usedBytes = 32
put(db, result.asDbKey, data.toOpenArray)
put(db, result.asDbKey, data)
proc appendAndSave(rlpWriter: var RlpWriter, data: BytesRange, db: DB) =
proc appendAndSave(rlpWriter: var RlpWriter, data: openArray[byte], db: DB) =
if data.len >= 32:
var nodeKey = dbPut(db, data)
rlpWriter.append(nodeKey.hash)
@ -373,7 +362,7 @@ proc appendAndSave(rlpWriter: var RlpWriter, data: BytesRange, db: DB) =
proc isTrieBranch(rlp: Rlp): bool =
rlp.isList and (var len = rlp.listLen; len == 2 or len == 17)
proc replaceValue(data: Rlp, key: NibblesRange, value: BytesRange): Bytes =
proc replaceValue(data: Rlp, key: NibblesSeq, value: openArray[byte]): seq[byte] =
if data.isEmpty:
let prefix = hexPrefixEncode(key, true)
return encodeList(prefix, value)
@ -403,11 +392,6 @@ proc isTwoItemNode(self: HexaryTrie; r: Rlp): bool =
else:
return r.isList and r.listLen == 2
proc isLeaf(r: Rlp): bool =
doAssert r.isList and r.listLen == 2
let b = r.listElem(0).toBytes()
return (b[0] and 0x20) != 0
proc findSingleChild(r: Rlp; childPos: var byte): Rlp =
result = zeroBytesRlp
var i: byte = 0
@ -421,10 +405,10 @@ proc findSingleChild(r: Rlp; childPos: var byte): Rlp =
return zeroBytesRlp
inc i
proc deleteAt(self: var HexaryTrie; origRlp: Rlp, key: NibblesRange): BytesRange {.gcsafe.}
proc deleteAt(self: var HexaryTrie; origRlp: Rlp, key: NibblesSeq): seq[byte] {.gcsafe.}
proc deleteAux(self: var HexaryTrie; rlpWriter: var RlpWriter;
origRlp: Rlp; path: NibblesRange): bool =
origRlp: Rlp; path: NibblesSeq): bool =
if origRlp.isEmpty:
return false
@ -439,16 +423,15 @@ proc deleteAux(self: var HexaryTrie; rlpWriter: var RlpWriter;
rlpWriter.appendAndSave(b, self.db)
return true
proc graft(self: var HexaryTrie; r: Rlp): Bytes =
proc graft(self: var HexaryTrie; r: Rlp): seq[byte] =
doAssert r.isList and r.listLen == 2
var (origIsLeaf, origPath) = r.extensionNodeKey
var (_, origPath) = r.extensionNodeKey
var value = r.listElem(1)
var n: Rlp
if not value.isList:
let nodeKey = value.expectHash
var resolvedData = self.db.get(nodeKey.toOpenArray).toRange
self.prune(nodeKey.toOpenArray)
var resolvedData = self.db.get(nodeKey)
self.prune(nodeKey)
value = rlpFromBytes resolvedData
doAssert value.listLen == 2
@ -460,10 +443,10 @@ proc graft(self: var HexaryTrie; r: Rlp): Bytes =
return rlpWriter.finish
proc mergeAndGraft(self: var HexaryTrie;
soleChild: Rlp, childPos: byte): Bytes =
soleChild: Rlp, childPos: byte): seq[byte] =
var output = initRlpList(2)
if childPos == 16:
output.append hexPrefixEncode(zeroNibblesRange, true)
output.append hexPrefixEncode(NibblesSeq(), true)
else:
doAssert(not soleChild.isEmpty)
output.append int(hexPrefixEncodeByte(childPos))
@ -471,20 +454,20 @@ proc mergeAndGraft(self: var HexaryTrie;
result = output.finish()
if self.isTwoItemNode(soleChild):
result = self.graft(rlpFromBytes(result.toRange))
result = self.graft(rlpFromBytes(result))
proc deleteAt(self: var HexaryTrie;
origRlp: Rlp, key: NibblesRange): BytesRange =
origRlp: Rlp, key: NibblesSeq): seq[byte] =
if origRlp.isEmpty:
return zeroBytesRange
return
doAssert origRlp.isTrieBranch
let origBytes = origRlp.rawData
let origBytes = @(origRlp.rawData)
if origRlp.listLen == 2:
let (isLeaf, k) = origRlp.extensionNodeKey
if k == key and isLeaf:
self.dbDel origBytes
return emptyRlp.toRange
return emptyRlp
if key.startsWith(k):
var
@ -493,22 +476,22 @@ proc deleteAt(self: var HexaryTrie;
value = origRlp.listElem(1)
rlpWriter.append(path)
if not self.deleteAux(rlpWriter, value, key.slice(k.len)):
return zeroBytesRange
return
self.dbDel origBytes
var finalBytes = rlpWriter.finish.toRange
var finalBytes = rlpWriter.finish
var rlp = rlpFromBytes(finalBytes)
if self.isTwoItemNode(rlp.listElem(1)):
return self.graft(rlp).toRange
return self.graft(rlp)
return finalBytes
else:
return zeroBytesRange
return
else:
if key.len == 0 and origRlp.listElem(16).isEmpty:
self.dbDel origBytes
var foundChildPos: byte
let singleChild = origRlp.findSingleChild(foundChildPos)
if singleChild.hasData and foundChildPos != 16:
result = self.mergeAndGraft(singleChild, foundChildPos).toRange
result = self.mergeAndGraft(singleChild, foundChildPos)
else:
var rlpRes = initRlpList(17)
var iter = origRlp
@ -518,7 +501,7 @@ proc deleteAt(self: var HexaryTrie;
rlpRes.append iter
iter.skipElem
rlpRes.append ""
return rlpRes.finish.toRange
return rlpRes.finish
else:
var rlpWriter = initRlpList(17)
let keyHead = int(key[0])
@ -527,20 +510,20 @@ proc deleteAt(self: var HexaryTrie;
for elem in items(origCopy):
if i == keyHead:
if not self.deleteAux(rlpWriter, elem, key.slice(1)):
return zeroBytesRange
return
else:
rlpWriter.append(elem)
inc i
self.dbDel origBytes
result = rlpWriter.finish.toRange
result = rlpWriter.finish
var resultRlp = rlpFromBytes(result)
var foundChildPos: byte
let singleChild = resultRlp.findSingleChild(foundChildPos)
if singleChild.hasData:
result = self.mergeAndGraft(singleChild, foundChildPos).toRange
result = self.mergeAndGraft(singleChild, foundChildPos)
proc del*(self: var HexaryTrie; key: BytesRange) =
proc del*(self: var HexaryTrie; key: openArray[byte]) =
var
rootBytes = keyToLocalBytes(self.db, self.root)
rootRlp = rlpFromBytes rootBytes
@ -552,16 +535,16 @@ proc del*(self: var HexaryTrie; key: BytesRange) =
self.root = self.db.dbPut(newRootBytes)
proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
key: NibblesRange, value: BytesRange,
isInline = false): BytesRange {.gcsafe.}
key: NibblesSeq, value: openArray[byte],
isInline = false): seq[byte] {.gcsafe.}
proc mergeAt(self: var HexaryTrie, rlp: Rlp,
key: NibblesRange, value: BytesRange,
isInline = false): BytesRange =
key: NibblesSeq, value: openArray[byte],
isInline = false): seq[byte] =
self.mergeAt(rlp, rlp.rawData.keccak, key, value, isInline)
proc mergeAtAux(self: var HexaryTrie, output: var RlpWriter, orig: Rlp,
key: NibblesRange, value: BytesRange) =
key: NibblesSeq, value: openArray[byte]) =
var resolved = orig
var isRemovable = false
if not (orig.isList or orig.isEmpty):
@ -572,11 +555,11 @@ proc mergeAtAux(self: var HexaryTrie, output: var RlpWriter, orig: Rlp,
output.appendAndSave(b, self.db)
proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
key: NibblesRange, value: BytesRange,
isInline = false): BytesRange =
key: NibblesSeq, value: openArray[byte],
isInline = false): seq[byte] =
template origWithNewValue: auto =
self.prune(origHash.data)
replaceValue(orig, key, value).toRange
replaceValue(orig, key, value)
if orig.isEmpty:
return origWithNewValue()
@ -595,7 +578,7 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
var r = initRlpList(2)
r.append orig.listElem(0)
self.mergeAtAux(r, origValue, key.slice(k.len), value)
return r.finish.toRange
return r.finish
if orig.rawData.len >= 32:
self.prune(origHash.data)
@ -608,9 +591,9 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
var top = initRlpList(2)
top.append hexPrefixEncode(k.slice(0, sharedNibbles), false)
top.appendAndSave(bottom.finish.toRange, self.db)
top.appendAndSave(bottom.finish, self.db)
return self.mergeAt(rlpFromBytes(top.finish.toRange), key, value, true)
return self.mergeAt(rlpFromBytes(top.finish), key, value, true)
else:
# Create a branch node
var branches = initRlpList(17)
@ -626,7 +609,7 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
if byte(i) == n:
if isLeaf or k.len > 1:
let childNode = encodeList(hexPrefixEncode(k.slice(1), isLeaf),
origValue).toRange
origValue)
branches.appendAndSave(childNode, self.db)
else:
branches.append origValue
@ -634,7 +617,7 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
branches.append ""
branches.append ""
return self.mergeAt(rlpFromBytes(branches.finish.toRange), key, value, true)
return self.mergeAt(rlpFromBytes(branches.finish), key, value, true)
else:
if key.len == 0:
return origWithNewValue()
@ -654,12 +637,12 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash,
r.append(elem)
inc i
return r.finish.toRange
return r.finish
proc put*(self: var HexaryTrie; key, value: BytesRange) =
proc put*(self: var HexaryTrie; key, value: openArray[byte]) =
let root = self.root.hash
var rootBytes = self.db.get(root.data).toRange
var rootBytes = self.db.get(root.data)
doAssert rootBytes.len > 0
let newRootBytes = self.mergeAt(rlpFromBytes(rootBytes), root,
@ -669,23 +652,19 @@ proc put*(self: var HexaryTrie; key, value: BytesRange) =
self.root = self.db.dbPut(newRootBytes)
proc put*(self: var SecureHexaryTrie; key, value: BytesRange) =
let keyHash = @(key.keccak.data)
put(HexaryTrie(self), keyHash.toRange, value)
proc put*(self: var SecureHexaryTrie; key, value: openArray[byte]) =
put(HexaryTrie(self), key.keccak.data, value)
proc get*(self: SecureHexaryTrie; key: BytesRange): BytesRange =
let keyHash = @(key.keccak.data)
return get(HexaryTrie(self), keyHash.toRange)
proc get*(self: SecureHexaryTrie; key: openArray[byte]): seq[byte] =
return get(HexaryTrie(self), key.keccak.data)
proc del*(self: var SecureHexaryTrie; key: BytesRange) =
let keyHash = @(key.keccak.data)
del(HexaryTrie(self), keyHash.toRange)
proc del*(self: var SecureHexaryTrie; key: openArray[byte]) =
del(HexaryTrie(self), key.keccak.data)
proc rootHash*(self: SecureHexaryTrie): KeccakHash {.borrow.}
proc rootHashHex*(self: SecureHexaryTrie): string {.borrow.}
proc isPruning*(self: SecureHexaryTrie): bool {.borrow.}
template contains*(self: HexaryTrie | SecureHexaryTrie;
key: BytesRange): bool =
key: openArray[byte]): bool =
self.get(key).len > 0

60
eth/trie/nibbles.nim
View File

@ -1,33 +1,26 @@
import
trie_defs
type
NibblesRange* = object
bytes: ByteRange
NibblesSeq* = object
bytes: seq[byte]
ibegin, iend: int
proc initNibbleRange*(bytes: ByteRange): NibblesRange =
result.bytes = bytes
proc initNibbleRange*(bytes: openArray[byte]): NibblesSeq =
result.bytes = @bytes
result.ibegin = 0
result.iend = bytes.len * 2
# can't be a const: https://github.com/status-im/nim-eth/issues/6
# we can't initialise it here, but since it's already zeroed memory, we don't need to
var zeroNibblesRange* {.threadvar.}: NibblesRange
proc `{}`(r: NibblesRange, pos: int): byte {.inline.} =
proc `{}`(r: NibblesSeq, pos: int): byte {.inline.} =
## This is a helper for a more raw access to the nibbles.
## It works with absolute positions.
if pos > r.iend: raise newException(RangeError, "index out of range")
return if (pos and 1) != 0: (r.bytes[pos div 2] and 0xf)
else: (r.bytes[pos div 2] shr 4)
template `[]`*(r: NibblesRange, i: int): byte = r{r.ibegin + i}
template `[]`*(r: NibblesSeq, i: int): byte = r{r.ibegin + i}
proc len*(r: NibblesRange): int =
proc len*(r: NibblesSeq): int =
r.iend - r.ibegin
proc `==`*(lhs, rhs: NibblesRange): bool =
proc `==`*(lhs, rhs: NibblesSeq): bool =
if lhs.len == rhs.len:
for i in 0 ..< lhs.len:
if lhs[i] != rhs[i]:
@ -36,7 +29,7 @@ proc `==`*(lhs, rhs: NibblesRange): bool =
else:
return false
proc `$`*(r: NibblesRange): string =
proc `$`*(r: NibblesSeq): string =
result = newStringOfCap(100)
for i in r.ibegin ..< r.iend:
let n = int r{i}
@ -44,7 +37,7 @@ proc `$`*(r: NibblesRange): string =
else: char(ord('0') + n)
result.add c
proc slice*(r: NibblesRange, ibegin: int, iend = -1): NibblesRange =
proc slice*(r: NibblesSeq, ibegin: int, iend = -1): NibblesSeq =
result.bytes = r.bytes
result.ibegin = r.ibegin + ibegin
let e = if iend < 0: r.iend + iend + 1
@ -69,11 +62,11 @@ template writeNibbles(r) {.dirty.} =
result[writeHead] = nextNibble shl 4
oddnessFlag = not oddnessFlag
proc hexPrefixEncode*(r: NibblesRange, isLeaf = false): Bytes =
proc hexPrefixEncode*(r: NibblesSeq, isLeaf = false): seq[byte] =
writeFirstByte(r.len)
writeNibbles(r)
proc hexPrefixEncode*(r1, r2: NibblesRange, isLeaf = false): Bytes =
proc hexPrefixEncode*(r1, r2: NibblesSeq, isLeaf = false): seq[byte] =
writeFirstByte(r1.len + r2.len)
writeNibbles(r1)
writeNibbles(r2)
@ -82,16 +75,16 @@ proc hexPrefixEncodeByte*(val: byte, isLeaf = false): byte =
doAssert val < 16
result = (((byte(isLeaf) * 2) + 1) shl 4) or val
proc sharedPrefixLen*(lhs, rhs: NibblesRange): int =
proc sharedPrefixLen*(lhs, rhs: NibblesSeq): int =
result = 0
while result < lhs.len and result < rhs.len:
if lhs[result] != rhs[result]: break
inc result
proc startsWith*(lhs, rhs: NibblesRange): bool =
proc startsWith*(lhs, rhs: NibblesSeq): bool =
sharedPrefixLen(lhs, rhs) == rhs.len
proc hexPrefixDecode*(r: ByteRange): tuple[isLeaf: bool, nibbles: NibblesRange] =
proc hexPrefixDecode*(r: openArray[byte]): tuple[isLeaf: bool, nibbles: NibblesSeq] =
result.nibbles = initNibbleRange(r)
if r.len > 0:
result.isLeaf = (r[0] and 0x20) != 0
@ -115,7 +108,7 @@ template putNibbles(bytes, src: untyped) =
template calcNeededBytes(len: int): int =
(len shr 1) + (len and 1)
proc `&`*(a, b: NibblesRange): NibblesRange =
proc `&`*(a, b: NibblesSeq): NibblesSeq =
let
len = a.len + b.len
bytesNeeded = calcNeededBytes(len)
@ -128,10 +121,10 @@ proc `&`*(a, b: NibblesRange): NibblesRange =
bytes.putNibbles(a)
bytes.putNibbles(b)
result = initNibbleRange(bytes.toRange)
result = initNibbleRange(bytes)
result.iend = len
proc cloneAndReserveNibble*(a: NibblesRange): NibblesRange =
proc cloneAndReserveNibble*(a: NibblesSeq): NibblesSeq =
let
len = a.len + 1
bytesNeeded = calcNeededBytes(len)
@ -142,24 +135,15 @@ proc cloneAndReserveNibble*(a: NibblesRange): NibblesRange =
pos = 0
bytes.putNibbles(a)
result = initNibbleRange(bytes.toRange)
result = initNibbleRange(bytes)
result.iend = len
proc replaceLastNibble*(a: var NibblesRange, b: byte) =
proc replaceLastNibble*(a: var NibblesSeq, b: byte) =
var
odd = (a.len and 1) == 0
pos = (a.len shr 1) - odd.int
putNibble(MutRange[byte](a.bytes), b)
putNibble(a.bytes, b)
proc getBytes*(a: NibblesRange): ByteRange =
proc getBytes*(a: NibblesSeq): seq[byte] =
a.bytes
when false:
proc keyOf(r: ByteRange): NibblesRange =
let firstIdx = if r.len == 0: 0
elif (r[0] and 0x10) != 0: 1
else: 2
return initNibbleRange(s).slice(firstIdx)

136
eth/trie/sparse_binary.nim
View File

@ -1,9 +1,9 @@
import
stew/ranges/[typedranges, bitranges], eth/rlp/types,
trie_defs, trie_utils, db, sparse_proofs
./trie_bitseq,
./trie_defs, ./trie_utils, ./db, ./sparse_proofs
export
types, trie_utils, bitranges,
trie_utils, trie_bitseq,
sparse_proofs.verifyProof
type
@ -11,11 +11,7 @@ type
SparseBinaryTrie* = object
db: DB
rootHash: ByteRange
proc `==`(a: ByteRange, b: KeccakHash): bool =
if a.len != b.data.len: return false
equalMem(a.baseAddr, b.data[0].unsafeAddr, a.len)
rootHash: seq[byte]
type
# 256 * 2 div 8
@ -24,83 +20,83 @@ type
proc initDoubleHash(a, b: openArray[byte]): DoubleHash =
doAssert(a.len == 32, $a.len)
doAssert(b.len == 32, $b.len)
copyMem(result[ 0].addr, a[0].unsafeAddr, 32)
copyMem(result[32].addr, b[0].unsafeAddr, 32)
result[0..31] = a
result[32..^1] = b
proc initDoubleHash(x: ByteRange): DoubleHash =
initDoubleHash(x.toOpenArray, x.toOpenArray)
proc initDoubleHash(x: openArray[byte]): DoubleHash =
initDoubleHash(x, x)
proc init*(x: typedesc[SparseBinaryTrie], db: DB): SparseBinaryTrie =
result.db = db
# Initialize an empty tree with one branch
var value = initDoubleHash(emptyNodeHashes[0])
result.rootHash = keccakHash(value)
result.db.put(result.rootHash.toOpenArray, value)
var value = initDoubleHash(emptyNodeHashes[0].data)
result.rootHash = @(keccakHash(value).data)
result.db.put(result.rootHash, value)
for i in 0..<treeHeight - 1:
value = initDoubleHash(emptyNodeHashes[i+1])
result.db.put(emptyNodeHashes[i].toOpenArray, value)
value = initDoubleHash(emptyNodeHashes[i+1].data)
result.db.put(emptyNodeHashes[i].data, value)
result.db.put(emptyLeafNodeHash.data, zeroBytesRange.toOpenArray)
result.db.put(emptyLeafNodeHash.data, [])
proc initSparseBinaryTrie*(db: DB): SparseBinaryTrie =
init(SparseBinaryTrie, db)
proc init*(x: typedesc[SparseBinaryTrie], db: DB,
rootHash: BytesContainer | KeccakHash): SparseBinaryTrie =
rootHash: openArray[byte]): SparseBinaryTrie =
checkValidHashZ(rootHash)
result.db = db
result.rootHash = rootHash
result.rootHash = @rootHash
proc initSparseBinaryTrie*(db: DB, rootHash: BytesContainer | KeccakHash): SparseBinaryTrie =
proc initSparseBinaryTrie*(db: DB, rootHash: openArray[byte]): SparseBinaryTrie =
init(SparseBinaryTrie, db, rootHash)
proc getDB*(t: SparseBinaryTrie): auto = t.db
proc getRootHash*(self: SparseBinaryTrie): ByteRange {.inline.} =
proc getRootHash*(self: SparseBinaryTrie): seq[byte] {.inline.} =
self.rootHash
proc getAux(self: SparseBinaryTrie, path: BitRange, rootHash: ByteRange): ByteRange =
var nodeHash = rootHash
proc getAux(self: SparseBinaryTrie, path: TrieBitSeq, rootHash: openArray[byte]): seq[byte] =
var nodeHash = @rootHash
for targetBit in path:
let value = self.db.get(nodeHash.toOpenArray).toRange
if value.len == 0: return zeroBytesRange
let value = self.db.get(nodeHash)
if value.len == 0: return
if targetBit: nodeHash = value[32..^1]
else: nodeHash = value[0..31]
if nodeHash.toOpenArray == emptyLeafNodeHash.data:
result = zeroBytesRange
if nodeHash == emptyLeafNodeHash.data:
result = @[]
else:
result = self.db.get(nodeHash.toOpenArray).toRange
result = self.db.get(nodeHash)
proc get*(self: SparseBinaryTrie, key: BytesContainer): ByteRange =
proc get*(self: SparseBinaryTrie, key: openArray[byte]): seq[byte] =
## gets a key from the tree.
doAssert(key.len == pathByteLen)
let path = MutByteRange(key.toRange).bits
let path = bits key
self.getAux(path, self.rootHash)
proc get*(self: SparseBinaryTrie, key, rootHash: distinct BytesContainer): ByteRange =
proc get*(self: SparseBinaryTrie, key, rootHash: openArray[byte]): seq[byte] =
## gets a key from the tree at a specific root.
doAssert(key.len == pathByteLen)
let path = MutByteRange(key.toRange).bits
self.getAux(path, rootHash.toRange)
let path = bits key
self.getAux(path, rootHash)
proc hashAndSave*(self: SparseBinaryTrie, node: ByteRange): ByteRange =
result = keccakHash(node)
self.db.put(result.toOpenArray, node.toOpenArray)
proc hashAndSave*(self: SparseBinaryTrie, node: openArray[byte]): seq[byte] =
result = @(keccakHash(node).data)
self.db.put(result, node)
proc hashAndSave*(self: SparseBinaryTrie, a, b: ByteRange): ByteRange =
let value = initDoubleHash(a.toOpenArray, b.toOpenArray)
result = keccakHash(value)
self.db.put(result.toOpenArray, value)
proc hashAndSave*(self: SparseBinaryTrie, a, b: openArray[byte]): seq[byte] =
let value = initDoubleHash(a, b)
result = @(keccakHash(value).data)
self.db.put(result, value)
proc setAux(self: var SparseBinaryTrie, value: ByteRange,
path: BitRange, depth: int, nodeHash: ByteRange): ByteRange =
proc setAux(self: var SparseBinaryTrie, value: openArray[byte],
path: TrieBitSeq, depth: int, nodeHash: openArray[byte]): seq[byte] =
if depth == treeHeight:
result = self.hashAndSave(value)
else:
let
node = self.db.get(nodeHash.toOpenArray).toRange
node = self.db.get(nodeHash)
leftNode = node[0..31]
rightNode = node[32..^1]
if path[depth]:
@ -108,75 +104,75 @@ proc setAux(self: var SparseBinaryTrie, value: ByteRange,
else:
result = self.hashAndSave(self.setAux(value, path, depth+1, leftNode), rightNode)
proc set*(self: var SparseBinaryTrie, key, value: distinct BytesContainer) =
proc set*(self: var SparseBinaryTrie, key, value: openArray[byte]) =
## sets a new value for a key in the tree, returns the new root,
## and sets the new current root of the tree.
doAssert(key.len == pathByteLen)
let path = MutByteRange(key.toRange).bits
self.rootHash = self.setAux(value.toRange, path, 0, self.rootHash)
let path = bits key
self.rootHash = self.setAux(value, path, 0, self.rootHash)
proc set*(self: var SparseBinaryTrie, key, value, rootHash: distinct BytesContainer): ByteRange =
proc set*(self: var SparseBinaryTrie, key, value, rootHash: openArray[byte]): seq[byte] =
## sets a new value for a key in the tree at a specific root,
## and returns the new root.
doAssert(key.len == pathByteLen)
let path = MutByteRange(key.toRange).bits
self.setAux(value.toRange, path, 0, rootHash.toRange)
let path = bits key
self.setAux(value, path, 0, rootHash)
template exists*(self: SparseBinaryTrie, key: BytesContainer): bool =
self.get(toRange(key)) != zeroBytesRange
template exists*(self: SparseBinaryTrie, key: openArray[byte]): bool =
self.get(key) != []
proc del*(self: var SparseBinaryTrie, key: BytesContainer) =
proc del*(self: var SparseBinaryTrie, key: openArray[byte]) =
## Equals to setting the value to zeroBytesRange
doAssert(key.len == pathByteLen)
self.set(key, zeroBytesRange)
self.set(key, [])
# Dictionary API
template `[]`*(self: SparseBinaryTrie, key: BytesContainer): ByteRange =
template `[]`*(self: SparseBinaryTrie, key: openArray[byte]): seq[byte] =
self.get(key)
template `[]=`*(self: var SparseBinaryTrie, key, value: distinct BytesContainer) =
template `[]=`*(self: var SparseBinaryTrie, key, value: openArray[byte]) =
self.set(key, value)
template contains*(self: SparseBinaryTrie, key: BytesContainer): bool =
template contains*(self: SparseBinaryTrie, key: openArray[byte]): bool =
self.exists(key)
proc proveAux(self: SparseBinaryTrie, key, rootHash: ByteRange, output: var seq[ByteRange]): bool =
proc proveAux(self: SparseBinaryTrie, key, rootHash: openArray[byte], output: var seq[seq[byte]]): bool =
doAssert(key.len == pathByteLen)
var currVal = self.db.get(rootHash.toOpenArray).toRange
var currVal = self.db.get(rootHash)
if currVal.len == 0: return false
let path = MutByteRange(key).bits
let path = bits key
for i, bit in path:
if bit:
# right side
output[i] = currVal[0..31]
currVal = self.db.get(currVal[32..^1].toOpenArray).toRange
currVal = self.db.get(currVal[32..^1])
if currVal.len == 0: return false
else:
output[i] = currVal[32..^1]
currVal = self.db.get(currVal[0..31].toOpenArray).toRange
currVal = self.db.get(currVal[0..31])
if currVal.len == 0: return false
result = true
# prove generates a Merkle proof for a key.
proc prove*(self: SparseBinaryTrie, key: BytesContainer): seq[ByteRange] =
result = newSeq[ByteRange](treeHeight)
if not self.proveAux(key.toRange, self.rootHash, result):
proc prove*(self: SparseBinaryTrie, key: openArray[byte]): seq[seq[byte]] =
result = newSeq[seq[byte]](treeHeight)
if not self.proveAux(key, self.rootHash, result):
result = @[]
# prove generates a Merkle proof for a key, at a specific root.
proc prove*(self: SparseBinaryTrie, key, rootHash: distinct BytesContainer): seq[ByteRange] =
result = newSeq[ByteRange](treeHeight)
if not self.proveAux(key.toRange, rootHash.toRange, result):
proc prove*(self: SparseBinaryTrie, key, rootHash: openArray[byte]): seq[seq[byte]] =
result = newSeq[seq[byte]](treeHeight)
if not self.proveAux(key, rootHash, result):
result = @[]
# proveCompact generates a compacted Merkle proof for a key.
proc proveCompact*(self: SparseBinaryTrie, key: BytesContainer): seq[ByteRange] =
proc proveCompact*(self: SparseBinaryTrie, key: openArray[byte]): seq[seq[byte]] =
var temp = self.prove(key)
temp.compactProof
# proveCompact generates a compacted Merkle proof for a key, at a specific root.
proc proveCompact*(self: SparseBinaryTrie, key, rootHash: distinct BytesContainer): seq[ByteRange] =
proc proveCompact*(self: SparseBinaryTrie, key, rootHash: openArray[byte]): seq[seq[byte]] =
var temp = self.prove(key, rootHash)
temp.compactProof

50
eth/trie/sparse_proofs.nim
View File

@ -1,26 +1,25 @@
import
stew/ranges/[typedranges, bitranges],
trie_defs, trie_utils
./trie_bitseq, ./trie_defs, /trie_utils
const
treeHeight* = 160
pathByteLen* = treeHeight div 8
emptyLeafNodeHash* = blankStringHash
proc makeInitialEmptyTreeHash(H: static[int]): array[H, ByteRange] =
result[^1] = @(emptyLeafNodeHash.data).toRange
proc makeInitialEmptyTreeHash(H: static[int]): array[H, KeccakHash] =
result[^1] = emptyLeafNodeHash
for i in countdown(H-1, 1):
result[i - 1] = keccakHash(result[i], result[i])
result[i - 1] = keccakHash(result[i].data, result[i].data)
# cannot yet turn this into compile time constant
let emptyNodeHashes* = makeInitialEmptyTreeHash(treeHeight)
# VerifyProof verifies a Merkle proof.
proc verifyProofAux*(proof: seq[ByteRange], root, key, value: ByteRange): bool =
proc verifyProofAux*(proof: seq[seq[byte]], root, key, value: openArray[byte]): bool =
doAssert(root.len == 32)
doAssert(key.len == pathByteLen)
var
path = MutByteRange(key).bits
path = bits key
curHash = keccakHash(value)
if proof.len != treeHeight: return false
@ -30,57 +29,58 @@ proc verifyProofAux*(proof: seq[ByteRange], root, key, value: ByteRange): bool =
if node.len != 32: return false
if path[i]: # right
# reuse curHash without more alloc
curHash.keccakHash(node, curHash)
curHash.data.keccakHash(node, curHash.data)
else:
curHash.keccakHash(curHash, node)
curHash.data.keccakHash(curHash.data, node)
result = curHash == root
result = curHash.data == root
template verifyProof*(proof: seq[ByteRange], root, key, value: distinct BytesContainer): bool =
verifyProofAux(proof, root.toRange, key.toRange, value.toRange)
template verifyProof*(proof: seq[seq[byte]], root, key, value: openArray[byte]): bool =
verifyProofAux(proof, root, key, value)
proc count(b: BitRange, val: bool): int =
proc count(b: TrieBitSeq, val: bool): int =
for c in b:
if c == val: inc result
# CompactProof compacts a proof, to reduce its size.
proc compactProof*(proof: seq[ByteRange]): seq[ByteRange] =
proc compactProof*(proof: seq[seq[byte]]): seq[seq[byte]] =
if proof.len != treeHeight: return
var
data = newRange[byte](pathByteLen)
bits = MutByteRange(data).bits
data = newSeq[byte](pathByteLen)
bits = bits data
result = @[]
result.add data
result.add @[]
for i in 0 ..< treeHeight:
var node = proof[i]
if node == emptyNodeHashes[i]:
if node == emptyNodeHashes[i].data:
bits[i] = true
else:
result.add node
result[0] = bits.toBytes
# decompactProof decompacts a proof, so that it can be used for VerifyProof.
proc decompactProof*(proof: seq[ByteRange]): seq[ByteRange] =
proc decompactProof*(proof: seq[seq[byte]]): seq[seq[byte]] =
if proof.len == 0: return
if proof[0].len != pathByteLen: return
var bits = MutByteRange(proof[0]).bits
let bits = bits proof[0]
if proof.len != bits.count(false) + 1: return
result = newSeq[ByteRange](treeHeight)
result = newSeq[seq[byte]](treeHeight)
var pos = 1 # skip bits
for i in 0 ..< treeHeight:
if bits[i]:
result[i] = emptyNodeHashes[i]
result[i] = @(emptyNodeHashes[i].data)
else:
result[i] = proof[pos]
inc pos
# verifyCompactProof verifies a compacted Merkle proof.
proc verifyCompactProofAux*(proof: seq[ByteRange], root, key, value: ByteRange): bool =
proc verifyCompactProofAux*(proof: seq[seq[byte]], root, key, value: openArray[byte]): bool =
var decompactedProof = decompactProof(proof)
if decompactedProof.len == 0: return false
verifyProofAux(decompactedProof, root, key, value)
template verifyCompactProof*(proof: seq[ByteRange], root, key, value: distinct BytesContainer): bool =
verifyCompactProofAux(proof, root.toRange, key.toRange, value.toRange)
template verifyCompactProof*(proof: seq[seq[byte]], root, key, value: openArray[byte]): bool =
verifyCompactProofAux(proof, root, key, value)

129
eth/trie/trie_bitseq.nim Normal file
View File

@ -0,0 +1,129 @@
import
stew/bitops2
type
TrieBitSeq* = object
## Bit sequence as used in ethereum tries
data: seq[byte]
start: int
mLen: int ## Length in bits
template `@`(s, idx: untyped): untyped =
(when idx is BackwardsIndex: s.len - int(idx) else: int(idx))
proc bits*(a: seq[byte], start, len: int): TrieBitSeq =
doAssert start <= len
doAssert len <= 8 * a.len
TrieBitSeq(data: a, start: start, mLen: len)
template bits*(a: seq[byte]): TrieBitSeq =
bits(a, 0, a.len * 8)
template bits*(a: seq[byte], len: int): TrieBitSeq =
bits(a, 0, len)
template bits*(a: openArray[byte], start, len: int): TrieBitSeq =
bits(@a, start, len)
template bits*(a: openArray[byte]): TrieBitSeq =
bits(@a, 0, a.len * 8)
template bits*(a: openArray[byte], len: int): TrieBitSeq =
bits(@a, 0, len)
template bits*(x: TrieBitSeq): TrieBitSeq = x
proc len*(r: TrieBitSeq): int = r.mLen
iterator enumerateBits(x: TrieBitSeq): (int, bool) =
var p = x.start
var i = 0
let e = x.len
while i != e:
yield (i, getBitBE(x.data, p))
inc p
inc i
iterator items*(x: TrieBitSeq): bool =
for _, v in enumerateBits(x): yield v
iterator pairs*(x: TrieBitSeq): (int, bool) =
for i, v in enumerateBits(x): yield (i, v)
proc `[]`*(x: TrieBitSeq, idx: int): bool =
doAssert idx < x.len
let p = x.start + idx
result = getBitBE(x.data, p)
proc sliceNormalized(x: TrieBitSeq, ibegin, iend: int): TrieBitSeq =
doAssert ibegin >= 0 and
ibegin < x.len and
iend < x.len and
iend + 1 >= ibegin # the +1 here allows the result to be
# an empty range
result.data = x.data
result.start = x.start + ibegin
result.mLen = iend - ibegin + 1
proc `[]`*(r: TrieBitSeq, s: HSlice): TrieBitSeq =
sliceNormalized(r, r @ s.a, r @ s.b)
proc `==`*(a, b: TrieBitSeq): bool =
if a.len != b.len: return false
for i in 0 ..< a.len:
if a[i] != b[i]: return false
true
proc `[]=`*(r: var TrieBitSeq, idx: Natural, val: bool) =
doAssert idx < r.len
let absIdx = r.start + idx
changeBitBE(r.data, absIdx, val)
proc pushFront*(x: var TrieBitSeq, val: bool) =
doAssert x.start > 0
dec x.start
x[0] = val
inc x.mLen
template neededBytes(nBits: int): int =
(nBits shr 3) + ord((nBits and 0b111) != 0)
static:
doAssert neededBytes(2) == 1
doAssert neededBytes(8) == 1
doAssert neededBytes(9) == 2
proc `&`*(a, b: TrieBitSeq): TrieBitSeq =
let totalLen = a.len + b.len
var bytes = newSeq[byte](totalLen.neededBytes)
result = bits(bytes, 0, totalLen)
for i in 0 ..< a.len: result.data.changeBitBE(i, a[i])
for i in 0 ..< b.len: result.data.changeBitBE(i + a.len, b[i])
proc `$`*(r: TrieBitSeq): string =
result = newStringOfCap(r.len)
for bit in r:
result.add(if bit: '1' else: '0')
proc fromBits*(T: type, r: TrieBitSeq, offset, num: Natural): T =
doAssert(num <= sizeof(T) * 8)
# XXX: Nim has a bug that a typedesc parameter cannot be used
# in a type coercion, so we must define an alias here:
type TT = T
for i in 0 ..< num:
result = (result shl 1) or TT(r[offset + i])
proc parse*(T: type TrieBitSeq, s: string): TrieBitSeq =
var bytes = newSeq[byte](s.len.neededBytes)
for i, c in s:
case c
of '0': discard
of '1': setBitBE(bytes, i)
else: doAssert false
result = bits(bytes, 0, s.len)
proc toBytes*(r: TrieBitSeq): seq[byte] =
r.data[(r.start div 8)..<((r.mLen - r.start + 7) div 8)]

17
eth/trie/trie_defs.nim
View File

@ -1,12 +1,8 @@
import
eth/rlp, stew/ranges/typedranges, nimcrypto/hash
export
typedranges, Bytes
eth/rlp, nimcrypto/hash
type
KeccakHash* = MDigest[256]
BytesContainer* = ByteRange | Bytes | string
TrieError* = object of CatchableError
# A common base type of all Trie errors.
@ -20,10 +16,6 @@ type
# operate if its database has been tampered with. A swift crash
# will be a more appropriate response.
# can't be a const: https://github.com/status-im/nim-eth/issues/6
# we can't initialise it here, but since it's already zeroed memory, we don't need to
var zeroBytesRange* {.threadvar.}: ByteRange
const
blankStringHash* = "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470".toDigest
emptyRlp* = @[128.byte]
@ -34,10 +26,3 @@ proc read*(rlp: var Rlp, T: typedesc[MDigest]): T {.inline.} =
proc append*(rlpWriter: var RlpWriter, a: MDigest) {.inline.} =
rlpWriter.append(a.data)
proc unnecessary_OpenArrayToRange*(key: openarray[byte]): ByteRange =
## XXX: The name of this proc is intentionally long, because it
## performs a memory allocation and data copying that may be eliminated
## in the future. Avoid renaming it to something similar as `toRange`, so
## it can remain searchable in the code.
toRange(@key)

45
eth/trie/trie_utils.nim
View File

@ -1,42 +1,22 @@
import
stew/byteutils,
stew/ranges/[typedranges, ptr_arith], nimcrypto/[hash, keccak],
trie_defs, binaries
proc toTrieNodeKey*(hash: KeccakHash): TrieNodeKey =
result = newRange[byte](32)
copyMem(result.baseAddr, hash.data.baseAddr, 32)
nimcrypto/[hash, keccak],
trie_defs
template checkValidHashZ*(x: untyped) =
when x.type isnot KeccakHash:
doAssert(x.len == 32 or x.len == 0)
template isZeroHash*(x: ByteRange): bool =
template isZeroHash*(x: openArray[byte]): bool =
x.len == 0
template toRange*(hash: KeccakHash): ByteRange =
toTrieNodeKey(hash)
proc toRange*(str: string): ByteRange =
var s = newSeq[byte](str.len)
if str.len > 0:
copyMem(s[0].addr, str[0].unsafeAddr, str.len)
result = toRange(s)
proc hashFromHex*(bits: static[int], input: string): MDigest[bits] =
MDigest(data: hexToByteArray[bits div 8](input))
template hashFromHex*(s: static[string]): untyped = hashFromHex(s.len * 4, s)
proc keccakHash*(input: openArray[byte]): ByteRange =
var s = newSeq[byte](32)
var ctx: keccak256
ctx.init()
if input.len > 0:
ctx.update(input[0].unsafeAddr, uint(input.len))
ctx.finish s
ctx.clear()
result = toRange(s)
proc keccakHash*(input: openArray[byte]): KeccakHash =
keccak256.digest(input)
proc keccakHash*(dest: var openArray[byte], a, b: openArray[byte]) =
var ctx: keccak256
@ -48,16 +28,7 @@ proc keccakHash*(dest: var openArray[byte], a, b: openArray[byte]) =
ctx.finish dest
ctx.clear()
proc keccakHash*(a, b: openArray[byte]): ByteRange =
var s = newSeq[byte](32)
proc keccakHash*(a, b: openArray[byte]): KeccakHash =
var s: array[32, byte]
keccakHash(s, a, b)
result = toRange(s)
template keccakHash*(input: ByteRange): ByteRange =
keccakHash(input.toOpenArray)
template keccakHash*(a, b: ByteRange): ByteRange =
keccakHash(a.toOpenArray, b.toOpenArray)
template keccakHash*(dest: var ByteRange, a, b: ByteRange) =
keccakHash(dest.toOpenArray, a.toOpenArray, b.toOpenArray)
KeccakHash(data: s)

12
tests/fuzzing/discovery/generate.nim
View File

@ -24,8 +24,8 @@ proc generate() =
# valid data for a Ping packet
block:
let payload = rlp.encode((4, fromAddr, toAddr, expiration())).toRange
let encodedData = @[1.byte] & payload.toSeq()
let payload = rlp.encode((4, fromAddr, toAddr, expiration()))
let encodedData = @[1.byte] & payload
debug "Ping", data=byteutils.toHex(encodedData)
encodedData.toFile(inputsDir & "ping")
@ -33,8 +33,8 @@ proc generate() =
# valid data for a Pong packet
block:
let token = keccak256.digest(@[0])
let payload = rlp.encode((toAddr, token , expiration())).toRange
let encodedData = @[2.byte] & payload.toSeq()
let payload = rlp.encode((toAddr, token , expiration()))
let encodedData = @[2.byte] & payload
debug "Pong", data=byteutils.toHex(encodedData)
encodedData.toFile(inputsDir & "pong")
@ -43,7 +43,7 @@ proc generate() =
block:
var data: array[64, byte]
data[32 .. ^1] = peerKey.toPublicKey().tryGet().toNodeId().toByteArrayBE()
let payload = rlp.encode((data, expiration())).toRange
let payload = rlp.encode((data, expiration()))
let encodedData = @[3.byte] & payload.toSeq()
debug "FindNode", data=byteutils.toHex(encodedData)
@ -65,7 +65,7 @@ proc generate() =
nodes.add((n1Addr.ip, n1Addr.udpPort, n1Addr.tcpPort, n1Key.toPublicKey().tryGet()))
nodes.add((n2Addr.ip, n2Addr.udpPort, n2Addr.tcpPort, n2Key.toPublicKey().tryGet()))
let payload = rlp.encode((nodes, expiration())).toRange
let payload = rlp.encode((nodes, expiration()))
let encodedData = @[4.byte] & payload.toSeq()
debug "Neighbours", data=byteutils.toHex(encodedData)

4
tests/fuzzing/readme.md
View File

@ -19,7 +19,7 @@ These are the mandatory `test` block and the optional `init` block.
Example usage:
```nim
test:
var rlp = rlpFromBytes(@payload.toRange)
var rlp = rlpFromBytes(payload)
discard rlp.inspect()
```
@ -30,7 +30,7 @@ E.g.:
```nim
test:
try:
var rlp = rlpFromBytes(@payload.toRange)
var rlp = rlpFromBytes(payload)
discard rlp.inspect()
except RlpError as e:
debug "Inspect failed", err = e.msg

2
tests/fuzzing/rlp/rlp_inspect.nim
View File

@ -2,7 +2,7 @@ import chronicles, eth/rlp, ../fuzztest
test:
try:
var rlp = rlpFromBytes(@payload.toRange)
var rlp = rlpFromBytes(payload)
discard rlp.inspect()
except RlpError as e:
debug "Inspect failed", err = e.msg

2
tests/p2p/p2p_test_helper.nim
View File

@ -52,7 +52,7 @@ proc packData*(payload: openArray[byte], pk: PrivateKey): seq[byte] =
template sourceDir*: string = currentSourcePath.rsplit(DirSep, 1)[0]
proc recvMsgMock*(msg: openArray[byte]): tuple[msgId: int, msgData: Rlp] =
var rlp = rlpFromBytes(@msg.toRange)
var rlp = rlpFromBytes(msg)
let msgId = rlp.read(int32)
return (msgId.int, rlp)

2
tests/p2p/test_enr.nim
View File

@ -1,7 +1,7 @@
import
net, unittest, options,
nimcrypto/utils,
eth/p2p/enode, eth/p2p/discoveryv5/enr, eth/keys, eth/rlp
eth/p2p/enode, eth/p2p/discoveryv5/enr, eth/keys
suite "ENR":
test "Serialization":

4
tests/p2p/test_waku_mail.nim
View File

@ -49,7 +49,7 @@ procSuite "Waku Mail Client":
let decoded = decode(response.envelope.data, symKey = some(symKey))
require decoded.isSome()
var rlp = rlpFromBytes(decoded.get().payload.toRange)
var rlp = rlpFromBytes(decoded.get().payload)
let output = rlp.read(MailRequest)
check:
output.lower == lower
@ -92,7 +92,7 @@ procSuite "Waku Mail Client":
var envelopes: seq[Envelope]
traceAsyncErrors peer.p2pMessage(envelopes)
var cursor: Bytes
var cursor: seq[byte]
count = count - 1
if count == 0:
cursor = @[]

42
tests/rlp/test_api_usage.nim
View File

@ -1,13 +1,15 @@
{.used.}
import
math, unittest, strutils,
eth/rlp, util/json_testing
math, unittest, strutils, stew/byteutils,
eth/rlp
proc q(s: string): string = "\"" & s & "\""
proc i(s: string): string = s.replace(" ").replace("\n")
proc inspectMatch(r: Rlp, s: string): bool = r.inspect.i == s.i
test "empty bytes are not a proper RLP":
var rlp = rlpFromBytes Bytes(@[]).toRange
var rlp = rlpFromBytes seq[byte](@[])
check:
not rlp.hasData
@ -52,7 +54,7 @@ test "encode/decode object":
var writer = initRlpWriter()
writer.append(input)
let bytes = writer.finish()
var rlp = rlpFromBytes(bytes.toRange)
var rlp = rlpFromBytes(bytes)
var output = rlp.read(MyObj)
check:
@ -66,10 +68,10 @@ test "encode and decode lists":
var
bytes = writer.finish
rlp = rlpFromBytes bytes.toRange
rlp = rlpFromBytes bytes
check:
bytes.hexRepr == "d183666f6fc8836261728362617ac31e2832"
bytes.toHex == "d183666f6fc8836261728362617ac31e2832"
rlp.inspectMatch """
{
"foo"
@ -89,7 +91,7 @@ test "encode and decode lists":
"Lorem ipsum dolor sit amet",
"Donec ligula tortor, egestas eu est vitae")
rlp = rlpFromBytes bytes.toRange
rlp = rlpFromBytes bytes
check:
rlp.listLen == 3
rlp.listElem(0).toInt(int) == 6000
@ -97,7 +99,7 @@ test "encode and decode lists":
rlp.listElem(2).toString == "Donec ligula tortor, egestas eu est vitae"
# test creating RLPs from other RLPs
var list = rlpFromBytes encodeList(rlp.listELem(1), rlp.listELem(0)).toRange
var list = rlpFromBytes encodeList(rlp.listELem(1), rlp.listELem(0))
# test that iteration with enterList/skipElem works as expected
doAssert list.enterList # We already know that we are working with a list
@ -117,11 +119,11 @@ test "toBytes":
let tok = rlp.listElem(1).toBytes()
check:
tok.len == 32
tok.hexRepr == "40ef02798f211da2e8173d37f255be908871ae65060dbb2f77fb29c0421447f4"
tok.toHex == "40ef02798f211da2e8173d37f255be908871ae65060dbb2f77fb29c0421447f4"
test "nested lists":
let listBytes = encode([[1, 2, 3], [5, 6, 7]])
let listRlp = rlpFromBytes listBytes.toRange
let listRlp = rlpFromBytes listBytes
let sublistRlp0 = listRlp.listElem(0)
let sublistRlp1 = listRlp.listElem(1)
check sublistRlp0.listElem(0).toInt(int) == 1
@ -133,12 +135,12 @@ test "nested lists":
test "encoding length":
let listBytes = encode([1,2,3,4,5])
let listRlp = rlpFromBytes listBytes.toRange
let listRlp = rlpFromBytes listBytes
check listRlp.listLen == 5
let emptyListBytes = encode ""
check emptyListBytes.len == 1
let emptyListRlp = rlpFromBytes emptyListBytes.toRange
let emptyListRlp = rlpFromBytes emptyListBytes
check emptyListRlp.blobLen == 0
test "basic decoding":
@ -159,21 +161,21 @@ test "encode byte arrays":
var b2 = [byte(6), 8, 12, 123]
var b3 = @[byte(122), 56, 65, 12]
let rlp = rlpFromBytes(encode((b1, b2, b3)).toRange)
let rlp = rlpFromBytes(encode((b1, b2, b3)))
check:
rlp.listLen == 3
rlp.listElem(0).toBytes().toSeq() == @b1
rlp.listElem(1).toBytes().toSeq() == @b2
rlp.listElem(2).toBytes().toSeq() == @b3
rlp.listElem(0).toBytes() == b1
rlp.listElem(1).toBytes() == b2
rlp.listElem(2).toBytes() == b3
# The first byte here is the length of the datum (132 - 128 => 4)
$(rlp.listElem(1).rawData) == "R[132, 6, 8, 12, 123]"
$(rlp.listElem(1).rawData) == "[132, 6, 8, 12, 123]"
test "empty byte arrays":
var
rlp = rlpFromBytes rlp.encode("").toRange
rlp = rlpFromBytes rlp.encode("")
b = rlp.toBytes
check $b == "R[]"
check $b == "@[]"
test "encode/decode floats":
for f in [high(float64), low(float64), 0.1, 122.23,
@ -202,7 +204,7 @@ test "invalid enum":
writer.append(2)
writer.append(-1)
let bytes = writer.finish()
var rlp = rlpFromBytes(bytes.toRange)
var rlp = rlpFromBytes(bytes)
expect RlpTypeMismatch:
discard rlp.read(MyEnum)
rlp.skipElem()

9
tests/rlp/test_object_serialization.nim
View File

@ -1,7 +1,7 @@
{.used.}
import
unittest, times, eth/rlp, util/json_testing
unittest, times, eth/rlp, stew/byteutils
type
Transaction = object
@ -46,7 +46,7 @@ test "encoding and decoding an object":
f: Foo(x: 5'u64, y: "hocus pocus", z: @[100, 200, 300]))
var bytes = encode(originalBar)
var r = rlpFromBytes(bytes.toRange)
var r = rlpFromBytes(bytes)
var restoredBar = r.read(Bar)
check:
@ -57,7 +57,7 @@ test "encoding and decoding an object":
var t2 = bytes.decode(Transaction)
check:
bytes.hexRepr == "cd85416c69636583426f628203e8" # verifies that Alice comes first
bytes.toHex == "cd85416c69636583426f628203e8" # verifies that Alice comes first
t2.time == default(Time)
t2.sender == "Alice"
t2.receiver == "Bob"
@ -66,7 +66,7 @@ test "encoding and decoding an object":
test "custom field serialization":
var origVal = CustomSerialized(customFoo: Foo(x: 10'u64, y: "y", z: @[]), ignored: 5)
var bytes = encode(origVal)
var r = rlpFromBytes(bytes.toRange)
var r = rlpFromBytes(bytes)
var restored = r.read(CustomSerialized)
check:
@ -79,4 +79,3 @@ test "RLP fields count":
Bar.rlpFieldsCount == 2
Foo.rlpFieldsCount == 3
Transaction.rlpFieldsCount == 3

9
tests/rlp/util/json_testing.nim
View File

@ -1,5 +1,5 @@
import
json, strutils, unittest, eth/rlp
json, stew/byteutils, unittest, eth/rlp
proc append(output: var RlpWriter, js: JsonNode) =
case js.kind
@ -14,11 +14,6 @@ proc append(output: var RlpWriter, js: JsonNode) =
of JArray:
output.append js.elems
proc hexRepr*(bytes: BytesRange|Bytes): string =
result = newStringOfCap(bytes.len * 2)
for byte in bytes:
result.add(toHex(int(byte), 2).toLowerAscii)
proc `==`(lhs: JsonNode, rhs: string): bool =
lhs.kind == JString and lhs.str == rhs
@ -58,7 +53,7 @@ proc runTests*(filename: string) =
var outRlp = initRlpWriter()
outRlp.append input
let
actual = outRlp.finish.hexRepr
actual = outRlp.finish.toHex
expected = output.str
check actual == expected

4
tests/trie/all_tests.nim
View File

@ -6,7 +6,7 @@ import
test_examples,
test_hexary_trie,
test_json_suite,
test_nibbles,
test_sparse_binary_trie,
test_storage_backends,
test_transaction_db
test_transaction_db,
test_trie_bitseq

46
tests/trie/test_bin_trie.nim
View File

@ -2,8 +2,8 @@
import
unittest, random,
eth/trie/[trie_defs, db, binary],
./testutils
eth/trie/[db, binary],
./testutils, stew/byteutils
suite "binary trie":
@ -19,7 +19,7 @@ suite "binary trie":
for i, c in kv_pairs:
trie.set(c.key, c.value)
let x = trie.get(c.key)
let y = toRange(c.value)
let y = c.value
check y == x
check result == zeroHash or trie.getRootHash() == result
@ -54,27 +54,27 @@ suite "binary trie":
let will_raise_error = data[4]
# First test case, delete subtrie of a kv node
trie.set(kv1[0], kv1[1])
trie.set(kv2[0], kv2[1])
check trie.get(kv1[0]) == toRange(kv1[1])
check trie.get(kv2[0]) == toRange(kv2[1])
trie.set(kv1[0].toBytes, kv1[1].toBytes)
trie.set(kv2[0].toBytes, kv2[1].toBytes)
check trie.get(kv1[0].toBytes) == kv1[1].toBytes
check trie.get(kv2[0].toBytes) == kv2[1].toBytes
if will_delete:
trie.deleteSubtrie(key_to_be_deleted)
check trie.get(kv1[0]) == zeroBytesRange
check trie.get(kv2[0]) == zeroBytesRange
trie.deleteSubtrie(key_to_be_deleted.toBytes)
check trie.get(kv1[0].toBytes) == []
check trie.get(kv2[0].toBytes) == []
check trie.getRootHash() == zeroHash
else:
if will_raise_error:
try:
trie.deleteSubtrie(key_to_be_deleted)
trie.deleteSubtrie(key_to_be_deleted.toBytes)
except NodeOverrideError:
discard
else:
let root_hash_before_delete = trie.getRootHash()
trie.deleteSubtrie(key_to_be_deleted)
check trie.get(kv1[0]) == toRange(kv1[1])
check trie.get(kv2[0]) == toRange(kv2[1])
trie.deleteSubtrie(key_to_be_deleted.toBytes)
check trie.get(kv1[0].toBytes) == toBytes(kv1[1])
check trie.get(kv2[0].toBytes) == toBytes(kv2[1])
check trie.getRootHash() == root_hash_before_delete
const invalidKeyData = [
@ -90,22 +90,22 @@ suite "binary trie":
var db = newMemoryDB()
var trie = initBinaryTrie(db)
trie.set("\x12\x34\x56\x78", "78")
trie.set("\x12\x34\x56\x79", "79")
trie.set("\x12\x34\x56\x78".toBytes, "78".toBytes)
trie.set("\x12\x34\x56\x79".toBytes, "79".toBytes)
let invalidKey = data[0]
let if_error = data[1]
check trie.get(invalidKey) == zeroBytesRange
check trie.get(invalidKey.toBytes) == []
if if_error:
try:
trie.delete(invalidKey)
trie.delete(invalidKey.toBytes)
except NodeOverrideError:
discard
else:
let previous_root_hash = trie.getRootHash()
trie.delete(invalidKey)
trie.delete(invalidKey.toBytes)
check previous_root_hash == trie.getRootHash()
test "update value":
@ -114,13 +114,13 @@ suite "binary trie":
var db = newMemoryDB()
var trie = initBinaryTrie(db)
for key in keys:
trie.set(key, "old")
trie.set(key.toBytes, "old".toBytes)
var current_root = trie.getRootHash()
for i in vals:
trie.set(keys[i], "old")
trie.set(keys[i].toBytes, "old".toBytes)
check current_root == trie.getRootHash()
trie.set(keys[i], "new")
trie.set(keys[i].toBytes, "new".toBytes)
check current_root != trie.getRootHash()
check trie.get(keys[i]) == toRange("new")
check trie.get(keys[i].toBytes) == toBytes("new")
current_root = trie.getRootHash()

44
tests/trie/test_binaries_utils.nim
View File

@ -2,11 +2,11 @@
import
unittest, strutils,
stew/ranges/bitranges, eth/rlp/types, nimcrypto/[keccak, hash],
eth/trie/[binaries, trie_utils],
./testutils
nimcrypto/[keccak, hash],
eth/trie/[binaries, trie_bitseq],
./testutils, stew/byteutils
proc parseBitVector(x: string): BitRange =
proc parseBitVector(x: string): TrieBitSeq =
result = genBitVec(x.len)
for i, c in x:
result[i] = (c == '1')
@ -53,7 +53,7 @@ suite "binaries utils":
test "node parsing":
for c in parseNodeData:
let input = toRange(c[0])
let input = toBytes(c[0])
let node = c[1]
let kind = TrieNodeKind(node[0])
let raiseError = node[3]
@ -69,12 +69,12 @@ suite "binaries utils":
case res.kind
of KV_TYPE:
check(res.keyPath == parseBitVector(node[1]))
check(res.child == toRange(node[2]))
check(res.child == toBytes(node[2]))
of BRANCH_TYPE:
check(res.leftChild == toRange(node[2]))
check(res.rightChild == toRange(node[2]))
check(res.leftChild == toBytes(node[2]))
check(res.rightChild == toBytes(node[2]))
of LEAF_TYPE:
check(res.value == toRange(node[2]))
check(res.value == toBytes(node[2]))
const
kvData = [
@ -88,7 +88,7 @@ suite "binaries utils":
test "kv node encoding":
for c in kvData:
let keyPath = parseBitVector(c[0])
let node = toRange(c[1])
let node = toBytes(c[1])
let output = toBytes(c[2])
let raiseError = c[3]
@ -110,8 +110,8 @@ suite "binaries utils":
test "branch node encode":
for c in branchData:
let left = toRange(c[0])
let right = toRange(c[1])
let left = toBytes(c[0])
let right = toBytes(c[1])
let output = toBytes(c[2])
let raiseError = c[3]
@ -132,34 +132,34 @@ suite "binaries utils":
let raiseError = c[2]
if raiseError:
expect(ValidationError):
check toBytes(c[1]) == encodeLeafNode(toRange(c[0]))
check toBytes(c[1]) == encodeLeafNode(toBytes(c[0]))
else:
check toBytes(c[1]) == encodeLeafNode(toRange(c[0]))
check toBytes(c[1]) == encodeLeafNode(toBytes(c[0]))
test "random kv encoding":
let lengths = randList(int, randGen(1, 999), randGen(100, 100), unique = false)
for len in lengths:
var k = len
var bitvec = genBitVec(len)
var nodeHash = keccak256.digest(cast[ptr byte](k.addr), uint(sizeof(int))).toRange
var kvnode = encodeKVNode(bitvec, nodeHash).toRange
var nodeHash = keccak256.digest(cast[ptr byte](k.addr), uint(sizeof(int)))
var kvnode = encodeKVNode(bitvec, @(nodeHash.data))
# first byte if KV_TYPE
# in the middle are 1..n bits of binary-encoded-keypath
# last 32 bytes are hash
var keyPath = decodeToBinKeypath(kvnode[1..^33])
check kvnode[0].ord == KV_TYPE.ord
check keyPath == bitvec
check kvnode[^32..^1] == nodeHash
check kvnode[^32..^1] == nodeHash.data
test "optimized single bit keypath kvnode encoding":
var k = 1
var nodeHash = keccak256.digest(cast[ptr byte](k.addr), uint(sizeof(int))).toRange
var nodeHash = keccak256.digest(cast[ptr byte](k.addr), uint(sizeof(int)))
var bitvec = genBitVec(1)
bitvec[0] = false
var kvnode = encodeKVNode(bitvec, nodeHash).toRange
var kvnode = encodeKVNode(bitvec, @(nodeHash.data))
var kp = decodeToBinKeypath(kvnode[1..^33])
var okv = encodeKVNode(false, nodeHash).toRange
var okv = encodeKVNode(false, @(nodeHash.data))
check okv == kvnode
var okp = decodeToBinKeypath(kvnode[1..^33])
check okp == kp
@ -167,10 +167,10 @@ suite "binaries utils":
check okp == bitvec
bitvec[0] = true
kvnode = encodeKVNode(bitvec, nodeHash).toRange
kvnode = encodeKVNode(bitvec, @(nodeHash.data))
kp = decodeToBinKeypath(kvnode[1..^33])
okv = encodeKVNode(true, nodeHash).toRange
okv = encodeKVNode(true, @(nodeHash.data))
check okv == kvnode
okp = decodeToBinKeypath(kvnode[1..^33])
check okp == kp

22
tests/trie/test_branches_utils.nim
View File

@ -1,7 +1,7 @@
{.used.}
import
sets, unittest, strutils, sets,
sets, unittest, strutils, stew/byteutils,
eth/trie/[db, binary, branches]
suite "branches utils":
@ -10,9 +10,9 @@ suite "branches utils":
var db = newMemoryDB()
var trie = initBinaryTrie(db)
trie.set("\x12\x34\x56\x78\x9a", "9a")
trie.set("\x12\x34\x56\x78\x9b", "9b")
trie.set("\x12\x34\x56\xff", "ff")
trie.set("\x12\x34\x56\x78\x9a".toBytes, "9a".toBytes)
trie.set("\x12\x34\x56\x78\x9b".toBytes, "9b".toBytes)
trie.set("\x12\x34\x56\xff".toBytes, "ff".toBytes)
trie
const branchExistData = [
@ -28,7 +28,7 @@ suite "branches utils":
var trie = testTrie()
var db = trie.getDB()
for c in branchExistData:
let keyPrefix = c[0].toRange
let keyPrefix = c[0].toBytes
let if_exist = c[1]
check checkIfBranchExist(db, trie.getRootHash(), keyPrefix) == if_exist
@ -45,7 +45,7 @@ suite "branches utils":
var trie = testTrie()
var db = trie.getDB()
for c in branchData:
let key = c[0].toRange
let key = c[0].toBytes
let keyValid = c[1]
if keyValid:
@ -83,15 +83,15 @@ suite "branches utils":
(repeat('0', 32), @[])
]
proc toRanges(x: seq[string]): seq[BytesRange] =
result = newSeq[BytesRange](x.len)
for i, c in x: result[i] = toRange(c)
proc toRanges(x: seq[string]): seq[seq[byte]] =
result = newSeq[seq[byte]](x.len)
for i, c in x: result[i] = toBytes(c)
test "get trie nodes":
var trie = testTrie()
var db = trie.getDB()
for c in trieNodesData:
let root = c[0].toRange()
let root = c[0].toBytes()
let nodes = toRanges(c[1])
check toHashSet(nodes) == toHashSet(getTrieNodes(db, root))
@ -135,7 +135,7 @@ suite "branches utils":
var trie = testTrie()
var db = trie.getDB()
for c in witnessData:
let key = c[0].toRange
let key = c[0].toBytes
let nodes = toRanges(c[1])
if nodes.len != 0:

62
tests/trie/test_examples.nim
View File

@ -1,9 +1,9 @@
{.used.}
import
unittest,
unittest, stew/byteutils,
nimcrypto/[keccak, hash],
eth/trie/[trie_defs, db, binary, binaries, trie_utils, branches]
eth/trie/[db, binary, binaries, trie_utils, branches]
suite "examples":
@ -11,84 +11,84 @@ suite "examples":
var trie = initBinaryTrie(db)
test "basic set/get":
trie.set("key1", "value1")
trie.set("key2", "value2")
check trie.get("key1") == "value1".toRange
check trie.get("key2") == "value2".toRange
trie.set("key1".toBytes(), "value1".toBytes())
trie.set("key2".toBytes(), "value2".toBytes())
check trie.get("key1".toBytes) == "value1".toBytes
check trie.get("key2".toBytes) == "value2".toBytes
test "check branch exists":
check checkIfBranchExist(db, trie.getRootHash(), "key") == true
check checkIfBranchExist(db, trie.getRootHash(), "key1") == true
check checkIfBranchExist(db, trie.getRootHash(), "ken") == false
check checkIfBranchExist(db, trie.getRootHash(), "key123") == false
check checkIfBranchExist(db, trie.getRootHash(), "key".toBytes) == true
check checkIfBranchExist(db, trie.getRootHash(), "key1".toBytes) == true
check checkIfBranchExist(db, trie.getRootHash(), "ken".toBytes) == false
check checkIfBranchExist(db, trie.getRootHash(), "key123".toBytes) == false
test "branches utils":
var branchA = getBranch(db, trie.getRootHash(), "key1")
var branchA = getBranch(db, trie.getRootHash(), "key1".toBytes)
# ==> [A, B, C1, D1]
check branchA.len == 4
var branchB = getBranch(db, trie.getRootHash(), "key2")
var branchB = getBranch(db, trie.getRootHash(), "key2".toBytes)
# ==> [A, B, C2, D2]
check branchB.len == 4
check isValidBranch(branchA, trie.getRootHash(), "key1", "value1") == true
check isValidBranch(branchA, trie.getRootHash(), "key5", "") == true
check isValidBranch(branchA, trie.getRootHash(), "key1".toBytes, "value1".toBytes) == true
check isValidBranch(branchA, trie.getRootHash(), "key5".toBytes, "".toBytes) == true
expect InvalidNode:
check isValidBranch(branchB, trie.getRootHash(), "key1", "value1")
check isValidBranch(branchB, trie.getRootHash(), "key1".toBytes, "value1".toBytes)
var x = getBranch(db, trie.getRootHash(), "key")
var x = getBranch(db, trie.getRootHash(), "key".toBytes)
# ==> [A]
check x.len == 1
expect InvalidKeyError:
x = getBranch(db, trie.getRootHash(), "key123") # InvalidKeyError
x = getBranch(db, trie.getRootHash(), "key123".toBytes) # InvalidKeyError
x = getBranch(db, trie.getRootHash(), "key5") # there is still branch for non-exist key
x = getBranch(db, trie.getRootHash(), "key5".toBytes) # there is still branch for non-exist key
# ==> [A]
check x.len == 1
test "getWitness":
var branch = getWitness(db, trie.getRootHash(), "key1")
var branch = getWitness(db, trie.getRootHash(), "key1".toBytes)
# equivalent to `getBranch(db, trie.getRootHash(), "key1")`
# ==> [A, B, C1, D1]
check branch.len == 4
branch = getWitness(db, trie.getRootHash(), "key")
branch = getWitness(db, trie.getRootHash(), "key".toBytes)
# this will include additional nodes of "key2"
# ==> [A, B, C1, D1, C2, D2]
check branch.len == 6
branch = getWitness(db, trie.getRootHash(), "")
branch = getWitness(db, trie.getRootHash(), "".toBytes)
# this will return the whole trie
# ==> [A, B, C1, D1, C2, D2]
check branch.len == 6
let beforeDeleteLen = db.totalRecordsInMemoryDB
test "verify intermediate entries existence":
var branchs = getWitness(db, trie.getRootHash, zeroBytesRange)
var branchs = getWitness(db, trie.getRootHash, [])
# set operation create new intermediate entries
check branchs.len < beforeDeleteLen
var node = branchs[1]
let nodeHash = keccak256.digest(node.baseAddr, uint(node.len))
var nodes = getTrieNodes(db, nodeHash)
let nodeHash = keccak256.digest(node)
var nodes = getTrieNodes(db, @(nodeHash.data))
check nodes.len == branchs.len - 1
test "delete sub trie":
# delete all subtrie with key prefixes "key"
trie.deleteSubtrie("key")
check trie.get("key1") == zeroBytesRange
check trie.get("key2") == zeroBytesRange
trie.deleteSubtrie("key".toBytes)
check trie.get("key1".toBytes) == []
check trie.get("key2".toBytes) == []
test "prove the lie":
# `delete` and `deleteSubtrie` not actually delete the nodes
check db.totalRecordsInMemoryDB == beforeDeleteLen
var branchs = getWitness(db, trie.getRootHash, zeroBytesRange)
var branchs = getWitness(db, trie.getRootHash, [])
check branchs.len == 0
test "dictionary syntax API":
# dictionary syntax API
trie["moon"] = "sun"
check "moon" in trie
check trie["moon"] == "sun".toRange
trie["moon".toBytes] = "sun".toBytes
check "moon".toBytes in trie
check trie["moon".toBytes] == "sun".toBytes

99
tests/trie/test_hexary_trie.nim
View File

@ -1,26 +1,17 @@
{.used.}
import
unittest, sequtils, os,
stew/ranges/typedranges, eth/trie/[hexary, db, trie_defs], nimcrypto/utils,
./testutils, algorithm, eth/rlp/types as rlpTypes, random
unittest, sequtils, os, stew/byteutils,
eth/trie/[hexary, db, trie_defs], nimcrypto/utils,
./testutils, algorithm, random
from strutils import split
template put(t: HexaryTrie|SecureHexaryTrie, key, val: string) =
t.put(key.toBytesRange, val.toBytesRange)
template del(t: HexaryTrie|SecureHexaryTrie, key) =
t.del(key.toBytesRange)
template get(t: HexaryTrie|SecureHexaryTrie, key): auto =
t.get(key.toBytesRange)
suite "hexary trie":
setup:
var
db = newMemoryDB()
tr = initHexaryTrie(db)
tr {.used.} = initHexaryTrie(db)
test "ref-counted keys crash":
proc addKey(intKey: int) =
@ -28,12 +19,12 @@ suite "hexary trie":
key[19] = byte(intKey)
var data = newSeqWith(29, 1.byte)
var k = key.toRange
var k = key
let v = tr.get(k)
doAssert(v.len == 0)
tr.put(k, toRange(data))
tr.put(k, data)
addKey(166)
addKey(193)
@ -78,12 +69,12 @@ suite "hexary trie":
key = fromHex(parts[0])
val = fromHex(parts[1])
SecureHexaryTrie(tr).put(key.toRange, val.toRange)
SecureHexaryTrie(tr).put(key, val)
check tr.rootHashHex == "D7F8974FB5AC78D9AC099B9AD5018BEDC2CE0A72DAD1827A1709DA30580F0544"
# lexicographic comparison
proc lexComp(a, b: BytesRange): bool =
proc lexComp(a, b: seq[byte]): bool =
var
x = 0
y = 0
@ -98,7 +89,7 @@ suite "hexary trie":
result = y != ylen
proc cmp(a, b: BytesRange): int =
proc cmp(a, b: seq[byte]): int =
if a == b: return 0
if a.lexComp(b): return 1
return -1
@ -109,17 +100,17 @@ suite "hexary trie":
memdb = newMemoryDB()
trie = initHexaryTrie(memdb)
keys = [
"key".toBytesRange,
"abc".toBytesRange,
"hola".toBytesRange,
"bubble".toBytesRange
"key".toBytes,
"abc".toBytes,
"hola".toBytes,
"bubble".toBytes
]
vals = [
"hello".toBytesRange,
"world".toBytesRange,
"block".toBytesRange,
"chain".toBytesRange
"hello".toBytes,
"world".toBytes,
"block".toBytes,
"chain".toBytes
]
for i in 0 ..< keys.len:
@ -157,11 +148,11 @@ suite "hexary trie":
var
memdb = newMemoryDB()
trie = initHexaryTrie(memdb)
keys = randList(BytesRange, randGen(5, 32), randGen(10))
vals = randList(BytesRange, randGen(5, 7), randGen(10))
keys = randList(seq[byte], randGen(5, 32), randGen(10))
vals = randList(seq[byte], randGen(5, 7), randGen(10))
keys2 = randList(BytesRange, randGen(5, 30), randGen(15))
vals2 = randList(BytesRange, randGen(5, 7), randGen(15))
keys2 = randList(seq[byte], randGen(5, 30), randGen(15))
vals2 = randList(seq[byte], randGen(5, 7), randGen(15))
for i in 0 ..< keys.len:
trie.put(keys[i], vals[i])
@ -226,11 +217,11 @@ suite "hexary trie":
var
memdb = newMemoryDB()
nonPruningTrie = initHexaryTrie(memdb, false)
keys = randList(BytesRange, randGen(5, 77), randGen(30))
vals = randList(BytesRange, randGen(1, 57), randGen(30))
keys = randList(seq[byte], randGen(5, 77), randGen(30))
vals = randList(seq[byte], randGen(1, 57), randGen(30))
moreKeys = randList(BytesRange, randGen(5, 33), randGen(45))
moreVals = randList(BytesRange, randGen(1, 47), randGen(45))
moreKeys = randList(seq[byte], randGen(5, 33), randGen(45))
moreVals = randList(seq[byte], randGen(1, 47), randGen(45))
for i in 0 ..< keys.len:
nonPruningTrie.put(keys[i], vals[i])
@ -265,8 +256,8 @@ suite "hexary trie":
test "elaborate non-pruning test":
type
History = object
keys: seq[BytesRange]
values: seq[BytesRange]
keys: seq[seq[byte]]
values: seq[seq[byte]]
rootHash: KeccakHash
const
@ -277,14 +268,14 @@ suite "hexary trie":
var
memdb = newMemoryDB()
nonPruningTrie = initHexaryTrie(memdb, false)
keys = randList(BytesRange, randGen(3, 33), randGen(listLength))
values = randList(BytesRange, randGen(5, 77), randGen(listLength))
keys = randList(seq[byte], randGen(3, 33), randGen(listLength))
values = randList(seq[byte], randGen(5, 77), randGen(listLength))
historyList = newSeq[History](listLength)
ok = true
for i, k in keys:
historyList[i].keys = newSeq[BytesRange](i + 1)
historyList[i].values = newSeq[BytesRange](i + 1)
historyList[i].keys = newSeq[seq[byte]](i + 1)
historyList[i].values = newSeq[seq[byte]](i + 1)
for x in 0 ..< i + 1:
historyList[i].keys[x] = keys[x]
historyList[i].values[x] = values[x]
@ -296,7 +287,7 @@ suite "hexary trie":
for h in historyList:
var
trie = initHexaryTrie(memdb, h.rootHash)
pKeys: seq[BytesRange] = @[]
pKeys: seq[seq[byte]] = @[]
pValues = trie.getValues()
for k in trie.keys:
@ -318,7 +309,7 @@ suite "hexary trie":
echo "ITERATION: ", iteration
break
proc isValidBranch(branch: seq[BytesRange], rootHash: KeccakHash, key, value: BytesRange): bool =
proc isValidBranch(branch: seq[seq[byte]], rootHash: KeccakHash, key, value: seq[byte]): bool =
# branch must not be empty
doAssert(branch.len != 0)
@ -326,17 +317,17 @@ suite "hexary trie":
for node in branch:
doAssert(node.len != 0)
let nodeHash = hexary.keccak(node)
db.put(nodeHash.data, node.toOpenArray)
db.put(nodeHash.data, node)
var trie = initHexaryTrie(db, rootHash)
result = trie.get(key) == toRange(value)
result = trie.get(key) == value
test "get branch with pruning trie":
var
memdb = newMemoryDB()
trie = initHexaryTrie(memdb)
keys = randList(BytesRange, randGen(5, 77), randGen(30))
vals = randList(BytesRange, randGen(1, 57), randGen(30))
keys = randList(seq[byte], randGen(5, 77), randGen(30))
vals = randList(seq[byte], randGen(1, 57), randGen(30))
for i in 0 ..< keys.len:
trie.put(keys[i], vals[i])
@ -352,8 +343,8 @@ suite "hexary trie":
var
memdb = newMemoryDB()
nonPruningTrie = initHexaryTrie(memdb, false)
keys = randList(BytesRange, randGen(5, 77), randGen(numKeyVal))
vals = randList(BytesRange, randGen(1, 57), randGen(numKeyVal))
keys = randList(seq[byte], randGen(5, 77), randGen(numKeyVal))
vals = randList(seq[byte], randGen(1, 57), randGen(numKeyVal))
roots = newSeq[KeccakHash](numKeyVal)
for i in 0 ..< keys.len:
@ -388,9 +379,9 @@ suite "hexary trie":
pruningTrie = initHexaryTrie(memdb, isPruning = true)
let
keys = randList(BytesRange, randGen(5, 77), randGen(numKeyVal))
vals = randList(BytesRange, randGen(1, 57), randGen(numKeyVal))
newVals = randList(BytesRange, randGen(1, 63), randGen(numKeyVal))
keys = randList(seq[byte], randGen(5, 77), randGen(numKeyVal))
vals = randList(seq[byte], randGen(1, 57), randGen(numKeyVal))
newVals = randList(seq[byte], randGen(1, 63), randGen(numKeyVal))
var tx1 = memdb.beginTransaction()
for i in 0 ..< numKeyVal:
@ -426,15 +417,15 @@ suite "hexary trie":
pruningTrie = initHexaryTrie(memdb, isPruning = true)
let
keys = randList(BytesRange, randGen(5, 77), randGen(numKeyVal))
vals = randList(BytesRange, randGen(1, 57), randGen(numKeyVal))
keys = randList(seq[byte], randGen(5, 77), randGen(numKeyVal))
vals = randList(seq[byte], randGen(1, 57), randGen(numKeyVal))
for i in 0 ..< numKeyVal:
pruningTrie.put(keys[i], vals[i])
let rootHash = pruningTrie.rootHash
for k, v in pruningTrie.replicate:
repdb.put(k.toOpenArray, v.toOpenArray)
repdb.put(k, v)
var trie = initHexaryTrie(repdb, rootHash, isPruning = true)
var numPairs = 0

25
tests/trie/test_json_suite.nim
View File

@ -2,15 +2,14 @@
import
os, json, tables, strutils, algorithm,
eth/rlp/types,
eth/trie/[trie_defs, db, hexary],
./testutils
eth/trie/[db, hexary],
stew/byteutils
type
TestOp = object
idx: int
key: BytesRange
value: BytesRange
key: seq[byte]
value: seq[byte]
proc cmp(lhs, rhs: TestOp): int = cmp(lhs.idx, rhs.idx)
proc `<=`(lhs, rhs: TestOp): bool = lhs.idx <= rhs.idx
@ -76,12 +75,12 @@ proc runTests*(filename: string) =
case v.kind
of JString:
inputs.add(TestOp(idx: inputs.len,
key: k.str.toBytesRange,
value: v.str.toBytesRange))
key: k.str.toBytes,
value: v.str.toBytes))
of JNull:
inputs.add(TestOp(idx: inputs.len,
key: k.str.toBytesRange,
value: zeroBytesRange))
key: k.str.toBytes,
value: @[]))
else: invalidTest()
else: invalidTest()
@ -91,12 +90,12 @@ proc runTests*(filename: string) =
case v.kind
of JString:
inputs.add(TestOp(idx: inputs.len,
key: k.toBytesRange,
value: v.str.toBytesRange))
key: k.toBytes,
value: v.str.toBytes))
of JNull:
inputs.add(TestOp(idx: inputs.len,
key: k.toBytesRange,
value: zeroBytesRange))
key: k.toBytes,
value: @[]))
else: invalidTest()
else: invalidTest()

11
tests/trie/test_nibbles.nim
View File

@ -1,11 +0,0 @@
{.used.}
import
unittest,
eth/trie/nibbles
suite "nibbles":
test "zeroNibblesRange":
# https://github.com/status-im/nim-eth/issues/6
check zeroNibblesRange.len == 0

66
tests/trie/test_sparse_binary_trie.nim
View File

@ -1,8 +1,8 @@
{.used.}
import
unittest, random,
eth/trie/[trie_defs, db, sparse_binary, sparse_proofs],
unittest, random, stew/byteutils,
eth/trie/[db, sparse_binary, sparse_proofs],
./testutils
suite "sparse binary trie":
@ -21,14 +21,14 @@ suite "sparse binary trie":
test "basic get":
for c in kv_pairs:
let x = trie.get(c.key)
let y = toRange(c.value)
let y = c.value
check x == y
trie.del(c.key)
for c in kv_pairs:
check trie.exists(c.key) == false
check trie.getRootHash() == keccakHash(emptyNodeHashes[0].toOpenArray, emptyNodeHashes[0].toOpenArray).toRange
check trie.getRootHash() == keccakHash(emptyNodeHashes[0].data, emptyNodeHashes[0].data).data
test "single update set":
random.shuffle(kv_pairs)
@ -42,11 +42,11 @@ suite "sparse binary trie":
test "single update get":
for i in numbers:
# If new value is the same as current value, skip the update
if toRange($i) == trie.get(kv_pairs[i].key):
if toBytes($i) == trie.get(kv_pairs[i].key):
continue
# Update
trie.set(kv_pairs[i].key, $i)
check trie.get(kv_pairs[i].key) == toRange($i)
trie.set(kv_pairs[i].key, toBytes($i))
check trie.get(kv_pairs[i].key) == toBytes($i)
check trie.getRootHash() != prior_to_update_root
# Un-update
@ -56,7 +56,7 @@ suite "sparse binary trie":
test "batch update with different update order":
# First batch update
for i in numbers:
trie.set(kv_pairs[i].key, $i)
trie.set(kv_pairs[i].key, toBytes($i))
let batch_updated_root = trie.getRootHash()
@ -70,14 +70,14 @@ suite "sparse binary trie":
# Second batch update
random.shuffle(numbers)
for i in numbers:
trie.set(kv_pairs[i].key, $i)
trie.set(kv_pairs[i].key, toBytes($i))
check trie.getRootHash() == batch_updated_root
test "dictionary API":
trie[kv_pairs[0].key] = kv_pairs[0].value
let x = trie[kv_pairs[0].key]
let y = toRange(kv_pairs[0].value)
let y = kv_pairs[0].value
check x == y
check kv_pairs[0].key in trie
@ -85,10 +85,10 @@ suite "sparse binary trie":
db = newMemoryDB()
trie = initSparseBinaryTrie(db)
let
testKey = toRange(kv_pairs[0].key)
testValue = toRange(kv_pairs[0].value)
testKey2 = toRange(kv_pairs[1].key)
testValue2 = toRange(kv_pairs[1].value)
testKey = kv_pairs[0].key
testValue = kv_pairs[0].value
testKey2 = kv_pairs[1].key
testValue2 = kv_pairs[1].value
trie.set(testKey, testValue)
var root = trie.getRootHash()
@ -102,11 +102,11 @@ suite "sparse binary trie":
value = trie.get(testKey, root)
check value == testValue
proc makeBadProof(size: int, width = 32): seq[BytesRange] =
let badProofStr = randList(string, randGen(width, width), randGen(size, size))
result = newSeq[BytesRange](size)
proc makeBadProof(size: int, width = 32): seq[seq[byte]] =
let badProofStr = randList(seq[byte], randGen(width, width), randGen(size, size))
result = newSeq[seq[byte]](size)
for i in 0 ..< result.len:
result[i] = toRange(badProofStr[i])
result[i] = badProofStr[i]
test "proofs":
const
@ -115,9 +115,9 @@ suite "sparse binary trie":
let
testKey = kv_pairs[0].key
badKey = kv_pairs[1].key
testValue = "testValue"
testValue2 = "testValue2"
badValue = "badValue"
testValue = "testValue".toBytes
testValue2 = "testValue2".toBytes
badValue = "badValue".toBytes
badProof = makeBadProof(MaxBadProof)
trie[testKey] = testValue
@ -131,7 +131,7 @@ suite "sparse binary trie":
let
testKey2 = kv_pairs[2].key
testKey3 = kv_pairs[3].key
defaultValue = zeroBytesRange
defaultValue = default(seq[byte])
trie.set(testKey2, testValue)
proof = trie.prove(testKey)
@ -169,7 +169,7 @@ suite "sparse binary trie":
check compactProof(badProof2).len == 0
check compactProof(badProof3).len == 0
check decompactProof(badProof3).len == 0
var zeroProof: seq[BytesRange]
var zeroProof: seq[seq[byte]]
check decompactProof(zeroProof).len == 0
proof = trie.proveCompact(testKey2)
@ -202,23 +202,23 @@ suite "sparse binary trie":
test "examples":
let
key1 = "01234567890123456789"
key2 = "abcdefghijklmnopqrst"
key1 = "01234567890123456789".toBytes
key2 = "abcdefghijklmnopqrst".toBytes
trie.set(key1, "value1")
trie.set(key2, "value2")
check trie.get(key1) == "value1".toRange
check trie.get(key2) == "value2".toRange
trie.set(key1, "value1".toBytes)
trie.set(key2, "value2".toBytes)
check trie.get(key1) == "value1".toBytes
check trie.get(key2) == "value2".toBytes
trie.del(key1)
check trie.get(key1) == zeroBytesRange
check trie.get(key1) == []
trie.del(key2)
check trie[key2] == zeroBytesRange
check trie[key2] == []
let
value1 = "hello world"
badValue = "bad value"
value1 = "hello world".toBytes
badValue = "bad value".toBytes
trie[key1] = value1
var proof = trie.prove(key1)

11
tests/trie/test_transaction_db.nim
View File

@ -2,8 +2,7 @@
import
unittest,
eth/trie/[db, trie_defs], ./testutils,
eth/rlp/types as rlpTypes
eth/trie/[db], ./testutils
suite "transaction db":
setup:
@ -11,10 +10,10 @@ suite "transaction db":
listLength = 30
var
keysA = randList(Bytes, randGen(3, 33), randGen(listLength))
valuesA = randList(Bytes, randGen(5, 77), randGen(listLength))
keysB = randList(Bytes, randGen(3, 33), randGen(listLength))
valuesB = randList(Bytes, randGen(5, 77), randGen(listLength))
keysA = randList(seq[byte], randGen(3, 33), randGen(listLength))
valuesA = randList(seq[byte], randGen(5, 77), randGen(listLength))
keysB = randList(seq[byte], randGen(3, 33), randGen(listLength))
valuesB = randList(seq[byte], randGen(5, 77), randGen(listLength))
proc populateA(db: TrieDatabaseRef) =
for i in 0 ..< listLength:

84
tests/trie/test_trie_bitseq.nim Normal file
View File

@ -0,0 +1,84 @@
{.used.}
import
random, unittest,
eth/trie/trie_bitseq
proc randomBytes(n: int): seq[byte] =
result = newSeq[byte](n)
for i in 0 ..< result.len:
result[i] = byte(rand(256))
suite "trie bitseq":
test "basic":
var a = @[byte 0b10101010, 0b11110000, 0b00001111, 0b01010101]
var bSeq = @[byte 0b10101010, 0b00000000, 0b00000000, 0b11111111]
var b = bits(bSeq, 8)
var cSeq = @[byte 0b11110000, 0b00001111, 0b00000000, 0b00000000]
var c = bits(cSeq, 16)
var dSeq = @[byte 0b00001111, 0b00000000, 0b00000000, 0b00000000]
var d = bits(dSeq, 8)
var eSeq = @[byte 0b01010101, 0b00000000, 0b00000000, 0b00000000]
var e = bits(eSeq, 8)
var m = a.bits
var n = m[0..7]
check n == b
check n.len == 8
check b.len == 8
check c == m[8..23]
check $(d) == "00001111"
check $(e) == "01010101"
var f = int.fromBits(e, 0, 4)
check f == 0b0101
let k = n & d
check(k.len == n.len + d.len)
check($k == $n & $d)
var asciiSeq = @[byte('A'),byte('S'),byte('C'),byte('I'),byte('I')]
let asciiBits = bits(asciiSeq)
check $asciiBits == "0100000101010011010000110100100101001001"
test "concat operator":
randomize(5000)
for i in 0..<256:
var xSeq = randomBytes(rand(i))
var ySeq = randomBytes(rand(i))
let x = xSeq.bits
let y = ySeq.bits
var z = x & y
check z.len == x.len + y.len
check($z == $x & $y)
test "get set bits":
randomize(1000)
for i in 0..<256:
# produce random vector
var xSeq = randomBytes(i)
var ySeq = randomBytes(i)
var x = xSeq.bits
var y = ySeq.bits
for idx, bit in x:
y[idx] = bit
check x == y
test "constructor with start":
var a = @[byte 0b10101010, 0b11110000, 0b00001111, 0b01010101]
var b = a.bits(1, 8)
check b.len == 8
check b[0] == false
check $b == "01010101"
b[0] = true
check $b == "11010101"
check b[0] == true
b.pushFront(false)
check b[0] == false
check $b == "011010101"

31
tests/trie/testutils.nim
View File

@ -1,6 +1,6 @@
import
random, sets, eth/trie/trie_utils as ethUtils,
eth/rlp/types as rlpTypes, stew/ranges/bitranges,
random, sets,
eth/trie/trie_bitseq,
nimcrypto/[utils, sysrand]
type
@ -8,8 +8,8 @@ type
minVal, maxVal: T
KVPair* = ref object
key*: string
value*: string
key*: seq[byte]
value*: seq[byte]
proc randGen*[T](minVal, maxVal: T): RandGen[T] =
doAssert(minVal <= maxVal)
@ -28,11 +28,11 @@ proc randString*(len: int): string =
for i in 0..<len:
result[i] = rand(255).char
proc randBytes*(len: int): Bytes =
proc randBytes*(len: int): seq[byte] =
result = newSeq[byte](len)
discard randomBytes(result[0].addr, len)
proc toBytesRange*(str: string): BytesRange =
proc toBytesRange*(str: string): seq[byte] =
var s: seq[byte]
if str[0] == '0' and str[1] == 'x':
s = fromHex(str.substr(2))
@ -40,16 +40,16 @@ proc toBytesRange*(str: string): BytesRange =
s = newSeq[byte](str.len)
for i in 0 ..< str.len:
s[i] = byte(str[i])
result = s.toRange
result = s
proc randPrimitives*[T](val: int): T =
when T is string:
randString(val)
elif T is int:
result = val
elif T is BytesRange:
result = randString(val).toRange
elif T is Bytes:
elif T is string:
result = randString(val)
elif T is seq[byte]:
result = randBytes(val)
proc randList*(T: typedesc, strGen, listGen: RandGen, unique: bool = true): seq[T] =
@ -71,19 +71,14 @@ proc randList*(T: typedesc, strGen, listGen: RandGen, unique: bool = true): seq[
proc randKVPair*(keySize = 32): seq[KVPair] =
const listLen = 100
let keys = randList(string, randGen(keySize, keySize), randGen(listLen, listLen))
let vals = randList(string, randGen(1, 100), randGen(listLen, listLen))
let keys = randList(seq[byte], randGen(keySize, keySize), randGen(listLen, listLen))
let vals = randList(seq[byte], randGen(1, 100), randGen(listLen, listLen))
result = newSeq[KVPair](listLen)
for i in 0..<listLen:
result[i] = KVPair(key: keys[i], value: vals[i])
proc toBytes*(str: string): Bytes =
result = newSeq[byte](str.len)
for i in 0..<str.len:
result[i] = byte(str[i])
proc genBitVec*(len: int): BitRange =
proc genBitVec*(len: int): TrieBitSeq =
let k = ((len + 7) and (not 7)) shr 3
var s = newSeq[byte](k)
result = bits(s, len)