diff --git a/doc/rlp.md b/doc/rlp.md index df15a5f..74a8889 100644 --- a/doc/rlp.md +++ b/doc/rlp.md @@ -10,15 +10,9 @@ and [Wiki](https://github.com/ethereum/wiki/wiki/RLP). ### Reading RLP data The `Rlp` type provided by this library represents a cursor over an RLP-encoded -byte stream. Before instantiating such a cursor, you must convert your -input data a `BytesRange` value provided by the [nim-ranges][RNG] library, -which represents an immutable and thus cheap-to-copy sub-range view over an -underlying `seq[byte]` instance: - -[RNG]: https://github.com/status-im/nim-ranges - +byte stream. ``` nim -proc rlpFromBytes*(data: BytesRange): Rlp +proc rlpFromBytes*(data: openArray[byte]): Rlp ``` ### Streaming API @@ -67,7 +61,7 @@ type RlpNode* = object case kind*: RlpNodeType of rlpBlob: - bytes*: BytesRange + bytes*: seq[byte] of rlpList: elems*: seq[RlpNode] ``` @@ -86,7 +80,7 @@ proc inspect*(self: Rlp, indent = 0): string The `RlpWriter` type can be used to encode RLP data. Instances are created with the `initRlpWriter` proc. This should be followed by one or more calls to `append` which is overloaded to accept arbitrary values. Finally, you can -call `finish` to obtain the final `BytesRange`. +call `finish` to obtain the final `seq[byte]`. If the end result should be a RLP list of particular length, you can replace the initial call to `initRlpWriter` with `initRlpList(n)`. Calling `finish` diff --git a/doc/trie.md b/doc/trie.md index f2a7bea..80902c8 100644 --- a/doc/trie.md +++ b/doc/trie.md @@ -62,12 +62,9 @@ The primary API for Binary-trie is `set` and `get`. * set(key, value) --- _store a value associated with a key_ * get(key): value --- _get a value using a key_ -Both `key` and `value` are of `BytesRange` type. And they cannot have zero length. -You can also use convenience API `get` and `set` which accepts -`Bytes` or `string` (a `string` is conceptually wrong in this context -and may costlier than a `BytesRange`, but it is good for testing purpose). +Both `key` and `value` are of `seq[byte]` type. And they cannot have zero length. -Getting a non-existent key will return zero length BytesRange. +Getting a non-existent key will return zero length seq[byte]. Binary-trie also provide dictionary syntax API for `set` and `get`. * trie[key] = value -- same as `set` @@ -81,11 +78,11 @@ Additional APIs are: that starts with the same key prefix * rootNode() -- get root node * rootNode(node) -- replace the root node - * getRootHash(): `KeccakHash` with `BytesRange` type + * getRootHash(): `KeccakHash` with `seq[byte]` type * getDB(): `DB` -- get flat-db pointer Constructor API: - * initBinaryTrie(DB, rootHash[optional]) -- rootHash has `BytesRange` or KeccakHash type + * initBinaryTrie(DB, rootHash[optional]) -- rootHash has `seq[byte]` or KeccakHash type * init(BinaryTrie, DB, rootHash[optional]) Normally you would not set the rootHash when constructing an empty Binary-trie. @@ -103,17 +100,17 @@ var db = newMemoryDB() var trie = initBinaryTrie(db) trie.set("key1", "value1") trie.set("key2", "value2") -doAssert trie.get("key1") == "value1".toRange -doAssert trie.get("key2") == "value2".toRange +doAssert trie.get("key1") == "value1".toBytes +doAssert trie.get("key2") == "value2".toBytes # delete all subtrie with key prefixes "key" trie.deleteSubtrie("key") -doAssert trie.get("key1") == zeroBytesRange -doAssert trie.get("key2") == zeroBytesRange +doAssert trie.get("key1") == [] +doAssert trie.get("key2") == []] trie["moon"] = "sun" doAssert "moon" in trie -doAssert trie["moon"] == "sun".toRange +doAssert trie["moon"] == "sun".toBytes ``` Remember, `set` and `get` are trie operations. A single `set` operation may invoke @@ -142,12 +139,12 @@ The branch utils consist of these API: * getTrieNodes(DB; nodeHash): branch `keyPrefix`, `key`, and `value` are bytes container with length greater than zero. -They can be BytesRange, Bytes, and string(again, for convenience and testing purpose). +They can be openArray[byte]. `rootHash` and `nodeHash` also bytes container, but they have constraint: must be 32 bytes in length, and it must be a keccak_256 hash value. -`branch` is a list of nodes, or in this case a seq[BytesRange]. +`branch` is a list of nodes, or in this case a `seq[seq[byte]]`. A list? yes, the structure is stored along with the encoded node. Therefore a list is enough to reconstruct the entire trie/branch. @@ -303,14 +300,14 @@ let trie.set(key1, "value1") trie.set(key2, "value2") -doAssert trie.get(key1) == "value1".toRange -doAssert trie.get(key2) == "value2".toRange +doAssert trie.get(key1) == "value1".toBytes +doAssert trie.get(key2) == "value2".toBytes trie.delete(key1) -doAssert trie.get(key1) == zeroBytesRange +doAssert trie.get(key1) == [] trie.delete(key2) -doAssert trie[key2] == zeroBytesRange +doAssert trie[key2] == [] ``` Remember, `set` and `get` are trie operations. A single `set` operation may invoke diff --git a/eth/common/eth_types.nim b/eth/common/eth_types.nim index 9360a27..a793803 100644 --- a/eth/common/eth_types.nim +++ b/eth/common/eth_types.nim @@ -244,7 +244,7 @@ proc read*(rlp: var Rlp, T: typedesc[StUint]): T {.inline.} = if bytes.len > 0: # be sure the amount of bytes matches the size of the stint if bytes.len <= sizeof(result): - result.initFromBytesBE(bytes.toOpenArray) + result.initFromBytesBE(bytes) else: raise newException(RlpTypeMismatch, "Unsigned integer expected, but the source RLP has the wrong length") else: @@ -375,7 +375,7 @@ method getTrieDB*(db: AbstractChainDB): TrieDatabaseRef {.base, gcsafe.} = method getCodeByHash*(db: AbstractChainDB, hash: KeccakHash): Blob {.base, gcsafe.} = notImplemented() -method getSetting*(db: AbstractChainDB, key: string): Bytes {.base, gcsafe.} = +method getSetting*(db: AbstractChainDB, key: string): seq[byte] {.base, gcsafe.} = notImplemented() method setSetting*(db: AbstractChainDB, key: string, val: openarray[byte]) {.base, gcsafe.} = diff --git a/eth/common/state_accessors.nim b/eth/common/state_accessors.nim index cce3c3c..b7e1408 100644 --- a/eth/common/state_accessors.nim +++ b/eth/common/state_accessors.nim @@ -6,7 +6,7 @@ proc getAccount*(db: TrieDatabaseRef, rootHash: KeccakHash, account: EthAddress): Account = let trie = initSecureHexaryTrie(db, rootHash) - let data = trie.get(unnecessary_OpenArrayToRange account) + let data = trie.get(account) if data.len > 0: result = rlp.decode(data, Account) else: @@ -21,6 +21,3 @@ proc getContractCode*(chain: AbstractChainDB, req: ContractCodeRequest): Blob {. proc getStorageNode*(chain: AbstractChainDB, hash: KeccakHash): Blob = let db = chain.getTrieDB return db.get(hash.data) - # let trie = initSecureHexaryTrie(db, emptyRlpHash) # TODO emptyRlpHash is not correct here - # return trie.get(unnecessary_OpenArrayToRange hash.data) - diff --git a/eth/p2p/auth.nim b/eth/p2p/auth.nim index 33804fe..1dea3b5 100644 --- a/eth/p2p/auth.nim +++ b/eth/p2p/auth.nim @@ -14,7 +14,7 @@ import eth/[keys, rlp], nimcrypto import ecies -import stew/[byteutils, endians2, results] +import stew/[byteutils, endians2, objects, results] export results @@ -362,9 +362,6 @@ proc decodeAuthMessageV4(h: var Handshake, m: openarray[byte]): AuthResult[void] proc decodeAuthMessageEip8(h: var Handshake, m: openarray[byte]): AuthResult[void] = ## Decodes EIP-8 AuthMessage. - var - nonce: Nonce - let size = uint16.fromBytesBE(m) h.expectedLength = int(size) + 2 if h.expectedLength > len(m): @@ -374,7 +371,7 @@ proc decodeAuthMessageEip8(h: var Handshake, m: openarray[byte]): AuthResult[voi toa(m, 0, 2)).isErr: return err(EciesError) try: - var reader = rlpFromBytes(buffer.toRange()) + var reader = rlpFromBytes(buffer) if not reader.isList() or reader.listLen() < 4: return err(InvalidAuth) if reader.listElem(0).blobLen != RawSignatureSize: @@ -385,29 +382,28 @@ proc decodeAuthMessageEip8(h: var Handshake, m: openarray[byte]): AuthResult[voi return err(InvalidAuth) if reader.listElem(3).blobLen != 1: return err(InvalidAuth) - var signatureBr = reader.listElem(0).toBytes() - var pubkeyBr = reader.listElem(1).toBytes() - var nonceBr = reader.listElem(2).toBytes() - var versionBr = reader.listElem(3).toBytes() + let + signatureBr = reader.listElem(0).toBytes() + pubkeyBr = reader.listElem(1).toBytes() + nonceBr = reader.listElem(2).toBytes() + versionBr = reader.listElem(3).toBytes() - let pubkey = - ? PublicKey.fromRaw(pubkeyBr.toOpenArray()).mapErrTo(InvalidPubKey) - - copyMem(addr nonce[0], nonceBr.baseAddr, KeyLength) + let + signature = ? Signature.fromRaw(signatureBr).mapErrTo(SignatureError) + pubkey = ? PublicKey.fromRaw(pubkeyBr).mapErrTo(InvalidPubKey) + nonce = toArray(KeyLength, nonceBr) var secret = ? ecdhRaw(h.host.seckey, pubkey).mapErrTo(EcdhError) let xornonce = nonce xor secret.data secret.clear() - let signature = - ? Signature.fromRaw(signatureBr.toOpenArray()).mapErrTo(SignatureError) h.remoteEPubkey = ? recover(signature, SkMessage(data: xornonce)).mapErrTo(SignatureError) h.initiatorNonce = nonce h.remoteHPubkey = pubkey - h.version = cast[ptr byte](versionBr.baseAddr)[] + h.version = versionBr[0] ok() except CatchableError: err(RlpError) @@ -424,7 +420,7 @@ proc decodeAckMessageEip8*(h: var Handshake, m: openarray[byte]): AuthResult[voi toa(m, 0, 2)).isErr: return err(EciesError) try: - var reader = rlpFromBytes(buffer.toRange()) + var reader = rlpFromBytes(buffer) if not reader.isList() or reader.listLen() < 3: return err(InvalidAck) if reader.listElem(0).blobLen != RawPublicKeySize: @@ -433,14 +429,14 @@ proc decodeAckMessageEip8*(h: var Handshake, m: openarray[byte]): AuthResult[voi return err(InvalidAck) if reader.listElem(2).blobLen != 1: return err(InvalidAck) - let pubkeyBr = reader.listElem(0).toBytes() - let nonceBr = reader.listElem(1).toBytes() - let versionBr = reader.listElem(2).toBytes() - h.remoteEPubkey = - ? PublicKey.fromRaw(pubkeyBr.toOpenArray()).mapErrTo(InvalidPubKey) + let + pubkeyBr = reader.listElem(0).toBytes() + nonceBr = reader.listElem(1).toBytes() + versionBr = reader.listElem(2).toBytes() - copyMem(addr h.responderNonce[0], nonceBr.baseAddr, KeyLength) - h.version = cast[ptr byte](versionBr.baseAddr)[] + h.remoteEPubkey = ? PublicKey.fromRaw(pubkeyBr).mapErrTo(InvalidPubKey) + h.responderNonce = toArray(KeyLength, nonceBr) + h.version = versionBr[0] ok() except CatchableError: diff --git a/eth/p2p/discovery.nim b/eth/p2p/discovery.nim index 2daae63..29029f2 100644 --- a/eth/p2p/discovery.nim +++ b/eth/p2p/discovery.nim @@ -13,7 +13,7 @@ import chronos, stint, nimcrypto, chronicles, eth/[keys, rlp], kademlia, enode, - stew/results + stew/[objects, results] export Node, results @@ -53,27 +53,27 @@ const MinListLen: array[CommandId, int] = [4, 3, 2, 2] proc append*(w: var RlpWriter, a: IpAddress) = case a.family of IpAddressFamily.IPv6: - w.append(a.address_v6.toMemRange) + w.append(a.address_v6) of IpAddressFamily.IPv4: - w.append(a.address_v4.toMemRange) + w.append(a.address_v4) proc append(w: var RlpWriter, p: Port) {.inline.} = w.append(p.int) proc append(w: var RlpWriter, pk: PublicKey) {.inline.} = w.append(pk.toRaw()) proc append(w: var RlpWriter, h: MDigest[256]) {.inline.} = w.append(h.data) -proc pack(cmdId: CommandId, payload: BytesRange, pk: PrivateKey): Bytes = +proc pack(cmdId: CommandId, payload: openArray[byte], pk: PrivateKey): seq[byte] = ## Create and sign a UDP message to be sent to a remote node. ## ## See https://github.com/ethereum/devp2p/blob/master/rlpx.md#node-discovery for information on ## how UDP packets are structured. # TODO: There is a lot of unneeded allocations here - let encodedData = @[cmdId.byte] & payload.toSeq() + let encodedData = @[cmdId.byte] & @payload let signature = @(pk.sign(encodedData).tryGet().toRaw()) let msgHash = keccak256.digest(signature & encodedData) result = @(msgHash.data) & signature & encodedData -proc validateMsgHash(msg: Bytes): DiscResult[MDigest[256]] = +proc validateMsgHash(msg: openArray[byte]): DiscResult[MDigest[256]] = if msg.len > HEAD_SIZE: var ret: MDigest[256] ret.data[0 .. ^1] = msg.toOpenArray(0, ret.data.high) @@ -90,7 +90,7 @@ proc recoverMsgPublicKey(msg: openArray[byte]): DiscResult[PublicKey] = let sig = ? Signature.fromRaw(msg.toOpenArray(MAC_SIZE, HEAD_SIZE)) recover(sig, msg.toOpenArray(HEAD_SIZE, msg.high)) -proc unpack(msg: Bytes): tuple[cmdId: CommandId, payload: Bytes] = +proc unpack(msg: openArray[byte]): tuple[cmdId: CommandId, payload: seq[byte]] = # Check against possible RangeError if msg[HEAD_SIZE].int < CommandId.low.ord or msg[HEAD_SIZE].int > CommandId.high.ord: @@ -112,14 +112,14 @@ proc send(d: DiscoveryProtocol, n: Node, data: seq[byte]) = proc sendPing*(d: DiscoveryProtocol, n: Node): seq[byte] = let payload = rlp.encode((PROTO_VERSION, d.address, n.node.address, - expiration())).toRange + expiration())) let msg = pack(cmdPing, payload, d.privKey) result = msg[0 ..< MAC_SIZE] trace ">>> ping ", n d.send(n, msg) proc sendPong*(d: DiscoveryProtocol, n: Node, token: MDigest[256]) = - let payload = rlp.encode((n.node.address, token, expiration())).toRange + let payload = rlp.encode((n.node.address, token, expiration())) let msg = pack(cmdPong, payload, d.privKey) trace ">>> pong ", n d.send(n, msg) @@ -127,7 +127,7 @@ proc sendPong*(d: DiscoveryProtocol, n: Node, token: MDigest[256]) = proc sendFindNode*(d: DiscoveryProtocol, n: Node, targetNodeId: NodeId) = var data: array[64, byte] data[32 .. ^1] = targetNodeId.toByteArrayBE() - let payload = rlp.encode((data, expiration())).toRange + let payload = rlp.encode((data, expiration())) let msg = pack(cmdFindNode, payload, d.privKey) trace ">>> find_node to ", n#, ": ", msg.toHex() d.send(n, msg) @@ -140,7 +140,7 @@ proc sendNeighbours*(d: DiscoveryProtocol, node: Node, neighbours: seq[Node]) = template flush() = block: - let payload = rlp.encode((nodes, expiration())).toRange + let payload = rlp.encode((nodes, expiration())) let msg = pack(cmdNeighbours, payload, d.privkey) trace "Neighbours to", node, nodes d.send(node, msg) @@ -169,14 +169,14 @@ proc recvPing(d: DiscoveryProtocol, node: Node, msgHash: MDigest[256]) {.inline.} = d.kademlia.recvPing(node, msgHash) -proc recvPong(d: DiscoveryProtocol, node: Node, payload: Bytes) {.inline.} = - let rlp = rlpFromBytes(payload.toRange) - let tok = rlp.listElem(1).toBytes().toSeq() +proc recvPong(d: DiscoveryProtocol, node: Node, payload: seq[byte]) {.inline.} = + let rlp = rlpFromBytes(payload) + let tok = rlp.listElem(1).toBytes() d.kademlia.recvPong(node, tok) proc recvNeighbours(d: DiscoveryProtocol, node: Node, - payload: Bytes) {.inline.} = - let rlp = rlpFromBytes(payload.toRange) + payload: seq[byte]) {.inline.} = + let rlp = rlpFromBytes(payload) let neighboursList = rlp.listElem(0) let sz = neighboursList.listLen() @@ -187,18 +187,18 @@ proc recvNeighbours(d: DiscoveryProtocol, node: Node, var ip: IpAddress case ipBlob.len of 4: - ip = IpAddress(family: IpAddressFamily.IPv4) - copyMem(addr ip.address_v4[0], baseAddr ipBlob, 4) + ip = IpAddress( + family: IpAddressFamily.IPv4, address_v4: toArray(4, ipBlob)) of 16: - ip = IpAddress(family: IpAddressFamily.IPv6) - copyMem(addr ip.address_v6[0], baseAddr ipBlob, 16) + ip = IpAddress( + family: IpAddressFamily.IPv6, address_v6: toArray(16, ipBlob)) else: error "Wrong ip address length!" continue let udpPort = n.listElem(1).toInt(uint16).Port let tcpPort = n.listElem(2).toInt(uint16).Port - let pk = PublicKey.fromRaw(n.listElem(3).toBytes.toOpenArray()) + let pk = PublicKey.fromRaw(n.listElem(3).toBytes) if pk.isErr: warn "Could not parse public key" continue @@ -206,24 +206,24 @@ proc recvNeighbours(d: DiscoveryProtocol, node: Node, neighbours.add(newNode(pk[], Address(ip: ip, udpPort: udpPort, tcpPort: tcpPort))) d.kademlia.recvNeighbours(node, neighbours) -proc recvFindNode(d: DiscoveryProtocol, node: Node, payload: Bytes) {.inline, gcsafe.} = - let rlp = rlpFromBytes(payload.toRange) +proc recvFindNode(d: DiscoveryProtocol, node: Node, payload: openArray[byte]) {.inline, gcsafe.} = + let rlp = rlpFromBytes(payload) trace "<<< find_node from ", node let rng = rlp.listElem(0).toBytes # Check for pubkey len if rng.len == 64: - let nodeId = readUIntBE[256](rng[32 .. ^1].toOpenArray()) + let nodeId = readUIntBE[256](rng[32 .. ^1]) d.kademlia.recvFindNode(node, nodeId) else: trace "Invalid target public key received" -proc expirationValid(cmdId: CommandId, rlpEncodedPayload: seq[byte]): +proc expirationValid(cmdId: CommandId, rlpEncodedPayload: openArray[byte]): bool {.inline, raises:[DiscProtocolError, RlpError].} = ## Can only raise `DiscProtocolError` and all of `RlpError` # Check if there is a payload if rlpEncodedPayload.len <= 0: raise newException(DiscProtocolError, "RLP stream is empty") - let rlp = rlpFromBytes(rlpEncodedPayload.toRange) + let rlp = rlpFromBytes(rlpEncodedPayload) # Check payload is an RLP list and if the list has the minimum items required # for this packet type if rlp.isList and rlp.listLen >= MinListLen[cmdId]: @@ -233,7 +233,7 @@ proc expirationValid(cmdId: CommandId, rlpEncodedPayload: seq[byte]): else: raise newException(DiscProtocolError, "Invalid RLP list for this packet id") -proc receive*(d: DiscoveryProtocol, a: Address, msg: Bytes) {.gcsafe.} = +proc receive*(d: DiscoveryProtocol, a: Address, msg: openArray[byte]) {.gcsafe.} = ## Can raise `DiscProtocolError` and all of `RlpError` # Note: export only needed for testing let msgHash = validateMsgHash(msg) diff --git a/eth/p2p/discoveryv5/encoding.nim b/eth/p2p/discoveryv5/encoding.nim index a049f93..fa43b7b 100644 --- a/eth/p2p/discoveryv5/encoding.nim +++ b/eth/p2p/discoveryv5/encoding.nim @@ -202,7 +202,7 @@ proc decodePacketBody(typ: byte, let kind = cast[PacketKind](typ) res = Packet(kind: kind) - var rlp = rlpFromBytes(@body.toRange) + var rlp = rlpFromBytes(body) if rlp.enterList: res.reqId = rlp.read(RequestId) @@ -258,12 +258,11 @@ proc decodeAuthResp(c: Codec, fromId: NodeId, head: AuthHeader, proc decodeEncrypted*(c: var Codec, fromId: NodeID, fromAddr: Address, - input: seq[byte], + input: openArray[byte], authTag: var AuthTag, newNode: var Node, packet: var Packet): DecodeStatus = - let input = input.toRange - var r = rlpFromBytes(input[tagSize .. ^1]) + var r = rlpFromBytes(input.toOpenArray(tagSize, input.high)) var auth: AuthHeader var readKey: AesKey @@ -312,10 +311,11 @@ proc decodeEncrypted*(c: var Codec, # doAssert(false, "TODO: HANDLE ME!") let headSize = tagSize + r.position - let bodyEnc = input[headSize .. ^1] - let body = decryptGCM(readKey, auth.auth, bodyEnc.toOpenArray, - input[0 .. tagSize - 1].toOpenArray) + let body = decryptGCM( + readKey, auth.auth, + input.toOpenArray(headSize, input.high), + input.toOpenArray(0, tagSize - 1)) if body.isNone(): discard c.db.deleteKeys(fromId, fromAddr) return DecryptError diff --git a/eth/p2p/discoveryv5/enr.nim b/eth/p2p/discoveryv5/enr.nim index 8599f33..7c43937 100644 --- a/eth/p2p/discoveryv5/enr.nim +++ b/eth/p2p/discoveryv5/enr.nim @@ -214,11 +214,11 @@ proc verifySignatureV4(r: Record, sigData: openarray[byte], content: seq[byte]): return verify(sig[], h, publicKey.get) proc verifySignature(r: Record): bool = - var rlp = rlpFromBytes(r.raw.toRange) + var rlp = rlpFromBytes(r.raw) let sz = rlp.listLen if not rlp.enterList: return false - let sigData = rlp.read(Bytes) + let sigData = rlp.read(seq[byte]) let content = block: var writer = initRlpList(sz - 1) var reader = rlp @@ -240,7 +240,7 @@ proc fromBytesAux(r: var Record): bool = if r.raw.len > maxEnrSize: return false - var rlp = rlpFromBytes(r.raw.toRange) + var rlp = rlpFromBytes(r.raw) if not rlp.isList: return false @@ -270,7 +270,7 @@ proc fromBytesAux(r: var Record): bool = let v = rlp.read(uint16) r.pairs.add((k, Field(kind: kNum, num: v))) else: - r.pairs.add((k, Field(kind: kBytes, bytes: rlp.read(Bytes)))) + r.pairs.add((k, Field(kind: kBytes, bytes: rlp.read(seq[byte])))) verifySignature(r) @@ -329,9 +329,9 @@ proc `$`*(r: Record): string = proc `==`*(a, b: Record): bool = a.raw == b.raw proc read*(rlp: var Rlp, T: typedesc[Record]): T {.inline.} = - if not result.fromBytes(rlp.rawData.toOpenArray): + if not result.fromBytes(rlp.rawData): raise newException(ValueError, "Could not deserialize") rlp.skipElem() proc append*(rlpWriter: var RlpWriter, value: Record) = - rlpWriter.appendRawBytes(value.raw.toRange) + rlpWriter.appendRawBytes(value.raw) diff --git a/eth/p2p/discoveryv5/protocol.nim b/eth/p2p/discoveryv5/protocol.nim index 2fd4fad..7e53237 100644 --- a/eth/p2p/discoveryv5/protocol.nim +++ b/eth/p2p/discoveryv5/protocol.nim @@ -97,13 +97,13 @@ proc whoareyouMagic(toNode: NodeId): array[magicSize, byte] = for i, c in prefix: data[sizeof(toNode) + i] = byte(c) sha256.digest(data).data -proc isWhoAreYou(d: Protocol, msg: Bytes): bool = +proc isWhoAreYou(d: Protocol, msg: openArray[byte]): bool = if msg.len > d.whoareyouMagic.len: result = d.whoareyouMagic == msg.toOpenArray(0, magicSize - 1) -proc decodeWhoAreYou(d: Protocol, msg: Bytes): Whoareyou = +proc decodeWhoAreYou(d: Protocol, msg: openArray[byte]): Whoareyou = result = Whoareyou() - result[] = rlp.decode(msg.toRange[magicSize .. ^1], WhoareyouObj) + result[] = rlp.decode(msg.toOpenArray(magicSize, msg.high), WhoareyouObj) proc sendWhoareyou(d: Protocol, address: Address, toNode: NodeId, authTag: AuthTag) = trace "sending who are you", to = $toNode, toAddress = $address @@ -172,7 +172,7 @@ proc handleFindNode(d: Protocol, fromId: NodeId, fromAddr: Address, d.sendNodes(fromId, fromAddr, reqId, d.routingTable.neighboursAtDistance(distance)) -proc receive*(d: Protocol, a: Address, msg: Bytes) {.gcsafe, +proc receive*(d: Protocol, a: Address, msg: openArray[byte]) {.gcsafe, raises: [ Defect, # TODO This is now coming from Chronos's callSoon diff --git a/eth/p2p/rlpx.nim b/eth/p2p/rlpx.nim index 194c3a2..bc01b9d 100644 --- a/eth/p2p/rlpx.nim +++ b/eth/p2p/rlpx.nim @@ -255,7 +255,7 @@ proc invokeThunk*(peer: Peer, msgId: int, msgData: var Rlp): Future[void] = return thunk(peer, msgId, msgData) -template compressMsg(peer: Peer, data: Bytes): Bytes = +template compressMsg(peer: Peer, data: seq[byte]): seq[byte] = when useSnappy: if peer.snappyEnabled: snappy.compress(data) @@ -263,7 +263,7 @@ template compressMsg(peer: Peer, data: Bytes): Bytes = else: data -proc sendMsg*(peer: Peer, data: Bytes) {.gcsafe, async.} = +proc sendMsg*(peer: Peer, data: seq[byte]) {.gcsafe, async.} = try: var cipherText = encryptMsg(peer.compressMsg(data), peer.secretsState) var res = await peer.transport.write(cipherText) @@ -426,7 +426,7 @@ proc recvMsg*(peer: Peer): Future[tuple[msgId: int, msgData: Rlp]] {.async.} = if decryptedBytes.len == 0: await peer.disconnectAndRaise(BreachOfProtocol, "Snappy uncompress encountered malformed data") - var rlp = rlpFromBytes(decryptedBytes.toRange) + var rlp = rlpFromBytes(decryptedBytes) try: # int32 as this seems more than big enough for the amount of msgIds @@ -561,7 +561,6 @@ proc p2pProtocolBackendImpl*(protocol: P2PProtocol): Backend = EthereumNode = bindSym "EthereumNode" initRlpWriter = bindSym "initRlpWriter" - rlpFromBytes = bindSym "rlpFromBytes" append = bindSym("append", brForceOpen) read = bindSym("read", brForceOpen) checkedRlpRead = bindSym "checkedRlpRead" diff --git a/eth/p2p/rlpx_protocols/les/flow_control.nim b/eth/p2p/rlpx_protocols/les/flow_control.nim index fa59b62..9aef7bd 100644 --- a/eth/p2p/rlpx_protocols/les/flow_control.nim +++ b/eth/p2p/rlpx_protocols/les/flow_control.nim @@ -103,7 +103,7 @@ proc loadMessageStats*(network: LesNetwork, break readFromDB try: - var statsRlp = rlpFromBytes(stats.toRange) + var statsRlp = rlpFromBytes(stats) if not statsRlp.enterList: notice "Found a corrupted LES stats record" break readFromDB diff --git a/eth/p2p/rlpx_protocols/waku_mail.nim b/eth/p2p/rlpx_protocols/waku_mail.nim index b9ed49d..850ccaf 100644 --- a/eth/p2p/rlpx_protocols/waku_mail.nim +++ b/eth/p2p/rlpx_protocols/waku_mail.nim @@ -15,12 +15,12 @@ const requestCompleteTimeout = chronos.seconds(5) type - Cursor = Bytes + Cursor = seq[byte] MailRequest* = object lower*: uint32 ## Unix timestamp; oldest requested envelope's creation time upper*: uint32 ## Unix timestamp; newest requested envelope's creation time - bloom*: Bytes ## Bloom filter to apply on the envelopes + bloom*: seq[byte] ## Bloom filter to apply on the envelopes limit*: uint32 ## Maximum amount of envelopes to return cursor*: Cursor ## Optional cursor diff --git a/eth/p2p/rlpx_protocols/waku_protocol.nim b/eth/p2p/rlpx_protocols/waku_protocol.nim index 8269c51..1208f5d 100644 --- a/eth/p2p/rlpx_protocols/waku_protocol.nim +++ b/eth/p2p/rlpx_protocols/waku_protocol.nim @@ -144,7 +144,7 @@ proc append*(rlpWriter: var RlpWriter, value: StatusOptions) = let bytes = list.finish() - rlpWriter.append(rlpFromBytes(bytes.toRange)) + rlpWriter.append(rlpFromBytes(bytes)) proc read*(rlp: var Rlp, T: typedesc[StatusOptions]): T = if not rlp.isList(): @@ -379,7 +379,7 @@ p2pProtocol Waku(version = wakuVersion, proc p2pRequestComplete(peer: Peer, requestId: Hash, lastEnvelopeHash: Hash, - cursor: Bytes) = discard + cursor: seq[byte]) = discard # TODO: # In the current specification the parameters are not wrapped in a regular # envelope as is done for the P2P Request packet. If we could alter this in @@ -488,8 +488,8 @@ proc queueMessage(node: EthereumNode, msg: Message): bool = proc postMessage*(node: EthereumNode, pubKey = none[PublicKey](), symKey = none[SymKey](), src = none[PrivateKey](), - ttl: uint32, topic: Topic, payload: Bytes, - padding = none[Bytes](), powTime = 1'f, + ttl: uint32, topic: Topic, payload: seq[byte], + padding = none[seq[byte]](), powTime = 1'f, powTarget = defaultMinPow, targetPeer = none[NodeId]()): bool = ## Post a message on the message queue which will be processed at the diff --git a/eth/p2p/rlpx_protocols/whisper/whisper_types.nim b/eth/p2p/rlpx_protocols/whisper/whisper_types.nim index 0fa81e9..aaa20aa 100644 --- a/eth/p2p/rlpx_protocols/whisper/whisper_types.nim +++ b/eth/p2p/rlpx_protocols/whisper/whisper_types.nim @@ -9,7 +9,7 @@ # import - algorithm, bitops, math, options, strutils, tables, times, chronicles, hashes, + algorithm, bitops, math, options, tables, times, chronicles, hashes, strutils, stew/[byteutils, endians2], metrics, nimcrypto/[bcmode, hash, keccak, rijndael, sysrand], eth/[keys, rlp, p2p], eth/p2p/ecies @@ -56,16 +56,16 @@ type src*: Option[PrivateKey] ## Optional key used for signing message dst*: Option[PublicKey] ## Optional key used for asymmetric encryption symKey*: Option[SymKey] ## Optional key used for symmetric encryption - payload*: Bytes ## Application data / message contents - padding*: Option[Bytes] ## Padding - if unset, will automatically pad up to + payload*: seq[byte] ## Application data / message contents + padding*: Option[seq[byte]] ## Padding - if unset, will automatically pad up to ## nearest maxPadLen-byte boundary DecodedPayload* = object ## The decoded payload of a received message. src*: Option[PublicKey] ## If the message was signed, this is the public key ## of the source - payload*: Bytes ## Application data / message contents - padding*: Option[Bytes] ## Message padding + payload*: seq[byte] ## Application data / message contents + padding*: Option[seq[byte]] ## Message padding Envelope* = object ## What goes on the wire in the whisper protocol - a payload and some @@ -74,7 +74,7 @@ type expiry*: uint32 ## Unix timestamp when message expires ttl*: uint32 ## Time-to-live, seconds - message was created at (expiry - ttl) topic*: Topic - data*: Bytes ## Payload, as given by user + data*: seq[byte] ## Payload, as given by user nonce*: uint64 ## Nonce used for proof-of-work calculation Message* = object @@ -177,7 +177,7 @@ proc `or`(a, b: Bloom): Bloom = for i in 0..= 2 and input[0] == '0' and input[1] == 'x': 2 - else: 0 - - let totalBytes = (input.len - startByte) div 2 - var backingStore = newSeq[byte](totalBytes) - - for i in 0 ..< totalBytes: - var nextByte: int - if parseHex(input, nextByte, startByte + i*2, 2) == 2: - backingStore[i] = byte(nextByte) - else: - doAssert false, "rlpFromHex expects a hexademical string, but the input contains non hexademical characters" - - result.bytes = backingStore.toRange() - -{.this: self.} + rlpFromBytes(hexToSeqByte(input)) proc hasData*(self: Rlp): bool = - position < bytes.len + self.position < self.bytes.len proc currentElemEnd*(self: Rlp): int {.gcsafe.} -proc rawData*(self: Rlp): BytesRange = - return self.bytes[position ..< self.currentElemEnd] +template rawData*(self: Rlp): openArray[byte] = + self.bytes.toOpenArray(self.position, self.currentElemEnd - 1) proc isBlob*(self: Rlp): bool = - hasData() and bytes[position] < LIST_START_MARKER + self.hasData() and self.bytes[self.position] < LIST_START_MARKER proc isEmpty*(self: Rlp): bool = ### Contains a blob or a list of zero length - hasData() and (bytes[position] == BLOB_START_MARKER or - bytes[position] == LIST_START_MARKER) + self.hasData() and (self.bytes[self.position] == BLOB_START_MARKER or + self.bytes[self.position] == LIST_START_MARKER) proc isList*(self: Rlp): bool = - hasData() and bytes[position] >= LIST_START_MARKER + self.hasData() and self.bytes[self.position] >= LIST_START_MARKER template eosError = raise newException(MalformedRlpError, "Read past the end of the RLP stream") template requireData {.dirty.} = - if not hasData(): + if not self.hasData(): raise newException(MalformedRlpError, "Illegal operation over an empty RLP stream") proc getType*(self: Rlp): RlpNodeType = requireData() - return if isBlob(): rlpBlob else: rlpList + return if self.isBlob(): rlpBlob else: rlpList proc lengthBytesCount(self: Rlp): int = - var marker = bytes[position] - if isBlob() and marker > LEN_PREFIXED_BLOB_MARKER: + var marker = self.bytes[self.position] + if self.isBlob() and marker > LEN_PREFIXED_BLOB_MARKER: return int(marker - LEN_PREFIXED_BLOB_MARKER) - if isList() and marker > LEN_PREFIXED_LIST_MARKER: + if self.isList() and marker > LEN_PREFIXED_LIST_MARKER: return int(marker - LEN_PREFIXED_LIST_MARKER) return 0 proc isSingleByte*(self: Rlp): bool = - hasData() and bytes[position] < BLOB_START_MARKER + self.hasData() and self.bytes[self.position] < BLOB_START_MARKER proc getByteValue*(self: Rlp): byte = doAssert self.isSingleByte() - return bytes[position] + return self.bytes[self.position] proc payloadOffset(self: Rlp): int = - if isSingleByte(): 0 else: 1 + lengthBytesCount() + if self.isSingleByte(): 0 else: 1 + self.lengthBytesCount() -template readAheadCheck(numberOfBytes) = +template readAheadCheck(numberOfBytes: int) = # important to add nothing to the left side of the equation as `numberOfBytes` # can in theory be at max size of its type already - if numberOfBytes > bytes.len - position - payloadOffset(): eosError() + if numberOfBytes > self.bytes.len - self.position - self.payloadOffset(): + eosError() template nonCanonicalNumberError = raise newException(MalformedRlpError, "Small number encoded in a non-canonical way") proc payloadBytesCount(self: Rlp): int = - if not hasData(): + if not self.hasData(): return 0 - var marker = bytes[position] + var marker = self.bytes[self.position] if marker < BLOB_START_MARKER: return 1 if marker <= LEN_PREFIXED_BLOB_MARKER: result = int(marker - BLOB_START_MARKER) readAheadCheck(result) if result == 1: - if bytes[position + 1] < BLOB_START_MARKER: + if self.bytes[self.position + 1] < BLOB_START_MARKER: nonCanonicalNumberError() return @@ -162,22 +147,22 @@ proc payloadBytesCount(self: Rlp): int = readAheadCheck(result) proc blobLen*(self: Rlp): int = - if isBlob(): payloadBytesCount() else: 0 + if self.isBlob(): self.payloadBytesCount() else: 0 proc isInt*(self: Rlp): bool = - if not hasData(): + if not self.hasData(): return false - var marker = bytes[position] + var marker = self.bytes[self.position] if marker < BLOB_START_MARKER: return marker != 0 if marker == BLOB_START_MARKER: return true if marker <= LEN_PREFIXED_BLOB_MARKER: - return bytes[position + 1] != 0 + return self.bytes[self.position + 1] != 0 if marker < LIST_START_MARKER: - let offset = position + int(marker + 1 - LEN_PREFIXED_BLOB_MARKER) - if offset >= bytes.len: eosError() - return bytes[offset] != 0 + let offset = self.position + int(marker + 1 - LEN_PREFIXED_BLOB_MARKER) + if offset >= self.bytes.len: eosError() + return self.bytes[offset] != 0 return false template maxBytes*(o: type[Ordinal | uint64 | uint]): int = sizeof(o) @@ -206,83 +191,83 @@ proc toInt*(self: Rlp, IntType: type): IntType = result = (result shl 8) or OutputType(self.bytes[self.position + i]) proc toString*(self: Rlp): string = - if not isBlob(): + if not self.isBlob(): raise newException(RlpTypeMismatch, "String expected, but the source RLP is not a blob") let - payloadOffset = payloadOffset() - payloadLen = payloadBytesCount() + payloadOffset = self.payloadOffset() + payloadLen = self.payloadBytesCount() result = newString(payloadLen) for i in 0 ..< payloadLen: # XXX: switch to copyMem here - result[i] = char(bytes[position + payloadOffset + i]) + result[i] = char(self.bytes[self.position + payloadOffset + i]) -proc toBytes*(self: Rlp): BytesRange = - if not isBlob(): +proc toBytes*(self: Rlp): seq[byte] = + if not self.isBlob(): raise newException(RlpTypeMismatch, "Bytes expected, but the source RLP in not a blob") - let payloadLen = payloadBytesCount() + let payloadLen = self.payloadBytesCount() if payloadLen > 0: let - payloadOffset = payloadOffset() - ibegin = position + payloadOffset + payloadOffset = self.payloadOffset() + ibegin = self.position + payloadOffset iend = ibegin + payloadLen - 1 - result = bytes.slice(ibegin, iend) + result = self.bytes[ibegin..iend] proc currentElemEnd*(self: Rlp): int = - doAssert hasData() - result = position + doAssert self.hasData() + result = self.position - if isSingleByte(): + if self.isSingleByte(): result += 1 - elif isBlob() or isList(): - result += payloadOffset() + payloadBytesCount() + elif self.isBlob() or self.isList(): + result += self.payloadOffset() + self.payloadBytesCount() proc enterList*(self: var Rlp): bool = - if not isList(): + if not self.isList(): return false - position += payloadOffset() + self.position += self.payloadOffset() return true proc tryEnterList*(self: var Rlp) = - if not enterList(): + if not self.enterList(): raise newException(RlpTypeMismatch, "List expected, but source RLP is not a list") proc skipElem*(rlp: var Rlp) = rlp.position = rlp.currentElemEnd iterator items*(self: var Rlp): var Rlp = - doAssert isList() + doAssert self.isList() var - payloadOffset = payloadOffset() - payloadEnd = position + payloadOffset + payloadBytesCount() + payloadOffset = self.payloadOffset() + payloadEnd = self.position + payloadOffset + self.payloadBytesCount() - if payloadEnd > bytes.len: + if payloadEnd > self.bytes.len: raise newException(MalformedRlpError, "List length extends past the end of the stream") - position += payloadOffset + self.position += payloadOffset - while position < payloadEnd: - let elemEnd = currentElemEnd() + while self.position < payloadEnd: + let elemEnd = self.currentElemEnd() yield self - position = elemEnd + self.position = elemEnd proc listElem*(self: Rlp, i: int): Rlp = - doAssert isList() + doAssert self.isList() let - payloadOffset = payloadOffset() + payloadOffset = self.payloadOffset() # This will only check if there is some data, not if it is correct according # to list length. Could also run here payloadBytesCount() instead. - if position + payloadOffset + 1 > bytes.len: eosError() + if self.position + payloadOffset + 1 > self.bytes.len: eosError() - let payload = bytes.slice(position + payloadOffset) + let payload = self.bytes[self.position + payloadOffset..^1] result = rlpFromBytes payload var pos = 0 while pos < i and result.hasData: @@ -290,7 +275,7 @@ proc listElem*(self: Rlp, i: int): Rlp = inc pos proc listLen*(self: Rlp): int = - if not isList(): + if not self.isList(): return 0 var rlp = self @@ -336,7 +321,7 @@ proc readImpl[R, E](rlp: var Rlp, T: type array[R, E]): T = if result.len != bytes.len: raise newException(RlpTypeMismatch, "Fixed-size array expected, but the source RLP contains a blob of different length") - copyMem(addr result[0], bytes.baseAddr, bytes.len) + copyMem(addr result[0], unsafeAddr bytes[0], bytes.len) rlp.skipElem @@ -356,10 +341,7 @@ proc readImpl[E](rlp: var Rlp, T: type seq[E]): T = mixin read when E is (byte or char): - var bytes = rlp.toBytes - if bytes.len != 0: - result = newSeq[byte](bytes.len) - copyMem(addr result[0], bytes.baseAddr, bytes.len) + result = rlp.toBytes rlp.skipElem else: if not rlp.isList: @@ -383,7 +365,9 @@ proc readImpl(rlp: var Rlp, T: type[object|tuple], "List expected, but the source RLP is not a list.") var payloadOffset = rlp.payloadOffset() - payloadEnd = rlp.position + payloadOffset + rlp.payloadBytesCount() + + # there's an exception-raising side effect in there *sigh* + discard rlp.payloadBytesCount() rlp.position += payloadOffset @@ -398,16 +382,16 @@ proc readImpl(rlp: var Rlp, T: type[object|tuple], proc toNodes*(self: var Rlp): RlpNode = requireData() - if isList(): + if self.isList(): result.kind = rlpList newSeq result.elems, 0 for e in self: result.elems.add e.toNodes else: - doAssert isBlob() + doAssert self.isBlob() result.kind = rlpBlob - result.bytes = toBytes() - position = currentElemEnd() + result.bytes = self.toBytes() + self.position = self.currentElemEnd() # We define a single `read` template with a pretty low specifity # score in order to facilitate easier overloading with user types: @@ -422,22 +406,18 @@ template readRecordType*(rlp: var Rlp, T: type, wrappedInList: bool): auto = readImpl(rlp, T, wrappedInList) proc decode*(bytes: openarray[byte]): RlpNode = - var - bytesCopy = @bytes - rlp = rlpFromBytes(bytesCopy.toRange()) - return rlp.toNodes + var rlp = rlpFromBytes(bytes) + rlp.toNodes -template decode*(bytes: BytesRange, T: type): untyped = +template decode*(bytes: openArray[byte], T: type): untyped = mixin read var rlp = rlpFromBytes(bytes) rlp.read(T) -template decode*(bytes: openarray[byte], T: type): T = - var bytesCopy = @bytes - decode(bytesCopy.toRange, T) - template decode*(bytes: seq[byte], T: type): untyped = - decode(bytes.toRange, T) + mixin read + var rlp = rlpFromBytes(bytes) + rlp.read(T) proc append*(writer: var RlpWriter; rlp: Rlp) = appendRawBytes(writer, rlp.rawData) @@ -450,7 +430,7 @@ proc isPrintable(s: string): bool = return true proc inspectAux(self: var Rlp, depth: int, hexOutput: bool, output: var string) = - if not hasData(): + if not self.hasData(): return template indent = @@ -461,7 +441,7 @@ proc inspectAux(self: var Rlp, depth: int, hexOutput: bool, output: var string) if self.isSingleByte: output.add "byte " - output.add $bytes[position] + output.add $self.bytes[self.position] elif self.isBlob: let str = self.toString if str.isPrintable: @@ -491,6 +471,5 @@ proc inspectAux(self: var Rlp, depth: int, hexOutput: bool, output: var string) proc inspect*(self: Rlp, indent = 0, hexOutput = true): string = var rlpCopy = self - result = newStringOfCap(bytes.len) + result = newStringOfCap(self.bytes.len) inspectAux(rlpCopy, indent, hexOutput, result) - diff --git a/eth/rlp/types.nim b/eth/rlp/types.nim deleted file mode 100644 index 73271ad..0000000 --- a/eth/rlp/types.nim +++ /dev/null @@ -1,6 +0,0 @@ -import stew/ranges -export ranges - -type - Bytes* = seq[byte] - BytesRange* = Range[byte] diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index b082f3f..4669a68 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -1,15 +1,11 @@ import - macros, types, - stew/ranges/[memranges, ptr_arith], + macros, object_serialization, priv/defs -export - memranges - type RlpWriter* = object pendingLists: seq[tuple[remainingItems, outBytes: int]] - output: Bytes + output: seq[byte] IntLike* = concept x, y type T = type(x) @@ -39,7 +35,7 @@ proc bytesNeeded(num: Integer): int = inc result n = n shr 8 -proc writeBigEndian(outStream: var Bytes, number: Integer, +proc writeBigEndian(outStream: var seq[byte], number: Integer, lastByteIdx: int, numberOfBytes: int) = mixin `and`, `shr` @@ -48,12 +44,12 @@ proc writeBigEndian(outStream: var Bytes, number: Integer, outStream[i] = byte(n and 0xff) n = n shr 8 -proc writeBigEndian(outStream: var Bytes, number: Integer, +proc writeBigEndian(outStream: var seq[byte], number: Integer, numberOfBytes: int) {.inline.} = outStream.setLen(outStream.len + numberOfBytes) outStream.writeBigEndian(number, outStream.len - 1, numberOfBytes) -proc writeCount(bytes: var Bytes, count: int, baseMarker: byte) = +proc writeCount(bytes: var seq[byte], count: int, baseMarker: byte) = if count < THRESHOLD_LIST_LEN: bytes.add(baseMarker + byte(count)) else: @@ -65,19 +61,6 @@ proc writeCount(bytes: var Bytes, count: int, baseMarker: byte) = bytes[origLen] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) bytes.writeBigEndian(count, bytes.len - 1, lenPrefixBytes) -proc add(outStream: var Bytes, newChunk: BytesRange) = - let prevLen = outStream.len - outStream.setLen(prevLen + newChunk.len) - # XXX: Use copyMem here - for i in 0 ..< newChunk.len: - outStream[prevLen + i] = newChunk[i] - -{.this: self.} -{.experimental.} - -using - self: var RlpWriter - proc initRlpWriter*: RlpWriter = newSeq(result.pendingLists, 0) newSeq(result.output, 0) @@ -86,88 +69,74 @@ proc decRet(n: var int, delta: int): int = n -= delta return n -proc maybeClosePendingLists(self) = - while pendingLists.len > 0: - let lastListIdx = pendingLists.len - 1 - doAssert pendingLists[lastListIdx].remainingItems >= 1 - if decRet(pendingLists[lastListIdx].remainingItems, 1) == 0: +proc maybeClosePendingLists(self: var RlpWriter) = + while self.pendingLists.len > 0: + let lastListIdx = self.pendingLists.len - 1 + doAssert self.pendingLists[lastListIdx].remainingItems >= 1 + if decRet(self.pendingLists[lastListIdx].remainingItems, 1) == 0: # A list have been just finished. It was started in `startList`. - let listStartPos = pendingLists[lastListIdx].outBytes - pendingLists.setLen lastListIdx + let listStartPos = self.pendingLists[lastListIdx].outBytes + self.pendingLists.setLen lastListIdx # How many bytes were written since the start? - let listLen = output.len - listStartPos + let listLen = self.output.len - listStartPos # Compute the number of bytes required to write down the list length let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 else: int(listLen.bytesNeeded) + 1 # Shift the written data to make room for the prefix length - output.setLen(output.len + totalPrefixBytes) - let outputBaseAddr = output.baseAddr + self.output.setLen(self.output.len + totalPrefixBytes) - moveMem(outputBaseAddr.shift(listStartPos + totalPrefixBytes), - outputBaseAddr.shift(listStartPos), + moveMem(addr self.output[listStartPos + totalPrefixBytes], + unsafeAddr self.output[listStartPos], listLen) # Write out the prefix length if listLen < THRESHOLD_LIST_LEN: - output[listStartPos] = LIST_START_MARKER + byte(listLen) + self.output[listStartPos] = LIST_START_MARKER + byte(listLen) else: let listLenBytes = totalPrefixBytes - 1 - output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) - output.writeBigEndian(listLen, listStartPos + listLenBytes, listLenBytes) + self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + self.output.writeBigEndian(listLen, listStartPos + listLenBytes, listLenBytes) else: # The currently open list is not finished yet. Nothing to do. return -proc appendRawList(self; bytes: BytesRange) = - output.writeCount(bytes.len, LIST_START_MARKER) - output.add(bytes) - maybeClosePendingLists() +proc appendRawList(self: var RlpWriter, bytes: openArray[byte]) = + self.output.writeCount(bytes.len, LIST_START_MARKER) + self.output.add(bytes) + self.maybeClosePendingLists() -proc appendRawBytes*(self; bytes: BytesRange) = - output.add(bytes) - maybeClosePendingLists() +proc appendRawBytes*(self: var RlpWriter, bytes: openArray[byte]) = + self.output.add(bytes) + self.maybeClosePendingLists() -proc startList*(self; listSize: int) = +proc startList*(self: var RlpWriter, listSize: int) = if listSize == 0: - appendRawList(BytesRange()) + self.appendRawList([]) else: - pendingLists.add((listSize, output.len)) - -template appendBlob(self; data, startMarker) = - mixin baseAddr + self.pendingLists.add((listSize, self.output.len)) +proc appendBlob(self: var RlpWriter, data: openArray[byte], startMarker: byte) = if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: self.output.add byte(data[0]) else: self.output.writeCount(data.len, startMarker) + self.output.add data - let startPos = output.len - self.output.setLen(startPos + data.len) - copyMem(shift(baseAddr(self.output), startPos), - baseAddr(data), - data.len) + self.maybeClosePendingLists() - maybeClosePendingLists() +proc appendImpl(self: var RlpWriter, data: string) = + appendBlob(self, data.toOpenArrayByte(0, data.high), BLOB_START_MARKER) -proc appendImpl(self; data: string) = +proc appendBlob(self: var RlpWriter, data: openarray[byte]) = appendBlob(self, data, BLOB_START_MARKER) -proc appendBlob(self; data: openarray[byte]) = - appendBlob(self, data, BLOB_START_MARKER) +proc appendBlob(self: var RlpWriter, data: openarray[char]) = + appendBlob(self, data.toOpenArrayByte(0, data.high), BLOB_START_MARKER) -proc appendBlob(self; data: openarray[char]) = - appendBlob(self, data, BLOB_START_MARKER) - -proc appendBytesRange(self; data: BytesRange) = - appendBlob(self, data, BLOB_START_MARKER) - -proc appendImpl(self; data: MemRange) = - appendBlob(self, data, BLOB_START_MARKER) - -proc appendInt(self; i: Integer) = +proc appendInt(self: var RlpWriter, i: Integer) = # this is created as a separate proc as an extra precaution against # any overloading resolution problems when matching the IntLike concept. type IntType = type(i) @@ -183,7 +152,7 @@ proc appendInt(self; i: Integer) = self.maybeClosePendingLists() -proc appendFloat(self; data: float64) = +proc appendFloat(self: var RlpWriter, data: float64) = # This is not covered in the RLP spec, but Geth uses Go's # `math.Float64bits`, which is defined here: # https://github.com/gopherjs/gopherjs/blob/master/compiler/natives/src/math/math.go @@ -191,16 +160,16 @@ proc appendFloat(self; data: float64) = let uint64bits = (uint64(uintWords[1]) shl 32) or uint64(uintWords[0]) self.appendInt(uint64bits) -template appendImpl(self; i: Integer) = +template appendImpl(self: var RlpWriter, i: Integer) = appendInt(self, i) -template appendImpl(self; e: enum) = +template appendImpl(self: var RlpWriter, e: enum) = appendImpl(self, int(e)) -template appendImpl(self; b: bool) = +template appendImpl(self: var RlpWriter, b: bool) = appendImpl(self, int(b)) -proc appendImpl[T](self; listOrBlob: openarray[T]) = +proc appendImpl[T](self: var RlpWriter, listOrBlob: openarray[T]) = mixin append # TODO: This append proc should be overloaded by `openarray[byte]` after @@ -212,7 +181,7 @@ proc appendImpl[T](self; listOrBlob: openarray[T]) = for i in 0 ..< listOrBlob.len: self.append listOrBlob[i] -proc appendRecordType*(self; obj: object|tuple, wrapInList = wrapObjsInList) = +proc appendRecordType*(self: var RlpWriter, obj: object|tuple, wrapInList = wrapObjsInList) = mixin enumerateRlpFields, append if wrapInList: @@ -226,15 +195,10 @@ proc appendRecordType*(self; obj: object|tuple, wrapInList = wrapObjsInList) = enumerateRlpFields(obj, op) -proc appendImpl(self; data: object) {.inline.} = - # TODO: This append proc should be overloaded by `BytesRange` after - # nim bug #7416 is fixed. - when data is BytesRange: - self.appendBytesRange(data) - else: +proc appendImpl(self: var RlpWriter, data: object) {.inline.} = self.appendRecordType(data) -proc appendImpl(self; data: tuple) {.inline.} = +proc appendImpl(self: var RlpWriter, data: tuple) {.inline.} = self.appendRecordType(data) # We define a single `append` template with a pretty low specifity @@ -253,22 +217,22 @@ proc initRlpList*(listSize: int): RlpWriter = startList(result, listSize) # TODO: This should return a lent value -proc finish*(self): Bytes = - doAssert pendingLists.len == 0, "Insufficient number of elements written to a started list" - result = output +template finish*(self: RlpWriter): seq[byte] = + doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" + self.output -proc encode*[T](v: T): Bytes = +proc encode*[T](v: T): seq[byte] = mixin append var writer = initRlpWriter() writer.append(v) return writer.finish -proc encodeInt*(i: Integer): Bytes = +proc encodeInt*(i: Integer): seq[byte] = var writer = initRlpWriter() writer.appendInt(i) return writer.finish -macro encodeList*(args: varargs[untyped]): Bytes = +macro encodeList*(args: varargs[untyped]): seq[byte] = var listLen = args.len writer = genSym(nskVar, "rlpWriter") @@ -286,10 +250,9 @@ macro encodeList*(args: varargs[untyped]): Bytes = when false: # XXX: Currently fails with a malformed AST error on the args.len expression - template encodeList*(args: varargs[untyped]): BytesRange = + template encodeList*(args: varargs[untyped]): seq[byte] = mixin append var writer = initRlpList(args.len) for arg in args: writer.append(arg) writer.finish - diff --git a/eth/trie/backends/caching_backend.nim b/eth/trie/backends/caching_backend.nim index b18cee5..3834502 100644 --- a/eth/trie/backends/caching_backend.nim +++ b/eth/trie/backends/caching_backend.nim @@ -1,5 +1,5 @@ import - stew/ranges, tables, sets, + tables, sets, eth/trie/db type diff --git a/eth/trie/backends/lmdb_backend.nim b/eth/trie/backends/lmdb_backend.nim index bc1ef34..b83c182 100644 --- a/eth/trie/backends/lmdb_backend.nim +++ b/eth/trie/backends/lmdb_backend.nim @@ -1,4 +1,4 @@ -import os, stew/ranges, eth/trie/[trie_defs, db_tracing] +import os, eth/trie/[trie_defs, db_tracing] import backend_defs when defined(windows): diff --git a/eth/trie/backends/rocksdb_backend.nim b/eth/trie/backends/rocksdb_backend.nim index 67ae4a7..9f8c3dd 100644 --- a/eth/trie/backends/rocksdb_backend.nim +++ b/eth/trie/backends/rocksdb_backend.nim @@ -1,4 +1,4 @@ -import os, rocksdb, stew/ranges, eth/trie/[trie_defs, db_tracing] +import os, rocksdb, eth/trie/[trie_defs, db_tracing] import backend_defs type diff --git a/eth/trie/backends/sqlite_backend.nim b/eth/trie/backends/sqlite_backend.nim index 81f016c..8cd6696 100644 --- a/eth/trie/backends/sqlite_backend.nim +++ b/eth/trie/backends/sqlite_backend.nim @@ -1,5 +1,5 @@ import - os, sqlite3, stew/ranges, stew/ranges/ptr_arith, eth/trie/[db_tracing, trie_defs], + os, sqlite3, stew/ranges/ptr_arith, eth/trie/[db_tracing, trie_defs], backend_defs type diff --git a/eth/trie/binaries.nim b/eth/trie/binaries.nim index a81b40d..b37608e 100644 --- a/eth/trie/binaries.nim +++ b/eth/trie/binaries.nim @@ -1,6 +1,7 @@ import sequtils, - stew/ranges/[ptr_arith, bitranges], eth/rlp/types, trie_defs + stew/ranges/ptr_arith, trie_defs, + ./trie_bitseq type TrieNodeKind* = enum @@ -8,31 +9,30 @@ type BRANCH_TYPE = 1 LEAF_TYPE = 2 - TrieNodeKey* = BytesRange - TrieBitRange* = BitRange + TrieNodeKey* = seq[byte] TrieNode* = object case kind*: TrieNodeKind of KV_TYPE: - keyPath*: TrieBitRange + keyPath*: TrieBitSeq child*: TrieNodeKey of BRANCH_TYPE: leftChild*: TrieNodeKey rightChild*: TrieNodeKey of LEAF_TYPE: - value*: BytesRange + value*: seq[byte] InvalidNode* = object of CorruptedTrieDatabase ValidationError* = object of CorruptedTrieDatabase # ---------------------------------------------- -template sliceToEnd*(r: TrieBitRange, index: int): TrieBitRange = - if r.len <= index: TrieBitRange() else: r[index .. ^1] +template sliceToEnd*(r: TrieBitSeq, index: int): TrieBitSeq = + if r.len <= index: TrieBitSeq() else: r[index .. ^1] -proc decodeToBinKeypath*(path: BytesRange): TrieBitRange = +proc decodeToBinKeypath*(path: seq[byte]): TrieBitSeq = ## Decodes bytes into a sequence of 0s and 1s ## Used in decoding key path of a KV-NODE - var path = MutByteRange(path).bits + var path = path.bits if path[0]: path = path[4..^1] @@ -42,11 +42,11 @@ proc decodeToBinKeypath*(path: BytesRange): TrieBitRange = bits = bits or path[3].int if path.len > 4: - result = path[4+((4 - bits) mod 4)..^1] + path[4+((4 - bits) mod 4)..^1] else: - result = BitRange() + TrieBitSeq() -proc parseNode*(node: BytesRange): TrieNode = +proc parseNode*(node: openArray[byte]): TrieNode = # Input: a serialized node if node.len == 0: @@ -76,7 +76,7 @@ proc parseNode*(node: BytesRange): TrieNode = # Output: node type, value return TrieNode(kind: LEAF_TYPE, value: node[1..^1]) -proc encodeKVNode*(keyPath: TrieBitRange, childHash: TrieNodeKey): Bytes = +proc encodeKVNode*(keyPath: TrieBitSeq, childHash: TrieNodeKey): seq[byte] = ## Serializes a key/value node if keyPath.len == 0: raise newException(ValidationError, "Key path can not be empty") @@ -110,13 +110,13 @@ proc encodeKVNode*(keyPath: TrieBitRange, childHash: TrieNodeKey): Bytes = inc(nbits, 8) copyMem(result[^32].addr, childHash.baseAddr, 32) -proc encodeKVNode*(keyPath: bool, childHash: TrieNodeKey): Bytes = +proc encodeKVNode*(keyPath: bool, childHash: TrieNodeKey): seq[byte] = result = newSeq[byte](34) result[0] = KV_TYPE.byte result[1] = byte(16) or byte(keyPath) copyMem(result[^32].addr, childHash.baseAddr, 32) -proc encodeBranchNode*(leftChildHash, rightChildHash: TrieNodeKey): Bytes = +proc encodeBranchNode*(leftChildHash, rightChildHash: TrieNodeKey): seq[byte] = ## Serializes a branch node const BRANCH_TYPE_PREFIX = @[BRANCH_TYPE.byte] @@ -126,7 +126,7 @@ proc encodeBranchNode*(leftChildHash, rightChildHash: TrieNodeKey): Bytes = result = BRANCH_TYPE_PREFIX.concat(leftChildHash, rightChildHash) -proc encodeLeafNode*(value: BytesRange | Bytes): Bytes = +proc encodeLeafNode*(value: openArray[byte]): seq[byte] = ## Serializes a leaf node const LEAF_TYPE_PREFIX = @[LEAF_TYPE.byte] @@ -134,9 +134,9 @@ proc encodeLeafNode*(value: BytesRange | Bytes): Bytes = if value.len == 0: raise newException(ValidationError, "Value of leaf node can not be empty") - result = LEAF_TYPE_PREFIX.concat(value) + result = LEAF_TYPE_PREFIX.concat(@value) -proc getCommonPrefixLength*(a, b: TrieBitRange): int = +proc getCommonPrefixLength*(a, b: TrieBitSeq): int = let len = min(a.len, b.len) for i in 0.. 0: let (nodeRlp, path) = stack.pop() if not nodeRlp.hasData or nodeRlp.isEmpty: @@ -172,15 +164,14 @@ proc getKeysAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesRange]]) raise newException(CorruptedTrieDatabase, "HexaryTrie node with an unexpected number of children") -iterator keys*(self: HexaryTrie): BytesRange = +iterator keys*(self: HexaryTrie): seq[byte] = var nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root) - path = newRange[byte](0) - stack = @[(nodeRlp, initNibbleRange(path))] + stack = @[(nodeRlp, initNibbleRange([]))] while stack.len > 0: yield getKeysAux(self.db, stack) -proc getValuesAux(db: DB, stack: var seq[Rlp]): BytesRange = +proc getValuesAux(db: DB, stack: var seq[Rlp]): seq[byte] = while stack.len > 0: let nodeRlp = stack.pop() if not nodeRlp.hasData or nodeRlp.isEmpty: @@ -211,14 +202,14 @@ proc getValuesAux(db: DB, stack: var seq[Rlp]): BytesRange = raise newException(CorruptedTrieDatabase, "HexaryTrie node with an unexpected number of children") -iterator values*(self: HexaryTrie): BytesRange = +iterator values*(self: HexaryTrie): seq[byte] = var nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root) stack = @[nodeRlp] while stack.len > 0: yield getValuesAux(self.db, stack) -proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesRange]]): (BytesRange, BytesRange) = +proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesSeq]]): (seq[byte], seq[byte]) = while stack.len > 0: let (nodeRlp, path) = stack.pop() if not nodeRlp.hasData or nodeRlp.isEmpty: @@ -254,11 +245,10 @@ proc getPairsAux(db: DB, stack: var seq[tuple[nodeRlp: Rlp, path: NibblesRange]] raise newException(CorruptedTrieDatabase, "HexaryTrie node with an unexpected number of children") -iterator pairs*(self: HexaryTrie): (BytesRange, BytesRange) = +iterator pairs*(self: HexaryTrie): (seq[byte], seq[byte]) = var nodeRlp = rlpFromBytes keyToLocalBytes(self.db, self.root) - path = newRange[byte](0) - stack = @[(nodeRlp, initNibbleRange(path))] + stack = @[(nodeRlp, initNibbleRange([]))] while stack.len > 0: # perhaps a Nim bug #9778 # cannot yield the helper proc directly @@ -266,7 +256,7 @@ iterator pairs*(self: HexaryTrie): (BytesRange, BytesRange) = let res = getPairsAux(self.db, stack) yield res -iterator replicate*(self: HexaryTrie): (BytesRange, BytesRange) = +iterator replicate*(self: HexaryTrie): (seq[byte], seq[byte]) = # this iterator helps 'rebuild' the entire trie without # going through a trie algorithm, but it will pull the entire # low level KV pairs. Thus the target db will only use put operations @@ -274,19 +264,18 @@ iterator replicate*(self: HexaryTrie): (BytesRange, BytesRange) = var localBytes = keyToLocalBytes(self.db, self.root) nodeRlp = rlpFromBytes localBytes - path = newRange[byte](0) - stack = @[(nodeRlp, initNibbleRange(path))] + stack = @[(nodeRlp, initNibbleRange([]))] template pushOrYield(elem: untyped) = if elem.isList: stack.add((elem, key)) else: - let rlpBytes = get(self.db, toOpenArray(elem.expectHash)).toRange + let rlpBytes = get(self.db, elem.expectHash) let nextLookup = rlpFromBytes(rlpBytes) stack.add((nextLookup, key)) yield (elem.toBytes, rlpBytes) - yield (self.rootHash.toRange, localBytes) + yield (@(self.rootHash.data), localBytes) while stack.len > 0: let (nodeRlp, path) = stack.pop() if not nodeRlp.hasData or nodeRlp.isEmpty: @@ -310,21 +299,21 @@ iterator replicate*(self: HexaryTrie): (BytesRange, BytesRange) = raise newException(CorruptedTrieDatabase, "HexaryTrie node with an unexpected number of children") -proc getValues*(self: HexaryTrie): seq[BytesRange] = +proc getValues*(self: HexaryTrie): seq[seq[byte]] = result = @[] for v in self.values: result.add v -proc getKeys*(self: HexaryTrie): seq[BytesRange] = +proc getKeys*(self: HexaryTrie): seq[seq[byte]] = result = @[] for k in self.keys: result.add k template getNode(elem: untyped): untyped = - if elem.isList: elem.rawData - else: get(db, toOpenArray(elem.expectHash)).toRange + if elem.isList: @(elem.rawData) + else: get(db, elem.expectHash) -proc getBranchAux(db: DB, node: BytesRange, path: NibblesRange, output: var seq[BytesRange]) = +proc getBranchAux(db: DB, node: openArray[byte], path: NibblesSeq, output: var seq[seq[byte]]) = var nodeRlp = rlpFromBytes node if not nodeRlp.hasData or nodeRlp.isEmpty: return @@ -349,21 +338,21 @@ proc getBranchAux(db: DB, node: BytesRange, path: NibblesRange, output: var seq[ raise newException(CorruptedTrieDatabase, "HexaryTrie node with an unexpected number of children") -proc getBranch*(self: HexaryTrie; key: BytesRange): seq[BytesRange] = +proc getBranch*(self: HexaryTrie; key: openArray[byte]): seq[seq[byte]] = result = @[] var node = keyToLocalBytes(self.db, self.root) result.add node getBranchAux(self.db, node, initNibbleRange(key), result) -proc dbDel(t: var HexaryTrie, data: BytesRange) = +proc dbDel(t: var HexaryTrie, data: openArray[byte]) = if data.len >= 32: t.prune(data.keccak.data) -proc dbPut(db: DB, data: BytesRange): TrieNodeKey = +proc dbPut(db: DB, data: openArray[byte]): TrieNodeKey = result.hash = data.keccak result.usedBytes = 32 - put(db, result.asDbKey, data.toOpenArray) + put(db, result.asDbKey, data) -proc appendAndSave(rlpWriter: var RlpWriter, data: BytesRange, db: DB) = +proc appendAndSave(rlpWriter: var RlpWriter, data: openArray[byte], db: DB) = if data.len >= 32: var nodeKey = dbPut(db, data) rlpWriter.append(nodeKey.hash) @@ -373,7 +362,7 @@ proc appendAndSave(rlpWriter: var RlpWriter, data: BytesRange, db: DB) = proc isTrieBranch(rlp: Rlp): bool = rlp.isList and (var len = rlp.listLen; len == 2 or len == 17) -proc replaceValue(data: Rlp, key: NibblesRange, value: BytesRange): Bytes = +proc replaceValue(data: Rlp, key: NibblesSeq, value: openArray[byte]): seq[byte] = if data.isEmpty: let prefix = hexPrefixEncode(key, true) return encodeList(prefix, value) @@ -403,11 +392,6 @@ proc isTwoItemNode(self: HexaryTrie; r: Rlp): bool = else: return r.isList and r.listLen == 2 -proc isLeaf(r: Rlp): bool = - doAssert r.isList and r.listLen == 2 - let b = r.listElem(0).toBytes() - return (b[0] and 0x20) != 0 - proc findSingleChild(r: Rlp; childPos: var byte): Rlp = result = zeroBytesRlp var i: byte = 0 @@ -421,10 +405,10 @@ proc findSingleChild(r: Rlp; childPos: var byte): Rlp = return zeroBytesRlp inc i -proc deleteAt(self: var HexaryTrie; origRlp: Rlp, key: NibblesRange): BytesRange {.gcsafe.} +proc deleteAt(self: var HexaryTrie; origRlp: Rlp, key: NibblesSeq): seq[byte] {.gcsafe.} proc deleteAux(self: var HexaryTrie; rlpWriter: var RlpWriter; - origRlp: Rlp; path: NibblesRange): bool = + origRlp: Rlp; path: NibblesSeq): bool = if origRlp.isEmpty: return false @@ -439,16 +423,15 @@ proc deleteAux(self: var HexaryTrie; rlpWriter: var RlpWriter; rlpWriter.appendAndSave(b, self.db) return true -proc graft(self: var HexaryTrie; r: Rlp): Bytes = +proc graft(self: var HexaryTrie; r: Rlp): seq[byte] = doAssert r.isList and r.listLen == 2 - var (origIsLeaf, origPath) = r.extensionNodeKey + var (_, origPath) = r.extensionNodeKey var value = r.listElem(1) - var n: Rlp if not value.isList: let nodeKey = value.expectHash - var resolvedData = self.db.get(nodeKey.toOpenArray).toRange - self.prune(nodeKey.toOpenArray) + var resolvedData = self.db.get(nodeKey) + self.prune(nodeKey) value = rlpFromBytes resolvedData doAssert value.listLen == 2 @@ -460,10 +443,10 @@ proc graft(self: var HexaryTrie; r: Rlp): Bytes = return rlpWriter.finish proc mergeAndGraft(self: var HexaryTrie; - soleChild: Rlp, childPos: byte): Bytes = + soleChild: Rlp, childPos: byte): seq[byte] = var output = initRlpList(2) if childPos == 16: - output.append hexPrefixEncode(zeroNibblesRange, true) + output.append hexPrefixEncode(NibblesSeq(), true) else: doAssert(not soleChild.isEmpty) output.append int(hexPrefixEncodeByte(childPos)) @@ -471,20 +454,20 @@ proc mergeAndGraft(self: var HexaryTrie; result = output.finish() if self.isTwoItemNode(soleChild): - result = self.graft(rlpFromBytes(result.toRange)) + result = self.graft(rlpFromBytes(result)) proc deleteAt(self: var HexaryTrie; - origRlp: Rlp, key: NibblesRange): BytesRange = + origRlp: Rlp, key: NibblesSeq): seq[byte] = if origRlp.isEmpty: - return zeroBytesRange + return doAssert origRlp.isTrieBranch - let origBytes = origRlp.rawData + let origBytes = @(origRlp.rawData) if origRlp.listLen == 2: let (isLeaf, k) = origRlp.extensionNodeKey if k == key and isLeaf: self.dbDel origBytes - return emptyRlp.toRange + return emptyRlp if key.startsWith(k): var @@ -493,22 +476,22 @@ proc deleteAt(self: var HexaryTrie; value = origRlp.listElem(1) rlpWriter.append(path) if not self.deleteAux(rlpWriter, value, key.slice(k.len)): - return zeroBytesRange + return self.dbDel origBytes - var finalBytes = rlpWriter.finish.toRange + var finalBytes = rlpWriter.finish var rlp = rlpFromBytes(finalBytes) if self.isTwoItemNode(rlp.listElem(1)): - return self.graft(rlp).toRange + return self.graft(rlp) return finalBytes else: - return zeroBytesRange + return else: if key.len == 0 and origRlp.listElem(16).isEmpty: self.dbDel origBytes var foundChildPos: byte let singleChild = origRlp.findSingleChild(foundChildPos) if singleChild.hasData and foundChildPos != 16: - result = self.mergeAndGraft(singleChild, foundChildPos).toRange + result = self.mergeAndGraft(singleChild, foundChildPos) else: var rlpRes = initRlpList(17) var iter = origRlp @@ -518,7 +501,7 @@ proc deleteAt(self: var HexaryTrie; rlpRes.append iter iter.skipElem rlpRes.append "" - return rlpRes.finish.toRange + return rlpRes.finish else: var rlpWriter = initRlpList(17) let keyHead = int(key[0]) @@ -527,20 +510,20 @@ proc deleteAt(self: var HexaryTrie; for elem in items(origCopy): if i == keyHead: if not self.deleteAux(rlpWriter, elem, key.slice(1)): - return zeroBytesRange + return else: rlpWriter.append(elem) inc i self.dbDel origBytes - result = rlpWriter.finish.toRange + result = rlpWriter.finish var resultRlp = rlpFromBytes(result) var foundChildPos: byte let singleChild = resultRlp.findSingleChild(foundChildPos) if singleChild.hasData: - result = self.mergeAndGraft(singleChild, foundChildPos).toRange + result = self.mergeAndGraft(singleChild, foundChildPos) -proc del*(self: var HexaryTrie; key: BytesRange) = +proc del*(self: var HexaryTrie; key: openArray[byte]) = var rootBytes = keyToLocalBytes(self.db, self.root) rootRlp = rlpFromBytes rootBytes @@ -552,16 +535,16 @@ proc del*(self: var HexaryTrie; key: BytesRange) = self.root = self.db.dbPut(newRootBytes) proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash, - key: NibblesRange, value: BytesRange, - isInline = false): BytesRange {.gcsafe.} + key: NibblesSeq, value: openArray[byte], + isInline = false): seq[byte] {.gcsafe.} proc mergeAt(self: var HexaryTrie, rlp: Rlp, - key: NibblesRange, value: BytesRange, - isInline = false): BytesRange = + key: NibblesSeq, value: openArray[byte], + isInline = false): seq[byte] = self.mergeAt(rlp, rlp.rawData.keccak, key, value, isInline) proc mergeAtAux(self: var HexaryTrie, output: var RlpWriter, orig: Rlp, - key: NibblesRange, value: BytesRange) = + key: NibblesSeq, value: openArray[byte]) = var resolved = orig var isRemovable = false if not (orig.isList or orig.isEmpty): @@ -572,11 +555,11 @@ proc mergeAtAux(self: var HexaryTrie, output: var RlpWriter, orig: Rlp, output.appendAndSave(b, self.db) proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash, - key: NibblesRange, value: BytesRange, - isInline = false): BytesRange = + key: NibblesSeq, value: openArray[byte], + isInline = false): seq[byte] = template origWithNewValue: auto = self.prune(origHash.data) - replaceValue(orig, key, value).toRange + replaceValue(orig, key, value) if orig.isEmpty: return origWithNewValue() @@ -595,7 +578,7 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash, var r = initRlpList(2) r.append orig.listElem(0) self.mergeAtAux(r, origValue, key.slice(k.len), value) - return r.finish.toRange + return r.finish if orig.rawData.len >= 32: self.prune(origHash.data) @@ -608,9 +591,9 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash, var top = initRlpList(2) top.append hexPrefixEncode(k.slice(0, sharedNibbles), false) - top.appendAndSave(bottom.finish.toRange, self.db) + top.appendAndSave(bottom.finish, self.db) - return self.mergeAt(rlpFromBytes(top.finish.toRange), key, value, true) + return self.mergeAt(rlpFromBytes(top.finish), key, value, true) else: # Create a branch node var branches = initRlpList(17) @@ -626,7 +609,7 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash, if byte(i) == n: if isLeaf or k.len > 1: let childNode = encodeList(hexPrefixEncode(k.slice(1), isLeaf), - origValue).toRange + origValue) branches.appendAndSave(childNode, self.db) else: branches.append origValue @@ -634,7 +617,7 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash, branches.append "" branches.append "" - return self.mergeAt(rlpFromBytes(branches.finish.toRange), key, value, true) + return self.mergeAt(rlpFromBytes(branches.finish), key, value, true) else: if key.len == 0: return origWithNewValue() @@ -654,12 +637,12 @@ proc mergeAt(self: var HexaryTrie, orig: Rlp, origHash: KeccakHash, r.append(elem) inc i - return r.finish.toRange + return r.finish -proc put*(self: var HexaryTrie; key, value: BytesRange) = +proc put*(self: var HexaryTrie; key, value: openArray[byte]) = let root = self.root.hash - var rootBytes = self.db.get(root.data).toRange + var rootBytes = self.db.get(root.data) doAssert rootBytes.len > 0 let newRootBytes = self.mergeAt(rlpFromBytes(rootBytes), root, @@ -669,23 +652,19 @@ proc put*(self: var HexaryTrie; key, value: BytesRange) = self.root = self.db.dbPut(newRootBytes) -proc put*(self: var SecureHexaryTrie; key, value: BytesRange) = - let keyHash = @(key.keccak.data) - put(HexaryTrie(self), keyHash.toRange, value) +proc put*(self: var SecureHexaryTrie; key, value: openArray[byte]) = + put(HexaryTrie(self), key.keccak.data, value) -proc get*(self: SecureHexaryTrie; key: BytesRange): BytesRange = - let keyHash = @(key.keccak.data) - return get(HexaryTrie(self), keyHash.toRange) +proc get*(self: SecureHexaryTrie; key: openArray[byte]): seq[byte] = + return get(HexaryTrie(self), key.keccak.data) -proc del*(self: var SecureHexaryTrie; key: BytesRange) = - let keyHash = @(key.keccak.data) - del(HexaryTrie(self), keyHash.toRange) +proc del*(self: var SecureHexaryTrie; key: openArray[byte]) = + del(HexaryTrie(self), key.keccak.data) proc rootHash*(self: SecureHexaryTrie): KeccakHash {.borrow.} proc rootHashHex*(self: SecureHexaryTrie): string {.borrow.} proc isPruning*(self: SecureHexaryTrie): bool {.borrow.} template contains*(self: HexaryTrie | SecureHexaryTrie; - key: BytesRange): bool = + key: openArray[byte]): bool = self.get(key).len > 0 - diff --git a/eth/trie/nibbles.nim b/eth/trie/nibbles.nim index a2f8c89..3e9928b 100644 --- a/eth/trie/nibbles.nim +++ b/eth/trie/nibbles.nim @@ -1,33 +1,26 @@ -import - trie_defs - type - NibblesRange* = object - bytes: ByteRange + NibblesSeq* = object + bytes: seq[byte] ibegin, iend: int -proc initNibbleRange*(bytes: ByteRange): NibblesRange = - result.bytes = bytes +proc initNibbleRange*(bytes: openArray[byte]): NibblesSeq = + result.bytes = @bytes result.ibegin = 0 result.iend = bytes.len * 2 -# can't be a const: https://github.com/status-im/nim-eth/issues/6 -# we can't initialise it here, but since it's already zeroed memory, we don't need to -var zeroNibblesRange* {.threadvar.}: NibblesRange - -proc `{}`(r: NibblesRange, pos: int): byte {.inline.} = +proc `{}`(r: NibblesSeq, pos: int): byte {.inline.} = ## This is a helper for a more raw access to the nibbles. ## It works with absolute positions. if pos > r.iend: raise newException(RangeError, "index out of range") return if (pos and 1) != 0: (r.bytes[pos div 2] and 0xf) else: (r.bytes[pos div 2] shr 4) -template `[]`*(r: NibblesRange, i: int): byte = r{r.ibegin + i} +template `[]`*(r: NibblesSeq, i: int): byte = r{r.ibegin + i} -proc len*(r: NibblesRange): int = +proc len*(r: NibblesSeq): int = r.iend - r.ibegin -proc `==`*(lhs, rhs: NibblesRange): bool = +proc `==`*(lhs, rhs: NibblesSeq): bool = if lhs.len == rhs.len: for i in 0 ..< lhs.len: if lhs[i] != rhs[i]: @@ -36,7 +29,7 @@ proc `==`*(lhs, rhs: NibblesRange): bool = else: return false -proc `$`*(r: NibblesRange): string = +proc `$`*(r: NibblesSeq): string = result = newStringOfCap(100) for i in r.ibegin ..< r.iend: let n = int r{i} @@ -44,7 +37,7 @@ proc `$`*(r: NibblesRange): string = else: char(ord('0') + n) result.add c -proc slice*(r: NibblesRange, ibegin: int, iend = -1): NibblesRange = +proc slice*(r: NibblesSeq, ibegin: int, iend = -1): NibblesSeq = result.bytes = r.bytes result.ibegin = r.ibegin + ibegin let e = if iend < 0: r.iend + iend + 1 @@ -69,11 +62,11 @@ template writeNibbles(r) {.dirty.} = result[writeHead] = nextNibble shl 4 oddnessFlag = not oddnessFlag -proc hexPrefixEncode*(r: NibblesRange, isLeaf = false): Bytes = +proc hexPrefixEncode*(r: NibblesSeq, isLeaf = false): seq[byte] = writeFirstByte(r.len) writeNibbles(r) -proc hexPrefixEncode*(r1, r2: NibblesRange, isLeaf = false): Bytes = +proc hexPrefixEncode*(r1, r2: NibblesSeq, isLeaf = false): seq[byte] = writeFirstByte(r1.len + r2.len) writeNibbles(r1) writeNibbles(r2) @@ -82,16 +75,16 @@ proc hexPrefixEncodeByte*(val: byte, isLeaf = false): byte = doAssert val < 16 result = (((byte(isLeaf) * 2) + 1) shl 4) or val -proc sharedPrefixLen*(lhs, rhs: NibblesRange): int = +proc sharedPrefixLen*(lhs, rhs: NibblesSeq): int = result = 0 while result < lhs.len and result < rhs.len: if lhs[result] != rhs[result]: break inc result -proc startsWith*(lhs, rhs: NibblesRange): bool = +proc startsWith*(lhs, rhs: NibblesSeq): bool = sharedPrefixLen(lhs, rhs) == rhs.len -proc hexPrefixDecode*(r: ByteRange): tuple[isLeaf: bool, nibbles: NibblesRange] = +proc hexPrefixDecode*(r: openArray[byte]): tuple[isLeaf: bool, nibbles: NibblesSeq] = result.nibbles = initNibbleRange(r) if r.len > 0: result.isLeaf = (r[0] and 0x20) != 0 @@ -115,7 +108,7 @@ template putNibbles(bytes, src: untyped) = template calcNeededBytes(len: int): int = (len shr 1) + (len and 1) -proc `&`*(a, b: NibblesRange): NibblesRange = +proc `&`*(a, b: NibblesSeq): NibblesSeq = let len = a.len + b.len bytesNeeded = calcNeededBytes(len) @@ -128,10 +121,10 @@ proc `&`*(a, b: NibblesRange): NibblesRange = bytes.putNibbles(a) bytes.putNibbles(b) - result = initNibbleRange(bytes.toRange) + result = initNibbleRange(bytes) result.iend = len -proc cloneAndReserveNibble*(a: NibblesRange): NibblesRange = +proc cloneAndReserveNibble*(a: NibblesSeq): NibblesSeq = let len = a.len + 1 bytesNeeded = calcNeededBytes(len) @@ -142,24 +135,15 @@ proc cloneAndReserveNibble*(a: NibblesRange): NibblesRange = pos = 0 bytes.putNibbles(a) - result = initNibbleRange(bytes.toRange) + result = initNibbleRange(bytes) result.iend = len -proc replaceLastNibble*(a: var NibblesRange, b: byte) = +proc replaceLastNibble*(a: var NibblesSeq, b: byte) = var odd = (a.len and 1) == 0 pos = (a.len shr 1) - odd.int - putNibble(MutRange[byte](a.bytes), b) + putNibble(a.bytes, b) -proc getBytes*(a: NibblesRange): ByteRange = +proc getBytes*(a: NibblesSeq): seq[byte] = a.bytes - -when false: - proc keyOf(r: ByteRange): NibblesRange = - let firstIdx = if r.len == 0: 0 - elif (r[0] and 0x10) != 0: 1 - else: 2 - - return initNibbleRange(s).slice(firstIdx) - diff --git a/eth/trie/sparse_binary.nim b/eth/trie/sparse_binary.nim index 9b62e27..9a4cb5f 100644 --- a/eth/trie/sparse_binary.nim +++ b/eth/trie/sparse_binary.nim @@ -1,9 +1,9 @@ import - stew/ranges/[typedranges, bitranges], eth/rlp/types, - trie_defs, trie_utils, db, sparse_proofs + ./trie_bitseq, + ./trie_defs, ./trie_utils, ./db, ./sparse_proofs export - types, trie_utils, bitranges, + trie_utils, trie_bitseq, sparse_proofs.verifyProof type @@ -11,11 +11,7 @@ type SparseBinaryTrie* = object db: DB - rootHash: ByteRange - -proc `==`(a: ByteRange, b: KeccakHash): bool = - if a.len != b.data.len: return false - equalMem(a.baseAddr, b.data[0].unsafeAddr, a.len) + rootHash: seq[byte] type # 256 * 2 div 8 @@ -24,83 +20,83 @@ type proc initDoubleHash(a, b: openArray[byte]): DoubleHash = doAssert(a.len == 32, $a.len) doAssert(b.len == 32, $b.len) - copyMem(result[ 0].addr, a[0].unsafeAddr, 32) - copyMem(result[32].addr, b[0].unsafeAddr, 32) + result[0..31] = a + result[32..^1] = b -proc initDoubleHash(x: ByteRange): DoubleHash = - initDoubleHash(x.toOpenArray, x.toOpenArray) +proc initDoubleHash(x: openArray[byte]): DoubleHash = + initDoubleHash(x, x) proc init*(x: typedesc[SparseBinaryTrie], db: DB): SparseBinaryTrie = result.db = db # Initialize an empty tree with one branch - var value = initDoubleHash(emptyNodeHashes[0]) - result.rootHash = keccakHash(value) - result.db.put(result.rootHash.toOpenArray, value) + var value = initDoubleHash(emptyNodeHashes[0].data) + result.rootHash = @(keccakHash(value).data) + result.db.put(result.rootHash, value) for i in 0..= 0 and + ibegin < x.len and + iend < x.len and + iend + 1 >= ibegin # the +1 here allows the result to be + # an empty range + + result.data = x.data + result.start = x.start + ibegin + result.mLen = iend - ibegin + 1 + +proc `[]`*(r: TrieBitSeq, s: HSlice): TrieBitSeq = + sliceNormalized(r, r @ s.a, r @ s.b) + +proc `==`*(a, b: TrieBitSeq): bool = + if a.len != b.len: return false + for i in 0 ..< a.len: + if a[i] != b[i]: return false + true + +proc `[]=`*(r: var TrieBitSeq, idx: Natural, val: bool) = + doAssert idx < r.len + let absIdx = r.start + idx + changeBitBE(r.data, absIdx, val) + +proc pushFront*(x: var TrieBitSeq, val: bool) = + doAssert x.start > 0 + dec x.start + x[0] = val + inc x.mLen + +template neededBytes(nBits: int): int = + (nBits shr 3) + ord((nBits and 0b111) != 0) + +static: + doAssert neededBytes(2) == 1 + doAssert neededBytes(8) == 1 + doAssert neededBytes(9) == 2 + +proc `&`*(a, b: TrieBitSeq): TrieBitSeq = + let totalLen = a.len + b.len + + var bytes = newSeq[byte](totalLen.neededBytes) + result = bits(bytes, 0, totalLen) + + for i in 0 ..< a.len: result.data.changeBitBE(i, a[i]) + for i in 0 ..< b.len: result.data.changeBitBE(i + a.len, b[i]) + +proc `$`*(r: TrieBitSeq): string = + result = newStringOfCap(r.len) + for bit in r: + result.add(if bit: '1' else: '0') + +proc fromBits*(T: type, r: TrieBitSeq, offset, num: Natural): T = + doAssert(num <= sizeof(T) * 8) + # XXX: Nim has a bug that a typedesc parameter cannot be used + # in a type coercion, so we must define an alias here: + type TT = T + for i in 0 ..< num: + result = (result shl 1) or TT(r[offset + i]) + +proc parse*(T: type TrieBitSeq, s: string): TrieBitSeq = + var bytes = newSeq[byte](s.len.neededBytes) + for i, c in s: + case c + of '0': discard + of '1': setBitBE(bytes, i) + else: doAssert false + result = bits(bytes, 0, s.len) + +proc toBytes*(r: TrieBitSeq): seq[byte] = + r.data[(r.start div 8)..<((r.mLen - r.start + 7) div 8)] diff --git a/eth/trie/trie_defs.nim b/eth/trie/trie_defs.nim index 72eb2a4..61ce3ad 100644 --- a/eth/trie/trie_defs.nim +++ b/eth/trie/trie_defs.nim @@ -1,12 +1,8 @@ import - eth/rlp, stew/ranges/typedranges, nimcrypto/hash - -export - typedranges, Bytes + eth/rlp, nimcrypto/hash type KeccakHash* = MDigest[256] - BytesContainer* = ByteRange | Bytes | string TrieError* = object of CatchableError # A common base type of all Trie errors. @@ -20,10 +16,6 @@ type # operate if its database has been tampered with. A swift crash # will be a more appropriate response. -# can't be a const: https://github.com/status-im/nim-eth/issues/6 -# we can't initialise it here, but since it's already zeroed memory, we don't need to -var zeroBytesRange* {.threadvar.}: ByteRange - const blankStringHash* = "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470".toDigest emptyRlp* = @[128.byte] @@ -34,10 +26,3 @@ proc read*(rlp: var Rlp, T: typedesc[MDigest]): T {.inline.} = proc append*(rlpWriter: var RlpWriter, a: MDigest) {.inline.} = rlpWriter.append(a.data) - -proc unnecessary_OpenArrayToRange*(key: openarray[byte]): ByteRange = - ## XXX: The name of this proc is intentionally long, because it - ## performs a memory allocation and data copying that may be eliminated - ## in the future. Avoid renaming it to something similar as `toRange`, so - ## it can remain searchable in the code. - toRange(@key) diff --git a/eth/trie/trie_utils.nim b/eth/trie/trie_utils.nim index f928b99..50989aa 100644 --- a/eth/trie/trie_utils.nim +++ b/eth/trie/trie_utils.nim @@ -1,42 +1,22 @@ import stew/byteutils, - stew/ranges/[typedranges, ptr_arith], nimcrypto/[hash, keccak], - trie_defs, binaries - -proc toTrieNodeKey*(hash: KeccakHash): TrieNodeKey = - result = newRange[byte](32) - copyMem(result.baseAddr, hash.data.baseAddr, 32) + nimcrypto/[hash, keccak], + trie_defs template checkValidHashZ*(x: untyped) = when x.type isnot KeccakHash: doAssert(x.len == 32 or x.len == 0) -template isZeroHash*(x: ByteRange): bool = +template isZeroHash*(x: openArray[byte]): bool = x.len == 0 -template toRange*(hash: KeccakHash): ByteRange = - toTrieNodeKey(hash) - -proc toRange*(str: string): ByteRange = - var s = newSeq[byte](str.len) - if str.len > 0: - copyMem(s[0].addr, str[0].unsafeAddr, str.len) - result = toRange(s) - proc hashFromHex*(bits: static[int], input: string): MDigest[bits] = MDigest(data: hexToByteArray[bits div 8](input)) template hashFromHex*(s: static[string]): untyped = hashFromHex(s.len * 4, s) -proc keccakHash*(input: openArray[byte]): ByteRange = - var s = newSeq[byte](32) - var ctx: keccak256 - ctx.init() - if input.len > 0: - ctx.update(input[0].unsafeAddr, uint(input.len)) - ctx.finish s - ctx.clear() - result = toRange(s) +proc keccakHash*(input: openArray[byte]): KeccakHash = + keccak256.digest(input) proc keccakHash*(dest: var openArray[byte], a, b: openArray[byte]) = var ctx: keccak256 @@ -48,16 +28,7 @@ proc keccakHash*(dest: var openArray[byte], a, b: openArray[byte]) = ctx.finish dest ctx.clear() -proc keccakHash*(a, b: openArray[byte]): ByteRange = - var s = newSeq[byte](32) +proc keccakHash*(a, b: openArray[byte]): KeccakHash = + var s: array[32, byte] keccakHash(s, a, b) - result = toRange(s) - -template keccakHash*(input: ByteRange): ByteRange = - keccakHash(input.toOpenArray) - -template keccakHash*(a, b: ByteRange): ByteRange = - keccakHash(a.toOpenArray, b.toOpenArray) - -template keccakHash*(dest: var ByteRange, a, b: ByteRange) = - keccakHash(dest.toOpenArray, a.toOpenArray, b.toOpenArray) + KeccakHash(data: s) diff --git a/tests/fuzzing/discovery/generate.nim b/tests/fuzzing/discovery/generate.nim index ea109e0..f0c1911 100644 --- a/tests/fuzzing/discovery/generate.nim +++ b/tests/fuzzing/discovery/generate.nim @@ -24,8 +24,8 @@ proc generate() = # valid data for a Ping packet block: - let payload = rlp.encode((4, fromAddr, toAddr, expiration())).toRange - let encodedData = @[1.byte] & payload.toSeq() + let payload = rlp.encode((4, fromAddr, toAddr, expiration())) + let encodedData = @[1.byte] & payload debug "Ping", data=byteutils.toHex(encodedData) encodedData.toFile(inputsDir & "ping") @@ -33,8 +33,8 @@ proc generate() = # valid data for a Pong packet block: let token = keccak256.digest(@[0]) - let payload = rlp.encode((toAddr, token , expiration())).toRange - let encodedData = @[2.byte] & payload.toSeq() + let payload = rlp.encode((toAddr, token , expiration())) + let encodedData = @[2.byte] & payload debug "Pong", data=byteutils.toHex(encodedData) encodedData.toFile(inputsDir & "pong") @@ -43,7 +43,7 @@ proc generate() = block: var data: array[64, byte] data[32 .. ^1] = peerKey.toPublicKey().tryGet().toNodeId().toByteArrayBE() - let payload = rlp.encode((data, expiration())).toRange + let payload = rlp.encode((data, expiration())) let encodedData = @[3.byte] & payload.toSeq() debug "FindNode", data=byteutils.toHex(encodedData) @@ -65,7 +65,7 @@ proc generate() = nodes.add((n1Addr.ip, n1Addr.udpPort, n1Addr.tcpPort, n1Key.toPublicKey().tryGet())) nodes.add((n2Addr.ip, n2Addr.udpPort, n2Addr.tcpPort, n2Key.toPublicKey().tryGet())) - let payload = rlp.encode((nodes, expiration())).toRange + let payload = rlp.encode((nodes, expiration())) let encodedData = @[4.byte] & payload.toSeq() debug "Neighbours", data=byteutils.toHex(encodedData) diff --git a/tests/fuzzing/readme.md b/tests/fuzzing/readme.md index 83cc3fa..98eba5f 100644 --- a/tests/fuzzing/readme.md +++ b/tests/fuzzing/readme.md @@ -19,7 +19,7 @@ These are the mandatory `test` block and the optional `init` block. Example usage: ```nim test: - var rlp = rlpFromBytes(@payload.toRange) + var rlp = rlpFromBytes(payload) discard rlp.inspect() ``` @@ -30,7 +30,7 @@ E.g.: ```nim test: try: - var rlp = rlpFromBytes(@payload.toRange) + var rlp = rlpFromBytes(payload) discard rlp.inspect() except RlpError as e: debug "Inspect failed", err = e.msg diff --git a/tests/fuzzing/rlp/rlp_inspect.nim b/tests/fuzzing/rlp/rlp_inspect.nim index ddb639e..ade7704 100644 --- a/tests/fuzzing/rlp/rlp_inspect.nim +++ b/tests/fuzzing/rlp/rlp_inspect.nim @@ -2,7 +2,7 @@ import chronicles, eth/rlp, ../fuzztest test: try: - var rlp = rlpFromBytes(@payload.toRange) + var rlp = rlpFromBytes(payload) discard rlp.inspect() except RlpError as e: debug "Inspect failed", err = e.msg diff --git a/tests/p2p/p2p_test_helper.nim b/tests/p2p/p2p_test_helper.nim index 54ffb59..d58242c 100644 --- a/tests/p2p/p2p_test_helper.nim +++ b/tests/p2p/p2p_test_helper.nim @@ -52,7 +52,7 @@ proc packData*(payload: openArray[byte], pk: PrivateKey): seq[byte] = template sourceDir*: string = currentSourcePath.rsplit(DirSep, 1)[0] proc recvMsgMock*(msg: openArray[byte]): tuple[msgId: int, msgData: Rlp] = - var rlp = rlpFromBytes(@msg.toRange) + var rlp = rlpFromBytes(msg) let msgId = rlp.read(int32) return (msgId.int, rlp) diff --git a/tests/p2p/test_enr.nim b/tests/p2p/test_enr.nim index 1c15acf..9edb39b 100644 --- a/tests/p2p/test_enr.nim +++ b/tests/p2p/test_enr.nim @@ -1,7 +1,7 @@ import net, unittest, options, nimcrypto/utils, - eth/p2p/enode, eth/p2p/discoveryv5/enr, eth/keys, eth/rlp + eth/p2p/enode, eth/p2p/discoveryv5/enr, eth/keys suite "ENR": test "Serialization": diff --git a/tests/p2p/test_waku_mail.nim b/tests/p2p/test_waku_mail.nim index 42b259b..88ec8bd 100644 --- a/tests/p2p/test_waku_mail.nim +++ b/tests/p2p/test_waku_mail.nim @@ -49,7 +49,7 @@ procSuite "Waku Mail Client": let decoded = decode(response.envelope.data, symKey = some(symKey)) require decoded.isSome() - var rlp = rlpFromBytes(decoded.get().payload.toRange) + var rlp = rlpFromBytes(decoded.get().payload) let output = rlp.read(MailRequest) check: output.lower == lower @@ -92,7 +92,7 @@ procSuite "Waku Mail Client": var envelopes: seq[Envelope] traceAsyncErrors peer.p2pMessage(envelopes) - var cursor: Bytes + var cursor: seq[byte] count = count - 1 if count == 0: cursor = @[] diff --git a/tests/rlp/test_api_usage.nim b/tests/rlp/test_api_usage.nim index 2ae6242..4db6f3b 100644 --- a/tests/rlp/test_api_usage.nim +++ b/tests/rlp/test_api_usage.nim @@ -1,13 +1,15 @@ +{.used.} + import - math, unittest, strutils, - eth/rlp, util/json_testing + math, unittest, strutils, stew/byteutils, + eth/rlp proc q(s: string): string = "\"" & s & "\"" proc i(s: string): string = s.replace(" ").replace("\n") proc inspectMatch(r: Rlp, s: string): bool = r.inspect.i == s.i test "empty bytes are not a proper RLP": - var rlp = rlpFromBytes Bytes(@[]).toRange + var rlp = rlpFromBytes seq[byte](@[]) check: not rlp.hasData @@ -52,7 +54,7 @@ test "encode/decode object": var writer = initRlpWriter() writer.append(input) let bytes = writer.finish() - var rlp = rlpFromBytes(bytes.toRange) + var rlp = rlpFromBytes(bytes) var output = rlp.read(MyObj) check: @@ -66,10 +68,10 @@ test "encode and decode lists": var bytes = writer.finish - rlp = rlpFromBytes bytes.toRange + rlp = rlpFromBytes bytes check: - bytes.hexRepr == "d183666f6fc8836261728362617ac31e2832" + bytes.toHex == "d183666f6fc8836261728362617ac31e2832" rlp.inspectMatch """ { "foo" @@ -89,7 +91,7 @@ test "encode and decode lists": "Lorem ipsum dolor sit amet", "Donec ligula tortor, egestas eu est vitae") - rlp = rlpFromBytes bytes.toRange + rlp = rlpFromBytes bytes check: rlp.listLen == 3 rlp.listElem(0).toInt(int) == 6000 @@ -97,7 +99,7 @@ test "encode and decode lists": rlp.listElem(2).toString == "Donec ligula tortor, egestas eu est vitae" # test creating RLPs from other RLPs - var list = rlpFromBytes encodeList(rlp.listELem(1), rlp.listELem(0)).toRange + var list = rlpFromBytes encodeList(rlp.listELem(1), rlp.listELem(0)) # test that iteration with enterList/skipElem works as expected doAssert list.enterList # We already know that we are working with a list @@ -117,11 +119,11 @@ test "toBytes": let tok = rlp.listElem(1).toBytes() check: tok.len == 32 - tok.hexRepr == "40ef02798f211da2e8173d37f255be908871ae65060dbb2f77fb29c0421447f4" + tok.toHex == "40ef02798f211da2e8173d37f255be908871ae65060dbb2f77fb29c0421447f4" test "nested lists": let listBytes = encode([[1, 2, 3], [5, 6, 7]]) - let listRlp = rlpFromBytes listBytes.toRange + let listRlp = rlpFromBytes listBytes let sublistRlp0 = listRlp.listElem(0) let sublistRlp1 = listRlp.listElem(1) check sublistRlp0.listElem(0).toInt(int) == 1 @@ -133,12 +135,12 @@ test "nested lists": test "encoding length": let listBytes = encode([1,2,3,4,5]) - let listRlp = rlpFromBytes listBytes.toRange + let listRlp = rlpFromBytes listBytes check listRlp.listLen == 5 let emptyListBytes = encode "" check emptyListBytes.len == 1 - let emptyListRlp = rlpFromBytes emptyListBytes.toRange + let emptyListRlp = rlpFromBytes emptyListBytes check emptyListRlp.blobLen == 0 test "basic decoding": @@ -159,21 +161,21 @@ test "encode byte arrays": var b2 = [byte(6), 8, 12, 123] var b3 = @[byte(122), 56, 65, 12] - let rlp = rlpFromBytes(encode((b1, b2, b3)).toRange) + let rlp = rlpFromBytes(encode((b1, b2, b3))) check: rlp.listLen == 3 - rlp.listElem(0).toBytes().toSeq() == @b1 - rlp.listElem(1).toBytes().toSeq() == @b2 - rlp.listElem(2).toBytes().toSeq() == @b3 + rlp.listElem(0).toBytes() == b1 + rlp.listElem(1).toBytes() == b2 + rlp.listElem(2).toBytes() == b3 # The first byte here is the length of the datum (132 - 128 => 4) - $(rlp.listElem(1).rawData) == "R[132, 6, 8, 12, 123]" + $(rlp.listElem(1).rawData) == "[132, 6, 8, 12, 123]" test "empty byte arrays": var - rlp = rlpFromBytes rlp.encode("").toRange + rlp = rlpFromBytes rlp.encode("") b = rlp.toBytes - check $b == "R[]" + check $b == "@[]" test "encode/decode floats": for f in [high(float64), low(float64), 0.1, 122.23, @@ -202,7 +204,7 @@ test "invalid enum": writer.append(2) writer.append(-1) let bytes = writer.finish() - var rlp = rlpFromBytes(bytes.toRange) + var rlp = rlpFromBytes(bytes) expect RlpTypeMismatch: discard rlp.read(MyEnum) rlp.skipElem() diff --git a/tests/rlp/test_object_serialization.nim b/tests/rlp/test_object_serialization.nim index d1fc261..67cd5bb 100644 --- a/tests/rlp/test_object_serialization.nim +++ b/tests/rlp/test_object_serialization.nim @@ -1,7 +1,7 @@ {.used.} import - unittest, times, eth/rlp, util/json_testing + unittest, times, eth/rlp, stew/byteutils type Transaction = object @@ -46,7 +46,7 @@ test "encoding and decoding an object": f: Foo(x: 5'u64, y: "hocus pocus", z: @[100, 200, 300])) var bytes = encode(originalBar) - var r = rlpFromBytes(bytes.toRange) + var r = rlpFromBytes(bytes) var restoredBar = r.read(Bar) check: @@ -57,7 +57,7 @@ test "encoding and decoding an object": var t2 = bytes.decode(Transaction) check: - bytes.hexRepr == "cd85416c69636583426f628203e8" # verifies that Alice comes first + bytes.toHex == "cd85416c69636583426f628203e8" # verifies that Alice comes first t2.time == default(Time) t2.sender == "Alice" t2.receiver == "Bob" @@ -66,7 +66,7 @@ test "encoding and decoding an object": test "custom field serialization": var origVal = CustomSerialized(customFoo: Foo(x: 10'u64, y: "y", z: @[]), ignored: 5) var bytes = encode(origVal) - var r = rlpFromBytes(bytes.toRange) + var r = rlpFromBytes(bytes) var restored = r.read(CustomSerialized) check: @@ -79,4 +79,3 @@ test "RLP fields count": Bar.rlpFieldsCount == 2 Foo.rlpFieldsCount == 3 Transaction.rlpFieldsCount == 3 - diff --git a/tests/rlp/util/json_testing.nim b/tests/rlp/util/json_testing.nim index 3a59f81..ca686aa 100644 --- a/tests/rlp/util/json_testing.nim +++ b/tests/rlp/util/json_testing.nim @@ -1,5 +1,5 @@ import - json, strutils, unittest, eth/rlp + json, stew/byteutils, unittest, eth/rlp proc append(output: var RlpWriter, js: JsonNode) = case js.kind @@ -14,11 +14,6 @@ proc append(output: var RlpWriter, js: JsonNode) = of JArray: output.append js.elems -proc hexRepr*(bytes: BytesRange|Bytes): string = - result = newStringOfCap(bytes.len * 2) - for byte in bytes: - result.add(toHex(int(byte), 2).toLowerAscii) - proc `==`(lhs: JsonNode, rhs: string): bool = lhs.kind == JString and lhs.str == rhs @@ -58,7 +53,7 @@ proc runTests*(filename: string) = var outRlp = initRlpWriter() outRlp.append input let - actual = outRlp.finish.hexRepr + actual = outRlp.finish.toHex expected = output.str check actual == expected diff --git a/tests/trie/all_tests.nim b/tests/trie/all_tests.nim index eeeb0e2..49e5b2b 100644 --- a/tests/trie/all_tests.nim +++ b/tests/trie/all_tests.nim @@ -6,7 +6,7 @@ import test_examples, test_hexary_trie, test_json_suite, - test_nibbles, test_sparse_binary_trie, test_storage_backends, - test_transaction_db + test_transaction_db, + test_trie_bitseq diff --git a/tests/trie/test_bin_trie.nim b/tests/trie/test_bin_trie.nim index e136a5b..8122638 100644 --- a/tests/trie/test_bin_trie.nim +++ b/tests/trie/test_bin_trie.nim @@ -2,8 +2,8 @@ import unittest, random, - eth/trie/[trie_defs, db, binary], - ./testutils + eth/trie/[db, binary], + ./testutils, stew/byteutils suite "binary trie": @@ -19,7 +19,7 @@ suite "binary trie": for i, c in kv_pairs: trie.set(c.key, c.value) let x = trie.get(c.key) - let y = toRange(c.value) + let y = c.value check y == x check result == zeroHash or trie.getRootHash() == result @@ -54,27 +54,27 @@ suite "binary trie": let will_raise_error = data[4] # First test case, delete subtrie of a kv node - trie.set(kv1[0], kv1[1]) - trie.set(kv2[0], kv2[1]) - check trie.get(kv1[0]) == toRange(kv1[1]) - check trie.get(kv2[0]) == toRange(kv2[1]) + trie.set(kv1[0].toBytes, kv1[1].toBytes) + trie.set(kv2[0].toBytes, kv2[1].toBytes) + check trie.get(kv1[0].toBytes) == kv1[1].toBytes + check trie.get(kv2[0].toBytes) == kv2[1].toBytes if will_delete: - trie.deleteSubtrie(key_to_be_deleted) - check trie.get(kv1[0]) == zeroBytesRange - check trie.get(kv2[0]) == zeroBytesRange + trie.deleteSubtrie(key_to_be_deleted.toBytes) + check trie.get(kv1[0].toBytes) == [] + check trie.get(kv2[0].toBytes) == [] check trie.getRootHash() == zeroHash else: if will_raise_error: try: - trie.deleteSubtrie(key_to_be_deleted) + trie.deleteSubtrie(key_to_be_deleted.toBytes) except NodeOverrideError: discard else: let root_hash_before_delete = trie.getRootHash() - trie.deleteSubtrie(key_to_be_deleted) - check trie.get(kv1[0]) == toRange(kv1[1]) - check trie.get(kv2[0]) == toRange(kv2[1]) + trie.deleteSubtrie(key_to_be_deleted.toBytes) + check trie.get(kv1[0].toBytes) == toBytes(kv1[1]) + check trie.get(kv2[0].toBytes) == toBytes(kv2[1]) check trie.getRootHash() == root_hash_before_delete const invalidKeyData = [ @@ -90,22 +90,22 @@ suite "binary trie": var db = newMemoryDB() var trie = initBinaryTrie(db) - trie.set("\x12\x34\x56\x78", "78") - trie.set("\x12\x34\x56\x79", "79") + trie.set("\x12\x34\x56\x78".toBytes, "78".toBytes) + trie.set("\x12\x34\x56\x79".toBytes, "79".toBytes) let invalidKey = data[0] let if_error = data[1] - check trie.get(invalidKey) == zeroBytesRange + check trie.get(invalidKey.toBytes) == [] if if_error: try: - trie.delete(invalidKey) + trie.delete(invalidKey.toBytes) except NodeOverrideError: discard else: let previous_root_hash = trie.getRootHash() - trie.delete(invalidKey) + trie.delete(invalidKey.toBytes) check previous_root_hash == trie.getRootHash() test "update value": @@ -114,13 +114,13 @@ suite "binary trie": var db = newMemoryDB() var trie = initBinaryTrie(db) for key in keys: - trie.set(key, "old") + trie.set(key.toBytes, "old".toBytes) var current_root = trie.getRootHash() for i in vals: - trie.set(keys[i], "old") + trie.set(keys[i].toBytes, "old".toBytes) check current_root == trie.getRootHash() - trie.set(keys[i], "new") + trie.set(keys[i].toBytes, "new".toBytes) check current_root != trie.getRootHash() - check trie.get(keys[i]) == toRange("new") + check trie.get(keys[i].toBytes) == toBytes("new") current_root = trie.getRootHash() diff --git a/tests/trie/test_binaries_utils.nim b/tests/trie/test_binaries_utils.nim index 98ef5d2..4d6c676 100644 --- a/tests/trie/test_binaries_utils.nim +++ b/tests/trie/test_binaries_utils.nim @@ -2,11 +2,11 @@ import unittest, strutils, - stew/ranges/bitranges, eth/rlp/types, nimcrypto/[keccak, hash], - eth/trie/[binaries, trie_utils], - ./testutils + nimcrypto/[keccak, hash], + eth/trie/[binaries, trie_bitseq], + ./testutils, stew/byteutils -proc parseBitVector(x: string): BitRange = +proc parseBitVector(x: string): TrieBitSeq = result = genBitVec(x.len) for i, c in x: result[i] = (c == '1') @@ -53,7 +53,7 @@ suite "binaries utils": test "node parsing": for c in parseNodeData: - let input = toRange(c[0]) + let input = toBytes(c[0]) let node = c[1] let kind = TrieNodeKind(node[0]) let raiseError = node[3] @@ -69,12 +69,12 @@ suite "binaries utils": case res.kind of KV_TYPE: check(res.keyPath == parseBitVector(node[1])) - check(res.child == toRange(node[2])) + check(res.child == toBytes(node[2])) of BRANCH_TYPE: - check(res.leftChild == toRange(node[2])) - check(res.rightChild == toRange(node[2])) + check(res.leftChild == toBytes(node[2])) + check(res.rightChild == toBytes(node[2])) of LEAF_TYPE: - check(res.value == toRange(node[2])) + check(res.value == toBytes(node[2])) const kvData = [ @@ -88,7 +88,7 @@ suite "binaries utils": test "kv node encoding": for c in kvData: let keyPath = parseBitVector(c[0]) - let node = toRange(c[1]) + let node = toBytes(c[1]) let output = toBytes(c[2]) let raiseError = c[3] @@ -110,8 +110,8 @@ suite "binaries utils": test "branch node encode": for c in branchData: - let left = toRange(c[0]) - let right = toRange(c[1]) + let left = toBytes(c[0]) + let right = toBytes(c[1]) let output = toBytes(c[2]) let raiseError = c[3] @@ -132,34 +132,34 @@ suite "binaries utils": let raiseError = c[2] if raiseError: expect(ValidationError): - check toBytes(c[1]) == encodeLeafNode(toRange(c[0])) + check toBytes(c[1]) == encodeLeafNode(toBytes(c[0])) else: - check toBytes(c[1]) == encodeLeafNode(toRange(c[0])) + check toBytes(c[1]) == encodeLeafNode(toBytes(c[0])) test "random kv encoding": let lengths = randList(int, randGen(1, 999), randGen(100, 100), unique = false) for len in lengths: var k = len var bitvec = genBitVec(len) - var nodeHash = keccak256.digest(cast[ptr byte](k.addr), uint(sizeof(int))).toRange - var kvnode = encodeKVNode(bitvec, nodeHash).toRange + var nodeHash = keccak256.digest(cast[ptr byte](k.addr), uint(sizeof(int))) + var kvnode = encodeKVNode(bitvec, @(nodeHash.data)) # first byte if KV_TYPE # in the middle are 1..n bits of binary-encoded-keypath # last 32 bytes are hash var keyPath = decodeToBinKeypath(kvnode[1..^33]) check kvnode[0].ord == KV_TYPE.ord check keyPath == bitvec - check kvnode[^32..^1] == nodeHash + check kvnode[^32..^1] == nodeHash.data test "optimized single bit keypath kvnode encoding": var k = 1 - var nodeHash = keccak256.digest(cast[ptr byte](k.addr), uint(sizeof(int))).toRange + var nodeHash = keccak256.digest(cast[ptr byte](k.addr), uint(sizeof(int))) var bitvec = genBitVec(1) bitvec[0] = false - var kvnode = encodeKVNode(bitvec, nodeHash).toRange + var kvnode = encodeKVNode(bitvec, @(nodeHash.data)) var kp = decodeToBinKeypath(kvnode[1..^33]) - var okv = encodeKVNode(false, nodeHash).toRange + var okv = encodeKVNode(false, @(nodeHash.data)) check okv == kvnode var okp = decodeToBinKeypath(kvnode[1..^33]) check okp == kp @@ -167,10 +167,10 @@ suite "binaries utils": check okp == bitvec bitvec[0] = true - kvnode = encodeKVNode(bitvec, nodeHash).toRange + kvnode = encodeKVNode(bitvec, @(nodeHash.data)) kp = decodeToBinKeypath(kvnode[1..^33]) - okv = encodeKVNode(true, nodeHash).toRange + okv = encodeKVNode(true, @(nodeHash.data)) check okv == kvnode okp = decodeToBinKeypath(kvnode[1..^33]) check okp == kp diff --git a/tests/trie/test_branches_utils.nim b/tests/trie/test_branches_utils.nim index 2a6f99e..46ada61 100644 --- a/tests/trie/test_branches_utils.nim +++ b/tests/trie/test_branches_utils.nim @@ -1,7 +1,7 @@ {.used.} import - sets, unittest, strutils, sets, + sets, unittest, strutils, stew/byteutils, eth/trie/[db, binary, branches] suite "branches utils": @@ -10,9 +10,9 @@ suite "branches utils": var db = newMemoryDB() var trie = initBinaryTrie(db) - trie.set("\x12\x34\x56\x78\x9a", "9a") - trie.set("\x12\x34\x56\x78\x9b", "9b") - trie.set("\x12\x34\x56\xff", "ff") + trie.set("\x12\x34\x56\x78\x9a".toBytes, "9a".toBytes) + trie.set("\x12\x34\x56\x78\x9b".toBytes, "9b".toBytes) + trie.set("\x12\x34\x56\xff".toBytes, "ff".toBytes) trie const branchExistData = [ @@ -28,7 +28,7 @@ suite "branches utils": var trie = testTrie() var db = trie.getDB() for c in branchExistData: - let keyPrefix = c[0].toRange + let keyPrefix = c[0].toBytes let if_exist = c[1] check checkIfBranchExist(db, trie.getRootHash(), keyPrefix) == if_exist @@ -45,7 +45,7 @@ suite "branches utils": var trie = testTrie() var db = trie.getDB() for c in branchData: - let key = c[0].toRange + let key = c[0].toBytes let keyValid = c[1] if keyValid: @@ -83,15 +83,15 @@ suite "branches utils": (repeat('0', 32), @[]) ] - proc toRanges(x: seq[string]): seq[BytesRange] = - result = newSeq[BytesRange](x.len) - for i, c in x: result[i] = toRange(c) + proc toRanges(x: seq[string]): seq[seq[byte]] = + result = newSeq[seq[byte]](x.len) + for i, c in x: result[i] = toBytes(c) test "get trie nodes": var trie = testTrie() var db = trie.getDB() for c in trieNodesData: - let root = c[0].toRange() + let root = c[0].toBytes() let nodes = toRanges(c[1]) check toHashSet(nodes) == toHashSet(getTrieNodes(db, root)) @@ -135,7 +135,7 @@ suite "branches utils": var trie = testTrie() var db = trie.getDB() for c in witnessData: - let key = c[0].toRange + let key = c[0].toBytes let nodes = toRanges(c[1]) if nodes.len != 0: diff --git a/tests/trie/test_examples.nim b/tests/trie/test_examples.nim index cc627a1..6ca2dc0 100644 --- a/tests/trie/test_examples.nim +++ b/tests/trie/test_examples.nim @@ -1,9 +1,9 @@ {.used.} import - unittest, + unittest, stew/byteutils, nimcrypto/[keccak, hash], - eth/trie/[trie_defs, db, binary, binaries, trie_utils, branches] + eth/trie/[db, binary, binaries, trie_utils, branches] suite "examples": @@ -11,84 +11,84 @@ suite "examples": var trie = initBinaryTrie(db) test "basic set/get": - trie.set("key1", "value1") - trie.set("key2", "value2") - check trie.get("key1") == "value1".toRange - check trie.get("key2") == "value2".toRange + trie.set("key1".toBytes(), "value1".toBytes()) + trie.set("key2".toBytes(), "value2".toBytes()) + check trie.get("key1".toBytes) == "value1".toBytes + check trie.get("key2".toBytes) == "value2".toBytes test "check branch exists": - check checkIfBranchExist(db, trie.getRootHash(), "key") == true - check checkIfBranchExist(db, trie.getRootHash(), "key1") == true - check checkIfBranchExist(db, trie.getRootHash(), "ken") == false - check checkIfBranchExist(db, trie.getRootHash(), "key123") == false + check checkIfBranchExist(db, trie.getRootHash(), "key".toBytes) == true + check checkIfBranchExist(db, trie.getRootHash(), "key1".toBytes) == true + check checkIfBranchExist(db, trie.getRootHash(), "ken".toBytes) == false + check checkIfBranchExist(db, trie.getRootHash(), "key123".toBytes) == false test "branches utils": - var branchA = getBranch(db, trie.getRootHash(), "key1") + var branchA = getBranch(db, trie.getRootHash(), "key1".toBytes) # ==> [A, B, C1, D1] check branchA.len == 4 - var branchB = getBranch(db, trie.getRootHash(), "key2") + var branchB = getBranch(db, trie.getRootHash(), "key2".toBytes) # ==> [A, B, C2, D2] check branchB.len == 4 - check isValidBranch(branchA, trie.getRootHash(), "key1", "value1") == true - check isValidBranch(branchA, trie.getRootHash(), "key5", "") == true + check isValidBranch(branchA, trie.getRootHash(), "key1".toBytes, "value1".toBytes) == true + check isValidBranch(branchA, trie.getRootHash(), "key5".toBytes, "".toBytes) == true expect InvalidNode: - check isValidBranch(branchB, trie.getRootHash(), "key1", "value1") + check isValidBranch(branchB, trie.getRootHash(), "key1".toBytes, "value1".toBytes) - var x = getBranch(db, trie.getRootHash(), "key") + var x = getBranch(db, trie.getRootHash(), "key".toBytes) # ==> [A] check x.len == 1 expect InvalidKeyError: - x = getBranch(db, trie.getRootHash(), "key123") # InvalidKeyError + x = getBranch(db, trie.getRootHash(), "key123".toBytes) # InvalidKeyError - x = getBranch(db, trie.getRootHash(), "key5") # there is still branch for non-exist key + x = getBranch(db, trie.getRootHash(), "key5".toBytes) # there is still branch for non-exist key # ==> [A] check x.len == 1 test "getWitness": - var branch = getWitness(db, trie.getRootHash(), "key1") + var branch = getWitness(db, trie.getRootHash(), "key1".toBytes) # equivalent to `getBranch(db, trie.getRootHash(), "key1")` # ==> [A, B, C1, D1] check branch.len == 4 - branch = getWitness(db, trie.getRootHash(), "key") + branch = getWitness(db, trie.getRootHash(), "key".toBytes) # this will include additional nodes of "key2" # ==> [A, B, C1, D1, C2, D2] check branch.len == 6 - branch = getWitness(db, trie.getRootHash(), "") + branch = getWitness(db, trie.getRootHash(), "".toBytes) # this will return the whole trie # ==> [A, B, C1, D1, C2, D2] check branch.len == 6 let beforeDeleteLen = db.totalRecordsInMemoryDB test "verify intermediate entries existence": - var branchs = getWitness(db, trie.getRootHash, zeroBytesRange) + var branchs = getWitness(db, trie.getRootHash, []) # set operation create new intermediate entries check branchs.len < beforeDeleteLen var node = branchs[1] - let nodeHash = keccak256.digest(node.baseAddr, uint(node.len)) - var nodes = getTrieNodes(db, nodeHash) + let nodeHash = keccak256.digest(node) + var nodes = getTrieNodes(db, @(nodeHash.data)) check nodes.len == branchs.len - 1 test "delete sub trie": # delete all subtrie with key prefixes "key" - trie.deleteSubtrie("key") - check trie.get("key1") == zeroBytesRange - check trie.get("key2") == zeroBytesRange + trie.deleteSubtrie("key".toBytes) + check trie.get("key1".toBytes) == [] + check trie.get("key2".toBytes) == [] test "prove the lie": # `delete` and `deleteSubtrie` not actually delete the nodes check db.totalRecordsInMemoryDB == beforeDeleteLen - var branchs = getWitness(db, trie.getRootHash, zeroBytesRange) + var branchs = getWitness(db, trie.getRootHash, []) check branchs.len == 0 test "dictionary syntax API": # dictionary syntax API - trie["moon"] = "sun" - check "moon" in trie - check trie["moon"] == "sun".toRange + trie["moon".toBytes] = "sun".toBytes + check "moon".toBytes in trie + check trie["moon".toBytes] == "sun".toBytes diff --git a/tests/trie/test_hexary_trie.nim b/tests/trie/test_hexary_trie.nim index 93c67dc..dbfaaa1 100644 --- a/tests/trie/test_hexary_trie.nim +++ b/tests/trie/test_hexary_trie.nim @@ -1,26 +1,17 @@ {.used.} import - unittest, sequtils, os, - stew/ranges/typedranges, eth/trie/[hexary, db, trie_defs], nimcrypto/utils, - ./testutils, algorithm, eth/rlp/types as rlpTypes, random + unittest, sequtils, os, stew/byteutils, + eth/trie/[hexary, db, trie_defs], nimcrypto/utils, + ./testutils, algorithm, random from strutils import split -template put(t: HexaryTrie|SecureHexaryTrie, key, val: string) = - t.put(key.toBytesRange, val.toBytesRange) - -template del(t: HexaryTrie|SecureHexaryTrie, key) = - t.del(key.toBytesRange) - -template get(t: HexaryTrie|SecureHexaryTrie, key): auto = - t.get(key.toBytesRange) - suite "hexary trie": setup: var db = newMemoryDB() - tr = initHexaryTrie(db) + tr {.used.} = initHexaryTrie(db) test "ref-counted keys crash": proc addKey(intKey: int) = @@ -28,12 +19,12 @@ suite "hexary trie": key[19] = byte(intKey) var data = newSeqWith(29, 1.byte) - var k = key.toRange + var k = key let v = tr.get(k) doAssert(v.len == 0) - tr.put(k, toRange(data)) + tr.put(k, data) addKey(166) addKey(193) @@ -78,12 +69,12 @@ suite "hexary trie": key = fromHex(parts[0]) val = fromHex(parts[1]) - SecureHexaryTrie(tr).put(key.toRange, val.toRange) + SecureHexaryTrie(tr).put(key, val) check tr.rootHashHex == "D7F8974FB5AC78D9AC099B9AD5018BEDC2CE0A72DAD1827A1709DA30580F0544" # lexicographic comparison - proc lexComp(a, b: BytesRange): bool = + proc lexComp(a, b: seq[byte]): bool = var x = 0 y = 0 @@ -98,7 +89,7 @@ suite "hexary trie": result = y != ylen - proc cmp(a, b: BytesRange): int = + proc cmp(a, b: seq[byte]): int = if a == b: return 0 if a.lexComp(b): return 1 return -1 @@ -109,17 +100,17 @@ suite "hexary trie": memdb = newMemoryDB() trie = initHexaryTrie(memdb) keys = [ - "key".toBytesRange, - "abc".toBytesRange, - "hola".toBytesRange, - "bubble".toBytesRange + "key".toBytes, + "abc".toBytes, + "hola".toBytes, + "bubble".toBytes ] vals = [ - "hello".toBytesRange, - "world".toBytesRange, - "block".toBytesRange, - "chain".toBytesRange + "hello".toBytes, + "world".toBytes, + "block".toBytes, + "chain".toBytes ] for i in 0 ..< keys.len: @@ -157,11 +148,11 @@ suite "hexary trie": var memdb = newMemoryDB() trie = initHexaryTrie(memdb) - keys = randList(BytesRange, randGen(5, 32), randGen(10)) - vals = randList(BytesRange, randGen(5, 7), randGen(10)) + keys = randList(seq[byte], randGen(5, 32), randGen(10)) + vals = randList(seq[byte], randGen(5, 7), randGen(10)) - keys2 = randList(BytesRange, randGen(5, 30), randGen(15)) - vals2 = randList(BytesRange, randGen(5, 7), randGen(15)) + keys2 = randList(seq[byte], randGen(5, 30), randGen(15)) + vals2 = randList(seq[byte], randGen(5, 7), randGen(15)) for i in 0 ..< keys.len: trie.put(keys[i], vals[i]) @@ -226,11 +217,11 @@ suite "hexary trie": var memdb = newMemoryDB() nonPruningTrie = initHexaryTrie(memdb, false) - keys = randList(BytesRange, randGen(5, 77), randGen(30)) - vals = randList(BytesRange, randGen(1, 57), randGen(30)) + keys = randList(seq[byte], randGen(5, 77), randGen(30)) + vals = randList(seq[byte], randGen(1, 57), randGen(30)) - moreKeys = randList(BytesRange, randGen(5, 33), randGen(45)) - moreVals = randList(BytesRange, randGen(1, 47), randGen(45)) + moreKeys = randList(seq[byte], randGen(5, 33), randGen(45)) + moreVals = randList(seq[byte], randGen(1, 47), randGen(45)) for i in 0 ..< keys.len: nonPruningTrie.put(keys[i], vals[i]) @@ -265,8 +256,8 @@ suite "hexary trie": test "elaborate non-pruning test": type History = object - keys: seq[BytesRange] - values: seq[BytesRange] + keys: seq[seq[byte]] + values: seq[seq[byte]] rootHash: KeccakHash const @@ -277,14 +268,14 @@ suite "hexary trie": var memdb = newMemoryDB() nonPruningTrie = initHexaryTrie(memdb, false) - keys = randList(BytesRange, randGen(3, 33), randGen(listLength)) - values = randList(BytesRange, randGen(5, 77), randGen(listLength)) + keys = randList(seq[byte], randGen(3, 33), randGen(listLength)) + values = randList(seq[byte], randGen(5, 77), randGen(listLength)) historyList = newSeq[History](listLength) ok = true for i, k in keys: - historyList[i].keys = newSeq[BytesRange](i + 1) - historyList[i].values = newSeq[BytesRange](i + 1) + historyList[i].keys = newSeq[seq[byte]](i + 1) + historyList[i].values = newSeq[seq[byte]](i + 1) for x in 0 ..< i + 1: historyList[i].keys[x] = keys[x] historyList[i].values[x] = values[x] @@ -296,7 +287,7 @@ suite "hexary trie": for h in historyList: var trie = initHexaryTrie(memdb, h.rootHash) - pKeys: seq[BytesRange] = @[] + pKeys: seq[seq[byte]] = @[] pValues = trie.getValues() for k in trie.keys: @@ -318,7 +309,7 @@ suite "hexary trie": echo "ITERATION: ", iteration break - proc isValidBranch(branch: seq[BytesRange], rootHash: KeccakHash, key, value: BytesRange): bool = + proc isValidBranch(branch: seq[seq[byte]], rootHash: KeccakHash, key, value: seq[byte]): bool = # branch must not be empty doAssert(branch.len != 0) @@ -326,17 +317,17 @@ suite "hexary trie": for node in branch: doAssert(node.len != 0) let nodeHash = hexary.keccak(node) - db.put(nodeHash.data, node.toOpenArray) + db.put(nodeHash.data, node) var trie = initHexaryTrie(db, rootHash) - result = trie.get(key) == toRange(value) + result = trie.get(key) == value test "get branch with pruning trie": var memdb = newMemoryDB() trie = initHexaryTrie(memdb) - keys = randList(BytesRange, randGen(5, 77), randGen(30)) - vals = randList(BytesRange, randGen(1, 57), randGen(30)) + keys = randList(seq[byte], randGen(5, 77), randGen(30)) + vals = randList(seq[byte], randGen(1, 57), randGen(30)) for i in 0 ..< keys.len: trie.put(keys[i], vals[i]) @@ -352,8 +343,8 @@ suite "hexary trie": var memdb = newMemoryDB() nonPruningTrie = initHexaryTrie(memdb, false) - keys = randList(BytesRange, randGen(5, 77), randGen(numKeyVal)) - vals = randList(BytesRange, randGen(1, 57), randGen(numKeyVal)) + keys = randList(seq[byte], randGen(5, 77), randGen(numKeyVal)) + vals = randList(seq[byte], randGen(1, 57), randGen(numKeyVal)) roots = newSeq[KeccakHash](numKeyVal) for i in 0 ..< keys.len: @@ -388,9 +379,9 @@ suite "hexary trie": pruningTrie = initHexaryTrie(memdb, isPruning = true) let - keys = randList(BytesRange, randGen(5, 77), randGen(numKeyVal)) - vals = randList(BytesRange, randGen(1, 57), randGen(numKeyVal)) - newVals = randList(BytesRange, randGen(1, 63), randGen(numKeyVal)) + keys = randList(seq[byte], randGen(5, 77), randGen(numKeyVal)) + vals = randList(seq[byte], randGen(1, 57), randGen(numKeyVal)) + newVals = randList(seq[byte], randGen(1, 63), randGen(numKeyVal)) var tx1 = memdb.beginTransaction() for i in 0 ..< numKeyVal: @@ -426,15 +417,15 @@ suite "hexary trie": pruningTrie = initHexaryTrie(memdb, isPruning = true) let - keys = randList(BytesRange, randGen(5, 77), randGen(numKeyVal)) - vals = randList(BytesRange, randGen(1, 57), randGen(numKeyVal)) + keys = randList(seq[byte], randGen(5, 77), randGen(numKeyVal)) + vals = randList(seq[byte], randGen(1, 57), randGen(numKeyVal)) for i in 0 ..< numKeyVal: pruningTrie.put(keys[i], vals[i]) let rootHash = pruningTrie.rootHash for k, v in pruningTrie.replicate: - repdb.put(k.toOpenArray, v.toOpenArray) + repdb.put(k, v) var trie = initHexaryTrie(repdb, rootHash, isPruning = true) var numPairs = 0 diff --git a/tests/trie/test_json_suite.nim b/tests/trie/test_json_suite.nim index 3d70c27..f4a9548 100644 --- a/tests/trie/test_json_suite.nim +++ b/tests/trie/test_json_suite.nim @@ -2,15 +2,14 @@ import os, json, tables, strutils, algorithm, - eth/rlp/types, - eth/trie/[trie_defs, db, hexary], - ./testutils + eth/trie/[db, hexary], + stew/byteutils type TestOp = object idx: int - key: BytesRange - value: BytesRange + key: seq[byte] + value: seq[byte] proc cmp(lhs, rhs: TestOp): int = cmp(lhs.idx, rhs.idx) proc `<=`(lhs, rhs: TestOp): bool = lhs.idx <= rhs.idx @@ -76,12 +75,12 @@ proc runTests*(filename: string) = case v.kind of JString: inputs.add(TestOp(idx: inputs.len, - key: k.str.toBytesRange, - value: v.str.toBytesRange)) + key: k.str.toBytes, + value: v.str.toBytes)) of JNull: inputs.add(TestOp(idx: inputs.len, - key: k.str.toBytesRange, - value: zeroBytesRange)) + key: k.str.toBytes, + value: @[])) else: invalidTest() else: invalidTest() @@ -91,12 +90,12 @@ proc runTests*(filename: string) = case v.kind of JString: inputs.add(TestOp(idx: inputs.len, - key: k.toBytesRange, - value: v.str.toBytesRange)) + key: k.toBytes, + value: v.str.toBytes)) of JNull: inputs.add(TestOp(idx: inputs.len, - key: k.toBytesRange, - value: zeroBytesRange)) + key: k.toBytes, + value: @[])) else: invalidTest() else: invalidTest() diff --git a/tests/trie/test_nibbles.nim b/tests/trie/test_nibbles.nim deleted file mode 100644 index 441a4f9..0000000 --- a/tests/trie/test_nibbles.nim +++ /dev/null @@ -1,11 +0,0 @@ -{.used.} - -import - unittest, - eth/trie/nibbles - -suite "nibbles": - test "zeroNibblesRange": - # https://github.com/status-im/nim-eth/issues/6 - check zeroNibblesRange.len == 0 - diff --git a/tests/trie/test_sparse_binary_trie.nim b/tests/trie/test_sparse_binary_trie.nim index f520a89..8ce508c 100644 --- a/tests/trie/test_sparse_binary_trie.nim +++ b/tests/trie/test_sparse_binary_trie.nim @@ -1,8 +1,8 @@ {.used.} import - unittest, random, - eth/trie/[trie_defs, db, sparse_binary, sparse_proofs], + unittest, random, stew/byteutils, + eth/trie/[db, sparse_binary, sparse_proofs], ./testutils suite "sparse binary trie": @@ -21,14 +21,14 @@ suite "sparse binary trie": test "basic get": for c in kv_pairs: let x = trie.get(c.key) - let y = toRange(c.value) + let y = c.value check x == y trie.del(c.key) for c in kv_pairs: check trie.exists(c.key) == false - check trie.getRootHash() == keccakHash(emptyNodeHashes[0].toOpenArray, emptyNodeHashes[0].toOpenArray).toRange + check trie.getRootHash() == keccakHash(emptyNodeHashes[0].data, emptyNodeHashes[0].data).data test "single update set": random.shuffle(kv_pairs) @@ -42,11 +42,11 @@ suite "sparse binary trie": test "single update get": for i in numbers: # If new value is the same as current value, skip the update - if toRange($i) == trie.get(kv_pairs[i].key): + if toBytes($i) == trie.get(kv_pairs[i].key): continue # Update - trie.set(kv_pairs[i].key, $i) - check trie.get(kv_pairs[i].key) == toRange($i) + trie.set(kv_pairs[i].key, toBytes($i)) + check trie.get(kv_pairs[i].key) == toBytes($i) check trie.getRootHash() != prior_to_update_root # Un-update @@ -56,7 +56,7 @@ suite "sparse binary trie": test "batch update with different update order": # First batch update for i in numbers: - trie.set(kv_pairs[i].key, $i) + trie.set(kv_pairs[i].key, toBytes($i)) let batch_updated_root = trie.getRootHash() @@ -70,14 +70,14 @@ suite "sparse binary trie": # Second batch update random.shuffle(numbers) for i in numbers: - trie.set(kv_pairs[i].key, $i) + trie.set(kv_pairs[i].key, toBytes($i)) check trie.getRootHash() == batch_updated_root test "dictionary API": trie[kv_pairs[0].key] = kv_pairs[0].value let x = trie[kv_pairs[0].key] - let y = toRange(kv_pairs[0].value) + let y = kv_pairs[0].value check x == y check kv_pairs[0].key in trie @@ -85,10 +85,10 @@ suite "sparse binary trie": db = newMemoryDB() trie = initSparseBinaryTrie(db) let - testKey = toRange(kv_pairs[0].key) - testValue = toRange(kv_pairs[0].value) - testKey2 = toRange(kv_pairs[1].key) - testValue2 = toRange(kv_pairs[1].value) + testKey = kv_pairs[0].key + testValue = kv_pairs[0].value + testKey2 = kv_pairs[1].key + testValue2 = kv_pairs[1].value trie.set(testKey, testValue) var root = trie.getRootHash() @@ -102,11 +102,11 @@ suite "sparse binary trie": value = trie.get(testKey, root) check value == testValue - proc makeBadProof(size: int, width = 32): seq[BytesRange] = - let badProofStr = randList(string, randGen(width, width), randGen(size, size)) - result = newSeq[BytesRange](size) + proc makeBadProof(size: int, width = 32): seq[seq[byte]] = + let badProofStr = randList(seq[byte], randGen(width, width), randGen(size, size)) + result = newSeq[seq[byte]](size) for i in 0 ..< result.len: - result[i] = toRange(badProofStr[i]) + result[i] = badProofStr[i] test "proofs": const @@ -115,9 +115,9 @@ suite "sparse binary trie": let testKey = kv_pairs[0].key badKey = kv_pairs[1].key - testValue = "testValue" - testValue2 = "testValue2" - badValue = "badValue" + testValue = "testValue".toBytes + testValue2 = "testValue2".toBytes + badValue = "badValue".toBytes badProof = makeBadProof(MaxBadProof) trie[testKey] = testValue @@ -131,7 +131,7 @@ suite "sparse binary trie": let testKey2 = kv_pairs[2].key testKey3 = kv_pairs[3].key - defaultValue = zeroBytesRange + defaultValue = default(seq[byte]) trie.set(testKey2, testValue) proof = trie.prove(testKey) @@ -169,7 +169,7 @@ suite "sparse binary trie": check compactProof(badProof2).len == 0 check compactProof(badProof3).len == 0 check decompactProof(badProof3).len == 0 - var zeroProof: seq[BytesRange] + var zeroProof: seq[seq[byte]] check decompactProof(zeroProof).len == 0 proof = trie.proveCompact(testKey2) @@ -202,23 +202,23 @@ suite "sparse binary trie": test "examples": let - key1 = "01234567890123456789" - key2 = "abcdefghijklmnopqrst" + key1 = "01234567890123456789".toBytes + key2 = "abcdefghijklmnopqrst".toBytes - trie.set(key1, "value1") - trie.set(key2, "value2") - check trie.get(key1) == "value1".toRange - check trie.get(key2) == "value2".toRange + trie.set(key1, "value1".toBytes) + trie.set(key2, "value2".toBytes) + check trie.get(key1) == "value1".toBytes + check trie.get(key2) == "value2".toBytes trie.del(key1) - check trie.get(key1) == zeroBytesRange + check trie.get(key1) == [] trie.del(key2) - check trie[key2] == zeroBytesRange + check trie[key2] == [] let - value1 = "hello world" - badValue = "bad value" + value1 = "hello world".toBytes + badValue = "bad value".toBytes trie[key1] = value1 var proof = trie.prove(key1) diff --git a/tests/trie/test_transaction_db.nim b/tests/trie/test_transaction_db.nim index 33591bf..0e8691a 100644 --- a/tests/trie/test_transaction_db.nim +++ b/tests/trie/test_transaction_db.nim @@ -2,8 +2,7 @@ import unittest, - eth/trie/[db, trie_defs], ./testutils, - eth/rlp/types as rlpTypes + eth/trie/[db], ./testutils suite "transaction db": setup: @@ -11,10 +10,10 @@ suite "transaction db": listLength = 30 var - keysA = randList(Bytes, randGen(3, 33), randGen(listLength)) - valuesA = randList(Bytes, randGen(5, 77), randGen(listLength)) - keysB = randList(Bytes, randGen(3, 33), randGen(listLength)) - valuesB = randList(Bytes, randGen(5, 77), randGen(listLength)) + keysA = randList(seq[byte], randGen(3, 33), randGen(listLength)) + valuesA = randList(seq[byte], randGen(5, 77), randGen(listLength)) + keysB = randList(seq[byte], randGen(3, 33), randGen(listLength)) + valuesB = randList(seq[byte], randGen(5, 77), randGen(listLength)) proc populateA(db: TrieDatabaseRef) = for i in 0 ..< listLength: diff --git a/tests/trie/test_trie_bitseq.nim b/tests/trie/test_trie_bitseq.nim new file mode 100644 index 0000000..adccb39 --- /dev/null +++ b/tests/trie/test_trie_bitseq.nim @@ -0,0 +1,84 @@ +{.used.} + +import + random, unittest, + eth/trie/trie_bitseq + +proc randomBytes(n: int): seq[byte] = + result = newSeq[byte](n) + for i in 0 ..< result.len: + result[i] = byte(rand(256)) + +suite "trie bitseq": + test "basic": + var a = @[byte 0b10101010, 0b11110000, 0b00001111, 0b01010101] + + var bSeq = @[byte 0b10101010, 0b00000000, 0b00000000, 0b11111111] + var b = bits(bSeq, 8) + + var cSeq = @[byte 0b11110000, 0b00001111, 0b00000000, 0b00000000] + var c = bits(cSeq, 16) + + var dSeq = @[byte 0b00001111, 0b00000000, 0b00000000, 0b00000000] + var d = bits(dSeq, 8) + + var eSeq = @[byte 0b01010101, 0b00000000, 0b00000000, 0b00000000] + var e = bits(eSeq, 8) + + var m = a.bits + var n = m[0..7] + check n == b + check n.len == 8 + check b.len == 8 + check c == m[8..23] + check $(d) == "00001111" + check $(e) == "01010101" + + var f = int.fromBits(e, 0, 4) + check f == 0b0101 + + let k = n & d + check(k.len == n.len + d.len) + check($k == $n & $d) + + var asciiSeq = @[byte('A'),byte('S'),byte('C'),byte('I'),byte('I')] + let asciiBits = bits(asciiSeq) + check $asciiBits == "0100000101010011010000110100100101001001" + + test "concat operator": + randomize(5000) + + for i in 0..<256: + var xSeq = randomBytes(rand(i)) + var ySeq = randomBytes(rand(i)) + let x = xSeq.bits + let y = ySeq.bits + var z = x & y + check z.len == x.len + y.len + check($z == $x & $y) + + test "get set bits": + randomize(1000) + + for i in 0..<256: + # produce random vector + var xSeq = randomBytes(i) + var ySeq = randomBytes(i) + var x = xSeq.bits + var y = ySeq.bits + for idx, bit in x: + y[idx] = bit + check x == y + + test "constructor with start": + var a = @[byte 0b10101010, 0b11110000, 0b00001111, 0b01010101] + var b = a.bits(1, 8) + check b.len == 8 + check b[0] == false + check $b == "01010101" + b[0] = true + check $b == "11010101" + check b[0] == true + b.pushFront(false) + check b[0] == false + check $b == "011010101" diff --git a/tests/trie/testutils.nim b/tests/trie/testutils.nim index 2b2d720..059fc0a 100644 --- a/tests/trie/testutils.nim +++ b/tests/trie/testutils.nim @@ -1,6 +1,6 @@ import - random, sets, eth/trie/trie_utils as ethUtils, - eth/rlp/types as rlpTypes, stew/ranges/bitranges, + random, sets, + eth/trie/trie_bitseq, nimcrypto/[utils, sysrand] type @@ -8,8 +8,8 @@ type minVal, maxVal: T KVPair* = ref object - key*: string - value*: string + key*: seq[byte] + value*: seq[byte] proc randGen*[T](minVal, maxVal: T): RandGen[T] = doAssert(minVal <= maxVal) @@ -28,11 +28,11 @@ proc randString*(len: int): string = for i in 0..