From 1646d78d833b386ff0d554e42ede83595871249f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sat, 18 Apr 2020 10:17:59 +0200 Subject: [PATCH] cleanups (#226) --- eth.nimble | 24 +++------------------ eth/common/eth_types.nim | 2 +- eth/p2p/blockchain_sync.nim | 2 +- eth/p2p/ecies.nim | 6 ++++-- eth/p2p/kademlia.nim | 10 ++++----- eth/p2p/peer_pool.nim | 2 +- eth/p2p/private/p2p_types.nim | 2 +- eth/p2p/rlpx.nim | 1 - eth/p2p/rlpx_protocols/les/flow_control.nim | 2 +- eth/p2p/rlpx_protocols/les_protocol.nim | 2 +- eth/rlp/priv/defs.nim | 3 --- eth/trie/backends/caching_backend.nim | 2 +- eth/trie/backends/lmdb_backend.nim | 4 ++-- eth/trie/binary.nim | 2 +- eth/trie/db.nim | 2 +- eth/trie/sparse_binary.nim | 2 +- eth/trie/sparse_proofs.nim | 2 +- eth/trie/trie_utils.nim | 16 ++------------ tests/rlp/all_tests.nim | 4 ++++ tests/rlp/test_api_usage.nim | 4 +--- tests/rlp/test_json_suite.nim | 2 ++ tests/rlp/test_object_serialization.nim | 4 ++-- tests/trie/all_tests.nim | 12 +++++++++++ tests/trie/test_bin_trie.nim | 10 ++++----- tests/trie/test_binaries_utils.nim | 2 ++ tests/trie/test_branches_utils.nim | 10 +++++---- tests/trie/test_caching_db_backend.nim | 2 ++ tests/trie/test_examples.nim | 2 ++ tests/trie/test_hexary_trie.nim | 2 ++ tests/trie/test_json_suite.nim | 9 ++++---- tests/trie/test_nibbles.nim | 2 ++ tests/trie/test_sparse_binary_trie.nim | 2 ++ tests/trie/test_storage_backends.nim | 2 ++ tests/trie/test_transaction_db.nim | 4 +++- tests/trie/testutils.nim | 2 +- 35 files changed, 80 insertions(+), 81 deletions(-) create mode 100644 tests/rlp/all_tests.nim create mode 100644 tests/trie/all_tests.nim diff --git a/eth.nimble b/eth.nimble index b342b4e..c5625be 100644 --- a/eth.nimble +++ b/eth.nimble @@ -17,7 +17,7 @@ requires "nim >= 1.2.0", proc runTest(path: string) = echo "\nRunning: ", path - exec "nim c -r -d:release -d:chronicles_log_level=ERROR --verbosity:0 --hints:off --warnings:off " & path + exec "nim c -r -d:release -d:chronicles_log_level=ERROR --verbosity:0 --hints:off " & path rmFile path proc runKeyfileTests() = @@ -66,31 +66,13 @@ task test_p2p, "run p2p tests": runP2pTests() proc runRlpTests() = - for filename in [ - "test_api_usage", - "test_json_suite", - "test_object_serialization", - ]: - runTest("tests/rlp/" & filename) + runTest("tests/rlp/all_tests") task test_rlp, "run rlp tests": runRlpTests() proc runTrieTests() = - for filename in [ - "test_binaries_utils", - "test_bin_trie", - "test_branches_utils", - "test_caching_db_backend", - "test_examples", - "test_hexary_trie", - "test_json_suite", - "test_nibbles", - "test_sparse_binary_trie", - "test_storage_backends", - "test_transaction_db", - ]: - runTest("tests/trie/" & filename) + runTest("tests/trie/all_tests") task test_trie, "run trie tests": runTrieTests() diff --git a/eth/common/eth_types.nim b/eth/common/eth_types.nim index a3fbc77..9360a27 100644 --- a/eth/common/eth_types.nim +++ b/eth/common/eth_types.nim @@ -261,7 +261,7 @@ proc append*(rlpWriter: var RlpWriter, value: StUint) = rlpWriter.append bytes.toOpenArray(bytes.len - nonZeroBytes, bytes.len - 1) else: - rlpWriter.append(value.toInt) + rlpWriter.append(value.truncate(int)) proc read*(rlp: var Rlp, T: typedesc[Stint]): T {.inline.} = # The Ethereum Yellow Paper defines the RLP serialization only diff --git a/eth/p2p/blockchain_sync.nim b/eth/p2p/blockchain_sync.nim index da0eceb..a1fb370 100644 --- a/eth/p2p/blockchain_sync.nim +++ b/eth/p2p/blockchain_sync.nim @@ -168,7 +168,7 @@ proc newSyncContext(chain: AbstractChainDB, peerPool: PeerPool): SyncContext = new result result.chain = chain result.peerPool = peerPool - result.trustedPeers = initSet[Peer]() + result.trustedPeers = initHashSet[Peer]() result.finalizedBlock = chain.getBestBlockHeader().blockNumber proc handleLostPeer(ctx: SyncContext) = diff --git a/eth/p2p/ecies.nim b/eth/p2p/ecies.nim index 723cebc..1544d59 100644 --- a/eth/p2p/ecies.nim +++ b/eth/p2p/ecies.nim @@ -122,7 +122,8 @@ proc eciesEncrypt*(input: openarray[byte], output: var openarray[byte], copyMem(addr encKey[0], addr material[0], aes128.sizeKey) - var macKey = sha256.digest(material, ostart = KeyLength div 2) + var macKey = + sha256.digest(material.toOpenArray(KeyLength div 2, material.high)) burnMem(material) var header = cast[ptr EciesHeader](addr output[0]) @@ -189,7 +190,8 @@ proc eciesDecrypt*(input: openarray[byte], burnMem(secret) copyMem(addr encKey[0], addr material[0], aes128.sizeKey) - var macKey = sha256.digest(material, ostart = KeyLength div 2) + var macKey = + sha256.digest(material.toOpenArray(KeyLength div 2, material.high)) burnMem(material) let macsize = eciesMacLength(len(input) - eciesOverheadLength()) diff --git a/eth/p2p/kademlia.nim b/eth/p2p/kademlia.nim index 7b29428..764a420 100644 --- a/eth/p2p/kademlia.nim +++ b/eth/p2p/kademlia.nim @@ -363,8 +363,8 @@ proc lookup*(k: KademliaProtocol, nodeId: NodeId): Future[seq[Node]] {.async.} = ## It approaches the target by querying nodes that are closer to it on each iteration. The ## given target does not need to be an actual node identifier. - var nodesAsked = initSet[Node]() - var nodesSeen = initSet[Node]() + var nodesAsked = initHashSet[Node]() + var nodesSeen = initHashSet[Node]() proc findNode(nodeId: NodeId, remote: Node): Future[seq[Node]] {.async.} = k.wire.sendFindNode(remote, nodeId) @@ -388,7 +388,7 @@ proc lookup*(k: KademliaProtocol, nodeId: NodeId): Future[seq[Node]] {.async.} = result = candidates proc excludeIfAsked(nodes: seq[Node]): seq[Node] = - result = toSeq(items(nodes.toSet() - nodesAsked)) + result = toSeq(items(nodes.toHashSet() - nodesAsked)) sortByDistance(result, nodeId, FIND_CONCURRENCY) var closest = k.routing.neighbours(nodeId) @@ -396,7 +396,7 @@ proc lookup*(k: KademliaProtocol, nodeId: NodeId): Future[seq[Node]] {.async.} = var nodesToAsk = excludeIfAsked(closest) while nodesToAsk.len != 0: trace "Node lookup; querying ", nodesToAsk - nodesAsked.incl(nodesToAsk.toSet()) + nodesAsked.incl(nodesToAsk.toHashSet()) let results = await all(nodesToAsk.mapIt(findNode(nodeId, it))) for candidates in results: closest.add(candidates) @@ -489,7 +489,7 @@ proc randomNodes*(k: KademliaProtocol, count: int): seq[Node] = count = sz result = newSeqOfCap[Node](count) - var seen = initSet[Node]() + var seen = initHashSet[Node]() # This is a rather inneficient way of randomizing nodes from all buckets, but even if we # iterate over all nodes in the routing table, the time it takes would still be diff --git a/eth/p2p/peer_pool.nim b/eth/p2p/peer_pool.nim index d1b7c5e..10d19bb 100644 --- a/eth/p2p/peer_pool.nim +++ b/eth/p2p/peer_pool.nim @@ -21,7 +21,7 @@ proc newPeerPool*(network: EthereumNode, result.networkId = networkId result.discovery = discovery result.connectedNodes = initTable[Node, Peer]() - result.connectingNodes = initSet[Node]() + result.connectingNodes = initHashSet[Node]() result.observers = initTable[int, PeerObserver]() result.listenPort = listenPort diff --git a/eth/p2p/private/p2p_types.nim b/eth/p2p/private/p2p_types.nim index a160938..86614cf 100644 --- a/eth/p2p/private/p2p_types.nim +++ b/eth/p2p/private/p2p_types.nim @@ -1,7 +1,7 @@ import deques, tables, eth/[rlp, keys], chronos, eth/common/eth_types, - ../enode, ../kademlia, ../discovery, ../options, ../rlpxcrypt + ../enode, ../kademlia, ../discovery, ../rlpxcrypt const useSnappy* = defined(useSnappy) diff --git a/eth/p2p/rlpx.nim b/eth/p2p/rlpx.nim index 03370b1..194c3a2 100644 --- a/eth/p2p/rlpx.nim +++ b/eth/p2p/rlpx.nim @@ -437,7 +437,6 @@ proc recvMsg*(peer: Peer): Future[tuple[msgId: int, msgData: Rlp]] {.async.} = "Cannot read RLPx message id") proc checkedRlpRead(peer: Peer, r: var Rlp, MsgType: type): auto {.inline.} = - let tmp = r when defined(release): return r.read(MsgType) else: diff --git a/eth/p2p/rlpx_protocols/les/flow_control.nim b/eth/p2p/rlpx_protocols/les/flow_control.nim index ec0c3a3..fa59b62 100644 --- a/eth/p2p/rlpx_protocols/les/flow_control.nim +++ b/eth/p2p/rlpx_protocols/les/flow_control.nim @@ -419,7 +419,7 @@ when defined(testing): setup: var lesNetwork = new LesNetwork - lesNetwork.peers = initSet[LesPeer]() + lesNetwork.peers = initHashSet[LesPeer]() lesNetwork.initFlowControl(dummyLes.protocolInfo, reqCostTarget = 300, maxReqCount = 5, diff --git a/eth/p2p/rlpx_protocols/les_protocol.nim b/eth/p2p/rlpx_protocols/les_protocol.nim index 4f9f6f5..5dbc777 100644 --- a/eth/p2p/rlpx_protocols/les_protocol.nim +++ b/eth/p2p/rlpx_protocols/les_protocol.nim @@ -77,7 +77,7 @@ const keyAnnounceSignature = "sign" proc initProtocolState(network: LesNetwork, node: EthereumNode) {.gcsafe.} = - network.peers = initSet[LesPeer]() + network.peers = initHashSet[LesPeer]() proc addPeer(network: LesNetwork, peer: LesPeer) = network.enlistInFlowControl peer diff --git a/eth/rlp/priv/defs.nim b/eth/rlp/priv/defs.nim index 772b112..13fda9a 100644 --- a/eth/rlp/priv/defs.nim +++ b/eth/rlp/priv/defs.nim @@ -1,6 +1,3 @@ -import - ../types - const MAX_LENGTH_BYTES* = 8 diff --git a/eth/trie/backends/caching_backend.nim b/eth/trie/backends/caching_backend.nim index 20a7e1b..b18cee5 100644 --- a/eth/trie/backends/caching_backend.nim +++ b/eth/trie/backends/caching_backend.nim @@ -12,7 +12,7 @@ proc newCachingDB*(backing: TrieDatabaseRef): CachingDB = result.new() result.backing = backing result.changed = initTable[seq[byte], seq[byte]]() - result.deleted = initSet[seq[byte]]() + result.deleted = initHashSet[seq[byte]]() proc get*(db: CachingDB, key: openarray[byte]): seq[byte] = let key = @key diff --git a/eth/trie/backends/lmdb_backend.nim b/eth/trie/backends/lmdb_backend.nim index 1b83d80..bc1ef34 100644 --- a/eth/trie/backends/lmdb_backend.nim +++ b/eth/trie/backends/lmdb_backend.nim @@ -30,7 +30,7 @@ type MDB_Dbi = distinct cuint MDB_val = object - mv_size: csize + mv_size: csize_t mv_data: pointer # this is only a subset of LMDB API needed in nimbus @@ -77,7 +77,7 @@ proc txCommit*(db: ChainDB, manualCommit = true): bool = mdb_dbi_close(db.env, db.dbi) proc toMdbVal(val: openArray[byte]): MDB_Val = - result.mv_size = val.len + result.mv_size = csize_t(val.len) result.mv_data = unsafeAddr val[0] proc get*(db: ChainDB, key: openarray[byte]): seq[byte] = diff --git a/eth/trie/binary.nim b/eth/trie/binary.nim index 44518a2..817f4db 100644 --- a/eth/trie/binary.nim +++ b/eth/trie/binary.nim @@ -1,5 +1,5 @@ import - stew/ranges/[ptr_arith, typedranges, bitranges], eth/rlp/types, + stew/ranges/[typedranges, bitranges], eth/rlp/types, trie_defs, db, binaries, trie_utils export diff --git a/eth/trie/db.nim b/eth/trie/db.nim index a9ed8fa..8f9d849 100644 --- a/eth/trie/db.nim +++ b/eth/trie/db.nim @@ -108,7 +108,7 @@ proc put*(db: MemoryLayer, key, val: openarray[byte]) = proc newMemoryLayer: MemoryLayer = result.new result.records = initTable[Bytes, MemDBRec]() - result.deleted = initSet[Bytes]() + result.deleted = initHashSet[Bytes]() proc commit(memDb: MemoryLayer, db: TrieDatabaseRef, applyDeletes: bool = true) = if applyDeletes: diff --git a/eth/trie/sparse_binary.nim b/eth/trie/sparse_binary.nim index 4a4df7f..9b62e27 100644 --- a/eth/trie/sparse_binary.nim +++ b/eth/trie/sparse_binary.nim @@ -1,5 +1,5 @@ import - stew/ranges/[ptr_arith, typedranges, bitranges], eth/rlp/types, + stew/ranges/[typedranges, bitranges], eth/rlp/types, trie_defs, trie_utils, db, sparse_proofs export diff --git a/eth/trie/sparse_proofs.nim b/eth/trie/sparse_proofs.nim index e284615..cfc6e8f 100644 --- a/eth/trie/sparse_proofs.nim +++ b/eth/trie/sparse_proofs.nim @@ -1,6 +1,6 @@ import stew/ranges/[typedranges, bitranges], - trie_defs, db, trie_utils + trie_defs, trie_utils const treeHeight* = 160 diff --git a/eth/trie/trie_utils.nim b/eth/trie/trie_utils.nim index 38bc2c2..f928b99 100644 --- a/eth/trie/trie_utils.nim +++ b/eth/trie/trie_utils.nim @@ -1,10 +1,8 @@ import - strutils, parseutils, + stew/byteutils, stew/ranges/[typedranges, ptr_arith], nimcrypto/[hash, keccak], trie_defs, binaries -#proc baseAddr*(x: Bytes): ptr byte = x[0].unsafeAddr - proc toTrieNodeKey*(hash: KeccakHash): TrieNodeKey = result = newRange[byte](32) copyMem(result.baseAddr, hash.data.baseAddr, 32) @@ -26,17 +24,7 @@ proc toRange*(str: string): ByteRange = result = toRange(s) proc hashFromHex*(bits: static[int], input: string): MDigest[bits] = - if input.len != bits div 4: - raise newException(ValueError, - "The input string has incorrect size") - - for i in 0 ..< bits div 8: - var nextByte: int - if parseHex(input, nextByte, i*2, 2) == 2: - result.data[i] = uint8(nextByte) - else: - raise newException(ValueError, -"The input string contains invalid characters") + MDigest(data: hexToByteArray[bits div 8](input)) template hashFromHex*(s: static[string]): untyped = hashFromHex(s.len * 4, s) diff --git a/tests/rlp/all_tests.nim b/tests/rlp/all_tests.nim new file mode 100644 index 0000000..5c6697d --- /dev/null +++ b/tests/rlp/all_tests.nim @@ -0,0 +1,4 @@ +import + ./test_api_usage, + ./test_json_suite, + ./test_object_serialization diff --git a/tests/rlp/test_api_usage.nim b/tests/rlp/test_api_usage.nim index 95136b8..2ae6242 100644 --- a/tests/rlp/test_api_usage.nim +++ b/tests/rlp/test_api_usage.nim @@ -31,9 +31,7 @@ test "you cannot finish a list without appending enough elements": writer.append "bar" expect Defect: - let result = writer.finish - -proc withNewLines(x: string): string = x & "\n" + discard writer.finish test "encode/decode object": type diff --git a/tests/rlp/test_json_suite.nim b/tests/rlp/test_json_suite.nim index c5d498d..7a83a00 100644 --- a/tests/rlp/test_json_suite.nim +++ b/tests/rlp/test_json_suite.nim @@ -1,3 +1,5 @@ +{.used.} + import os, strutils, strformat, util/json_testing diff --git a/tests/rlp/test_object_serialization.nim b/tests/rlp/test_object_serialization.nim index 6f58196..d1fc261 100644 --- a/tests/rlp/test_object_serialization.nim +++ b/tests/rlp/test_object_serialization.nim @@ -1,3 +1,5 @@ +{.used.} + import unittest, times, eth/rlp, util/json_testing @@ -17,8 +19,6 @@ type b: string f: Foo - CompressedFoo = object - CustomSerialized = object customFoo {.rlpCustomSerialization.}: Foo ignored {.rlpIgnore.}: int diff --git a/tests/trie/all_tests.nim b/tests/trie/all_tests.nim new file mode 100644 index 0000000..eeeb0e2 --- /dev/null +++ b/tests/trie/all_tests.nim @@ -0,0 +1,12 @@ +import + test_bin_trie, + test_binaries_utils, + test_branches_utils, + test_caching_db_backend, + test_examples, + test_hexary_trie, + test_json_suite, + test_nibbles, + test_sparse_binary_trie, + test_storage_backends, + test_transaction_db diff --git a/tests/trie/test_bin_trie.nim b/tests/trie/test_bin_trie.nim index 1fc3c1e..e136a5b 100644 --- a/tests/trie/test_bin_trie.nim +++ b/tests/trie/test_bin_trie.nim @@ -1,3 +1,5 @@ +{.used.} + import unittest, random, eth/trie/[trie_defs, db, binary], @@ -66,10 +68,8 @@ suite "binary trie": if will_raise_error: try: trie.deleteSubtrie(key_to_be_deleted) - except NodeOverrideError as E: + except NodeOverrideError: discard - except: - check(false) else: let root_hash_before_delete = trie.getRootHash() trie.deleteSubtrie(key_to_be_deleted) @@ -101,10 +101,8 @@ suite "binary trie": if if_error: try: trie.delete(invalidKey) - except NodeOverrideError as E: + except NodeOverrideError: discard - except: - check(false) else: let previous_root_hash = trie.getRootHash() trie.delete(invalidKey) diff --git a/tests/trie/test_binaries_utils.nim b/tests/trie/test_binaries_utils.nim index 1daf488..98ef5d2 100644 --- a/tests/trie/test_binaries_utils.nim +++ b/tests/trie/test_binaries_utils.nim @@ -1,3 +1,5 @@ +{.used.} + import unittest, strutils, stew/ranges/bitranges, eth/rlp/types, nimcrypto/[keccak, hash], diff --git a/tests/trie/test_branches_utils.nim b/tests/trie/test_branches_utils.nim index 96064cf..2a6f99e 100644 --- a/tests/trie/test_branches_utils.nim +++ b/tests/trie/test_branches_utils.nim @@ -1,5 +1,7 @@ +{.used.} + import - random, sets, unittest, strutils, sets, + sets, unittest, strutils, sets, eth/trie/[db, binary, branches] suite "branches utils": @@ -91,7 +93,7 @@ suite "branches utils": for c in trieNodesData: let root = c[0].toRange() let nodes = toRanges(c[1]) - check toSet(nodes) == toSet(getTrieNodes(db, root)) + check toHashSet(nodes) == toHashSet(getTrieNodes(db, root)) const witnessData = [ ("\x12\x34\x56\x78\x9b", @@ -137,6 +139,6 @@ suite "branches utils": let nodes = toRanges(c[1]) if nodes.len != 0: - let x = toSet(nodes) - let y = toSet(getWitness(db, trie.getRootHash(), key)) + let x = toHashSet(nodes) + let y = toHashSet(getWitness(db, trie.getRootHash(), key)) check x == y diff --git a/tests/trie/test_caching_db_backend.nim b/tests/trie/test_caching_db_backend.nim index eaf996c..0577044 100644 --- a/tests/trie/test_caching_db_backend.nim +++ b/tests/trie/test_caching_db_backend.nim @@ -1,3 +1,5 @@ +{.used.} + import unittest, eth/trie/db, diff --git a/tests/trie/test_examples.nim b/tests/trie/test_examples.nim index 44c6f85..cc627a1 100644 --- a/tests/trie/test_examples.nim +++ b/tests/trie/test_examples.nim @@ -1,3 +1,5 @@ +{.used.} + import unittest, nimcrypto/[keccak, hash], diff --git a/tests/trie/test_hexary_trie.nim b/tests/trie/test_hexary_trie.nim index 3ebe583..93c67dc 100644 --- a/tests/trie/test_hexary_trie.nim +++ b/tests/trie/test_hexary_trie.nim @@ -1,3 +1,5 @@ +{.used.} + import unittest, sequtils, os, stew/ranges/typedranges, eth/trie/[hexary, db, trie_defs], nimcrypto/utils, diff --git a/tests/trie/test_json_suite.nim b/tests/trie/test_json_suite.nim index 51f77d8..3d70c27 100644 --- a/tests/trie/test_json_suite.nim +++ b/tests/trie/test_json_suite.nim @@ -1,12 +1,11 @@ +{.used.} + import - os, json, tables, sequtils, strutils, algorithm, - eth/rlp/types, nimcrypto/utils, + os, json, tables, strutils, algorithm, + eth/rlp/types, eth/trie/[trie_defs, db, hexary], ./testutils -proc `==`(lhs: JsonNode, rhs: string): bool = - lhs.kind == JString and lhs.str == rhs - type TestOp = object idx: int diff --git a/tests/trie/test_nibbles.nim b/tests/trie/test_nibbles.nim index cf81751..441a4f9 100644 --- a/tests/trie/test_nibbles.nim +++ b/tests/trie/test_nibbles.nim @@ -1,3 +1,5 @@ +{.used.} + import unittest, eth/trie/nibbles diff --git a/tests/trie/test_sparse_binary_trie.nim b/tests/trie/test_sparse_binary_trie.nim index 9682fd9..f520a89 100644 --- a/tests/trie/test_sparse_binary_trie.nim +++ b/tests/trie/test_sparse_binary_trie.nim @@ -1,3 +1,5 @@ +{.used.} + import unittest, random, eth/trie/[trie_defs, db, sparse_binary, sparse_proofs], diff --git a/tests/trie/test_storage_backends.nim b/tests/trie/test_storage_backends.nim index 4f62026..28820cc 100644 --- a/tests/trie/test_storage_backends.nim +++ b/tests/trie/test_storage_backends.nim @@ -1,3 +1,5 @@ +{.used.} + import unittest, macros, os, eth/trie/backends/[rocksdb_backend, sqlite_backend, lmdb_backend] diff --git a/tests/trie/test_transaction_db.nim b/tests/trie/test_transaction_db.nim index f55926f..33591bf 100644 --- a/tests/trie/test_transaction_db.nim +++ b/tests/trie/test_transaction_db.nim @@ -1,5 +1,7 @@ +{.used.} + import - unittest, strutils, sequtils, os, + unittest, eth/trie/[db, trie_defs], ./testutils, eth/rlp/types as rlpTypes diff --git a/tests/trie/testutils.nim b/tests/trie/testutils.nim index 49279fd..2b2d720 100644 --- a/tests/trie/testutils.nim +++ b/tests/trie/testutils.nim @@ -56,7 +56,7 @@ proc randList*(T: typedesc, strGen, listGen: RandGen, unique: bool = true): seq[ let listLen = listGen.getVal() result = newSeqOfCap[T](listLen) if unique: - var set = initSet[T]() + var set = initHashSet[T]() for len in 0..