From 7a0a48e4a58851b9e522ff9d6915ce602227a08a Mon Sep 17 00:00:00 2001 From: markspanbroek Date: Thu, 9 Mar 2023 12:23:45 +0100 Subject: [PATCH] Fix warnings (drops Nim 1.2) (#348) * [build] disable XCannotRaiseY hint There are too many {.raises:[Defect].} in the libraries that we use, drowning out all other warnings and hints * [build] disable BareExcept warning Not yet enabled in a released version of Nim, so libraries that we depend on have not fixed this yet, drowning out our own hints and warnings * [build] disable DotLikeOps warning dot-like ops were an experiment that is not going land in Nim * [build] compile log statements in tests When running tests, all log statements are compiled. They are filtered out at runtime during a test run. * [build] do not build executable when running unit test It's already built in the integration test * [build] Fix warnings - remove unused code - remove unused imports - stop using deprecated stuff * [build] Put compiler flags behind nim version checks * [CI] remove Nim 1.2 compatibility --- .github/workflows/ci.yml | 17 -- codex.nim | 8 +- codex/blockexchange/engine/engine.nim | 12 +- codex/codex.nim | 1 - codex/conf.nim | 8 +- codex/manifest/coders.nim | 1 - codex/node.nim | 22 +-- codex/proving.nim | 1 - codex/rest/api.nim | 6 +- codex/sales/salesagent.nim | 1 - codex/sales/states/filling.nim | 1 - codex/storageproofs/por/por.nim | 53 ------ codex/storageproofs/timing/proofs.nim | 1 - codex/stores/cachestore.nim | 1 - codex/stores/keyutils.nim | 1 - codex/stores/repostore.nim | 2 - codex/utils/asyncheapqueue.nim | 2 +- codex/utils/keyutils.nim | 5 - config.nims | 9 + nim.cfg | 3 - .../blockexchange/discovery/testdiscovery.nim | 6 +- .../discovery/testdiscoveryengine.nim | 1 - .../blockexchange/engine/testblockexc.nim | 11 +- .../codex/blockexchange/engine/testengine.nim | 2 - tests/codex/blockexchange/testnetwork.nim | 1 - .../codex/blockexchange/testpendingblocks.nim | 8 +- tests/codex/helpers.nim | 2 - tests/codex/helpers/mockdiscovery.nim | 2 - tests/codex/helpers/mockrepostore.nim | 1 - tests/codex/storageproofs/testnetwork.nim | 14 +- tests/codex/storageproofs/teststpstore.nim | 2 - tests/codex/stores/testcachestore.nim | 1 - tests/codex/stores/testmaintenance.nim | 1 - tests/codex/stores/testrepostore.nim | 7 +- tests/codex/testerasure.nim | 1 - tests/codex/testmanifest.nim | 1 - tests/codex/testnode.nim | 1 - tests/codex/testpurchasing.nim | 3 +- tests/codex/utils/testtimer.nim | 2 - tests/contracts/testClock.nim | 4 +- tests/contracts/testMarket.nim | 5 +- tests/contracts/testProofs.nim | 2 - tests/integration/testIntegration.nim | 156 +++++++++++++++++ tests/integration/testblockexpiration.nim | 10 +- tests/logging.nim | 6 + tests/nim.cfg | 4 +- tests/testCodex.nim | 4 +- tests/testCodex.nim.cfg | 1 - tests/testContracts.nim | 1 + tests/testContracts.nim.cfg | 1 - tests/testIntegration.nim | 160 +----------------- 51 files changed, 232 insertions(+), 344 deletions(-) delete mode 100644 nim.cfg create mode 100644 tests/integration/testIntegration.nim create mode 100644 tests/logging.nim delete mode 100644 tests/testCodex.nim.cfg delete mode 100644 tests/testContracts.nim.cfg diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70f2dc1f..305077e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,20 +90,3 @@ jobs: flags: unittests name: codecov-umbrella verbose: true - - nim_1_2: - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Setup Nimbus Build System - uses: ./.github/actions/nimbus-build-system - with: - os: linux - nim_version: version-1-2 - - - name: Unit tests - run: make -j${ncpu} test diff --git a/codex.nim b/codex.nim index dedc6344..b5d71eaa 100644 --- a/codex.nim +++ b/codex.nim @@ -59,7 +59,7 @@ when isMainModule: trace "Data dir initialized", dir = $config.dataDir - if not(checkAndCreateDataDir((config.dataDir / "repo").string)): + if not(checkAndCreateDataDir((config.dataDir / "repo"))): # We are unable to access/create data folder or data folder's # permissions are insecure. quit QuitFailure @@ -72,10 +72,10 @@ when isMainModule: let keyPath = - if isAbsolute(string config.netPrivKeyFile): - string config.netPrivKeyFile + if isAbsolute(config.netPrivKeyFile): + config.netPrivKeyFile else: - string config.dataDir / string config.netPrivKeyFile + config.dataDir / config.netPrivKeyFile privateKey = setupKey(keyPath).expect("Should setup private key!") server = CodexServer.new(config, privateKey) diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index 91a3e8d4..ab0ae6a2 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -40,12 +40,12 @@ const DefaultMaxPeersPerRequest* = 10 DefaultTaskQueueSize = 100 DefaultConcurrentTasks = 10 - DefaultMaxRetries = 3 - DefaultConcurrentDiscRequests = 10 - DefaultConcurrentAdvertRequests = 10 - DefaultDiscoveryTimeout = 1.minutes - DefaultMaxQueriedBlocksCache = 1000 - DefaultMinPeersPerBlock = 3 + # DefaultMaxRetries = 3 + # DefaultConcurrentDiscRequests = 10 + # DefaultConcurrentAdvertRequests = 10 + # DefaultDiscoveryTimeout = 1.minutes + # DefaultMaxQueriedBlocksCache = 1000 + # DefaultMinPeersPerBlock = 3 type TaskHandler* = proc(task: BlockExcPeerCtx): Future[void] {.gcsafe.} diff --git a/codex/codex.nim b/codex/codex.nim index 8e761961..e43dadb7 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -9,7 +9,6 @@ import std/sequtils import std/os -import std/sugar import std/tables import pkg/chronicles diff --git a/codex/conf.nim b/codex/conf.nim index 431f7821..1daf8070 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -245,11 +245,11 @@ proc defaultDataDir*(): string = getHomeDir() / dataDir -func parseCmdArg*(T: type MultiAddress, input: TaintedString): T +func parseCmdArg*(T: type MultiAddress, input: string): T {.raises: [ValueError, LPError, Defect].} = MultiAddress.init($input).tryGet() -proc parseCmdArg*(T: type SignedPeerRecord, uri: TaintedString): T = +proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T = var res: SignedPeerRecord try: if not res.fromURI(uri): @@ -260,11 +260,11 @@ proc parseCmdArg*(T: type SignedPeerRecord, uri: TaintedString): T = quit QuitFailure res -func parseCmdArg*(T: type EthAddress, address: TaintedString): T = +func parseCmdArg*(T: type EthAddress, address: string): T = EthAddress.init($address).get() # no idea why confutils needs this: -proc completeCmdArg*(T: type EthAddress; val: TaintedString): seq[string] = +proc completeCmdArg*(T: type EthAddress; val: string): seq[string] = discard # silly chronicles, colors is a compile-time property diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index fbf2eba6..36f7d743 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -136,7 +136,6 @@ func decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = if pbNode.getRepeatedField(2, linksBuf).isOk: for pbLinkBuf in linksBuf: var - blocksBuf: seq[seq[byte]] blockBuf: seq[byte] pbLink = initProtoBuffer(pbLinkBuf) diff --git a/codex/node.nim b/codex/node.nim index b7d4af43..56e86dde 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -131,18 +131,18 @@ proc retrieve*( without res =? (await node.erasure.decode(manifest)), error: trace "Unable to erasure decode manifest", cid, exc = error.msg except CatchableError as exc: - trace "Exception decoding manifest", cid + trace "Exception decoding manifest", cid, exc = exc.msg # asyncSpawn erasureJob() - else: - # Prefetch the entire dataset into the local store - proc prefetchBlocks() {.async, raises: [Defect].} = - try: - discard await node.fetchBatched(manifest) - except CatchableError as exc: - trace "Exception prefetching blocks", exc = exc.msg - # - # asyncSpawn prefetchBlocks() - temporarily commented out + # else: + # # Prefetch the entire dataset into the local store + # proc prefetchBlocks() {.async, raises: [Defect].} = + # try: + # discard await node.fetchBatched(manifest) + # except CatchableError as exc: + # trace "Exception prefetching blocks", exc = exc.msg + # # + # # asyncSpawn prefetchBlocks() - temporarily commented out # # Retrieve all blocks of the dataset sequentially from the local store or network trace "Creating store stream for manifest", cid @@ -158,7 +158,7 @@ proc retrieve*( try: await stream.pushData(blk.data) except CatchableError as exc: - trace "Unable to send block", cid + trace "Unable to send block", cid, exc = exc.msg discard finally: await stream.pushEof() diff --git a/codex/proving.nim b/codex/proving.nim index bf14b234..53bc467e 100644 --- a/codex/proving.nim +++ b/codex/proving.nim @@ -35,7 +35,6 @@ proc waitUntilPeriod(proving: Proving, period: Period) {.async.} = await proving.clock.waitUntil(periodicity.periodStart(period).truncate(int64)) proc removeEndedContracts(proving: Proving) {.async.} = - let now = proving.clock.now().u256 var ended: HashSet[SlotId] for id in proving.slots: let state = await proving.proofs.slotState(id) diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 1c2c43a0..eab7c7f9 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -76,9 +76,9 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = try: await node.connect(peerId.get(), addresses) return RestApiResponse.response("Successfully connected to peer") - except DialFailedError as e: + except DialFailedError: return RestApiResponse.error(Http400, "Unable to dial peer") - except CatchableError as e: + except CatchableError: return RestApiResponse.error(Http400, "Unknown error dialling peer") router.api( @@ -187,7 +187,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = trace "Uploaded file", cid return RestApiResponse.response($cid) - except CancelledError as exc: + except CancelledError: return RestApiResponse.error(Http500) except AsyncStreamError: return RestApiResponse.error(Http500) diff --git a/codex/sales/salesagent.nim b/codex/sales/salesagent.nim index f75ff84f..3bad1e93 100644 --- a/codex/sales/salesagent.nim +++ b/codex/sales/salesagent.nim @@ -1,5 +1,4 @@ import pkg/chronos -import pkg/upraises import pkg/stint import ../contracts/requests import ../utils/asyncspawn diff --git a/codex/sales/states/filling.nim b/codex/sales/states/filling.nim index b9b1d060..9743c769 100644 --- a/codex/sales/states/filling.nim +++ b/codex/sales/states/filling.nim @@ -3,7 +3,6 @@ import ../statemachine import ../salesagent import ./errorhandling import ./filled -import ./errored import ./cancelled import ./failed diff --git a/codex/storageproofs/por/por.nim b/codex/storageproofs/por/por.nim index f2f465a3..04e0b0ea 100644 --- a/codex/storageproofs/por/por.nim +++ b/codex/storageproofs/por/por.nim @@ -134,13 +134,6 @@ type tau*: Tau authenticators*: seq[blst_p1] -proc fromBytesBE(a: array[32, byte]): blst_scalar = - ## Convert data to blst native form - ## - - blst_scalar_from_bendian(result, a) - doAssert(blst_scalar_fr_check(result).bool) - proc fromBytesBE(a: openArray[byte]): blst_scalar = ## Convert data to blst native form ## @@ -269,26 +262,6 @@ proc hashNameI(name: array[Namelen, byte], i: int64): blst_p1 = bigEndian64(addr(namei[sizeof(name)]), unsafeAddr(i)) return hashToG1(namei) -proc generateAuthenticatorNaive( - stream: SeekableStream, - ssk: SecretKey, - i: int64, - s: int64, - t: TauZero): Future[blst_p1] {.async.} = - ## Naive implementation of authenticator as in the S&W paper. - ## With the paper's multiplicative notation: - ## \sigmai=\(H(file||i)\cdot\prod{j=0}^{s-1}{uj^{m[i][j]}})^{\alpha} - ## - - var sum: blst_p1 - for j in 0.. 0: - waiters.delete(0, i - 1) + waiters.delete(0..(i-1)) proc heapCmp[T](x, y: T, max: bool = false): bool {.inline.} = if max: diff --git a/codex/utils/keyutils.nim b/codex/utils/keyutils.nim index 5179a0bc..6e14f0b0 100644 --- a/codex/utils/keyutils.nim +++ b/codex/utils/keyutils.nim @@ -13,15 +13,10 @@ push: {.upraises: [].} import pkg/chronicles import pkg/questionable/results import pkg/libp2p -import pkg/datastore import ./fileutils import ../errors import ../rng -import ../namespaces - -const - SafePermissions = {UserRead, UserWrite} type CodexKeyError = object of CodexError diff --git a/config.nims b/config.nims index 01d95d77..7e80515b 100644 --- a/config.nims +++ b/config.nims @@ -70,6 +70,15 @@ else: # for heap-usage-by-instance-type metrics and object base-type strings --define:nimTypeNames +when (NimMajor, NimMinor) >= (1, 4): + --warning:"ObservableStores:off" + --warning:"LockLevel:off" + --hint:"XCannotRaiseY:off" +when (NimMajor, NimMinor) >= (1, 6): + --warning:"DotLikeOps:off" +when (NimMajor, NimMinor, NimPatch) >= (1, 6, 11): + --warning:"BareExcept:off" + switch("define", "withoutPCRE") # the default open files limit is too low on macOS (512), breaking the diff --git a/nim.cfg b/nim.cfg deleted file mode 100644 index ec563332..00000000 --- a/nim.cfg +++ /dev/null @@ -1,3 +0,0 @@ --d:"chronicles_log_level=INFO" ---warning:LockLevel:off ---warning:ObservableStores:off diff --git a/tests/codex/blockexchange/discovery/testdiscovery.nim b/tests/codex/blockexchange/discovery/testdiscovery.nim index 93668f19..0eb87e1f 100644 --- a/tests/codex/blockexchange/discovery/testdiscovery.nim +++ b/tests/codex/blockexchange/discovery/testdiscovery.nim @@ -231,15 +231,15 @@ suite "E2E - Multiple Nodes Discovery": MockDiscovery(blockexc[1].engine.discovery.discovery) .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = - advertised.add(cid, switch[1].peerInfo.signedPeerRecord) + advertised[cid] = switch[1].peerInfo.signedPeerRecord MockDiscovery(blockexc[2].engine.discovery.discovery) .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = - advertised.add(cid, switch[2].peerInfo.signedPeerRecord) + advertised[cid] = switch[2].peerInfo.signedPeerRecord MockDiscovery(blockexc[3].engine.discovery.discovery) .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = - advertised.add(cid, switch[3].peerInfo.signedPeerRecord) + advertised[cid] = switch[3].peerInfo.signedPeerRecord await blockexc[1].engine.blocksHandler(switch[0].peerInfo.peerId, blocks[0..5]) await blockexc[2].engine.blocksHandler(switch[0].peerInfo.peerId, blocks[4..10]) diff --git a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim index a8f59fec..f194408d 100644 --- a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim @@ -28,7 +28,6 @@ suite "Test Discovery Engine": peerStore: PeerCtxStore blockDiscovery: MockDiscovery pendingBlocks: PendingBlocksManager - localStore: CacheStore network: BlockExcNetwork setup: diff --git a/tests/codex/blockexchange/engine/testblockexc.nim b/tests/codex/blockexchange/engine/testblockexc.nim index 9965979b..0ccfa438 100644 --- a/tests/codex/blockexchange/engine/testblockexc.nim +++ b/tests/codex/blockexchange/engine/testblockexc.nim @@ -169,13 +169,10 @@ suite "NetworkStore engine - 2 nodes": nodeCmps2.networkStore.putBlock(it) )) - let - blocks = await allFinished( - blocks2[4..7].mapIt( - nodeCmps1.networkStore.getBlock(it.cid) - )) - - # await sleepAsync(10.seconds) + discard await allFinished( + blocks2[4..7].mapIt( + nodeCmps1.networkStore.getBlock(it.cid) + )) let channel = !peerCtx1.paymentChannel diff --git a/tests/codex/blockexchange/engine/testengine.nim b/tests/codex/blockexchange/engine/testengine.nim index 212dc8c3..62be8ad3 100644 --- a/tests/codex/blockexchange/engine/testengine.nim +++ b/tests/codex/blockexchange/engine/testengine.nim @@ -140,7 +140,6 @@ suite "NetworkStore engine handlers": discovery: DiscoveryEngine peerCtx: BlockExcPeerCtx localStore: BlockStore - done: Future[void] blocks: seq[bt.Block] setup: @@ -366,7 +365,6 @@ suite "Task Handler": network: BlockExcNetwork engine: BlockExcEngine discovery: DiscoveryEngine - peerCtx: BlockExcPeerCtx localStore: BlockStore peersCtx: seq[BlockExcPeerCtx] diff --git a/tests/codex/blockexchange/testnetwork.nim b/tests/codex/blockexchange/testnetwork.nim index 9f3c399e..385c86d9 100644 --- a/tests/codex/blockexchange/testnetwork.nim +++ b/tests/codex/blockexchange/testnetwork.nim @@ -264,7 +264,6 @@ suite "Network - Test Limits": var switch1, switch2: Switch network1, network2: BlockExcNetwork - blocks: seq[bt.Block] done: Future[void] setup: diff --git a/tests/codex/blockexchange/testpendingblocks.nim b/tests/codex/blockexchange/testpendingblocks.nim index d713cdee..a267f9f7 100644 --- a/tests/codex/blockexchange/testpendingblocks.nim +++ b/tests/codex/blockexchange/testpendingblocks.nim @@ -9,14 +9,13 @@ import pkg/stew/byteutils import pkg/codex/blocktype as bt import pkg/codex/blockexchange -import ../examples - suite "Pending Blocks": test "Should add want handle": let pendingBlocks = PendingBlocksManager.new() blk = bt.Block.new("Hello".toBytes).tryGet - handle = pendingBlocks.getWantHandle(blk.cid) + + discard pendingBlocks.getWantHandle(blk.cid) check pendingBlocks.pending(blk.cid) @@ -59,7 +58,8 @@ suite "Pending Blocks": let pendingBlocks = PendingBlocksManager.new() blks = (0..9).mapIt( bt.Block.new(("Hello " & $it).toBytes).tryGet ) - handles = blks.mapIt( pendingBlocks.getWantHandle( it.cid ) ) + + discard blks.mapIt( pendingBlocks.getWantHandle( it.cid ) ) check: blks.mapIt( $it.cid ).sorted(cmp[string]) == diff --git a/tests/codex/helpers.nim b/tests/codex/helpers.nim index 6d431870..306ecf8e 100644 --- a/tests/codex/helpers.nim +++ b/tests/codex/helpers.nim @@ -1,5 +1,3 @@ -import std/options - import pkg/chronos import pkg/libp2p import pkg/libp2p/varint diff --git a/tests/codex/helpers/mockdiscovery.nim b/tests/codex/helpers/mockdiscovery.nim index 86e517a6..978c688b 100644 --- a/tests/codex/helpers/mockdiscovery.nim +++ b/tests/codex/helpers/mockdiscovery.nim @@ -10,8 +10,6 @@ import pkg/chronos import pkg/libp2p import pkg/questionable -import pkg/questionable/results -import pkg/stew/shims/net import pkg/codex/discovery import pkg/contractabi/address as ca diff --git a/tests/codex/helpers/mockrepostore.nim b/tests/codex/helpers/mockrepostore.nim index aa7f5d70..dd86593e 100644 --- a/tests/codex/helpers/mockrepostore.nim +++ b/tests/codex/helpers/mockrepostore.nim @@ -12,7 +12,6 @@ import pkg/chronos import pkg/libp2p import pkg/questionable import pkg/questionable/results -import pkg/codex/blocktype as bt import codex/stores/repostore diff --git a/tests/codex/storageproofs/testnetwork.nim b/tests/codex/storageproofs/testnetwork.nim index 3d31d7c0..7d7cd582 100644 --- a/tests/codex/storageproofs/testnetwork.nim +++ b/tests/codex/storageproofs/testnetwork.nim @@ -1,4 +1,3 @@ -import std/os import std/sequtils import pkg/asynctest @@ -22,17 +21,11 @@ import ../helpers const BlockSize = 31 * 64 - SectorSize = 31 - SectorsPerBlock = BlockSize div SectorSize DataSetSize = BlockSize * 100 suite "Storage Proofs Network": let - rng = Rng.instance() - seckey1 = PrivateKey.random(rng[]).tryGet() - seckey2 = PrivateKey.random(rng[]).tryGet() - hostAddr1 = ca.Address.example - hostAddr2 = ca.Address.example + hostAddr = ca.Address.example blocks = toSeq([1, 5, 10, 14, 20, 12, 22]) # TODO: maybe make them random var @@ -48,7 +41,6 @@ suite "Storage Proofs Network": store: BlockStore ssk: st.SecretKey spk: st.PublicKey - stpstore: st.StpStore porMsg: PorMessage cid: Cid por: PoR @@ -104,7 +96,7 @@ suite "Storage Proofs Network": discovery1.findHostProvidersHandler = proc(d: MockDiscovery, host: ca.Address): Future[seq[SignedPeerRecord]] {.async, gcsafe.} = - check hostAddr2 == host + check hostAddr == host return @[switch2.peerInfo.signedPeerRecord] proc tagsHandler(msg: TagsMessage) {.async, gcsafe.} = @@ -119,6 +111,6 @@ suite "Storage Proofs Network": cid, blocks, porMsg.authenticators, - hostAddr2)).tryGet() + hostAddr)).tryGet() await done.wait(1.seconds) diff --git a/tests/codex/storageproofs/teststpstore.nim b/tests/codex/storageproofs/teststpstore.nim index f8619bb2..190e67bb 100644 --- a/tests/codex/storageproofs/teststpstore.nim +++ b/tests/codex/storageproofs/teststpstore.nim @@ -13,8 +13,6 @@ import ../helpers const BlockSize = 31 * 64 - SectorSize = 31 - SectorsPerBlock = BlockSize div SectorSize DataSetSize = BlockSize * 100 suite "Test PoR store": diff --git a/tests/codex/stores/testcachestore.nim b/tests/codex/stores/testcachestore.nim index 11bbff84..427ea0b7 100644 --- a/tests/codex/stores/testcachestore.nim +++ b/tests/codex/stores/testcachestore.nim @@ -1,5 +1,4 @@ import std/strutils -import std/options import pkg/chronos import pkg/asynctest diff --git a/tests/codex/stores/testmaintenance.nim b/tests/codex/stores/testmaintenance.nim index 40c18d58..900c37cd 100644 --- a/tests/codex/stores/testmaintenance.nim +++ b/tests/codex/stores/testmaintenance.nim @@ -10,7 +10,6 @@ import pkg/chronos import pkg/libp2p import pkg/asynctest -import pkg/questionable import pkg/questionable/results import pkg/codex/blocktype as bt import pkg/codex/stores/repostore diff --git a/tests/codex/stores/testrepostore.nim b/tests/codex/stores/testrepostore.nim index 46131fe4..30f0fb0d 100644 --- a/tests/codex/stores/testrepostore.nim +++ b/tests/codex/stores/testrepostore.nim @@ -1,5 +1,4 @@ import std/os -import std/options import std/strutils import std/sequtils @@ -127,7 +126,7 @@ suite "RepoStore": discard (await metaDs.get(QuotaReservedKey)).tryGet test "Should release bytes": - let blk = createTestBlock(100) + discard createTestBlock(100) check repo.totalUsed == 0 (await repo.reserve(100)).tryGet @@ -179,7 +178,7 @@ suite "RepoStore": check: response.len == 1 - response[0].key.get == expectedKey + !response[0].key == expectedKey response[0].data == expectedExpiration.toBytes test "Should store block with default expiration timestamp when not provided": @@ -196,7 +195,7 @@ suite "RepoStore": check: response.len == 1 - response[0].key.get == expectedKey + !response[0].key == expectedKey response[0].data == expectedExpiration.toBytes test "delBlock should remove expiration metadata": diff --git a/tests/codex/testerasure.nim b/tests/codex/testerasure.nim index 9b50c558..b8a9228a 100644 --- a/tests/codex/testerasure.nim +++ b/tests/codex/testerasure.nim @@ -3,7 +3,6 @@ import std/sequtils import pkg/asynctest import pkg/chronos import pkg/libp2p -import pkg/questionable import pkg/questionable/results import pkg/codex/erasure diff --git a/tests/codex/testmanifest.nim b/tests/codex/testmanifest.nim index 7cd43b6e..451ab2f4 100644 --- a/tests/codex/testmanifest.nim +++ b/tests/codex/testmanifest.nim @@ -1,7 +1,6 @@ import std/sequtils import pkg/chronos -import pkg/questionable import pkg/questionable/results import pkg/asynctest import pkg/libp2p diff --git a/tests/codex/testnode.nim b/tests/codex/testnode.nim index 64a711aa..3c01db35 100644 --- a/tests/codex/testnode.nim +++ b/tests/codex/testnode.nim @@ -4,7 +4,6 @@ import std/math import pkg/asynctest import pkg/chronos -import pkg/chronicles import pkg/stew/byteutils import pkg/nitro diff --git a/tests/codex/testpurchasing.nim b/tests/codex/testpurchasing.nim index f2c00a67..1944cae8 100644 --- a/tests/codex/testpurchasing.nim +++ b/tests/codex/testpurchasing.nim @@ -1,10 +1,9 @@ import std/times import pkg/asynctest import pkg/chronos -import pkg/upraises import pkg/stint import pkg/codex/purchasing -import pkg/codex/purchasing/states/[finished, failed, error, started, submitted, unknown] +import pkg/codex/purchasing/states/[finished, error, started, submitted, unknown] import ./helpers/mockmarket import ./helpers/mockclock import ./helpers/eventually diff --git a/tests/codex/utils/testtimer.nim b/tests/codex/utils/testtimer.nim index 481406cb..7251ab10 100644 --- a/tests/codex/utils/testtimer.nim +++ b/tests/codex/utils/testtimer.nim @@ -7,8 +7,6 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/questionable - import pkg/chronos import pkg/asynctest diff --git a/tests/contracts/testClock.nim b/tests/contracts/testClock.nim index db97d70e..8f6f42c6 100644 --- a/tests/contracts/testClock.nim +++ b/tests/contracts/testClock.nim @@ -31,12 +31,12 @@ ethersuite "On-Chain Clock": check clock.now() > past test "raises when not started": - expect AssertionError: + expect AssertionDefect: discard OnChainClock.new(provider).now() test "raises when stopped": await clock.stop() - expect AssertionError: + expect AssertionDefect: discard clock.now() test "handles starting multiple times": diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index d7013aa1..1f0cb22b 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -1,6 +1,5 @@ import std/options import pkg/chronos -import pkg/ethers/testing import pkg/stew/byteutils import codex/contracts import codex/contracts/testtoken @@ -49,7 +48,7 @@ ethersuite "On-Chain Market": test "fails to instantiate when contract does not have a signer": let storageWithoutSigner = marketplace.connect(provider) - expect AssertionError: + expect AssertionDefect: discard OnChainMarket.new(storageWithoutSigner) test "knows signer address": @@ -119,7 +118,7 @@ ethersuite "On-Chain Market": proc onSlotFilled(requestId: RequestId, slotIndex: UInt256) = receivedSlotIndices.add(slotIndex) let subscription = await market.subscribeSlotFilled(request.id, slotIndex, onSlotFilled) - await market.fillSlot(request.id, slotIndex - 1, proof) + await market.fillSlot(request.id, otherSlot, proof) check receivedSlotIndices.len == 0 await market.fillSlot(request.id, slotIndex, proof) check receivedSlotIndices == @[slotIndex] diff --git a/tests/contracts/testProofs.nim b/tests/contracts/testProofs.nim index e37374a8..edf1417d 100644 --- a/tests/contracts/testProofs.nim +++ b/tests/contracts/testProofs.nim @@ -1,8 +1,6 @@ import codex/contracts -import codex/contracts/testtoken import ../ethertest import ./examples -import ./time ethersuite "On-Chain Proofs": diff --git a/tests/integration/testIntegration.nim b/tests/integration/testIntegration.nim new file mode 100644 index 00000000..307bf06d --- /dev/null +++ b/tests/integration/testIntegration.nim @@ -0,0 +1,156 @@ +import std/os +import std/httpclient +import std/json +import std/strutils +import pkg/chronos +import ../ethertest +import ../contracts/time +import ../codex/helpers/eventually +import ./nodes +import ./tokens + +ethersuite "Integration tests": + + var node1, node2: NodeProcess + var baseurl1, baseurl2: string + var client: HttpClient + + let dataDir1 = getTempDir() / "Codex1" + let dataDir2 = getTempDir() / "Codex2" + + setup: + await provider.getSigner(accounts[0]).mint() + await provider.getSigner(accounts[1]).mint() + await provider.getSigner(accounts[1]).deposit() + + baseurl1 = "http://localhost:8080/api/codex/v1" + baseurl2 = "http://localhost:8081/api/codex/v1" + client = newHttpClient() + + node1 = startNode([ + "--api-port=8080", + "--data-dir=" & dataDir1, + "--nat=127.0.0.1", + "--disc-ip=127.0.0.1", + "--disc-port=8090", + "--persistence", + "--eth-account=" & $accounts[0] + ], debug = false) + + let + bootstrap = strip( + $(parseJson(client.get(baseurl1 & "/debug/info").body)["spr"]), + chars = {'"'}) + + node2 = startNode([ + "--api-port=8081", + "--data-dir=" & dataDir2, + "--nat=127.0.0.1", + "--disc-ip=127.0.0.1", + "--disc-port=8091", + "--bootstrap-node=" & bootstrap, + "--persistence", + "--eth-account=" & $accounts[1] + ], debug = false) + + teardown: + client.close() + node1.stop() + node2.stop() + + dataDir1.removeDir() + dataDir2.removeDir() + + test "nodes can print their peer information": + let info1 = client.get(baseurl1 & "/debug/info").body + let info2 = client.get(baseurl2 & "/debug/info").body + check info1 != info2 + + test "nodes should set chronicles log level": + client.headers = newHttpHeaders({ "Content-Type": "text/plain" }) + let filter = "/debug/chronicles/loglevel?level=DEBUG;TRACE:codex" + check client.request(baseurl1 & filter, httpMethod = HttpPost, body = "").status == "200 OK" + + test "node accepts file uploads": + let url = baseurl1 & "/upload" + let response = client.post(url, "some file contents") + check response.status == "200 OK" + + test "node handles new storage availability": + let url = baseurl1 & "/sales/availability" + let json = %*{"size": "0x1", "duration": "0x2", "minPrice": "0x3"} + check client.post(url, $json).status == "200 OK" + + test "node lists storage that is for sale": + let url = baseurl1 & "/sales/availability" + let json = %*{"size": "0x1", "duration": "0x2", "minPrice": "0x3"} + let availability = parseJson(client.post(url, $json).body) + let response = client.get(url) + check response.status == "200 OK" + check %*availability in parseJson(response.body) + + test "node handles storage request": + let cid = client.post(baseurl1 & "/upload", "some file contents").body + let url = baseurl1 & "/storage/request/" & cid + let json = %*{"duration": "0x1", "reward": "0x2"} + let response = client.post(url, $json) + check response.status == "200 OK" + + test "node retrieves purchase status": + let cid = client.post(baseurl1 & "/upload", "some file contents").body + let request = %*{"duration": "0x1", "reward": "0x2"} + let id = client.post(baseurl1 & "/storage/request/" & cid, $request).body + let response = client.get(baseurl1 & "/storage/purchases/" & id) + check response.status == "200 OK" + let json = parseJson(response.body) + check json["request"]["ask"]["duration"].getStr == "0x1" + check json["request"]["ask"]["reward"].getStr == "0x2" + + test "node remembers purchase status after restart": + let cid = client.post(baseurl1 & "/upload", "some file contents").body + let request = %*{"duration": "0x1", "reward": "0x2"} + let id = client.post(baseurl1 & "/storage/request/" & cid, $request).body + + proc getPurchase(id: string): JsonNode = + let response = client.get(baseurl1 & "/storage/purchases/" & id) + return parseJson(response.body).catch |? nil + + check eventually getPurchase(id){"state"}.getStr == "submitted" + + node1.restart() + + client.close() + client = newHttpClient() + + check eventually (not isNil getPurchase(id){"request"}{"ask"}) + check getPurchase(id){"request"}{"ask"}{"duration"}.getStr == "0x1" + check getPurchase(id){"request"}{"ask"}{"reward"}.getStr == "0x2" + + test "nodes negotiate contracts on the marketplace": + proc sell = + let json = %*{"size": "0xFFFFF", "duration": "0x200", "minPrice": "0x300"} + discard client.post(baseurl2 & "/sales/availability", $json) + + proc available: JsonNode = + client.get(baseurl2 & "/sales/availability").body.parseJson + + proc upload: string = + client.post(baseurl1 & "/upload", "some file contents").body + + proc buy(cid: string): string = + let expiry = ((waitFor provider.currentTime()) + 30).toHex + let json = %*{"duration": "0x1", "reward": "0x400", "expiry": expiry} + client.post(baseurl1 & "/storage/request/" & cid, $json).body + + proc finish(purchase: string): Future[JsonNode] {.async.} = + while true: + let response = client.get(baseurl1 & "/storage/purchases/" & purchase) + let json = parseJson(response.body) + if json["state"].getStr == "finished": return json + await sleepAsync(1.seconds) + + sell() + let purchase = waitFor upload().buy().finish() + + check purchase["error"].getStr == "" + check available().len == 0 diff --git a/tests/integration/testblockexpiration.nim b/tests/integration/testblockexpiration.nim index 6178553a..462aa26b 100644 --- a/tests/integration/testblockexpiration.nim +++ b/tests/integration/testblockexpiration.nim @@ -1,15 +1,9 @@ -import std/osproc import std/os import std/httpclient -import std/strutils -import std/times import pkg/chronos import ../ethertest -import ../contracts/time -import ../codex/helpers/eventually import ./nodes -import ./tokens ethersuite "Node block expiration tests": @@ -59,7 +53,7 @@ ethersuite "Node block expiration tests": let contentId = uploadTestFile() - await sleepAsync(10 * 1000) + await sleepAsync(10.seconds) let response = downloadTestFile(contentId) check: @@ -71,7 +65,7 @@ ethersuite "Node block expiration tests": let contentId = uploadTestFile() - await sleepAsync(10 * 1000) + await sleepAsync(10.seconds) expect TimeoutError: discard downloadTestFile(contentId) diff --git a/tests/logging.nim b/tests/logging.nim new file mode 100644 index 00000000..0a9bfad5 --- /dev/null +++ b/tests/logging.nim @@ -0,0 +1,6 @@ +import pkg/chronicles + +proc ignoreLogging(level: LogLevel, message: LogOutputStr) = + discard + +defaultChroniclesStream.output.writer = ignoreLogging diff --git a/tests/nim.cfg b/tests/nim.cfg index 11b328d0..fff5b569 100644 --- a/tests/nim.cfg +++ b/tests/nim.cfg @@ -1,4 +1,6 @@ --path:".." --threads:on --tlsEmulation:off --d:chronicles_enabled=off + +-d:chronicles_log_level:"NONE" # compile all log statements +-d:chronicles_sinks:"textlines[dynamic]" # allow logs to be filtered by tests diff --git a/tests/testCodex.nim b/tests/testCodex.nim index 0afc5894..efd7a9dd 100644 --- a/tests/testCodex.nim +++ b/tests/testCodex.nim @@ -1,3 +1,4 @@ +import ./logging import ./codex/teststores import ./codex/testblockexchange import ./codex/teststorageproofs @@ -14,7 +15,4 @@ import ./codex/testutils import ./codex/testclock import ./codex/testsystemclock -# to check that everything compiles -import ../codex - {.warning[UnusedImport]: off.} diff --git a/tests/testCodex.nim.cfg b/tests/testCodex.nim.cfg deleted file mode 100644 index 0e237893..00000000 --- a/tests/testCodex.nim.cfg +++ /dev/null @@ -1 +0,0 @@ --d:chronicles_log_level=WARN diff --git a/tests/testContracts.nim b/tests/testContracts.nim index 0e8fde1d..5d9dbad1 100644 --- a/tests/testContracts.nim +++ b/tests/testContracts.nim @@ -1,3 +1,4 @@ +import ./logging import ./contracts/testCollateral import ./contracts/testContracts import ./contracts/testMarket diff --git a/tests/testContracts.nim.cfg b/tests/testContracts.nim.cfg deleted file mode 100644 index 0e237893..00000000 --- a/tests/testContracts.nim.cfg +++ /dev/null @@ -1 +0,0 @@ --d:chronicles_log_level=WARN diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index b428f336..3f5148e9 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -1,159 +1,5 @@ -import std/osproc -import std/os -import std/httpclient -import std/json -import std/strutils - -import pkg/chronos -import ./ethertest -import ./contracts/time -import ./integration/nodes -import ./integration/tokens +import ./logging +import ./integration/testIntegration import ./integration/testblockexpiration -import ./codex/helpers/eventually -ethersuite "Integration tests": - - var node1, node2: NodeProcess - var baseurl1, baseurl2: string - var client: HttpClient - - let dataDir1 = getTempDir() / "Codex1" - let dataDir2 = getTempDir() / "Codex2" - - setup: - await provider.getSigner(accounts[0]).mint() - await provider.getSigner(accounts[1]).mint() - await provider.getSigner(accounts[1]).deposit() - - baseurl1 = "http://localhost:8080/api/codex/v1" - baseurl2 = "http://localhost:8081/api/codex/v1" - client = newHttpClient() - - node1 = startNode([ - "--api-port=8080", - "--data-dir=" & dataDir1, - "--nat=127.0.0.1", - "--disc-ip=127.0.0.1", - "--disc-port=8090", - "--persistence", - "--eth-account=" & $accounts[0] - ], debug = false) - - let - bootstrap = strip( - $(parseJson(client.get(baseurl1 & "/debug/info").body)["spr"]), - chars = {'"'}) - - node2 = startNode([ - "--api-port=8081", - "--data-dir=" & dataDir2, - "--nat=127.0.0.1", - "--disc-ip=127.0.0.1", - "--disc-port=8091", - "--bootstrap-node=" & bootstrap, - "--persistence", - "--eth-account=" & $accounts[1] - ], debug = false) - - teardown: - client.close() - node1.stop() - node2.stop() - - dataDir1.removeDir() - dataDir2.removeDir() - - test "nodes can print their peer information": - let info1 = client.get(baseurl1 & "/debug/info").body - let info2 = client.get(baseurl2 & "/debug/info").body - check info1 != info2 - - test "nodes should set chronicles log level": - client.headers = newHttpHeaders({ "Content-Type": "text/plain" }) - let filter = "/debug/chronicles/loglevel?level=DEBUG;TRACE:codex" - check client.request(baseurl1 & filter, httpMethod = HttpPost, body = "").status == "200 OK" - - test "node accepts file uploads": - let url = baseurl1 & "/upload" - let response = client.post(url, "some file contents") - check response.status == "200 OK" - - test "node handles new storage availability": - let url = baseurl1 & "/sales/availability" - let json = %*{"size": "0x1", "duration": "0x2", "minPrice": "0x3"} - check client.post(url, $json).status == "200 OK" - - test "node lists storage that is for sale": - let url = baseurl1 & "/sales/availability" - let json = %*{"size": "0x1", "duration": "0x2", "minPrice": "0x3"} - let availability = parseJson(client.post(url, $json).body) - let response = client.get(url) - check response.status == "200 OK" - check %*availability in parseJson(response.body) - - test "node handles storage request": - let cid = client.post(baseurl1 & "/upload", "some file contents").body - let url = baseurl1 & "/storage/request/" & cid - let json = %*{"duration": "0x1", "reward": "0x2"} - let response = client.post(url, $json) - check response.status == "200 OK" - - test "node retrieves purchase status": - let cid = client.post(baseurl1 & "/upload", "some file contents").body - let request = %*{"duration": "0x1", "reward": "0x2"} - let id = client.post(baseurl1 & "/storage/request/" & cid, $request).body - let response = client.get(baseurl1 & "/storage/purchases/" & id) - check response.status == "200 OK" - let json = parseJson(response.body) - check json["request"]["ask"]["duration"].getStr == "0x1" - check json["request"]["ask"]["reward"].getStr == "0x2" - - test "node remembers purchase status after restart": - let cid = client.post(baseurl1 & "/upload", "some file contents").body - let request = %*{"duration": "0x1", "reward": "0x2"} - let id = client.post(baseurl1 & "/storage/request/" & cid, $request).body - - proc getPurchase(id: string): JsonNode = - let response = client.get(baseurl1 & "/storage/purchases/" & id) - return parseJson(response.body).catch |? nil - - check eventually getPurchase(id){"state"}.getStr == "submitted" - - node1.restart() - - client.close() - client = newHttpClient() - - check eventually (not isNil getPurchase(id){"request"}{"ask"}) - check getPurchase(id){"request"}{"ask"}{"duration"}.getStr == "0x1" - check getPurchase(id){"request"}{"ask"}{"reward"}.getStr == "0x2" - - test "nodes negotiate contracts on the marketplace": - proc sell = - let json = %*{"size": "0xFFFFF", "duration": "0x200", "minPrice": "0x300"} - discard client.post(baseurl2 & "/sales/availability", $json) - - proc available: JsonNode = - client.get(baseurl2 & "/sales/availability").body.parseJson - - proc upload: string = - client.post(baseurl1 & "/upload", "some file contents").body - - proc buy(cid: string): string = - let expiry = ((waitFor provider.currentTime()) + 30).toHex - let json = %*{"duration": "0x1", "reward": "0x400", "expiry": expiry} - client.post(baseurl1 & "/storage/request/" & cid, $json).body - - proc finish(purchase: string): Future[JsonNode] {.async.} = - while true: - let response = client.get(baseurl1 & "/storage/purchases/" & purchase) - let json = parseJson(response.body) - if json["state"].getStr == "finished": return json - await sleepAsync(1.seconds) - - sell() - let purchase = waitFor upload().buy().finish() - - check purchase["error"].getStr == "" - check available().len == 0 +{.warning[UnusedImport]:off.}