diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65080675..9fcb1de7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,28 +7,43 @@ on: workflow_dispatch: env: cache_nonce: 0 # Allows for easily busting actions/cache caches - nim_version: v1.6.10 + nim_version: v1.6.14 jobs: build: strategy: matrix: - os: [linux, macos, windows] include: - os: linux + cpu: amd64 builder: ubuntu-latest shell: bash --noprofile --norc -e -o pipefail + tests: all - os: macos + cpu: amd64 builder: macos-latest shell: bash --noprofile --norc -e -o pipefail + tests: all - os: windows + cpu: amd64 builder: windows-latest shell: msys2 + tests: unittest + - os: windows + cpu: amd64 + builder: windows-latest + shell: msys2 + tests: contract + - os: windows + cpu: amd64 + builder: windows-latest + shell: msys2 + tests: integration defaults: run: shell: ${{ matrix.shell }} {0} - name: '${{ matrix.os }}' + name: '${{ matrix.os }}-${{ matrix.cpu }}-tests-${{ matrix.tests }}' runs-on: ${{ matrix.builder }} timeout-minutes: 80 steps: @@ -44,7 +59,9 @@ jobs: shell: ${{ matrix.shell }} nim_version: ${{ env.nim_version }} + ## Part 1 Tests ## - name: Unit tests + if: matrix.tests == 'unittest' || matrix.tests == 'all' run: make -j${ncpu} test # workaround for https://github.com/NomicFoundation/hardhat/issues/3877 @@ -54,6 +71,7 @@ jobs: node-version: 18.15 - name: Start Ethereum node with Codex contracts + if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'all' working-directory: vendor/codex-contracts-eth env: MSYS2_PATH_TYPE: inherit @@ -61,10 +79,14 @@ jobs: npm install npm start & + ## Part 2 Tests ## - name: Contract tests + if: matrix.tests == 'contract' || matrix.tests == 'all' run: make -j${ncpu} testContracts + ## Part 3 Tests ## - name: Integration tests + if: matrix.tests == 'integration' || matrix.tests == 'all' run: make -j${ncpu} testIntegration coverage: @@ -83,7 +105,9 @@ jobs: nim_version: ${{ env.nim_version }} - name: Generate coverage data - run: make -j${ncpu} coverage + run: | + # make -j${ncpu} coverage + make -j${ncpu} coverage-script shell: bash - name: Upload coverage data to Codecov diff --git a/.gitignore b/.gitignore index 05d53c20..c85aa931 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,8 @@ coverage/ # Nimble packages /vendor/.nimble +/vendor/packages/ +# /vendor/*/ # Nimble user files nimble.develop @@ -36,3 +38,4 @@ nimbus-build-system.paths docker/hostdatadir docker/prometheus-data .DS_Store +nim.cfg diff --git a/.gitmodules b/.gitmodules index b74ec86f..8cc85d0e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -181,3 +181,15 @@ [submodule "vendor/codex-contracts-eth"] path = vendor/codex-contracts-eth url = https://github.com/status-im/codex-contracts-eth +[submodule "vendor/nim-protobuf-serialization"] + path = vendor/nim-protobuf-serialization + url = https://github.com/status-im/nim-protobuf-serialization +[submodule "vendor/nim-results"] + path = vendor/nim-results + url = https://github.com/arnetheduck/nim-results +[submodule "vendor/nim-testutils"] + path = vendor/nim-testutils + url = https://github.com/status-im/nim-testutils +[submodule "vendor/npeg"] + path = vendor/npeg + url = https://github.com/zevv/npeg diff --git a/Makefile b/Makefile index e629d710..0f7545f4 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ else # "variables.mk" was included. Business as usual until the end of this file # Builds the codex binary all: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim codex $(NIM_PARAMS) codex.nims + $(ENV_SCRIPT) nim codex $(NIM_PARAMS) build.nims # must be included after the default target -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk @@ -60,15 +60,12 @@ else NIM_PARAMS := $(NIM_PARAMS) -d:release endif -deps: | deps-common nat-libs codex.nims +deps: | deps-common nat-libs ifneq ($(USE_LIBBACKTRACE), 0) deps: | libbacktrace endif -#- deletes and recreates "codex.nims" which on Windows is a copy instead of a proper symlink update: | update-common - rm -rf codex.nims && \ - $(MAKE) codex.nims $(HANDLE_OUTPUT) # detecting the os ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... @@ -83,26 +80,22 @@ endif # Builds and run a part of the test suite test: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim test $(NIM_PARAMS) codex.nims + $(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims # Builds and runs the smart contract tests testContracts: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) codex.nims + $(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) build.nims # Builds and runs the integration tests testIntegration: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) codex.nims + $(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) build.nims # Builds and runs all tests testAll: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) codex.nims - -# symlink -codex.nims: - ln -s codex.nimble $@ + $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) build.nims # nim-libbacktrace LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 @@ -127,8 +120,15 @@ coverage: shopt -s globstar && lcov --extract coverage/coverage.info $$(pwd)/codex/{*,**/*}.nim --output-file coverage/coverage.f.info echo -e $(BUILD_MSG) "coverage/report/index.html" genhtml coverage/coverage.f.info --output-directory coverage/report + +show-coverage: if which open >/dev/null; then (echo -e "\e[92mOpening\e[39m HTML coverage report in browser..." && open coverage/report/index.html) || true; fi +coverage-script: build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim coverage $(NIM_PARAMS) build.nims + echo "Run `make show-coverage` to view coverage results" + # usual cleaning clean: | clean-common rm -rf build diff --git a/atlas.lock b/atlas.lock new file mode 100644 index 00000000..b37d03e3 --- /dev/null +++ b/atlas.lock @@ -0,0 +1,209 @@ +{ + "clangVersion": "", + "gccVersion": "", + "hostCPU": "arm64", + "hostOS": "macosx", + "items": { + "asynctest": { + "commit": "fe1a34caf572b05f8bdba3b650f1871af9fce31e", + "dir": "vendor/asynctest", + "url": "https://github.com/codex-storage/asynctest" + }, + "dnsclient.nim": { + "commit": "23214235d4784d24aceed99bbfe153379ea557c8", + "dir": "vendor/dnsclient.nim", + "url": "https://github.com/ba0f3/dnsclient.nim" + }, + "lrucache.nim": { + "commit": "8767ade0b76ea5b5d4ce24a52d0c58a6ebeb66cd", + "dir": "vendor/lrucache.nim", + "url": "https://github.com/status-im/lrucache.nim" + }, + "nim-bearssl": { + "commit": "99fcb3405c55b27cfffbf60f5368c55da7346f23", + "dir": "vendor/nim-bearssl", + "url": "https://github.com/status-im/nim-bearssl" + }, + "nim-blscurve": { + "commit": "48d8668c5a9a350d3a7ee0c3713ef9a11980a40d", + "dir": "vendor/nim-blscurve", + "url": "https://github.com/status-im/nim-blscurve" + }, + "nim-chronicles": { + "commit": "c9c8e58ec3f89b655a046c485f622f9021c68b61", + "dir": "vendor/nim-chronicles", + "url": "https://github.com/status-im/nim-chronicles" + }, + "nim-chronos": { + "commit": "0277b65be2c7a365ac13df002fba6e172be55537", + "dir": "vendor/nim-chronos", + "url": "https://github.com/status-im/nim-chronos" + }, + "nim-confutils": { + "commit": "2028b41602b3abf7c9bf450744efde7b296707a2", + "dir": "vendor/nim-confutils", + "url": "https://github.com/status-im/nim-confutils" + }, + "nim-contract-abi": { + "commit": "61f8f59b3917d8e27c6eb4330a6d8cf428e98b2d", + "dir": "vendor/nim-contract-abi", + "url": "https://github.com/status-im/nim-contract-abi" + }, + "nim-datastore": { + "commit": "0cde8aeb67c59fd0ac95496dc6b5e1168d6632aa", + "dir": "vendor/nim-datastore", + "url": "https://github.com/codex-storage/nim-datastore" + }, + "nim-faststreams": { + "commit": "720fc5e5c8e428d9d0af618e1e27c44b42350309", + "dir": "vendor/nim-faststreams", + "url": "https://github.com/status-im/nim-faststreams" + }, + "nim-http-utils": { + "commit": "3b491a40c60aad9e8d3407443f46f62511e63b18", + "dir": "vendor/nim-http-utils", + "url": "https://github.com/status-im/nim-http-utils" + }, + "nim-json-rpc": { + "commit": "0bf2bcbe74a18a3c7a709d57108bb7b51e748a92", + "dir": "vendor/nim-json-rpc", + "url": "https://github.com/status-im/nim-json-rpc" + }, + "nim-json-serialization": { + "commit": "bb53d49caf2a6c6cf1df365ba84af93cdcfa7aa3", + "dir": "vendor/nim-json-serialization", + "url": "https://github.com/status-im/nim-json-serialization" + }, + "nim-leopard": { + "commit": "1a6f2ab7252426a6ac01482a68b75d0c3b134cf0", + "dir": "vendor/nim-leopard", + "url": "https://github.com/status-im/nim-leopard" + }, + "nim-libbacktrace": { + "commit": "b29c22ba0ef13de50b779c776830dbea1d50cd33", + "dir": "vendor/nim-libbacktrace", + "url": "https://github.com/status-im/nim-libbacktrace" + }, + "nim-libp2p": { + "commit": "440461b24b9e66542b34d26a0b908c17f6549d05", + "dir": "vendor/nim-libp2p", + "url": "https://github.com/status-im/nim-libp2p" + }, + "nim-libp2p-dht": { + "commit": "fdd02450aa6979add7dabd29a3ba0f8738bf89f8", + "dir": "vendor/nim-libp2p-dht", + "url": "https://github.com/status-im/nim-libp2p-dht" + }, + "nim-metrics": { + "commit": "6142e433fc8ea9b73379770a788017ac528d46ff", + "dir": "vendor/nim-metrics", + "url": "https://github.com/status-im/nim-metrics" + }, + "nim-nat-traversal": { + "commit": "27d314d65c9078924b3239fe4e2f5af0c512b28c", + "dir": "vendor/nim-nat-traversal", + "url": "https://github.com/status-im/nim-nat-traversal" + }, + "nim-nitro": { + "commit": "6b4c455bf4dad7449c1580055733a1738fcd5aec", + "dir": "vendor/nim-nitro", + "url": "https://github.com/status-im/nim-nitro" + }, + "nim-presto": { + "commit": "3984431dc0fc829eb668e12e57e90542b041d298", + "dir": "vendor/nim-presto", + "url": "https://github.com/status-im/nim-presto" + }, + "nim-protobuf-serialization": { + "commit": "28214b3e40c755a9886d2ec8f261ec48fbb6bec6", + "dir": "vendor/nim-protobuf-serialization", + "url": "https://github.com/status-im/nim-protobuf-serialization" + }, + "nim-results": { + "commit": "f3c666a272c69d70cb41e7245e7f6844797303ad", + "dir": "vendor/nim-results", + "url": "https://github.com/arnetheduck/nim-results" + }, + "nim-secp256k1": { + "commit": "2acbbdcc0e63002a013fff49f015708522875832", + "dir": "vendor/nim-secp256k1", + "url": "https://github.com/status-im/nim-secp256k1" + }, + "nim-serialization": { + "commit": "384eb2561ee755446cff512a8e057325848b86a7", + "dir": "vendor/nim-serialization", + "url": "https://github.com/status-im/nim-serialization" + }, + "nim-sqlite3-abi": { + "commit": "362e1bd9f689ad9f5380d9d27f0705b3d4dfc7d3", + "dir": "vendor/nim-sqlite3-abi", + "url": "https://github.com/arnetheduck/nim-sqlite3-abi" + }, + "nim-stew": { + "commit": "7afe7e3c070758cac1f628e4330109f3ef6fc853", + "dir": "vendor/nim-stew", + "url": "https://github.com/status-im/nim-stew" + }, + "nim-taskpools": { + "commit": "b3673c7a7a959ccacb393bd9b47e997bbd177f5a", + "dir": "vendor/nim-taskpools", + "url": "https://github.com/status-im/nim-taskpools" + }, + "nim-testutils": { + "commit": "b56a5953e37fc5117bd6ea6dfa18418c5e112815", + "dir": "vendor/nim-testutils", + "url": "https://github.com/status-im/nim-testutils" + }, + "nim-toml-serialization": { + "commit": "86d477136f105f04bfd0dd7c0e939593d81fc581", + "dir": "vendor/nim-toml-serialization", + "url": "https://github.com/status-im/nim-toml-serialization" + }, + "nim-unittest2": { + "commit": "b178f47527074964f76c395ad0dfc81cf118f379", + "dir": "vendor/nim-unittest2", + "url": "https://github.com/status-im/nim-unittest2" + }, + "nim-websock": { + "commit": "2c3ae3137f3c9cb48134285bd4a47186fa51f0e8", + "dir": "vendor/nim-websock", + "url": "https://github.com/status-im/nim-websock" + }, + "nim-zlib": { + "commit": "f34ca261efd90f118dc1647beefd2f7a69b05d93", + "dir": "vendor/nim-zlib", + "url": "https://github.com/status-im/nim-zlib" + }, + "nim-stint": { + "dir": "vendor/stint", + "url": "https://github.com/status-im/nim-stint", + "commit": "86621eced1dcfb5e25903019ebcfc76ed9128ec5" + }, + "nimcrypto": { + "commit": "24e006df85927f64916e60511620583b11403178", + "dir": "vendor/nimcrypto", + "url": "https://github.com/status-im/nimcrypto" + }, + "npeg": { + "commit": "b15a10e388b91b898c581dbbcb6a718d46b27d2f", + "dir": "vendor/npeg", + "url": "https://github.com/zevv/npeg" + }, + "questionable": { + "commit": "b3cf35ac450fd42c9ea83dc084f5cba2efc55da3", + "dir": "vendor/questionable", + "url": "https://github.com/codex-storage/questionable" + }, + "upraises": { + "commit": "ff4f8108e44fba9b35cac535ab63d3927e8fd3c2", + "dir": "vendor/upraises", + "url": "https://github.com/markspanbroek/upraises" + } + }, + "nimVersion": "1.6.14", + "nimbleFile": { + "content": "# Package\n\nversion = \"0.3.2\"\nauthor = \"Status Research & Development GmbH\"\ndescription = \"DHT based on the libp2p Kademlia spec\"\nlicense = \"MIT\"\nskipDirs = @[\"tests\"]\n\n\n# Dependencies\nrequires \"nim >= 1.2.0\"\nrequires \"secp256k1#2acbbdcc0e63002a013fff49f015708522875832\" # >= 0.5.2 & < 0.6.0\nrequires \"protobuf_serialization\" # >= 0.2.0 & < 0.3.0\nrequires \"nimcrypto == 0.5.4\"\nrequires \"bearssl#head\"\nrequires \"chronicles >= 0.10.2 & < 0.11.0\"\nrequires \"chronos == 3.2.0\" # >= 3.0.11 & < 3.1.0\nrequires \"libp2p#unstable\"\nrequires \"metrics\"\nrequires \"stew#head\"\nrequires \"stint\"\nrequires \"asynctest >= 0.3.1 & < 0.4.0\"\nrequires \"https://github.com/codex-storage/nim-datastore#head\"\nrequires \"questionable\"\n\ninclude \"build.nims\"\n\n", + "filename": "" + }, + "nimcfg": "############# begin Atlas config section ##########\n--noNimblePath\n--path:\"vendor/nim-secp256k1\"\n--path:\"vendor/nim-protobuf-serialization\"\n--path:\"vendor/nimcrypto\"\n--path:\"vendor/nim-bearssl\"\n--path:\"vendor/nim-chronicles\"\n--path:\"vendor/nim-chronos\"\n--path:\"vendor/nim-libp2p\"\n--path:\"vendor/nim-metrics\"\n--path:\"vendor/nim-stew\"\n--path:\"vendor/nim-stint\"\n--path:\"vendor/asynctest\"\n--path:\"vendor/nim-datastore\"\n--path:\"vendor/questionable\"\n--path:\"vendor/nim-faststreams\"\n--path:\"vendor/nim-serialization\"\n--path:\"vendor/npeg/src\"\n--path:\"vendor/nim-unittest2\"\n--path:\"vendor/nim-testutils\"\n--path:\"vendor/nim-json-serialization\"\n--path:\"vendor/nim-http-utils\"\n--path:\"vendor/dnsclient.nim/src\"\n--path:\"vendor/nim-websock\"\n--path:\"vendor/nim-results\"\n--path:\"vendor/nim-sqlite3-abi\"\n--path:\"vendor/upraises\"\n--path:\"vendor/nim-zlib\"\n############# end Atlas config section ##########\n" +} diff --git a/build.nims b/build.nims new file mode 100644 index 00000000..bf89c0f0 --- /dev/null +++ b/build.nims @@ -0,0 +1,87 @@ +mode = ScriptMode.Verbose + + +### Helper functions +proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = + if not dirExists "build": + mkDir "build" + # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims" + var extra_params = params + when compiles(commandLineParams): + for param in commandLineParams(): + extra_params &= " " & param + else: + for i in 2..= 1.2.0" requires "asynctest >= 0.3.2 & < 0.4.0" @@ -13,7 +12,7 @@ requires "bearssl >= 0.1.4" requires "chronicles >= 0.7.2" requires "chronos >= 2.5.2" requires "confutils" -requires "ethers >= 0.2.4 & < 0.3.0" +requires "ethers >= 0.5.0 & < 0.6.0" requires "libbacktrace" requires "libp2p" requires "metrics" @@ -32,50 +31,4 @@ requires "blscurve" requires "libp2pdht" requires "eth" -when declared(namedBin): - namedBin = { - "codex/codex": "codex" - }.toTable() - -when not declared(getPathsClause): - proc getPathsClause(): string = "" - -### Helper functions -proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = - if not dirExists "build": - mkDir "build" - # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims" - var extra_params = params - when compiles(commandLineParams): - for param in commandLineParams: - extra_params &= " " & param - else: - for i in 2.. + PastStorageRequest( + requestId: event.requestId, + ask: event.ask, + expiry: event.expiry + ) + ) diff --git a/codex/contracts/marketplace.nim b/codex/contracts/marketplace.nim index 45d92335..f09e8720 100644 --- a/codex/contracts/marketplace.nim +++ b/codex/contracts/marketplace.nim @@ -18,13 +18,13 @@ type StorageRequested* = object of Event requestId*: RequestId ask*: StorageAsk + expiry*: UInt256 SlotFilled* = object of Event requestId* {.indexed.}: RequestId - slotIndex* {.indexed.}: UInt256 - slotId*: SlotId + slotIndex*: UInt256 SlotFreed* = object of Event requestId* {.indexed.}: RequestId - slotId*: SlotId + slotIndex*: UInt256 RequestFulfilled* = object of Event requestId* {.indexed.}: RequestId RequestCancelled* = object of Event diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 4c6e8b10..7393f278 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -4,6 +4,8 @@ import pkg/nimcrypto import pkg/ethers/fields import pkg/questionable/results import pkg/stew/byteutils +import pkg/json_serialization +import pkg/upraises export contractabi @@ -203,3 +205,17 @@ func price*(request: StorageRequest): UInt256 = func size*(ask: StorageAsk): UInt256 = ask.slots.u256 * ask.slotSize + +proc writeValue*( + writer: var JsonWriter, + value: SlotId | RequestId) {.upraises:[IOError].} = + + mixin writeValue + writer.writeValue value.toArray + +proc readValue*[T: SlotId | RequestId]( + reader: var JsonReader, + value: var T) {.upraises: [SerializationError, IOError].} = + + mixin readValue + value = T reader.readValue(T.distinctBase) diff --git a/codex/discovery.nim b/codex/discovery.nim index 58b83857..dda8607e 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -11,14 +11,12 @@ import std/algorithm import pkg/chronos import pkg/chronicles -import pkg/libp2p -import pkg/libp2p/routing_record -import pkg/libp2p/signed_envelope +import pkg/libp2p/[cid, multicodec, routing_record, signed_envelope] import pkg/questionable import pkg/questionable/results import pkg/stew/shims/net import pkg/contractabi/address as ca -import pkg/libp2pdht/discv5/protocol as discv5 +import pkg/codexdht/discv5/protocol as discv5 import ./rng import ./errors diff --git a/codex/errors.nim b/codex/errors.nim index 40bd7749..135af278 100644 --- a/codex/errors.nim +++ b/codex/errors.nim @@ -9,15 +9,20 @@ import pkg/stew/results +export results + type CodexError* = object of CatchableError # base codex error CodexResult*[T] = Result[T, ref CodexError] -template mapFailure*( - exp: untyped, - exc: typed = type CodexError -): untyped = +template mapFailure*[T, V, E]( + exp: Result[T, V], + exc: typedesc[E], +): Result[T, ref CatchableError] = ## Convert `Result[T, E]` to `Result[E, ref CatchableError]` ## - ((exp.mapErr do (e: auto) -> ref CatchableError: (ref exc)(msg: $e))) + exp.mapErr(proc (e: V): ref CatchableError = (ref exc)(msg: $e)) + +template mapFailure*[T, V](exp: Result[T, V]): Result[T, ref CatchableError] = + mapFailure(exp, CodexError) diff --git a/codex/formats.nim b/codex/formats.nim index ec79dabe..38881bc9 100644 --- a/codex/formats.nim +++ b/codex/formats.nim @@ -10,7 +10,7 @@ import std/strutils import pkg/chronicles -import pkg/libp2p +import pkg/libp2p/cid func shortLog*(cid: Cid): string = ## Returns compact string representation of ``pid``. diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index e0515ba0..0a3c894d 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -26,7 +26,7 @@ import ../errors import ../blocktype import ./types -func encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = +proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = ## Encode the manifest into a ``ManifestCodec`` ## multicodec container (Dag-pb) for now ## @@ -60,7 +60,7 @@ func encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = # ``` # - let cid = !manifest.rootHash + let cid = ? manifest.cid var header = initProtoBuffer() header.write(1, cid.data.buffer) header.write(2, manifest.blockSize.uint32) @@ -145,22 +145,31 @@ func decode*(_: DagPBCoder, data: openArray[byte]): ?!Manifest = if blocksLen.int != blocks.len: return failure("Total blocks and length of blocks in header don't match!") - var - self = Manifest( - rootHash: rootHashCid.some, - originalBytes: originalBytes.NBytes, - blockSize: blockSize.NBytes, - blocks: blocks, - hcodec: (? rootHashCid.mhash.mapFailure).mcodec, - codec: rootHashCid.mcodec, - version: rootHashCid.cidver, - protected: pbErasureInfo.buffer.len > 0) - - if self.protected: - self.ecK = ecK.int - self.ecM = ecM.int - self.originalCid = ? Cid.init(originalCid).mapFailure - self.originalLen = originalLen.int + let + self = if pbErasureInfo.buffer.len > 0: + Manifest.new( + rootHash = rootHashCid, + originalBytes = originalBytes.NBytes, + blockSize = blockSize.NBytes, + blocks = blocks, + version = rootHashCid.cidver, + hcodec = (? rootHashCid.mhash.mapFailure).mcodec, + codec = rootHashCid.mcodec, + ecK = ecK.int, + ecM = ecM.int, + originalCid = ? Cid.init(originalCid).mapFailure, + originalLen = originalLen.int + ) + else: + Manifest.new( + rootHash = rootHashCid, + originalBytes = originalBytes.NBytes, + blockSize = blockSize.NBytes, + blocks = blocks, + version = rootHashCid.cidver, + hcodec = (? rootHashCid.mhash.mapFailure).mcodec, + codec = rootHashCid.mcodec + ) ? self.verify() self.success @@ -172,9 +181,6 @@ proc encode*( ## Encode a manifest using `encoder` ## - if self.rootHash.isNone: - ? self.makeRoot() - encoder.encode(self) func decode*( diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index d29b8fea..1bf307ae 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -27,6 +27,58 @@ import ./types export types +type + Manifest* = ref object of RootObj + rootHash: ?Cid # Root (tree) hash of the contained data set + originalBytes*: NBytes # Exact size of the original (uploaded) file + blockSize: NBytes # Size of each contained block (might not be needed if blocks are len-prefixed) + blocks: seq[Cid] # Block Cid + version: CidVersion # Cid version + hcodec: MultiCodec # Multihash codec + codec: MultiCodec # Data set codec + case protected: bool # Protected datasets have erasure coded info + of true: + ecK: int # Number of blocks to encode + ecM: int # Number of resulting parity blocks + originalCid: Cid # The original Cid of the dataset being erasure coded + originalLen: int # The length of the original manifest + else: + discard + +############################################################ +# Accessors +############################################################ + +proc blockSize*(self: Manifest): NBytes = + self.blockSize + +proc blocks*(self: Manifest): seq[Cid] = + self.blocks + +proc version*(self: Manifest): CidVersion = + self.version + +proc hcodec*(self: Manifest): MultiCodec = + self.hcodec + +proc codec*(self: Manifest): MultiCodec = + self.codec + +proc protected*(self: Manifest): bool = + self.protected + +proc ecK*(self: Manifest): int = + self.ecK + +proc ecM*(self: Manifest): int = + self.ecM + +proc originalCid*(self: Manifest): Cid = + self.originalCid + +proc originalLen*(self: Manifest): int = + self.originalLen + ############################################################ # Operations on block list ############################################################ @@ -49,7 +101,8 @@ func `[]=`*(self: Manifest, i: BackwardsIndex, item: Cid) = self.blocks[self.len - i.int] = item func isManifest*(cid: Cid): ?!bool = - ($(?cid.contentType().mapFailure) in ManifestContainers).success + let res = ?cid.contentType().mapFailure(CodexError) + ($(res) in ManifestContainers).success func isManifest*(mc: MultiCodec): ?!bool = ($mc in ManifestContainers).success @@ -137,11 +190,8 @@ proc makeRoot*(self: Manifest): ?!void = stack.add(mh) if stack.len == 1: - let cid = ? Cid.init( - self.version, - self.codec, - (? EmptyDigests[self.version][self.hcodec].catch)) - .mapFailure + let digest = ? EmptyDigests[self.version][self.hcodec].catch + let cid = ? Cid.init(self.version, self.codec, digest).mapFailure self.rootHash = cid.some @@ -173,8 +223,8 @@ proc new*( ## Create a manifest using an array of `Cid`s ## - if hcodec notin EmptyDigests[version]: - return failure("Unsupported manifest hash codec!") + # if hcodec notin EmptyDigests[version]: + # return failure("Unsupported manifest hash codec!") T( blocks: @blocks, @@ -231,5 +281,55 @@ proc new*( decoder = ManifestContainers[$DagPBCodec] ): ?!Manifest = ## Create a manifest instance from given data - ## + ## Manifest.decode(data, decoder) + +proc new*( + T: type Manifest, + rootHash: Cid, + originalBytes: NBytes, + blockSize: NBytes, + blocks: seq[Cid], + version: CidVersion, + hcodec: MultiCodec, + codec: MultiCodec, + ecK: int, + ecM: int, + originalCid: Cid, + originalLen: int +): Manifest = + Manifest( + rootHash: rootHash.some, + originalBytes: originalBytes, + blockSize: blockSize, + blocks: blocks, + version: version, + hcodec: hcodec, + codec: codec, + protected: true, + ecK: ecK, + ecM: ecM, + originalCid: originalCid, + originalLen: originalLen + ) + +proc new*( + T: type Manifest, + rootHash: Cid, + originalBytes: NBytes, + blockSize: NBytes, + blocks: seq[Cid], + version: CidVersion, + hcodec: MultiCodec, + codec: MultiCodec +): Manifest = + Manifest( + rootHash: rootHash.some, + originalBytes: originalBytes, + blockSize: blockSize, + blocks: blocks, + version: version, + hcodec: hcodec, + codec: codec, + protected: false, + ) diff --git a/codex/manifest/types.nim b/codex/manifest/types.nim index f37c995f..cf940601 100644 --- a/codex/manifest/types.nim +++ b/codex/manifest/types.nim @@ -28,21 +28,3 @@ const ManifestContainers* = { $DagPBCodec: DagPBCoder() }.toTable - -type - Manifest* = ref object of RootObj - rootHash*: ?Cid # Root (tree) hash of the contained data set - originalBytes*: NBytes # Exact size of the original (uploaded) file - blockSize*: NBytes # Size of each contained block (might not be needed if blocks are len-prefixed) - blocks*: seq[Cid] # Block Cid - version*: CidVersion # Cid version - hcodec*: MultiCodec # Multihash codec - codec*: MultiCodec # Data set codec - case protected*: bool # Protected datasets have erasure coded info - of true: - ecK*: int # Number of blocks to encode - ecM*: int # Number of resulting parity blocks - originalCid*: Cid # The original Cid of the dataset being erasure coded - originalLen*: int # The length of the original manifest - else: - discard diff --git a/codex/market.nim b/codex/market.nim index e2a233a6..be0d06fc 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -15,13 +15,19 @@ export periods type Market* = ref object of RootObj Subscription* = ref object of RootObj - OnRequest* = proc(id: RequestId, ask: StorageAsk) {.gcsafe, upraises:[].} + OnRequest* = proc(id: RequestId, + ask: StorageAsk, + expiry: UInt256) {.gcsafe, upraises:[].} OnFulfillment* = proc(requestId: RequestId) {.gcsafe, upraises: [].} OnSlotFilled* = proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises:[].} - OnSlotFreed* = proc(slotId: SlotId) {.gcsafe, upraises: [].} + OnSlotFreed* = proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises: [].} OnRequestCancelled* = proc(requestId: RequestId) {.gcsafe, upraises:[].} OnRequestFailed* = proc(requestId: RequestId) {.gcsafe, upraises:[].} OnProofSubmitted* = proc(id: SlotId, proof: seq[byte]) {.gcsafe, upraises:[].} + PastStorageRequest* = object + requestId*: RequestId + ask*: StorageAsk + expiry*: UInt256 method getSigner*(market: Market): Future[Address] {.base, async.} = raiseAssert("not implemented") @@ -112,6 +118,11 @@ method canProofBeMarkedAsMissing*(market: Market, period: Period): Future[bool] {.base, async.} = raiseAssert("not implemented") +method subscribeFulfillment*(market: Market, + callback: OnFulfillment): + Future[Subscription] {.base, async.} = + raiseAssert("not implemented") + method subscribeFulfillment*(market: Market, requestId: RequestId, callback: OnFulfillment): @@ -135,12 +146,22 @@ method subscribeSlotFreed*(market: Market, Future[Subscription] {.base, async.} = raiseAssert("not implemented") +method subscribeRequestCancelled*(market: Market, + callback: OnRequestCancelled): + Future[Subscription] {.base, async.} = + raiseAssert("not implemented") + method subscribeRequestCancelled*(market: Market, requestId: RequestId, callback: OnRequestCancelled): Future[Subscription] {.base, async.} = raiseAssert("not implemented") +method subscribeRequestFailed*(market: Market, + callback: OnRequestFailed): + Future[Subscription] {.base, async.} = + raiseAssert("not implemented") + method subscribeRequestFailed*(market: Market, requestId: RequestId, callback: OnRequestFailed): @@ -154,3 +175,8 @@ method subscribeProofSubmission*(market: Market, method unsubscribe*(subscription: Subscription) {.base, async, upraises:[].} = raiseAssert("not implemented") + +method queryPastStorageRequests*(market: Market, + blocksAgo: int): + Future[seq[PastStorageRequest]] {.base, async.} = + raiseAssert("not implemented") diff --git a/codex/merkletree/merkletree.nim b/codex/merkletree/merkletree.nim new file mode 100644 index 00000000..6fcb4682 --- /dev/null +++ b/codex/merkletree/merkletree.nim @@ -0,0 +1,189 @@ +## Nim-Codex +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/sequtils +import std/math +import std/bitops +import std/sugar + +import pkg/libp2p +import pkg/stew/byteutils +import pkg/questionable +import pkg/questionable/results + +type + MerkleHash* = MultiHash + MerkleTree* = object + leavesCount: int + nodes: seq[MerkleHash] + MerkleProof* = object + index: int + path: seq[MerkleHash] + +# Tree constructed from leaves H0..H2 is +# H5=H(H3 & H4) +# / \ +# H3=H(H0 & H1) H4=H(H2 & H2) +# / \ / +# H0=H(A) H1=H(B) H2=H(C) +# | | | +# A B C +# +# Memory layout is [H0, H1, H2, H3, H4, H5] +# +# Proofs of inclusion are +# - [H1, H4] for A +# - [H0, H4] for B +# - [H2, H3] for C + + +func computeTreeHeight(leavesCount: int): int = + if isPowerOfTwo(leavesCount): + fastLog2(leavesCount) + 1 + else: + fastLog2(leavesCount) + 2 + +func getLowHigh(leavesCount, level: int): (int, int) = + var width = leavesCount + var low = 0 + for _ in 0..= self.leavesCount or index < 0: + return failure("Index " & $index & " out of range [0.." & $self.leaves.high & "]" ) + + var path = newSeq[MerkleHash](self.height - 1) + for level in 0.. newException(CatchableError, "Error calculating hash using codec " & $mcodec & ": " & $c) + ) + + # copy leaves + for i in 0.. + SlotQueueItem.init(event.requestId, event.ask, event.expiry) + ) + + trace "found past storage requested events to add to queue", + events = events.len + + for slots in requests: + for slot in slots: + if err =? (await queue.push(slot)).errorOption: + # continue on error + if err of QueueNotRunningError: + warn "cannot push items to queue, queue is not running" + elif err of NoMatchingAvailabilityError: + info "slot in queue had no matching availabilities, ignoring" + elif err of SlotsOutOfRangeError: + warn "Too many slots, cannot add to queue" + elif err of SlotQueueItemExistsError: + trace "item already exists, ignoring" + discard + else: raise err + except CatchableError as e: - error "Unable to start sales", msg = e.msg + warn "Error adding request to SlotQueue", error = e.msg + discard + +proc onStorageRequested(sales: Sales, + requestId: RequestId, + ask: StorageAsk, + expiry: UInt256) = + + logScope: + topics = "sales onStorageRequested" + requestId + slots = ask.slots + expiry + + let slotQueue = sales.context.slotQueue + + trace "storage requested, adding slots to queue" + + without items =? SlotQueueItem.init(requestId, ask, expiry).catch, err: + if err of SlotsOutOfRangeError: + warn "Too many slots, cannot add to queue" + else: + warn "Failed to create slot queue items from request", error = err.msg + + for item in items: + # continue on failure + slotQueue.push(item) + .track(sales) + .catch(proc(err: ref CatchableError) = + if err of NoMatchingAvailabilityError: + info "slot in queue had no matching availabilities, ignoring" + elif err of SlotQueueItemExistsError: + error "Failed to push item to queue becaue it already exists" + elif err of QueueNotRunningError: + warn "Failed to push item to queue becaue queue is not running" + else: + warn "Error adding request to SlotQueue", error = err.msg + ) + +proc onSlotFreed(sales: Sales, + requestId: RequestId, + slotIndex: UInt256) = + + logScope: + topics = "sales onSlotFreed" + requestId + slotIndex + + trace "slot freed, adding to queue" + + proc addSlotToQueue() {.async.} = + let context = sales.context + let market = context.market + let queue = context.slotQueue + + # first attempt to populate request using existing slot metadata in queue + without var found =? queue.populateItem(requestId, + slotIndex.truncate(uint16)): + trace "no existing request metadata, getting request info from contract" + # if there's no existing slot for that request, retrieve the request + # from the contract. + without request =? await market.getRequest(requestId): + error "unknown request in contract" + return + + found = SlotQueueItem.init(request, slotIndex.truncate(uint16)) + + if err =? (await queue.push(found)).errorOption: + raise err + + addSlotToQueue() + .track(sales) + .catch(proc(err: ref CatchableError) = + if err of NoMatchingAvailabilityError: + info "slot in queue had no matching availabilities, ignoring" + elif err of SlotQueueItemExistsError: + error "Failed to push item to queue becaue it already exists" + elif err of QueueNotRunningError: + warn "Failed to push item to queue becaue queue is not running" + else: + warn "Error adding request to SlotQueue", error = err.msg + ) + +proc subscribeRequested(sales: Sales) {.async.} = + let context = sales.context + let market = context.market + + proc onStorageRequested(requestId: RequestId, + ask: StorageAsk, + expiry: UInt256) = + sales.onStorageRequested(requestId, ask, expiry) + + try: + let sub = await market.subscribeRequests(onStorageRequested) + sales.subscriptions.add(sub) + except CatchableError as e: + error "Unable to subscribe to storage request events", msg = e.msg + +proc subscribeCancellation(sales: Sales) {.async.} = + let context = sales.context + let market = context.market + let queue = context.slotQueue + + proc onCancelled(requestId: RequestId) = + trace "request cancelled, removing all request slots from queue" + queue.delete(requestId) + + try: + let sub = await market.subscribeRequestCancelled(onCancelled) + sales.subscriptions.add(sub) + except CatchableError as e: + error "Unable to subscribe to cancellation events", msg = e.msg + +proc subscribeFulfilled*(sales: Sales) {.async.} = + let context = sales.context + let market = context.market + let queue = context.slotQueue + + proc onFulfilled(requestId: RequestId) = + trace "request fulfilled, removing all request slots from queue" + queue.delete(requestId) + + for agent in sales.agents: + agent.onFulfilled(requestId) + + try: + let sub = await market.subscribeFulfillment(onFulfilled) + sales.subscriptions.add(sub) + except CatchableError as e: + error "Unable to subscribe to storage fulfilled events", msg = e.msg + +proc subscribeFailure(sales: Sales) {.async.} = + let context = sales.context + let market = context.market + let queue = context.slotQueue + + proc onFailed(requestId: RequestId) = + trace "request failed, removing all request slots from queue" + queue.delete(requestId) + + for agent in sales.agents: + agent.onFailed(requestId) + + try: + let sub = await market.subscribeRequestFailed(onFailed) + sales.subscriptions.add(sub) + except CatchableError as e: + error "Unable to subscribe to storage failure events", msg = e.msg + +proc subscribeSlotFilled(sales: Sales) {.async.} = + let context = sales.context + let market = context.market + let queue = context.slotQueue + + proc onSlotFilled(requestId: RequestId, slotIndex: UInt256) = + trace "slot filled, removing from slot queue", requestId, slotIndex + queue.delete(requestId, slotIndex.truncate(uint16)) + + for agent in sales.agents: + agent.onSlotFilled(requestId, slotIndex) + + try: + let sub = await market.subscribeSlotFilled(onSlotFilled) + sales.subscriptions.add(sub) + except CatchableError as e: + error "Unable to subscribe to slot filled events", msg = e.msg + +proc subscribeSlotFreed(sales: Sales) {.async.} = + let context = sales.context + let market = context.market + + proc onSlotFreed(requestId: RequestId, slotIndex: UInt256) = + sales.onSlotFreed(requestId, slotIndex) + + try: + let sub = await market.subscribeSlotFreed(onSlotFreed) + sales.subscriptions.add(sub) + except CatchableError as e: + error "Unable to subscribe to slot freed events", msg = e.msg + +proc startSlotQueue(sales: Sales) {.async.} = + let slotQueue = sales.context.slotQueue + let reservations = sales.context.reservations + + slotQueue.onProcessSlot = + proc(item: SlotQueueItem, done: Future[void]) {.async.} = + sales.processSlot(item, done) + + asyncSpawn slotQueue.start() + + reservations.onReservationAdded = + proc(availability: Availability) {.async.} = + await sales.onReservationAdded(availability) + + +proc subscribe(sales: Sales) {.async.} = + await sales.subscribeRequested() + await sales.subscribeFulfilled() + await sales.subscribeFailure() + await sales.subscribeSlotFilled() + await sales.subscribeSlotFreed() + await sales.subscribeCancellation() + +proc unsubscribe(sales: Sales) {.async.} = + for sub in sales.subscriptions: + try: + await sub.unsubscribe() + except CatchableError as e: + error "Unable to unsubscribe from subscription", error = e.msg + +proc start*(sales: Sales) {.async.} = + await sales.startSlotQueue() + await sales.subscribe() + await sales.load() proc stop*(sales: Sales) {.async.} = - if subscription =? sales.subscription: - sales.subscription = market.Subscription.none - try: - await subscription.unsubscribe() - except CatchableError as e: - warn "Unsubscribe failed", msg = e.msg + trace "stopping sales" + sales.running = false + await sales.context.slotQueue.stop() + await sales.unsubscribe() + await sales.trackedFutures.cancelTracked() for agent in sales.agents: await agent.stop() + + sales.agents = @[] diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 86aacd64..0895307c 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -42,7 +42,9 @@ type used*: bool Reservations* = ref object repo: RepoStore + onReservationAdded: ?OnReservationAdded GetNext* = proc(): Future[?Availability] {.upraises: [], gcsafe, closure.} + OnReservationAdded* = proc(availability: Availability): Future[void] {.upraises: [], gcsafe.} AvailabilityIter* = ref object finished*: bool next*: GetNext @@ -96,18 +98,22 @@ proc toErr[E1: ref CatchableError, E2: AvailabilityError]( proc writeValue*( writer: var JsonWriter, - value: SlotId | AvailabilityId) {.upraises:[IOError].} = + value: AvailabilityId) {.upraises:[IOError].} = mixin writeValue writer.writeValue value.toArray -proc readValue*[T: SlotId | AvailabilityId]( +proc readValue*[T: AvailabilityId]( reader: var JsonReader, value: var T) {.upraises: [SerializationError, IOError].} = mixin readValue value = T reader.readValue(T.distinctBase) +proc `onReservationAdded=`*(self: Reservations, + onReservationAdded: OnReservationAdded) = + self.onReservationAdded = some onReservationAdded + func key(id: AvailabilityId): ?!Key = (ReservationsKey / id.toArray.toHex) @@ -210,6 +216,15 @@ proc reserve*( return failure(updateErr) + if onReservationAdded =? self.onReservationAdded: + try: + await onReservationAdded(availability) + except CatchableError as e: + # we don't have any insight into types of errors that `onProcessSlot` can + # throw because it is caller-defined + warn "Unknown error during 'onReservationAdded' callback", + availabilityId = availability.id, error = e.msg + return success() proc release*( @@ -320,7 +335,7 @@ proc unused*(r: Reservations): Future[?!seq[Availability]] {.async.} = proc find*( self: Reservations, - size, duration, minPrice: UInt256, collateral: UInt256, + size, duration, minPrice, collateral: UInt256, used: bool): Future[?Availability] {.async.} = diff --git a/codex/sales/salesagent.nim b/codex/sales/salesagent.nim index 7a1f7876..3f84ff9b 100644 --- a/codex/sales/salesagent.nim +++ b/codex/sales/salesagent.nim @@ -1,8 +1,11 @@ import pkg/chronos import pkg/chronicles +import pkg/questionable +import pkg/questionable/results import pkg/stint +import pkg/upraises import ../contracts/requests -import ../utils/asyncspawn +import ../errors import ./statemachine import ./salescontext import ./salesdata @@ -13,10 +16,13 @@ export reservations logScope: topics = "marketplace sales" -type SalesAgent* = ref object of Machine - context*: SalesContext - data*: SalesData - subscribed: bool +type + SalesAgent* = ref object of Machine + context*: SalesContext + data*: SalesData + subscribed: bool + SalesAgentError = object of CodexError + AllSlotsFilledError* = object of SalesAgentError func `==`*(a, b: SalesAgent): bool = a.data.requestId == b.data.requestId and @@ -26,12 +32,13 @@ proc newSalesAgent*(context: SalesContext, requestId: RequestId, slotIndex: UInt256, request: ?StorageRequest): SalesAgent = - SalesAgent( - context: context, - data: SalesData( - requestId: requestId, - slotIndex: slotIndex, - request: request)) + var agent = SalesAgent.new() + agent.context = context + agent.data = SalesData( + requestId: requestId, + slotIndex: slotIndex, + request: request) + return agent proc retrieveRequest*(agent: SalesAgent) {.async.} = let data = agent.data @@ -41,7 +48,6 @@ proc retrieveRequest*(agent: SalesAgent) {.async.} = proc subscribeCancellation(agent: SalesAgent) {.async.} = let data = agent.data - let market = agent.context.market let clock = agent.context.clock proc onCancelled() {.async.} = @@ -49,51 +55,34 @@ proc subscribeCancellation(agent: SalesAgent) {.async.} = return await clock.waitUntil(request.expiry.truncate(int64)) - if not data.fulfilled.isNil: - asyncSpawn data.fulfilled.unsubscribe(), ignore = CatchableError agent.schedule(cancelledEvent(request)) data.cancelled = onCancelled() - proc onFulfilled(_: RequestId) = - data.cancelled.cancel() +method onFulfilled*(agent: SalesAgent, requestId: RequestId) {.base, gcsafe, upraises: [].} = + if agent.data.requestId == requestId and + not agent.data.cancelled.isNil: + agent.data.cancelled.cancel() - data.fulfilled = - await market.subscribeFulfillment(data.requestId, onFulfilled) - -proc subscribeFailure(agent: SalesAgent) {.async.} = - let data = agent.data - let market = agent.context.market - - proc onFailed(_: RequestId) = - without request =? data.request: - return - asyncSpawn data.failed.unsubscribe(), ignore = CatchableError +method onFailed*(agent: SalesAgent, requestId: RequestId) {.base, gcsafe, upraises: [].} = + without request =? agent.data.request: + return + if agent.data.requestId == requestId: agent.schedule(failedEvent(request)) - data.failed = - await market.subscribeRequestFailed(data.requestId, onFailed) +method onSlotFilled*(agent: SalesAgent, + requestId: RequestId, + slotIndex: UInt256) {.base, gcsafe, upraises: [].} = -proc subscribeSlotFilled(agent: SalesAgent) {.async.} = - let data = agent.data - let market = agent.context.market - - proc onSlotFilled(requestId: RequestId, slotIndex: UInt256) = - asyncSpawn data.slotFilled.unsubscribe(), ignore = CatchableError - agent.schedule(slotFilledEvent(requestId, data.slotIndex)) - - data.slotFilled = - await market.subscribeSlotFilled(data.requestId, - data.slotIndex, - onSlotFilled) + if agent.data.requestId == requestId and + agent.data.slotIndex == slotIndex: + agent.schedule(slotFilledEvent(requestId, slotIndex)) proc subscribe*(agent: SalesAgent) {.async.} = if agent.subscribed: return await agent.subscribeCancellation() - await agent.subscribeFailure() - await agent.subscribeSlotFilled() agent.subscribed = true proc unsubscribe*(agent: SalesAgent) {.async.} = @@ -101,30 +90,12 @@ proc unsubscribe*(agent: SalesAgent) {.async.} = return let data = agent.data - try: - if not data.fulfilled.isNil: - await data.fulfilled.unsubscribe() - data.fulfilled = nil - except CatchableError: - discard - try: - if not data.failed.isNil: - await data.failed.unsubscribe() - data.failed = nil - except CatchableError: - discard - try: - if not data.slotFilled.isNil: - await data.slotFilled.unsubscribe() - data.slotFilled = nil - except CatchableError: - discard - if not data.cancelled.isNil: + if not data.cancelled.isNil and not data.cancelled.finished: await data.cancelled.cancelAndWait() data.cancelled = nil agent.subscribed = false proc stop*(agent: SalesAgent) {.async.} = - procCall Machine(agent).stop() + await Machine(agent).stop() await agent.unsubscribe() diff --git a/codex/sales/salescontext.nim b/codex/sales/salescontext.nim index ede2b1a6..9063ba31 100644 --- a/codex/sales/salescontext.nim +++ b/codex/sales/salescontext.nim @@ -5,6 +5,7 @@ import ../node/batch import ../market import ../clock import ../proving +import ./slotqueue import ./reservations type @@ -14,9 +15,10 @@ type onStore*: ?OnStore onClear*: ?OnClear onSale*: ?OnSale - onIgnored*: OnIgnored + onCleanUp*: OnCleanUp proving*: Proving reservations*: Reservations + slotQueue*: SlotQueue OnStore* = proc(request: StorageRequest, slot: UInt256, @@ -27,4 +29,4 @@ type slotIndex: UInt256) {.gcsafe, upraises: [].} OnSale* = proc(request: StorageRequest, slotIndex: UInt256) {.gcsafe, upraises: [].} - OnIgnored* = proc() {.gcsafe, upraises: [].} + OnCleanUp* = proc: Future[void] {.gcsafe, upraises: [].} diff --git a/codex/sales/salesdata.nim b/codex/sales/salesdata.nim index d8226877..0e975ac1 100644 --- a/codex/sales/salesdata.nim +++ b/codex/sales/salesdata.nim @@ -9,7 +9,4 @@ type ask*: StorageAsk request*: ?StorageRequest slotIndex*: UInt256 - failed*: market.Subscription - fulfilled*: market.Subscription - slotFilled*: market.Subscription cancelled*: Future[void] diff --git a/codex/sales/slotqueue.nim b/codex/sales/slotqueue.nim new file mode 100644 index 00000000..80e95aec --- /dev/null +++ b/codex/sales/slotqueue.nim @@ -0,0 +1,395 @@ +import std/sequtils +import std/tables +import pkg/chronicles +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +import pkg/upraises +import ./reservations +import ../errors +import ../rng +import ../utils +import ../contracts/requests +import ../utils/asyncheapqueue +import ../utils/then +import ../utils/trackedfutures + +logScope: + topics = "marketplace slotqueue" + +type + OnProcessSlot* = + proc(item: SlotQueueItem, done: Future[void]): Future[void] {.gcsafe, upraises:[].} + + # Non-ref obj copies value when assigned, preventing accidental modification + # of values which could cause an incorrect order (eg + # ``slotQueue[1].collateral = 1`` would cause ``collateral`` to be updated, + # but the heap invariant would no longer be honoured. When non-ref, the + # compiler can ensure that statement will fail). + SlotQueueWorker = object + doneProcessing*: Future[void] + + SlotQueueItem* = object + requestId: RequestId + slotIndex: uint16 + slotSize: UInt256 + duration: UInt256 + reward: UInt256 + collateral: UInt256 + expiry: UInt256 + + # don't need to -1 to prevent overflow when adding 1 (to always allow push) + # because AsyncHeapQueue size is of type `int`, which is larger than `uint16` + SlotQueueSize = range[1'u16..uint16.high] + + SlotQueue* = ref object + maxWorkers: int + onProcessSlot: ?OnProcessSlot + queue: AsyncHeapQueue[SlotQueueItem] + reservations: Reservations + running: bool + workers: AsyncQueue[SlotQueueWorker] + trackedFutures: TrackedFutures + + SlotQueueError = object of CodexError + SlotQueueItemExistsError* = object of SlotQueueError + SlotQueueItemNotExistsError* = object of SlotQueueError + SlotsOutOfRangeError* = object of SlotQueueError + NoMatchingAvailabilityError* = object of SlotQueueError + QueueNotRunningError* = object of SlotQueueError + +# Number of concurrent workers used for processing SlotQueueItems +const DefaultMaxWorkers = 3 + +# Cap slot queue size to prevent unbounded growth and make sifting more +# efficient. Max size is not equivalent to the number of slots a host can +# service, which is limited by host availabilities and new requests circulating +# the network. Additionally, each new request/slot in the network will be +# included in the queue if it is higher priority than any of the exisiting +# items. Older slots should be unfillable over time as other hosts fill the +# slots. +const DefaultMaxSize = 64'u16 + +proc profitability(item: SlotQueueItem): UInt256 = + StorageAsk(collateral: item.collateral, + duration: item.duration, + reward: item.reward, + slotSize: item.slotSize).pricePerSlot + +proc `<`*(a, b: SlotQueueItem): bool = + # for A to have a higher priority than B (in a min queue), A must be less than + # B. + var scoreA: uint8 = 0 + var scoreB: uint8 = 0 + + proc addIf(score: var uint8, condition: bool, addition: int) = + if condition: + score += 1'u8 shl addition + + scoreA.addIf(a.profitability > b.profitability, 3) + scoreB.addIf(a.profitability < b.profitability, 3) + + scoreA.addIf(a.collateral < b.collateral, 2) + scoreB.addIf(a.collateral > b.collateral, 2) + + scoreA.addIf(a.expiry > b.expiry, 1) + scoreB.addIf(a.expiry < b.expiry, 1) + + scoreA.addIf(a.slotSize < b.slotSize, 0) + scoreB.addIf(a.slotSize > b.slotSize, 0) + + return scoreA > scoreB + +proc `==`*(a, b: SlotQueueItem): bool = + a.requestId == b.requestId and + a.slotIndex == b.slotIndex + +proc new*(_: type SlotQueue, + reservations: Reservations, + maxWorkers = DefaultMaxWorkers, + maxSize: SlotQueueSize = DefaultMaxSize): SlotQueue = + + if maxWorkers <= 0: + raise newException(ValueError, "maxWorkers must be positive") + if maxWorkers.uint16 > maxSize: + raise newException(ValueError, "maxWorkers must be less than maxSize") + + SlotQueue( + maxWorkers: maxWorkers, + # Add 1 to always allow for an extra item to be pushed onto the queue + # temporarily. After push (and sort), the bottom-most item will be deleted + queue: newAsyncHeapQueue[SlotQueueItem](maxSize.int + 1), + reservations: reservations, + running: false, + trackedFutures: TrackedFutures.new() + ) + # avoid instantiating `workers` in constructor to avoid side effects in + # `newAsyncQueue` procedure + +proc init*(_: type SlotQueueWorker): SlotQueueWorker = + SlotQueueWorker( + doneProcessing: newFuture[void]("slotqueue.worker.processing") + ) + +proc init*(_: type SlotQueueItem, + requestId: RequestId, + slotIndex: uint16, + ask: StorageAsk, + expiry: UInt256): SlotQueueItem = + + SlotQueueItem( + requestId: requestId, + slotIndex: slotIndex, + slotSize: ask.slotSize, + duration: ask.duration, + reward: ask.reward, + collateral: ask.collateral, + expiry: expiry + ) + +proc init*(_: type SlotQueueItem, + request: StorageRequest, + slotIndex: uint16): SlotQueueItem = + + SlotQueueItem.init(request.id, + slotIndex, + request.ask, + request.expiry) + +proc init*(_: type SlotQueueItem, + requestId: RequestId, + ask: StorageAsk, + expiry: UInt256): seq[SlotQueueItem] = + + if not ask.slots.inRange: + raise newException(SlotsOutOfRangeError, "Too many slots") + + var i = 0'u16 + proc initSlotQueueItem: SlotQueueItem = + let item = SlotQueueItem.init(requestId, i, ask, expiry) + inc i + return item + + var items = newSeqWith(ask.slots.int, initSlotQueueItem()) + Rng.instance.shuffle(items) + return items + +proc init*(_: type SlotQueueItem, + request: StorageRequest): seq[SlotQueueItem] = + + return SlotQueueItem.init(request.id, request.ask, request.expiry) + +proc inRange*(val: SomeUnsignedInt): bool = + val.uint16 in SlotQueueSize.low..SlotQueueSize.high + +proc requestId*(self: SlotQueueItem): RequestId = self.requestId +proc slotIndex*(self: SlotQueueItem): uint16 = self.slotIndex +proc slotSize*(self: SlotQueueItem): UInt256 = self.slotSize +proc duration*(self: SlotQueueItem): UInt256 = self.duration +proc reward*(self: SlotQueueItem): UInt256 = self.reward +proc collateral*(self: SlotQueueItem): UInt256 = self.collateral + +proc running*(self: SlotQueue): bool = self.running + +proc len*(self: SlotQueue): int = self.queue.len + +proc size*(self: SlotQueue): int = self.queue.size - 1 + +proc `$`*(self: SlotQueue): string = $self.queue + +proc `onProcessSlot=`*(self: SlotQueue, onProcessSlot: OnProcessSlot) = + self.onProcessSlot = some onProcessSlot + +proc activeWorkers*(self: SlotQueue): int = + if not self.running: return 0 + + # active = capacity - available + self.maxWorkers - self.workers.len + +proc contains*(self: SlotQueue, item: SlotQueueItem): bool = + self.queue.contains(item) + +proc populateItem*(self: SlotQueue, + requestId: RequestId, + slotIndex: uint16): ?SlotQueueItem = + + trace "populate item, items in queue", len = self.queue.len + for item in self.queue.items: + trace "populate item search", itemRequestId = item.requestId, requestId + if item.requestId == requestId: + return some SlotQueueItem( + requestId: requestId, + slotIndex: slotIndex, + slotSize: item.slotSize, + duration: item.duration, + reward: item.reward, + collateral: item.collateral, + expiry: item.expiry + ) + return none SlotQueueItem + +proc push*(self: SlotQueue, item: SlotQueueItem): Future[?!void] {.async.} = + + trace "pushing item to queue", + requestId = item.requestId, slotIndex = item.slotIndex + + if not self.running: + let err = newException(QueueNotRunningError, "queue not running") + return failure(err) + + without availability =? await self.reservations.find(item.slotSize, + item.duration, + item.profitability, + item.collateral, + used = false): + let err = newException(NoMatchingAvailabilityError, "no availability") + return failure(err) + + if self.contains(item): + let err = newException(SlotQueueItemExistsError, "item already exists") + return failure(err) + + if err =? self.queue.pushNoWait(item).mapFailure.errorOption: + return failure(err) + + if self.queue.full(): + # delete the last item + self.queue.del(self.queue.size - 1) + + doAssert self.queue.len <= self.queue.size - 1 + return success() + +proc push*(self: SlotQueue, items: seq[SlotQueueItem]): Future[?!void] {.async.} = + for item in items: + if err =? (await self.push(item)).errorOption: + return failure(err) + + return success() + +proc findByRequest(self: SlotQueue, requestId: RequestId): seq[SlotQueueItem] = + var items: seq[SlotQueueItem] = @[] + for item in self.queue.items: + if item.requestId == requestId: + items.add item + return items + +proc delete*(self: SlotQueue, item: SlotQueueItem) = + logScope: + requestId = item.requestId + slotIndex = item.slotIndex + + trace "removing item from queue" + + if not self.running: + trace "cannot delete item from queue, queue not running" + return + + self.queue.delete(item) + +proc delete*(self: SlotQueue, requestId: RequestId, slotIndex: uint16) = + let item = SlotQueueItem(requestId: requestId, slotIndex: slotIndex) + self.delete(item) + +proc delete*(self: SlotQueue, requestId: RequestId) = + let items = self.findByRequest(requestId) + for item in items: + self.delete(item) + +proc `[]`*(self: SlotQueue, i: Natural): SlotQueueItem = + self.queue[i] + +proc addWorker(self: SlotQueue): ?!void = + if not self.running: + let err = newException(QueueNotRunningError, "queue must be running") + return failure(err) + + trace "adding new worker to worker queue" + + let worker = SlotQueueWorker.init() + try: + self.workers.addLastNoWait(worker) + except AsyncQueueFullError: + return failure("failed to add worker, worker queue full") + + return success() + +proc dispatch(self: SlotQueue, + worker: SlotQueueWorker, + item: SlotQueueItem) {.async.} = + logScope: + requestId = item.requestId + slotIndex = item.slotIndex + + if not self.running: + warn "Could not dispatch worker because queue is not running" + return + + if onProcessSlot =? self.onProcessSlot: + try: + await onProcessSlot(item, worker.doneProcessing) + await worker.doneProcessing + + if err =? self.addWorker().errorOption: + raise err # catch below + + except QueueNotRunningError as e: + info "could not re-add worker to worker queue, queue not running", + error = e.msg + except CancelledError: + # do not bubble exception up as it is called with `asyncSpawn` which would + # convert the exception into a `FutureDefect` + discard + except CatchableError as e: + # we don't have any insight into types of errors that `onProcessSlot` can + # throw because it is caller-defined + warn "Unknown error processing slot in worker", error = e.msg + +proc start*(self: SlotQueue) {.async.} = + if self.running: + return + + trace "starting slot queue" + + self.running = true + + # must be called in `start` to avoid sideeffects in `new` + self.workers = newAsyncQueue[SlotQueueWorker](self.maxWorkers) + + # Add initial workers to the `AsyncHeapQueue`. Once a worker has completed its + # task, a new worker will be pushed to the queue + for i in 0.. 0: + self.totalBlocks = uint64.fromBytesBE(total).uint + trace "Number of blocks in store at start", total = self.totalBlocks + ## load current persist and cache bytes from meta ds without quotaUsedBytes =? await self.metaDs.get(QuotaUsedKey), err: if not (err of DatastoreKeyNotFound): @@ -386,6 +427,7 @@ proc start*(self: RepoStore): Future[void] {.async.} = notice "Current bytes used for persist quota", bytes = self.quotaReservedBytes + self.updateMetrics() self.started = true proc stop*(self: RepoStore): Future[void] {.async.} = @@ -410,8 +452,8 @@ func new*( quotaMaxBytes = DefaultQuotaBytes, blockTtl = DefaultBlockTtl ): RepoStore = - ## Create new instance of a RepoStore - ## + ## Create new instance of a RepoStore + ## RepoStore( repoDs: repoDs, metaDs: metaDs, diff --git a/codex/streams/seekablestream.nim b/codex/streams/seekablestream.nim index 54c13380..785d9afe 100644 --- a/codex/streams/seekablestream.nim +++ b/codex/streams/seekablestream.nim @@ -7,11 +7,11 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/libp2p +import pkg/libp2p/stream/lpstream import pkg/chronos import pkg/chronicles -export libp2p, chronos, chronicles +export lpstream, chronos, chronicles logScope: topics = "codex seekablestream" diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index 0f174aee..809d961c 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -13,7 +13,6 @@ import pkg/upraises push: {.upraises: [].} -import pkg/libp2p import pkg/chronos import pkg/chronicles import pkg/stew/ptrops diff --git a/codex/utils/asyncheapqueue.nim b/codex/utils/asyncheapqueue.nim index 17ca1f78..e7d7edad 100644 --- a/codex/utils/asyncheapqueue.nim +++ b/codex/utils/asyncheapqueue.nim @@ -283,7 +283,7 @@ proc len*[T](heap: AsyncHeapQueue[T]): int {.inline.} = proc size*[T](heap: AsyncHeapQueue[T]): int {.inline.} = ## Return the maximum number of elements in ``heap``. - len(heap.maxsize) + heap.maxsize proc `[]`*[T](heap: AsyncHeapQueue[T], i: Natural) : T {.inline.} = ## Access the i-th element of ``heap`` by order from first to last. diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 13392008..3d49f741 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -1,7 +1,10 @@ +import std/sugar import pkg/questionable import pkg/chronos import pkg/chronicles import pkg/upraises +import ./trackedfutures +import ./then push: {.upraises:[].} @@ -10,8 +13,8 @@ type state: State running: Future[void] scheduled: AsyncQueue[Event] - scheduling: Future[void] started: bool + trackedFutures: TrackedFutures State* = ref object of RootObj Query*[T] = proc(state: State): T Event* = proc(state: State): ?State {.gcsafe, upraises:[].} @@ -19,6 +22,9 @@ type logScope: topics = "statemachine" +proc new*[T: Machine](_: type T): T = + T(trackedFutures: TrackedFutures.new()) + method `$`*(state: State): string {.base.} = raiseAssert "not implemented" @@ -60,21 +66,21 @@ proc run(machine: Machine, state: State) {.async.} = discard proc scheduler(machine: Machine) {.async.} = - proc onRunComplete(udata: pointer) {.gcsafe.} = - var fut = cast[FutureBase](udata) - if fut.failed(): - machine.schedule(machine.onError(fut.error)) - + var running: Future[void] try: - while true: - let event = await machine.scheduled.get() + while machine.started: + let event = await machine.scheduled.get().track(machine) if next =? event(machine.state): - if not machine.running.isNil: - await machine.running.cancelAndWait() + if not running.isNil and not running.finished: + await running.cancelAndWait() machine.state = next debug "enter state", state = machine.state - machine.running = machine.run(machine.state) - machine.running.addCallback(onRunComplete) + running = machine.run(machine.state) + running + .track(machine) + .catch((err: ref CatchableError) => + machine.schedule(machine.onError(err)) + ) except CancelledError: discard @@ -84,18 +90,20 @@ proc start*(machine: Machine, initialState: State) = if machine.scheduled.isNil: machine.scheduled = newAsyncQueue[Event]() - machine.scheduling = machine.scheduler() + machine.started = true + machine.scheduler() + .track(machine) + .catch((err: ref CatchableError) => + error("Error in scheduler", error = err.msg) + ) machine.schedule(Event.transition(machine.state, initialState)) -proc stop*(machine: Machine) = +proc stop*(machine: Machine) {.async.} = if not machine.started: return - if not machine.scheduling.isNil: - machine.scheduling.cancel() - if not machine.running.isNil: - machine.running.cancel() + machine.started = false + await machine.trackedFutures.cancelTracked() machine.state = nil - machine.started = false diff --git a/codex/utils/keyutils.nim b/codex/utils/keyutils.nim index 6e14f0b0..ef6f6246 100644 --- a/codex/utils/keyutils.nim +++ b/codex/utils/keyutils.nim @@ -12,12 +12,14 @@ push: {.upraises: [].} import pkg/chronicles import pkg/questionable/results -import pkg/libp2p +import pkg/libp2p/crypto/crypto import ./fileutils import ../errors import ../rng +export crypto + type CodexKeyError = object of CodexError CodexKeyUnsafeError = object of CodexKeyError @@ -37,7 +39,6 @@ proc setupKey*(path: string): ?!PrivateKey = warn "The network private key file is not safe, aborting" return failure newException( CodexKeyUnsafeError, "The network private key file is not safe") - - return PrivateKey.init( - ? path.readAllBytes().mapFailure(CodexKeyError)) - .mapFailure(CodexKeyError) + + let kb = ? path.readAllBytes().mapFailure(CodexKeyError) + return PrivateKey.init(kb).mapFailure(CodexKeyError) diff --git a/codex/utils/then.nim b/codex/utils/then.nim new file mode 100644 index 00000000..fbcf7bf3 --- /dev/null +++ b/codex/utils/then.nim @@ -0,0 +1,207 @@ +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +import pkg/upraises + +# Similar to JavaScript's Promise API, `.then` and `.catch` can be used to +# handle results and errors of async `Futures` within a synchronous closure. +# They can be used as an alternative to `asyncSpawn` which does not return a +# value and will raise a `FutureDefect` if there are unhandled errors +# encountered. Both `.then` and `.catch` act as callbacks that do not block the +# synchronous closure's flow. + +# `.then` is called when the `Future` is successfully completed and can be +# chained as many times as desired, calling each `.then` callback in order. When +# the `Future` returns `Result[T, ref CatchableError]` (or `?!T`), the value +# called in the `.then` callback will be unpacked from the `Result` as a +# convenience. In other words, for `Future[?!T]`, the `.then` callback will take +# a single parameter `T`. See `tests/utils/testthen.nim` for more examples. To +# allow for chaining, `.then` returns its future. If the future is already +# complete, the `.then` callback will be executed immediately. + +# `.catch` is called when the `Future` fails. In the case when the `Future` +# returns a `Result[T, ref CatchableError` (or `?!T`), `.catch` will be called +# if the `Result` contains an error. If the `Future` is already failed (or +# `Future[?!T]` contains an error), the `.catch` callback will be executed +# immediately. + +# `.cancelled` is called when the `Future` is cancelled. If the `Future` is +# already cancelled, the `.cancelled` callback will be executed immediately. + +# More info on JavaScript's Promise API can be found at: +# https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise + +runnableExamples: + proc asyncProc(): Future[int] {.async.} = + await sleepAsync(1.millis) + return 1 + + asyncProc() + .then(proc(i: int) = echo "returned ", i) + .catch(proc(e: ref CatchableError) = doAssert false, "will not be triggered") + + # outputs "returned 1" + + proc asyncProcWithError(): Future[int] {.async.} = + await sleepAsync(1.millis) + raise newException(ValueError, "some error") + + asyncProcWithError() + .then(proc(i: int) = doAssert false, "will not be triggered") + .catch(proc(e: ref CatchableError) = echo "errored: ", e.msg) + + # outputs "errored: some error" + +type + OnSuccess*[T] = proc(val: T) {.gcsafe, upraises: [].} + OnError* = proc(err: ref CatchableError) {.gcsafe, upraises: [].} + OnCancelled* = proc() {.gcsafe, upraises: [].} + +proc ignoreError(err: ref CatchableError) = discard +proc ignoreCancelled() = discard + +template handleFinished(future: FutureBase, + onError: OnError, + onCancelled: OnCancelled) = + + if not future.finished: + return + + if future.cancelled: + onCancelled() + return + + if future.failed: + onError(future.error) + return + +proc then*(future: Future[void], onSuccess: OnSuccess[void]): Future[void] = + + proc cb(udata: pointer) = + future.handleFinished(ignoreError, ignoreCancelled) + onSuccess() + + proc cancellation(udata: pointer) = + if not future.finished(): + future.removeCallback(cb) + + future.addCallback(cb) + future.cancelCallback = cancellation + return future + +proc then*[T](future: Future[T], onSuccess: OnSuccess[T]): Future[T] = + + proc cb(udata: pointer) = + future.handleFinished(ignoreError, ignoreCancelled) + + if val =? future.read.catch: + onSuccess(val) + + proc cancellation(udata: pointer) = + if not future.finished(): + future.removeCallback(cb) + + future.addCallback(cb) + future.cancelCallback = cancellation + return future + +proc then*[T](future: Future[?!T], onSuccess: OnSuccess[T]): Future[?!T] = + + proc cb(udata: pointer) = + future.handleFinished(ignoreError, ignoreCancelled) + + try: + if val =? future.read: + onSuccess(val) + except CatchableError as e: + ignoreError(e) + + proc cancellation(udata: pointer) = + if not future.finished(): + future.removeCallback(cb) + + future.addCallback(cb) + future.cancelCallback = cancellation + return future + +proc then*(future: Future[?!void], onSuccess: OnSuccess[void]): Future[?!void] = + + proc cb(udata: pointer) = + future.handleFinished(ignoreError, ignoreCancelled) + + try: + if future.read.isOk: + onSuccess() + except CatchableError as e: + ignoreError(e) + return + + proc cancellation(udata: pointer) = + if not future.finished(): + future.removeCallback(cb) + + future.addCallback(cb) + future.cancelCallback = cancellation + return future + +proc catch*[T](future: Future[T], onError: OnError) = + + if future.isNil: return + + proc cb(udata: pointer) = + future.handleFinished(onError, ignoreCancelled) + + proc cancellation(udata: pointer) = + if not future.finished(): + future.removeCallback(cb) + + future.addCallback(cb) + future.cancelCallback = cancellation + +proc catch*[T](future: Future[?!T], onError: OnError) = + + if future.isNil: return + + proc cb(udata: pointer) = + future.handleFinished(onError, ignoreCancelled) + + try: + if err =? future.read.errorOption: + onError(err) + except CatchableError as e: + onError(e) + + proc cancellation(udata: pointer) = + if not future.finished(): + future.removeCallback(cb) + + future.addCallback(cb) + future.cancelCallback = cancellation + +proc cancelled*[T](future: Future[T], onCancelled: OnCancelled): Future[T] = + + proc cb(udata: pointer) = + future.handleFinished(ignoreError, onCancelled) + + proc cancellation(udata: pointer) = + if not future.finished(): + future.removeCallback(cb) + onCancelled() + + future.addCallback(cb) + future.cancelCallback = cancellation + return future + +proc cancelled*[T](future: Future[?!T], onCancelled: OnCancelled): Future[?!T] = + + proc cb(udata: pointer) = + future.handleFinished(ignoreError, onCancelled) + + proc cancellation(udata: pointer) = + if not future.finished(): + future.removeCallback(cb) + onCancelled() + + future.addCallback(cb) + future.cancelCallback = cancellation + return future diff --git a/codex/utils/trackedfutures.nim b/codex/utils/trackedfutures.nim new file mode 100644 index 00000000..ea26c4ae --- /dev/null +++ b/codex/utils/trackedfutures.nim @@ -0,0 +1,51 @@ +import std/sugar +import std/tables +import pkg/chronicles +import pkg/chronos +import ../utils/then + +type + TrackedFutures* = ref object + futures: Table[uint, FutureBase] + cancelling: bool + +logScope: + topics = "trackable futures" + +proc len*(self: TrackedFutures): int = self.futures.len + +proc removeFuture(self: TrackedFutures, future: FutureBase) = + if not self.cancelling and not future.isNil: + trace "removing tracked future" + self.futures.del(future.id) + +proc track*[T](self: TrackedFutures, fut: Future[T]): Future[T] = + if self.cancelling: + return fut + + trace "tracking future", id = fut.id + self.futures[fut.id] = FutureBase(fut) + + fut + .then((val: T) => self.removeFuture(fut)) + .cancelled(() => self.removeFuture(fut)) + .catch((e: ref CatchableError) => self.removeFuture(fut)) + + return fut + +proc track*[T, U](future: Future[T], self: U): Future[T] = + ## Convenience method that allows chaining future, eg: + ## `await someFut().track(sales)`, where `sales` has declared a + ## `trackedFutures` property. + self.trackedFutures.track(future) + +proc cancelTracked*(self: TrackedFutures) {.async.} = + self.cancelling = true + + for future in self.futures.values: + if not future.isNil and not future.finished: + trace "cancelling tracked future", id = future.id + await future.cancelAndWait() + + self.futures.clear() + self.cancelling = false diff --git a/config.nims b/config.nims index dcb9d272..1f6d01ca 100644 --- a/config.nims +++ b/config.nims @@ -1,5 +1,7 @@ -import std/os +include "build.nims" + +import std/os const currentDir = currentSourcePath()[0 .. ^(len("config.nims") + 1)] when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and @@ -11,12 +13,12 @@ when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and include "nimbus-build-system.paths" -if defined(release): +when defined(release): switch("nimcache", joinPath(currentSourcePath.parentDir, "nimcache/release/$projectName")) else: switch("nimcache", joinPath(currentSourcePath.parentDir, "nimcache/debug/$projectName")) -if defined(limitStackUsage): +when defined(limitStackUsage): # This limits stack usage of each individual function to 1MB - the option is # available on some GCC versions but not all - run with `-d:limitStackUsage` # and look for .su files in "./build/", "./nimcache/" or $TMPDIR that list the @@ -24,7 +26,7 @@ if defined(limitStackUsage): switch("passC", "-fstack-usage -Werror=stack-usage=1048576") switch("passL", "-fstack-usage -Werror=stack-usage=1048576") -if defined(windows): +when defined(windows): # https://github.com/nim-lang/Nim/pull/19891 switch("define", "nimRawSetjmp") @@ -48,8 +50,8 @@ if defined(windows): # engineering a more portable binary release, this should be tweaked but still # use at least -msse2 or -msse3. -if defined(disableMarchNative): - if defined(i386) or defined(amd64): +when defined(disableMarchNative): + when defined(i386) or defined(amd64): switch("passC", "-mssse3") elif defined(macosx) and defined(arm64): # Apple's Clang can't handle "-march=native" on M1: https://github.com/status-im/nimbus-eth2/issues/2758 @@ -94,7 +96,7 @@ if not defined(macosx): --define:nimStackTraceOverride switch("import", "libbacktrace") ---define:nimOldCaseObjects # https://github.com/status-im/nim-confutils/issues/9 +switch("define", "codex_enable_proof_failures=true") # `switch("warning[CaseTransition]", "off")` fails with "Error: invalid command line option: '--warning[CaseTransition]'" switch("warning", "CaseTransition:off") diff --git a/docs/TWOCLIENTTEST.md b/docs/TWOCLIENTTEST.md index 1f888fc6..be8fc040 100644 --- a/docs/TWOCLIENTTEST.md +++ b/docs/TWOCLIENTTEST.md @@ -78,7 +78,7 @@ This GET request will return the node's debug information. The response JSON sho Replace `` in the next command with the string value for `spr`, returned by the first node's `debug/info` response. Open a new terminal and run: -- Mac/Unx: `"build/codex" --data-dir="$(pwd)\Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=` +- Mac/Unx: `"build/codex" --data-dir="$(pwd)/Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=` - Windows: `"build/codex.exe" --data-dir="Data2" --listen-addrs=/ip4/127.0.0.1/tcp/8071 --api-port=8081 --disc-port=8091 --bootstrap-node=` Notice we're using a new data-dir, and we've increased each port number by one. This is needed so that the new node won't try to open ports already in use by the first node. diff --git a/tests/codex/blockexchange/discovery/testdiscovery.nim b/tests/codex/blockexchange/discovery/testdiscovery.nim index 94c8ebd2..6a64255a 100644 --- a/tests/codex/blockexchange/discovery/testdiscovery.nim +++ b/tests/codex/blockexchange/discovery/testdiscovery.nim @@ -5,7 +5,6 @@ import std/tables import pkg/asynctest import pkg/chronos -import pkg/libp2p import pkg/libp2p/errors import pkg/codex/rng diff --git a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim index 4273bbb8..c9259768 100644 --- a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim @@ -5,7 +5,6 @@ import std/tables import pkg/asynctest import pkg/chronos -import pkg/libp2p import pkg/codex/rng import pkg/codex/stores diff --git a/tests/codex/blockexchange/engine/testblockexc.nim b/tests/codex/blockexchange/engine/testblockexc.nim index 4b9e7de4..2a4c9d76 100644 --- a/tests/codex/blockexchange/engine/testblockexc.nim +++ b/tests/codex/blockexchange/engine/testblockexc.nim @@ -5,9 +5,6 @@ import pkg/asynctest import pkg/chronos import pkg/stew/byteutils -import pkg/libp2p -import pkg/libp2p/errors - import pkg/codex/rng import pkg/codex/stores import pkg/codex/blockexchange diff --git a/tests/codex/blockexchange/engine/testengine.nim b/tests/codex/blockexchange/engine/testengine.nim index a7c8786a..a4e496ed 100644 --- a/tests/codex/blockexchange/engine/testengine.nim +++ b/tests/codex/blockexchange/engine/testengine.nim @@ -5,9 +5,9 @@ import std/algorithm import pkg/stew/byteutils import pkg/asynctest import pkg/chronos -import pkg/libp2p +import pkg/libp2p/errors import pkg/libp2p/routing_record -import pkg/libp2pdht/discv5/protocol as discv5 +import pkg/codexdht/discv5/protocol as discv5 import pkg/codex/rng import pkg/codex/blockexchange diff --git a/tests/codex/blockexchange/protobuf/testpresence.nim b/tests/codex/blockexchange/protobuf/testpresence.nim index 1eaf476e..e23a51dc 100644 --- a/tests/codex/blockexchange/protobuf/testpresence.nim +++ b/tests/codex/blockexchange/protobuf/testpresence.nim @@ -1,6 +1,5 @@ import pkg/asynctest import pkg/chronos -import pkg/libp2p import pkg/codex/blockexchange/protobuf/presence import ../../examples diff --git a/tests/codex/blockexchange/testnetwork.nim b/tests/codex/blockexchange/testnetwork.nim index 51d197de..6e791032 100644 --- a/tests/codex/blockexchange/testnetwork.nim +++ b/tests/codex/blockexchange/testnetwork.nim @@ -3,8 +3,6 @@ import std/tables import pkg/asynctest import pkg/chronos -import pkg/libp2p -import pkg/libp2p/errors import pkg/codex/rng import pkg/codex/chunker diff --git a/tests/codex/blockexchange/testpendingblocks.nim b/tests/codex/blockexchange/testpendingblocks.nim index f319b562..03634f9b 100644 --- a/tests/codex/blockexchange/testpendingblocks.nim +++ b/tests/codex/blockexchange/testpendingblocks.nim @@ -3,7 +3,6 @@ import std/algorithm import pkg/chronos import pkg/asynctest -import pkg/libp2p import pkg/stew/byteutils import pkg/codex/blocktype as bt diff --git a/tests/codex/helpers.nim b/tests/codex/helpers.nim index 60b1cd14..20734109 100644 --- a/tests/codex/helpers.nim +++ b/tests/codex/helpers.nim @@ -14,6 +14,8 @@ import ../checktest export randomchunker, nodeutils, mockdiscovery, eventually, checktest, manifest +export libp2p except setup, eventually + # NOTE: The meaning of equality for blocks # is changed here, because blocks are now `ref` # types. This is only in tests!!! diff --git a/tests/codex/helpers/eventually.nim b/tests/codex/helpers/eventually.nim index bbeef3be..20a957e7 100644 --- a/tests/codex/helpers/eventually.nim +++ b/tests/codex/helpers/eventually.nim @@ -1,6 +1,6 @@ import pkg/chronos -template eventually*(condition: untyped, timeout = 5.seconds): bool = +template eventuallyCheck*(condition: untyped, timeout = 5.seconds): bool = proc loop: Future[bool] {.async.} = let start = Moment.now() while true: @@ -11,3 +11,15 @@ template eventually*(condition: untyped, timeout = 5.seconds): bool = else: await sleepAsync(1.millis) await loop() + +template always*(condition: untyped, timeout = 50.millis): bool = + proc loop: Future[bool] {.async.} = + let start = Moment.now() + while true: + if not condition: + return false + if Moment.now() > (start + timeout): + return true + else: + await sleepAsync(1.millis) + await loop() diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index 6616ef57..867a3ef5 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -2,6 +2,8 @@ import std/sequtils import std/tables import std/hashes import std/sets +import std/sugar +import pkg/questionable import pkg/codex/market import pkg/codex/contracts/requests import pkg/codex/contracts/config @@ -53,7 +55,7 @@ type callback: OnRequest FulfillmentSubscription* = ref object of Subscription market: MockMarket - requestId: RequestId + requestId: ?RequestId callback: OnFulfillment SlotFilledSubscription* = ref object of Subscription market: MockMarket @@ -65,11 +67,11 @@ type callback: OnSlotFreed RequestCancelledSubscription* = ref object of Subscription market: MockMarket - requestId: RequestId + requestId: ?RequestId callback: OnRequestCancelled RequestFailedSubscription* = ref object of Subscription market: MockMarket - requestId: RequestId + requestId: ?RequestId callback: OnRequestCancelled ProofSubmittedSubscription = ref object of Subscription market: MockMarket @@ -83,7 +85,7 @@ proc hash*(requestId: RequestId): Hash = proc new*(_: type MockMarket): MockMarket = ## Create a new mocked Market instance - ## + ## let config = MarketplaceConfig( collateral: CollateralConfig( repairRewardPercentage: 10, @@ -112,7 +114,9 @@ method requestStorage*(market: MockMarket, request: StorageRequest) {.async.} = market.requested.add(request) var subscriptions = market.subscriptions.onRequest for subscription in subscriptions: - subscription.callback(request.id, request.ask) + subscription.callback(request.id, + request.ask, + request.expiry) method myRequests*(market: MockMarket): Future[seq[RequestId]] {.async.} = return market.activeRequests[market.signer] @@ -173,28 +177,32 @@ proc emitSlotFilled*(market: MockMarket, if requestMatches and slotMatches: subscription.callback(requestId, slotIndex) -proc emitSlotFreed*(market: MockMarket, slotId: SlotId) = +proc emitSlotFreed*(market: MockMarket, + requestId: RequestId, + slotIndex: UInt256) = var subscriptions = market.subscriptions.onSlotFreed for subscription in subscriptions: - subscription.callback(slotId) + subscription.callback(requestId, slotIndex) -proc emitRequestCancelled*(market: MockMarket, - requestId: RequestId) = +proc emitRequestCancelled*(market: MockMarket, requestId: RequestId) = var subscriptions = market.subscriptions.onRequestCancelled for subscription in subscriptions: - if subscription.requestId == requestId: + if subscription.requestId == requestId.some or + subscription.requestId.isNone: subscription.callback(requestId) proc emitRequestFulfilled*(market: MockMarket, requestId: RequestId) = var subscriptions = market.subscriptions.onFulfillment for subscription in subscriptions: - if subscription.requestId == requestId: + if subscription.requestId == requestId.some or + subscription.requestId.isNone: subscription.callback(requestId) proc emitRequestFailed*(market: MockMarket, requestId: RequestId) = var subscriptions = market.subscriptions.onRequestFailed for subscription in subscriptions: - if subscription.requestId == requestId: + if subscription.requestId == requestId.some or + subscription.requestId.isNone: subscription.callback(requestId) proc fillSlot*(market: MockMarket, @@ -221,7 +229,12 @@ method fillSlot*(market: MockMarket, method freeSlot*(market: MockMarket, slotId: SlotId) {.async.} = market.freed.add(slotId) - market.emitSlotFreed(slotId) + for s in market.filled: + if slotId(s.requestId, s.slotIndex) == slotId: + market.emitSlotFreed(s.requestId, s.slotIndex) + break + market.slotState[slotId] = SlotState.Free + method withdrawFunds*(market: MockMarket, requestId: RequestId) {.async.} = @@ -281,13 +294,24 @@ method subscribeRequests*(market: MockMarket, market.subscriptions.onRequest.add(subscription) return subscription +method subscribeFulfillment*(market: MockMarket, + callback: OnFulfillment): + Future[Subscription] {.async.} = + let subscription = FulfillmentSubscription( + market: market, + requestId: none RequestId, + callback: callback + ) + market.subscriptions.onFulfillment.add(subscription) + return subscription + method subscribeFulfillment*(market: MockMarket, requestId: RequestId, callback: OnFulfillment): Future[Subscription] {.async.} = let subscription = FulfillmentSubscription( market: market, - requestId: requestId, + requestId: some requestId, callback: callback ) market.subscriptions.onFulfillment.add(subscription) @@ -321,25 +345,47 @@ method subscribeSlotFreed*(market: MockMarket, market.subscriptions.onSlotFreed.add(subscription) return subscription +method subscribeRequestCancelled*(market: MockMarket, + callback: OnRequestCancelled): + Future[Subscription] {.async.} = + let subscription = RequestCancelledSubscription( + market: market, + requestId: none RequestId, + callback: callback + ) + market.subscriptions.onRequestCancelled.add(subscription) + return subscription + method subscribeRequestCancelled*(market: MockMarket, requestId: RequestId, callback: OnRequestCancelled): Future[Subscription] {.async.} = let subscription = RequestCancelledSubscription( market: market, - requestId: requestId, + requestId: some requestId, callback: callback ) market.subscriptions.onRequestCancelled.add(subscription) return subscription +method subscribeRequestFailed*(market: MockMarket, + callback: OnRequestFailed): + Future[Subscription] {.async.} = + let subscription = RequestFailedSubscription( + market: market, + requestId: none RequestId, + callback: callback + ) + market.subscriptions.onRequestFailed.add(subscription) + return subscription + method subscribeRequestFailed*(market: MockMarket, requestId: RequestId, callback: OnRequestFailed): Future[Subscription] {.async.} = let subscription = RequestFailedSubscription( market: market, - requestId: requestId, + requestId: some requestId, callback: callback ) market.subscriptions.onRequestFailed.add(subscription) @@ -355,6 +401,17 @@ method subscribeProofSubmission*(mock: MockMarket, mock.subscriptions.onProofSubmitted.add(subscription) return subscription +method queryPastStorageRequests*(market: MockMarket, + blocksAgo: int): + Future[seq[PastStorageRequest]] {.async.} = + # MockMarket does not have the concept of blocks, so simply return all + # previous events + return market.requested.map(request => + PastStorageRequest(requestId: request.id, + ask: request.ask, + expiry: request.expiry) + ) + method unsubscribe*(subscription: RequestSubscription) {.async.} = subscription.market.subscriptions.onRequest.keepItIf(it != subscription) diff --git a/tests/codex/helpers/mocksalesagent.nim b/tests/codex/helpers/mocksalesagent.nim new file mode 100644 index 00000000..43b0be87 --- /dev/null +++ b/tests/codex/helpers/mocksalesagent.nim @@ -0,0 +1,16 @@ +import pkg/codex/sales/salesagent + +type + MockSalesAgent = ref object of SalesAgent + fulfilledCalled*: bool + failedCalled*: bool + slotFilledCalled*: bool + +method onFulfilled*(agent: SalesAgent, requestId: RequestId) = + fulfilledCalled = true + +method onFailed*(agent: SalesAgent, requestId: RequestId) = + failedCalled = true + +method onSlotFilled*(agent: SalesAgent, requestId: RequestId, slotIndex: UInt256) {.base.} = + slotFilledCalled = true diff --git a/tests/codex/helpers/nodeutils.nim b/tests/codex/helpers/nodeutils.nim index df4b470e..83e59a69 100644 --- a/tests/codex/helpers/nodeutils.nim +++ b/tests/codex/helpers/nodeutils.nim @@ -2,6 +2,7 @@ import std/sequtils import pkg/chronos import pkg/libp2p +import pkg/libp2p/errors import pkg/codex/discovery import pkg/codex/stores diff --git a/tests/codex/merkletree/testmerkletree.nim b/tests/codex/merkletree/testmerkletree.nim new file mode 100644 index 00000000..18e4ae9a --- /dev/null +++ b/tests/codex/merkletree/testmerkletree.nim @@ -0,0 +1,108 @@ +import std/unittest +import std/bitops +import std/random +import std/sequtils +import pkg/libp2p +import codex/merkletree/merkletree +import ../helpers +import pkg/questionable/results + +checksuite "merkletree": + const sha256 = multiCodec("sha2-256") + const sha512 = multiCodec("sha2-512") + + proc randomHash(codec: MultiCodec = sha256): MerkleHash = + var data: array[0..31, byte] + for i in 0..31: + data[i] = rand(uint8) + return MultiHash.digest($codec, data).tryGet() + + proc combine(a, b: MerkleHash, codec: MultiCodec = sha256): MerkleHash = + var buf = newSeq[byte](a.size + b.size) + for i in 0.. agent.data.requestId == request.id and agent.data.slotIndex == 0.u256) + check sales.agents.any(agent => agent.data.requestId == request.id and agent.data.slotIndex == 1.u256) + +asyncchecksuite "Sales": + let proof = exampleProof() + + var availability: Availability + var request: StorageRequest + var sales: Sales + var market: MockMarket + var clock: MockClock + var proving: Proving + var reservations: Reservations + var repo: RepoStore + var queue: SlotQueue + var itemsProcessed: seq[SlotQueueItem] + + setup: + availability = Availability.init( + size=100.u256, + duration=60.u256, + minPrice=600.u256, + maxCollateral=400.u256 + ) + request = StorageRequest( + ask: StorageAsk( + slots: 4, + slotSize: 100.u256, + duration: 60.u256, + reward: 10.u256, + collateral: 200.u256, + ), + content: StorageContent( + cid: "some cid" + ), + expiry: (getTime() + initDuration(hours=1)).toUnix.u256 + ) + + market = MockMarket.new() + + let me = await market.getSigner() + market.activeSlots[me] = @[] + + clock = MockClock.new() + proving = Proving.new() + let repoDs = SQLiteDatastore.new(Memory).tryGet() + let metaDs = SQLiteDatastore.new(Memory).tryGet() + repo = RepoStore.new(repoDs, metaDs) + await repo.start() + sales = Sales.new(market, clock, proving, repo) + reservations = sales.context.reservations + sales.onStore = proc(request: StorageRequest, + slot: UInt256, + onBatch: BatchProc): Future[?!void] {.async.} = + return success() + queue = sales.context.slotQueue proving.onProve = proc(slot: Slot): Future[seq[byte]] {.async.} = return proof await sales.start() request.expiry = (clock.now() + 42).u256 + itemsProcessed = @[] teardown: - await repo.stop() await sales.stop() + await repo.stop() proc getAvailability: ?!Availability = waitFor reservations.get(availability.id) - proc wasIgnored: Future[bool] {.async.} = - return - eventually sales.agents.len == 1 and # agent created at first - eventually sales.agents.len == 0 # then removed once ignored + proc notProcessed(itemsProcessed: seq[SlotQueueItem], + request: StorageRequest): bool = + let items = SlotQueueItem.init(request) + for i in 0.. agent.data == expected) + check sales.agents.any(agent => agent.data.requestId == request.id and agent.data.slotIndex == 0.u256) + check sales.agents.any(agent => agent.data.requestId == request.id and agent.data.slotIndex == 1.u256) diff --git a/tests/codex/sales/testsalesagent.nim b/tests/codex/sales/testsalesagent.nim index 264a000c..690d7902 100644 --- a/tests/codex/sales/testsalesagent.nim +++ b/tests/codex/sales/testsalesagent.nim @@ -1,6 +1,3 @@ -import std/sets -import std/sequtils -import std/sugar import std/times import pkg/asynctest import pkg/chronos @@ -13,6 +10,7 @@ import pkg/codex/proving import ../helpers/mockmarket import ../helpers/mockclock import ../helpers/eventually +import ../helpers import ../examples var onCancelCalled = false @@ -25,6 +23,7 @@ type MockErrorState = ref object of ErrorHandlingState method `$`*(state: MockState): string = "MockState" +method `$`*(state: MockErrorState): string = "MockErrorState" method onCancelled*(state: MockState, request: StorageRequest): ?State = onCancelCalled = true @@ -88,45 +87,26 @@ asyncchecksuite "Sales agent": await agent.retrieveRequest() check agent.data.request == some request - test "subscribe assigns subscriptions/futures": + test "subscribe assigns cancelled future": await agent.subscribe() check not agent.data.cancelled.isNil - check not agent.data.failed.isNil - check not agent.data.fulfilled.isNil - check not agent.data.slotFilled.isNil - test "unsubscribe deassigns subscriptions/futures": + test "unsubscribe deassigns canceleld future": await agent.subscribe() await agent.unsubscribe() check agent.data.cancelled.isNil - check agent.data.failed.isNil - check agent.data.fulfilled.isNil - check agent.data.slotFilled.isNil test "subscribe can be called multiple times, without overwriting subscriptions/futures": await agent.subscribe() let cancelled = agent.data.cancelled - let failed = agent.data.failed - let fulfilled = agent.data.fulfilled - let slotFilled = agent.data.slotFilled await agent.subscribe() check cancelled == agent.data.cancelled - check failed == agent.data.failed - check fulfilled == agent.data.fulfilled - check slotFilled == agent.data.slotFilled test "unsubscribe can be called multiple times": await agent.subscribe() await agent.unsubscribe() await agent.unsubscribe() - test "subscribe can be called when request expiry has lapsed": - # succeeds when agent.data.fulfilled.isNil - request.expiry = (getTime() - initDuration(seconds=1)).toUnix.u256 - agent.data.request = some request - check agent.data.fulfilled.isNil - await agent.subscribe() - test "current state onCancelled called when cancel emitted": let state = MockState.new() agent.start(state) @@ -134,22 +114,20 @@ asyncchecksuite "Sales agent": clock.set(request.expiry.truncate(int64)) check eventually onCancelCalled - test "cancelled future is finished (cancelled) when fulfillment emitted": + test "cancelled future is finished (cancelled) when onFulfilled called": agent.start(MockState.new()) await agent.subscribe() - market.emitRequestFulfilled(request.id) + agent.onFulfilled(request.id) check eventually agent.data.cancelled.cancelled() - test "current state onFailed called when failed emitted": + test "current state onFailed called when onFailed called": agent.start(MockState.new()) - await agent.subscribe() - market.emitRequestFailed(request.id) + agent.onFailed(request.id) check eventually onFailedCalled test "current state onSlotFilled called when slot filled emitted": agent.start(MockState.new()) - await agent.subscribe() - market.emitSlotFilled(request.id, slotIndex) + agent.onSlotFilled(request.id, slotIndex) check eventually onSlotFilledCalled test "ErrorHandlingState.onError can be overridden at the state level": diff --git a/tests/codex/sales/testslotqueue.nim b/tests/codex/sales/testslotqueue.nim new file mode 100644 index 00000000..b7cc2059 --- /dev/null +++ b/tests/codex/sales/testslotqueue.nim @@ -0,0 +1,451 @@ +import std/sequtils +import pkg/asynctest +import pkg/chronicles +import pkg/chronos +import pkg/datastore +import pkg/questionable +import pkg/questionable/results + +import pkg/codex/sales/reservations +import pkg/codex/sales/slotqueue +import pkg/codex/stores + +import ../helpers +import ../helpers/mockmarket +import ../helpers/eventually +import ../examples + +suite "Slot queue start/stop": + + var repo: RepoStore + var repoDs: Datastore + var metaDs: SQLiteDatastore + var reservations: Reservations + var queue: SlotQueue + + setup: + repoDs = SQLiteDatastore.new(Memory).tryGet() + metaDs = SQLiteDatastore.new(Memory).tryGet() + repo = RepoStore.new(repoDs, metaDs) + reservations = Reservations.new(repo) + queue = SlotQueue.new(reservations) + + teardown: + await queue.stop() + + test "starts out not running": + check not queue.running + + test "can call start multiple times, and when already running": + asyncSpawn queue.start() + asyncSpawn queue.start() + check queue.running + + test "can call stop when alrady stopped": + await queue.stop() + check not queue.running + + test "can call stop when running": + asyncSpawn queue.start() + await queue.stop() + check not queue.running + + test "can call stop multiple times": + asyncSpawn queue.start() + await queue.stop() + await queue.stop() + check not queue.running + +suite "Slot queue workers": + + var repo: RepoStore + var repoDs: Datastore + var metaDs: SQLiteDatastore + var availability: Availability + var reservations: Reservations + var queue: SlotQueue + + proc onProcessSlot(item: SlotQueueItem, doneProcessing: Future[void]) {.async.} = + await sleepAsync(1000.millis) + # this is not illustrative of the realistic scenario as the + # `doneProcessing` future would be passed to another context before being + # completed and therefore is not as simple as making the callback async + doneProcessing.complete() + + setup: + let request = StorageRequest.example + repoDs = SQLiteDatastore.new(Memory).tryGet() + metaDs = SQLiteDatastore.new(Memory).tryGet() + let quota = request.ask.slotSize.truncate(uint) * 100 + 1 + repo = RepoStore.new(repoDs, metaDs, quotaMaxBytes = quota) + reservations = Reservations.new(repo) + # create an availability that should always match + availability = Availability.init( + size = request.ask.slotSize * 100, + duration = request.ask.duration * 100, + minPrice = request.ask.pricePerSlot div 100, + maxCollateral = request.ask.collateral * 100 + ) + queue = SlotQueue.new(reservations, maxSize = 5, maxWorkers = 3) + queue.onProcessSlot = onProcessSlot + discard await reservations.reserve(availability) + + proc startQueue = asyncSpawn queue.start() + + teardown: + await queue.stop() + + test "activeWorkers should be 0 when not running": + check queue.activeWorkers == 0 + + test "maxWorkers cannot be 0": + expect ValueError: + discard SlotQueue.new(reservations, maxSize = 1, maxWorkers = 0) + + test "maxWorkers cannot surpass maxSize": + expect ValueError: + discard SlotQueue.new(reservations, maxSize = 1, maxWorkers = 2) + + test "does not surpass max workers": + startQueue() + let item1 = SlotQueueItem.example + let item2 = SlotQueueItem.example + let item3 = SlotQueueItem.example + let item4 = SlotQueueItem.example + check (await queue.push(item1)).isOk + check (await queue.push(item2)).isOk + check (await queue.push(item3)).isOk + check (await queue.push(item4)).isOk + check eventually queue.activeWorkers == 3 + + test "discards workers once processing completed": + proc processSlot(item: SlotQueueItem, done: Future[void]) {.async.} = + await sleepAsync(1.millis) + done.complete() + + queue.onProcessSlot = processSlot + + startQueue() + let item1 = SlotQueueItem.example + let item2 = SlotQueueItem.example + let item3 = SlotQueueItem.example + let item4 = SlotQueueItem.example + check (await queue.push(item1)).isOk # finishes after 1.millis + check (await queue.push(item2)).isOk # finishes after 1.millis + check (await queue.push(item3)).isOk # finishes after 1.millis + check (await queue.push(item4)).isOk + check eventually queue.activeWorkers == 1 + +suite "Slot queue": + + var onProcessSlotCalled = false + var onProcessSlotCalledWith: seq[(RequestId, uint16)] + var repo: RepoStore + var repoDs: Datastore + var metaDs: SQLiteDatastore + var availability: Availability + var reservations: Reservations + var queue: SlotQueue + let maxWorkers = 2 + var unpauseQueue: Future[void] + var paused: bool + + proc newSlotQueue(maxSize, maxWorkers: int, processSlotDelay = 1.millis) = + queue = SlotQueue.new(reservations, maxWorkers, maxSize.uint16) + queue.onProcessSlot = proc(item: SlotQueueItem, done: Future[void]) {.async.} = + await sleepAsync(processSlotDelay) + trace "processing item", requestId = item.requestId, slotIndex = item.slotIndex + onProcessSlotCalled = true + onProcessSlotCalledWith.add (item.requestId, item.slotIndex) + done.complete() + asyncSpawn queue.start() + + setup: + onProcessSlotCalled = false + onProcessSlotCalledWith = @[] + let request = StorageRequest.example + repoDs = SQLiteDatastore.new(Memory).tryGet() + metaDs = SQLiteDatastore.new(Memory).tryGet() + let quota = request.ask.slotSize.truncate(uint) * 100 + 1 + repo = RepoStore.new(repoDs, metaDs, quotaMaxBytes = quota) + reservations = Reservations.new(repo) + # create an availability that should always match + availability = Availability.init( + size = request.ask.slotSize * 100, + duration = request.ask.duration * 100, + minPrice = request.ask.pricePerSlot div 100, + maxCollateral = request.ask.collateral * 100 + ) + discard await reservations.reserve(availability) + + teardown: + paused = false + + await queue.stop() + + test "starts out empty": + newSlotQueue(maxSize = 2, maxWorkers = 2) + check queue.len == 0 + check $queue == "[]" + + test "reports correct size": + newSlotQueue(maxSize = 2, maxWorkers = 2) + check queue.size == 2 + + test "correctly compares SlotQueueItems": + var requestA = StorageRequest.example + requestA.ask.duration = 1.u256 + requestA.ask.reward = 1.u256 + check requestA.ask.pricePerSlot == 1.u256 + requestA.ask.collateral = 100000.u256 + requestA.expiry = 1001.u256 + + var requestB = StorageRequest.example + requestB.ask.duration = 100.u256 + requestB.ask.reward = 1000.u256 + check requestB.ask.pricePerSlot == 100000.u256 + requestB.ask.collateral = 1.u256 + requestB.expiry = 1000.u256 + + let itemA = SlotQueueItem.init(requestA, 0) + let itemB = SlotQueueItem.init(requestB, 0) + check itemB < itemA # B higher priority than A + check itemA > itemB + + test "expands available all possible slot indices on init": + let request = StorageRequest.example + let items = SlotQueueItem.init(request) + check items.len.uint64 == request.ask.slots + var checked = 0 + for slotIndex in 0'u16.. lldb.SBValue: + if isinstance(name, str): + return self.GetChildMemberWithName(name) + else: + return self.GetChildAtIndex(name) + + +# Make this easier to work with +lldb.SBValue.__getitem__ = sbvaluegetitem + +NIM_IS_V2 = True + + +def get_nti(value: lldb.SBValue, nim_name=None): + name_split = value.type.name.split("_") + type_nim_name = nim_name or name_split[1] + id_string = name_split[-1].split(" ")[0] + + type_info_name = "NTI" + type_nim_name.lower() + "__" + id_string + "_" + nti = value.target.FindFirstGlobalVariable(type_info_name) + if not nti.IsValid(): + type_info_name = "NTI" + "__" + id_string + "_" + nti = value.target.FindFirstGlobalVariable(type_info_name) + if not nti.IsValid(): + print(f"NimEnumPrinter: lookup global symbol: '{type_info_name}' failed for {value.type.name}.\n") + return type_nim_name, nti + + +def enum_to_string(value: lldb.SBValue, int_val=None, nim_name=None): + tname = nim_name or value.type.name.split("_")[1] + + enum_val = value.signed + if int_val is not None: + enum_val = int_val + + default_val = f"{tname}.{str(enum_val)}" + + fn_syms = value.target.FindFunctions("reprEnum") + if not fn_syms.GetSize() > 0: + return default_val + + fn_sym: lldb.SBSymbolContext = fn_syms.GetContextAtIndex(0) + + fn: lldb.SBFunction = fn_sym.function + + fn_type: lldb.SBType = fn.type + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + if arg_types.GetSize() < 2: + return default_val + + arg1_type: lldb.SBType = arg_types.GetTypeAtIndex(0) + arg2_type: lldb.SBType = arg_types.GetTypeAtIndex(1) + + ty_info_name, nti = get_nti(value, nim_name=tname) + + if not nti.IsValid(): + return default_val + + call = f"{fn.name}(({arg1_type.name}){enum_val}, ({arg2_type.name})" + str(nti.GetLoadAddress()) + ");" + + res = executeCommand(call) + + if res.error.fail: + return default_val + + return f"{tname}.{res.summary[1:-1]}" + + +def to_string(value: lldb.SBValue): + # For getting NimStringDesc * value + value = value.GetNonSyntheticValue() + + # Check if data pointer is Null + if value.type.is_pointer and value.unsigned == 0: + return None + + size = int(value["Sup"]["len"].unsigned) + + if size == 0: + return "" + + if size > 2**14: + return "... (too long) ..." + + data = value["data"] + + # Check if first element is NULL + base_data_type = value.target.FindFirstType("char") + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return None + + cast = data.Cast(value.target.FindFirstType("char").GetArrayType(size)) + return bytearray(cast.data.uint8s).decode("utf-8") + + +def to_stringV2(value: lldb.SBValue): + # For getting NimStringV2 value + value = value.GetNonSyntheticValue() + + data = value["p"]["data"] + + # Check if data pointer is Null + if value["p"].unsigned == 0: + return None + + size = int(value["len"].signed) + + if size == 0: + return "" + + if size > 2**14: + return "... (too long) ..." + + # Check if first element is NULL + base_data_type = data.type.GetArrayElementType().GetTypedefedType() + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return None + + cast = data.Cast(base_data_type.GetArrayType(size)) + return bytearray(cast.data.uint8s).decode("utf-8") + + +def NimString(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + if NIM_IS_V2: + res = to_stringV2(value) + else: + res = to_string(value) + + if res is not None: + return f'"{res}"' + else: + return "nil" + + +def rope_helper(value: lldb.SBValue) -> str: + value = value.GetNonSyntheticValue() + if value.type.is_pointer and value.unsigned == 0: + return "" + + if value["length"].unsigned == 0: + return "" + + if NIM_IS_V2: + str_val = to_stringV2(value["data"]) + else: + str_val = to_string(value["data"]) + + if str_val is None: + str_val = "" + + return rope_helper(value["left"]) + str_val + rope_helper(value["right"]) + + +def Rope(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + rope_str = rope_helper(value) + + if len(rope_str) == 0: + rope_str = "nil" + else: + rope_str = f'"{rope_str}"' + + return f"Rope({rope_str})" + + +def NCSTRING(value: lldb.SBValue, internal_dict=None): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + ty = value.Dereference().type + val = value.target.CreateValueFromAddress( + value.name or "temp", lldb.SBAddress(value.unsigned, value.target), ty + ).AddressOf() + return val.summary + + +def ObjectV2(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + orig_value = value.GetNonSyntheticValue() + if orig_value.type.is_pointer and orig_value.unsigned == 0: + return "nil" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + while orig_value.type.is_pointer: + orig_value = orig_value.Dereference() + + if "_" in orig_value.type.name: + obj_name = orig_value.type.name.split("_")[1].replace("colonObjectType", "") + else: + obj_name = orig_value.type.name + + num_children = value.num_children + fields = [] + + for i in range(num_children): + fields.append(f"{value[i].name}: {value[i].summary}") + + res = f"{obj_name}(" + ", ".join(fields) + ")" + return res + + +def Number(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + if value.type.is_pointer and value.signed == 0: + return "nil" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.signed) + + +def Float(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.value) + + +def UnsignedNumber(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.unsigned) + + +def Bool(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.value) + + +def CharArray(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str([f"'{char}'" for char in value.uint8s]) + + +def Array(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + value = value.GetNonSyntheticValue() + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + value = value.GetNonSyntheticValue() + return "[" + ", ".join([value[i].summary for i in range(value.num_children)]) + "]" + + +def Tuple(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + while value.type.is_pointer: + value = value.Dereference() + + num_children = value.num_children + + fields = [] + + for i in range(num_children): + key = value[i].name + val = value[i].summary + if key.startswith("Field"): + fields.append(f"{val}") + else: + fields.append(f"{key}: {val}") + + return "(" + ", ".join(fields) + f")" + + +def is_local(value: lldb.SBValue) -> bool: + line: lldb.SBLineEntry = value.frame.GetLineEntry() + decl: lldb.SBDeclaration = value.GetDeclaration() + + if line.file == decl.file and decl.line != 0: + return True + + return False + + +def is_in_scope(value: lldb.SBValue) -> bool: + line: lldb.SBLineEntry = value.frame.GetLineEntry() + decl: lldb.SBDeclaration = value.GetDeclaration() + + if is_local(value) and decl.line < line.line: + return True + + return False + + +def Enum(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_value_summary(value) + if custom_summary is not None: + return custom_summary + + return enum_to_string(value) + + +def EnumSet(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + vals = [] + max_vals = 7 + for child in value.children: + vals.append(child.summary) + if len(vals) > max_vals: + vals.append("...") + break + + return "{" + ", ".join(vals) + "}" + + +def Set(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + vals = [] + max_vals = 7 + for child in value.children: + vals.append(child.value) + if len(vals) > max_vals: + vals.append("...") + break + + return "{" + ", ".join(vals) + "}" + + +def Table(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + fields = [] + + for i in range(value.num_children): + key = value[i].name + val = value[i].summary + fields.append(f"{key}: {val}") + + return "Table({" + ", ".join(fields) + "})" + + +def HashSet(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + fields = [] + + for i in range(value.num_children): + fields.append(f"{value[i].summary}") + + return "HashSet({" + ", ".join(fields) + "})" + + +def StringTable(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + fields = [] + + for i in range(value.num_children - 1): + key = value[i].name + val = value[i].summary + fields.append(f"{key}: {val}") + + mode = value[value.num_children - 1].summary + + return "StringTable({" + ", ".join(fields) + f"}}, mode={mode})" + + +def Sequence(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return "@[" + ", ".join([value[i].summary for i in range(value.num_children)]) + "]" + + +class StringChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + if not NIM_IS_V2: + self.data_type = self.value.target.FindFirstType("char") + + self.first_element: lldb.SBValue + self.update() + self.count = 0 + + def num_children(self): + return self.count + + def get_child_index(self, name): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.data_size + return self.first_element.CreateChildAtOffset("[" + str(index) + "]", offset, self.data_type) + + def get_data(self) -> lldb.SBValue: + return self.value["p"]["data"] if NIM_IS_V2 else self.value["data"] + + def get_len(self) -> int: + if NIM_IS_V2: + if self.value["p"].unsigned == 0: + return 0 + + size = int(self.value["len"].signed) + + if size == 0: + return 0 + + data = self.value["p"]["data"] + + # Check if first element is NULL + base_data_type = data.type.GetArrayElementType().GetTypedefedType() + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return 0 + else: + if self.value.type.is_pointer and self.value.unsigned == 0: + return 0 + + size = int(self.value["Sup"]["len"].unsigned) + + if size == 0: + return 0 + + data = self.value["data"] + + # Check if first element is NULL + base_data_type = self.value.target.FindFirstType("char") + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return 0 + + return size + + def update(self): + if is_local(self.value): + if not is_in_scope(self.value): + return + + data = self.get_data() + size = self.get_len() + + self.count = size + self.first_element = data + + if NIM_IS_V2: + self.data_type = data.type.GetArrayElementType().GetTypedefedType() + + self.data_size = self.data_type.GetByteSize() + + def has_children(self): + return bool(self.num_children()) + + +class ArrayChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.update() + + def num_children(self): + return self.has_children() and self.value.num_children + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.value[index].GetByteSize() + return self.first_element.CreateChildAtOffset("[" + str(index) + "]", offset, self.data_type) + + def update(self): + if not self.has_children(): + return + + self.first_element = self.value[0] + self.data_type = self.value.type.GetArrayElementType() + + def has_children(self): + if is_local(self.value): + if not is_in_scope(self.value): + return False + return bool(self.value.num_children) + + +class SeqChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.data: lldb.SBValue + self.count = 0 + self.update() + + def num_children(self): + return self.count + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.data[index].GetByteSize() + return self.first_element.CreateChildAtOffset("[" + str(index) + "]", offset, self.data_type) + + def get_data(self) -> lldb.SBValue: + return self.value["p"]["data"] if NIM_IS_V2 else self.value["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["len"] if NIM_IS_V2 else self.value["Sup"]["len"] + + def update(self): + self.count = 0 + + if is_local(self.value): + if not is_in_scope(self.value): + return + + self.count = self.get_len().unsigned + + if not self.has_children(): + return + + data = self.get_data() + self.data_type = data.type.GetArrayElementType() + + self.data = data.Cast(self.data_type.GetArrayType(self.num_children())) + self.first_element = self.data + + def has_children(self): + return bool(self.num_children()) + + +class ObjectChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.data: lldb.SBValue + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.children) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def populate_children(self): + self.children.clear() + self.child_list = [] + + if is_local(self.value): + if not is_in_scope(self.value): + return + + stack = [self.value.GetNonSyntheticValue()] + + index = 0 + + while stack: + cur_val = stack.pop() + if cur_val.type.is_pointer and cur_val.unsigned == 0: + continue + + while cur_val.type.is_pointer: + cur_val = cur_val.Dereference() + + # Add super objects if they exist + if cur_val.num_children > 0 and cur_val[0].name == "Sup" and cur_val[0].type.name.startswith("tyObject"): + stack.append(cur_val[0]) + + for child in cur_val.children: + child = child.GetNonSyntheticValue() + if child.name == "Sup": + continue + self.children[child.name] = index + self.child_list.append(child) + index += 1 + + def update(self): + self.populate_children() + + def has_children(self): + return bool(self.num_children()) + + +class HashSetChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def get_data(self) -> lldb.SBValue: + return self.value["data"]["p"]["data"] if NIM_IS_V2 else self.value["data"]["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["data"]["len"] if NIM_IS_V2 else self.value["data"]["Sup"]["len"] + + def update(self): + self.child_list = [] + + if is_local(self.value): + if not is_in_scope(self.value): + return + + tuple_len = int(self.get_len().unsigned) + tuple = self.get_data() + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el[0].unsigned) + if field0 == 0: + continue + key = el[1] + child = key.CreateValueFromAddress(f"[{str(index)}]", key.GetLoadAddress(), key.GetType()) + index += 1 + + self.child_list.append(child) + + def has_children(self): + return bool(self.num_children()) + + +class SetCharChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType("char") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + + cur_pos = 0 + for child in self.value.children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data(f"[{len(self.child_list)}]", data, self.ty) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +def create_set_children(value: lldb.SBValue, child_type: lldb.SBType, starting_pos: int) -> list[lldb.SBValue]: + child_list: list[lldb.SBValue] = [] + cur_pos = starting_pos + + if value.num_children > 0: + children = value.children + else: + children = [value] + + for child in children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = value.synthetic_child_from_data(f"[{len(child_list)}]", data, child_type) + child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + return child_list + + +class SetIntChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NI64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + bits = self.value.GetByteSize() * 8 + starting_pos = -(bits // 2) + self.child_list = create_set_children(self.value, self.ty, starting_pos) + + def has_children(self): + return bool(self.num_children()) + + +class SetUIntChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NU64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + self.child_list = create_set_children(self.value, self.ty, starting_pos=0) + + def has_children(self): + return bool(self.num_children()) + + +class SetEnumChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(self.value.type.name.replace("tySet_", "")) + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + if is_local(self.value): + if not is_in_scope(self.value): + return + self.child_list = create_set_children(self.value, self.ty, starting_pos=0) + + def has_children(self): + return bool(self.num_children()) + + +class TableChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def get_data(self) -> lldb.SBValue: + return self.value["data"]["p"]["data"] if NIM_IS_V2 else self.value["data"]["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["data"]["len"] if NIM_IS_V2 else self.value["data"]["Sup"]["len"] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + + tuple_len = int(self.get_len().unsigned) + tuple = self.get_data() + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el[0].unsigned) + if field0 == 0: + continue + key = el[1] + val = el[2] + key_summary = key.summary + child = self.value.CreateValueFromAddress(key_summary, val.GetLoadAddress(), val.GetType()) + self.child_list.append(child) + self.children[key_summary] = index + index += 1 + + def has_children(self): + return bool(self.num_children()) + + +class StringTableChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def get_data(self) -> lldb.SBValue: + return self.value["data"]["p"]["data"] if NIM_IS_V2 else self.value["data"]["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["data"]["len"] if NIM_IS_V2 else self.value["data"]["Sup"]["len"] + + def update(self): + self.children.clear() + self.child_list = [] + + if is_local(self.value): + if not is_in_scope(self.value): + return + + tuple_len = int(self.get_len().unsigned) + tuple = self.get_data() + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el[2].unsigned) + if field0 == 0: + continue + key = el[0] + val = el[1] + child = val.CreateValueFromAddress(key.summary, val.GetLoadAddress(), val.GetType()) + self.child_list.append(child) + self.children[key.summary] = index + index += 1 + + self.child_list.append(self.value["mode"]) + self.children["mode"] = index + + def has_children(self): + return bool(self.num_children()) + + +class LLDBDynamicObjectProvider: + def __init__(self, value: lldb.SBValue, internalDict): + value = value.GetNonSyntheticValue() + self.value: lldb.SBValue = value[0] + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + + while self.value.type.is_pointer: + self.value = self.value.Dereference() + + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.children.clear() + self.child_list = [] + + for i, child in enumerate(self.value.children): + name = child.name.strip('"') + new_child = child.CreateValueFromAddress(name, child.GetLoadAddress(), child.GetType()) + + self.children[name] = i + self.child_list.append(new_child) + + def has_children(self): + return bool(self.num_children()) + + +class LLDBBasicObjectProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value: lldb.SBValue = value + + def num_children(self): + if self.value is not None: + return self.value.num_children + return 0 + + def get_child_index(self, name: str): + return self.value.GetIndexOfChildWithName(name) + + def get_child_at_index(self, index): + return self.value.GetChildAtIndex(index) + + def update(self): + pass + + def has_children(self): + return self.num_children() > 0 + + +class CustomObjectChildrenProvider: + """ + This children provider handles values returned from lldbDebugSynthetic* + Nim procedures + """ + + def __init__(self, value: lldb.SBValue, internalDict): + self.value: lldb.SBValue = get_custom_synthetic(value) or value + if "lldbdynamicobject" in self.value.type.name.lower(): + self.provider = LLDBDynamicObjectProvider(self.value, internalDict) + else: + self.provider = LLDBBasicObjectProvider(self.value, internalDict) + + def num_children(self): + return self.provider.num_children() + + def get_child_index(self, name: str): + return self.provider.get_child_index(name) + + def get_child_at_index(self, index): + return self.provider.get_child_at_index(index) + + def update(self): + self.provider.update() + + def has_children(self): + return self.provider.has_children() + + +def echo(debugger: lldb.SBDebugger, command: str, result, internal_dict): + debugger.HandleCommand("po " + command) + + +SUMMARY_FUNCTIONS: dict[str, lldb.SBFunction] = {} +SYNTHETIC_FUNCTIONS: dict[str, lldb.SBFunction] = {} + + +def get_custom_summary(value: lldb.SBValue) -> Union[str, None]: + """Get a custom summary if a function exists for it""" + value = value.GetNonSyntheticValue() + if value.GetAddress().GetOffset() == 0: + return None + + base_type = get_base_type(value.type) + + fn = SUMMARY_FUNCTIONS.get(base_type.name) + if fn is None: + return None + + fn_type: lldb.SBType = fn.type + + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + first_type = arg_types.GetTypeAtIndex(0) + + while value.type.is_pointer: + value = value.Dereference() + + if first_type.is_pointer: + command = f"{fn.name}(({first_type.name})" + str(value.GetLoadAddress()) + ");" + else: + command = f"{fn.name}(*({first_type.GetPointerType().name})" + str(value.GetLoadAddress()) + ");" + + res = executeCommand(command) + + if res.error.fail: + return None + + return res.summary.strip('"') + + +def get_custom_value_summary(value: lldb.SBValue) -> Union[str, None]: + """Get a custom summary if a function exists for it""" + + fn: lldb.SBFunction = SUMMARY_FUNCTIONS.get(value.type.name) + if fn is None: + return None + + command = f"{fn.name}(({value.type.name})" + str(value.signed) + ");" + res = executeCommand(command) + + if res.error.fail: + return None + + return res.summary.strip('"') + + +def get_custom_synthetic(value: lldb.SBValue) -> Union[lldb.SBValue, None]: + """Get a custom synthetic object if a function exists for it""" + value = value.GetNonSyntheticValue() + if value.GetAddress().GetOffset() == 0: + return None + + base_type = get_base_type(value.type) + + fn = SYNTHETIC_FUNCTIONS.get(base_type.name) + if fn is None: + return None + + fn_type: lldb.SBType = fn.type + + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + first_type = arg_types.GetTypeAtIndex(0) + + while value.type.is_pointer: + value = value.Dereference() + + if first_type.is_pointer: + first_arg = f"({first_type.name}){value.GetLoadAddress()}" + else: + first_arg = f"*({first_type.GetPointerType().name}){value.GetLoadAddress()}" + + if arg_types.GetSize() > 1 and fn.GetArgumentName(1) == "Result": + ret_type = arg_types.GetTypeAtIndex(1) + ret_type = get_base_type(ret_type) + + command = f""" + {ret_type.name} lldbT; + nimZeroMem((void*)(&lldbT), sizeof({ret_type.name})); + {fn.name}(({first_arg}), (&lldbT)); + lldbT; + """ + else: + command = f"{fn.name}({first_arg});" + + res = executeCommand(command) + + if res.error.fail: + print(res.error) + return None + + return res + + +def get_base_type(ty: lldb.SBType) -> lldb.SBType: + """Get the base type of the type""" + temp = ty + while temp.IsPointerType(): + temp = temp.GetPointeeType() + return temp + + +def use_base_type(ty: lldb.SBType) -> bool: + types_to_check = [ + "NF", + "NF32", + "NF64", + "NI", + "NI8", + "NI16", + "NI32", + "NI64", + "bool", + "NIM_BOOL", + "NU", + "NU8", + "NU16", + "NU32", + "NU64", + ] + + for type_to_check in types_to_check: + if ty.name.startswith(type_to_check): + return False + + return True + + +def breakpoint_function_wrapper(frame: lldb.SBFrame, bp_loc, internal_dict): + """This allows function calls to Nim for custom object summaries and synthetic children""" + debugger = lldb.debugger + + global SUMMARY_FUNCTIONS + global SYNTHETIC_FUNCTIONS + + global NIM_IS_V2 + + for tname, fn in SYNTHETIC_FUNCTIONS.items(): + debugger.HandleCommand(f"type synthetic delete -w nim {tname}") + + SUMMARY_FUNCTIONS = {} + SYNTHETIC_FUNCTIONS = {} + + target: lldb.SBTarget = debugger.GetSelectedTarget() + + NIM_IS_V2 = target.FindFirstType("TNimTypeV2").IsValid() + + module = frame.GetSymbolContext(lldb.eSymbolContextModule).module + + for sym in module: + if ( + not sym.name.startswith("lldbDebugSummary") + and not sym.name.startswith("lldbDebugSynthetic") + and not sym.name.startswith("dollar___") + ): + continue + + fn_syms: lldb.SBSymbolContextList = target.FindFunctions(sym.name) + if not fn_syms.GetSize() > 0: + continue + + fn_sym: lldb.SBSymbolContext = fn_syms.GetContextAtIndex(0) + + fn: lldb.SBFunction = fn_sym.function + fn_type: lldb.SBType = fn.type + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + + if arg_types.GetSize() > 1 and fn.GetArgumentName(1) == "Result": + pass # don't continue + elif arg_types.GetSize() != 1: + continue + + arg_type: lldb.SBType = arg_types.GetTypeAtIndex(0) + if use_base_type(arg_type): + arg_type = get_base_type(arg_type) + + if sym.name.startswith("lldbDebugSummary") or sym.name.startswith("dollar___"): + SUMMARY_FUNCTIONS[arg_type.name] = fn + elif sym.name.startswith("lldbDebugSynthetic"): + SYNTHETIC_FUNCTIONS[arg_type.name] = fn + debugger.HandleCommand( + f"type synthetic add -w nim -l {__name__}.CustomObjectChildrenProvider {arg_type.name}" + ) + + +def executeCommand(command, *args): + debugger = lldb.debugger + process = debugger.GetSelectedTarget().GetProcess() + frame: lldb.SBFrame = process.GetSelectedThread().GetSelectedFrame() + + expr_options = lldb.SBExpressionOptions() + expr_options.SetIgnoreBreakpoints(False) + expr_options.SetFetchDynamicValue(lldb.eDynamicCanRunTarget) + expr_options.SetTimeoutInMicroSeconds(30 * 1000 * 1000) # 30 second timeout + expr_options.SetTryAllThreads(True) + expr_options.SetUnwindOnError(False) + expr_options.SetGenerateDebugInfo(True) + expr_options.SetLanguage(lldb.eLanguageTypeC) + expr_options.SetCoerceResultToId(True) + res = frame.EvaluateExpression(command, expr_options) + + return res + + +def __lldb_init_module(debugger, internal_dict): + # fmt: off + print("internal_dict: ", internal_dict.keys()) + debugger.HandleCommand(f"breakpoint command add -F {__name__}.breakpoint_function_wrapper --script-type python 1") + debugger.HandleCommand(f"type summary add -w nim -n sequence -F {__name__}.Sequence -x tySequence_+[[:alnum:]]+$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SeqChildrenProvider -x tySequence_+[[:alnum:]]+$") + + debugger.HandleCommand(f"type summary add -w nim -n chararray -F {__name__}.CharArray -x char\s+[\d+]") + debugger.HandleCommand(f"type summary add -w nim -n array -F {__name__}.Array -x tyArray_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.ArrayChildrenProvider -x tyArray_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n string -F {__name__}.NimString NimStringDesc") + + debugger.HandleCommand(f"type summary add -w nim -n stringv2 -F {__name__}.NimString -x NimStringV2$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringChildrenProvider -x NimStringV2$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringChildrenProvider -x NimStringDesc$") + + debugger.HandleCommand(f"type summary add -w nim -n cstring -F {__name__}.NCSTRING NCSTRING") + + debugger.HandleCommand(f"type summary add -w nim -n object -F {__name__}.ObjectV2 -x tyObject_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.ObjectChildrenProvider -x tyObject_+[[:alnum:]]+_+[[:alnum:]]+$") + + debugger.HandleCommand(f"type summary add -w nim -n tframe -F {__name__}.ObjectV2 -x TFrame$") + + debugger.HandleCommand(f"type summary add -w nim -n rootobj -F {__name__}.ObjectV2 -x RootObj$") + + debugger.HandleCommand(f"type summary add -w nim -n enum -F {__name__}.Enum -x tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n hashset -F {__name__}.HashSet -x tyObject_+HashSet_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.HashSetChildrenProvider -x tyObject_+HashSet_+[[:alnum:]]+") + + debugger.HandleCommand(f"type summary add -w nim -n rope -F {__name__}.Rope -x tyObject_+Rope[[:alnum:]]+_+[[:alnum:]]+") + + debugger.HandleCommand(f"type summary add -w nim -n setuint -F {__name__}.Set -x tySet_+tyInt_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetIntChildrenProvider -x tySet_+tyInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setint -F {__name__}.Set -x tySet_+tyInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setuint2 -F {__name__}.Set -x tySet_+tyUInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyUInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyInt_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setenum -F {__name__}.EnumSet -x tySet_+tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetEnumChildrenProvider -x tySet_+tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setchar -F {__name__}.Set -x tySet_+tyChar_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetCharChildrenProvider -x tySet_+tyChar_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n table -F {__name__}.Table -x tyObject_+Table_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.TableChildrenProvider -x tyObject_+Table_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n stringtable -F {__name__}.StringTable -x tyObject_+StringTableObj_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringTableChildrenProvider -x tyObject_+StringTableObj_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n tuple2 -F {__name__}.Tuple -x tyObject_+Tuple_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n tuple -F {__name__}.Tuple -x tyTuple_+[[:alnum:]]+") + + debugger.HandleCommand(f"type summary add -w nim -n float -F {__name__}.Float NF") + debugger.HandleCommand(f"type summary add -w nim -n float32 -F {__name__}.Float NF32") + debugger.HandleCommand(f"type summary add -w nim -n float64 -F {__name__}.Float NF64") + debugger.HandleCommand(f"type summary add -w nim -n integer -F {__name__}.Number -x NI") + debugger.HandleCommand(f"type summary add -w nim -n integer8 -F {__name__}.Number -x NI8") + debugger.HandleCommand(f"type summary add -w nim -n integer16 -F {__name__}.Number -x NI16") + debugger.HandleCommand(f"type summary add -w nim -n integer32 -F {__name__}.Number -x NI32") + debugger.HandleCommand(f"type summary add -w nim -n integer64 -F {__name__}.Number -x NI64") + debugger.HandleCommand(f"type summary add -w nim -n bool -F {__name__}.Bool -x bool") + debugger.HandleCommand(f"type summary add -w nim -n bool2 -F {__name__}.Bool -x NIM_BOOL") + debugger.HandleCommand(f"type summary add -w nim -n uinteger -F {__name__}.UnsignedNumber -x NU") + debugger.HandleCommand(f"type summary add -w nim -n uinteger8 -F {__name__}.UnsignedNumber -x NU8") + debugger.HandleCommand(f"type summary add -w nim -n uinteger16 -F {__name__}.UnsignedNumber -x NU16") + debugger.HandleCommand(f"type summary add -w nim -n uinteger32 -F {__name__}.UnsignedNumber -x NU32") + debugger.HandleCommand(f"type summary add -w nim -n uinteger64 -F {__name__}.UnsignedNumber -x NU64") + debugger.HandleCommand("type category enable nim") + debugger.HandleCommand(f"command script add -f {__name__}.echo echo") + # fmt: on diff --git a/tests/testCodex.nim b/tests/testCodex.nim index 4f59e747..0c1d138f 100644 --- a/tests/testCodex.nim +++ b/tests/testCodex.nim @@ -15,5 +15,6 @@ import ./codex/testclock import ./codex/testsystemclock import ./codex/testvalidation import ./codex/testasyncstreamwrapper +import ./codex/testmerkletree {.warning[UnusedImport]: off.} diff --git a/vendor/asynctest b/vendor/asynctest index a236a5f0..fe1a34ca 160000 --- a/vendor/asynctest +++ b/vendor/asynctest @@ -1 +1 @@ -Subproject commit a236a5f0f3031573ac2cb082b63dbf6e170e06e7 +Subproject commit fe1a34caf572b05f8bdba3b650f1871af9fce31e diff --git a/vendor/atlas.workspace b/vendor/atlas.workspace new file mode 100644 index 00000000..812bfb2d --- /dev/null +++ b/vendor/atlas.workspace @@ -0,0 +1,3 @@ +deps="" +resolver="MaxVer" +overrides="urls.rules" diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth index 30affa0d..230e7276 160000 --- a/vendor/codex-contracts-eth +++ b/vendor/codex-contracts-eth @@ -1 +1 @@ -Subproject commit 30affa0da85985f6dc90b62f6293de46a9e26130 +Subproject commit 230e7276e271ce53bce36fffdbb25a50621c33b9 diff --git a/vendor/nim-bearssl b/vendor/nim-bearssl index 9ee8b136..99fcb340 160000 --- a/vendor/nim-bearssl +++ b/vendor/nim-bearssl @@ -1 +1 @@ -Subproject commit 9ee8b136e32b2995bad0b16b597d6b50485a9078 +Subproject commit 99fcb3405c55b27cfffbf60f5368c55da7346f23 diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles index 7631f7b2..c9c8e58e 160000 --- a/vendor/nim-chronicles +++ b/vendor/nim-chronicles @@ -1 +1 @@ -Subproject commit 7631f7b2ee03398cb1512a79923264e8f9410af6 +Subproject commit c9c8e58ec3f89b655a046c485f622f9021c68b61 diff --git a/vendor/nim-chronos b/vendor/nim-chronos index 6525f4ce..0277b65b 160000 --- a/vendor/nim-chronos +++ b/vendor/nim-chronos @@ -1 +1 @@ -Subproject commit 6525f4ce1d1a7eba146e5f1a53f6f105077ae686 +Subproject commit 0277b65be2c7a365ac13df002fba6e172be55537 diff --git a/vendor/nim-eth b/vendor/nim-eth index 5885f638..15a09fab 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit 5885f638e47b8607683ef9e1e77fc21ce1aede44 +Subproject commit 15a09fab737d08a2545284c727199c377bb0f4b7 diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 0321e6d7..9f4f762e 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 0321e6d7bd9c703c9e9bf31ee8664adac1d6cbe7 +Subproject commit 9f4f762e21b433aa31549964d723f47d45da7990 diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams index 1b561a9e..720fc5e5 160000 --- a/vendor/nim-faststreams +++ b/vendor/nim-faststreams @@ -1 +1 @@ -Subproject commit 1b561a9e71b6bdad1c1cdff753418906037e9d09 +Subproject commit 720fc5e5c8e428d9d0af618e1e27c44b42350309 diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils index e88e231d..3b491a40 160000 --- a/vendor/nim-http-utils +++ b/vendor/nim-http-utils @@ -1 +1 @@ -Subproject commit e88e231dfcef4585fe3b2fbd9b664dbd28a88040 +Subproject commit 3b491a40c60aad9e8d3407443f46f62511e63b18 diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc index 5a281760..0bf2bcbe 160000 --- a/vendor/nim-json-rpc +++ b/vendor/nim-json-rpc @@ -1 +1 @@ -Subproject commit 5a281760803907f4989cacf109b516381dfbbe11 +Subproject commit 0bf2bcbe74a18a3c7a709d57108bb7b51e748a92 diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index e5b18fb7..bb53d49c 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4 +Subproject commit bb53d49caf2a6c6cf1df365ba84af93cdcfa7aa3 diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index 8c2eca18..440461b2 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit 8c2eca18dcc538c57a8fbc0c53fd0b9d24d56cff +Subproject commit 440461b24b9e66542b34d26a0b908c17f6549d05 diff --git a/vendor/nim-libp2p-dht b/vendor/nim-libp2p-dht index bd517f0e..3c940ea8 160000 --- a/vendor/nim-libp2p-dht +++ b/vendor/nim-libp2p-dht @@ -1 +1 @@ -Subproject commit bd517f0e8da38a1b5da15f7deb2d5c652ca389f1 +Subproject commit 3c940ea8901ae6118e66cc4df423b8ff53699eb4 diff --git a/vendor/nim-metrics b/vendor/nim-metrics index 743f81d4..6142e433 160000 --- a/vendor/nim-metrics +++ b/vendor/nim-metrics @@ -1 +1 @@ -Subproject commit 743f81d4f6c6ebf0ac02389f2392ff8b4235bee5 +Subproject commit 6142e433fc8ea9b73379770a788017ac528d46ff diff --git a/vendor/nim-protobuf-serialization b/vendor/nim-protobuf-serialization new file mode 160000 index 00000000..28214b3e --- /dev/null +++ b/vendor/nim-protobuf-serialization @@ -0,0 +1 @@ +Subproject commit 28214b3e40c755a9886d2ec8f261ec48fbb6bec6 diff --git a/vendor/nim-results b/vendor/nim-results new file mode 160000 index 00000000..f3c666a2 --- /dev/null +++ b/vendor/nim-results @@ -0,0 +1 @@ +Subproject commit f3c666a272c69d70cb41e7245e7f6844797303ad diff --git a/vendor/nim-secp256k1 b/vendor/nim-secp256k1 index 5340cf18..2acbbdcc 160000 --- a/vendor/nim-secp256k1 +++ b/vendor/nim-secp256k1 @@ -1 +1 @@ -Subproject commit 5340cf188168d6afcafc8023770d880f067c0b2f +Subproject commit 2acbbdcc0e63002a013fff49f015708522875832 diff --git a/vendor/nim-serialization b/vendor/nim-serialization index 493d18b8..384eb256 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit 493d18b8292fc03aa4f835fd825dea1183f97466 +Subproject commit 384eb2561ee755446cff512a8e057325848b86a7 diff --git a/vendor/nim-sqlite3-abi b/vendor/nim-sqlite3-abi index fda455cf..362e1bd9 160000 --- a/vendor/nim-sqlite3-abi +++ b/vendor/nim-sqlite3-abi @@ -1 +1 @@ -Subproject commit fda455cfea2df707dde052034411ce63de218453 +Subproject commit 362e1bd9f689ad9f5380d9d27f0705b3d4dfc7d3 diff --git a/vendor/nim-stew b/vendor/nim-stew index e18f5a62..7afe7e3c 160000 --- a/vendor/nim-stew +++ b/vendor/nim-stew @@ -1 +1 @@ -Subproject commit e18f5a62af2ade7a1fd1d39635d4e04d944def08 +Subproject commit 7afe7e3c070758cac1f628e4330109f3ef6fc853 diff --git a/vendor/nim-testutils b/vendor/nim-testutils new file mode 160000 index 00000000..b56a5953 --- /dev/null +++ b/vendor/nim-testutils @@ -0,0 +1 @@ +Subproject commit b56a5953e37fc5117bd6ea6dfa18418c5e112815 diff --git a/vendor/nim-websock b/vendor/nim-websock index 7b2ed397..2c3ae313 160000 --- a/vendor/nim-websock +++ b/vendor/nim-websock @@ -1 +1 @@ -Subproject commit 7b2ed397d6e4c37ea4df08ae82aeac7ff04cd180 +Subproject commit 2c3ae3137f3c9cb48134285bd4a47186fa51f0e8 diff --git a/vendor/nim-zlib b/vendor/nim-zlib index 74cdeb54..f34ca261 160000 --- a/vendor/nim-zlib +++ b/vendor/nim-zlib @@ -1 +1 @@ -Subproject commit 74cdeb54b21bededb5a515d36f608bc1850555a2 +Subproject commit f34ca261efd90f118dc1647beefd2f7a69b05d93 diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index 1cf6a1b1..fe9bc3f3 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit 1cf6a1b18ca5aa0d24e7a2861dd85d79ad9cb0cd +Subproject commit fe9bc3f3759ae1add6bf8c899db2e75327f03782 diff --git a/vendor/nimcrypto b/vendor/nimcrypto index a5742a9a..24e006df 160000 --- a/vendor/nimcrypto +++ b/vendor/nimcrypto @@ -1 +1 @@ -Subproject commit a5742a9a214ac33f91615f3862c7b099aec43b00 +Subproject commit 24e006df85927f64916e60511620583b11403178 diff --git a/vendor/npeg b/vendor/npeg new file mode 160000 index 00000000..b15a10e3 --- /dev/null +++ b/vendor/npeg @@ -0,0 +1 @@ +Subproject commit b15a10e388b91b898c581dbbcb6a718d46b27d2f diff --git a/vendor/questionable b/vendor/questionable index 30e4184a..e56cf86c 160000 --- a/vendor/questionable +++ b/vendor/questionable @@ -1 +1 @@ -Subproject commit 30e4184a99c8c1ba329925912d2c5d4b09acf8cc +Subproject commit e56cf86c4a089c78a1b7c3005f13343bfbbe3b48 diff --git a/vendor/stint b/vendor/stint index 036c71d0..86621ece 160000 --- a/vendor/stint +++ b/vendor/stint @@ -1 +1 @@ -Subproject commit 036c71d06a6b22f8f967ba9d54afd2189c3872ca +Subproject commit 86621eced1dcfb5e25903019ebcfc76ed9128ec5 diff --git a/vendor/urls.rules b/vendor/urls.rules new file mode 100644 index 00000000..7636ff34 --- /dev/null +++ b/vendor/urls.rules @@ -0,0 +1,8 @@ +https://github.com/status-im/nim-libp2p-dht.git -> https://github.com/codex-storage/nim-codex-dht.git +https://github.com/markspanbroek/questionable -> https://github.com/codex-storage/questionable +https://github.com/status-im/questionable -> https://github.com/codex-storage/questionable +https://github.com/status-im/asynctest -> https://github.com/codex-storage/asynctest +https://github.com/status-im/nim-datastore -> https://github.com/codex-storage/nim-datastore +https://github.com/cheatfate/nimcrypto -> https://github.com/status-im/nimcrypto +protobufserialization -> protobuf_serialization +protobufserialization -> https://github.com/status-im/nim-protobuf-serialization