From a3f10f7f2079c670810141b802d9b072ccf25f20 Mon Sep 17 00:00:00 2001 From: Mark Spanbroek Date: Thu, 19 May 2022 09:56:12 +0200 Subject: [PATCH 1/7] [ci] disable pull-request comments by codecov --- codecov.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/codecov.yml b/codecov.yml index 429a94c8..b0dd0256 100644 --- a/codecov.yml +++ b/codecov.yml @@ -20,3 +20,4 @@ coverage: # `informational`: https://docs.codecov.com/docs/commit-status#informational # `threshold`: https://docs.codecov.com/docs/commit-status#threshold informational: true +comment: false From 209343087c4acb00c5f9d9c448bff434146ccf10 Mon Sep 17 00:00:00 2001 From: Michael Bradley Date: Thu, 19 May 2022 14:56:03 -0500 Subject: [PATCH 2/7] Change every dagger to codex (#102) --- .github/workflows/ci.yml | 10 +++--- .gitignore | 2 +- Makefile | 26 +++++++-------- dagger.nim => codex.nim | 16 +++++----- dagger.nimble => codex.nimble | 20 ++++++------ {dagger => codex}/blockexchange.nim | 0 {dagger => codex}/blockexchange/engine.nim | 4 +-- .../blockexchange/engine/discovery.nim | 4 +-- .../blockexchange/engine/payments.nim | 0 {dagger => codex}/blockexchange/network.nim | 6 ++-- .../blockexchange/networkpeer.nim | 4 +-- .../blockexchange/peercontext.nim | 0 {dagger => codex}/blockexchange/peers.nim | 0 .../blockexchange/peers/peerctxstore.nim | 4 +-- .../blockexchange/pendingblocks.nim | 4 +-- .../blockexchange/protobuf/blockexc.nim | 2 +- .../blockexchange/protobuf/message.proto | 0 .../protobuf/message.proto.license | 0 .../blockexchange/protobuf/payments.nim | 0 .../blockexchange/protobuf/presence.nim | 0 {dagger => codex}/blocktype.nim | 2 +- {dagger => codex}/chunker.nim | 2 +- {dagger => codex}/clock.nim | 0 dagger/dagger.nim => codex/codex.nim | 24 +++++++------- {dagger => codex}/conf.nim | 24 +++++++------- {dagger => codex}/contracts.nim | 0 {dagger => codex}/contracts/Readme.md | 10 +++--- {dagger => codex}/contracts/clock.nim | 0 {dagger => codex}/contracts/deployment.nim | 0 {dagger => codex}/contracts/interactions.nim | 0 {dagger => codex}/contracts/market.nim | 0 {dagger => codex}/contracts/offers.nim | 0 {dagger => codex}/contracts/proofs.nim | 0 {dagger => codex}/contracts/requests.nim | 0 {dagger => codex}/contracts/storage.nim | 0 {dagger => codex}/contracts/testtoken.nim | 0 {dagger => codex}/discovery.nim | 2 +- {dagger => codex}/erasure.nim | 2 +- {dagger => codex}/erasure/backend.nim | 2 +- .../erasure/backends/leopard.nim | 2 +- {dagger => codex}/erasure/erasure.nim | 4 +-- {dagger => codex}/errors.nim | 8 ++--- {dagger => codex}/manifest.nim | 0 {dagger => codex}/manifest/coders.nim | 2 +- {dagger => codex}/manifest/manifest.nim | 2 +- {dagger => codex}/manifest/types.nim | 2 +- {dagger => codex}/market.nim | 0 {dagger => codex}/node.nim | 32 +++++++++---------- {dagger => codex}/por/README.md | 0 {dagger => codex}/por/benchmark.nim | 0 {dagger => codex}/por/example.txt | 0 {dagger => codex}/por/por.nim | 4 +-- {dagger => codex}/por/testpor.nim | 0 {dagger => codex}/por/timing/periods.nim | 0 {dagger => codex}/por/timing/proofs.nim | 0 {dagger => codex}/proving.nim | 0 {dagger => codex}/purchasing.nim | 0 {dagger => codex}/rest/api.nim | 20 ++++++------ {dagger => codex}/rest/json.nim | 0 {dagger => codex}/rng.nim | 2 +- {dagger => codex}/sales.nim | 0 {dagger => codex}/stores.nim | 0 {dagger => codex}/stores/blockstore.nim | 2 +- {dagger => codex}/stores/cachestore.nim | 6 ++-- {dagger => codex}/stores/fsstore.nim | 4 +-- {dagger => codex}/stores/networkstore.nim | 4 +-- {dagger => codex}/streams.nim | 0 {dagger => codex}/streams/seekablestream.nim | 4 +-- {dagger => codex}/streams/storestream.nim | 4 +-- {dagger => codex}/utils.nim | 0 {dagger => codex}/utils/asyncheapqueue.nim | 2 +- {dagger => codex}/utils/fileutils.nim | 2 +- config.nims | 4 +-- .../blockexc/discovery/testdiscovery.nim | 10 +++--- .../discovery/testdiscoveryengine.nim | 12 +++---- .../blockexc/engine/testpayments.nim | 2 +- .../blockexc/protobuf/testpayments.nim | 2 +- .../blockexc/protobuf/testpresence.nim | 2 +- .../blockexc/testblockexc.nim | 12 +++---- .../{dagger => codex}/blockexc/testengine.nim | 14 ++++---- .../blockexc/testnetwork.nim | 8 ++--- .../blockexc/testpeerctxstore.nim | 4 +-- tests/{dagger => codex}/examples.nim | 8 ++--- tests/{dagger => codex}/helpers.nim | 2 +- tests/{dagger => codex}/helpers/mockclock.nim | 2 +- .../helpers/mockdiscovery.nim | 4 +-- .../{dagger => codex}/helpers/mockmarket.nim | 2 +- .../{dagger => codex}/helpers/mockproofs.nim | 2 +- tests/{dagger => codex}/helpers/nodeutils.nim | 8 ++--- .../helpers/randomchunker.nim | 4 +-- .../stores/testcachestore.nim | 4 +-- .../{dagger => codex}/stores/testfsstore.nim | 8 ++--- .../{dagger => codex}/testasyncheapqueue.nim | 4 +-- tests/{dagger => codex}/testblockexc.nim | 0 tests/{dagger => codex}/testchunking.nim | 2 +- tests/{dagger => codex}/testerasure.nim | 10 +++--- tests/{dagger => codex}/testmanifest.nim | 6 ++-- tests/{dagger => codex}/testnode.nim | 20 ++++++------ tests/{dagger => codex}/testproving.nim | 2 +- tests/{dagger => codex}/testpurchasing.nim | 2 +- tests/{dagger => codex}/testsales.nim | 2 +- tests/{dagger => codex}/teststores.nim | 0 tests/{dagger => codex}/teststorestream.nim | 10 +++--- tests/contracts/examples.nim | 2 +- tests/contracts/testClock.nim | 2 +- tests/contracts/testCollateral.nim | 4 +-- tests/contracts/testContracts.nim | 6 ++-- tests/contracts/testInteractions.nim | 2 +- tests/contracts/testMarket.nim | 4 +-- tests/contracts/testProofs.nim | 2 +- tests/integration/nodes.nim | 4 +-- tests/integration/tokens.nim | 4 +-- tests/testCodex.nim | 16 ++++++++++ .../{testDagger.nim.cfg => testCodex.nim.cfg} | 0 tests/testDagger.nim | 16 ---------- tests/testIntegration.nim | 4 +-- 116 files changed, 257 insertions(+), 257 deletions(-) rename dagger.nim => codex.nim (88%) rename dagger.nimble => codex.nimble (83%) rename {dagger => codex}/blockexchange.nim (100%) rename {dagger => codex}/blockexchange/engine.nim (99%) rename {dagger => codex}/blockexchange/engine/discovery.nim (99%) rename {dagger => codex}/blockexchange/engine/payments.nim (100%) rename {dagger => codex}/blockexchange/network.nim (99%) rename {dagger => codex}/blockexchange/networkpeer.nim (97%) rename {dagger => codex}/blockexchange/peercontext.nim (100%) rename {dagger => codex}/blockexchange/peers.nim (100%) rename {dagger => codex}/blockexchange/peers/peerctxstore.nim (97%) rename {dagger => codex}/blockexchange/pendingblocks.nim (97%) rename {dagger => codex}/blockexchange/protobuf/blockexc.nim (98%) rename {dagger => codex}/blockexchange/protobuf/message.proto (100%) rename {dagger => codex}/blockexchange/protobuf/message.proto.license (100%) rename {dagger => codex}/blockexchange/protobuf/payments.nim (100%) rename {dagger => codex}/blockexchange/protobuf/presence.nim (100%) rename {dagger => codex}/blocktype.nim (99%) rename {dagger => codex}/chunker.nim (99%) rename {dagger => codex}/clock.nim (100%) rename dagger/dagger.nim => codex/codex.nim (89%) rename {dagger => codex}/conf.nim (94%) rename {dagger => codex}/contracts.nim (100%) rename {dagger => codex}/contracts/Readme.md (94%) rename {dagger => codex}/contracts/clock.nim (100%) rename {dagger => codex}/contracts/deployment.nim (100%) rename {dagger => codex}/contracts/interactions.nim (100%) rename {dagger => codex}/contracts/market.nim (100%) rename {dagger => codex}/contracts/offers.nim (100%) rename {dagger => codex}/contracts/proofs.nim (100%) rename {dagger => codex}/contracts/requests.nim (100%) rename {dagger => codex}/contracts/storage.nim (100%) rename {dagger => codex}/contracts/testtoken.nim (100%) rename {dagger => codex}/discovery.nim (99%) rename {dagger => codex}/erasure.nim (97%) rename {dagger => codex}/erasure/backend.nim (98%) rename {dagger => codex}/erasure/backends/leopard.nim (99%) rename {dagger => codex}/erasure/erasure.nim (99%) rename {dagger => codex}/errors.nim (75%) rename {dagger => codex}/manifest.nim (100%) rename {dagger => codex}/manifest/coders.nim (99%) rename {dagger => codex}/manifest/manifest.nim (99%) rename {dagger => codex}/manifest/types.nim (99%) rename {dagger => codex}/market.nim (100%) rename {dagger => codex}/node.nim (92%) rename {dagger => codex}/por/README.md (100%) rename {dagger => codex}/por/benchmark.nim (100%) rename {dagger => codex}/por/example.txt (100%) rename {dagger => codex}/por/por.nim (99%) rename {dagger => codex}/por/testpor.nim (100%) rename {dagger => codex}/por/timing/periods.nim (100%) rename {dagger => codex}/por/timing/proofs.nim (100%) rename {dagger => codex}/proving.nim (100%) rename {dagger => codex}/purchasing.nim (100%) rename {dagger => codex}/rest/api.nim (94%) rename {dagger => codex}/rest/json.nim (100%) rename {dagger => codex}/rng.nim (99%) rename {dagger => codex}/sales.nim (100%) rename {dagger => codex}/stores.nim (100%) rename {dagger => codex}/stores/blockstore.nim (99%) rename {dagger => codex}/stores/cachestore.nim (97%) rename {dagger => codex}/stores/fsstore.nim (98%) rename {dagger => codex}/stores/networkstore.nim (97%) rename {dagger => codex}/streams.nim (100%) rename {dagger => codex}/streams/seekablestream.nim (92%) rename {dagger => codex}/streams/storestream.nim (97%) rename {dagger => codex}/utils.nim (100%) rename {dagger => codex}/utils/asyncheapqueue.nim (99%) rename {dagger => codex}/utils/fileutils.nim (99%) rename tests/{dagger => codex}/blockexc/discovery/testdiscovery.nim (98%) rename tests/{dagger => codex}/blockexc/discovery/testdiscoveryengine.nim (97%) rename tests/{dagger => codex}/blockexc/engine/testpayments.nim (97%) rename tests/{dagger => codex}/blockexc/protobuf/testpayments.nim (97%) rename tests/{dagger => codex}/blockexc/protobuf/testpresence.nim (96%) rename tests/{dagger => codex}/blockexc/testblockexc.nim (97%) rename tests/{dagger => codex}/blockexc/testengine.nim (98%) rename tests/{dagger => codex}/blockexc/testnetwork.nim (98%) rename tests/{dagger => codex}/blockexc/testpeerctxstore.nim (96%) rename tests/{dagger => codex}/examples.nim (93%) rename tests/{dagger => codex}/helpers.nim (95%) rename tests/{dagger => codex}/helpers/mockclock.nim (95%) rename tests/{dagger => codex}/helpers/mockdiscovery.nim (97%) rename tests/{dagger => codex}/helpers/mockmarket.nim (99%) rename tests/{dagger => codex}/helpers/mockproofs.nim (98%) rename tests/{dagger => codex}/helpers/nodeutils.nim (93%) rename tests/{dagger => codex}/helpers/randomchunker.nim (94%) rename tests/{dagger => codex}/stores/testcachestore.nim (98%) rename tests/{dagger => codex}/stores/testfsstore.nim (93%) rename tests/{dagger => codex}/testasyncheapqueue.nim (98%) rename tests/{dagger => codex}/testblockexc.nim (100%) rename tests/{dagger => codex}/testchunking.nim (98%) rename tests/{dagger => codex}/testerasure.nim (97%) rename tests/{dagger => codex}/testmanifest.nim (96%) rename tests/{dagger => codex}/testnode.nim (90%) rename tests/{dagger => codex}/testproving.nim (99%) rename tests/{dagger => codex}/testpurchasing.nim (99%) rename tests/{dagger => codex}/testsales.nim (99%) rename tests/{dagger => codex}/teststores.nim (100%) rename tests/{dagger => codex}/teststorestream.nim (94%) create mode 100644 tests/testCodex.nim rename tests/{testDagger.nim.cfg => testCodex.nim.cfg} (100%) delete mode 100644 tests/testDagger.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e9ed94a..34f58414 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,12 +209,12 @@ jobs: make NIM_COMMIT="${{ matrix.branch }}" testAll if [[ ${{ runner.os }} = macOS ]]; then echo - echo otool -L build/testDagger - otool -L build/testDagger + echo otool -L build/testCodex + otool -L build/testCodex else echo - echo ldd build/testDagger - ldd build/testDagger + echo ldd build/testCodex + ldd build/testCodex fi echo "Testing TRACE log level" - ./env.sh nim c -d:chronicles_log_level=TRACE dagger.nim + ./env.sh nim c -d:chronicles_log_level=TRACE codex.nim diff --git a/.gitignore b/.gitignore index c9bad6d9..b1c8f7e0 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,4 @@ nimble.paths .env .update.timestamp -dagger.nims +codex.nims diff --git a/Makefile b/Makefile index b6bdb8c7..30adaf84 100644 --- a/Makefile +++ b/Makefile @@ -56,15 +56,15 @@ else NIM_PARAMS := $(NIM_PARAMS) -d:release endif -deps: | deps-common nat-libs dagger.nims +deps: | deps-common nat-libs codex.nims ifneq ($(USE_LIBBACKTRACE), 0) deps: | libbacktrace endif -#- deletes and recreates "dagger.nims" which on Windows is a copy instead of a proper symlink +#- deletes and recreates "codex.nims" which on Windows is a copy instead of a proper symlink update: | update-common - rm -rf dagger.nims && \ - $(MAKE) dagger.nims $(HANDLE_OUTPUT) + rm -rf codex.nims && \ + $(MAKE) codex.nims $(HANDLE_OUTPUT) # detecting the os ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... @@ -79,16 +79,16 @@ endif # Builds and run a part of the test suite test: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim test $(NIM_PARAMS) dagger.nims + $(ENV_SCRIPT) nim test $(NIM_PARAMS) codex.nims # Builds and runs all tests testAll: | build deps - echo -e $(BUILD_MSG) "build/testDagger" "build/testContracts" && \ - $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) dagger.nims + echo -e $(BUILD_MSG) "build/testCodex" "build/testContracts" && \ + $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) codex.nims # symlink -dagger.nims: - ln -s dagger.nimble $@ +codex.nims: + ln -s codex.nimble $@ # nim-libbacktrace libbacktrace: @@ -96,12 +96,12 @@ libbacktrace: coverage: $(MAKE) NIMFLAGS="--lineDir:on --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage" testAll - cd nimcache/release/testDagger && rm -f *.c + cd nimcache/release/testCodex && rm -f *.c cd nimcache/release/testContracts && rm -f *.c mkdir -p coverage - lcov --capture --directory nimcache/release/testDagger --directory nimcache/release/testContracts --output-file coverage/coverage.info - shopt -s globstar && ls $$(pwd)/dagger/{*,**/*}.nim - shopt -s globstar && lcov --extract coverage/coverage.info $$(pwd)/dagger/{*,**/*}.nim --output-file coverage/coverage.f.info + lcov --capture --directory nimcache/release/testCodex --directory nimcache/release/testContracts --output-file coverage/coverage.info + shopt -s globstar && ls $$(pwd)/codex/{*,**/*}.nim + shopt -s globstar && lcov --extract coverage/coverage.info $$(pwd)/codex/{*,**/*}.nim --output-file coverage/coverage.f.info echo -e $(BUILD_MSG) "coverage/report/index.html" genhtml coverage/coverage.f.info --output-directory coverage/report if which open >/dev/null; then (echo -e "\e[92mOpening\e[39m HTML coverage report in browser..." && open coverage/report/index.html) || true; fi diff --git a/dagger.nim b/codex.nim similarity index 88% rename from dagger.nim rename to codex.nim index 72249041..1003d21a 100644 --- a/dagger.nim +++ b/codex.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -12,23 +12,23 @@ import pkg/chronos import pkg/confutils import pkg/libp2p -import ./dagger/conf -import ./dagger/dagger +import ./codex/conf +import ./codex/codex -export dagger, conf, libp2p, chronos, chronicles +export codex, conf, libp2p, chronos, chronicles when isMainModule: import std/os import pkg/confutils/defs - import ./dagger/utils/fileutils + import ./codex/utils/fileutils when defined(posix): import system/ansi_c - let config = DaggerConf.load( - version = daggerFullVersion + let config = CodexConf.load( + version = codexFullVersion ) config.setupLogging() config.setupMetrics() @@ -50,7 +50,7 @@ when isMainModule: trace "Repo dir initialized", dir = config.dataDir / "repo" - let server = DaggerServer.new(config) + let server = CodexServer.new(config) ## Ctrl+C handling proc controlCHandler() {.noconv.} = diff --git a/dagger.nimble b/codex.nimble similarity index 83% rename from dagger.nimble rename to codex.nimble index 34bbc6cc..d3e017aa 100644 --- a/dagger.nimble +++ b/codex.nimble @@ -1,7 +1,7 @@ mode = ScriptMode.Verbose version = "0.1.0" -author = "Dagger Team" +author = "Codex Team" description = "p2p data durability engine" license = "MIT" binDir = "build" @@ -28,7 +28,7 @@ requires "nim >= 1.2.0", when declared(namedBin): namedBin = { - "dagger/dagger": "dagger" + "codex/codex": "codex" }.toTable() ### Helper functions @@ -45,23 +45,23 @@ proc test(name: string, srcDir = "tests/", lang = "c") = buildBinary name, srcDir exec "build/" & name -task dagger, "build dagger binary": - buildBinary "dagger" +task codex, "build codex binary": + buildBinary "codex" -task testDagger, "Build & run Dagger tests": - test "testDagger" +task testCodex, "Build & run Codex tests": + test "testCodex" -task testContracts, "Build & run Dagger Contract tests": +task testContracts, "Build & run Codex Contract tests": test "testContracts" task testIntegration, "Run integration tests": - daggerTask() + codexTask() test "testIntegration" task test, "Run tests": - testDaggerTask() + testCodexTask() task testAll, "Run all tests": - testDaggerTask() + testCodexTask() testContractsTask() testIntegrationTask() diff --git a/dagger/blockexchange.nim b/codex/blockexchange.nim similarity index 100% rename from dagger/blockexchange.nim rename to codex/blockexchange.nim diff --git a/dagger/blockexchange/engine.nim b/codex/blockexchange/engine.nim similarity index 99% rename from dagger/blockexchange/engine.nim rename to codex/blockexchange/engine.nim index 6a67d964..6223ee13 100644 --- a/dagger/blockexchange/engine.nim +++ b/codex/blockexchange/engine.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -30,7 +30,7 @@ import ./pendingblocks export peers, pendingblocks, payments, discovery logScope: - topics = "dagger blockexc engine" + topics = "codex blockexc engine" const DefaultMaxPeersPerRequest* = 10 diff --git a/dagger/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim similarity index 99% rename from dagger/blockexchange/engine/discovery.nim rename to codex/blockexchange/engine/discovery.nim index aba3ccd0..aeb88557 100644 --- a/dagger/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -25,7 +25,7 @@ import ../../stores/blockstore import ../pendingblocks logScope: - topics = "dagger discovery engine" + topics = "codex discovery engine" const DefaultConcurrentDiscRequests = 10 diff --git a/dagger/blockexchange/engine/payments.nim b/codex/blockexchange/engine/payments.nim similarity index 100% rename from dagger/blockexchange/engine/payments.nim rename to codex/blockexchange/engine/payments.nim diff --git a/dagger/blockexchange/network.nim b/codex/blockexchange/network.nim similarity index 99% rename from dagger/blockexchange/network.nim rename to codex/blockexchange/network.nim index 60fe6a28..0c14e51c 100644 --- a/dagger/blockexchange/network.nim +++ b/codex/blockexchange/network.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -26,9 +26,9 @@ import ./networkpeer export networkpeer, payments logScope: - topics = "dagger blockexc network" + topics = "codex blockexc network" -const Codec* = "/dagger/blockexc/1.0.0" +const Codec* = "/codex/blockexc/1.0.0" type WantListHandler* = proc(peer: PeerID, wantList: WantList): Future[void] {.gcsafe.} diff --git a/dagger/blockexchange/networkpeer.nim b/codex/blockexchange/networkpeer.nim similarity index 97% rename from dagger/blockexchange/networkpeer.nim rename to codex/blockexchange/networkpeer.nim index bf84384a..1c44c3c2 100644 --- a/dagger/blockexchange/networkpeer.nim +++ b/codex/blockexchange/networkpeer.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -15,7 +15,7 @@ import pkg/libp2p import ./protobuf/blockexc logScope: - topics = "dagger blockexc networkpeer" + topics = "codex blockexc networkpeer" const MaxMessageSize = 100 * 1024 * 1024 # manifest files can be big diff --git a/dagger/blockexchange/peercontext.nim b/codex/blockexchange/peercontext.nim similarity index 100% rename from dagger/blockexchange/peercontext.nim rename to codex/blockexchange/peercontext.nim diff --git a/dagger/blockexchange/peers.nim b/codex/blockexchange/peers.nim similarity index 100% rename from dagger/blockexchange/peers.nim rename to codex/blockexchange/peers.nim diff --git a/dagger/blockexchange/peers/peerctxstore.nim b/codex/blockexchange/peers/peerctxstore.nim similarity index 97% rename from dagger/blockexchange/peers/peerctxstore.nim rename to codex/blockexchange/peers/peerctxstore.nim index 9453a6f8..77ad6f60 100644 --- a/dagger/blockexchange/peers/peerctxstore.nim +++ b/codex/blockexchange/peers/peerctxstore.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -25,7 +25,7 @@ import ../peercontext export peercontext logScope: - topics = "dagger blockexc peerctxstore" + topics = "codex blockexc peerctxstore" type PeerCtxStore* = ref object of RootObj diff --git a/dagger/blockexchange/pendingblocks.nim b/codex/blockexchange/pendingblocks.nim similarity index 97% rename from dagger/blockexchange/pendingblocks.nim rename to codex/blockexchange/pendingblocks.nim index f1e30f6c..5539734f 100644 --- a/dagger/blockexchange/pendingblocks.nim +++ b/codex/blockexchange/pendingblocks.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -21,7 +21,7 @@ import pkg/libp2p import ../blocktype logScope: - topics = "dagger blockexc pendingblocks" + topics = "codex blockexc pendingblocks" const DefaultBlockTimeout* = 10.minutes diff --git a/dagger/blockexchange/protobuf/blockexc.nim b/codex/blockexchange/protobuf/blockexc.nim similarity index 98% rename from dagger/blockexchange/protobuf/blockexc.nim rename to codex/blockexchange/protobuf/blockexc.nim index b7b7bf47..326a098a 100644 --- a/dagger/blockexchange/protobuf/blockexc.nim +++ b/codex/blockexchange/protobuf/blockexc.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/blockexchange/protobuf/message.proto b/codex/blockexchange/protobuf/message.proto similarity index 100% rename from dagger/blockexchange/protobuf/message.proto rename to codex/blockexchange/protobuf/message.proto diff --git a/dagger/blockexchange/protobuf/message.proto.license b/codex/blockexchange/protobuf/message.proto.license similarity index 100% rename from dagger/blockexchange/protobuf/message.proto.license rename to codex/blockexchange/protobuf/message.proto.license diff --git a/dagger/blockexchange/protobuf/payments.nim b/codex/blockexchange/protobuf/payments.nim similarity index 100% rename from dagger/blockexchange/protobuf/payments.nim rename to codex/blockexchange/protobuf/payments.nim diff --git a/dagger/blockexchange/protobuf/presence.nim b/codex/blockexchange/protobuf/presence.nim similarity index 100% rename from dagger/blockexchange/protobuf/presence.nim rename to codex/blockexchange/protobuf/presence.nim diff --git a/dagger/blocktype.nim b/codex/blocktype.nim similarity index 99% rename from dagger/blocktype.nim rename to codex/blocktype.nim index ce48d497..b324c2a4 100644 --- a/dagger/blocktype.nim +++ b/codex/blocktype.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/chunker.nim b/codex/chunker.nim similarity index 99% rename from dagger/chunker.nim rename to codex/chunker.nim index 1ecb5cb2..6b0b9cc2 100644 --- a/dagger/chunker.nim +++ b/codex/chunker.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/clock.nim b/codex/clock.nim similarity index 100% rename from dagger/clock.nim rename to codex/clock.nim diff --git a/dagger/dagger.nim b/codex/codex.nim similarity index 89% rename from dagger/dagger.nim rename to codex/codex.nim index f938bd8b..27baa2a2 100644 --- a/dagger/dagger.nim +++ b/codex/codex.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -33,26 +33,26 @@ import ./discovery import ./contracts type - DaggerServer* = ref object + CodexServer* = ref object runHandle: Future[void] - config: DaggerConf + config: CodexConf restServer: RestServerRef - daggerNode: DaggerNodeRef + codexNode: CodexNodeRef -proc start*(s: DaggerServer) {.async.} = +proc start*(s: CodexServer) {.async.} = s.restServer.start() - await s.daggerNode.start() + await s.codexNode.start() s.runHandle = newFuture[void]() await s.runHandle -proc stop*(s: DaggerServer) {.async.} = +proc stop*(s: CodexServer) {.async.} = await allFuturesThrowing( - s.restServer.stop(), s.daggerNode.stop()) + s.restServer.stop(), s.codexNode.stop()) s.runHandle.complete() -proc new*(T: type DaggerServer, config: DaggerConf): T = +proc new*(T: type CodexServer, config: CodexConf): T = const SafePermissions = {UserRead, UserWrite} let @@ -130,9 +130,9 @@ proc new*(T: type DaggerServer, config: DaggerConf): T = config.ethDeployment, config.ethAccount ) - daggerNode = DaggerNodeRef.new(switch, store, engine, erasure, blockDiscovery, contracts) + codexNode = CodexNodeRef.new(switch, store, engine, erasure, blockDiscovery, contracts) restServer = RestServerRef.new( - daggerNode.initRestApi(config), + codexNode.initRestApi(config), initTAddress("127.0.0.1" , config.apiPort), bufferSize = (1024 * 64), maxRequestBodySize = int.high) @@ -141,6 +141,6 @@ proc new*(T: type DaggerServer, config: DaggerConf): T = switch.mount(network) T( config: config, - daggerNode: daggerNode, + codexNode: codexNode, restServer: restServer, ) diff --git a/dagger/conf.nim b/codex/conf.nim similarity index 94% rename from dagger/conf.nim rename to codex/conf.nim index 5ef7f770..6f16adb2 100644 --- a/dagger/conf.nim +++ b/codex/conf.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -44,7 +44,7 @@ type Json = "json" None = "none" - DaggerConf* = object + CodexConf* = object logLevel* {. defaultValue: LogLevel.INFO desc: "Sets the log level", @@ -74,7 +74,7 @@ type name: "metrics-port" }: Port dataDir* {. - desc: "The directory where dagger will store configuration and data." + desc: "The directory where codex will store configuration and data." defaultValue: defaultDataDir() defaultValueDesc: "" abbr: "d" @@ -124,7 +124,7 @@ type name: "max-peers" }: int agentString* {. - defaultValue: "Dagger" + defaultValue: "Codex" desc: "Node agent string which is used as identifier in network" name: "agent-string" }: string @@ -171,20 +171,20 @@ const nimBanner* = staticExec("nim --version | grep Version") #TODO add versionMajor, Minor & Fix when we switch to semver - daggerVersion* = gitRevision + codexVersion* = gitRevision - daggerFullVersion* = - "Dagger build " & daggerVersion & "\p" & + codexFullVersion* = + "Codex build " & codexVersion & "\p" & nimBanner proc defaultDataDir*(): string = let dataDir = when defined(windows): - "AppData" / "Roaming" / "Dagger" + "AppData" / "Roaming" / "Codex" elif defined(macosx): - "Library" / "Application Support" / "Dagger" + "Library" / "Application Support" / "Codex" else: - ".cache" / "dagger" + ".cache" / "codex" getHomeDir() / dataDir @@ -246,7 +246,7 @@ proc stripAnsi(v: string): string = res -proc setupLogging*(conf: DaggerConf) = +proc setupLogging*(conf: CodexConf) = when defaultChroniclesStream.outputs.type.arity != 2: warn "Logging configuration options not enabled in the current build" else: @@ -283,7 +283,7 @@ proc setupLogging*(conf: DaggerConf) = setLogLevel(conf.logLevel) -proc setupMetrics*(config: DaggerConf) = +proc setupMetrics*(config: CodexConf) = if config.metricsEnabled: let metricsAddress = config.metricsAddress notice "Starting metrics HTTP server", diff --git a/dagger/contracts.nim b/codex/contracts.nim similarity index 100% rename from dagger/contracts.nim rename to codex/contracts.nim diff --git a/dagger/contracts/Readme.md b/codex/contracts/Readme.md similarity index 94% rename from dagger/contracts/Readme.md rename to codex/contracts/Readme.md index 687b0acc..6a4ea8d0 100644 --- a/dagger/contracts/Readme.md +++ b/codex/contracts/Readme.md @@ -1,13 +1,13 @@ -Dagger Contracts in Nim +Codex Contracts in Nim ======================= -Nim API for the [Dagger smart contracts][1]. +Nim API for the [Codex smart contracts][1]. Usage ----- For a global overview of the steps involved in starting and fulfilling a -storage contract, see [Dagger Contracts][1]. +storage contract, see [Codex Contracts][1]. Smart contract -------------- @@ -15,7 +15,7 @@ Smart contract Connecting to the smart contract on an Ethereum node: ```nim -import dagger/contracts +import codex/contracts import ethers let address = # fill in address where the contract was deployed @@ -178,4 +178,4 @@ await storage [1]: https://github.com/status-im/dagger-contracts/ [2]: https://ethereum.org/en/developers/docs/standards/tokens/erc-20/ -[3]: https://github.com/status-im/dagger-research/blob/main/design/storage-proof-timing.md +[3]: https://github.com/status-im/codex-research/blob/main/design/storage-proof-timing.md diff --git a/dagger/contracts/clock.nim b/codex/contracts/clock.nim similarity index 100% rename from dagger/contracts/clock.nim rename to codex/contracts/clock.nim diff --git a/dagger/contracts/deployment.nim b/codex/contracts/deployment.nim similarity index 100% rename from dagger/contracts/deployment.nim rename to codex/contracts/deployment.nim diff --git a/dagger/contracts/interactions.nim b/codex/contracts/interactions.nim similarity index 100% rename from dagger/contracts/interactions.nim rename to codex/contracts/interactions.nim diff --git a/dagger/contracts/market.nim b/codex/contracts/market.nim similarity index 100% rename from dagger/contracts/market.nim rename to codex/contracts/market.nim diff --git a/dagger/contracts/offers.nim b/codex/contracts/offers.nim similarity index 100% rename from dagger/contracts/offers.nim rename to codex/contracts/offers.nim diff --git a/dagger/contracts/proofs.nim b/codex/contracts/proofs.nim similarity index 100% rename from dagger/contracts/proofs.nim rename to codex/contracts/proofs.nim diff --git a/dagger/contracts/requests.nim b/codex/contracts/requests.nim similarity index 100% rename from dagger/contracts/requests.nim rename to codex/contracts/requests.nim diff --git a/dagger/contracts/storage.nim b/codex/contracts/storage.nim similarity index 100% rename from dagger/contracts/storage.nim rename to codex/contracts/storage.nim diff --git a/dagger/contracts/testtoken.nim b/codex/contracts/testtoken.nim similarity index 100% rename from dagger/contracts/testtoken.nim rename to codex/contracts/testtoken.nim diff --git a/dagger/discovery.nim b/codex/discovery.nim similarity index 99% rename from dagger/discovery.nim rename to codex/discovery.nim index 44dfd04d..5ead151a 100644 --- a/dagger/discovery.nim +++ b/codex/discovery.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/erasure.nim b/codex/erasure.nim similarity index 97% rename from dagger/erasure.nim rename to codex/erasure.nim index b55ba413..b14d8e52 100644 --- a/dagger/erasure.nim +++ b/codex/erasure.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/erasure/backend.nim b/codex/erasure/backend.nim similarity index 98% rename from dagger/erasure/backend.nim rename to codex/erasure/backend.nim index 35a30cd2..9feac36b 100644 --- a/dagger/erasure/backend.nim +++ b/codex/erasure/backend.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/erasure/backends/leopard.nim b/codex/erasure/backends/leopard.nim similarity index 99% rename from dagger/erasure/backends/leopard.nim rename to codex/erasure/backends/leopard.nim index afeb13e7..bf0a8161 100644 --- a/dagger/erasure/backends/leopard.nim +++ b/codex/erasure/backends/leopard.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/erasure/erasure.nim b/codex/erasure/erasure.nim similarity index 99% rename from dagger/erasure/erasure.nim rename to codex/erasure/erasure.nim index c47f57cd..87fc6a17 100644 --- a/dagger/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -26,7 +26,7 @@ import ./backend export backend logScope: - topics = "dagger erasure" + topics = "codex erasure" type ## Encode a manifest into one that is erasure protected. diff --git a/dagger/errors.nim b/codex/errors.nim similarity index 75% rename from dagger/errors.nim rename to codex/errors.nim index 847db46b..b3777f2a 100644 --- a/dagger/errors.nim +++ b/codex/errors.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -10,12 +10,12 @@ import pkg/stew/results type - DaggerError* = object of CatchableError # base dagger error - DaggerResult*[T] = Result[T, ref DaggerError] + CodexError* = object of CatchableError # base codex error + CodexResult*[T] = Result[T, ref CodexError] template mapFailure*( exp: untyped, - exc: typed = type DaggerError): untyped = + exc: typed = type CodexError): untyped = ## Convert `Result[T, E]` to `Result[E, ref CatchableError]` ## diff --git a/dagger/manifest.nim b/codex/manifest.nim similarity index 100% rename from dagger/manifest.nim rename to codex/manifest.nim diff --git a/dagger/manifest/coders.nim b/codex/manifest/coders.nim similarity index 99% rename from dagger/manifest/coders.nim rename to codex/manifest/coders.nim index 02b76c6f..f99277bd 100644 --- a/dagger/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/manifest/manifest.nim b/codex/manifest/manifest.nim similarity index 99% rename from dagger/manifest/manifest.nim rename to codex/manifest/manifest.nim index e797ef4b..ebfe039a 100644 --- a/dagger/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/manifest/types.nim b/codex/manifest/types.nim similarity index 99% rename from dagger/manifest/types.nim rename to codex/manifest/types.nim index b0b7ddec..3ae0e497 100644 --- a/dagger/manifest/types.nim +++ b/codex/manifest/types.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/market.nim b/codex/market.nim similarity index 100% rename from dagger/market.nim rename to codex/market.nim diff --git a/dagger/node.nim b/codex/node.nim similarity index 92% rename from dagger/node.nim rename to codex/node.nim index 8a40baaf..82e73763 100644 --- a/dagger/node.nim +++ b/codex/node.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -32,12 +32,12 @@ import ./discovery import ./contracts logScope: - topics = "dagger node" + topics = "codex node" type - DaggerError = object of CatchableError + CodexError = object of CatchableError - DaggerNodeRef* = ref object + CodexNodeRef* = ref object switch*: Switch networkId*: PeerID blockStore*: BlockStore @@ -46,7 +46,7 @@ type discovery*: Discovery contracts*: ?ContractInteractions -proc start*(node: DaggerNodeRef) {.async.} = +proc start*(node: CodexNodeRef) {.async.} = if not node.switch.isNil: await node.switch.start() @@ -63,9 +63,9 @@ proc start*(node: DaggerNodeRef) {.async.} = await contracts.start() node.networkId = node.switch.peerInfo.peerId - notice "Started dagger node", id = $node.networkId, addrs = node.switch.peerInfo.addrs + notice "Started codex node", id = $node.networkId, addrs = node.switch.peerInfo.addrs -proc stop*(node: DaggerNodeRef) {.async.} = +proc stop*(node: CodexNodeRef) {.async.} = trace "Stopping node" if not node.engine.isNil: @@ -84,28 +84,28 @@ proc stop*(node: DaggerNodeRef) {.async.} = await contracts.stop() proc findPeer*( - node: DaggerNodeRef, + node: CodexNodeRef, peerId: PeerID): Future[?PeerRecord] {.async.} = return await node.discovery.findPeer(peerId) proc connect*( - node: DaggerNodeRef, + node: CodexNodeRef, peerId: PeerID, addrs: seq[MultiAddress]): Future[void] = node.switch.connect(peerId, addrs) proc retrieve*( - node: DaggerNodeRef, + node: CodexNodeRef, cid: Cid): Future[?!LPStream] {.async.} = trace "Received retrieval request", cid without blk =? await node.blockStore.getBlock(cid): return failure( - newException(DaggerError, "Couldn't retrieve block for Cid!")) + newException(CodexError, "Couldn't retrieve block for Cid!")) without mc =? blk.cid.contentType(): return failure( - newException(DaggerError, "Couldn't identify Cid!")) + newException(CodexError, "Couldn't identify Cid!")) # if we got a manifest, stream the blocks if $mc in ManifestContainers: @@ -152,7 +152,7 @@ proc retrieve*( return LPStream(stream).success() proc store*( - node: DaggerNodeRef, + node: CodexNodeRef, stream: LPStream): Future[?!Cid] {.async.} = trace "Storing data" @@ -186,7 +186,7 @@ proc store*( # Generate manifest without data =? blockManifest.encode(): return failure( - newException(DaggerError, "Could not generate dataset manifest!")) + newException(CodexError, "Could not generate dataset manifest!")) # Store as a dag-pb block without manifest =? bt.Block.new(data = data, codec = DagPBCodec): @@ -207,7 +207,7 @@ proc store*( return manifest.cid.success -proc requestStorage*(self: DaggerNodeRef, +proc requestStorage*(self: CodexNodeRef, cid: Cid, duration: UInt256, nodes: uint, @@ -289,7 +289,7 @@ proc requestStorage*(self: DaggerNodeRef, return success purchase.id proc new*( - T: type DaggerNodeRef, + T: type CodexNodeRef, switch: Switch, store: BlockStore, engine: BlockExcEngine, diff --git a/dagger/por/README.md b/codex/por/README.md similarity index 100% rename from dagger/por/README.md rename to codex/por/README.md diff --git a/dagger/por/benchmark.nim b/codex/por/benchmark.nim similarity index 100% rename from dagger/por/benchmark.nim rename to codex/por/benchmark.nim diff --git a/dagger/por/example.txt b/codex/por/example.txt similarity index 100% rename from dagger/por/example.txt rename to codex/por/example.txt diff --git a/dagger/por/por.nim b/codex/por/por.nim similarity index 99% rename from dagger/por/por.nim rename to codex/por/por.nim index fa021be4..d0d1f7c4 100644 --- a/dagger/por/por.nim +++ b/codex/por/por.nim @@ -199,8 +199,8 @@ proc split(f: File, s: int64): int64 = return n proc hashToG1[T: byte|char](msg: openArray[T]): blst_p1 = - ## Hash to curve with Dagger specific domain separation - const dst = "DAGGER-PROOF-OF-CONCEPT" + ## Hash to curve with Codex specific domain separation + const dst = "CODEX-PROOF-OF-CONCEPT" result.blst_hash_to_g1(msg, dst, aug = "") proc hashNameI(name: array[namelen, byte], i: int64): blst_p1 = diff --git a/dagger/por/testpor.nim b/codex/por/testpor.nim similarity index 100% rename from dagger/por/testpor.nim rename to codex/por/testpor.nim diff --git a/dagger/por/timing/periods.nim b/codex/por/timing/periods.nim similarity index 100% rename from dagger/por/timing/periods.nim rename to codex/por/timing/periods.nim diff --git a/dagger/por/timing/proofs.nim b/codex/por/timing/proofs.nim similarity index 100% rename from dagger/por/timing/proofs.nim rename to codex/por/timing/proofs.nim diff --git a/dagger/proving.nim b/codex/proving.nim similarity index 100% rename from dagger/proving.nim rename to codex/proving.nim diff --git a/dagger/purchasing.nim b/codex/purchasing.nim similarity index 100% rename from dagger/purchasing.nim rename to codex/purchasing.nim diff --git a/dagger/rest/api.nim b/codex/rest/api.nim similarity index 94% rename from dagger/rest/api.nim rename to codex/rest/api.nim index d0aef0ae..22a27c1a 100644 --- a/dagger/rest/api.nim +++ b/codex/rest/api.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -102,11 +102,11 @@ proc decodeString(_: type array[32, byte], except ValueError as e: err e.msg.cstring -proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = +proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = var router = RestRouter.init(validate) router.api( MethodGet, - "/api/dagger/v1/connect/{peerId}") do ( + "/api/codex/v1/connect/{peerId}") do ( peerId: PeerID, addrs: seq[MultiAddress]) -> RestApiResponse: ## Connect to a peer @@ -140,7 +140,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.api( MethodGet, - "/api/dagger/v1/download/{id}") do ( + "/api/codex/v1/download/{id}") do ( id: Cid, resp: HttpResponseRef) -> RestApiResponse: ## Download a file from the node in a streaming ## manner @@ -185,7 +185,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.rawApi( MethodPost, - "/api/dagger/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse: + "/api/codex/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse: ## Create a request for storage ## ## cid - the cid of a previously uploaded dataset @@ -216,7 +216,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.rawApi( MethodPost, - "/api/dagger/v1/upload") do ( + "/api/codex/v1/upload") do ( ) -> RestApiResponse: ## Upload a file in a streamming manner ## @@ -273,7 +273,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.api( MethodGet, - "/api/dagger/v1/info") do () -> RestApiResponse: + "/api/codex/v1/info") do () -> RestApiResponse: ## Print rudimentary node information ## @@ -288,7 +288,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.api( MethodGet, - "/api/dagger/v1/sales/availability") do () -> RestApiResponse: + "/api/codex/v1/sales/availability") do () -> RestApiResponse: ## Returns storage that is for sale without contracts =? node.contracts: @@ -299,7 +299,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.rawApi( MethodPost, - "/api/dagger/v1/sales/availability") do () -> RestApiResponse: + "/api/codex/v1/sales/availability") do () -> RestApiResponse: ## Add available storage to sell ## ## size - size of available storage in bytes @@ -321,7 +321,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.api( MethodGet, - "/api/dagger/v1/storage/purchases/{id}") do ( + "/api/codex/v1/storage/purchases/{id}") do ( id: array[32, byte]) -> RestApiResponse: without contracts =? node.contracts: diff --git a/dagger/rest/json.nim b/codex/rest/json.nim similarity index 100% rename from dagger/rest/json.nim rename to codex/rest/json.nim diff --git a/dagger/rng.nim b/codex/rng.nim similarity index 99% rename from dagger/rng.nim rename to codex/rng.nim index 61ff24aa..b1f6266d 100644 --- a/dagger/rng.nim +++ b/codex/rng.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/sales.nim b/codex/sales.nim similarity index 100% rename from dagger/sales.nim rename to codex/sales.nim diff --git a/dagger/stores.nim b/codex/stores.nim similarity index 100% rename from dagger/stores.nim rename to codex/stores.nim diff --git a/dagger/stores/blockstore.nim b/codex/stores/blockstore.nim similarity index 99% rename from dagger/stores/blockstore.nim rename to codex/stores/blockstore.nim index ddcbc550..6ae774be 100644 --- a/dagger/stores/blockstore.nim +++ b/codex/stores/blockstore.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/stores/cachestore.nim b/codex/stores/cachestore.nim similarity index 97% rename from dagger/stores/cachestore.nim rename to codex/stores/cachestore.nim index 50cfaf9d..0864c55d 100644 --- a/dagger/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -29,7 +29,7 @@ import ../errors export blockstore logScope: - topics = "dagger cachestore" + topics = "codex cachestore" type CacheStore* = ref object of BlockStore @@ -37,7 +37,7 @@ type size*: Positive # in bytes cache: LruCache[Cid, Block] - InvalidBlockSize* = object of DaggerError + InvalidBlockSize* = object of CodexError const MiB* = 1024 * 1024 # bytes, 1 mebibyte = 1,048,576 bytes diff --git a/dagger/stores/fsstore.nim b/codex/stores/fsstore.nim similarity index 98% rename from dagger/stores/fsstore.nim rename to codex/stores/fsstore.nim index a7c76732..ac98d42d 100644 --- a/dagger/stores/fsstore.nim +++ b/codex/stores/fsstore.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -26,7 +26,7 @@ import ./blockstore export blockstore logScope: - topics = "dagger fsstore" + topics = "codex fsstore" type FSStore* = ref object of BlockStore diff --git a/dagger/stores/networkstore.nim b/codex/stores/networkstore.nim similarity index 97% rename from dagger/stores/networkstore.nim rename to codex/stores/networkstore.nim index a3cd9a71..951d853f 100644 --- a/dagger/stores/networkstore.nim +++ b/codex/stores/networkstore.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -24,7 +24,7 @@ import ../blockexchange export blockstore, blockexchange, asyncheapqueue logScope: - topics = "dagger networkstore" + topics = "codex networkstore" type NetworkStore* = ref object of BlockStore diff --git a/dagger/streams.nim b/codex/streams.nim similarity index 100% rename from dagger/streams.nim rename to codex/streams.nim diff --git a/dagger/streams/seekablestream.nim b/codex/streams/seekablestream.nim similarity index 92% rename from dagger/streams/seekablestream.nim rename to codex/streams/seekablestream.nim index 2c1ec1b8..ab5ae0fd 100644 --- a/dagger/streams/seekablestream.nim +++ b/codex/streams/seekablestream.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -14,7 +14,7 @@ import pkg/chronicles export libp2p, chronos, chronicles logScope: - topics = "dagger seekablestream" + topics = "codex seekablestream" type SeekableStream* = ref object of LPStream diff --git a/dagger/streams/storestream.nim b/codex/streams/storestream.nim similarity index 97% rename from dagger/streams/storestream.nim rename to codex/streams/storestream.nim index 9e87710c..88b9b8f8 100644 --- a/dagger/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -25,7 +25,7 @@ import ./seekablestream export stores, blocktype, manifest, chronos logScope: - topics = "dagger storestream" + topics = "codex storestream" type StoreStream* = ref object of SeekableStream diff --git a/dagger/utils.nim b/codex/utils.nim similarity index 100% rename from dagger/utils.nim rename to codex/utils.nim diff --git a/dagger/utils/asyncheapqueue.nim b/codex/utils/asyncheapqueue.nim similarity index 99% rename from dagger/utils/asyncheapqueue.nim rename to codex/utils/asyncheapqueue.nim index efbc7a66..b85f7de7 100644 --- a/dagger/utils/asyncheapqueue.nim +++ b/codex/utils/asyncheapqueue.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/utils/fileutils.nim b/codex/utils/fileutils.nim similarity index 99% rename from dagger/utils/fileutils.nim rename to codex/utils/fileutils.nim index eeaba77f..a900460a 100644 --- a/dagger/utils/fileutils.nim +++ b/codex/utils/fileutils.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/config.nims b/config.nims index d43c6d7c..12ad5c0e 100644 --- a/config.nims +++ b/config.nims @@ -75,8 +75,8 @@ switch("warning", "LockLevel:off") switch("define", "libp2p_pki_schemes=secp256k1") #TODO this infects everything in this folder, ideally it would only -# apply to dagger.nim, but since dagger.nims is used for other purpose -# we can't use it. And dagger.cfg doesn't work +# apply to codex.nim, but since codex.nims is used for other purpose +# we can't use it. And codex.cfg doesn't work switch("define", "chronicles_sinks=textlines[dynamic],json[dynamic]") # begin Nimble config (version 1) diff --git a/tests/dagger/blockexc/discovery/testdiscovery.nim b/tests/codex/blockexc/discovery/testdiscovery.nim similarity index 98% rename from tests/dagger/blockexc/discovery/testdiscovery.nim rename to tests/codex/blockexc/discovery/testdiscovery.nim index 520a5bee..37a8a271 100644 --- a/tests/dagger/blockexc/discovery/testdiscovery.nim +++ b/tests/codex/blockexc/discovery/testdiscovery.nim @@ -8,11 +8,11 @@ import pkg/chronos import pkg/libp2p import pkg/libp2p/errors -import pkg/dagger/rng -import pkg/dagger/stores -import pkg/dagger/blockexchange -import pkg/dagger/chunker -import pkg/dagger/blocktype as bt +import pkg/codex/rng +import pkg/codex/stores +import pkg/codex/blockexchange +import pkg/codex/chunker +import pkg/codex/blocktype as bt import ../../helpers/mockdiscovery diff --git a/tests/dagger/blockexc/discovery/testdiscoveryengine.nim b/tests/codex/blockexc/discovery/testdiscoveryengine.nim similarity index 97% rename from tests/dagger/blockexc/discovery/testdiscoveryengine.nim rename to tests/codex/blockexc/discovery/testdiscoveryengine.nim index 766fe0f4..3531206a 100644 --- a/tests/dagger/blockexc/discovery/testdiscoveryengine.nim +++ b/tests/codex/blockexc/discovery/testdiscoveryengine.nim @@ -8,12 +8,12 @@ import pkg/chronos import pkg/chronicles import pkg/libp2p -import pkg/dagger/rng -import pkg/dagger/stores -import pkg/dagger/blockexchange -import pkg/dagger/chunker -import pkg/dagger/blocktype as bt -import pkg/dagger/blockexchange/engine +import pkg/codex/rng +import pkg/codex/stores +import pkg/codex/blockexchange +import pkg/codex/chunker +import pkg/codex/blocktype as bt +import pkg/codex/blockexchange/engine import ../../helpers/mockdiscovery diff --git a/tests/dagger/blockexc/engine/testpayments.nim b/tests/codex/blockexc/engine/testpayments.nim similarity index 97% rename from tests/dagger/blockexc/engine/testpayments.nim rename to tests/codex/blockexc/engine/testpayments.nim index fa8dfe6d..b50a1720 100644 --- a/tests/dagger/blockexc/engine/testpayments.nim +++ b/tests/codex/blockexc/engine/testpayments.nim @@ -1,6 +1,6 @@ import std/unittest -import pkg/dagger/stores +import pkg/codex/stores import ../../examples suite "engine payments": diff --git a/tests/dagger/blockexc/protobuf/testpayments.nim b/tests/codex/blockexc/protobuf/testpayments.nim similarity index 97% rename from tests/dagger/blockexc/protobuf/testpayments.nim rename to tests/codex/blockexc/protobuf/testpayments.nim index a341255e..c9d6f9c6 100644 --- a/tests/dagger/blockexc/protobuf/testpayments.nim +++ b/tests/codex/blockexc/protobuf/testpayments.nim @@ -2,7 +2,7 @@ import pkg/asynctest import pkg/chronos import pkg/stew/byteutils import ../../examples -import pkg/dagger/stores +import pkg/codex/stores suite "account protobuf messages": diff --git a/tests/dagger/blockexc/protobuf/testpresence.nim b/tests/codex/blockexc/protobuf/testpresence.nim similarity index 96% rename from tests/dagger/blockexc/protobuf/testpresence.nim rename to tests/codex/blockexc/protobuf/testpresence.nim index 65969d0c..43593e13 100644 --- a/tests/dagger/blockexc/protobuf/testpresence.nim +++ b/tests/codex/blockexc/protobuf/testpresence.nim @@ -2,7 +2,7 @@ import pkg/asynctest import pkg/chronos import pkg/libp2p -import pkg/dagger/blockexchange/protobuf/presence +import pkg/codex/blockexchange/protobuf/presence import ../../examples suite "block presence protobuf messages": diff --git a/tests/dagger/blockexc/testblockexc.nim b/tests/codex/blockexc/testblockexc.nim similarity index 97% rename from tests/dagger/blockexc/testblockexc.nim rename to tests/codex/blockexc/testblockexc.nim index b878b6ff..351dfe0c 100644 --- a/tests/dagger/blockexc/testblockexc.nim +++ b/tests/codex/blockexc/testblockexc.nim @@ -8,12 +8,12 @@ import pkg/stew/byteutils import pkg/libp2p import pkg/libp2p/errors -import pkg/dagger/rng -import pkg/dagger/stores -import pkg/dagger/blockexchange -import pkg/dagger/chunker -import pkg/dagger/discovery -import pkg/dagger/blocktype as bt +import pkg/codex/rng +import pkg/codex/stores +import pkg/codex/blockexchange +import pkg/codex/chunker +import pkg/codex/discovery +import pkg/codex/blocktype as bt import ../helpers import ../examples diff --git a/tests/dagger/blockexc/testengine.nim b/tests/codex/blockexc/testengine.nim similarity index 98% rename from tests/dagger/blockexc/testengine.nim rename to tests/codex/blockexc/testengine.nim index 7813b626..eb128327 100644 --- a/tests/dagger/blockexc/testengine.nim +++ b/tests/codex/blockexc/testengine.nim @@ -9,13 +9,13 @@ import pkg/libp2p import pkg/libp2p/routing_record import pkg/libp2pdht/discv5/protocol as discv5 -import pkg/dagger/rng -import pkg/dagger/blockexchange -import pkg/dagger/stores -import pkg/dagger/chunker -import pkg/dagger/discovery -import pkg/dagger/blocktype as bt -import pkg/dagger/utils/asyncheapqueue +import pkg/codex/rng +import pkg/codex/blockexchange +import pkg/codex/stores +import pkg/codex/chunker +import pkg/codex/discovery +import pkg/codex/blocktype as bt +import pkg/codex/utils/asyncheapqueue import ../helpers import ../examples diff --git a/tests/dagger/blockexc/testnetwork.nim b/tests/codex/blockexc/testnetwork.nim similarity index 98% rename from tests/dagger/blockexc/testnetwork.nim rename to tests/codex/blockexc/testnetwork.nim index 6aa5b589..fe955d35 100644 --- a/tests/dagger/blockexc/testnetwork.nim +++ b/tests/codex/blockexc/testnetwork.nim @@ -7,10 +7,10 @@ import pkg/libp2p import pkg/libp2p/errors import pkg/protobuf_serialization -import pkg/dagger/rng -import pkg/dagger/chunker -import pkg/dagger/blocktype as bt -import pkg/dagger/blockexchange +import pkg/codex/rng +import pkg/codex/chunker +import pkg/codex/blocktype as bt +import pkg/codex/blockexchange import ../helpers import ../examples diff --git a/tests/dagger/blockexc/testpeerctxstore.nim b/tests/codex/blockexc/testpeerctxstore.nim similarity index 96% rename from tests/dagger/blockexc/testpeerctxstore.nim rename to tests/codex/blockexc/testpeerctxstore.nim index 21f75cb3..a1c826e3 100644 --- a/tests/dagger/blockexc/testpeerctxstore.nim +++ b/tests/codex/blockexc/testpeerctxstore.nim @@ -4,8 +4,8 @@ import std/sequtils import pkg/unittest2 import pkg/libp2p -import pkg/dagger/blockexchange/peers -import pkg/dagger/blockexchange/protobuf/blockexc +import pkg/codex/blockexchange/peers +import pkg/codex/blockexchange/protobuf/blockexc import ../examples diff --git a/tests/dagger/examples.nim b/tests/codex/examples.nim similarity index 93% rename from tests/dagger/examples.nim rename to tests/codex/examples.nim index c4eb8fe4..8f75b02b 100644 --- a/tests/dagger/examples.nim +++ b/tests/codex/examples.nim @@ -3,10 +3,10 @@ import std/sequtils import pkg/libp2p import pkg/nitro import pkg/stint -import pkg/dagger/rng -import pkg/dagger/stores -import pkg/dagger/blocktype as bt -import pkg/dagger/sales +import pkg/codex/rng +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/sales import ../examples export examples diff --git a/tests/dagger/helpers.nim b/tests/codex/helpers.nim similarity index 95% rename from tests/dagger/helpers.nim rename to tests/codex/helpers.nim index 82ff7aaf..cf68c45e 100644 --- a/tests/dagger/helpers.nim +++ b/tests/codex/helpers.nim @@ -1,6 +1,6 @@ import pkg/libp2p import pkg/libp2p/varint -import pkg/dagger/blocktype +import pkg/codex/blocktype import ./helpers/nodeutils import ./helpers/randomchunker diff --git a/tests/dagger/helpers/mockclock.nim b/tests/codex/helpers/mockclock.nim similarity index 95% rename from tests/dagger/helpers/mockclock.nim rename to tests/codex/helpers/mockclock.nim index 6ff8a411..55283f8a 100644 --- a/tests/dagger/helpers/mockclock.nim +++ b/tests/codex/helpers/mockclock.nim @@ -1,5 +1,5 @@ import std/times -import dagger/clock +import codex/clock export clock diff --git a/tests/dagger/helpers/mockdiscovery.nim b/tests/codex/helpers/mockdiscovery.nim similarity index 97% rename from tests/dagger/helpers/mockdiscovery.nim rename to tests/codex/helpers/mockdiscovery.nim index 9b671a9a..51192427 100644 --- a/tests/dagger/helpers/mockdiscovery.nim +++ b/tests/codex/helpers/mockdiscovery.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -12,7 +12,7 @@ import pkg/libp2p import pkg/questionable import pkg/questionable/results import pkg/stew/shims/net -import pkg/dagger/discovery +import pkg/codex/discovery type MockDiscovery* = ref object of Discovery diff --git a/tests/dagger/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim similarity index 99% rename from tests/dagger/helpers/mockmarket.nim rename to tests/codex/helpers/mockmarket.nim index 1852be03..48b7f591 100644 --- a/tests/dagger/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -1,7 +1,7 @@ import std/sequtils import std/heapqueue import pkg/questionable -import pkg/dagger/market +import pkg/codex/market export market diff --git a/tests/dagger/helpers/mockproofs.nim b/tests/codex/helpers/mockproofs.nim similarity index 98% rename from tests/dagger/helpers/mockproofs.nim rename to tests/codex/helpers/mockproofs.nim index f9fdbe5b..90dcc6bb 100644 --- a/tests/dagger/helpers/mockproofs.nim +++ b/tests/codex/helpers/mockproofs.nim @@ -2,7 +2,7 @@ import std/sets import std/tables import std/sequtils import pkg/upraises -import pkg/dagger/por/timing/proofs +import pkg/codex/por/timing/proofs type MockProofs* = ref object of Proofs diff --git a/tests/dagger/helpers/nodeutils.nim b/tests/codex/helpers/nodeutils.nim similarity index 93% rename from tests/dagger/helpers/nodeutils.nim rename to tests/codex/helpers/nodeutils.nim index bfc21a24..a792aa81 100644 --- a/tests/dagger/helpers/nodeutils.nim +++ b/tests/codex/helpers/nodeutils.nim @@ -3,10 +3,10 @@ import std/sequtils import pkg/chronos import pkg/libp2p -import pkg/dagger/discovery -import pkg/dagger/stores -import pkg/dagger/blocktype as bt -import pkg/dagger/blockexchange +import pkg/codex/discovery +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/blockexchange import ../examples diff --git a/tests/dagger/helpers/randomchunker.nim b/tests/codex/helpers/randomchunker.nim similarity index 94% rename from tests/dagger/helpers/randomchunker.nim rename to tests/codex/helpers/randomchunker.nim index 4e3e5aad..87726953 100644 --- a/tests/dagger/helpers/randomchunker.nim +++ b/tests/codex/helpers/randomchunker.nim @@ -2,8 +2,8 @@ import std/sequtils import pkg/chronos -import pkg/dagger/chunker -import pkg/dagger/rng +import pkg/codex/chunker +import pkg/codex/rng export chunker diff --git a/tests/dagger/stores/testcachestore.nim b/tests/codex/stores/testcachestore.nim similarity index 98% rename from tests/dagger/stores/testcachestore.nim rename to tests/codex/stores/testcachestore.nim index 3bd128dd..2fcdf736 100644 --- a/tests/dagger/stores/testcachestore.nim +++ b/tests/codex/stores/testcachestore.nim @@ -5,8 +5,8 @@ import pkg/asynctest import pkg/libp2p import pkg/stew/byteutils import pkg/questionable/results -import pkg/dagger/stores/cachestore -import pkg/dagger/chunker +import pkg/codex/stores/cachestore +import pkg/codex/chunker import ../helpers diff --git a/tests/dagger/stores/testfsstore.nim b/tests/codex/stores/testfsstore.nim similarity index 93% rename from tests/dagger/stores/testfsstore.nim rename to tests/codex/stores/testfsstore.nim index 254c146d..070a602c 100644 --- a/tests/dagger/stores/testfsstore.nim +++ b/tests/codex/stores/testfsstore.nim @@ -8,10 +8,10 @@ import pkg/asynctest import pkg/libp2p import pkg/stew/byteutils -import pkg/dagger/stores/cachestore -import pkg/dagger/chunker -import pkg/dagger/stores -import pkg/dagger/blocktype as bt +import pkg/codex/stores/cachestore +import pkg/codex/chunker +import pkg/codex/stores +import pkg/codex/blocktype as bt import ../helpers diff --git a/tests/dagger/testasyncheapqueue.nim b/tests/codex/testasyncheapqueue.nim similarity index 98% rename from tests/dagger/testasyncheapqueue.nim rename to tests/codex/testasyncheapqueue.nim index 7837e2e0..d7fe3ec1 100644 --- a/tests/dagger/testasyncheapqueue.nim +++ b/tests/codex/testasyncheapqueue.nim @@ -2,8 +2,8 @@ import pkg/chronos import pkg/asynctest import pkg/stew/results -import pkg/dagger/utils/asyncheapqueue -import pkg/dagger/rng +import pkg/codex/utils/asyncheapqueue +import pkg/codex/rng type Task* = tuple[name: string, priority: int] diff --git a/tests/dagger/testblockexc.nim b/tests/codex/testblockexc.nim similarity index 100% rename from tests/dagger/testblockexc.nim rename to tests/codex/testblockexc.nim diff --git a/tests/dagger/testchunking.nim b/tests/codex/testchunking.nim similarity index 98% rename from tests/dagger/testchunking.nim rename to tests/codex/testchunking.nim index 4d4668a8..626c332c 100644 --- a/tests/dagger/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -1,6 +1,6 @@ import pkg/asynctest import pkg/stew/byteutils -import pkg/dagger/chunker +import pkg/codex/chunker import pkg/chronicles import pkg/chronos import pkg/libp2p diff --git a/tests/dagger/testerasure.nim b/tests/codex/testerasure.nim similarity index 97% rename from tests/dagger/testerasure.nim rename to tests/codex/testerasure.nim index 70d46f19..23eab4e2 100644 --- a/tests/dagger/testerasure.nim +++ b/tests/codex/testerasure.nim @@ -6,11 +6,11 @@ import pkg/libp2p import pkg/questionable import pkg/questionable/results -import pkg/dagger/erasure -import pkg/dagger/manifest -import pkg/dagger/stores -import pkg/dagger/blocktype as bt -import pkg/dagger/rng +import pkg/codex/erasure +import pkg/codex/manifest +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/rng import ./helpers diff --git a/tests/dagger/testmanifest.nim b/tests/codex/testmanifest.nim similarity index 96% rename from tests/dagger/testmanifest.nim rename to tests/codex/testmanifest.nim index e78f1beb..7cd43b6e 100644 --- a/tests/dagger/testmanifest.nim +++ b/tests/codex/testmanifest.nim @@ -7,9 +7,9 @@ import pkg/asynctest import pkg/libp2p import pkg/stew/byteutils -import pkg/dagger/chunker -import pkg/dagger/blocktype as bt -import pkg/dagger/manifest +import pkg/codex/chunker +import pkg/codex/blocktype as bt +import pkg/codex/manifest import ./helpers diff --git a/tests/dagger/testnode.nim b/tests/codex/testnode.nim similarity index 90% rename from tests/dagger/testnode.nim rename to tests/codex/testnode.nim index b0038fdf..98cbff01 100644 --- a/tests/dagger/testnode.nim +++ b/tests/codex/testnode.nim @@ -10,14 +10,14 @@ import pkg/nitro import pkg/libp2p import pkg/libp2pdht/discv5/protocol as discv5 -import pkg/dagger/stores -import pkg/dagger/blockexchange -import pkg/dagger/chunker -import pkg/dagger/node -import pkg/dagger/manifest -import pkg/dagger/discovery -import pkg/dagger/blocktype as bt -import pkg/dagger/contracts +import pkg/codex/stores +import pkg/codex/blockexchange +import pkg/codex/chunker +import pkg/codex/node +import pkg/codex/manifest +import pkg/codex/discovery +import pkg/codex/blocktype as bt +import pkg/codex/contracts import ./helpers @@ -34,7 +34,7 @@ suite "Test Node": localStore: CacheStore engine: BlockExcEngine store: NetworkStore - node: DaggerNodeRef + node: CodexNodeRef blockDiscovery: Discovery peerStore: PeerCtxStore pendingBlocks: PendingBlocksManager @@ -55,7 +55,7 @@ suite "Test Node": engine = BlockExcEngine.new(localStore, wallet, network, discovery, peerStore, pendingBlocks) store = NetworkStore.new(engine, localStore) contracts = ContractInteractions.new() - node = DaggerNodeRef.new(switch, store, engine, nil, blockDiscovery, contracts) # TODO: pass `Erasure` + node = CodexNodeRef.new(switch, store, engine, nil, blockDiscovery, contracts) # TODO: pass `Erasure` await node.start() diff --git a/tests/dagger/testproving.nim b/tests/codex/testproving.nim similarity index 99% rename from tests/dagger/testproving.nim rename to tests/codex/testproving.nim index ae5ea01f..a2779efd 100644 --- a/tests/dagger/testproving.nim +++ b/tests/codex/testproving.nim @@ -1,6 +1,6 @@ import pkg/asynctest import pkg/chronos -import pkg/dagger/proving +import pkg/codex/proving import ./helpers/mockproofs import ./helpers/mockclock import ./examples diff --git a/tests/dagger/testpurchasing.nim b/tests/codex/testpurchasing.nim similarity index 99% rename from tests/dagger/testpurchasing.nim rename to tests/codex/testpurchasing.nim index 4b260ec5..e4456593 100644 --- a/tests/dagger/testpurchasing.nim +++ b/tests/codex/testpurchasing.nim @@ -2,7 +2,7 @@ import std/times import pkg/asynctest import pkg/chronos import pkg/stint -import pkg/dagger/purchasing +import pkg/codex/purchasing import ./helpers/mockmarket import ./helpers/mockclock import ./examples diff --git a/tests/dagger/testsales.nim b/tests/codex/testsales.nim similarity index 99% rename from tests/dagger/testsales.nim rename to tests/codex/testsales.nim index 72997a44..af8dd648 100644 --- a/tests/dagger/testsales.nim +++ b/tests/codex/testsales.nim @@ -1,7 +1,7 @@ import std/times import pkg/asynctest import pkg/chronos -import pkg/dagger/sales +import pkg/codex/sales import ./helpers/mockmarket import ./helpers/mockclock import ./examples diff --git a/tests/dagger/teststores.nim b/tests/codex/teststores.nim similarity index 100% rename from tests/dagger/teststores.nim rename to tests/codex/teststores.nim diff --git a/tests/dagger/teststorestream.nim b/tests/codex/teststorestream.nim similarity index 94% rename from tests/dagger/teststorestream.nim rename to tests/codex/teststorestream.nim index 3d457fee..1f810ba5 100644 --- a/tests/dagger/teststorestream.nim +++ b/tests/codex/teststorestream.nim @@ -5,11 +5,11 @@ import pkg/questionable/results import ./helpers -import pkg/dagger/streams -import pkg/dagger/stores -import pkg/dagger/manifest -import pkg/dagger/rng -import pkg/dagger/blocktype as bt +import pkg/codex/streams +import pkg/codex/stores +import pkg/codex/manifest +import pkg/codex/rng +import pkg/codex/blocktype as bt suite "StoreStream": var diff --git a/tests/contracts/examples.nim b/tests/contracts/examples.nim index 3dbbd888..2fc2b035 100644 --- a/tests/contracts/examples.nim +++ b/tests/contracts/examples.nim @@ -1,7 +1,7 @@ import std/times import pkg/stint import pkg/ethers -import dagger/contracts +import codex/contracts import ../examples export examples diff --git a/tests/contracts/testClock.nim b/tests/contracts/testClock.nim index 1b325af7..db97d70e 100644 --- a/tests/contracts/testClock.nim +++ b/tests/contracts/testClock.nim @@ -1,6 +1,6 @@ import std/times import pkg/chronos -import dagger/contracts/clock +import codex/contracts/clock import ../ethertest ethersuite "On-Chain Clock": diff --git a/tests/contracts/testCollateral.nim b/tests/contracts/testCollateral.nim index 523fdf36..6d2079b8 100644 --- a/tests/contracts/testCollateral.nim +++ b/tests/contracts/testCollateral.nim @@ -1,7 +1,7 @@ import pkg/chronos import pkg/stint -import dagger/contracts -import dagger/contracts/testtoken +import codex/contracts +import codex/contracts/testtoken import ../ethertest ethersuite "Collateral": diff --git a/tests/contracts/testContracts.nim b/tests/contracts/testContracts.nim index b7f345b1..efb4d6a7 100644 --- a/tests/contracts/testContracts.nim +++ b/tests/contracts/testContracts.nim @@ -1,9 +1,9 @@ import std/json import pkg/chronos import pkg/nimcrypto -import dagger/contracts -import dagger/contracts/testtoken -import dagger/por/timing/periods +import codex/contracts +import codex/contracts/testtoken +import codex/por/timing/periods import ../ethertest import ./examples import ./time diff --git a/tests/contracts/testInteractions.nim b/tests/contracts/testInteractions.nim index 84eed9f5..fbfbd0fe 100644 --- a/tests/contracts/testInteractions.nim +++ b/tests/contracts/testInteractions.nim @@ -1,5 +1,5 @@ import std/os -import dagger/contracts +import codex/contracts import ../ethertest import ./examples diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 7b32579a..4c3b7b52 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -1,6 +1,6 @@ import pkg/chronos -import dagger/contracts -import dagger/contracts/testtoken +import codex/contracts +import codex/contracts/testtoken import ../ethertest import ./examples import ./time diff --git a/tests/contracts/testProofs.nim b/tests/contracts/testProofs.nim index ade20bef..359f4ecc 100644 --- a/tests/contracts/testProofs.nim +++ b/tests/contracts/testProofs.nim @@ -1,4 +1,4 @@ -import dagger/contracts +import codex/contracts import ../ethertest import ./examples import ./time diff --git a/tests/integration/nodes.nim b/tests/integration/nodes.nim index 57c66f96..12a0b71e 100644 --- a/tests/integration/nodes.nim +++ b/tests/integration/nodes.nim @@ -4,7 +4,7 @@ import std/streams import std/strutils const workingDir = currentSourcePath() / ".." / ".." / ".." -const executable = "build" / "dagger" +const executable = "build" / "codex" proc startNode*(args: openArray[string], debug = false): Process = if debug: @@ -13,7 +13,7 @@ proc startNode*(args: openArray[string], debug = false): Process = else: result = startProcess(executable, workingDir, args) for line in result.outputStream.lines: - if line.contains("Started dagger node"): + if line.contains("Started codex node"): break proc stop*(node: Process) = diff --git a/tests/integration/tokens.nim b/tests/integration/tokens.nim index 4c7c8d47..8c1a00d7 100644 --- a/tests/integration/tokens.nim +++ b/tests/integration/tokens.nim @@ -1,5 +1,5 @@ -import dagger/contracts -import dagger/contracts/testtoken +import codex/contracts +import codex/contracts/testtoken proc mint*(signer: Signer, amount = 1_000_000.u256) {.async.} = ## Mints a considerable amount of tokens and approves them for transfer to diff --git a/tests/testCodex.nim b/tests/testCodex.nim new file mode 100644 index 00000000..b5088732 --- /dev/null +++ b/tests/testCodex.nim @@ -0,0 +1,16 @@ +import ./codex/teststores +import ./codex/testblockexc +import ./codex/testasyncheapqueue +import ./codex/testchunking +import ./codex/testmanifest +import ./codex/testnode +import ./codex/teststorestream +import ./codex/testpurchasing +import ./codex/testsales +import ./codex/testerasure +import ./codex/testproving + +# to check that everything compiles +import ../codex + +{.warning[UnusedImport]: off.} diff --git a/tests/testDagger.nim.cfg b/tests/testCodex.nim.cfg similarity index 100% rename from tests/testDagger.nim.cfg rename to tests/testCodex.nim.cfg diff --git a/tests/testDagger.nim b/tests/testDagger.nim deleted file mode 100644 index 332baa0a..00000000 --- a/tests/testDagger.nim +++ /dev/null @@ -1,16 +0,0 @@ -import ./dagger/teststores -import ./dagger/testblockexc -import ./dagger/testasyncheapqueue -import ./dagger/testchunking -import ./dagger/testmanifest -import ./dagger/testnode -import ./dagger/teststorestream -import ./dagger/testpurchasing -import ./dagger/testsales -import ./dagger/testerasure -import ./dagger/testproving - -# to check that everything compiles -import ../dagger - -{.warning[UnusedImport]: off.} diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 5f67fd37..422bf396 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -27,8 +27,8 @@ ethersuite "Integration tests": "--udp-port=8091", "--eth-account=" & $accounts[1] ] - baseurl1 = "http://localhost:8080/api/dagger/v1" - baseurl2 = "http://localhost:8081/api/dagger/v1" + baseurl1 = "http://localhost:8080/api/codex/v1" + baseurl2 = "http://localhost:8081/api/codex/v1" client = newHttpClient() teardown: From d4f3ebc867e2ade0dfffe479f442f69a5561e572 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 19 May 2022 16:28:53 -0600 Subject: [PATCH 3/7] reorg engine directory structure (#104) --- codex/blockexchange/engine.nim | 479 +----------------- codex/blockexchange/engine/discovery.nim | 2 +- codex/blockexchange/engine/engine.nim | 479 ++++++++++++++++++ codex/blockexchange/engine/payments.nim | 9 + .../{ => engine}/pendingblocks.nim | 2 +- codex/blockexchange/network.nim | 377 +------------- codex/blockexchange/network/network.nim | 375 ++++++++++++++ .../{ => network}/networkpeer.nim | 2 +- codex/blockexchange/peers.nim | 3 +- .../blockexchange/{ => peers}/peercontext.nim | 15 +- codex/blockexchange/peers/peerctxstore.nim | 2 +- .../discovery/testdiscovery.nim | 0 .../discovery/testdiscoveryengine.nim | 0 .../engine}/testblockexc.nim | 4 +- .../engine}/testengine.nim | 4 +- .../engine/testpayments.nim | 0 .../protobuf/testpayments.nim | 0 .../protobuf/testpresence.nim | 0 tests/codex/blockexchange/testdiscovery.nim | 4 + tests/codex/blockexchange/testengine.nim | 5 + .../testnetwork.nim | 0 .../testpeerctxstore.nim | 0 tests/codex/blockexchange/testprotobuf.nim | 4 + tests/codex/testblockexc.nim | 14 +- 24 files changed, 909 insertions(+), 871 deletions(-) create mode 100644 codex/blockexchange/engine/engine.nim rename codex/blockexchange/{ => engine}/pendingblocks.nim (98%) create mode 100644 codex/blockexchange/network/network.nim rename codex/blockexchange/{ => network}/networkpeer.nim (98%) rename codex/blockexchange/{ => peers}/peercontext.nim (76%) rename tests/codex/{blockexc => blockexchange}/discovery/testdiscovery.nim (100%) rename tests/codex/{blockexc => blockexchange}/discovery/testdiscoveryengine.nim (100%) rename tests/codex/{blockexc => blockexchange/engine}/testblockexc.nim (99%) rename tests/codex/{blockexc => blockexchange/engine}/testengine.nim (99%) rename tests/codex/{blockexc => blockexchange}/engine/testpayments.nim (100%) rename tests/codex/{blockexc => blockexchange}/protobuf/testpayments.nim (100%) rename tests/codex/{blockexc => blockexchange}/protobuf/testpresence.nim (100%) create mode 100644 tests/codex/blockexchange/testdiscovery.nim create mode 100644 tests/codex/blockexchange/testengine.nim rename tests/codex/{blockexc => blockexchange}/testnetwork.nim (100%) rename tests/codex/{blockexc => blockexchange}/testpeerctxstore.nim (100%) create mode 100644 tests/codex/blockexchange/testprotobuf.nim diff --git a/codex/blockexchange/engine.nim b/codex/blockexchange/engine.nim index 6223ee13..a9cd9160 100644 --- a/codex/blockexchange/engine.nim +++ b/codex/blockexchange/engine.nim @@ -1,478 +1,5 @@ -## Nim-Codex -## Copyright (c) 2021 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import std/sequtils -import std/sets - -import pkg/chronos -import pkg/chronicles -import pkg/libp2p - -import ../stores/blockstore -import ../blocktype as bt -import ../utils - -import ./protobuf/blockexc -import ./protobuf/presence - -import ./network -import ./peers -import ./engine/payments import ./engine/discovery -import ./pendingblocks +import ./engine/engine +import ./engine/payments -export peers, pendingblocks, payments, discovery - -logScope: - topics = "codex blockexc engine" - -const - DefaultMaxPeersPerRequest* = 10 - DefaultTaskQueueSize = 100 - DefaultConcurrentTasks = 10 - DefaultMaxRetries = 3 - DefaultConcurrentDiscRequests = 10 - DefaultConcurrentAdvertRequests = 10 - DefaultDiscoveryTimeout = 1.minutes - DefaultMaxQueriedBlocksCache = 1000 - DefaultMinPeersPerBlock = 3 - -type - TaskHandler* = proc(task: BlockExcPeerCtx): Future[void] {.gcsafe.} - TaskScheduler* = proc(task: BlockExcPeerCtx): bool {.gcsafe.} - - BlockExcEngine* = ref object of RootObj - localStore*: BlockStore # Local block store for this instance - network*: BlockExcNetwork # Petwork interface - peers*: PeerCtxStore # Peers we're currently actively exchanging with - taskQueue*: AsyncHeapQueue[BlockExcPeerCtx] # Peers we're currently processing tasks for - concurrentTasks: int # Number of concurrent peers we're serving at any given time - blockexcTasks: seq[Future[void]] # Future to control blockexc task - blockexcRunning: bool # Indicates if the blockexc task is running - pendingBlocks*: PendingBlocksManager # Blocks we're awaiting to be resolved - peersPerRequest: int # Max number of peers to request from - wallet*: WalletRef # Nitro wallet for micropayments - pricing*: ?Pricing # Optional bandwidth pricing - discovery*: DiscoveryEngine - - Pricing* = object - address*: EthAddress - price*: UInt256 - -proc contains*(a: AsyncHeapQueue[Entry], b: Cid): bool = - ## Convenience method to check for entry prepense - ## - - a.anyIt( it.cid == b ) - -# attach task scheduler to engine -proc scheduleTask(b: BlockExcEngine, task: BlockExcPeerCtx): bool {.gcsafe} = - b.taskQueue.pushOrUpdateNoWait(task).isOk() - -proc blockexcTaskRunner(b: BlockExcEngine): Future[void] {.gcsafe.} - -proc start*(b: BlockExcEngine) {.async.} = - ## Start the blockexc task - ## - - await b.discovery.start() - - trace "Blockexc starting with concurrent tasks", tasks = b.concurrentTasks - if b.blockexcRunning: - warn "Starting blockexc twice" - return - - b.blockexcRunning = true - for i in 0.. 0: - b.network.request.sendWantList( - peer, - cids, - wantType = WantType.wantBlock) # we want this remote to send us a block - - # if none of the connected peers report our wants in their have list, - # fire up discovery - b.discovery.queueFindBlocksReq( - toSeq(b.pendingBlocks.wantList) - .filter do(cid: Cid) -> bool: - not b.peers.anyIt( cid in it.peerHave )) - -proc scheduleTasks(b: BlockExcEngine, blocks: seq[bt.Block]) = - trace "Schedule a task for new blocks" - - let - cids = blocks.mapIt( it.cid ) - - # schedule any new peers to provide blocks to - for p in b.peers: - for c in cids: # for each cid - # schedule a peer if it wants at least one - # cid and we have it in our local store - if c in p.peerWants and c in b.localStore: - if b.scheduleTask(p): - trace "Task scheduled for peer", peer = p.id - else: - trace "Unable to schedule task for peer", peer = p.id - - break # do next peer - -proc resolveBlocks*(b: BlockExcEngine, blocks: seq[bt.Block]) = - ## Resolve pending blocks from the pending blocks manager - ## and schedule any new task to be ran - ## - - trace "Resolving blocks", blocks = blocks.len - - b.pendingBlocks.resolve(blocks) - b.scheduleTasks(blocks) - b.discovery.queueProvideBlocksReq(blocks.mapIt( it.cid )) - -proc payForBlocks(engine: BlockExcEngine, - peer: BlockExcPeerCtx, - blocks: seq[bt.Block]) = - let sendPayment = engine.network.request.sendPayment - if sendPayment.isNil: - return - - let cids = blocks.mapIt(it.cid) - if payment =? engine.wallet.pay(peer, peer.price(cids)): - sendPayment(peer.id, payment) - -proc blocksHandler*( - b: BlockExcEngine, - peer: PeerID, - blocks: seq[bt.Block]) {.async.} = - ## handle incoming blocks - ## - - trace "Got blocks from peer", peer, len = blocks.len - for blk in blocks: - if not (await b.localStore.putBlock(blk)): - trace "Unable to store block", cid = blk.cid - continue - - b.resolveBlocks(blocks) - let peerCtx = b.peers.get(peer) - if peerCtx != nil: - b.payForBlocks(peerCtx, blocks) - -proc wantListHandler*( - b: BlockExcEngine, - peer: PeerID, - wantList: WantList) {.async.} = - ## Handle incoming want lists - ## - - trace "Got want list for peer", peer - let peerCtx = b.peers.get(peer) - if isNil(peerCtx): - return - - var dontHaves: seq[Cid] - let entries = wantList.entries - for e in entries: - let idx = peerCtx.peerWants.find(e) - if idx > -1: - # peer doesn't want this block anymore - if e.cancel: - peerCtx.peerWants.del(idx) - continue - - peerCtx.peerWants[idx] = e # update entry - else: - peerCtx.peerWants.add(e) - - trace "Added entry to peer's want list", peer = peerCtx.id, cid = $e.cid - - # peer might want to ask for the same cid with - # different want params - if e.sendDontHave and e.cid notin b.localStore: - dontHaves.add(e.cid) - - # send don't have's to remote - if dontHaves.len > 0: - b.network.request.sendPresence( - peer, - dontHaves.mapIt( - BlockPresence( - cid: it.data.buffer, - `type`: BlockPresenceType.presenceDontHave))) - - if not b.scheduleTask(peerCtx): - trace "Unable to schedule task for peer", peer - -proc accountHandler*( - engine: BlockExcEngine, - peer: PeerID, - account: Account) {.async.} = - let context = engine.peers.get(peer) - if context.isNil: - return - - context.account = account.some - -proc paymentHandler*( - engine: BlockExcEngine, - peer: PeerId, - payment: SignedState) {.async.} = - without context =? engine.peers.get(peer).option and - account =? context.account: - return - - if channel =? context.paymentChannel: - let sender = account.address - discard engine.wallet.acceptPayment(channel, Asset, sender, payment) - else: - context.paymentChannel = engine.wallet.acceptChannel(payment).option - -proc setupPeer*(b: BlockExcEngine, peer: PeerID) = - ## Perform initial setup, such as want - ## list exchange - ## - - trace "Setting up new peer", peer - if peer notin b.peers: - b.peers.add(BlockExcPeerCtx( - id: peer - )) - - # broadcast our want list, the other peer will do the same - if b.pendingBlocks.len > 0: - b.network.request.sendWantList(peer, toSeq(b.pendingBlocks.wantList), full = true) - - if address =? b.pricing.?address: - b.network.request.sendAccount(peer, Account(address: address)) - -proc dropPeer*(b: BlockExcEngine, peer: PeerID) = - ## Cleanup disconnected peer - ## - - trace "Dropping peer", peer - - # drop the peer from the peers table - b.peers.remove(peer) - -proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} = - trace "Handling task for peer", peer = task.id - - var wantsBlocks = newAsyncHeapQueue[Entry](queueType = QueueType.Max) - # get blocks and wants to send to the remote - for e in task.peerWants: - if e.wantType == WantType.wantBlock: - await wantsBlocks.push(e) - - # TODO: There should be all sorts of accounting of - # bytes sent/received here - if wantsBlocks.len > 0: - let blockFuts = await allFinished(wantsBlocks.mapIt( - b.localStore.getBlock(it.cid) - )) - - let blocks = blockFuts - .filterIt((not it.failed) and it.read.isOk) - .mapIt(!it.read) - - if blocks.len > 0: - trace "Sending blocks to peer", peer = task.id, blocks = blocks.len - b.network.request.sendBlocks( - task.id, - blocks) - - # Remove successfully sent blocks - task.peerWants.keepIf( - proc(e: Entry): bool = - not blocks.anyIt( it.cid == e.cid ) - ) - - var wants: seq[BlockPresence] - # do not remove wants from the queue unless - # we send the block or get a cancel - for e in task.peerWants: - if e.wantType == WantType.wantHave: - var presence = Presence(cid: e.cid) - presence.have = b.localStore.hasblock(presence.cid) - if presence.have and price =? b.pricing.?price: - presence.price = price - wants.add(BlockPresence.init(presence)) - - if wants.len > 0: - b.network.request.sendPresence(task.id, wants) - -proc blockexcTaskRunner(b: BlockExcEngine) {.async.} = - ## process tasks - ## - - trace "Starting blockexc task runner" - while b.blockexcRunning: - let - peerCtx = await b.taskQueue.pop() - - trace "Got new task from queue", peerId = peerCtx.id - await b.taskHandler(peerCtx) - - trace "Exiting blockexc task runner" - -proc new*( - T: type BlockExcEngine, - localStore: BlockStore, - wallet: WalletRef, - network: BlockExcNetwork, - discovery: DiscoveryEngine, - peerStore: PeerCtxStore, - pendingBlocks: PendingBlocksManager, - concurrentTasks = DefaultConcurrentTasks, - peersPerRequest = DefaultMaxPeersPerRequest): T = - - let - engine = BlockExcEngine( - localStore: localStore, - peers: peerStore, - pendingBlocks: pendingBlocks, - peersPerRequest: peersPerRequest, - network: network, - wallet: wallet, - concurrentTasks: concurrentTasks, - taskQueue: newAsyncHeapQueue[BlockExcPeerCtx](DefaultTaskQueueSize), - discovery: discovery) - - proc peerEventHandler(peerId: PeerID, event: PeerEvent) {.async.} = - if event.kind == PeerEventKind.Joined: - engine.setupPeer(peerId) - else: - engine.dropPeer(peerId) - - if not isNil(network.switch): - network.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Joined) - network.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Left) - - proc blockWantListHandler( - peer: PeerID, - wantList: WantList): Future[void] {.gcsafe.} = - engine.wantListHandler(peer, wantList) - - proc blockPresenceHandler( - peer: PeerID, - presence: seq[BlockPresence]): Future[void] {.gcsafe.} = - engine.blockPresenceHandler(peer, presence) - - proc blocksHandler( - peer: PeerID, - blocks: seq[bt.Block]): Future[void] {.gcsafe.} = - engine.blocksHandler(peer, blocks) - - proc accountHandler(peer: PeerId, account: Account): Future[void] {.gcsafe.} = - engine.accountHandler(peer, account) - - proc paymentHandler(peer: PeerId, payment: SignedState): Future[void] {.gcsafe.} = - engine.paymentHandler(peer, payment) - - network.handlers = BlockExcHandlers( - onWantList: blockWantListHandler, - onBlocks: blocksHandler, - onPresence: blockPresenceHandler, - onAccount: accountHandler, - onPayment: paymentHandler) - - return engine +export discovery, engine, payments diff --git a/codex/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim index aeb88557..107084d8 100644 --- a/codex/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -22,7 +22,7 @@ import ../../utils import ../../discovery import ../../stores/blockstore -import ../pendingblocks +import ./pendingblocks logScope: topics = "codex discovery engine" diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim new file mode 100644 index 00000000..69fda741 --- /dev/null +++ b/codex/blockexchange/engine/engine.nim @@ -0,0 +1,479 @@ +## Nim-Codex +## Copyright (c) 2021 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/sequtils +import std/sets + +import pkg/chronos +import pkg/chronicles +import pkg/libp2p + +import ../../stores/blockstore +import ../../blocktype as bt +import ../../utils + +import ../protobuf/blockexc +import ../protobuf/presence + +import ../network +import ../peers + +import ./payments +import ./discovery +import ./pendingblocks + +export peers, pendingblocks, payments, discovery + +logScope: + topics = "codex blockexc engine" + +const + DefaultMaxPeersPerRequest* = 10 + DefaultTaskQueueSize = 100 + DefaultConcurrentTasks = 10 + DefaultMaxRetries = 3 + DefaultConcurrentDiscRequests = 10 + DefaultConcurrentAdvertRequests = 10 + DefaultDiscoveryTimeout = 1.minutes + DefaultMaxQueriedBlocksCache = 1000 + DefaultMinPeersPerBlock = 3 + +type + TaskHandler* = proc(task: BlockExcPeerCtx): Future[void] {.gcsafe.} + TaskScheduler* = proc(task: BlockExcPeerCtx): bool {.gcsafe.} + + BlockExcEngine* = ref object of RootObj + localStore*: BlockStore # Local block store for this instance + network*: BlockExcNetwork # Petwork interface + peers*: PeerCtxStore # Peers we're currently actively exchanging with + taskQueue*: AsyncHeapQueue[BlockExcPeerCtx] # Peers we're currently processing tasks for + concurrentTasks: int # Number of concurrent peers we're serving at any given time + blockexcTasks: seq[Future[void]] # Future to control blockexc task + blockexcRunning: bool # Indicates if the blockexc task is running + pendingBlocks*: PendingBlocksManager # Blocks we're awaiting to be resolved + peersPerRequest: int # Max number of peers to request from + wallet*: WalletRef # Nitro wallet for micropayments + pricing*: ?Pricing # Optional bandwidth pricing + discovery*: DiscoveryEngine + + Pricing* = object + address*: EthAddress + price*: UInt256 + +proc contains*(a: AsyncHeapQueue[Entry], b: Cid): bool = + ## Convenience method to check for entry prepense + ## + + a.anyIt( it.cid == b ) + +# attach task scheduler to engine +proc scheduleTask(b: BlockExcEngine, task: BlockExcPeerCtx): bool {.gcsafe} = + b.taskQueue.pushOrUpdateNoWait(task).isOk() + +proc blockexcTaskRunner(b: BlockExcEngine): Future[void] {.gcsafe.} + +proc start*(b: BlockExcEngine) {.async.} = + ## Start the blockexc task + ## + + await b.discovery.start() + + trace "Blockexc starting with concurrent tasks", tasks = b.concurrentTasks + if b.blockexcRunning: + warn "Starting blockexc twice" + return + + b.blockexcRunning = true + for i in 0.. 0: + b.network.request.sendWantList( + peer, + cids, + wantType = WantType.wantBlock) # we want this remote to send us a block + + # if none of the connected peers report our wants in their have list, + # fire up discovery + b.discovery.queueFindBlocksReq( + toSeq(b.pendingBlocks.wantList) + .filter do(cid: Cid) -> bool: + not b.peers.anyIt( cid in it.peerHave )) + +proc scheduleTasks(b: BlockExcEngine, blocks: seq[bt.Block]) = + trace "Schedule a task for new blocks" + + let + cids = blocks.mapIt( it.cid ) + + # schedule any new peers to provide blocks to + for p in b.peers: + for c in cids: # for each cid + # schedule a peer if it wants at least one + # cid and we have it in our local store + if c in p.peerWants and c in b.localStore: + if b.scheduleTask(p): + trace "Task scheduled for peer", peer = p.id + else: + trace "Unable to schedule task for peer", peer = p.id + + break # do next peer + +proc resolveBlocks*(b: BlockExcEngine, blocks: seq[bt.Block]) = + ## Resolve pending blocks from the pending blocks manager + ## and schedule any new task to be ran + ## + + trace "Resolving blocks", blocks = blocks.len + + b.pendingBlocks.resolve(blocks) + b.scheduleTasks(blocks) + b.discovery.queueProvideBlocksReq(blocks.mapIt( it.cid )) + +proc payForBlocks(engine: BlockExcEngine, + peer: BlockExcPeerCtx, + blocks: seq[bt.Block]) = + let sendPayment = engine.network.request.sendPayment + if sendPayment.isNil: + return + + let cids = blocks.mapIt(it.cid) + if payment =? engine.wallet.pay(peer, peer.price(cids)): + sendPayment(peer.id, payment) + +proc blocksHandler*( + b: BlockExcEngine, + peer: PeerID, + blocks: seq[bt.Block]) {.async.} = + ## handle incoming blocks + ## + + trace "Got blocks from peer", peer, len = blocks.len + for blk in blocks: + if not (await b.localStore.putBlock(blk)): + trace "Unable to store block", cid = blk.cid + continue + + b.resolveBlocks(blocks) + let peerCtx = b.peers.get(peer) + if peerCtx != nil: + b.payForBlocks(peerCtx, blocks) + +proc wantListHandler*( + b: BlockExcEngine, + peer: PeerID, + wantList: WantList) {.async.} = + ## Handle incoming want lists + ## + + trace "Got want list for peer", peer + let peerCtx = b.peers.get(peer) + if isNil(peerCtx): + return + + var dontHaves: seq[Cid] + let entries = wantList.entries + for e in entries: + let idx = peerCtx.peerWants.find(e) + if idx > -1: + # peer doesn't want this block anymore + if e.cancel: + peerCtx.peerWants.del(idx) + continue + + peerCtx.peerWants[idx] = e # update entry + else: + peerCtx.peerWants.add(e) + + trace "Added entry to peer's want list", peer = peerCtx.id, cid = $e.cid + + # peer might want to ask for the same cid with + # different want params + if e.sendDontHave and e.cid notin b.localStore: + dontHaves.add(e.cid) + + # send don't have's to remote + if dontHaves.len > 0: + b.network.request.sendPresence( + peer, + dontHaves.mapIt( + BlockPresence( + cid: it.data.buffer, + `type`: BlockPresenceType.presenceDontHave))) + + if not b.scheduleTask(peerCtx): + trace "Unable to schedule task for peer", peer + +proc accountHandler*( + engine: BlockExcEngine, + peer: PeerID, + account: Account) {.async.} = + let context = engine.peers.get(peer) + if context.isNil: + return + + context.account = account.some + +proc paymentHandler*( + engine: BlockExcEngine, + peer: PeerId, + payment: SignedState) {.async.} = + without context =? engine.peers.get(peer).option and + account =? context.account: + return + + if channel =? context.paymentChannel: + let sender = account.address + discard engine.wallet.acceptPayment(channel, Asset, sender, payment) + else: + context.paymentChannel = engine.wallet.acceptChannel(payment).option + +proc setupPeer*(b: BlockExcEngine, peer: PeerID) = + ## Perform initial setup, such as want + ## list exchange + ## + + trace "Setting up new peer", peer + if peer notin b.peers: + b.peers.add(BlockExcPeerCtx( + id: peer + )) + + # broadcast our want list, the other peer will do the same + if b.pendingBlocks.len > 0: + b.network.request.sendWantList(peer, toSeq(b.pendingBlocks.wantList), full = true) + + if address =? b.pricing.?address: + b.network.request.sendAccount(peer, Account(address: address)) + +proc dropPeer*(b: BlockExcEngine, peer: PeerID) = + ## Cleanup disconnected peer + ## + + trace "Dropping peer", peer + + # drop the peer from the peers table + b.peers.remove(peer) + +proc taskHandler*(b: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, async.} = + trace "Handling task for peer", peer = task.id + + var wantsBlocks = newAsyncHeapQueue[Entry](queueType = QueueType.Max) + # get blocks and wants to send to the remote + for e in task.peerWants: + if e.wantType == WantType.wantBlock: + await wantsBlocks.push(e) + + # TODO: There should be all sorts of accounting of + # bytes sent/received here + if wantsBlocks.len > 0: + let blockFuts = await allFinished(wantsBlocks.mapIt( + b.localStore.getBlock(it.cid) + )) + + let blocks = blockFuts + .filterIt((not it.failed) and it.read.isOk) + .mapIt(!it.read) + + if blocks.len > 0: + trace "Sending blocks to peer", peer = task.id, blocks = blocks.len + b.network.request.sendBlocks( + task.id, + blocks) + + # Remove successfully sent blocks + task.peerWants.keepIf( + proc(e: Entry): bool = + not blocks.anyIt( it.cid == e.cid ) + ) + + var wants: seq[BlockPresence] + # do not remove wants from the queue unless + # we send the block or get a cancel + for e in task.peerWants: + if e.wantType == WantType.wantHave: + var presence = Presence(cid: e.cid) + presence.have = b.localStore.hasblock(presence.cid) + if presence.have and price =? b.pricing.?price: + presence.price = price + wants.add(BlockPresence.init(presence)) + + if wants.len > 0: + b.network.request.sendPresence(task.id, wants) + +proc blockexcTaskRunner(b: BlockExcEngine) {.async.} = + ## process tasks + ## + + trace "Starting blockexc task runner" + while b.blockexcRunning: + let + peerCtx = await b.taskQueue.pop() + + trace "Got new task from queue", peerId = peerCtx.id + await b.taskHandler(peerCtx) + + trace "Exiting blockexc task runner" + +proc new*( + T: type BlockExcEngine, + localStore: BlockStore, + wallet: WalletRef, + network: BlockExcNetwork, + discovery: DiscoveryEngine, + peerStore: PeerCtxStore, + pendingBlocks: PendingBlocksManager, + concurrentTasks = DefaultConcurrentTasks, + peersPerRequest = DefaultMaxPeersPerRequest): T = + + let + engine = BlockExcEngine( + localStore: localStore, + peers: peerStore, + pendingBlocks: pendingBlocks, + peersPerRequest: peersPerRequest, + network: network, + wallet: wallet, + concurrentTasks: concurrentTasks, + taskQueue: newAsyncHeapQueue[BlockExcPeerCtx](DefaultTaskQueueSize), + discovery: discovery) + + proc peerEventHandler(peerId: PeerID, event: PeerEvent) {.async.} = + if event.kind == PeerEventKind.Joined: + engine.setupPeer(peerId) + else: + engine.dropPeer(peerId) + + if not isNil(network.switch): + network.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Joined) + network.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Left) + + proc blockWantListHandler( + peer: PeerID, + wantList: WantList): Future[void] {.gcsafe.} = + engine.wantListHandler(peer, wantList) + + proc blockPresenceHandler( + peer: PeerID, + presence: seq[BlockPresence]): Future[void] {.gcsafe.} = + engine.blockPresenceHandler(peer, presence) + + proc blocksHandler( + peer: PeerID, + blocks: seq[bt.Block]): Future[void] {.gcsafe.} = + engine.blocksHandler(peer, blocks) + + proc accountHandler(peer: PeerId, account: Account): Future[void] {.gcsafe.} = + engine.accountHandler(peer, account) + + proc paymentHandler(peer: PeerId, payment: SignedState): Future[void] {.gcsafe.} = + engine.paymentHandler(peer, payment) + + network.handlers = BlockExcHandlers( + onWantList: blockWantListHandler, + onBlocks: blocksHandler, + onPresence: blockPresenceHandler, + onAccount: accountHandler, + onPayment: paymentHandler) + + return engine diff --git a/codex/blockexchange/engine/payments.nim b/codex/blockexchange/engine/payments.nim index 15079086..35d38e29 100644 --- a/codex/blockexchange/engine/payments.nim +++ b/codex/blockexchange/engine/payments.nim @@ -1,3 +1,12 @@ +## Nim-Codex +## Copyright (c) 2021 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + import std/math import pkg/nitro import pkg/questionable/results diff --git a/codex/blockexchange/pendingblocks.nim b/codex/blockexchange/engine/pendingblocks.nim similarity index 98% rename from codex/blockexchange/pendingblocks.nim rename to codex/blockexchange/engine/pendingblocks.nim index 5539734f..c24c7519 100644 --- a/codex/blockexchange/pendingblocks.nim +++ b/codex/blockexchange/engine/pendingblocks.nim @@ -18,7 +18,7 @@ import pkg/chronicles import pkg/chronos import pkg/libp2p -import ../blocktype +import ../../blocktype logScope: topics = "codex blockexc pendingblocks" diff --git a/codex/blockexchange/network.nim b/codex/blockexchange/network.nim index 0c14e51c..ccaa6207 100644 --- a/codex/blockexchange/network.nim +++ b/codex/blockexchange/network.nim @@ -1,375 +1,4 @@ -## Nim-Codex -## Copyright (c) 2021 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. +import ./network/network +import ./network/networkpeer -import std/tables -import std/sequtils - -import pkg/chronicles -import pkg/chronos - -import pkg/libp2p -import pkg/questionable -import pkg/questionable/results - -import ../blocktype as bt -import ./protobuf/blockexc as pb -import ./protobuf/payments - -import ./networkpeer - -export networkpeer, payments - -logScope: - topics = "codex blockexc network" - -const Codec* = "/codex/blockexc/1.0.0" - -type - WantListHandler* = proc(peer: PeerID, wantList: WantList): Future[void] {.gcsafe.} - BlocksHandler* = proc(peer: PeerID, blocks: seq[bt.Block]): Future[void] {.gcsafe.} - BlockPresenceHandler* = proc(peer: PeerID, precense: seq[BlockPresence]): Future[void] {.gcsafe.} - AccountHandler* = proc(peer: PeerID, account: Account): Future[void] {.gcsafe.} - PaymentHandler* = proc(peer: PeerID, payment: SignedState): Future[void] {.gcsafe.} - - BlockExcHandlers* = object - onWantList*: WantListHandler - onBlocks*: BlocksHandler - onPresence*: BlockPresenceHandler - onAccount*: AccountHandler - onPayment*: PaymentHandler - - WantListBroadcaster* = proc( - id: PeerID, - cids: seq[Cid], - priority: int32 = 0, - cancel: bool = false, - wantType: WantType = WantType.wantHave, - full: bool = false, - sendDontHave: bool = false) {.gcsafe.} - - BlocksBroadcaster* = proc(peer: PeerID, presence: seq[bt.Block]) {.gcsafe.} - PresenceBroadcaster* = proc(peer: PeerID, presence: seq[BlockPresence]) {.gcsafe.} - AccountBroadcaster* = proc(peer: PeerID, account: Account) {.gcsafe.} - PaymentBroadcaster* = proc(peer: PeerID, payment: SignedState) {.gcsafe.} - - BlockExcRequest* = object - sendWantList*: WantListBroadcaster - sendBlocks*: BlocksBroadcaster - sendPresence*: PresenceBroadcaster - sendAccount*: AccountBroadcaster - sendPayment*: PaymentBroadcaster - - BlockExcNetwork* = ref object of LPProtocol - peers*: Table[PeerID, NetworkPeer] - switch*: Switch - handlers*: BlockExcHandlers - request*: BlockExcRequest - getConn: ConnProvider - -proc handleWantList( - b: BlockExcNetwork, - peer: NetworkPeer, - list: WantList): Future[void] = - ## Handle incoming want list - ## - - if isNil(b.handlers.onWantList): - return - - trace "Handling want list for peer", peer = peer.id - b.handlers.onWantList(peer.id, list) - -# TODO: make into a template -proc makeWantList*( - cids: seq[Cid], - priority: int = 0, - cancel: bool = false, - wantType: WantType = WantType.wantHave, - full: bool = false, - sendDontHave: bool = false): WantList = - var entries: seq[Entry] - for cid in cids: - entries.add(Entry( - `block`: cid.data.buffer, - priority: priority.int32, - cancel: cancel, - wantType: wantType, - sendDontHave: sendDontHave)) - - WantList(entries: entries, full: full) - -proc broadcastWantList*( - b: BlockExcNetwork, - id: PeerID, - cids: seq[Cid], - priority: int32 = 0, - cancel: bool = false, - wantType: WantType = WantType.wantHave, - full: bool = false, - sendDontHave: bool = false) = - ## send a want message to peer - ## - - if id notin b.peers: - return - - trace "Sending want list to peer", peer = id, `type` = $wantType, len = cids.len - - let - wantList = makeWantList( - cids, - priority, - cancel, - wantType, - full, - sendDontHave) - b.peers.withValue(id, peer): - peer[].broadcast(Message(wantlist: wantList)) - -proc handleBlocks( - b: BlockExcNetwork, - peer: NetworkPeer, - blocks: seq[pb.Block]): Future[void] = - ## Handle incoming blocks - ## - - if isNil(b.handlers.onBlocks): - return - - trace "Handling blocks for peer", peer = peer.id - - var blks: seq[bt.Block] - for blob in blocks: - without cid =? Cid.init(blob.prefix): - trace "Unable to initialize Cid from protobuf message" - - without blk =? bt.Block.new(cid, blob.data, verify = true): - trace "Unable to initialize Block from data" - - blks.add(blk) - - b.handlers.onBlocks(peer.id, blks) - -template makeBlocks*(blocks: seq[bt.Block]): seq[pb.Block] = - var blks: seq[pb.Block] - for blk in blocks: - blks.add(pb.Block( - prefix: blk.cid.data.buffer, - data: blk.data - )) - - blks - -proc broadcastBlocks*( - b: BlockExcNetwork, - id: PeerID, - blocks: seq[bt.Block]) = - ## Send blocks to remote - ## - - if id notin b.peers: - trace "Unable to send blocks, peer disconnected", peer = id - return - - b.peers.withValue(id, peer): - trace "Sending blocks to peer", peer = id, len = blocks.len - peer[].broadcast(pb.Message(payload: makeBlocks(blocks))) - -proc handleBlockPresence( - b: BlockExcNetwork, - peer: NetworkPeer, - presence: seq[BlockPresence]): Future[void] = - ## Handle block presence - ## - - if isNil(b.handlers.onPresence): - return - - trace "Handling block presence for peer", peer = peer.id - b.handlers.onPresence(peer.id, presence) - -proc broadcastBlockPresence*( - b: BlockExcNetwork, - id: PeerID, - presence: seq[BlockPresence]) = - ## Send presence to remote - ## - - if id notin b.peers: - return - - trace "Sending presence to peer", peer = id - b.peers.withValue(id, peer): - peer[].broadcast(Message(blockPresences: @presence)) - -proc handleAccount(network: BlockExcNetwork, - peer: NetworkPeer, - account: Account): Future[void] = - if network.handlers.onAccount.isNil: - return - network.handlers.onAccount(peer.id, account) - -proc broadcastAccount*(network: BlockExcNetwork, - id: PeerId, - account: Account) = - if id notin network.peers: - return - - let message = Message(account: AccountMessage.init(account)) - network.peers.withValue(id, peer): - peer[].broadcast(message) - -proc broadcastPayment*(network: BlockExcNetwork, - id: PeerId, - payment: SignedState) = - if id notin network.peers: - return - - let message = Message(payment: StateChannelUpdate.init(payment)) - network.peers.withValue(id, peer): - peer[].broadcast(message) - -proc handlePayment(network: BlockExcNetwork, - peer: NetworkPeer, - payment: SignedState): Future[void] = - if network.handlers.onPayment.isNil: - return - network.handlers.onPayment(peer.id, payment) - -proc rpcHandler(b: BlockExcNetwork, peer: NetworkPeer, msg: Message) {.async.} = - try: - if msg.wantlist.entries.len > 0: - asyncSpawn b.handleWantList(peer, msg.wantlist) - - if msg.payload.len > 0: - asyncSpawn b.handleBlocks(peer, msg.payload) - - if msg.blockPresences.len > 0: - asyncSpawn b.handleBlockPresence(peer, msg.blockPresences) - - if account =? Account.init(msg.account): - asyncSpawn b.handleAccount(peer, account) - - if payment =? SignedState.init(msg.payment): - asyncSpawn b.handlePayment(peer, payment) - - except CatchableError as exc: - trace "Exception in blockexc rpc handler", exc = exc.msg - -proc getOrCreatePeer(b: BlockExcNetwork, peer: PeerID): NetworkPeer = - ## Creates or retrieves a BlockExcNetwork Peer - ## - - if peer in b.peers: - return b.peers.getOrDefault(peer, nil) - - var getConn = proc(): Future[Connection] {.async.} = - try: - return await b.switch.dial(peer, Codec) - except CatchableError as exc: - trace "Unable to connect to blockexc peer", exc = exc.msg - - if not isNil(b.getConn): - getConn = b.getConn - - let rpcHandler = proc (p: NetworkPeer, msg: Message): Future[void] = - b.rpcHandler(p, msg) - - # create new pubsub peer - let blockExcPeer = NetworkPeer.new(peer, getConn, rpcHandler) - debug "Created new blockexc peer", peer - - b.peers[peer] = blockExcPeer - - return blockExcPeer - -proc setupPeer*(b: BlockExcNetwork, peer: PeerID) = - ## Perform initial setup, such as want - ## list exchange - ## - - discard b.getOrCreatePeer(peer) - -proc dialPeer*(b: BlockExcNetwork, peer: PeerRecord) {.async.} = - try: - await b.switch.connect(peer.peerId, peer.addresses.mapIt(it.address)) - except CatchableError as exc: - debug "Failed to connect to peer", error = exc.msg, peer - -proc dropPeer*(b: BlockExcNetwork, peer: PeerID) = - ## Cleanup disconnected peer - ## - - b.peers.del(peer) - -method init*(b: BlockExcNetwork) = - ## Perform protocol initialization - ## - - proc peerEventHandler(peerId: PeerID, event: PeerEvent) {.async.} = - if event.kind == PeerEventKind.Joined: - b.setupPeer(peerId) - else: - b.dropPeer(peerId) - - b.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Joined) - b.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Left) - - proc handle(conn: Connection, proto: string) {.async, gcsafe, closure.} = - let peerId = conn.peerId - let blockexcPeer = b.getOrCreatePeer(peerId) - await blockexcPeer.readLoop(conn) # attach read loop - - b.handler = handle - b.codec = Codec - -proc new*( - T: type BlockExcNetwork, - switch: Switch, - connProvider: ConnProvider = nil): T = - ## Create a new BlockExcNetwork instance - ## - - let b = BlockExcNetwork( - switch: switch, - getConn: connProvider) - - proc sendWantList( - id: PeerID, - cids: seq[Cid], - priority: int32 = 0, - cancel: bool = false, - wantType: WantType = WantType.wantHave, - full: bool = false, - sendDontHave: bool = false) {.gcsafe.} = - b.broadcastWantList( - id, cids, priority, cancel, - wantType, full, sendDontHave) - - proc sendBlocks(id: PeerID, blocks: seq[bt.Block]) {.gcsafe.} = - b.broadcastBlocks(id, blocks) - - proc sendPresence(id: PeerID, presence: seq[BlockPresence]) {.gcsafe.} = - b.broadcastBlockPresence(id, presence) - - proc sendAccount(id: PeerID, account: Account) = - b.broadcastAccount(id, account) - - proc sendPayment(id: PeerID, payment: SignedState) = - b.broadcastPayment(id, payment) - - b.request = BlockExcRequest( - sendWantList: sendWantList, - sendBlocks: sendBlocks, - sendPresence: sendPresence, - sendAccount: sendAccount, - sendPayment: sendPayment) - - b.init() - return b +export network, networkpeer diff --git a/codex/blockexchange/network/network.nim b/codex/blockexchange/network/network.nim new file mode 100644 index 00000000..d50bfd84 --- /dev/null +++ b/codex/blockexchange/network/network.nim @@ -0,0 +1,375 @@ +## Nim-Codex +## Copyright (c) 2021 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/tables +import std/sequtils + +import pkg/chronicles +import pkg/chronos + +import pkg/libp2p +import pkg/questionable +import pkg/questionable/results + +import ../../blocktype as bt +import ../protobuf/blockexc as pb +import ../protobuf/payments + +import ./networkpeer + +export network, payments + +logScope: + topics = "codex blockexc network" + +const Codec* = "/codex/blockexc/1.0.0" + +type + WantListHandler* = proc(peer: PeerID, wantList: WantList): Future[void] {.gcsafe.} + BlocksHandler* = proc(peer: PeerID, blocks: seq[bt.Block]): Future[void] {.gcsafe.} + BlockPresenceHandler* = proc(peer: PeerID, precense: seq[BlockPresence]): Future[void] {.gcsafe.} + AccountHandler* = proc(peer: PeerID, account: Account): Future[void] {.gcsafe.} + PaymentHandler* = proc(peer: PeerID, payment: SignedState): Future[void] {.gcsafe.} + + BlockExcHandlers* = object + onWantList*: WantListHandler + onBlocks*: BlocksHandler + onPresence*: BlockPresenceHandler + onAccount*: AccountHandler + onPayment*: PaymentHandler + + WantListBroadcaster* = proc( + id: PeerID, + cids: seq[Cid], + priority: int32 = 0, + cancel: bool = false, + wantType: WantType = WantType.wantHave, + full: bool = false, + sendDontHave: bool = false) {.gcsafe.} + + BlocksBroadcaster* = proc(peer: PeerID, presence: seq[bt.Block]) {.gcsafe.} + PresenceBroadcaster* = proc(peer: PeerID, presence: seq[BlockPresence]) {.gcsafe.} + AccountBroadcaster* = proc(peer: PeerID, account: Account) {.gcsafe.} + PaymentBroadcaster* = proc(peer: PeerID, payment: SignedState) {.gcsafe.} + + BlockExcRequest* = object + sendWantList*: WantListBroadcaster + sendBlocks*: BlocksBroadcaster + sendPresence*: PresenceBroadcaster + sendAccount*: AccountBroadcaster + sendPayment*: PaymentBroadcaster + + BlockExcNetwork* = ref object of LPProtocol + peers*: Table[PeerID, NetworkPeer] + switch*: Switch + handlers*: BlockExcHandlers + request*: BlockExcRequest + getConn: ConnProvider + +proc handleWantList( + b: BlockExcNetwork, + peer: NetworkPeer, + list: WantList): Future[void] = + ## Handle incoming want list + ## + + if isNil(b.handlers.onWantList): + return + + trace "Handling want list for peer", peer = peer.id + b.handlers.onWantList(peer.id, list) + +# TODO: make into a template +proc makeWantList*( + cids: seq[Cid], + priority: int = 0, + cancel: bool = false, + wantType: WantType = WantType.wantHave, + full: bool = false, + sendDontHave: bool = false): WantList = + var entries: seq[Entry] + for cid in cids: + entries.add(Entry( + `block`: cid.data.buffer, + priority: priority.int32, + cancel: cancel, + wantType: wantType, + sendDontHave: sendDontHave)) + + WantList(entries: entries, full: full) + +proc broadcastWantList*( + b: BlockExcNetwork, + id: PeerID, + cids: seq[Cid], + priority: int32 = 0, + cancel: bool = false, + wantType: WantType = WantType.wantHave, + full: bool = false, + sendDontHave: bool = false) = + ## send a want message to peer + ## + + if id notin b.peers: + return + + trace "Sending want list to peer", peer = id, `type` = $wantType, len = cids.len + + let + wantList = makeWantList( + cids, + priority, + cancel, + wantType, + full, + sendDontHave) + b.peers.withValue(id, peer): + peer[].broadcast(Message(wantlist: wantList)) + +proc handleBlocks( + b: BlockExcNetwork, + peer: NetworkPeer, + blocks: seq[pb.Block]): Future[void] = + ## Handle incoming blocks + ## + + if isNil(b.handlers.onBlocks): + return + + trace "Handling blocks for peer", peer = peer.id + + var blks: seq[bt.Block] + for blob in blocks: + without cid =? Cid.init(blob.prefix): + trace "Unable to initialize Cid from protobuf message" + + without blk =? bt.Block.new(cid, blob.data, verify = true): + trace "Unable to initialize Block from data" + + blks.add(blk) + + b.handlers.onBlocks(peer.id, blks) + +template makeBlocks*(blocks: seq[bt.Block]): seq[pb.Block] = + var blks: seq[pb.Block] + for blk in blocks: + blks.add(pb.Block( + prefix: blk.cid.data.buffer, + data: blk.data + )) + + blks + +proc broadcastBlocks*( + b: BlockExcNetwork, + id: PeerID, + blocks: seq[bt.Block]) = + ## Send blocks to remote + ## + + if id notin b.peers: + trace "Unable to send blocks, peer disconnected", peer = id + return + + b.peers.withValue(id, peer): + trace "Sending blocks to peer", peer = id, len = blocks.len + peer[].broadcast(pb.Message(payload: makeBlocks(blocks))) + +proc handleBlockPresence( + b: BlockExcNetwork, + peer: NetworkPeer, + presence: seq[BlockPresence]): Future[void] = + ## Handle block presence + ## + + if isNil(b.handlers.onPresence): + return + + trace "Handling block presence for peer", peer = peer.id + b.handlers.onPresence(peer.id, presence) + +proc broadcastBlockPresence*( + b: BlockExcNetwork, + id: PeerID, + presence: seq[BlockPresence]) = + ## Send presence to remote + ## + + if id notin b.peers: + return + + trace "Sending presence to peer", peer = id + b.peers.withValue(id, peer): + peer[].broadcast(Message(blockPresences: @presence)) + +proc handleAccount(network: BlockExcNetwork, + peer: NetworkPeer, + account: Account): Future[void] = + if network.handlers.onAccount.isNil: + return + network.handlers.onAccount(peer.id, account) + +proc broadcastAccount*(network: BlockExcNetwork, + id: PeerId, + account: Account) = + if id notin network.peers: + return + + let message = Message(account: AccountMessage.init(account)) + network.peers.withValue(id, peer): + peer[].broadcast(message) + +proc broadcastPayment*(network: BlockExcNetwork, + id: PeerId, + payment: SignedState) = + if id notin network.peers: + return + + let message = Message(payment: StateChannelUpdate.init(payment)) + network.peers.withValue(id, peer): + peer[].broadcast(message) + +proc handlePayment(network: BlockExcNetwork, + peer: NetworkPeer, + payment: SignedState): Future[void] = + if network.handlers.onPayment.isNil: + return + network.handlers.onPayment(peer.id, payment) + +proc rpcHandler(b: BlockExcNetwork, peer: NetworkPeer, msg: Message) {.async.} = + try: + if msg.wantlist.entries.len > 0: + asyncSpawn b.handleWantList(peer, msg.wantlist) + + if msg.payload.len > 0: + asyncSpawn b.handleBlocks(peer, msg.payload) + + if msg.blockPresences.len > 0: + asyncSpawn b.handleBlockPresence(peer, msg.blockPresences) + + if account =? Account.init(msg.account): + asyncSpawn b.handleAccount(peer, account) + + if payment =? SignedState.init(msg.payment): + asyncSpawn b.handlePayment(peer, payment) + + except CatchableError as exc: + trace "Exception in blockexc rpc handler", exc = exc.msg + +proc getOrCreatePeer(b: BlockExcNetwork, peer: PeerID): NetworkPeer = + ## Creates or retrieves a BlockExcNetwork Peer + ## + + if peer in b.peers: + return b.peers.getOrDefault(peer, nil) + + var getConn = proc(): Future[Connection] {.async.} = + try: + return await b.switch.dial(peer, Codec) + except CatchableError as exc: + trace "Unable to connect to blockexc peer", exc = exc.msg + + if not isNil(b.getConn): + getConn = b.getConn + + let rpcHandler = proc (p: NetworkPeer, msg: Message): Future[void] = + b.rpcHandler(p, msg) + + # create new pubsub peer + let blockExcPeer = NetworkPeer.new(peer, getConn, rpcHandler) + debug "Created new blockexc peer", peer + + b.peers[peer] = blockExcPeer + + return blockExcPeer + +proc setupPeer*(b: BlockExcNetwork, peer: PeerID) = + ## Perform initial setup, such as want + ## list exchange + ## + + discard b.getOrCreatePeer(peer) + +proc dialPeer*(b: BlockExcNetwork, peer: PeerRecord) {.async.} = + try: + await b.switch.connect(peer.peerId, peer.addresses.mapIt(it.address)) + except CatchableError as exc: + debug "Failed to connect to peer", error = exc.msg, peer + +proc dropPeer*(b: BlockExcNetwork, peer: PeerID) = + ## Cleanup disconnected peer + ## + + b.peers.del(peer) + +method init*(b: BlockExcNetwork) = + ## Perform protocol initialization + ## + + proc peerEventHandler(peerId: PeerID, event: PeerEvent) {.async.} = + if event.kind == PeerEventKind.Joined: + b.setupPeer(peerId) + else: + b.dropPeer(peerId) + + b.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Joined) + b.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Left) + + proc handle(conn: Connection, proto: string) {.async, gcsafe, closure.} = + let peerId = conn.peerId + let blockexcPeer = b.getOrCreatePeer(peerId) + await blockexcPeer.readLoop(conn) # attach read loop + + b.handler = handle + b.codec = Codec + +proc new*( + T: type BlockExcNetwork, + switch: Switch, + connProvider: ConnProvider = nil): T = + ## Create a new BlockExcNetwork instance + ## + + let b = BlockExcNetwork( + switch: switch, + getConn: connProvider) + + proc sendWantList( + id: PeerID, + cids: seq[Cid], + priority: int32 = 0, + cancel: bool = false, + wantType: WantType = WantType.wantHave, + full: bool = false, + sendDontHave: bool = false) {.gcsafe.} = + b.broadcastWantList( + id, cids, priority, cancel, + wantType, full, sendDontHave) + + proc sendBlocks(id: PeerID, blocks: seq[bt.Block]) {.gcsafe.} = + b.broadcastBlocks(id, blocks) + + proc sendPresence(id: PeerID, presence: seq[BlockPresence]) {.gcsafe.} = + b.broadcastBlockPresence(id, presence) + + proc sendAccount(id: PeerID, account: Account) = + b.broadcastAccount(id, account) + + proc sendPayment(id: PeerID, payment: SignedState) = + b.broadcastPayment(id, payment) + + b.request = BlockExcRequest( + sendWantList: sendWantList, + sendBlocks: sendBlocks, + sendPresence: sendPresence, + sendAccount: sendAccount, + sendPayment: sendPayment) + + b.init() + return b diff --git a/codex/blockexchange/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim similarity index 98% rename from codex/blockexchange/networkpeer.nim rename to codex/blockexchange/network/networkpeer.nim index 1c44c3c2..964a8586 100644 --- a/codex/blockexchange/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -12,7 +12,7 @@ import pkg/chronicles import pkg/protobuf_serialization import pkg/libp2p -import ./protobuf/blockexc +import ../protobuf/blockexc logScope: topics = "codex blockexc networkpeer" diff --git a/codex/blockexchange/peers.nim b/codex/blockexchange/peers.nim index e159806a..940f66e4 100644 --- a/codex/blockexchange/peers.nim +++ b/codex/blockexchange/peers.nim @@ -1,3 +1,4 @@ import ./peers/peerctxstore +import ./peers/peercontext -export peerctxstore +export peerctxstore, peercontext diff --git a/codex/blockexchange/peercontext.nim b/codex/blockexchange/peers/peercontext.nim similarity index 76% rename from codex/blockexchange/peercontext.nim rename to codex/blockexchange/peers/peercontext.nim index 0d303703..9aab35b2 100644 --- a/codex/blockexchange/peercontext.nim +++ b/codex/blockexchange/peers/peercontext.nim @@ -1,3 +1,12 @@ +## Nim-Codex +## Copyright (c) 2021 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + import std/sequtils import std/tables import pkg/libp2p @@ -5,9 +14,9 @@ import pkg/chronos import pkg/nitro import pkg/questionable -import ./protobuf/blockexc -import ./protobuf/payments -import ./protobuf/presence +import ../protobuf/blockexc +import ../protobuf/payments +import ../protobuf/presence export payments, nitro diff --git a/codex/blockexchange/peers/peerctxstore.nim b/codex/blockexchange/peers/peerctxstore.nim index 77ad6f60..442dd206 100644 --- a/codex/blockexchange/peers/peerctxstore.nim +++ b/codex/blockexchange/peers/peerctxstore.nim @@ -21,7 +21,7 @@ import pkg/libp2p import ../protobuf/blockexc -import ../peercontext +import ./peercontext export peercontext logScope: diff --git a/tests/codex/blockexc/discovery/testdiscovery.nim b/tests/codex/blockexchange/discovery/testdiscovery.nim similarity index 100% rename from tests/codex/blockexc/discovery/testdiscovery.nim rename to tests/codex/blockexchange/discovery/testdiscovery.nim diff --git a/tests/codex/blockexc/discovery/testdiscoveryengine.nim b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim similarity index 100% rename from tests/codex/blockexc/discovery/testdiscoveryengine.nim rename to tests/codex/blockexchange/discovery/testdiscoveryengine.nim diff --git a/tests/codex/blockexc/testblockexc.nim b/tests/codex/blockexchange/engine/testblockexc.nim similarity index 99% rename from tests/codex/blockexc/testblockexc.nim rename to tests/codex/blockexchange/engine/testblockexc.nim index 351dfe0c..6c2d6557 100644 --- a/tests/codex/blockexc/testblockexc.nim +++ b/tests/codex/blockexchange/engine/testblockexc.nim @@ -15,8 +15,8 @@ import pkg/codex/chunker import pkg/codex/discovery import pkg/codex/blocktype as bt -import ../helpers -import ../examples +import ../../helpers +import ../../examples suite "NetworkStore engine - 2 nodes": let diff --git a/tests/codex/blockexc/testengine.nim b/tests/codex/blockexchange/engine/testengine.nim similarity index 99% rename from tests/codex/blockexc/testengine.nim rename to tests/codex/blockexchange/engine/testengine.nim index eb128327..e6193e70 100644 --- a/tests/codex/blockexc/testengine.nim +++ b/tests/codex/blockexchange/engine/testengine.nim @@ -17,8 +17,8 @@ import pkg/codex/discovery import pkg/codex/blocktype as bt import pkg/codex/utils/asyncheapqueue -import ../helpers -import ../examples +import ../../helpers +import ../../examples suite "NetworkStore engine basic": var diff --git a/tests/codex/blockexc/engine/testpayments.nim b/tests/codex/blockexchange/engine/testpayments.nim similarity index 100% rename from tests/codex/blockexc/engine/testpayments.nim rename to tests/codex/blockexchange/engine/testpayments.nim diff --git a/tests/codex/blockexc/protobuf/testpayments.nim b/tests/codex/blockexchange/protobuf/testpayments.nim similarity index 100% rename from tests/codex/blockexc/protobuf/testpayments.nim rename to tests/codex/blockexchange/protobuf/testpayments.nim diff --git a/tests/codex/blockexc/protobuf/testpresence.nim b/tests/codex/blockexchange/protobuf/testpresence.nim similarity index 100% rename from tests/codex/blockexc/protobuf/testpresence.nim rename to tests/codex/blockexchange/protobuf/testpresence.nim diff --git a/tests/codex/blockexchange/testdiscovery.nim b/tests/codex/blockexchange/testdiscovery.nim new file mode 100644 index 00000000..6f443014 --- /dev/null +++ b/tests/codex/blockexchange/testdiscovery.nim @@ -0,0 +1,4 @@ +import ./discovery/testdiscovery +import ./discovery/testdiscoveryengine + +{.warning[UnusedImport]: off.} diff --git a/tests/codex/blockexchange/testengine.nim b/tests/codex/blockexchange/testengine.nim new file mode 100644 index 00000000..5277e027 --- /dev/null +++ b/tests/codex/blockexchange/testengine.nim @@ -0,0 +1,5 @@ +import ./engine/testengine +import ./engine/testblockexc +import ./engine/testpayments + +{.warning[UnusedImport]: off.} diff --git a/tests/codex/blockexc/testnetwork.nim b/tests/codex/blockexchange/testnetwork.nim similarity index 100% rename from tests/codex/blockexc/testnetwork.nim rename to tests/codex/blockexchange/testnetwork.nim diff --git a/tests/codex/blockexc/testpeerctxstore.nim b/tests/codex/blockexchange/testpeerctxstore.nim similarity index 100% rename from tests/codex/blockexc/testpeerctxstore.nim rename to tests/codex/blockexchange/testpeerctxstore.nim diff --git a/tests/codex/blockexchange/testprotobuf.nim b/tests/codex/blockexchange/testprotobuf.nim new file mode 100644 index 00000000..1d41d6b4 --- /dev/null +++ b/tests/codex/blockexchange/testprotobuf.nim @@ -0,0 +1,4 @@ +import ./protobuf/testpayments +import ./protobuf/testpresence + +{.warning[UnusedImport]: off.} diff --git a/tests/codex/testblockexc.nim b/tests/codex/testblockexc.nim index e12e9e0d..bb7ace3d 100644 --- a/tests/codex/testblockexc.nim +++ b/tests/codex/testblockexc.nim @@ -1,11 +1,7 @@ -import ./blockexc/testengine -import ./blockexc/testnetwork -import ./blockexc/testblockexc -import ./blockexc/testpeerctxstore -import ./blockexc/discovery/testdiscovery -import ./blockexc/discovery/testdiscoveryengine -import ./blockexc/protobuf/testpayments as testprotobufpayments -import ./blockexc/protobuf/testpresence -import ./blockexc/engine/testpayments as testenginepayments +import ./blockexchange/testengine +import ./blockexchange/testnetwork +import ./blockexchange/testpeerctxstore +import ./blockexchange/testdiscovery +import ./blockexchange/testprotobuf {.warning[UnusedImport]: off.} From 6ad7a6bb96cc789d5038ab6a064be59ea6e1a9fb Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 20 May 2022 10:53:34 -0600 Subject: [PATCH 4/7] Batched blocks (#105) * remove sleep * throttle prefetch * break loop if conn closed * logging * prefetch blocks in batches * add blocks batch constant --- codex/blockexchange/network/network.nim | 12 ++++++------ codex/blockexchange/network/networkpeer.nim | 2 +- codex/node.nim | 11 +++++++++-- codex/stores/fsstore.nim | 4 ---- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/codex/blockexchange/network/network.nim b/codex/blockexchange/network/network.nim index d50bfd84..40ba33a2 100644 --- a/codex/blockexchange/network/network.nim +++ b/codex/blockexchange/network/network.nim @@ -82,7 +82,7 @@ proc handleWantList( if isNil(b.handlers.onWantList): return - trace "Handling want list for peer", peer = peer.id + trace "Handling want list for peer", peer = peer.id, items = list.entries.len b.handlers.onWantList(peer.id, list) # TODO: make into a template @@ -119,7 +119,7 @@ proc broadcastWantList*( if id notin b.peers: return - trace "Sending want list to peer", peer = id, `type` = $wantType, len = cids.len + trace "Sending want list to peer", peer = id, `type` = $wantType, items = cids.len let wantList = makeWantList( @@ -142,7 +142,7 @@ proc handleBlocks( if isNil(b.handlers.onBlocks): return - trace "Handling blocks for peer", peer = peer.id + trace "Handling blocks for peer", peer = peer.id, items = blocks.len var blks: seq[bt.Block] for blob in blocks: @@ -178,7 +178,7 @@ proc broadcastBlocks*( return b.peers.withValue(id, peer): - trace "Sending blocks to peer", peer = id, len = blocks.len + trace "Sending blocks to peer", peer = id, items = blocks.len peer[].broadcast(pb.Message(payload: makeBlocks(blocks))) proc handleBlockPresence( @@ -191,7 +191,7 @@ proc handleBlockPresence( if isNil(b.handlers.onPresence): return - trace "Handling block presence for peer", peer = peer.id + trace "Handling block presence for peer", peer = peer.id, items = presence.len b.handlers.onPresence(peer.id, presence) proc broadcastBlockPresence*( @@ -204,7 +204,7 @@ proc broadcastBlockPresence*( if id notin b.peers: return - trace "Sending presence to peer", peer = id + trace "Sending presence to peer", peer = id, items = presence.len b.peers.withValue(id, peer): peer[].broadcast(Message(blockPresences: @presence)) diff --git a/codex/blockexchange/network/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim index 964a8586..1a213383 100644 --- a/codex/blockexchange/network/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -38,7 +38,7 @@ proc readLoop*(b: NetworkPeer, conn: Connection) {.async.} = return try: - while not conn.atEof: + while not conn.atEof or not conn.closed: let data = await conn.readLp(MaxMessageSize) msg: Message = Protobuf.decode(data, Message) diff --git a/codex/node.nim b/codex/node.nim index 82e73763..8ae3a1c9 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -34,6 +34,9 @@ import ./contracts logScope: topics = "codex node" +const + PrefetchBatch = 100 + type CodexError = object of CatchableError @@ -128,8 +131,12 @@ proc retrieve*( ## Initiates requests to all blocks in the manifest ## try: - discard await allFinished( - manifest.mapIt( node.blockStore.getBlock( it ) )) + let + batch = manifest.blocks.len div PrefetchBatch + trace "Prefetching in batches of", batch + for blks in manifest.blocks.distribute(batch, true): + discard await allFinished( + blks.mapIt( node.blockStore.getBlock( it ) )) except CatchableError as exc: trace "Exception prefetching blocks", exc = exc.msg diff --git a/codex/stores/fsstore.nim b/codex/stores/fsstore.nim index ac98d42d..de846e50 100644 --- a/codex/stores/fsstore.nim +++ b/codex/stores/fsstore.nim @@ -150,10 +150,6 @@ method listBlocks*(self: FSStore, onBlock: OnBlock) {.async.} = except CatchableError as exc: trace "Couldn't get block", cid = $(cid.get()) - # TODO: this should run on a thread which - # wouldn't need the sleep - await sleepAsync(100.millis) # avoid blocking - proc new*( T: type FSStore, repoDir: string, From 56b80d6f6d05c8c4c4dee76a310e7eda52da8fa0 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Mon, 23 May 2022 23:24:15 -0600 Subject: [PATCH 5/7] Por serialize (#106) * move por into storage proofs * use SeekableStream * adding serialization for por * remove leftovers * add empty block support * add basic por test * rename block exchange for consistency * add storageproofstests * moving timing to storageproofs * fix imports * fix imports * fix imports * add top level exports * move delete blocks helper to helpers * more import/export fixes * cleanup * more import fixes * fix unused warnings * detect corrupt blocks tests * add serialization tests * move init method around * bump asynctest * fix CID version * get rid of warning * wip: fix CI * increase CI timeout --- .github/workflows/ci.yml | 2 +- codex/blocktype.nim | 2 +- codex/contracts/proofs.nim | 2 +- codex/por/README.md | 1 - codex/por/benchmark.nim | 9 - codex/por/example.txt | 1 - codex/por/testpor.nim | 37 -- codex/proving.nim | 4 +- codex/rest/api.nim | 8 +- codex/storageproofs.nim | 4 + codex/storageproofs/por.nim | 4 + codex/{ => storageproofs}/por/por.nim | 363 ++++++++++++------ codex/storageproofs/por/serialization.nim | 3 + .../storageproofs/por/serialization/por.proto | 33 ++ .../por/serialization/serialization.nim | 170 ++++++++ codex/storageproofs/timing.nim | 4 + .../{por => storageproofs}/timing/periods.nim | 0 .../{por => storageproofs}/timing/proofs.nim | 0 codex/streams/storestream.nim | 20 +- .../discovery/testdiscoveryengine.nim | 1 - .../blockexchange/engine/testblockexc.nim | 1 - tests/codex/helpers.nim | 41 +- tests/codex/helpers/mockproofs.nim | 2 +- tests/codex/storageproofs/testpor.nim | 164 ++++++++ ...testblockexc.nim => testblockexchange.nim} | 0 tests/codex/teststorageproofs.nim | 3 + tests/codex/teststorestream.nim | 1 - tests/contracts/testContracts.nim | 2 +- tests/testCodex.nim | 3 +- vendor/asynctest | 2 +- 30 files changed, 697 insertions(+), 190 deletions(-) delete mode 100644 codex/por/README.md delete mode 100644 codex/por/benchmark.nim delete mode 100644 codex/por/example.txt delete mode 100644 codex/por/testpor.nim create mode 100644 codex/storageproofs.nim create mode 100644 codex/storageproofs/por.nim rename codex/{ => storageproofs}/por/por.nim (67%) create mode 100644 codex/storageproofs/por/serialization.nim create mode 100644 codex/storageproofs/por/serialization/por.proto create mode 100644 codex/storageproofs/por/serialization/serialization.nim create mode 100644 codex/storageproofs/timing.nim rename codex/{por => storageproofs}/timing/periods.nim (100%) rename codex/{por => storageproofs}/timing/proofs.nim (100%) create mode 100644 tests/codex/storageproofs/testpor.nim rename tests/codex/{testblockexc.nim => testblockexchange.nim} (100%) create mode 100644 tests/codex/teststorageproofs.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34f58414..bc1ca755 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: builder: windows-2019 name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.branch }}' runs-on: ${{ matrix.builder }} - timeout-minutes: 40 + timeout-minutes: 60 steps: - name: Checkout nim-codex uses: actions/checkout@v2 diff --git a/codex/blocktype.nim b/codex/blocktype.nim index b324c2a4..f16d155f 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -59,7 +59,7 @@ template EmptyDigests*: untyped = .get() }.toTable, CIDv1: { - multiCodec("sha2-256"): EmptyCid[CIDv0] + multiCodec("sha2-256"): EmptyCid[CIDv1] .catch .get()[multiCodec("sha2-256")] .catch diff --git a/codex/contracts/proofs.nim b/codex/contracts/proofs.nim index 0c835e68..4b6a7535 100644 --- a/codex/contracts/proofs.nim +++ b/codex/contracts/proofs.nim @@ -1,5 +1,5 @@ import pkg/ethers -import ../por/timing/proofs +import ../storageproofs/timing/proofs import ./storage export proofs diff --git a/codex/por/README.md b/codex/por/README.md deleted file mode 100644 index 1ee54b5e..00000000 --- a/codex/por/README.md +++ /dev/null @@ -1 +0,0 @@ -Nim implementation of Proof of Storage related schemes diff --git a/codex/por/benchmark.nim b/codex/por/benchmark.nim deleted file mode 100644 index 332b971a..00000000 --- a/codex/por/benchmark.nim +++ /dev/null @@ -1,9 +0,0 @@ -import times, strutils -export strutils.formatFloat - -template benchmark*(benchmarkName: string, code: untyped) = - let t0 = epochTime() - code - let elapsed = epochTime() - t0 - let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 3) - echo "CPU Time [", benchmarkName, "] ", elapsedStr, "s" diff --git a/codex/por/example.txt b/codex/por/example.txt deleted file mode 100644 index 5be377b5..00000000 --- a/codex/por/example.txt +++ /dev/null @@ -1 +0,0 @@ -The quick brown fox jumps over the lazy dog! diff --git a/codex/por/testpor.nim b/codex/por/testpor.nim deleted file mode 100644 index 718ee9d7..00000000 --- a/codex/por/testpor.nim +++ /dev/null @@ -1,37 +0,0 @@ -## Nim-POS -## Copyright (c) 2021 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import por -import benchmark -import strutils - -const sectorsperblock = 1024.int64 -const querylen = 22 - -proc testbls() : bool = - benchmark "Key generation": - let (spk, ssk) = por.keygen() - - benchmark "Auth generation (s=" & $sectorsperblock & ")": - let (tau, authenticators) = por.setup(ssk, sectorsperblock, "example.txt") - #echo "Auth: ", authenticators - - benchmark "Generating challenge (q=" & $querylen & ")": - let q = por.generateQuery(tau, spk, querylen) - #echo "Generated!" #, " q:", q - - benchmark "Issuing proof": - let (mu, sigma) = por.generateProof(q, authenticators, spk, sectorsperblock, "example.txt") - #echo "Issued!" #, " mu:", mu, " sigma:", sigma - - benchmark "Verifying proof": - result = por.verifyProof(tau, q, mu, sigma, spk) - echo "Result: ", result - -let r = testbls() diff --git a/codex/proving.nim b/codex/proving.nim index 26107296..6c65b775 100644 --- a/codex/proving.nim +++ b/codex/proving.nim @@ -2,11 +2,11 @@ import std/sets import pkg/upraises import pkg/questionable import pkg/chronicles -import ./por/timing/proofs +import ./storageproofs import ./clock export sets -export proofs +export storageproofs type Proving* = ref object diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 22a27c1a..4b6efac9 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -47,10 +47,10 @@ proc decodeString(T: type Cid, value: string): Result[Cid, cstring] = .init(value) .mapErr do(e: CidError) -> cstring: case e - of CidError.Incorrect: "Incorrect Cid" - of CidError.Unsupported: "Unsupported Cid" - of CidError.Overrun: "Overrun Cid" - else: "Error parsing Cid" + of CidError.Incorrect: "Incorrect Cid".cstring + of CidError.Unsupported: "Unsupported Cid".cstring + of CidError.Overrun: "Overrun Cid".cstring + else: "Error parsing Cid".cstring proc encodeString(peerId: PeerID): Result[string, cstring] = ok($peerId) diff --git a/codex/storageproofs.nim b/codex/storageproofs.nim new file mode 100644 index 00000000..e23ecfbf --- /dev/null +++ b/codex/storageproofs.nim @@ -0,0 +1,4 @@ +import ./storageproofs/por +import ./storageproofs/timing + +export por, timing diff --git a/codex/storageproofs/por.nim b/codex/storageproofs/por.nim new file mode 100644 index 00000000..79ada5dd --- /dev/null +++ b/codex/storageproofs/por.nim @@ -0,0 +1,4 @@ +import ./por/serialization +import ./por/por + +export por, serialization diff --git a/codex/por/por.nim b/codex/storageproofs/por/por.nim similarity index 67% rename from codex/por/por.nim rename to codex/storageproofs/por/por.nim index d0d1f7c4..03207e32 100644 --- a/codex/por/por.nim +++ b/codex/storageproofs/por/por.nim @@ -39,7 +39,7 @@ # Our implementation uses additive cyclic groups instead of the multiplicative # cyclic group in the paper, thus changing the name of the group operation as in # blscurve and blst. Thus, point multiplication becomes point addition, and scalar -# exponentiation becomes scalar multiplication. +# exponentiation becomes scalar multiplicaiton. # # Number of operations: # The following table summarizes the number of operations in different phases @@ -77,77 +77,108 @@ # q * (8 + 48) bytes # The size of the proof is instead # s * 32 + 48 bytes +import std/endians -import blscurve -import blscurve/blst/blst_abi -import ../rng -import endians +import pkg/chronos +import pkg/blscurve +import pkg/blscurve/blst/blst_abi + +import ../../rng +import ../../streams # sector size in bytes. Must be smaller than the subgroup order r # which is 255 bits long for BLS12-381 -const bytespersector = 31 +const + BytesPerSector* = 31 -# length in bytes of the unique (random) name -const namelen = 512 + # length in bytes of the unique (random) name + Namelen = 512 type # a single sector - ZChar = array[bytespersector, byte] + ZChar* = array[BytesPerSector, byte] # secret key combining the metadata signing key and the POR generation key - SecretKey = object - signkey: blscurve.SecretKey - key: blst_scalar + SecretKey* = object + signkey*: blscurve.SecretKey + key*: blst_scalar # public key combining the metadata signing key and the POR validation key - PublicKey = object - signkey: blscurve.PublicKey - key: blst_p2 + PublicKey* = object + signkey*: blscurve.PublicKey + key*: blst_p2 # POR metadata (called "file tag t_0" in the original paper) - TauZero = object - name: array[namelen, byte] - n: int64 - u: seq[blst_p1] + TauZero* = object + name*: array[Namelen, byte] + n*: int64 + u*: seq[blst_p1] # signed POR metadata (called "signed file tag t" in the original paper) - Tau = object - t: TauZero - signature: array[96, byte] + Tau* = object + t*: TauZero + signature*: array[96, byte] + + Proof* = object + mu*: seq[blst_scalar] + sigma*: blst_p1 # PoR query element - QElement = object - I: int64 - V: blst_scalar + QElement* = object + I*: int64 + V*: blst_scalar + + PoR* = object + ssk*: SecretKey + spk*: PublicKey + tau*: Tau + authenticators*: seq[blst_p1] proc fromBytesBE(a: array[32, byte]): blst_scalar = ## Convert data to blst native form + ## + blst_scalar_from_bendian(result, a) doAssert(blst_scalar_fr_check(result).bool) proc fromBytesBE(a: openArray[byte]): blst_scalar = ## Convert data to blst native form + ## + var b: array[32, byte] doAssert(a.len <= b.len) + let d = b.len - a.len - for i in 0 ..< a.len: + for i in 0.. postion - f.setFilePos((blockid * spb + sectorid) * sizeof(result)) - let r = f.readBytes(result, 0, sizeof(result)) + ## + + var res: ZChar + stream.setPos(((blockid * spb + sectorid) * ZChar.len).int) + discard await stream.readOnce(addr res[0], ZChar.len) + return res proc rndScalar(): blst_scalar = ## Generate random scalar within the subroup order r - var scal{.noInit.}: array[32, byte] - var scalar{.noInit.}: blst_scalar + ## + + var scal {.noInit.}: array[32, byte] + var scalar {.noInit.}: blst_scalar while true: for val in scal.mitems: val = byte Rng.instance.rand(0xFF) + scalar.blst_scalar_from_bendian(scal) if blst_scalar_fr_check(scalar).bool: break @@ -156,55 +187,77 @@ proc rndScalar(): blst_scalar = proc rndP2(): (blst_p2, blst_scalar) = ## Generate random point on G2 - var x{.noInit.}: blst_p2 + ## + + var + x {.noInit.}: blst_p2 x.blst_p2_from_affine(BLS12_381_G2) # init from generator - let scalar = rndScalar() + + let + scalar = rndScalar() x.blst_p2_mult(x, scalar, 255) + return (x, scalar) proc rndP1(): (blst_p1, blst_scalar) = ## Generate random point on G1 - var x{.noInit.}: blst_p1 + var + x {.noInit.}: blst_p1 x.blst_p1_from_affine(BLS12_381_G1) # init from generator - let scalar = rndScalar() + + let + scalar = rndScalar() x.blst_p1_mult(x, scalar, 255) + return (x, scalar) -proc posKeygen(): (blst_p2, blst_scalar) = +template posKeygen(): (blst_p2, blst_scalar) = ## Generate POS key pair + ## + rndP2() -proc keygen*(): (PublicKey, SecretKey) = +proc keyGen*(): (PublicKey, SecretKey) = ## Generate key pair for signing metadata and for POS tags - var pk: PublicKey - var sk: SecretKey - var ikm: array[32, byte] + ## + + var + pk: PublicKey + sk: SecretKey + ikm: array[32, byte] for b in ikm.mitems: b = byte Rng.instance.rand(0xFF) + doAssert ikm.keyGen(pk.signkey, sk.signkey) (pk.key, sk.key) = posKeygen() return (pk, sk) -proc split(f: File, s: int64): int64 = +proc sectorsCount(stream: SeekableStream, s: int64): int64 = ## Calculate number of blocks for a file - let size = f.getFileSize() - let n = ((size - 1) div (s * sizeof(ZChar))) + 1 - echo "File size=", size, " bytes", - ", blocks=", n, - ", sectors/block=", $s, - ", sectorsize=", $sizeof(ZChar), " bytes" + ## + + let + size = stream.size() + n = ((size - 1) div (s * sizeof(ZChar))) + 1 + # debugEcho "File size=", size, " bytes", + # ", blocks=", n, + # ", sectors/block=", $s, + # ", sectorsize=", $sizeof(ZChar), " bytes" return n proc hashToG1[T: byte|char](msg: openArray[T]): blst_p1 = - ## Hash to curve with Codex specific domain separation - const dst = "CODEX-PROOF-OF-CONCEPT" + ## Hash to curve with Dagger specific domain separation + ## + + const dst = "DAGGER-PROOF-OF-CONCEPT" result.blst_hash_to_g1(msg, dst, aug = "") -proc hashNameI(name: array[namelen, byte], i: int64): blst_p1 = - ## Calculate unique filname and block index based hash +proc hashNameI(name: array[Namelen, byte], i: int64): blst_p1 = + ## Calculate unique filename and block index based hash + ## # # naive implementation, hashing a long string representation # # such as "[255, 242, 23]1" @@ -216,20 +269,33 @@ proc hashNameI(name: array[namelen, byte], i: int64): blst_p1 = bigEndian64(addr(namei[sizeof(name)]), unsafeAddr(i)) return hashToG1(namei) -proc generateAuthenticatorNaive(i: int64, s: int64, t: TauZero, f: File, ssk: SecretKey): blst_p1 = +proc generateAuthenticatorNaive( + stream: SeekableStream, + ssk: SecretKey, + i: int64, + s: int64, + t: TauZero): Future[blst_p1] {.async.} = ## Naive implementation of authenticator as in the S&W paper. ## With the paper's multiplicative notation: ## \sigmai=\(H(file||i)\cdot\prod{j=0}^{s-1}{uj^{m[i][j]}})^{\alpha} + ## + var sum: blst_p1 - for j in 0 ..< s: + for j in 0.. G_T - var aa: blst_p1_affine - var bb: blst_p2_affine + ## + + var + aa: blst_p1_affine + bb: blst_p2_affine + l: blst_fp12 + blst_p1_to_affine(aa, a) blst_p2_to_affine(bb, b) - var l: blst_fp12 + blst_miller_loop(l, bb, aa) blst_final_exp(result, l) @@ -346,6 +415,8 @@ proc verifyPairingsNaive(a1: blst_p1, a2: blst_p2, b1: blst_p1, b2: blst_p2) : b proc verifyPairingsNeg(a1: blst_p1, a2: blst_p2, b1: blst_p1, b2: blst_p2) : bool = ## Faster pairing verification using 2 miller loops but ony one final exponentiation ## based on https://github.com/benjaminion/c-kzg/blob/main/src/bls12_381.c + ## + var loop0, loop1, gt_point: blst_fp12 aa1, bb1: blst_p1_affine @@ -369,29 +440,37 @@ proc verifyPairingsNeg(a1: blst_p1, a2: blst_p2, b1: blst_p1, b2: blst_p2) : boo proc verifyPairings(a1: blst_p1, a2: blst_p2, b1: blst_p1, b2: blst_p2) : bool = ## Wrapper to select verify pairings implementation + ## + verifyPairingsNaive(a1, a2, b1, b2) #verifyPairingsNeg(a1, a2, b1, b2) -proc verifyProof*(tau: Tau, q: openArray[QElement], mus: openArray[blst_scalar], sigma: blst_p1, spk: PublicKey): bool = +proc verifyProof*( + self: PoR, + q: seq[QElement], + mus: seq[blst_scalar], + sigma: blst_p1): bool = ## Verify a BLS proof given a query + ## # verify signature on Tau - var signature: Signature - if not signature.fromBytes(tau.signature): + var signature: blscurve.Signature + if not signature.fromBytes(self.tau.signature): return false - if not verify(spk.signkey, $tau.t, signature): + + if not verify(self.spk.signkey, $self.tau.t, signature): return false var first: blst_p1 - for qelem in q : + for qelem in q: var prod: blst_p1 - prod.blst_p1_mult(hashNameI(tau.t.name, qelem.I), qelem.V, 255) + prod.blst_p1_mult(hashNameI(self.tau.t.name, qelem.I), qelem.V, 255) first.blst_p1_add_or_double(first, prod) doAssert(blst_p1_on_curve(first).bool) - let us = tau.t.u + let us = self.tau.t.u var second: blst_p1 - for j in 0 ..< len(us) : + for j in 0..= self.manifest.blockSize: self.offset mod self.manifest.blockSize @@ -75,7 +77,15 @@ method readOnce*( else: min(nbytes - read, self.manifest.blockSize) - copyMem(pbytes.offset(read), unsafeAddr blk.data[blockOffset], readBytes) + trace "Reading bytes from store stream", pos, cid = blk.cid, bytes = readBytes, blockOffset = blockOffset + copyMem( + pbytes.offset(read), + if blk.isEmpty: + self.emptyBlock[blockOffset].addr + else: + blk.data[blockOffset].addr, + readBytes) + self.offset += readBytes read += readBytes diff --git a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim index 3531206a..e6dde554 100644 --- a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim @@ -5,7 +5,6 @@ import std/tables import pkg/asynctest import pkg/chronos -import pkg/chronicles import pkg/libp2p import pkg/codex/rng diff --git a/tests/codex/blockexchange/engine/testblockexc.nim b/tests/codex/blockexchange/engine/testblockexc.nim index 6c2d6557..f898bb79 100644 --- a/tests/codex/blockexchange/engine/testblockexc.nim +++ b/tests/codex/blockexchange/engine/testblockexc.nim @@ -16,7 +16,6 @@ import pkg/codex/discovery import pkg/codex/blocktype as bt import ../../helpers -import ../../examples suite "NetworkStore engine - 2 nodes": let diff --git a/tests/codex/helpers.nim b/tests/codex/helpers.nim index cf68c45e..c9931608 100644 --- a/tests/codex/helpers.nim +++ b/tests/codex/helpers.nim @@ -1,6 +1,10 @@ +import pkg/chronos import pkg/libp2p import pkg/libp2p/varint -import pkg/codex/blocktype +import pkg/codex/blocktype as bt +import pkg/codex/stores +import pkg/codex/manifest +import pkg/codex/rng import ./helpers/nodeutils import ./helpers/randomchunker @@ -10,8 +14,8 @@ export randomchunker, nodeutils # NOTE: The meaning of equality for blocks # is changed here, because blocks are now `ref` # types. This is only in tests!!! -func `==`*(a, b: Block): bool = - (a.cid == b.cid) and (a.data == b.data) +func `==`*(a, b: bt.Block): bool = + (a.cid == b.cid) and (a.data == b. data) proc lenPrefix*(msg: openArray[byte]): seq[byte] = ## Write `msg` with a varint-encoded length prefix @@ -23,3 +27,34 @@ proc lenPrefix*(msg: openArray[byte]): seq[byte] = buf[vbytes.len..= blks: + break + + var i = -1 + if (i = Rng.instance.rand(manifest.len - 1); pos.find(i) >= 0): + continue + + pos.add(i) + var + blk = (await store.getBlock(manifest[i])).tryGet() + bytePos: seq[int] + + while true: + if bytePos.len > bytes: + break + + var ii = -1 + if (ii = Rng.instance.rand(blk.data.len - 1); bytePos.find(ii) >= 0): + continue + + bytePos.add(ii) + blk.data[ii] = byte 0 + + return pos diff --git a/tests/codex/helpers/mockproofs.nim b/tests/codex/helpers/mockproofs.nim index 90dcc6bb..605775fb 100644 --- a/tests/codex/helpers/mockproofs.nim +++ b/tests/codex/helpers/mockproofs.nim @@ -2,7 +2,7 @@ import std/sets import std/tables import std/sequtils import pkg/upraises -import pkg/codex/por/timing/proofs +import pkg/codex/storageproofs type MockProofs* = ref object of Proofs diff --git a/tests/codex/storageproofs/testpor.nim b/tests/codex/storageproofs/testpor.nim new file mode 100644 index 00000000..81ca3d09 --- /dev/null +++ b/tests/codex/storageproofs/testpor.nim @@ -0,0 +1,164 @@ +import pkg/chronos +import pkg/asynctest + +import pkg/blscurve/blst/blst_abi + +import pkg/codex/streams +import pkg/codex/storageproofs as st +import pkg/codex/stores +import pkg/codex/manifest +import pkg/codex/chunker +import pkg/codex/rng +import pkg/codex/blocktype as bt + + +import ../helpers + +const + SectorSize = 31 + SectorsPerBlock = BlockSize div SectorSize + DataSetSize = BlockSize * 100 + +suite "BLS PoR": + var + chunker: RandomChunker + manifest: Manifest + store: BlockStore + ssk: st.SecretKey + spk: st.PublicKey + + setup: + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) + manifest = Manifest.new(blockSize = BlockSize).tryGet() + (spk, ssk) = st.keyGen() + + while ( + let chunk = await chunker.getBytes(); + chunk.len > 0): + + let + blk = bt.Block.new(chunk).tryGet() + + manifest.add(blk.cid) + if not (await store.putBlock(blk)): + raise newException(CatchableError, "Unable to store block " & $blk.cid) + + test "Test PoR without corruption": + let + por = await PoR.init( + StoreStream.new(store, manifest), + ssk, + spk, + BlockSize) + q = generateQuery(por.tau, 22) + proof = await generateProof( + StoreStream.new(store, manifest), + q, + por.authenticators, + SectorsPerBlock) + + check por.verifyProof(q, proof.mu, proof.sigma) + + test "Test PoR with corruption - query: 22, corrupted blocks: 300, bytes: 10": + let + por = await PoR.init( + StoreStream.new(store, manifest), + ssk, + spk, + BlockSize) + pos = await store.corruptBlocks(manifest, 30, 10) + q = generateQuery(por.tau, 22) + proof = await generateProof( + StoreStream.new(store, manifest), + q, + por.authenticators, + SectorsPerBlock) + + check pos.len == 30 + check not por.verifyProof(q, proof.mu, proof.sigma) + +suite "Test Serialization": + var + chunker: RandomChunker + manifest: Manifest + store: BlockStore + ssk: st.SecretKey + spk: st.PublicKey + por: PoR + q: seq[QElement] + proof: Proof + + setupAll: + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) + manifest = Manifest.new(blockSize = BlockSize).tryGet() + + while ( + let chunk = await chunker.getBytes(); + chunk.len > 0): + + let + blk = bt.Block.new(chunk).tryGet() + + manifest.add(blk.cid) + if not (await store.putBlock(blk)): + raise newException(CatchableError, "Unable to store block " & $blk.cid) + + (spk, ssk) = st.keyGen() + por = await PoR.init( + StoreStream.new(store, manifest), + ssk, + spk, + BlockSize) + q = generateQuery(por.tau, 22) + proof = await generateProof( + StoreStream.new(store, manifest), + q, + por.authenticators, + SectorsPerBlock) + + test "Serialize Public Key": + var + spkMessage = spk.toMessage() + + check: + spk.signkey == spkMessage.fromMessage().tryGet().signkey + spk.key.blst_p2_is_equal(spkMessage.fromMessage().tryGet().key).bool + + test "Serialize TauZero": + var + tauZeroMessage = por.tau.t.toMessage() + tauZero = tauZeroMessage.fromMessage().tryGet() + + check: + por.tau.t.name == tauZero.name + por.tau.t.n == tauZero.n + + for i in 0.. Date: Tue, 24 May 2022 18:42:49 -0600 Subject: [PATCH 6/7] Adding storage proofs store (#109) * add por store * rename por store to stp store * add stp tests --- codex/storageproofs.nim | 3 +- codex/storageproofs/stpstore.nim | 74 ++++++++++++++++++++++ tests/codex/storageproofs/teststpstore.nim | 68 ++++++++++++++++++++ tests/codex/teststorageproofs.nim | 1 + 4 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 codex/storageproofs/stpstore.nim create mode 100644 tests/codex/storageproofs/teststpstore.nim diff --git a/codex/storageproofs.nim b/codex/storageproofs.nim index e23ecfbf..7ad106ea 100644 --- a/codex/storageproofs.nim +++ b/codex/storageproofs.nim @@ -1,4 +1,5 @@ import ./storageproofs/por import ./storageproofs/timing +import ./storageproofs/stpstore -export por, timing +export por, timing, stpstore diff --git a/codex/storageproofs/stpstore.nim b/codex/storageproofs/stpstore.nim new file mode 100644 index 00000000..43a8ca14 --- /dev/null +++ b/codex/storageproofs/stpstore.nim @@ -0,0 +1,74 @@ +## Nim-Dagger +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/os +import std/strformat + +import pkg/libp2p +import pkg/chronos +import pkg/chronicles +import pkg/stew/io2 +import pkg/questionable +import pkg/questionable/results +import pkg/protobuf_serialization + +import ./por + +type + StpStore* = object + authDir*: string + postfixLen*: int + +template stpPath*(self: StpStore, cid: Cid): string = + self.authDir / ($cid)[^self.postfixLen..^1] / $cid + +proc retrieve*(self: StpStore, cid: Cid): Future[?!PorMessage] {.async.} = + ## Retrieve authenticators from data store + ## + + let path = self.stpPath(cid) + var data: seq[byte] + if ( + let res = io2.readFile(path, data); + res.isErr): + let error = io2.ioErrorMsg(res.error) + trace "Cannot retrieve authenticators from fs", path , error + return failure("Cannot retrieve authenticators from fs") + + return Protobuf.decode(data, PorMessage).success + +proc store*(self: StpStore, por: PoR, cid: Cid): Future[?!void] {.async.} = + ## Persist storage proofs + ## + + let + dir = self.stpPath(cid).parentDir + + if io2.createPath(dir).isErr: + trace "Unable to create storage proofs prefix dir", dir + return failure(&"Unable to create storage proofs prefix dir ${dir}") + + let path = self.stpPath(cid) + if ( + let res = io2.writeFile(path, Protobuf.encode(por.toMessage())); + res.isErr): + let error = io2.ioErrorMsg(res.error) + trace "Unable to store storage proofs", path, cid = cid, error + return failure( + &"Unable to store storage proofs path = ${path} cid = ${$cid} error = ${error}") + + return success() + +proc init*( + T: type StpStore, + authDir: string, + postfixLen: int = 2): StpStore = + T( + authDir: authDir, + postfixLen: postfixLen) diff --git a/tests/codex/storageproofs/teststpstore.nim b/tests/codex/storageproofs/teststpstore.nim new file mode 100644 index 00000000..c2839c1e --- /dev/null +++ b/tests/codex/storageproofs/teststpstore.nim @@ -0,0 +1,68 @@ +import std/os + +import pkg/chronos +import pkg/asynctest + +import pkg/codex/rng +import pkg/codex/streams +import pkg/codex/storageproofs as st +import pkg/codex/blocktype as bt + +import ../helpers + +const + SectorSize = 31 + SectorsPerBlock = BlockSize div SectorSize + DataSetSize = BlockSize * 100 + +suite "Test PoR store": + let + (path, _, _) = instantiationInfo(-2, fullPaths = true) # get this file's name + + var + chunker: RandomChunker + manifest: Manifest + store: BlockStore + ssk: st.SecretKey + spk: st.PublicKey + repoDir: string + stpstore: st.StpStore + por: PoR + cid: Cid + + setupAll: + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) + manifest = Manifest.new(blockSize = BlockSize).tryGet() + (spk, ssk) = st.keyGen() + + while ( + let chunk = await chunker.getBytes(); + chunk.len > 0): + + let + blk = bt.Block.new(chunk).tryGet() + + manifest.add(blk.cid) + if not (await store.putBlock(blk)): + raise newException(CatchableError, "Unable to store block " & $blk.cid) + + cid = manifest.cid.tryGet() + por = await PoR.init( + StoreStream.new(store, manifest), + ssk, spk, + BlockSize) + + repoDir = path.parentDir / "stp" + createDir(repoDir) + stpstore = st.StpStore.init(repoDir) + + teardownAll: + removeDir(repoDir) + + test "Should store Storage Proofs": + check (await stpstore.store(por, cid)).isOk + check fileExists(stpstore.stpPath(cid)) + + test "Should retrieve Storage Proofs": + discard (await stpstore.retrieve(cid)).tryGet() diff --git a/tests/codex/teststorageproofs.nim b/tests/codex/teststorageproofs.nim index 25a85538..71b2577a 100644 --- a/tests/codex/teststorageproofs.nim +++ b/tests/codex/teststorageproofs.nim @@ -1,3 +1,4 @@ +import ./storageproofs/teststpstore import ./storageproofs/testpor {.warning[UnusedImport]: off.} From 6ce7e237677ff6dedbbdc708702889195c572c16 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 25 May 2022 20:29:31 -0600 Subject: [PATCH 7/7] Upload authenticators (#108) * initial implementation of storage proofs upload * make sure proof verifies with after deserializing * add por store * rename por store to stp store * rename porstore to stpstore * add support for host discovery to discovery mock * add tags upload network tests --- codex/blockexchange/engine/discovery.nim | 4 +- codex/blockexchange/network/networkpeer.nim | 2 +- codex/discovery.nim | 70 ++++++++-- codex/storageproofs.nim | 4 +- .../storageproofs/por/serialization/por.proto | 10 +- codex/storageproofs/storageproofs.nim | 98 ++++++++++++++ codex/storageproofs/stp.proto | 15 ++ codex/storageproofs/stpnetwork.nim | 104 ++++++++++++++ codex/storageproofs/stpproto.nim | 7 + codex/storageproofs/stpstore.nim | 65 +++++++-- codex/utils.nim | 2 +- .../blockexchange/discovery/testdiscovery.nim | 20 +-- .../discovery/testdiscoveryengine.nim | 6 +- tests/codex/helpers.nim | 3 +- tests/codex/helpers/mockdiscovery.nim | 39 +++--- tests/codex/storageproofs/testnetwork.nim | 128 ++++++++++++++++++ tests/codex/storageproofs/testpor.nim | 3 +- tests/codex/storageproofs/teststpstore.nim | 22 ++- tests/codex/teststorageproofs.nim | 1 + 19 files changed, 541 insertions(+), 62 deletions(-) create mode 100644 codex/storageproofs/storageproofs.nim create mode 100644 codex/storageproofs/stp.proto create mode 100644 codex/storageproofs/stpnetwork.nim create mode 100644 codex/storageproofs/stpproto.nim create mode 100644 tests/codex/storageproofs/testnetwork.nim diff --git a/codex/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim index 107084d8..dacfb213 100644 --- a/codex/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -105,7 +105,7 @@ proc advertiseTaskLoop(b: DiscoveryEngine) {.async.} = try: trace "Advertising block", cid = $cid - let request = b.discovery.provideBlock(cid) + let request = b.discovery.provide(cid) b.inFlightAdvReqs[cid] = request await request finally: @@ -137,7 +137,7 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async.} = try: let request = b.discovery - .findBlockProviders(cid) + .find(cid) .wait(DefaultDiscoveryTimeout) b.inFlightDiscReqs[cid] = request diff --git a/codex/blockexchange/network/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim index 1a213383..9f648e37 100644 --- a/codex/blockexchange/network/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -18,7 +18,7 @@ logScope: topics = "codex blockexc networkpeer" const - MaxMessageSize = 100 * 1024 * 1024 # manifest files can be big + MaxMessageSize = 100 * 1 shl 20 # manifest files can be big type RPCHandler* = proc(peer: NetworkPeer, msg: Message): Future[void] {.gcsafe.} diff --git a/codex/discovery.nim b/codex/discovery.nim index 5ead151a..98089383 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -7,12 +7,17 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. +import std/algorithm + import pkg/chronos import pkg/chronicles import pkg/libp2p +import pkg/libp2p/routing_record +import pkg/libp2p/signed_envelope import pkg/questionable import pkg/questionable/results import pkg/stew/shims/net +import pkg/contractabi/address as ca import pkg/libp2pdht/discv5/protocol as discv5 import ./rng @@ -20,6 +25,10 @@ import ./errors export discv5 +# TODO: If generics in methods had not been +# deprecated, this could have been implemented +# much more elegantly. + type Discovery* = ref object of RootObj protocol: discv5.Protocol @@ -42,21 +51,31 @@ proc new*( ), localInfo: localInfo) +proc toNodeId*(cid: Cid): NodeId = + ## Cid to discovery id + ## + + readUintBE[256](keccak256.digest(cid.data.buffer).data) + +proc toNodeId*(host: ca.Address): NodeId = + ## Eth address to discovery id + ## + + readUintBE[256](keccak256.digest(host.toArray).data) + proc findPeer*( d: Discovery, peerId: PeerID): Future[?PeerRecord] {.async.} = - let node = await d.protocol.resolve(toNodeId(peerId)) + let + node = await d.protocol.resolve(toNodeId(peerId)) + return if node.isSome(): some(node.get().record.data) else: none(PeerRecord) -proc toDiscoveryId*(cid: Cid): NodeId = - ## To discovery id - readUintBE[256](keccak256.digest(cid.data.buffer).data) - -method findBlockProviders*( +method find*( d: Discovery, cid: Cid): Future[seq[SignedPeerRecord]] {.async, base.} = ## Find block providers @@ -64,19 +83,19 @@ method findBlockProviders*( trace "Finding providers for block", cid = $cid without providers =? - (await d.protocol.getProviders(cid.toDiscoveryId())).mapFailure, error: + (await d.protocol.getProviders(cid.toNodeId())).mapFailure, error: trace "Error finding providers for block", cid = $cid, error = error.msg return providers -method provideBlock*(d: Discovery, cid: Cid) {.async, base.} = +method provide*(d: Discovery, cid: Cid) {.async, base.} = ## Provide a bock Cid ## trace "Providing block", cid = $cid let nodes = await d.protocol.addProvider( - cid.toDiscoveryId(), + cid.toNodeId(), d.localInfo.signedPeerRecord) if nodes.len <= 0: @@ -84,6 +103,39 @@ method provideBlock*(d: Discovery, cid: Cid) {.async, base.} = trace "Provided to nodes", nodes = nodes.len +method find*( + d: Discovery, + host: ca.Address): Future[seq[SignedPeerRecord]] {.async, base.} = + ## Find host providers + ## + + trace "Finding providers for host", host = $host + without var providers =? + (await d.protocol.getProviders(host.toNodeId())).mapFailure, error: + trace "Error finding providers for host", host = $host, exc = error.msg + return + + if providers.len <= 0: + trace "No providers found", host = $host + return + + providers.sort do(a, b: SignedPeerRecord) -> int: + system.cmp[uint64](a.data.seqNo, b.data.seqNo) + + return providers + +method provide*(d: Discovery, host: ca.Address) {.async, base.} = + ## Provide hosts + ## + + trace "Providing host", host = $host + let + nodes = await d.protocol.addProvider( + host.toNodeId(), + d.localInfo.signedPeerRecord) + if nodes.len > 0: + trace "Provided to nodes", nodes = nodes.len + proc start*(d: Discovery) {.async.} = d.protocol.updateRecord(d.localInfo.signedPeerRecord).expect("updating SPR") d.protocol.open() diff --git a/codex/storageproofs.nim b/codex/storageproofs.nim index 7ad106ea..ef0a8f00 100644 --- a/codex/storageproofs.nim +++ b/codex/storageproofs.nim @@ -1,5 +1,7 @@ import ./storageproofs/por import ./storageproofs/timing import ./storageproofs/stpstore +import ./storageproofs/stpnetwork +import ./storageproofs/stpproto -export por, timing, stpstore +export por, timing, stpstore, stpnetwork, stpproto diff --git a/codex/storageproofs/por/serialization/por.proto b/codex/storageproofs/por/serialization/por.proto index 76ee1f0c..82670e18 100644 --- a/codex/storageproofs/por/serialization/por.proto +++ b/codex/storageproofs/por/serialization/por.proto @@ -12,11 +12,6 @@ message PoREnvelope { bytes signature = 2; } - message ProofMessage { - repeated bytes mu = 1; - bytes sigma = 2; - } - message PubKeyMessage { bytes signkey = 1; bytes key = 2; @@ -28,6 +23,11 @@ message PoREnvelope { repeated bytes authenticators = 3; } + message ProofMessage { + repeated bytes mu = 1; + bytes sigma = 2; + } + PorMessage por = 1; ProofMessage proof = 2; } diff --git a/codex/storageproofs/storageproofs.nim b/codex/storageproofs/storageproofs.nim new file mode 100644 index 00000000..eb9ae6cf --- /dev/null +++ b/codex/storageproofs/storageproofs.nim @@ -0,0 +1,98 @@ +## Nim-Dagger +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import pkg/chronos +import pkg/chronicles +import pkg/questionable +import pkg/questionable/results +import pkg/contractabi/address as ca + +import ../stores +import ../manifest +import ../streams +import ../utils + +import ./por +import ./stpnetwork +import ./stpproto +import ./stpstore +import ./timing + +export stpnetwork, stpstore, por, timing, stpproto + +type + StorageProofs* = object + store*: BlockStore + network*: StpNetwork + stpStore*: StpStore + +proc upload*( + self: StorageProofs, + cid: Cid, + indexes: seq[int], + host: ca.Address): Future[?!void] {.async.} = + ## Upload authenticators + ## + + without por =? (await self.stpStore.retrieve(cid)): + trace "Unable to retrieve por data from store", cid + return failure("Unable to retrieve por data from store") + + return await self.network.uploadTags( + cid, + indexes, + por.authenticators, + host) + +# proc proof*() = +# discard + +# proc verify*() = +# discard + +proc setupProofs*( + self: StorageProofs, + manifest: Manifest): Future[?!void] {.async.} = + ## Setup storage authentication + ## + + without cid =? manifest.cid: + return failure("Unable to retrieve Cid from manifest!") + + let + (spk, ssk) = keyGen() + por = await PoR.init( + StoreStream.new(self.store, manifest), + ssk, + spk, + manifest.blockSize) + + return await self.stpStore.store(por.toMessage(), cid) + +proc init*( + T: type StorageProofs, + network: StpNetwork, + store: BlockStore, + stpStore: StpStore): StorageProofs = + + var + self = T( + store: store, + stpStore: stpStore, + network: network) + + proc tagsHandler(msg: TagsMessage) {.async, gcsafe.} = + try: + await self.stpStore.store(msg.cid, msg.tags).tryGet() + trace "Stored tags", cid = $msg.cid, tags = msg.tags.len + except CatchableError as exc: + trace "Exception attempting to store tags", exc = exc.msg + + self.network.tagsHandler = tagsHandler + self diff --git a/codex/storageproofs/stp.proto b/codex/storageproofs/stp.proto new file mode 100644 index 00000000..800e9035 --- /dev/null +++ b/codex/storageproofs/stp.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +message StorageProofsMessage { + message Tag { + int64 idx = 1; + bytes tag = 2; + } + + message TagsMessage { + bytes cid = 1; + repeated Tag tags = 2; + } + + TagsMessage tagsMsg = 1; +} diff --git a/codex/storageproofs/stpnetwork.nim b/codex/storageproofs/stpnetwork.nim new file mode 100644 index 00000000..afafb7b5 --- /dev/null +++ b/codex/storageproofs/stpnetwork.nim @@ -0,0 +1,104 @@ +## Nim-Dagger +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/sequtils + +import pkg/chronos +import pkg/libp2p +import pkg/chronicles +import pkg/questionable +import pkg/questionable/results +import pkg/contractabi/address as ca +import pkg/protobuf_serialization + +import ./stpproto +import ../discovery + +const + Codec* = "/dagger/storageproofs/1.0.0" + MaxMessageSize* = 1 shl 22 # 4MB + +logScope: + topics = "dagger storageproofs network" + +type + TagsHandler* = proc(msg: TagsMessage): + Future[void] {.raises: [Defect], gcsafe.} + + StpNetwork* = ref object of LPProtocol + switch*: Switch + discovery*: Discovery + tagsHandle*: TagsHandler + +proc uploadTags*( + self: StpNetwork, + cid: Cid, + indexes: seq[int], + tags: seq[seq[byte]], + host: ca.Address): Future[?!void] {.async.} = + # Upload tags to `host` + # + + var msg = TagsMessage(cid: cid.data.buffer) + for i in indexes: + msg.tags.add(Tag(idx: i, tag: tags[i])) + + let + peers = await self.discovery.find(host) + connFut = await one(peers.mapIt( + self.switch.dial( + it.data.peerId, + it.data.addresses.mapIt( it.address ), + @[Codec]))) + conn = await connFut + + try: + await conn.writeLp( + Protobuf.encode(StorageProofsMessage(tagsMsg: msg))) + except CancelledError as exc: + raise exc + except CatchableError as exc: + trace "Exception submitting tags", cid, exc = exc.msg + return failure(exc.msg) + finally: + await conn.close() + + return success() + +method init*(self: StpNetwork) = + ## Perform protocol initialization + ## + + proc handle(conn: Connection, proto: string) {.async, gcsafe.} = + try: + let + msg = await conn.readLp(MaxMessageSize) + message = Protobuf.decode(msg, StorageProofsMessage) + + if message.tagsMsg.tags.len > 0 and not self.tagsHandle.isNil: + await self.tagsHandle(message.tagsMsg) + except CatchableError as exc: + trace "Exception handling Storage Proofs message", exc = exc.msg + finally: + await conn.close() + + self.handler = handle + self.codec = Codec + +proc new*( + T: type StpNetwork, + switch: Switch, + discovery: Discovery): StpNetwork = + let + self = StpNetwork( + switch: switch, + discovery: discovery) + + self.init() + self diff --git a/codex/storageproofs/stpproto.nim b/codex/storageproofs/stpproto.nim new file mode 100644 index 00000000..39303099 --- /dev/null +++ b/codex/storageproofs/stpproto.nim @@ -0,0 +1,7 @@ +import pkg/protobuf_serialization + +import_proto3 "stp.proto" + +export StorageProofsMessage +export TagsMessage +export Tag diff --git a/codex/storageproofs/stpstore.nim b/codex/storageproofs/stpstore.nim index 43a8ca14..e64cd107 100644 --- a/codex/storageproofs/stpstore.nim +++ b/codex/storageproofs/stpstore.nim @@ -18,6 +18,7 @@ import pkg/questionable import pkg/questionable/results import pkg/protobuf_serialization +import ./stpproto import ./por type @@ -28,40 +29,86 @@ type template stpPath*(self: StpStore, cid: Cid): string = self.authDir / ($cid)[^self.postfixLen..^1] / $cid -proc retrieve*(self: StpStore, cid: Cid): Future[?!PorMessage] {.async.} = +proc retrieve*( + self: StpStore, + cid: Cid): Future[?!PorMessage] {.async.} = ## Retrieve authenticators from data store ## - let path = self.stpPath(cid) + let path = self.stpPath(cid) / "por" var data: seq[byte] if ( let res = io2.readFile(path, data); res.isErr): let error = io2.ioErrorMsg(res.error) - trace "Cannot retrieve authenticators from fs", path , error - return failure("Cannot retrieve authenticators from fs") + trace "Cannot retrieve storage proof data from fs", path , error + return failure("Cannot retrieve storage proof data from fs") return Protobuf.decode(data, PorMessage).success -proc store*(self: StpStore, por: PoR, cid: Cid): Future[?!void] {.async.} = +proc store*( + self: StpStore, + por: PorMessage, + cid: Cid): Future[?!void] {.async.} = ## Persist storage proofs ## let - dir = self.stpPath(cid).parentDir + dir = self.stpPath(cid) if io2.createPath(dir).isErr: trace "Unable to create storage proofs prefix dir", dir return failure(&"Unable to create storage proofs prefix dir ${dir}") - let path = self.stpPath(cid) + let path = dir / "por" if ( - let res = io2.writeFile(path, Protobuf.encode(por.toMessage())); + let res = io2.writeFile(path, Protobuf.encode(por)); res.isErr): let error = io2.ioErrorMsg(res.error) trace "Unable to store storage proofs", path, cid = cid, error return failure( - &"Unable to store storage proofs path = ${path} cid = ${$cid} error = ${error}") + &"Unable to store storage proofs - path = ${path} cid = ${$cid} error = ${error}") + + return success() + +proc retrieve*( + self: StpStore, + cid: Cid, + blocks: seq[int]): Future[?!seq[Tag]] {.async.} = + var tags: seq[Tag] + for b in blocks: + var tag = Tag(idx: b) + let path = self.stpPath(cid) / $b + if ( + let res = io2.readFile(path, tag.tag); + res.isErr): + let error = io2.ioErrorMsg(res.error) + trace "Cannot retrieve tags from fs", path , error + return failure("Cannot retrieve tags from fs") + tags.add(tag) + + return tags.success + +proc store*( + self: StpStore, + tags: seq[Tag], + cid: Cid): Future[?!void] {.async.} = + let + dir = self.stpPath(cid) + + if io2.createPath(dir).isErr: + trace "Unable to create storage proofs prefix dir", dir + return failure(&"Unable to create storage proofs prefix dir ${dir}") + + for t in tags: + let path = dir / $t.idx + if ( + let res = io2.writeFile(path, t.tag); + res.isErr): + let error = io2.ioErrorMsg(res.error) + trace "Unable to store tags", path, cid = cid, error + return failure( + &"Unable to store tags - path = ${path} cid = ${$cid} error = ${error}") return success() diff --git a/codex/utils.nim b/codex/utils.nim index 2f481e62..8d561d40 100644 --- a/codex/utils.nim +++ b/codex/utils.nim @@ -1,4 +1,4 @@ import ./utils/asyncheapqueue import ./utils/fileutils -export asyncheapqueue, fileutils \ No newline at end of file +export asyncheapqueue, fileutils diff --git a/tests/codex/blockexchange/discovery/testdiscovery.nim b/tests/codex/blockexchange/discovery/testdiscovery.nim index 37a8a271..e71b78b3 100644 --- a/tests/codex/blockexchange/discovery/testdiscovery.nim +++ b/tests/codex/blockexchange/discovery/testdiscovery.nim @@ -43,7 +43,7 @@ suite "Block Advertising and Discovery": blocks.add(bt.Block.new(chunk).tryGet()) switch = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr}) - blockDiscovery = MockDiscovery.new(switch.peerInfo, 0.Port) + blockDiscovery = MockDiscovery.new() wallet = WalletRef.example network = BlockExcNetwork.new(switch) localStore = CacheStore.new(blocks.mapIt( it )) @@ -76,7 +76,7 @@ suite "Block Advertising and Discovery": await engine.start() - blockDiscovery.publishProvideHandler = + blockDiscovery.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async, gcsafe.} = return @@ -94,7 +94,7 @@ suite "Block Advertising and Discovery": advertised = initTable.collect: for b in blocks: {b.cid: newFuture[void]()} - blockDiscovery.publishProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} = + blockDiscovery.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} = if cid in advertised and not advertised[cid].finished(): advertised[cid].complete() @@ -150,7 +150,7 @@ suite "E2E - Multiple Nodes Discovery": for _ in 0..<4: let s = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr}) - blockDiscovery = MockDiscovery.new(s.peerInfo, 0.Port) + blockDiscovery = MockDiscovery.new() wallet = WalletRef.example network = BlockExcNetwork.new(s) localStore = CacheStore.new() @@ -189,15 +189,15 @@ suite "E2E - Multiple Nodes Discovery": var advertised: Table[Cid, SignedPeerRecord] MockDiscovery(blockexc[1].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised.add(cid, switch[1].peerInfo.signedPeerRecord) MockDiscovery(blockexc[2].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised.add(cid, switch[2].peerInfo.signedPeerRecord) MockDiscovery(blockexc[3].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised.add(cid, switch[3].peerInfo.signedPeerRecord) await blockexc[1].engine.blocksHandler(switch[0].peerInfo.peerId, blocks[0..5]) @@ -231,15 +231,15 @@ suite "E2E - Multiple Nodes Discovery": var advertised: Table[Cid, SignedPeerRecord] MockDiscovery(blockexc[1].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised[cid] = switch[1].peerInfo.signedPeerRecord MockDiscovery(blockexc[2].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised[cid] = switch[2].peerInfo.signedPeerRecord MockDiscovery(blockexc[3].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised[cid] = switch[3].peerInfo.signedPeerRecord await blockexc[1].engine.blocksHandler(switch[0].peerInfo.peerId, blocks[0..5]) diff --git a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim index e6dde554..f8fd316f 100644 --- a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim @@ -79,7 +79,7 @@ suite "Test Discovery Engine": for b in blocks: { b.cid: newFuture[void]() } - blockDiscovery.publishProvideHandler = + blockDiscovery.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} = if not haves[cid].finished: haves[cid].complete @@ -124,7 +124,7 @@ suite "Test Discovery Engine": discoveryLoopSleep = 100.millis) have = newFuture[void]() - blockDiscovery.publishProvideHandler = + blockDiscovery.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} = check cid == blocks[0].cid if not have.finished: @@ -216,7 +216,7 @@ suite "Test Discovery Engine": reqs = newFuture[void]() count = 0 - blockDiscovery.publishProvideHandler = + blockDiscovery.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} = check cid == blocks[0].cid if count > 0: diff --git a/tests/codex/helpers.nim b/tests/codex/helpers.nim index c9931608..9f57fd9c 100644 --- a/tests/codex/helpers.nim +++ b/tests/codex/helpers.nim @@ -8,8 +8,9 @@ import pkg/codex/rng import ./helpers/nodeutils import ./helpers/randomchunker +import ./helpers/mockdiscovery -export randomchunker, nodeutils +export randomchunker, nodeutils, mockdiscovery # NOTE: The meaning of equality for blocks # is changed here, because blocks are now `ref` diff --git a/tests/codex/helpers/mockdiscovery.nim b/tests/codex/helpers/mockdiscovery.nim index 51192427..86e517a6 100644 --- a/tests/codex/helpers/mockdiscovery.nim +++ b/tests/codex/helpers/mockdiscovery.nim @@ -13,21 +13,20 @@ import pkg/questionable import pkg/questionable/results import pkg/stew/shims/net import pkg/codex/discovery +import pkg/contractabi/address as ca type MockDiscovery* = ref object of Discovery findBlockProvidersHandler*: proc(d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] {.gcsafe.} - publishProvideHandler*: proc(d: MockDiscovery, cid: Cid): + publishBlockProvideHandler*: proc(d: MockDiscovery, cid: Cid): + Future[void] {.gcsafe.} + findHostProvidersHandler*: proc(d: MockDiscovery, host: ca.Address): + Future[seq[SignedPeerRecord]] {.gcsafe.} + publishHostProvideHandler*: proc(d: MockDiscovery, host: ca.Address): Future[void] {.gcsafe.} -proc new*( - T: type MockDiscovery, - localInfo: PeerInfo, - discoveryPort: Port, - bootstrapNodes = newSeq[SignedPeerRecord](), - ): T = - +proc new*(T: type MockDiscovery): T = T() proc findPeer*( @@ -35,7 +34,7 @@ proc findPeer*( peerId: PeerID): Future[?PeerRecord] {.async.} = return none(PeerRecord) -method findBlockProviders*( +method find*( d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] {.async.} = if isNil(d.findBlockProvidersHandler): @@ -43,14 +42,22 @@ method findBlockProviders*( return await d.findBlockProvidersHandler(d, cid) -method provideBlock*(d: MockDiscovery, cid: Cid): Future[void] {.async.} = - if isNil(d.publishProvideHandler): +method provide*(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + if isNil(d.publishBlockProvideHandler): return - await d.publishProvideHandler(d, cid) + await d.publishBlockProvideHandler(d, cid) -proc start*(d: Discovery) {.async.} = - discard +method find*( + d: MockDiscovery, + host: ca.Address): Future[seq[SignedPeerRecord]] {.async.} = + if isNil(d.findHostProvidersHandler): + return -proc stop*(d: Discovery) {.async.} = - discard + return await d.findHostProvidersHandler(d, host) + +method provide*(d: MockDiscovery, host: ca.Address): Future[void] {.async.} = + if isNil(d.publishHostProvideHandler): + return + + await d.publishHostProvideHandler(d, host) diff --git a/tests/codex/storageproofs/testnetwork.nim b/tests/codex/storageproofs/testnetwork.nim new file mode 100644 index 00000000..10357ba1 --- /dev/null +++ b/tests/codex/storageproofs/testnetwork.nim @@ -0,0 +1,128 @@ +import std/os +import std/sequtils + +import pkg/asynctest +import pkg/chronos +import pkg/libp2p +import pkg/libp2p/errors +import pkg/protobuf_serialization +import pkg/contractabi as ca + +import pkg/codex/rng +import pkg/codex/chunker +import pkg/codex/storageproofs +import pkg/codex/discovery +import pkg/codex/manifest +import pkg/codex/stores +import pkg/codex/storageproofs as st +import pkg/codex/blocktype as bt +import pkg/codex/streams + +import ../examples +import ../helpers + +const + SectorSize = 31 + SectorsPerBlock = BlockSize div SectorSize + DataSetSize = BlockSize * 100 + +suite "Storage Proofs Network": + let + rng = Rng.instance() + seckey1 = PrivateKey.random(rng[]).tryGet() + seckey2 = PrivateKey.random(rng[]).tryGet() + hostAddr1 = ca.Address.example + hostAddr2 = ca.Address.example + blocks = toSeq([1, 5, 10, 14, 20, 12, 22]) # TODO: maybe make them random + + var + stpNetwork1: StpNetwork + stpNetwork2: StpNetwork + switch1: Switch + switch2: Switch + discovery1: MockDiscovery + discovery2: MockDiscovery + + chunker: RandomChunker + manifest: Manifest + store: BlockStore + ssk: st.SecretKey + spk: st.PublicKey + repoDir: string + stpstore: st.StpStore + porMsg: PorMessage + cid: Cid + por: PoR + tags: seq[Tag] + + setupAll: + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) + manifest = Manifest.new(blockSize = BlockSize).tryGet() + (spk, ssk) = st.keyGen() + + while ( + let chunk = await chunker.getBytes(); + chunk.len > 0): + + let + blk = bt.Block.new(chunk).tryGet() + + manifest.add(blk.cid) + if not (await store.putBlock(blk)): + raise newException(CatchableError, "Unable to store block " & $blk.cid) + + cid = manifest.cid.tryGet() + por = await PoR.init( + StoreStream.new(store, manifest), + ssk, spk, + BlockSize) + + porMsg = por.toMessage() + tags = blocks.mapIt( + Tag(idx: it, tag: porMsg.authenticators[it]) ) + + setup: + switch1 = newStandardSwitch() + switch2 = newStandardSwitch() + + discovery1 = MockDiscovery.new(switch1.peerInfo) + discovery2 = MockDiscovery.new(switch2.peerInfo) + + stpNetwork1 = StpNetwork.new(switch1, discovery1) + stpNetwork2 = StpNetwork.new(switch2, discovery2) + + switch1.mount(stpNetwork1) + switch2.mount(stpNetwork2) + + await switch1.start() + await switch2.start() + + teardown: + await switch1.stop() + await switch2.stop() + + test "Should upload to host": + var + done = newFuture[void]() + + discovery1.findHostProvidersHandler = proc(d: MockDiscovery, host: ca.Address): + Future[seq[SignedPeerRecord]] {.async, gcsafe.} = + check hostAddr2 == host + return @[switch2.peerInfo.signedPeerRecord] + + proc tagsHandler(msg: TagsMessage) {.async, gcsafe.} = + check: + Cid.init(msg.cid).tryGet() == cid + msg.tags == tags + + done.complete() + + stpNetwork2.tagsHandle = tagsHandler + (await stpNetwork1.uploadTags( + cid, + blocks, + porMsg.authenticators, + hostAddr2)).tryGet() + + await done.wait(1.seconds) diff --git a/tests/codex/storageproofs/testpor.nim b/tests/codex/storageproofs/testpor.nim index 81ca3d09..5b9c8481 100644 --- a/tests/codex/storageproofs/testpor.nim +++ b/tests/codex/storageproofs/testpor.nim @@ -11,7 +11,6 @@ import pkg/codex/chunker import pkg/codex/rng import pkg/codex/blocktype as bt - import ../helpers const @@ -162,3 +161,5 @@ suite "Test Serialization": check: proof.sigma.blst_p1_is_equal(pproof.sigma).bool proof.mu == pproof.mu + + check por.verifyProof(q, pproof.mu, pproof.sigma) diff --git a/tests/codex/storageproofs/teststpstore.nim b/tests/codex/storageproofs/teststpstore.nim index c2839c1e..545ff68f 100644 --- a/tests/codex/storageproofs/teststpstore.nim +++ b/tests/codex/storageproofs/teststpstore.nim @@ -1,4 +1,5 @@ import std/os +import std/sequtils import pkg/chronos import pkg/asynctest @@ -18,6 +19,7 @@ const suite "Test PoR store": let (path, _, _) = instantiationInfo(-2, fullPaths = true) # get this file's name + blocks = toSeq([1, 5, 10, 14, 20, 12, 22]) # TODO: maybe make them random var chunker: RandomChunker @@ -28,7 +30,9 @@ suite "Test PoR store": repoDir: string stpstore: st.StpStore por: PoR + porMsg: PorMessage cid: Cid + tags: seq[Tag] setupAll: chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) @@ -53,6 +57,10 @@ suite "Test PoR store": ssk, spk, BlockSize) + porMsg = por.toMessage() + tags = blocks.mapIt( + Tag(idx: it, tag: porMsg.authenticators[it]) ) + repoDir = path.parentDir / "stp" createDir(repoDir) stpstore = st.StpStore.init(repoDir) @@ -61,8 +69,16 @@ suite "Test PoR store": removeDir(repoDir) test "Should store Storage Proofs": - check (await stpstore.store(por, cid)).isOk - check fileExists(stpstore.stpPath(cid)) + check (await stpstore.store(por.toMessage(), cid)).isOk + check fileExists(stpstore.stpPath(cid) / "por") test "Should retrieve Storage Proofs": - discard (await stpstore.retrieve(cid)).tryGet() + check (await stpstore.retrieve(cid)).tryGet() == porMsg + + test "Should store tags": + check (await stpstore.store(tags, cid)).isOk + for t in tags: + check fileExists(stpstore.stpPath(cid) / $t.idx ) + + test "Should retrieve tags": + check (await stpstore.retrieve(cid, blocks)).tryGet() == tags diff --git a/tests/codex/teststorageproofs.nim b/tests/codex/teststorageproofs.nim index 71b2577a..e90a8c0b 100644 --- a/tests/codex/teststorageproofs.nim +++ b/tests/codex/teststorageproofs.nim @@ -1,4 +1,5 @@ import ./storageproofs/teststpstore import ./storageproofs/testpor +import ./storageproofs/testnetwork {.warning[UnusedImport]: off.}