diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e9ed94a..bc1ca755 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: builder: windows-2019 name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.branch }}' runs-on: ${{ matrix.builder }} - timeout-minutes: 40 + timeout-minutes: 60 steps: - name: Checkout nim-codex uses: actions/checkout@v2 @@ -209,12 +209,12 @@ jobs: make NIM_COMMIT="${{ matrix.branch }}" testAll if [[ ${{ runner.os }} = macOS ]]; then echo - echo otool -L build/testDagger - otool -L build/testDagger + echo otool -L build/testCodex + otool -L build/testCodex else echo - echo ldd build/testDagger - ldd build/testDagger + echo ldd build/testCodex + ldd build/testCodex fi echo "Testing TRACE log level" - ./env.sh nim c -d:chronicles_log_level=TRACE dagger.nim + ./env.sh nim c -d:chronicles_log_level=TRACE codex.nim diff --git a/.gitignore b/.gitignore index c9bad6d9..b1c8f7e0 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,4 @@ nimble.paths .env .update.timestamp -dagger.nims +codex.nims diff --git a/Makefile b/Makefile index b6bdb8c7..30adaf84 100644 --- a/Makefile +++ b/Makefile @@ -56,15 +56,15 @@ else NIM_PARAMS := $(NIM_PARAMS) -d:release endif -deps: | deps-common nat-libs dagger.nims +deps: | deps-common nat-libs codex.nims ifneq ($(USE_LIBBACKTRACE), 0) deps: | libbacktrace endif -#- deletes and recreates "dagger.nims" which on Windows is a copy instead of a proper symlink +#- deletes and recreates "codex.nims" which on Windows is a copy instead of a proper symlink update: | update-common - rm -rf dagger.nims && \ - $(MAKE) dagger.nims $(HANDLE_OUTPUT) + rm -rf codex.nims && \ + $(MAKE) codex.nims $(HANDLE_OUTPUT) # detecting the os ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... @@ -79,16 +79,16 @@ endif # Builds and run a part of the test suite test: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim test $(NIM_PARAMS) dagger.nims + $(ENV_SCRIPT) nim test $(NIM_PARAMS) codex.nims # Builds and runs all tests testAll: | build deps - echo -e $(BUILD_MSG) "build/testDagger" "build/testContracts" && \ - $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) dagger.nims + echo -e $(BUILD_MSG) "build/testCodex" "build/testContracts" && \ + $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) codex.nims # symlink -dagger.nims: - ln -s dagger.nimble $@ +codex.nims: + ln -s codex.nimble $@ # nim-libbacktrace libbacktrace: @@ -96,12 +96,12 @@ libbacktrace: coverage: $(MAKE) NIMFLAGS="--lineDir:on --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage" testAll - cd nimcache/release/testDagger && rm -f *.c + cd nimcache/release/testCodex && rm -f *.c cd nimcache/release/testContracts && rm -f *.c mkdir -p coverage - lcov --capture --directory nimcache/release/testDagger --directory nimcache/release/testContracts --output-file coverage/coverage.info - shopt -s globstar && ls $$(pwd)/dagger/{*,**/*}.nim - shopt -s globstar && lcov --extract coverage/coverage.info $$(pwd)/dagger/{*,**/*}.nim --output-file coverage/coverage.f.info + lcov --capture --directory nimcache/release/testCodex --directory nimcache/release/testContracts --output-file coverage/coverage.info + shopt -s globstar && ls $$(pwd)/codex/{*,**/*}.nim + shopt -s globstar && lcov --extract coverage/coverage.info $$(pwd)/codex/{*,**/*}.nim --output-file coverage/coverage.f.info echo -e $(BUILD_MSG) "coverage/report/index.html" genhtml coverage/coverage.f.info --output-directory coverage/report if which open >/dev/null; then (echo -e "\e[92mOpening\e[39m HTML coverage report in browser..." && open coverage/report/index.html) || true; fi diff --git a/codecov.yml b/codecov.yml index 429a94c8..b0dd0256 100644 --- a/codecov.yml +++ b/codecov.yml @@ -20,3 +20,4 @@ coverage: # `informational`: https://docs.codecov.com/docs/commit-status#informational # `threshold`: https://docs.codecov.com/docs/commit-status#threshold informational: true +comment: false diff --git a/dagger.nim b/codex.nim similarity index 88% rename from dagger.nim rename to codex.nim index 72249041..1003d21a 100644 --- a/dagger.nim +++ b/codex.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -12,23 +12,23 @@ import pkg/chronos import pkg/confutils import pkg/libp2p -import ./dagger/conf -import ./dagger/dagger +import ./codex/conf +import ./codex/codex -export dagger, conf, libp2p, chronos, chronicles +export codex, conf, libp2p, chronos, chronicles when isMainModule: import std/os import pkg/confutils/defs - import ./dagger/utils/fileutils + import ./codex/utils/fileutils when defined(posix): import system/ansi_c - let config = DaggerConf.load( - version = daggerFullVersion + let config = CodexConf.load( + version = codexFullVersion ) config.setupLogging() config.setupMetrics() @@ -50,7 +50,7 @@ when isMainModule: trace "Repo dir initialized", dir = config.dataDir / "repo" - let server = DaggerServer.new(config) + let server = CodexServer.new(config) ## Ctrl+C handling proc controlCHandler() {.noconv.} = diff --git a/dagger.nimble b/codex.nimble similarity index 83% rename from dagger.nimble rename to codex.nimble index 34bbc6cc..d3e017aa 100644 --- a/dagger.nimble +++ b/codex.nimble @@ -1,7 +1,7 @@ mode = ScriptMode.Verbose version = "0.1.0" -author = "Dagger Team" +author = "Codex Team" description = "p2p data durability engine" license = "MIT" binDir = "build" @@ -28,7 +28,7 @@ requires "nim >= 1.2.0", when declared(namedBin): namedBin = { - "dagger/dagger": "dagger" + "codex/codex": "codex" }.toTable() ### Helper functions @@ -45,23 +45,23 @@ proc test(name: string, srcDir = "tests/", lang = "c") = buildBinary name, srcDir exec "build/" & name -task dagger, "build dagger binary": - buildBinary "dagger" +task codex, "build codex binary": + buildBinary "codex" -task testDagger, "Build & run Dagger tests": - test "testDagger" +task testCodex, "Build & run Codex tests": + test "testCodex" -task testContracts, "Build & run Dagger Contract tests": +task testContracts, "Build & run Codex Contract tests": test "testContracts" task testIntegration, "Run integration tests": - daggerTask() + codexTask() test "testIntegration" task test, "Run tests": - testDaggerTask() + testCodexTask() task testAll, "Run all tests": - testDaggerTask() + testCodexTask() testContractsTask() testIntegrationTask() diff --git a/dagger/blockexchange.nim b/codex/blockexchange.nim similarity index 100% rename from dagger/blockexchange.nim rename to codex/blockexchange.nim diff --git a/codex/blockexchange/engine.nim b/codex/blockexchange/engine.nim new file mode 100644 index 00000000..a9cd9160 --- /dev/null +++ b/codex/blockexchange/engine.nim @@ -0,0 +1,5 @@ +import ./engine/discovery +import ./engine/engine +import ./engine/payments + +export discovery, engine, payments diff --git a/dagger/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim similarity index 98% rename from dagger/blockexchange/engine/discovery.nim rename to codex/blockexchange/engine/discovery.nim index aba3ccd0..dacfb213 100644 --- a/dagger/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -22,10 +22,10 @@ import ../../utils import ../../discovery import ../../stores/blockstore -import ../pendingblocks +import ./pendingblocks logScope: - topics = "dagger discovery engine" + topics = "codex discovery engine" const DefaultConcurrentDiscRequests = 10 @@ -105,7 +105,7 @@ proc advertiseTaskLoop(b: DiscoveryEngine) {.async.} = try: trace "Advertising block", cid = $cid - let request = b.discovery.provideBlock(cid) + let request = b.discovery.provide(cid) b.inFlightAdvReqs[cid] = request await request finally: @@ -137,7 +137,7 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async.} = try: let request = b.discovery - .findBlockProviders(cid) + .find(cid) .wait(DefaultDiscoveryTimeout) b.inFlightDiscReqs[cid] = request diff --git a/dagger/blockexchange/engine.nim b/codex/blockexchange/engine/engine.nim similarity index 98% rename from dagger/blockexchange/engine.nim rename to codex/blockexchange/engine/engine.nim index 6a67d964..69fda741 100644 --- a/dagger/blockexchange/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -14,23 +14,24 @@ import pkg/chronos import pkg/chronicles import pkg/libp2p -import ../stores/blockstore -import ../blocktype as bt -import ../utils +import ../../stores/blockstore +import ../../blocktype as bt +import ../../utils -import ./protobuf/blockexc -import ./protobuf/presence +import ../protobuf/blockexc +import ../protobuf/presence -import ./network -import ./peers -import ./engine/payments -import ./engine/discovery +import ../network +import ../peers + +import ./payments +import ./discovery import ./pendingblocks export peers, pendingblocks, payments, discovery logScope: - topics = "dagger blockexc engine" + topics = "codex blockexc engine" const DefaultMaxPeersPerRequest* = 10 diff --git a/dagger/blockexchange/engine/payments.nim b/codex/blockexchange/engine/payments.nim similarity index 78% rename from dagger/blockexchange/engine/payments.nim rename to codex/blockexchange/engine/payments.nim index 15079086..35d38e29 100644 --- a/dagger/blockexchange/engine/payments.nim +++ b/codex/blockexchange/engine/payments.nim @@ -1,3 +1,12 @@ +## Nim-Codex +## Copyright (c) 2021 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + import std/math import pkg/nitro import pkg/questionable/results diff --git a/dagger/blockexchange/pendingblocks.nim b/codex/blockexchange/engine/pendingblocks.nim similarity index 96% rename from dagger/blockexchange/pendingblocks.nim rename to codex/blockexchange/engine/pendingblocks.nim index f1e30f6c..c24c7519 100644 --- a/dagger/blockexchange/pendingblocks.nim +++ b/codex/blockexchange/engine/pendingblocks.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -18,10 +18,10 @@ import pkg/chronicles import pkg/chronos import pkg/libp2p -import ../blocktype +import ../../blocktype logScope: - topics = "dagger blockexc pendingblocks" + topics = "codex blockexc pendingblocks" const DefaultBlockTimeout* = 10.minutes diff --git a/codex/blockexchange/network.nim b/codex/blockexchange/network.nim new file mode 100644 index 00000000..ccaa6207 --- /dev/null +++ b/codex/blockexchange/network.nim @@ -0,0 +1,4 @@ +import ./network/network +import ./network/networkpeer + +export network, networkpeer diff --git a/dagger/blockexchange/network.nim b/codex/blockexchange/network/network.nim similarity index 94% rename from dagger/blockexchange/network.nim rename to codex/blockexchange/network/network.nim index 60fe6a28..40ba33a2 100644 --- a/dagger/blockexchange/network.nim +++ b/codex/blockexchange/network/network.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -17,18 +17,18 @@ import pkg/libp2p import pkg/questionable import pkg/questionable/results -import ../blocktype as bt -import ./protobuf/blockexc as pb -import ./protobuf/payments +import ../../blocktype as bt +import ../protobuf/blockexc as pb +import ../protobuf/payments import ./networkpeer -export networkpeer, payments +export network, payments logScope: - topics = "dagger blockexc network" + topics = "codex blockexc network" -const Codec* = "/dagger/blockexc/1.0.0" +const Codec* = "/codex/blockexc/1.0.0" type WantListHandler* = proc(peer: PeerID, wantList: WantList): Future[void] {.gcsafe.} @@ -82,7 +82,7 @@ proc handleWantList( if isNil(b.handlers.onWantList): return - trace "Handling want list for peer", peer = peer.id + trace "Handling want list for peer", peer = peer.id, items = list.entries.len b.handlers.onWantList(peer.id, list) # TODO: make into a template @@ -119,7 +119,7 @@ proc broadcastWantList*( if id notin b.peers: return - trace "Sending want list to peer", peer = id, `type` = $wantType, len = cids.len + trace "Sending want list to peer", peer = id, `type` = $wantType, items = cids.len let wantList = makeWantList( @@ -142,7 +142,7 @@ proc handleBlocks( if isNil(b.handlers.onBlocks): return - trace "Handling blocks for peer", peer = peer.id + trace "Handling blocks for peer", peer = peer.id, items = blocks.len var blks: seq[bt.Block] for blob in blocks: @@ -178,7 +178,7 @@ proc broadcastBlocks*( return b.peers.withValue(id, peer): - trace "Sending blocks to peer", peer = id, len = blocks.len + trace "Sending blocks to peer", peer = id, items = blocks.len peer[].broadcast(pb.Message(payload: makeBlocks(blocks))) proc handleBlockPresence( @@ -191,7 +191,7 @@ proc handleBlockPresence( if isNil(b.handlers.onPresence): return - trace "Handling block presence for peer", peer = peer.id + trace "Handling block presence for peer", peer = peer.id, items = presence.len b.handlers.onPresence(peer.id, presence) proc broadcastBlockPresence*( @@ -204,7 +204,7 @@ proc broadcastBlockPresence*( if id notin b.peers: return - trace "Sending presence to peer", peer = id + trace "Sending presence to peer", peer = id, items = presence.len b.peers.withValue(id, peer): peer[].broadcast(Message(blockPresences: @presence)) diff --git a/dagger/blockexchange/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim similarity index 92% rename from dagger/blockexchange/networkpeer.nim rename to codex/blockexchange/network/networkpeer.nim index bf84384a..9f648e37 100644 --- a/dagger/blockexchange/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -12,13 +12,13 @@ import pkg/chronicles import pkg/protobuf_serialization import pkg/libp2p -import ./protobuf/blockexc +import ../protobuf/blockexc logScope: - topics = "dagger blockexc networkpeer" + topics = "codex blockexc networkpeer" const - MaxMessageSize = 100 * 1024 * 1024 # manifest files can be big + MaxMessageSize = 100 * 1 shl 20 # manifest files can be big type RPCHandler* = proc(peer: NetworkPeer, msg: Message): Future[void] {.gcsafe.} @@ -38,7 +38,7 @@ proc readLoop*(b: NetworkPeer, conn: Connection) {.async.} = return try: - while not conn.atEof: + while not conn.atEof or not conn.closed: let data = await conn.readLp(MaxMessageSize) msg: Message = Protobuf.decode(data, Message) diff --git a/codex/blockexchange/peers.nim b/codex/blockexchange/peers.nim new file mode 100644 index 00000000..940f66e4 --- /dev/null +++ b/codex/blockexchange/peers.nim @@ -0,0 +1,4 @@ +import ./peers/peerctxstore +import ./peers/peercontext + +export peerctxstore, peercontext diff --git a/dagger/blockexchange/peercontext.nim b/codex/blockexchange/peers/peercontext.nim similarity index 76% rename from dagger/blockexchange/peercontext.nim rename to codex/blockexchange/peers/peercontext.nim index 0d303703..9aab35b2 100644 --- a/dagger/blockexchange/peercontext.nim +++ b/codex/blockexchange/peers/peercontext.nim @@ -1,3 +1,12 @@ +## Nim-Codex +## Copyright (c) 2021 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + import std/sequtils import std/tables import pkg/libp2p @@ -5,9 +14,9 @@ import pkg/chronos import pkg/nitro import pkg/questionable -import ./protobuf/blockexc -import ./protobuf/payments -import ./protobuf/presence +import ../protobuf/blockexc +import ../protobuf/payments +import ../protobuf/presence export payments, nitro diff --git a/dagger/blockexchange/peers/peerctxstore.nim b/codex/blockexchange/peers/peerctxstore.nim similarity index 96% rename from dagger/blockexchange/peers/peerctxstore.nim rename to codex/blockexchange/peers/peerctxstore.nim index 9453a6f8..442dd206 100644 --- a/dagger/blockexchange/peers/peerctxstore.nim +++ b/codex/blockexchange/peers/peerctxstore.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -21,11 +21,11 @@ import pkg/libp2p import ../protobuf/blockexc -import ../peercontext +import ./peercontext export peercontext logScope: - topics = "dagger blockexc peerctxstore" + topics = "codex blockexc peerctxstore" type PeerCtxStore* = ref object of RootObj diff --git a/dagger/blockexchange/protobuf/blockexc.nim b/codex/blockexchange/protobuf/blockexc.nim similarity index 98% rename from dagger/blockexchange/protobuf/blockexc.nim rename to codex/blockexchange/protobuf/blockexc.nim index b7b7bf47..326a098a 100644 --- a/dagger/blockexchange/protobuf/blockexc.nim +++ b/codex/blockexchange/protobuf/blockexc.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/blockexchange/protobuf/message.proto b/codex/blockexchange/protobuf/message.proto similarity index 100% rename from dagger/blockexchange/protobuf/message.proto rename to codex/blockexchange/protobuf/message.proto diff --git a/dagger/blockexchange/protobuf/message.proto.license b/codex/blockexchange/protobuf/message.proto.license similarity index 100% rename from dagger/blockexchange/protobuf/message.proto.license rename to codex/blockexchange/protobuf/message.proto.license diff --git a/dagger/blockexchange/protobuf/payments.nim b/codex/blockexchange/protobuf/payments.nim similarity index 100% rename from dagger/blockexchange/protobuf/payments.nim rename to codex/blockexchange/protobuf/payments.nim diff --git a/dagger/blockexchange/protobuf/presence.nim b/codex/blockexchange/protobuf/presence.nim similarity index 100% rename from dagger/blockexchange/protobuf/presence.nim rename to codex/blockexchange/protobuf/presence.nim diff --git a/dagger/blocktype.nim b/codex/blocktype.nim similarity index 98% rename from dagger/blocktype.nim rename to codex/blocktype.nim index ce48d497..f16d155f 100644 --- a/dagger/blocktype.nim +++ b/codex/blocktype.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -59,7 +59,7 @@ template EmptyDigests*: untyped = .get() }.toTable, CIDv1: { - multiCodec("sha2-256"): EmptyCid[CIDv0] + multiCodec("sha2-256"): EmptyCid[CIDv1] .catch .get()[multiCodec("sha2-256")] .catch diff --git a/dagger/chunker.nim b/codex/chunker.nim similarity index 99% rename from dagger/chunker.nim rename to codex/chunker.nim index 1ecb5cb2..6b0b9cc2 100644 --- a/dagger/chunker.nim +++ b/codex/chunker.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/clock.nim b/codex/clock.nim similarity index 100% rename from dagger/clock.nim rename to codex/clock.nim diff --git a/dagger/dagger.nim b/codex/codex.nim similarity index 89% rename from dagger/dagger.nim rename to codex/codex.nim index f938bd8b..27baa2a2 100644 --- a/dagger/dagger.nim +++ b/codex/codex.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -33,26 +33,26 @@ import ./discovery import ./contracts type - DaggerServer* = ref object + CodexServer* = ref object runHandle: Future[void] - config: DaggerConf + config: CodexConf restServer: RestServerRef - daggerNode: DaggerNodeRef + codexNode: CodexNodeRef -proc start*(s: DaggerServer) {.async.} = +proc start*(s: CodexServer) {.async.} = s.restServer.start() - await s.daggerNode.start() + await s.codexNode.start() s.runHandle = newFuture[void]() await s.runHandle -proc stop*(s: DaggerServer) {.async.} = +proc stop*(s: CodexServer) {.async.} = await allFuturesThrowing( - s.restServer.stop(), s.daggerNode.stop()) + s.restServer.stop(), s.codexNode.stop()) s.runHandle.complete() -proc new*(T: type DaggerServer, config: DaggerConf): T = +proc new*(T: type CodexServer, config: CodexConf): T = const SafePermissions = {UserRead, UserWrite} let @@ -130,9 +130,9 @@ proc new*(T: type DaggerServer, config: DaggerConf): T = config.ethDeployment, config.ethAccount ) - daggerNode = DaggerNodeRef.new(switch, store, engine, erasure, blockDiscovery, contracts) + codexNode = CodexNodeRef.new(switch, store, engine, erasure, blockDiscovery, contracts) restServer = RestServerRef.new( - daggerNode.initRestApi(config), + codexNode.initRestApi(config), initTAddress("127.0.0.1" , config.apiPort), bufferSize = (1024 * 64), maxRequestBodySize = int.high) @@ -141,6 +141,6 @@ proc new*(T: type DaggerServer, config: DaggerConf): T = switch.mount(network) T( config: config, - daggerNode: daggerNode, + codexNode: codexNode, restServer: restServer, ) diff --git a/dagger/conf.nim b/codex/conf.nim similarity index 94% rename from dagger/conf.nim rename to codex/conf.nim index 5ef7f770..6f16adb2 100644 --- a/dagger/conf.nim +++ b/codex/conf.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -44,7 +44,7 @@ type Json = "json" None = "none" - DaggerConf* = object + CodexConf* = object logLevel* {. defaultValue: LogLevel.INFO desc: "Sets the log level", @@ -74,7 +74,7 @@ type name: "metrics-port" }: Port dataDir* {. - desc: "The directory where dagger will store configuration and data." + desc: "The directory where codex will store configuration and data." defaultValue: defaultDataDir() defaultValueDesc: "" abbr: "d" @@ -124,7 +124,7 @@ type name: "max-peers" }: int agentString* {. - defaultValue: "Dagger" + defaultValue: "Codex" desc: "Node agent string which is used as identifier in network" name: "agent-string" }: string @@ -171,20 +171,20 @@ const nimBanner* = staticExec("nim --version | grep Version") #TODO add versionMajor, Minor & Fix when we switch to semver - daggerVersion* = gitRevision + codexVersion* = gitRevision - daggerFullVersion* = - "Dagger build " & daggerVersion & "\p" & + codexFullVersion* = + "Codex build " & codexVersion & "\p" & nimBanner proc defaultDataDir*(): string = let dataDir = when defined(windows): - "AppData" / "Roaming" / "Dagger" + "AppData" / "Roaming" / "Codex" elif defined(macosx): - "Library" / "Application Support" / "Dagger" + "Library" / "Application Support" / "Codex" else: - ".cache" / "dagger" + ".cache" / "codex" getHomeDir() / dataDir @@ -246,7 +246,7 @@ proc stripAnsi(v: string): string = res -proc setupLogging*(conf: DaggerConf) = +proc setupLogging*(conf: CodexConf) = when defaultChroniclesStream.outputs.type.arity != 2: warn "Logging configuration options not enabled in the current build" else: @@ -283,7 +283,7 @@ proc setupLogging*(conf: DaggerConf) = setLogLevel(conf.logLevel) -proc setupMetrics*(config: DaggerConf) = +proc setupMetrics*(config: CodexConf) = if config.metricsEnabled: let metricsAddress = config.metricsAddress notice "Starting metrics HTTP server", diff --git a/dagger/contracts.nim b/codex/contracts.nim similarity index 100% rename from dagger/contracts.nim rename to codex/contracts.nim diff --git a/dagger/contracts/Readme.md b/codex/contracts/Readme.md similarity index 94% rename from dagger/contracts/Readme.md rename to codex/contracts/Readme.md index 687b0acc..6a4ea8d0 100644 --- a/dagger/contracts/Readme.md +++ b/codex/contracts/Readme.md @@ -1,13 +1,13 @@ -Dagger Contracts in Nim +Codex Contracts in Nim ======================= -Nim API for the [Dagger smart contracts][1]. +Nim API for the [Codex smart contracts][1]. Usage ----- For a global overview of the steps involved in starting and fulfilling a -storage contract, see [Dagger Contracts][1]. +storage contract, see [Codex Contracts][1]. Smart contract -------------- @@ -15,7 +15,7 @@ Smart contract Connecting to the smart contract on an Ethereum node: ```nim -import dagger/contracts +import codex/contracts import ethers let address = # fill in address where the contract was deployed @@ -178,4 +178,4 @@ await storage [1]: https://github.com/status-im/dagger-contracts/ [2]: https://ethereum.org/en/developers/docs/standards/tokens/erc-20/ -[3]: https://github.com/status-im/dagger-research/blob/main/design/storage-proof-timing.md +[3]: https://github.com/status-im/codex-research/blob/main/design/storage-proof-timing.md diff --git a/dagger/contracts/clock.nim b/codex/contracts/clock.nim similarity index 100% rename from dagger/contracts/clock.nim rename to codex/contracts/clock.nim diff --git a/dagger/contracts/deployment.nim b/codex/contracts/deployment.nim similarity index 100% rename from dagger/contracts/deployment.nim rename to codex/contracts/deployment.nim diff --git a/dagger/contracts/interactions.nim b/codex/contracts/interactions.nim similarity index 100% rename from dagger/contracts/interactions.nim rename to codex/contracts/interactions.nim diff --git a/dagger/contracts/market.nim b/codex/contracts/market.nim similarity index 100% rename from dagger/contracts/market.nim rename to codex/contracts/market.nim diff --git a/dagger/contracts/offers.nim b/codex/contracts/offers.nim similarity index 100% rename from dagger/contracts/offers.nim rename to codex/contracts/offers.nim diff --git a/dagger/contracts/proofs.nim b/codex/contracts/proofs.nim similarity index 98% rename from dagger/contracts/proofs.nim rename to codex/contracts/proofs.nim index 0c835e68..4b6a7535 100644 --- a/dagger/contracts/proofs.nim +++ b/codex/contracts/proofs.nim @@ -1,5 +1,5 @@ import pkg/ethers -import ../por/timing/proofs +import ../storageproofs/timing/proofs import ./storage export proofs diff --git a/dagger/contracts/requests.nim b/codex/contracts/requests.nim similarity index 100% rename from dagger/contracts/requests.nim rename to codex/contracts/requests.nim diff --git a/dagger/contracts/storage.nim b/codex/contracts/storage.nim similarity index 100% rename from dagger/contracts/storage.nim rename to codex/contracts/storage.nim diff --git a/dagger/contracts/testtoken.nim b/codex/contracts/testtoken.nim similarity index 100% rename from dagger/contracts/testtoken.nim rename to codex/contracts/testtoken.nim diff --git a/dagger/discovery.nim b/codex/discovery.nim similarity index 56% rename from dagger/discovery.nim rename to codex/discovery.nim index 44dfd04d..98089383 100644 --- a/dagger/discovery.nim +++ b/codex/discovery.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -7,12 +7,17 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. +import std/algorithm + import pkg/chronos import pkg/chronicles import pkg/libp2p +import pkg/libp2p/routing_record +import pkg/libp2p/signed_envelope import pkg/questionable import pkg/questionable/results import pkg/stew/shims/net +import pkg/contractabi/address as ca import pkg/libp2pdht/discv5/protocol as discv5 import ./rng @@ -20,6 +25,10 @@ import ./errors export discv5 +# TODO: If generics in methods had not been +# deprecated, this could have been implemented +# much more elegantly. + type Discovery* = ref object of RootObj protocol: discv5.Protocol @@ -42,21 +51,31 @@ proc new*( ), localInfo: localInfo) +proc toNodeId*(cid: Cid): NodeId = + ## Cid to discovery id + ## + + readUintBE[256](keccak256.digest(cid.data.buffer).data) + +proc toNodeId*(host: ca.Address): NodeId = + ## Eth address to discovery id + ## + + readUintBE[256](keccak256.digest(host.toArray).data) + proc findPeer*( d: Discovery, peerId: PeerID): Future[?PeerRecord] {.async.} = - let node = await d.protocol.resolve(toNodeId(peerId)) + let + node = await d.protocol.resolve(toNodeId(peerId)) + return if node.isSome(): some(node.get().record.data) else: none(PeerRecord) -proc toDiscoveryId*(cid: Cid): NodeId = - ## To discovery id - readUintBE[256](keccak256.digest(cid.data.buffer).data) - -method findBlockProviders*( +method find*( d: Discovery, cid: Cid): Future[seq[SignedPeerRecord]] {.async, base.} = ## Find block providers @@ -64,19 +83,19 @@ method findBlockProviders*( trace "Finding providers for block", cid = $cid without providers =? - (await d.protocol.getProviders(cid.toDiscoveryId())).mapFailure, error: + (await d.protocol.getProviders(cid.toNodeId())).mapFailure, error: trace "Error finding providers for block", cid = $cid, error = error.msg return providers -method provideBlock*(d: Discovery, cid: Cid) {.async, base.} = +method provide*(d: Discovery, cid: Cid) {.async, base.} = ## Provide a bock Cid ## trace "Providing block", cid = $cid let nodes = await d.protocol.addProvider( - cid.toDiscoveryId(), + cid.toNodeId(), d.localInfo.signedPeerRecord) if nodes.len <= 0: @@ -84,6 +103,39 @@ method provideBlock*(d: Discovery, cid: Cid) {.async, base.} = trace "Provided to nodes", nodes = nodes.len +method find*( + d: Discovery, + host: ca.Address): Future[seq[SignedPeerRecord]] {.async, base.} = + ## Find host providers + ## + + trace "Finding providers for host", host = $host + without var providers =? + (await d.protocol.getProviders(host.toNodeId())).mapFailure, error: + trace "Error finding providers for host", host = $host, exc = error.msg + return + + if providers.len <= 0: + trace "No providers found", host = $host + return + + providers.sort do(a, b: SignedPeerRecord) -> int: + system.cmp[uint64](a.data.seqNo, b.data.seqNo) + + return providers + +method provide*(d: Discovery, host: ca.Address) {.async, base.} = + ## Provide hosts + ## + + trace "Providing host", host = $host + let + nodes = await d.protocol.addProvider( + host.toNodeId(), + d.localInfo.signedPeerRecord) + if nodes.len > 0: + trace "Provided to nodes", nodes = nodes.len + proc start*(d: Discovery) {.async.} = d.protocol.updateRecord(d.localInfo.signedPeerRecord).expect("updating SPR") d.protocol.open() diff --git a/dagger/erasure.nim b/codex/erasure.nim similarity index 97% rename from dagger/erasure.nim rename to codex/erasure.nim index b55ba413..b14d8e52 100644 --- a/dagger/erasure.nim +++ b/codex/erasure.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/erasure/backend.nim b/codex/erasure/backend.nim similarity index 98% rename from dagger/erasure/backend.nim rename to codex/erasure/backend.nim index 35a30cd2..9feac36b 100644 --- a/dagger/erasure/backend.nim +++ b/codex/erasure/backend.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/erasure/backends/leopard.nim b/codex/erasure/backends/leopard.nim similarity index 99% rename from dagger/erasure/backends/leopard.nim rename to codex/erasure/backends/leopard.nim index afeb13e7..bf0a8161 100644 --- a/dagger/erasure/backends/leopard.nim +++ b/codex/erasure/backends/leopard.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/erasure/erasure.nim b/codex/erasure/erasure.nim similarity index 99% rename from dagger/erasure/erasure.nim rename to codex/erasure/erasure.nim index c47f57cd..87fc6a17 100644 --- a/dagger/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -26,7 +26,7 @@ import ./backend export backend logScope: - topics = "dagger erasure" + topics = "codex erasure" type ## Encode a manifest into one that is erasure protected. diff --git a/dagger/errors.nim b/codex/errors.nim similarity index 75% rename from dagger/errors.nim rename to codex/errors.nim index 847db46b..b3777f2a 100644 --- a/dagger/errors.nim +++ b/codex/errors.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -10,12 +10,12 @@ import pkg/stew/results type - DaggerError* = object of CatchableError # base dagger error - DaggerResult*[T] = Result[T, ref DaggerError] + CodexError* = object of CatchableError # base codex error + CodexResult*[T] = Result[T, ref CodexError] template mapFailure*( exp: untyped, - exc: typed = type DaggerError): untyped = + exc: typed = type CodexError): untyped = ## Convert `Result[T, E]` to `Result[E, ref CatchableError]` ## diff --git a/dagger/manifest.nim b/codex/manifest.nim similarity index 100% rename from dagger/manifest.nim rename to codex/manifest.nim diff --git a/dagger/manifest/coders.nim b/codex/manifest/coders.nim similarity index 99% rename from dagger/manifest/coders.nim rename to codex/manifest/coders.nim index 02b76c6f..f99277bd 100644 --- a/dagger/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/manifest/manifest.nim b/codex/manifest/manifest.nim similarity index 99% rename from dagger/manifest/manifest.nim rename to codex/manifest/manifest.nim index e797ef4b..ebfe039a 100644 --- a/dagger/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/manifest/types.nim b/codex/manifest/types.nim similarity index 99% rename from dagger/manifest/types.nim rename to codex/manifest/types.nim index b0b7ddec..3ae0e497 100644 --- a/dagger/manifest/types.nim +++ b/codex/manifest/types.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/market.nim b/codex/market.nim similarity index 100% rename from dagger/market.nim rename to codex/market.nim diff --git a/dagger/node.nim b/codex/node.nim similarity index 89% rename from dagger/node.nim rename to codex/node.nim index 8a40baaf..8ae3a1c9 100644 --- a/dagger/node.nim +++ b/codex/node.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -32,12 +32,15 @@ import ./discovery import ./contracts logScope: - topics = "dagger node" + topics = "codex node" + +const + PrefetchBatch = 100 type - DaggerError = object of CatchableError + CodexError = object of CatchableError - DaggerNodeRef* = ref object + CodexNodeRef* = ref object switch*: Switch networkId*: PeerID blockStore*: BlockStore @@ -46,7 +49,7 @@ type discovery*: Discovery contracts*: ?ContractInteractions -proc start*(node: DaggerNodeRef) {.async.} = +proc start*(node: CodexNodeRef) {.async.} = if not node.switch.isNil: await node.switch.start() @@ -63,9 +66,9 @@ proc start*(node: DaggerNodeRef) {.async.} = await contracts.start() node.networkId = node.switch.peerInfo.peerId - notice "Started dagger node", id = $node.networkId, addrs = node.switch.peerInfo.addrs + notice "Started codex node", id = $node.networkId, addrs = node.switch.peerInfo.addrs -proc stop*(node: DaggerNodeRef) {.async.} = +proc stop*(node: CodexNodeRef) {.async.} = trace "Stopping node" if not node.engine.isNil: @@ -84,28 +87,28 @@ proc stop*(node: DaggerNodeRef) {.async.} = await contracts.stop() proc findPeer*( - node: DaggerNodeRef, + node: CodexNodeRef, peerId: PeerID): Future[?PeerRecord] {.async.} = return await node.discovery.findPeer(peerId) proc connect*( - node: DaggerNodeRef, + node: CodexNodeRef, peerId: PeerID, addrs: seq[MultiAddress]): Future[void] = node.switch.connect(peerId, addrs) proc retrieve*( - node: DaggerNodeRef, + node: CodexNodeRef, cid: Cid): Future[?!LPStream] {.async.} = trace "Received retrieval request", cid without blk =? await node.blockStore.getBlock(cid): return failure( - newException(DaggerError, "Couldn't retrieve block for Cid!")) + newException(CodexError, "Couldn't retrieve block for Cid!")) without mc =? blk.cid.contentType(): return failure( - newException(DaggerError, "Couldn't identify Cid!")) + newException(CodexError, "Couldn't identify Cid!")) # if we got a manifest, stream the blocks if $mc in ManifestContainers: @@ -128,8 +131,12 @@ proc retrieve*( ## Initiates requests to all blocks in the manifest ## try: - discard await allFinished( - manifest.mapIt( node.blockStore.getBlock( it ) )) + let + batch = manifest.blocks.len div PrefetchBatch + trace "Prefetching in batches of", batch + for blks in manifest.blocks.distribute(batch, true): + discard await allFinished( + blks.mapIt( node.blockStore.getBlock( it ) )) except CatchableError as exc: trace "Exception prefetching blocks", exc = exc.msg @@ -152,7 +159,7 @@ proc retrieve*( return LPStream(stream).success() proc store*( - node: DaggerNodeRef, + node: CodexNodeRef, stream: LPStream): Future[?!Cid] {.async.} = trace "Storing data" @@ -186,7 +193,7 @@ proc store*( # Generate manifest without data =? blockManifest.encode(): return failure( - newException(DaggerError, "Could not generate dataset manifest!")) + newException(CodexError, "Could not generate dataset manifest!")) # Store as a dag-pb block without manifest =? bt.Block.new(data = data, codec = DagPBCodec): @@ -207,7 +214,7 @@ proc store*( return manifest.cid.success -proc requestStorage*(self: DaggerNodeRef, +proc requestStorage*(self: CodexNodeRef, cid: Cid, duration: UInt256, nodes: uint, @@ -289,7 +296,7 @@ proc requestStorage*(self: DaggerNodeRef, return success purchase.id proc new*( - T: type DaggerNodeRef, + T: type CodexNodeRef, switch: Switch, store: BlockStore, engine: BlockExcEngine, diff --git a/dagger/proving.nim b/codex/proving.nim similarity index 98% rename from dagger/proving.nim rename to codex/proving.nim index 26107296..6c65b775 100644 --- a/dagger/proving.nim +++ b/codex/proving.nim @@ -2,11 +2,11 @@ import std/sets import pkg/upraises import pkg/questionable import pkg/chronicles -import ./por/timing/proofs +import ./storageproofs import ./clock export sets -export proofs +export storageproofs type Proving* = ref object diff --git a/dagger/purchasing.nim b/codex/purchasing.nim similarity index 100% rename from dagger/purchasing.nim rename to codex/purchasing.nim diff --git a/dagger/rest/api.nim b/codex/rest/api.nim similarity index 93% rename from dagger/rest/api.nim rename to codex/rest/api.nim index d0aef0ae..4b6efac9 100644 --- a/dagger/rest/api.nim +++ b/codex/rest/api.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -47,10 +47,10 @@ proc decodeString(T: type Cid, value: string): Result[Cid, cstring] = .init(value) .mapErr do(e: CidError) -> cstring: case e - of CidError.Incorrect: "Incorrect Cid" - of CidError.Unsupported: "Unsupported Cid" - of CidError.Overrun: "Overrun Cid" - else: "Error parsing Cid" + of CidError.Incorrect: "Incorrect Cid".cstring + of CidError.Unsupported: "Unsupported Cid".cstring + of CidError.Overrun: "Overrun Cid".cstring + else: "Error parsing Cid".cstring proc encodeString(peerId: PeerID): Result[string, cstring] = ok($peerId) @@ -102,11 +102,11 @@ proc decodeString(_: type array[32, byte], except ValueError as e: err e.msg.cstring -proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = +proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = var router = RestRouter.init(validate) router.api( MethodGet, - "/api/dagger/v1/connect/{peerId}") do ( + "/api/codex/v1/connect/{peerId}") do ( peerId: PeerID, addrs: seq[MultiAddress]) -> RestApiResponse: ## Connect to a peer @@ -140,7 +140,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.api( MethodGet, - "/api/dagger/v1/download/{id}") do ( + "/api/codex/v1/download/{id}") do ( id: Cid, resp: HttpResponseRef) -> RestApiResponse: ## Download a file from the node in a streaming ## manner @@ -185,7 +185,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.rawApi( MethodPost, - "/api/dagger/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse: + "/api/codex/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse: ## Create a request for storage ## ## cid - the cid of a previously uploaded dataset @@ -216,7 +216,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.rawApi( MethodPost, - "/api/dagger/v1/upload") do ( + "/api/codex/v1/upload") do ( ) -> RestApiResponse: ## Upload a file in a streamming manner ## @@ -273,7 +273,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.api( MethodGet, - "/api/dagger/v1/info") do () -> RestApiResponse: + "/api/codex/v1/info") do () -> RestApiResponse: ## Print rudimentary node information ## @@ -288,7 +288,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.api( MethodGet, - "/api/dagger/v1/sales/availability") do () -> RestApiResponse: + "/api/codex/v1/sales/availability") do () -> RestApiResponse: ## Returns storage that is for sale without contracts =? node.contracts: @@ -299,7 +299,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.rawApi( MethodPost, - "/api/dagger/v1/sales/availability") do () -> RestApiResponse: + "/api/codex/v1/sales/availability") do () -> RestApiResponse: ## Add available storage to sell ## ## size - size of available storage in bytes @@ -321,7 +321,7 @@ proc initRestApi*(node: DaggerNodeRef, conf: DaggerConf): RestRouter = router.api( MethodGet, - "/api/dagger/v1/storage/purchases/{id}") do ( + "/api/codex/v1/storage/purchases/{id}") do ( id: array[32, byte]) -> RestApiResponse: without contracts =? node.contracts: diff --git a/dagger/rest/json.nim b/codex/rest/json.nim similarity index 100% rename from dagger/rest/json.nim rename to codex/rest/json.nim diff --git a/dagger/rng.nim b/codex/rng.nim similarity index 99% rename from dagger/rng.nim rename to codex/rng.nim index 61ff24aa..b1f6266d 100644 --- a/dagger/rng.nim +++ b/codex/rng.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/sales.nim b/codex/sales.nim similarity index 100% rename from dagger/sales.nim rename to codex/sales.nim diff --git a/codex/storageproofs.nim b/codex/storageproofs.nim new file mode 100644 index 00000000..ef0a8f00 --- /dev/null +++ b/codex/storageproofs.nim @@ -0,0 +1,7 @@ +import ./storageproofs/por +import ./storageproofs/timing +import ./storageproofs/stpstore +import ./storageproofs/stpnetwork +import ./storageproofs/stpproto + +export por, timing, stpstore, stpnetwork, stpproto diff --git a/dagger/por/backends/backend_blst.nim b/codex/storageproofs/backends/backend_blst.nim similarity index 100% rename from dagger/por/backends/backend_blst.nim rename to codex/storageproofs/backends/backend_blst.nim diff --git a/dagger/por/backends/backend_constantine.nim b/codex/storageproofs/backends/backend_constantine.nim similarity index 100% rename from dagger/por/backends/backend_constantine.nim rename to codex/storageproofs/backends/backend_constantine.nim diff --git a/codex/storageproofs/por.nim b/codex/storageproofs/por.nim new file mode 100644 index 00000000..79ada5dd --- /dev/null +++ b/codex/storageproofs/por.nim @@ -0,0 +1,4 @@ +import ./por/serialization +import ./por/por + +export por, serialization diff --git a/dagger/por/por.nim b/codex/storageproofs/por/por.nim similarity index 66% rename from dagger/por/por.nim rename to codex/storageproofs/por/por.nim index 01a0b984..ad2860a8 100644 --- a/dagger/por/por.nim +++ b/codex/storageproofs/por/por.nim @@ -47,7 +47,7 @@ # Our implementation uses additive cyclic groups instead of the multiplicative # cyclic group in the paper, thus changing the name of the group operation as in # blscurve and blst. Thus, point multiplication becomes point addition, and scalar -# exponentiation becomes scalar multiplication. +# exponentiation becomes scalar multiplicaiton. # # Number of operations: # The following table summarizes the number of operations in different phases @@ -85,134 +85,184 @@ # q * (8 + 48) bytes # The size of the proof is instead # s * 32 + 48 bytes +import std/endians # Select backend to use # - blst supports only the BLS12-381 curve # - constantine is more experimental, supports BLS and BN curves as well # As of now configuration of backends is in the backend_* file itself when defined(por_backend_constantine): - import ./backends/backend_constantine + import ../backends/backend_constantine else: - import ./backends/backend_blst + import ../backends/backend_blst -import ../rng -import endians +import pkg/chronos +import pkg/blscurve +import pkg/blscurve/blst/blst_abi + +import ../../rng +import ../../streams # sector size in bytes. Must be smaller than the subgroup order r # which is 255 bits long for BLS12-381 -const bytespersector = 31 +const + BytesPerSector* = 31 -# length in bytes of the unique (random) name -const namelen = 512 + # length in bytes of the unique (random) name + Namelen = 512 type # a single sector - ZChar = array[bytespersector, byte] + ZChar* = array[BytesPerSector, byte] # secret key combining the metadata signing key and the POR generation key - SecretKey = object - signkey: ec_SecretKey - key: ec_scalar + SecretKey* = object + signkey*: ec_SecretKey + key*: ec_scalar # public key combining the metadata signing key and the POR validation key - PublicKey = object - signkey: ec_PublicKey - key: ec_p2 + PublicKey* = object + signkey*: ec_PublicKey + key*: ec_p2 # POR metadata (called "file tag t_0" in the original paper) - TauZero = object - name: array[namelen, byte] - n: int64 - u: seq[ec_p1] + TauZero* = object + name*: array[Namelen, byte] + n*: int64 + u*: seq[ec_p1] # signed POR metadata (called "signed file tag t" in the original paper) - Tau = object - t: TauZero - signature: array[96, byte] + Tau* = object + t*: TauZero + signature*: array[96, byte] + + Proof* = object + mu*: seq[blst_scalar] + sigma*: blst_p1 # PoR query element - QElement = object - I: int64 - V: ec_scalar + QElement* = object + I*: int64 + V*: ec_scalar + + PoR* = object + ssk*: SecretKey + spk*: PublicKey + tau*: Tau + authenticators*: seq[ec_p1] proc fromBytesBE(a: array[32, byte]): ec_scalar = - ## Convert data to blst native form + ## Convert data to native form + ## + ec_scalar_from_bendian(result, a) doAssert(ec_scalar_fr_check(result).bool) proc fromBytesBE(a: openArray[byte]): ec_scalar = - ## Convert data to blst native form + ## Convert data to native form + ## + var b: array[32, byte] doAssert(a.len <= b.len) + let d = b.len - a.len - for i in 0 ..< a.len: + for i in 0.. postion - f.setFilePos((blockid * spb + sectorid) * sizeof(result)) - let r = f.readBytes(result, 0, sizeof(result)) + ## + + var res: ZChar + stream.setPos(((blockid * spb + sectorid) * ZChar.len).int) + discard await stream.readOnce(addr res[0], ZChar.len) + return res proc rndScalar(scalar: var ec_scalar): void = ## Generate random scalar within the subroup order r - var scal{.noInit.}: array[32, byte] + ## + + var scal {.noInit.}: array[32, byte] while true: for val in scal.mitems: val = byte Rng.instance.rand(0xFF) + scalar.ec_scalar_from_bendian(scal) if ec_scalar_fr_check(scalar).bool: break proc rndP2(x: var ec_p2, scalar: var ec_scalar): void = ## Generate random point on G2 + ## + x.ec_p2_from_affine(EC_G2) # init from generator scalar.rndScalar() x.ec_p2_mult(x, scalar, 255) proc rndP1(x: var ec_p1, scalar: var ec_scalar): void = ## Generate random point on G1 + ## + x.ec_p1_from_affine(EC_G1) # init from generator scalar.rndScalar() x.ec_p1_mult(x, scalar, 255) -let posKeygen = rndP2 +template posKeygen(x: var ec_p2, scalar: var ec_scalar): void = ## Generate POS key pair + ## -proc keygen*(): (PublicKey, SecretKey) = + rndP2(x, scalar) + +proc keyGen*(): (PublicKey, SecretKey) = ## Generate key pair for signing metadata and for POS tags - var pk: PublicKey - var sk: SecretKey - var ikm: array[32, byte] + ## + + var + pk: PublicKey + sk: SecretKey + ikm: array[32, byte] for b in ikm.mitems: b = byte Rng.instance.rand(0xFF) + doAssert ikm.ec_keygen(pk.signkey, sk.signkey) posKeygen(pk.key, sk.key) return (pk, sk) -proc split(f: File, s: int64): int64 = +proc sectorsCount(stream: SeekableStream, s: int64): int64 = ## Calculate number of blocks for a file - let size = f.getFileSize() - let n = ((size - 1) div (s * sizeof(ZChar))) + 1 - echo "File size=", size, " bytes", - ", blocks=", n, - ", sectors/block=", $s, - ", sectorsize=", $sizeof(ZChar), " bytes" + ## + + let + size = stream.size() + n = ((size - 1) div (s * sizeof(ZChar))) + 1 + # debugEcho "File size=", size, " bytes", + # ", blocks=", n, + # ", sectors/block=", $s, + # ", sectorsize=", $sizeof(ZChar), " bytes" return n proc hashToG1(msg: openArray[byte]): ec_p1 = ## Hash to curve with Dagger specific domain separation + ## + const dst = "DAGGER-PROOF-OF-CONCEPT" result.ec_hash_to_g1(msg, dst, aug = "") -proc hashNameI(name: array[namelen, byte], i: int64): ec_p1 = +proc hashNameI(name: array[Namelen, byte], i: int64): ec_p1 = ## Calculate unique filname and block index based hash + ## # # naive implementation, hashing a long string representation # # such as "[255, 242, 23]1" @@ -224,20 +274,33 @@ proc hashNameI(name: array[namelen, byte], i: int64): ec_p1 = bigEndian64(addr(namei[sizeof(name)]), unsafeAddr(i)) return hashToG1(namei) -proc generateAuthenticatorNaive(i: int64, s: int64, t: TauZero, f: File, ssk: SecretKey): ec_p1 = +proc generateAuthenticatorNaive( + stream: SeekableStream, + ssk: SecretKey, + i: int64, + s: int64, + t: TauZero): Future[ec_p1] {.async.} = ## Naive implementation of authenticator as in the S&W paper. ## With the paper's multiplicative notation: ## \sigmai=\(H(file||i)\cdot\prod{j=0}^{s-1}{uj^{m[i][j]}})^{\alpha} + ## + var sum: ec_p1 - for j in 0 ..< s: + for j in 0.. 0 and not self.tagsHandle.isNil: + await self.tagsHandle(message.tagsMsg) + except CatchableError as exc: + trace "Exception handling Storage Proofs message", exc = exc.msg + finally: + await conn.close() + + self.handler = handle + self.codec = Codec + +proc new*( + T: type StpNetwork, + switch: Switch, + discovery: Discovery): StpNetwork = + let + self = StpNetwork( + switch: switch, + discovery: discovery) + + self.init() + self diff --git a/codex/storageproofs/stpproto.nim b/codex/storageproofs/stpproto.nim new file mode 100644 index 00000000..39303099 --- /dev/null +++ b/codex/storageproofs/stpproto.nim @@ -0,0 +1,7 @@ +import pkg/protobuf_serialization + +import_proto3 "stp.proto" + +export StorageProofsMessage +export TagsMessage +export Tag diff --git a/codex/storageproofs/stpstore.nim b/codex/storageproofs/stpstore.nim new file mode 100644 index 00000000..e64cd107 --- /dev/null +++ b/codex/storageproofs/stpstore.nim @@ -0,0 +1,121 @@ +## Nim-Dagger +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/os +import std/strformat + +import pkg/libp2p +import pkg/chronos +import pkg/chronicles +import pkg/stew/io2 +import pkg/questionable +import pkg/questionable/results +import pkg/protobuf_serialization + +import ./stpproto +import ./por + +type + StpStore* = object + authDir*: string + postfixLen*: int + +template stpPath*(self: StpStore, cid: Cid): string = + self.authDir / ($cid)[^self.postfixLen..^1] / $cid + +proc retrieve*( + self: StpStore, + cid: Cid): Future[?!PorMessage] {.async.} = + ## Retrieve authenticators from data store + ## + + let path = self.stpPath(cid) / "por" + var data: seq[byte] + if ( + let res = io2.readFile(path, data); + res.isErr): + let error = io2.ioErrorMsg(res.error) + trace "Cannot retrieve storage proof data from fs", path , error + return failure("Cannot retrieve storage proof data from fs") + + return Protobuf.decode(data, PorMessage).success + +proc store*( + self: StpStore, + por: PorMessage, + cid: Cid): Future[?!void] {.async.} = + ## Persist storage proofs + ## + + let + dir = self.stpPath(cid) + + if io2.createPath(dir).isErr: + trace "Unable to create storage proofs prefix dir", dir + return failure(&"Unable to create storage proofs prefix dir ${dir}") + + let path = dir / "por" + if ( + let res = io2.writeFile(path, Protobuf.encode(por)); + res.isErr): + let error = io2.ioErrorMsg(res.error) + trace "Unable to store storage proofs", path, cid = cid, error + return failure( + &"Unable to store storage proofs - path = ${path} cid = ${$cid} error = ${error}") + + return success() + +proc retrieve*( + self: StpStore, + cid: Cid, + blocks: seq[int]): Future[?!seq[Tag]] {.async.} = + var tags: seq[Tag] + for b in blocks: + var tag = Tag(idx: b) + let path = self.stpPath(cid) / $b + if ( + let res = io2.readFile(path, tag.tag); + res.isErr): + let error = io2.ioErrorMsg(res.error) + trace "Cannot retrieve tags from fs", path , error + return failure("Cannot retrieve tags from fs") + tags.add(tag) + + return tags.success + +proc store*( + self: StpStore, + tags: seq[Tag], + cid: Cid): Future[?!void] {.async.} = + let + dir = self.stpPath(cid) + + if io2.createPath(dir).isErr: + trace "Unable to create storage proofs prefix dir", dir + return failure(&"Unable to create storage proofs prefix dir ${dir}") + + for t in tags: + let path = dir / $t.idx + if ( + let res = io2.writeFile(path, t.tag); + res.isErr): + let error = io2.ioErrorMsg(res.error) + trace "Unable to store tags", path, cid = cid, error + return failure( + &"Unable to store tags - path = ${path} cid = ${$cid} error = ${error}") + + return success() + +proc init*( + T: type StpStore, + authDir: string, + postfixLen: int = 2): StpStore = + T( + authDir: authDir, + postfixLen: postfixLen) diff --git a/codex/storageproofs/timing.nim b/codex/storageproofs/timing.nim new file mode 100644 index 00000000..163295e0 --- /dev/null +++ b/codex/storageproofs/timing.nim @@ -0,0 +1,4 @@ +import ./timing/periods +import ./timing/proofs + +export periods, proofs diff --git a/dagger/por/timing/periods.nim b/codex/storageproofs/timing/periods.nim similarity index 100% rename from dagger/por/timing/periods.nim rename to codex/storageproofs/timing/periods.nim diff --git a/dagger/por/timing/proofs.nim b/codex/storageproofs/timing/proofs.nim similarity index 100% rename from dagger/por/timing/proofs.nim rename to codex/storageproofs/timing/proofs.nim diff --git a/dagger/stores.nim b/codex/stores.nim similarity index 100% rename from dagger/stores.nim rename to codex/stores.nim diff --git a/dagger/stores/blockstore.nim b/codex/stores/blockstore.nim similarity index 99% rename from dagger/stores/blockstore.nim rename to codex/stores/blockstore.nim index ddcbc550..6ae774be 100644 --- a/dagger/stores/blockstore.nim +++ b/codex/stores/blockstore.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/stores/cachestore.nim b/codex/stores/cachestore.nim similarity index 97% rename from dagger/stores/cachestore.nim rename to codex/stores/cachestore.nim index 50cfaf9d..0864c55d 100644 --- a/dagger/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -29,7 +29,7 @@ import ../errors export blockstore logScope: - topics = "dagger cachestore" + topics = "codex cachestore" type CacheStore* = ref object of BlockStore @@ -37,7 +37,7 @@ type size*: Positive # in bytes cache: LruCache[Cid, Block] - InvalidBlockSize* = object of DaggerError + InvalidBlockSize* = object of CodexError const MiB* = 1024 * 1024 # bytes, 1 mebibyte = 1,048,576 bytes diff --git a/dagger/stores/fsstore.nim b/codex/stores/fsstore.nim similarity index 95% rename from dagger/stores/fsstore.nim rename to codex/stores/fsstore.nim index a7c76732..de846e50 100644 --- a/dagger/stores/fsstore.nim +++ b/codex/stores/fsstore.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -26,7 +26,7 @@ import ./blockstore export blockstore logScope: - topics = "dagger fsstore" + topics = "codex fsstore" type FSStore* = ref object of BlockStore @@ -150,10 +150,6 @@ method listBlocks*(self: FSStore, onBlock: OnBlock) {.async.} = except CatchableError as exc: trace "Couldn't get block", cid = $(cid.get()) - # TODO: this should run on a thread which - # wouldn't need the sleep - await sleepAsync(100.millis) # avoid blocking - proc new*( T: type FSStore, repoDir: string, diff --git a/dagger/stores/networkstore.nim b/codex/stores/networkstore.nim similarity index 97% rename from dagger/stores/networkstore.nim rename to codex/stores/networkstore.nim index a3cd9a71..951d853f 100644 --- a/dagger/stores/networkstore.nim +++ b/codex/stores/networkstore.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -24,7 +24,7 @@ import ../blockexchange export blockstore, blockexchange, asyncheapqueue logScope: - topics = "dagger networkstore" + topics = "codex networkstore" type NetworkStore* = ref object of BlockStore diff --git a/dagger/streams.nim b/codex/streams.nim similarity index 100% rename from dagger/streams.nim rename to codex/streams.nim diff --git a/dagger/streams/seekablestream.nim b/codex/streams/seekablestream.nim similarity index 92% rename from dagger/streams/seekablestream.nim rename to codex/streams/seekablestream.nim index 2c1ec1b8..ab5ae0fd 100644 --- a/dagger/streams/seekablestream.nim +++ b/codex/streams/seekablestream.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2022 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) @@ -14,7 +14,7 @@ import pkg/chronicles export libp2p, chronos, chronicles logScope: - topics = "dagger seekablestream" + topics = "codex seekablestream" type SeekableStream* = ref object of LPStream diff --git a/dagger/streams/storestream.nim b/codex/streams/storestream.nim similarity index 82% rename from dagger/streams/storestream.nim rename to codex/streams/storestream.nim index 9e87710c..cece19a5 100644 --- a/dagger/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -31,6 +31,7 @@ type StoreStream* = ref object of SeekableStream store*: BlockStore manifest*: Manifest + emptyBlock*: seq[byte] proc new*( T: type StoreStream, @@ -39,7 +40,8 @@ proc new*( result = T( store: store, manifest: manifest, - offset: 0) + offset: 0, + emptyBlock: newSeq[byte](manifest.blockSize)) result.initStream() @@ -57,12 +59,12 @@ method readOnce*( var read = 0 + trace "Reading from manifest", cid = self.manifest.cid.get(), blocks = self.manifest.len while read < nbytes and not self.atEof: let pos = self.offset div self.manifest.blockSize blk = (await self.store.getBlock(self.manifest[pos])).tryGet() - let blockOffset = if self.offset >= self.manifest.blockSize: self.offset mod self.manifest.blockSize @@ -75,7 +77,15 @@ method readOnce*( else: min(nbytes - read, self.manifest.blockSize) - copyMem(pbytes.offset(read), unsafeAddr blk.data[blockOffset], readBytes) + trace "Reading bytes from store stream", pos, cid = blk.cid, bytes = readBytes, blockOffset = blockOffset + copyMem( + pbytes.offset(read), + if blk.isEmpty: + self.emptyBlock[blockOffset].addr + else: + blk.data[blockOffset].addr, + readBytes) + self.offset += readBytes read += readBytes diff --git a/dagger/utils.nim b/codex/utils.nim similarity index 62% rename from dagger/utils.nim rename to codex/utils.nim index 2f481e62..8d561d40 100644 --- a/dagger/utils.nim +++ b/codex/utils.nim @@ -1,4 +1,4 @@ import ./utils/asyncheapqueue import ./utils/fileutils -export asyncheapqueue, fileutils \ No newline at end of file +export asyncheapqueue, fileutils diff --git a/dagger/utils/asyncheapqueue.nim b/codex/utils/asyncheapqueue.nim similarity index 99% rename from dagger/utils/asyncheapqueue.nim rename to codex/utils/asyncheapqueue.nim index efbc7a66..b85f7de7 100644 --- a/dagger/utils/asyncheapqueue.nim +++ b/codex/utils/asyncheapqueue.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/dagger/utils/fileutils.nim b/codex/utils/fileutils.nim similarity index 99% rename from dagger/utils/fileutils.nim rename to codex/utils/fileutils.nim index eeaba77f..a900460a 100644 --- a/dagger/utils/fileutils.nim +++ b/codex/utils/fileutils.nim @@ -1,4 +1,4 @@ -## Nim-Dagger +## Nim-Codex ## Copyright (c) 2021 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) diff --git a/config.nims b/config.nims index d43c6d7c..12ad5c0e 100644 --- a/config.nims +++ b/config.nims @@ -75,8 +75,8 @@ switch("warning", "LockLevel:off") switch("define", "libp2p_pki_schemes=secp256k1") #TODO this infects everything in this folder, ideally it would only -# apply to dagger.nim, but since dagger.nims is used for other purpose -# we can't use it. And dagger.cfg doesn't work +# apply to codex.nim, but since codex.nims is used for other purpose +# we can't use it. And codex.cfg doesn't work switch("define", "chronicles_sinks=textlines[dynamic],json[dynamic]") # begin Nimble config (version 1) diff --git a/dagger/blockexchange/peers.nim b/dagger/blockexchange/peers.nim deleted file mode 100644 index e159806a..00000000 --- a/dagger/blockexchange/peers.nim +++ /dev/null @@ -1,3 +0,0 @@ -import ./peers/peerctxstore - -export peerctxstore diff --git a/dagger/por/README.md b/dagger/por/README.md deleted file mode 100644 index 258606b2..00000000 --- a/dagger/por/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Nim implementation of Proof of Storage related schemes - -Implementation of the BLS-based public PoS scheme from -Shacham H., Waters B., "Compact Proofs of Retrievability" -using pairing over BLS12-381 ECC. - -For a detailed description of the implementation, see `bls.nim` - -For a usage example, see `testbls.nim` diff --git a/dagger/por/benchmark.nim b/dagger/por/benchmark.nim deleted file mode 100644 index 332b971a..00000000 --- a/dagger/por/benchmark.nim +++ /dev/null @@ -1,9 +0,0 @@ -import times, strutils -export strutils.formatFloat - -template benchmark*(benchmarkName: string, code: untyped) = - let t0 = epochTime() - code - let elapsed = epochTime() - t0 - let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 3) - echo "CPU Time [", benchmarkName, "] ", elapsedStr, "s" diff --git a/dagger/por/example.txt b/dagger/por/example.txt deleted file mode 100644 index 5be377b5..00000000 --- a/dagger/por/example.txt +++ /dev/null @@ -1 +0,0 @@ -The quick brown fox jumps over the lazy dog! diff --git a/dagger/por/testpor.nim b/dagger/por/testpor.nim deleted file mode 100644 index 718ee9d7..00000000 --- a/dagger/por/testpor.nim +++ /dev/null @@ -1,37 +0,0 @@ -## Nim-POS -## Copyright (c) 2021 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import por -import benchmark -import strutils - -const sectorsperblock = 1024.int64 -const querylen = 22 - -proc testbls() : bool = - benchmark "Key generation": - let (spk, ssk) = por.keygen() - - benchmark "Auth generation (s=" & $sectorsperblock & ")": - let (tau, authenticators) = por.setup(ssk, sectorsperblock, "example.txt") - #echo "Auth: ", authenticators - - benchmark "Generating challenge (q=" & $querylen & ")": - let q = por.generateQuery(tau, spk, querylen) - #echo "Generated!" #, " q:", q - - benchmark "Issuing proof": - let (mu, sigma) = por.generateProof(q, authenticators, spk, sectorsperblock, "example.txt") - #echo "Issued!" #, " mu:", mu, " sigma:", sigma - - benchmark "Verifying proof": - result = por.verifyProof(tau, q, mu, sigma, spk) - echo "Result: ", result - -let r = testbls() diff --git a/tests/dagger/blockexc/discovery/testdiscovery.nim b/tests/codex/blockexchange/discovery/testdiscovery.nim similarity index 88% rename from tests/dagger/blockexc/discovery/testdiscovery.nim rename to tests/codex/blockexchange/discovery/testdiscovery.nim index 520a5bee..e71b78b3 100644 --- a/tests/dagger/blockexc/discovery/testdiscovery.nim +++ b/tests/codex/blockexchange/discovery/testdiscovery.nim @@ -8,11 +8,11 @@ import pkg/chronos import pkg/libp2p import pkg/libp2p/errors -import pkg/dagger/rng -import pkg/dagger/stores -import pkg/dagger/blockexchange -import pkg/dagger/chunker -import pkg/dagger/blocktype as bt +import pkg/codex/rng +import pkg/codex/stores +import pkg/codex/blockexchange +import pkg/codex/chunker +import pkg/codex/blocktype as bt import ../../helpers/mockdiscovery @@ -43,7 +43,7 @@ suite "Block Advertising and Discovery": blocks.add(bt.Block.new(chunk).tryGet()) switch = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr}) - blockDiscovery = MockDiscovery.new(switch.peerInfo, 0.Port) + blockDiscovery = MockDiscovery.new() wallet = WalletRef.example network = BlockExcNetwork.new(switch) localStore = CacheStore.new(blocks.mapIt( it )) @@ -76,7 +76,7 @@ suite "Block Advertising and Discovery": await engine.start() - blockDiscovery.publishProvideHandler = + blockDiscovery.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async, gcsafe.} = return @@ -94,7 +94,7 @@ suite "Block Advertising and Discovery": advertised = initTable.collect: for b in blocks: {b.cid: newFuture[void]()} - blockDiscovery.publishProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} = + blockDiscovery.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} = if cid in advertised and not advertised[cid].finished(): advertised[cid].complete() @@ -150,7 +150,7 @@ suite "E2E - Multiple Nodes Discovery": for _ in 0..<4: let s = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr}) - blockDiscovery = MockDiscovery.new(s.peerInfo, 0.Port) + blockDiscovery = MockDiscovery.new() wallet = WalletRef.example network = BlockExcNetwork.new(s) localStore = CacheStore.new() @@ -189,15 +189,15 @@ suite "E2E - Multiple Nodes Discovery": var advertised: Table[Cid, SignedPeerRecord] MockDiscovery(blockexc[1].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised.add(cid, switch[1].peerInfo.signedPeerRecord) MockDiscovery(blockexc[2].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised.add(cid, switch[2].peerInfo.signedPeerRecord) MockDiscovery(blockexc[3].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised.add(cid, switch[3].peerInfo.signedPeerRecord) await blockexc[1].engine.blocksHandler(switch[0].peerInfo.peerId, blocks[0..5]) @@ -231,15 +231,15 @@ suite "E2E - Multiple Nodes Discovery": var advertised: Table[Cid, SignedPeerRecord] MockDiscovery(blockexc[1].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised[cid] = switch[1].peerInfo.signedPeerRecord MockDiscovery(blockexc[2].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised[cid] = switch[2].peerInfo.signedPeerRecord MockDiscovery(blockexc[3].engine.discovery.discovery) - .publishProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = advertised[cid] = switch[3].peerInfo.signedPeerRecord await blockexc[1].engine.blocksHandler(switch[0].peerInfo.peerId, blocks[0..5]) diff --git a/tests/dagger/blockexc/discovery/testdiscoveryengine.nim b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim similarity index 95% rename from tests/dagger/blockexc/discovery/testdiscoveryengine.nim rename to tests/codex/blockexchange/discovery/testdiscoveryengine.nim index 766fe0f4..f8fd316f 100644 --- a/tests/dagger/blockexc/discovery/testdiscoveryengine.nim +++ b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim @@ -5,15 +5,14 @@ import std/tables import pkg/asynctest import pkg/chronos -import pkg/chronicles import pkg/libp2p -import pkg/dagger/rng -import pkg/dagger/stores -import pkg/dagger/blockexchange -import pkg/dagger/chunker -import pkg/dagger/blocktype as bt -import pkg/dagger/blockexchange/engine +import pkg/codex/rng +import pkg/codex/stores +import pkg/codex/blockexchange +import pkg/codex/chunker +import pkg/codex/blocktype as bt +import pkg/codex/blockexchange/engine import ../../helpers/mockdiscovery @@ -80,7 +79,7 @@ suite "Test Discovery Engine": for b in blocks: { b.cid: newFuture[void]() } - blockDiscovery.publishProvideHandler = + blockDiscovery.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} = if not haves[cid].finished: haves[cid].complete @@ -125,7 +124,7 @@ suite "Test Discovery Engine": discoveryLoopSleep = 100.millis) have = newFuture[void]() - blockDiscovery.publishProvideHandler = + blockDiscovery.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} = check cid == blocks[0].cid if not have.finished: @@ -217,7 +216,7 @@ suite "Test Discovery Engine": reqs = newFuture[void]() count = 0 - blockDiscovery.publishProvideHandler = + blockDiscovery.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} = check cid == blocks[0].cid if count > 0: diff --git a/tests/dagger/blockexc/testblockexc.nim b/tests/codex/blockexchange/engine/testblockexc.nim similarity index 97% rename from tests/dagger/blockexc/testblockexc.nim rename to tests/codex/blockexchange/engine/testblockexc.nim index b878b6ff..f898bb79 100644 --- a/tests/dagger/blockexc/testblockexc.nim +++ b/tests/codex/blockexchange/engine/testblockexc.nim @@ -8,15 +8,14 @@ import pkg/stew/byteutils import pkg/libp2p import pkg/libp2p/errors -import pkg/dagger/rng -import pkg/dagger/stores -import pkg/dagger/blockexchange -import pkg/dagger/chunker -import pkg/dagger/discovery -import pkg/dagger/blocktype as bt +import pkg/codex/rng +import pkg/codex/stores +import pkg/codex/blockexchange +import pkg/codex/chunker +import pkg/codex/discovery +import pkg/codex/blocktype as bt -import ../helpers -import ../examples +import ../../helpers suite "NetworkStore engine - 2 nodes": let diff --git a/tests/dagger/blockexc/testengine.nim b/tests/codex/blockexchange/engine/testengine.nim similarity index 97% rename from tests/dagger/blockexc/testengine.nim rename to tests/codex/blockexchange/engine/testengine.nim index 7813b626..e6193e70 100644 --- a/tests/dagger/blockexc/testengine.nim +++ b/tests/codex/blockexchange/engine/testengine.nim @@ -9,16 +9,16 @@ import pkg/libp2p import pkg/libp2p/routing_record import pkg/libp2pdht/discv5/protocol as discv5 -import pkg/dagger/rng -import pkg/dagger/blockexchange -import pkg/dagger/stores -import pkg/dagger/chunker -import pkg/dagger/discovery -import pkg/dagger/blocktype as bt -import pkg/dagger/utils/asyncheapqueue +import pkg/codex/rng +import pkg/codex/blockexchange +import pkg/codex/stores +import pkg/codex/chunker +import pkg/codex/discovery +import pkg/codex/blocktype as bt +import pkg/codex/utils/asyncheapqueue -import ../helpers -import ../examples +import ../../helpers +import ../../examples suite "NetworkStore engine basic": var diff --git a/tests/dagger/blockexc/engine/testpayments.nim b/tests/codex/blockexchange/engine/testpayments.nim similarity index 97% rename from tests/dagger/blockexc/engine/testpayments.nim rename to tests/codex/blockexchange/engine/testpayments.nim index fa8dfe6d..b50a1720 100644 --- a/tests/dagger/blockexc/engine/testpayments.nim +++ b/tests/codex/blockexchange/engine/testpayments.nim @@ -1,6 +1,6 @@ import std/unittest -import pkg/dagger/stores +import pkg/codex/stores import ../../examples suite "engine payments": diff --git a/tests/dagger/blockexc/protobuf/testpayments.nim b/tests/codex/blockexchange/protobuf/testpayments.nim similarity index 97% rename from tests/dagger/blockexc/protobuf/testpayments.nim rename to tests/codex/blockexchange/protobuf/testpayments.nim index a341255e..c9d6f9c6 100644 --- a/tests/dagger/blockexc/protobuf/testpayments.nim +++ b/tests/codex/blockexchange/protobuf/testpayments.nim @@ -2,7 +2,7 @@ import pkg/asynctest import pkg/chronos import pkg/stew/byteutils import ../../examples -import pkg/dagger/stores +import pkg/codex/stores suite "account protobuf messages": diff --git a/tests/dagger/blockexc/protobuf/testpresence.nim b/tests/codex/blockexchange/protobuf/testpresence.nim similarity index 96% rename from tests/dagger/blockexc/protobuf/testpresence.nim rename to tests/codex/blockexchange/protobuf/testpresence.nim index 65969d0c..43593e13 100644 --- a/tests/dagger/blockexc/protobuf/testpresence.nim +++ b/tests/codex/blockexchange/protobuf/testpresence.nim @@ -2,7 +2,7 @@ import pkg/asynctest import pkg/chronos import pkg/libp2p -import pkg/dagger/blockexchange/protobuf/presence +import pkg/codex/blockexchange/protobuf/presence import ../../examples suite "block presence protobuf messages": diff --git a/tests/codex/blockexchange/testdiscovery.nim b/tests/codex/blockexchange/testdiscovery.nim new file mode 100644 index 00000000..6f443014 --- /dev/null +++ b/tests/codex/blockexchange/testdiscovery.nim @@ -0,0 +1,4 @@ +import ./discovery/testdiscovery +import ./discovery/testdiscoveryengine + +{.warning[UnusedImport]: off.} diff --git a/tests/codex/blockexchange/testengine.nim b/tests/codex/blockexchange/testengine.nim new file mode 100644 index 00000000..5277e027 --- /dev/null +++ b/tests/codex/blockexchange/testengine.nim @@ -0,0 +1,5 @@ +import ./engine/testengine +import ./engine/testblockexc +import ./engine/testpayments + +{.warning[UnusedImport]: off.} diff --git a/tests/dagger/blockexc/testnetwork.nim b/tests/codex/blockexchange/testnetwork.nim similarity index 98% rename from tests/dagger/blockexc/testnetwork.nim rename to tests/codex/blockexchange/testnetwork.nim index 6aa5b589..fe955d35 100644 --- a/tests/dagger/blockexc/testnetwork.nim +++ b/tests/codex/blockexchange/testnetwork.nim @@ -7,10 +7,10 @@ import pkg/libp2p import pkg/libp2p/errors import pkg/protobuf_serialization -import pkg/dagger/rng -import pkg/dagger/chunker -import pkg/dagger/blocktype as bt -import pkg/dagger/blockexchange +import pkg/codex/rng +import pkg/codex/chunker +import pkg/codex/blocktype as bt +import pkg/codex/blockexchange import ../helpers import ../examples diff --git a/tests/dagger/blockexc/testpeerctxstore.nim b/tests/codex/blockexchange/testpeerctxstore.nim similarity index 96% rename from tests/dagger/blockexc/testpeerctxstore.nim rename to tests/codex/blockexchange/testpeerctxstore.nim index 21f75cb3..a1c826e3 100644 --- a/tests/dagger/blockexc/testpeerctxstore.nim +++ b/tests/codex/blockexchange/testpeerctxstore.nim @@ -4,8 +4,8 @@ import std/sequtils import pkg/unittest2 import pkg/libp2p -import pkg/dagger/blockexchange/peers -import pkg/dagger/blockexchange/protobuf/blockexc +import pkg/codex/blockexchange/peers +import pkg/codex/blockexchange/protobuf/blockexc import ../examples diff --git a/tests/codex/blockexchange/testprotobuf.nim b/tests/codex/blockexchange/testprotobuf.nim new file mode 100644 index 00000000..1d41d6b4 --- /dev/null +++ b/tests/codex/blockexchange/testprotobuf.nim @@ -0,0 +1,4 @@ +import ./protobuf/testpayments +import ./protobuf/testpresence + +{.warning[UnusedImport]: off.} diff --git a/tests/dagger/examples.nim b/tests/codex/examples.nim similarity index 93% rename from tests/dagger/examples.nim rename to tests/codex/examples.nim index c4eb8fe4..8f75b02b 100644 --- a/tests/dagger/examples.nim +++ b/tests/codex/examples.nim @@ -3,10 +3,10 @@ import std/sequtils import pkg/libp2p import pkg/nitro import pkg/stint -import pkg/dagger/rng -import pkg/dagger/stores -import pkg/dagger/blocktype as bt -import pkg/dagger/sales +import pkg/codex/rng +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/sales import ../examples export examples diff --git a/tests/codex/helpers.nim b/tests/codex/helpers.nim new file mode 100644 index 00000000..9f57fd9c --- /dev/null +++ b/tests/codex/helpers.nim @@ -0,0 +1,61 @@ +import pkg/chronos +import pkg/libp2p +import pkg/libp2p/varint +import pkg/codex/blocktype as bt +import pkg/codex/stores +import pkg/codex/manifest +import pkg/codex/rng + +import ./helpers/nodeutils +import ./helpers/randomchunker +import ./helpers/mockdiscovery + +export randomchunker, nodeutils, mockdiscovery + +# NOTE: The meaning of equality for blocks +# is changed here, because blocks are now `ref` +# types. This is only in tests!!! +func `==`*(a, b: bt.Block): bool = + (a.cid == b.cid) and (a.data == b. data) + +proc lenPrefix*(msg: openArray[byte]): seq[byte] = + ## Write `msg` with a varint-encoded length prefix + ## + + let vbytes = PB.toBytes(msg.len().uint64) + var buf = newSeqUninitialized[byte](msg.len() + vbytes.len) + buf[0..= blks: + break + + var i = -1 + if (i = Rng.instance.rand(manifest.len - 1); pos.find(i) >= 0): + continue + + pos.add(i) + var + blk = (await store.getBlock(manifest[i])).tryGet() + bytePos: seq[int] + + while true: + if bytePos.len > bytes: + break + + var ii = -1 + if (ii = Rng.instance.rand(blk.data.len - 1); bytePos.find(ii) >= 0): + continue + + bytePos.add(ii) + blk.data[ii] = byte 0 + + return pos diff --git a/tests/dagger/helpers/mockclock.nim b/tests/codex/helpers/mockclock.nim similarity index 95% rename from tests/dagger/helpers/mockclock.nim rename to tests/codex/helpers/mockclock.nim index 6ff8a411..55283f8a 100644 --- a/tests/dagger/helpers/mockclock.nim +++ b/tests/codex/helpers/mockclock.nim @@ -1,5 +1,5 @@ import std/times -import dagger/clock +import codex/clock export clock diff --git a/tests/codex/helpers/mockdiscovery.nim b/tests/codex/helpers/mockdiscovery.nim new file mode 100644 index 00000000..86e517a6 --- /dev/null +++ b/tests/codex/helpers/mockdiscovery.nim @@ -0,0 +1,63 @@ +## Nim-Codex +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import pkg/chronos +import pkg/libp2p +import pkg/questionable +import pkg/questionable/results +import pkg/stew/shims/net +import pkg/codex/discovery +import pkg/contractabi/address as ca + +type + MockDiscovery* = ref object of Discovery + findBlockProvidersHandler*: proc(d: MockDiscovery, cid: Cid): + Future[seq[SignedPeerRecord]] {.gcsafe.} + publishBlockProvideHandler*: proc(d: MockDiscovery, cid: Cid): + Future[void] {.gcsafe.} + findHostProvidersHandler*: proc(d: MockDiscovery, host: ca.Address): + Future[seq[SignedPeerRecord]] {.gcsafe.} + publishHostProvideHandler*: proc(d: MockDiscovery, host: ca.Address): + Future[void] {.gcsafe.} + +proc new*(T: type MockDiscovery): T = + T() + +proc findPeer*( + d: Discovery, + peerId: PeerID): Future[?PeerRecord] {.async.} = + return none(PeerRecord) + +method find*( + d: MockDiscovery, + cid: Cid): Future[seq[SignedPeerRecord]] {.async.} = + if isNil(d.findBlockProvidersHandler): + return + + return await d.findBlockProvidersHandler(d, cid) + +method provide*(d: MockDiscovery, cid: Cid): Future[void] {.async.} = + if isNil(d.publishBlockProvideHandler): + return + + await d.publishBlockProvideHandler(d, cid) + +method find*( + d: MockDiscovery, + host: ca.Address): Future[seq[SignedPeerRecord]] {.async.} = + if isNil(d.findHostProvidersHandler): + return + + return await d.findHostProvidersHandler(d, host) + +method provide*(d: MockDiscovery, host: ca.Address): Future[void] {.async.} = + if isNil(d.publishHostProvideHandler): + return + + await d.publishHostProvideHandler(d, host) diff --git a/tests/dagger/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim similarity index 99% rename from tests/dagger/helpers/mockmarket.nim rename to tests/codex/helpers/mockmarket.nim index 1852be03..48b7f591 100644 --- a/tests/dagger/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -1,7 +1,7 @@ import std/sequtils import std/heapqueue import pkg/questionable -import pkg/dagger/market +import pkg/codex/market export market diff --git a/tests/dagger/helpers/mockproofs.nim b/tests/codex/helpers/mockproofs.nim similarity index 98% rename from tests/dagger/helpers/mockproofs.nim rename to tests/codex/helpers/mockproofs.nim index f9fdbe5b..605775fb 100644 --- a/tests/dagger/helpers/mockproofs.nim +++ b/tests/codex/helpers/mockproofs.nim @@ -2,7 +2,7 @@ import std/sets import std/tables import std/sequtils import pkg/upraises -import pkg/dagger/por/timing/proofs +import pkg/codex/storageproofs type MockProofs* = ref object of Proofs diff --git a/tests/dagger/helpers/nodeutils.nim b/tests/codex/helpers/nodeutils.nim similarity index 93% rename from tests/dagger/helpers/nodeutils.nim rename to tests/codex/helpers/nodeutils.nim index bfc21a24..a792aa81 100644 --- a/tests/dagger/helpers/nodeutils.nim +++ b/tests/codex/helpers/nodeutils.nim @@ -3,10 +3,10 @@ import std/sequtils import pkg/chronos import pkg/libp2p -import pkg/dagger/discovery -import pkg/dagger/stores -import pkg/dagger/blocktype as bt -import pkg/dagger/blockexchange +import pkg/codex/discovery +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/blockexchange import ../examples diff --git a/tests/dagger/helpers/randomchunker.nim b/tests/codex/helpers/randomchunker.nim similarity index 94% rename from tests/dagger/helpers/randomchunker.nim rename to tests/codex/helpers/randomchunker.nim index 4e3e5aad..87726953 100644 --- a/tests/dagger/helpers/randomchunker.nim +++ b/tests/codex/helpers/randomchunker.nim @@ -2,8 +2,8 @@ import std/sequtils import pkg/chronos -import pkg/dagger/chunker -import pkg/dagger/rng +import pkg/codex/chunker +import pkg/codex/rng export chunker diff --git a/tests/codex/storageproofs/testnetwork.nim b/tests/codex/storageproofs/testnetwork.nim new file mode 100644 index 00000000..10357ba1 --- /dev/null +++ b/tests/codex/storageproofs/testnetwork.nim @@ -0,0 +1,128 @@ +import std/os +import std/sequtils + +import pkg/asynctest +import pkg/chronos +import pkg/libp2p +import pkg/libp2p/errors +import pkg/protobuf_serialization +import pkg/contractabi as ca + +import pkg/codex/rng +import pkg/codex/chunker +import pkg/codex/storageproofs +import pkg/codex/discovery +import pkg/codex/manifest +import pkg/codex/stores +import pkg/codex/storageproofs as st +import pkg/codex/blocktype as bt +import pkg/codex/streams + +import ../examples +import ../helpers + +const + SectorSize = 31 + SectorsPerBlock = BlockSize div SectorSize + DataSetSize = BlockSize * 100 + +suite "Storage Proofs Network": + let + rng = Rng.instance() + seckey1 = PrivateKey.random(rng[]).tryGet() + seckey2 = PrivateKey.random(rng[]).tryGet() + hostAddr1 = ca.Address.example + hostAddr2 = ca.Address.example + blocks = toSeq([1, 5, 10, 14, 20, 12, 22]) # TODO: maybe make them random + + var + stpNetwork1: StpNetwork + stpNetwork2: StpNetwork + switch1: Switch + switch2: Switch + discovery1: MockDiscovery + discovery2: MockDiscovery + + chunker: RandomChunker + manifest: Manifest + store: BlockStore + ssk: st.SecretKey + spk: st.PublicKey + repoDir: string + stpstore: st.StpStore + porMsg: PorMessage + cid: Cid + por: PoR + tags: seq[Tag] + + setupAll: + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) + manifest = Manifest.new(blockSize = BlockSize).tryGet() + (spk, ssk) = st.keyGen() + + while ( + let chunk = await chunker.getBytes(); + chunk.len > 0): + + let + blk = bt.Block.new(chunk).tryGet() + + manifest.add(blk.cid) + if not (await store.putBlock(blk)): + raise newException(CatchableError, "Unable to store block " & $blk.cid) + + cid = manifest.cid.tryGet() + por = await PoR.init( + StoreStream.new(store, manifest), + ssk, spk, + BlockSize) + + porMsg = por.toMessage() + tags = blocks.mapIt( + Tag(idx: it, tag: porMsg.authenticators[it]) ) + + setup: + switch1 = newStandardSwitch() + switch2 = newStandardSwitch() + + discovery1 = MockDiscovery.new(switch1.peerInfo) + discovery2 = MockDiscovery.new(switch2.peerInfo) + + stpNetwork1 = StpNetwork.new(switch1, discovery1) + stpNetwork2 = StpNetwork.new(switch2, discovery2) + + switch1.mount(stpNetwork1) + switch2.mount(stpNetwork2) + + await switch1.start() + await switch2.start() + + teardown: + await switch1.stop() + await switch2.stop() + + test "Should upload to host": + var + done = newFuture[void]() + + discovery1.findHostProvidersHandler = proc(d: MockDiscovery, host: ca.Address): + Future[seq[SignedPeerRecord]] {.async, gcsafe.} = + check hostAddr2 == host + return @[switch2.peerInfo.signedPeerRecord] + + proc tagsHandler(msg: TagsMessage) {.async, gcsafe.} = + check: + Cid.init(msg.cid).tryGet() == cid + msg.tags == tags + + done.complete() + + stpNetwork2.tagsHandle = tagsHandler + (await stpNetwork1.uploadTags( + cid, + blocks, + porMsg.authenticators, + hostAddr2)).tryGet() + + await done.wait(1.seconds) diff --git a/tests/codex/storageproofs/testpor.nim b/tests/codex/storageproofs/testpor.nim new file mode 100644 index 00000000..5b9c8481 --- /dev/null +++ b/tests/codex/storageproofs/testpor.nim @@ -0,0 +1,165 @@ +import pkg/chronos +import pkg/asynctest + +import pkg/blscurve/blst/blst_abi + +import pkg/codex/streams +import pkg/codex/storageproofs as st +import pkg/codex/stores +import pkg/codex/manifest +import pkg/codex/chunker +import pkg/codex/rng +import pkg/codex/blocktype as bt + +import ../helpers + +const + SectorSize = 31 + SectorsPerBlock = BlockSize div SectorSize + DataSetSize = BlockSize * 100 + +suite "BLS PoR": + var + chunker: RandomChunker + manifest: Manifest + store: BlockStore + ssk: st.SecretKey + spk: st.PublicKey + + setup: + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) + manifest = Manifest.new(blockSize = BlockSize).tryGet() + (spk, ssk) = st.keyGen() + + while ( + let chunk = await chunker.getBytes(); + chunk.len > 0): + + let + blk = bt.Block.new(chunk).tryGet() + + manifest.add(blk.cid) + if not (await store.putBlock(blk)): + raise newException(CatchableError, "Unable to store block " & $blk.cid) + + test "Test PoR without corruption": + let + por = await PoR.init( + StoreStream.new(store, manifest), + ssk, + spk, + BlockSize) + q = generateQuery(por.tau, 22) + proof = await generateProof( + StoreStream.new(store, manifest), + q, + por.authenticators, + SectorsPerBlock) + + check por.verifyProof(q, proof.mu, proof.sigma) + + test "Test PoR with corruption - query: 22, corrupted blocks: 300, bytes: 10": + let + por = await PoR.init( + StoreStream.new(store, manifest), + ssk, + spk, + BlockSize) + pos = await store.corruptBlocks(manifest, 30, 10) + q = generateQuery(por.tau, 22) + proof = await generateProof( + StoreStream.new(store, manifest), + q, + por.authenticators, + SectorsPerBlock) + + check pos.len == 30 + check not por.verifyProof(q, proof.mu, proof.sigma) + +suite "Test Serialization": + var + chunker: RandomChunker + manifest: Manifest + store: BlockStore + ssk: st.SecretKey + spk: st.PublicKey + por: PoR + q: seq[QElement] + proof: Proof + + setupAll: + chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) + store = CacheStore.new(cacheSize = DataSetSize, chunkSize = BlockSize) + manifest = Manifest.new(blockSize = BlockSize).tryGet() + + while ( + let chunk = await chunker.getBytes(); + chunk.len > 0): + + let + blk = bt.Block.new(chunk).tryGet() + + manifest.add(blk.cid) + if not (await store.putBlock(blk)): + raise newException(CatchableError, "Unable to store block " & $blk.cid) + + (spk, ssk) = st.keyGen() + por = await PoR.init( + StoreStream.new(store, manifest), + ssk, + spk, + BlockSize) + q = generateQuery(por.tau, 22) + proof = await generateProof( + StoreStream.new(store, manifest), + q, + por.authenticators, + SectorsPerBlock) + + test "Serialize Public Key": + var + spkMessage = spk.toMessage() + + check: + spk.signkey == spkMessage.fromMessage().tryGet().signkey + spk.key.blst_p2_is_equal(spkMessage.fromMessage().tryGet().key).bool + + test "Serialize TauZero": + var + tauZeroMessage = por.tau.t.toMessage() + tauZero = tauZeroMessage.fromMessage().tryGet() + + check: + por.tau.t.name == tauZero.name + por.tau.t.n == tauZero.n + + for i in 0.. 0): + + let + blk = bt.Block.new(chunk).tryGet() + + manifest.add(blk.cid) + if not (await store.putBlock(blk)): + raise newException(CatchableError, "Unable to store block " & $blk.cid) + + cid = manifest.cid.tryGet() + por = await PoR.init( + StoreStream.new(store, manifest), + ssk, spk, + BlockSize) + + porMsg = por.toMessage() + tags = blocks.mapIt( + Tag(idx: it, tag: porMsg.authenticators[it]) ) + + repoDir = path.parentDir / "stp" + createDir(repoDir) + stpstore = st.StpStore.init(repoDir) + + teardownAll: + removeDir(repoDir) + + test "Should store Storage Proofs": + check (await stpstore.store(por.toMessage(), cid)).isOk + check fileExists(stpstore.stpPath(cid) / "por") + + test "Should retrieve Storage Proofs": + check (await stpstore.retrieve(cid)).tryGet() == porMsg + + test "Should store tags": + check (await stpstore.store(tags, cid)).isOk + for t in tags: + check fileExists(stpstore.stpPath(cid) / $t.idx ) + + test "Should retrieve tags": + check (await stpstore.retrieve(cid, blocks)).tryGet() == tags diff --git a/tests/dagger/stores/testcachestore.nim b/tests/codex/stores/testcachestore.nim similarity index 98% rename from tests/dagger/stores/testcachestore.nim rename to tests/codex/stores/testcachestore.nim index 3bd128dd..2fcdf736 100644 --- a/tests/dagger/stores/testcachestore.nim +++ b/tests/codex/stores/testcachestore.nim @@ -5,8 +5,8 @@ import pkg/asynctest import pkg/libp2p import pkg/stew/byteutils import pkg/questionable/results -import pkg/dagger/stores/cachestore -import pkg/dagger/chunker +import pkg/codex/stores/cachestore +import pkg/codex/chunker import ../helpers diff --git a/tests/dagger/stores/testfsstore.nim b/tests/codex/stores/testfsstore.nim similarity index 93% rename from tests/dagger/stores/testfsstore.nim rename to tests/codex/stores/testfsstore.nim index 254c146d..070a602c 100644 --- a/tests/dagger/stores/testfsstore.nim +++ b/tests/codex/stores/testfsstore.nim @@ -8,10 +8,10 @@ import pkg/asynctest import pkg/libp2p import pkg/stew/byteutils -import pkg/dagger/stores/cachestore -import pkg/dagger/chunker -import pkg/dagger/stores -import pkg/dagger/blocktype as bt +import pkg/codex/stores/cachestore +import pkg/codex/chunker +import pkg/codex/stores +import pkg/codex/blocktype as bt import ../helpers diff --git a/tests/dagger/testasyncheapqueue.nim b/tests/codex/testasyncheapqueue.nim similarity index 98% rename from tests/dagger/testasyncheapqueue.nim rename to tests/codex/testasyncheapqueue.nim index 7837e2e0..d7fe3ec1 100644 --- a/tests/dagger/testasyncheapqueue.nim +++ b/tests/codex/testasyncheapqueue.nim @@ -2,8 +2,8 @@ import pkg/chronos import pkg/asynctest import pkg/stew/results -import pkg/dagger/utils/asyncheapqueue -import pkg/dagger/rng +import pkg/codex/utils/asyncheapqueue +import pkg/codex/rng type Task* = tuple[name: string, priority: int] diff --git a/tests/codex/testblockexchange.nim b/tests/codex/testblockexchange.nim new file mode 100644 index 00000000..bb7ace3d --- /dev/null +++ b/tests/codex/testblockexchange.nim @@ -0,0 +1,7 @@ +import ./blockexchange/testengine +import ./blockexchange/testnetwork +import ./blockexchange/testpeerctxstore +import ./blockexchange/testdiscovery +import ./blockexchange/testprotobuf + +{.warning[UnusedImport]: off.} diff --git a/tests/dagger/testchunking.nim b/tests/codex/testchunking.nim similarity index 98% rename from tests/dagger/testchunking.nim rename to tests/codex/testchunking.nim index 4d4668a8..626c332c 100644 --- a/tests/dagger/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -1,6 +1,6 @@ import pkg/asynctest import pkg/stew/byteutils -import pkg/dagger/chunker +import pkg/codex/chunker import pkg/chronicles import pkg/chronos import pkg/libp2p diff --git a/tests/dagger/testerasure.nim b/tests/codex/testerasure.nim similarity index 97% rename from tests/dagger/testerasure.nim rename to tests/codex/testerasure.nim index 70d46f19..23eab4e2 100644 --- a/tests/dagger/testerasure.nim +++ b/tests/codex/testerasure.nim @@ -6,11 +6,11 @@ import pkg/libp2p import pkg/questionable import pkg/questionable/results -import pkg/dagger/erasure -import pkg/dagger/manifest -import pkg/dagger/stores -import pkg/dagger/blocktype as bt -import pkg/dagger/rng +import pkg/codex/erasure +import pkg/codex/manifest +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/rng import ./helpers diff --git a/tests/dagger/testmanifest.nim b/tests/codex/testmanifest.nim similarity index 96% rename from tests/dagger/testmanifest.nim rename to tests/codex/testmanifest.nim index e78f1beb..7cd43b6e 100644 --- a/tests/dagger/testmanifest.nim +++ b/tests/codex/testmanifest.nim @@ -7,9 +7,9 @@ import pkg/asynctest import pkg/libp2p import pkg/stew/byteutils -import pkg/dagger/chunker -import pkg/dagger/blocktype as bt -import pkg/dagger/manifest +import pkg/codex/chunker +import pkg/codex/blocktype as bt +import pkg/codex/manifest import ./helpers diff --git a/tests/dagger/testnode.nim b/tests/codex/testnode.nim similarity index 90% rename from tests/dagger/testnode.nim rename to tests/codex/testnode.nim index b0038fdf..98cbff01 100644 --- a/tests/dagger/testnode.nim +++ b/tests/codex/testnode.nim @@ -10,14 +10,14 @@ import pkg/nitro import pkg/libp2p import pkg/libp2pdht/discv5/protocol as discv5 -import pkg/dagger/stores -import pkg/dagger/blockexchange -import pkg/dagger/chunker -import pkg/dagger/node -import pkg/dagger/manifest -import pkg/dagger/discovery -import pkg/dagger/blocktype as bt -import pkg/dagger/contracts +import pkg/codex/stores +import pkg/codex/blockexchange +import pkg/codex/chunker +import pkg/codex/node +import pkg/codex/manifest +import pkg/codex/discovery +import pkg/codex/blocktype as bt +import pkg/codex/contracts import ./helpers @@ -34,7 +34,7 @@ suite "Test Node": localStore: CacheStore engine: BlockExcEngine store: NetworkStore - node: DaggerNodeRef + node: CodexNodeRef blockDiscovery: Discovery peerStore: PeerCtxStore pendingBlocks: PendingBlocksManager @@ -55,7 +55,7 @@ suite "Test Node": engine = BlockExcEngine.new(localStore, wallet, network, discovery, peerStore, pendingBlocks) store = NetworkStore.new(engine, localStore) contracts = ContractInteractions.new() - node = DaggerNodeRef.new(switch, store, engine, nil, blockDiscovery, contracts) # TODO: pass `Erasure` + node = CodexNodeRef.new(switch, store, engine, nil, blockDiscovery, contracts) # TODO: pass `Erasure` await node.start() diff --git a/tests/dagger/testproving.nim b/tests/codex/testproving.nim similarity index 99% rename from tests/dagger/testproving.nim rename to tests/codex/testproving.nim index ae5ea01f..a2779efd 100644 --- a/tests/dagger/testproving.nim +++ b/tests/codex/testproving.nim @@ -1,6 +1,6 @@ import pkg/asynctest import pkg/chronos -import pkg/dagger/proving +import pkg/codex/proving import ./helpers/mockproofs import ./helpers/mockclock import ./examples diff --git a/tests/dagger/testpurchasing.nim b/tests/codex/testpurchasing.nim similarity index 99% rename from tests/dagger/testpurchasing.nim rename to tests/codex/testpurchasing.nim index 4b260ec5..e4456593 100644 --- a/tests/dagger/testpurchasing.nim +++ b/tests/codex/testpurchasing.nim @@ -2,7 +2,7 @@ import std/times import pkg/asynctest import pkg/chronos import pkg/stint -import pkg/dagger/purchasing +import pkg/codex/purchasing import ./helpers/mockmarket import ./helpers/mockclock import ./examples diff --git a/tests/dagger/testsales.nim b/tests/codex/testsales.nim similarity index 99% rename from tests/dagger/testsales.nim rename to tests/codex/testsales.nim index 72997a44..af8dd648 100644 --- a/tests/dagger/testsales.nim +++ b/tests/codex/testsales.nim @@ -1,7 +1,7 @@ import std/times import pkg/asynctest import pkg/chronos -import pkg/dagger/sales +import pkg/codex/sales import ./helpers/mockmarket import ./helpers/mockclock import ./examples diff --git a/tests/codex/teststorageproofs.nim b/tests/codex/teststorageproofs.nim new file mode 100644 index 00000000..e90a8c0b --- /dev/null +++ b/tests/codex/teststorageproofs.nim @@ -0,0 +1,5 @@ +import ./storageproofs/teststpstore +import ./storageproofs/testpor +import ./storageproofs/testnetwork + +{.warning[UnusedImport]: off.} diff --git a/tests/dagger/teststores.nim b/tests/codex/teststores.nim similarity index 100% rename from tests/dagger/teststores.nim rename to tests/codex/teststores.nim diff --git a/tests/dagger/teststorestream.nim b/tests/codex/teststorestream.nim similarity index 94% rename from tests/dagger/teststorestream.nim rename to tests/codex/teststorestream.nim index 3d457fee..0a82785e 100644 --- a/tests/dagger/teststorestream.nim +++ b/tests/codex/teststorestream.nim @@ -5,11 +5,10 @@ import pkg/questionable/results import ./helpers -import pkg/dagger/streams -import pkg/dagger/stores -import pkg/dagger/manifest -import pkg/dagger/rng -import pkg/dagger/blocktype as bt +import pkg/codex/streams +import pkg/codex/stores +import pkg/codex/manifest +import pkg/codex/blocktype as bt suite "StoreStream": var diff --git a/tests/contracts/examples.nim b/tests/contracts/examples.nim index 3dbbd888..2fc2b035 100644 --- a/tests/contracts/examples.nim +++ b/tests/contracts/examples.nim @@ -1,7 +1,7 @@ import std/times import pkg/stint import pkg/ethers -import dagger/contracts +import codex/contracts import ../examples export examples diff --git a/tests/contracts/testClock.nim b/tests/contracts/testClock.nim index 1b325af7..db97d70e 100644 --- a/tests/contracts/testClock.nim +++ b/tests/contracts/testClock.nim @@ -1,6 +1,6 @@ import std/times import pkg/chronos -import dagger/contracts/clock +import codex/contracts/clock import ../ethertest ethersuite "On-Chain Clock": diff --git a/tests/contracts/testCollateral.nim b/tests/contracts/testCollateral.nim index 523fdf36..6d2079b8 100644 --- a/tests/contracts/testCollateral.nim +++ b/tests/contracts/testCollateral.nim @@ -1,7 +1,7 @@ import pkg/chronos import pkg/stint -import dagger/contracts -import dagger/contracts/testtoken +import codex/contracts +import codex/contracts/testtoken import ../ethertest ethersuite "Collateral": diff --git a/tests/contracts/testContracts.nim b/tests/contracts/testContracts.nim index b7f345b1..58f6391a 100644 --- a/tests/contracts/testContracts.nim +++ b/tests/contracts/testContracts.nim @@ -1,9 +1,9 @@ import std/json import pkg/chronos import pkg/nimcrypto -import dagger/contracts -import dagger/contracts/testtoken -import dagger/por/timing/periods +import codex/contracts +import codex/contracts/testtoken +import codex/storageproofs import ../ethertest import ./examples import ./time diff --git a/tests/contracts/testInteractions.nim b/tests/contracts/testInteractions.nim index 84eed9f5..fbfbd0fe 100644 --- a/tests/contracts/testInteractions.nim +++ b/tests/contracts/testInteractions.nim @@ -1,5 +1,5 @@ import std/os -import dagger/contracts +import codex/contracts import ../ethertest import ./examples diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 7b32579a..4c3b7b52 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -1,6 +1,6 @@ import pkg/chronos -import dagger/contracts -import dagger/contracts/testtoken +import codex/contracts +import codex/contracts/testtoken import ../ethertest import ./examples import ./time diff --git a/tests/contracts/testProofs.nim b/tests/contracts/testProofs.nim index ade20bef..359f4ecc 100644 --- a/tests/contracts/testProofs.nim +++ b/tests/contracts/testProofs.nim @@ -1,4 +1,4 @@ -import dagger/contracts +import codex/contracts import ../ethertest import ./examples import ./time diff --git a/tests/dagger/helpers.nim b/tests/dagger/helpers.nim deleted file mode 100644 index 82ff7aaf..00000000 --- a/tests/dagger/helpers.nim +++ /dev/null @@ -1,25 +0,0 @@ -import pkg/libp2p -import pkg/libp2p/varint -import pkg/dagger/blocktype - -import ./helpers/nodeutils -import ./helpers/randomchunker - -export randomchunker, nodeutils - -# NOTE: The meaning of equality for blocks -# is changed here, because blocks are now `ref` -# types. This is only in tests!!! -func `==`*(a, b: Block): bool = - (a.cid == b.cid) and (a.data == b.data) - -proc lenPrefix*(msg: openArray[byte]): seq[byte] = - ## Write `msg` with a varint-encoded length prefix - ## - - let vbytes = PB.toBytes(msg.len().uint64) - var buf = newSeqUninitialized[byte](msg.len() + vbytes.len) - buf[0..