From ab413bdfcf0175d585f1d1a1c174926e2bfcfeca Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 19 Feb 2026 15:59:15 +1100 Subject: [PATCH] chore!: Finish renaming Codex to Logos Storage (#1399) --- .github/workflows/ci-reusable.yml | 2 +- .github/workflows/ci.yml | 2 +- .gitignore | 4 +- .nph.toml | 5 + Makefile | 18 +-- README.md | 25 ++- build.nims | 8 +- codex/contentids_exts.nim | 2 - codex/multicodec_exts.nim | 2 - codex/namespaces.nim | 25 --- config.nims | 10 +- docker/README.md | 6 +- library/README.md | 2 +- library/libstorage.nim | 6 +- library/storage_context.nim | 4 +- .../requests/node_debug_request.nim | 16 +- .../requests/node_download_request.nim | 24 +-- .../requests/node_info_request.nim | 16 +- .../requests/node_lifecycle_request.nim | 24 +-- .../requests/node_p2p_request.nim | 8 +- .../requests/node_storage_request.nim | 22 +-- .../requests/node_upload_request.nim | 20 +-- .../storage_thread_request.nim | 4 +- metrics/README.md | 6 +- metrics/prometheus.yml | 2 +- ...rd.json => storage-grafana-dashboard.json} | 0 nix/default.nix | 2 +- nix/service.nix | 2 +- openapi.yaml | 4 +- codex.nim => storage.nim | 36 ++--- codex.nimble => storage.nimble | 0 {codex => storage}/blockexchange.nim | 0 {codex => storage}/blockexchange/engine.nim | 0 .../blockexchange/engine/advertiser.nim | 8 +- .../blockexchange/engine/discovery.nim | 8 +- .../blockexchange/engine/engine.nim | 52 +++--- .../blockexchange/engine/pendingblocks.nim | 13 +- {codex => storage}/blockexchange/network.nim | 0 .../blockexchange/network/network.nim | 4 +- .../blockexchange/network/networkpeer.nim | 2 +- {codex => storage}/blockexchange/peers.nim | 0 .../blockexchange/peers/peercontext.nim | 0 .../blockexchange/peers/peerctxstore.nim | 2 +- .../blockexchange/protobuf/blockexc.nim | 0 .../blockexchange/protobuf/message.nim | 0 .../blockexchange/protobuf/message.proto | 0 .../protobuf/message.proto.license | 0 .../blockexchange/protobuf/presence.nim | 0 {codex => storage}/blocktype.nim | 4 +- {codex => storage}/chunker.nim | 0 {codex => storage}/clock.nim | 0 {codex => storage}/conf.nim | 24 +-- storage/contentids_exts.nim | 5 + {codex => storage}/discovery.nim | 2 +- {codex => storage}/errors.nim | 6 +- {codex => storage}/logutils.nim | 12 +- {codex => storage}/manifest.nim | 0 {codex => storage}/manifest/coders.nim | 0 {codex => storage}/manifest/manifest.nim | 2 +- {codex => storage}/merkletree.nim | 1 + {codex => storage}/merkletree/coders.nim | 0 {codex => storage}/merkletree/merkletree.nim | 5 +- storage/multicodec_exts.nim | 2 + {codex => storage}/multihash_exts.nim | 0 storage/namespaces.nim | 25 +++ {codex => storage}/nat.nim | 2 +- {codex => storage}/node.nim | 68 ++++---- {codex => storage}/rest/api.nim | 26 +-- {codex => storage}/rest/coders.nim | 0 {codex => storage}/rest/json.nim | 0 {codex => storage}/rng.nim | 0 codex/codex.nim => storage/storage.nim | 64 ++++---- .../storagetypes.nim | 8 +- {codex => storage}/stores.nim | 0 {codex => storage}/stores/blockstore.nim | 2 +- {codex => storage}/stores/cachestore.nim | 4 +- {codex => storage}/stores/keyutils.nim | 20 +-- {codex => storage}/stores/maintenance.nim | 2 +- {codex => storage}/stores/networkstore.nim | 2 +- {codex => storage}/stores/queryiterhelper.nim | 0 {codex => storage}/stores/repostore.nim | 0 .../stores/repostore/coders.nim | 0 .../stores/repostore/operations.nim | 16 +- {codex => storage}/stores/repostore/store.nim | 14 +- {codex => storage}/stores/repostore/types.nim | 2 +- {codex => storage}/stores/treehelper.nim | 0 {codex => storage}/streams.nim | 0 .../streams/asyncstreamwrapper.nim | 0 {codex => storage}/streams/seekablestream.nim | 2 +- {codex => storage}/streams/storestream.nim | 2 +- {codex => storage}/systemclock.nim | 0 {codex => storage}/units.nim | 0 {codex => storage}/utils.nim | 0 {codex => storage}/utils/addrutils.nim | 0 {codex => storage}/utils/arrayutils.nim | 0 {codex => storage}/utils/asyncheapqueue.nim | 0 {codex => storage}/utils/asynciter.nim | 0 .../utils/asyncstatemachine.nim | 0 {codex => storage}/utils/digest.nim | 0 {codex => storage}/utils/exceptions.nim | 0 {codex => storage}/utils/fileutils.nim | 0 {codex => storage}/utils/iter.nim | 0 {codex => storage}/utils/json.nim | 0 {codex => storage}/utils/keyutils.nim | 20 +-- {codex => storage}/utils/natutils.nim | 0 {codex => storage}/utils/options.nim | 0 {codex => storage}/utils/safeasynciter.nim | 0 {codex => storage}/utils/sharedbuf.nim | 0 {codex => storage}/utils/stintutils.nim | 0 {codex => storage}/utils/timer.nim | 0 {codex => storage}/utils/trackedfutures.nim | 0 {codex => storage}/utils/uniqueptr.nim | 0 tests/codex/testmerkletree.nim | 4 - tests/coverage.nim | 2 +- tests/examples.nim | 8 +- tests/helpers/trackers.nim | 2 +- .../1_minute/testblockexpiration.nim | 4 +- tests/integration/5_minutes/testrestapi.nim | 10 +- .../5_minutes/testrestapivalidation.nim | 12 +- .../integration/5_minutes/testupdownload.nim | 6 +- tests/integration/multinodes.nim | 76 ++++----- tests/integration/nodeconfigs.nim | 4 +- tests/integration/nodeprocess.nim | 10 +- .../{codexclient.nim => storageclient.nim} | 62 ++++---- .../{codexconfig.nim => storageconfig.nim} | 150 +++++++++--------- .../{codexprocess.nim => storageprocess.nim} | 70 ++++---- tests/integration/twonodes.nim | 18 +-- tests/integration/utils.nim | 4 +- tests/logging.nim | 2 +- .../blockexchange/discovery/testdiscovery.nim | 14 +- .../discovery/testdiscoveryengine.nim | 16 +- .../blockexchange/engine/testadvertiser.nim | 12 +- .../blockexchange/engine/testblockexc.nim | 10 +- .../blockexchange/engine/testengine.nim | 14 +- .../blockexchange/protobuf/testpresence.nim | 2 +- .../blockexchange/testdiscovery.nim | 0 .../blockexchange/testengine.nim | 0 .../blockexchange/testnetwork.nim | 8 +- .../blockexchange/testpeerctxstore.nim | 6 +- .../blockexchange/testpendingblocks.nim | 4 +- .../blockexchange/testprotobuf.nim | 0 tests/{codex => storage}/examples.nim | 10 +- tests/{codex => storage}/helpers.nim | 14 +- tests/{codex => storage}/helpers/always.nim | 0 .../helpers/datasetutils.nim | 8 +- .../helpers/mockchunker.nim | 2 +- .../{codex => storage}/helpers/mockclock.nim | 2 +- .../helpers/mockdiscovery.nim | 2 +- .../helpers/mockrepostore.nim | 6 +- .../{codex => storage}/helpers/mocktimer.nim | 2 +- .../{codex => storage}/helpers/nodeutils.nim | 30 ++-- .../helpers/randomchunker.nim | 4 +- .../{codex => storage}/merkletree/helpers.nim | 2 +- .../merkletree/teststoragecoders.nim} | 2 +- .../merkletree/teststoragetree.nim} | 9 +- tests/{codex => storage}/node/helpers.nim | 10 +- tests/{codex => storage}/node/testnode.nim | 26 +-- tests/{codex => storage}/slots/helpers.nim | 14 +- .../stores/commonstoretests.nim | 10 +- .../stores/repostore/testcoders.nim | 6 +- .../stores/testcachestore.nim | 4 +- .../stores/testkeyutils.nim | 18 +-- .../stores/testmaintenance.nim | 8 +- .../stores/testqueryiterhelper.nim | 4 +- .../stores/testrepostore.nim | 16 +- .../{codex => storage}/testasyncheapqueue.nim | 4 +- .../testasyncstreamwrapper.nim | 2 +- .../{codex => storage}/testblockexchange.nim | 0 tests/{codex => storage}/testblocktype.nim | 2 +- tests/{codex => storage}/testchunking.nim | 4 +- tests/{codex => storage}/testclock.nim | 2 +- tests/{codex => storage}/testlogutils.nim | 10 +- tests/{codex => storage}/testmanifest.nim | 8 +- tests/storage/testmerkletree.nim | 4 + tests/{codex => storage}/testnat.nim | 6 +- tests/{codex => storage}/testnode.nim | 0 tests/{codex => storage}/testrng.nim | 2 +- tests/{codex => storage}/teststores.nim | 0 tests/{codex => storage}/teststorestream.nim | 2 +- tests/{codex => storage}/testsystemclock.nim | 2 +- tests/{codex => storage}/testutils.nim | 0 .../utils/testasynciter.nim | 2 +- .../utils/testasyncstatemachine.nim | 2 +- tests/{codex => storage}/utils/testiter.nim | 2 +- .../{codex => storage}/utils/testkeyutils.nim | 4 +- .../{codex => storage}/utils/testoptions.nim | 2 +- .../utils/testsafeasynciter.nim | 4 +- tests/{codex => storage}/utils/testtimer.nim | 2 +- .../utils/testtrackedfutures.nim | 2 +- tests/{codex => storage}/utils/testutils.nim | 2 +- tests/testIntegration.nim | 4 +- tests/{testCodex.nim => testStorage.nim} | 2 +- 192 files changed, 787 insertions(+), 766 deletions(-) create mode 100644 .nph.toml delete mode 100644 codex/contentids_exts.nim delete mode 100644 codex/multicodec_exts.nim delete mode 100644 codex/namespaces.nim rename metrics/{codex-grafana-dashboard.json => storage-grafana-dashboard.json} (100%) rename codex.nim => storage.nim (87%) rename codex.nimble => storage.nimble (100%) rename {codex => storage}/blockexchange.nim (100%) rename {codex => storage}/blockexchange/engine.nim (100%) rename {codex => storage}/blockexchange/engine/advertiser.nim (95%) rename {codex => storage}/blockexchange/engine/discovery.nim (96%) rename {codex => storage}/blockexchange/engine/engine.nim (95%) rename {codex => storage}/blockexchange/engine/pendingblocks.nim (94%) rename {codex => storage}/blockexchange/network.nim (100%) rename {codex => storage}/blockexchange/network/network.nim (99%) rename {codex => storage}/blockexchange/network/networkpeer.nim (98%) rename {codex => storage}/blockexchange/peers.nim (100%) rename {codex => storage}/blockexchange/peers/peercontext.nim (100%) rename {codex => storage}/blockexchange/peers/peerctxstore.nim (98%) rename {codex => storage}/blockexchange/protobuf/blockexc.nim (100%) rename {codex => storage}/blockexchange/protobuf/message.nim (100%) rename {codex => storage}/blockexchange/protobuf/message.proto (100%) rename {codex => storage}/blockexchange/protobuf/message.proto.license (100%) rename {codex => storage}/blockexchange/protobuf/presence.nim (100%) rename {codex => storage}/blocktype.nim (98%) rename {codex => storage}/chunker.nim (100%) rename {codex => storage}/clock.nim (100%) rename {codex => storage}/conf.nim (96%) create mode 100644 storage/contentids_exts.nim rename {codex => storage}/discovery.nim (99%) rename {codex => storage}/errors.nim (93%) rename {codex => storage}/logutils.nim (96%) rename {codex => storage}/manifest.nim (100%) rename {codex => storage}/manifest/coders.nim (100%) rename {codex => storage}/manifest/manifest.nim (98%) rename {codex => storage}/merkletree.nim (75%) rename {codex => storage}/merkletree/coders.nim (100%) rename {codex => storage}/merkletree/merkletree.nim (99%) create mode 100644 storage/multicodec_exts.nim rename {codex => storage}/multihash_exts.nim (100%) create mode 100644 storage/namespaces.nim rename {codex => storage}/nat.nim (99%) rename {codex => storage}/node.nim (90%) rename {codex => storage}/rest/api.nim (96%) rename {codex => storage}/rest/coders.nim (100%) rename {codex => storage}/rest/json.nim (100%) rename {codex => storage}/rng.nim (100%) rename codex/codex.nim => storage/storage.nim (80%) rename codex/codextypes.nim => storage/storagetypes.nim (90%) rename {codex => storage}/stores.nim (100%) rename {codex => storage}/stores/blockstore.nim (99%) rename {codex => storage}/stores/cachestore.nim (99%) rename {codex => storage}/stores/keyutils.nim (65%) rename {codex => storage}/stores/maintenance.nim (99%) rename {codex => storage}/stores/networkstore.nim (99%) rename {codex => storage}/stores/queryiterhelper.nim (100%) rename {codex => storage}/stores/repostore.nim (100%) rename {codex => storage}/stores/repostore/coders.nim (100%) rename {codex => storage}/stores/repostore/operations.nim (94%) rename {codex => storage}/stores/repostore/store.nim (97%) rename {codex => storage}/stores/repostore/types.nim (98%) rename {codex => storage}/stores/treehelper.nim (100%) rename {codex => storage}/streams.nim (100%) rename {codex => storage}/streams/asyncstreamwrapper.nim (100%) rename {codex => storage}/streams/seekablestream.nim (94%) rename {codex => storage}/streams/storestream.nim (99%) rename {codex => storage}/systemclock.nim (100%) rename {codex => storage}/units.nim (100%) rename {codex => storage}/utils.nim (100%) rename {codex => storage}/utils/addrutils.nim (100%) rename {codex => storage}/utils/arrayutils.nim (100%) rename {codex => storage}/utils/asyncheapqueue.nim (100%) rename {codex => storage}/utils/asynciter.nim (100%) rename {codex => storage}/utils/asyncstatemachine.nim (100%) rename {codex => storage}/utils/digest.nim (100%) rename {codex => storage}/utils/exceptions.nim (100%) rename {codex => storage}/utils/fileutils.nim (100%) rename {codex => storage}/utils/iter.nim (100%) rename {codex => storage}/utils/json.nim (100%) rename {codex => storage}/utils/keyutils.nim (57%) rename {codex => storage}/utils/natutils.nim (100%) rename {codex => storage}/utils/options.nim (100%) rename {codex => storage}/utils/safeasynciter.nim (100%) rename {codex => storage}/utils/sharedbuf.nim (100%) rename {codex => storage}/utils/stintutils.nim (100%) rename {codex => storage}/utils/timer.nim (100%) rename {codex => storage}/utils/trackedfutures.nim (100%) rename {codex => storage}/utils/uniqueptr.nim (100%) delete mode 100644 tests/codex/testmerkletree.nim rename tests/integration/{codexclient.nim => storageclient.nim} (81%) rename tests/integration/{codexconfig.nim => storageconfig.nim} (56%) rename tests/integration/{codexprocess.nim => storageprocess.nim} (53%) rename tests/{codex => storage}/blockexchange/discovery/testdiscovery.nim (98%) rename tests/{codex => storage}/blockexchange/discovery/testdiscoveryengine.nim (95%) rename tests/{codex => storage}/blockexchange/engine/testadvertiser.nim (93%) rename tests/{codex => storage}/blockexchange/engine/testblockexc.nim (97%) rename tests/{codex => storage}/blockexchange/engine/testengine.nim (98%) rename tests/{codex => storage}/blockexchange/protobuf/testpresence.nim (94%) rename tests/{codex => storage}/blockexchange/testdiscovery.nim (100%) rename tests/{codex => storage}/blockexchange/testengine.nim (100%) rename tests/{codex => storage}/blockexchange/testnetwork.nim (98%) rename tests/{codex => storage}/blockexchange/testpeerctxstore.nim (94%) rename tests/{codex => storage}/blockexchange/testpendingblocks.nim (97%) rename tests/{codex => storage}/blockexchange/testprotobuf.nim (100%) rename tests/{codex => storage}/examples.nim (88%) rename tests/{codex => storage}/helpers.nim (93%) rename tests/{codex => storage}/helpers/always.nim (100%) rename tests/{codex => storage}/helpers/datasetutils.nim (90%) rename tests/{codex => storage}/helpers/mockchunker.nim (96%) rename tests/{codex => storage}/helpers/mockclock.nim (98%) rename tests/{codex => storage}/helpers/mockdiscovery.nim (99%) rename tests/{codex => storage}/helpers/mockrepostore.nim (93%) rename tests/{codex => storage}/helpers/mocktimer.nim (97%) rename tests/{codex => storage}/helpers/nodeutils.nim (94%) rename tests/{codex => storage}/helpers/randomchunker.nim (94%) rename tests/{codex => storage}/merkletree/helpers.nim (91%) rename tests/{codex/merkletree/testcodexcoders.nim => storage/merkletree/teststoragecoders.nim} (97%) rename tests/{codex/merkletree/testcodextree.nim => storage/merkletree/teststoragetree.nim} (94%) rename tests/{codex => storage}/node/helpers.nim (95%) rename tests/{codex => storage}/node/testnode.nim (93%) rename tests/{codex => storage}/slots/helpers.nim (91%) rename tests/{codex => storage}/stores/commonstoretests.nim (96%) rename tests/{codex => storage}/stores/repostore/testcoders.nim (93%) rename tests/{codex => storage}/stores/testcachestore.nim (97%) rename tests/{codex => storage}/stores/testkeyutils.nim (87%) rename tests/{codex => storage}/stores/testmaintenance.nim (97%) rename tests/{codex => storage}/stores/testqueryiterhelper.nim (94%) rename tests/{codex => storage}/stores/testrepostore.nim (98%) rename tests/{codex => storage}/testasyncheapqueue.nim (98%) rename tests/{codex => storage}/testasyncstreamwrapper.nim (98%) rename tests/{codex => storage}/testblockexchange.nim (100%) rename tests/{codex => storage}/testblocktype.nim (97%) rename tests/{codex => storage}/testchunking.nim (98%) rename tests/{codex => storage}/testclock.nim (95%) rename tests/{codex => storage}/testlogutils.nim (97%) rename tests/{codex => storage}/testmanifest.nim (85%) create mode 100644 tests/storage/testmerkletree.nim rename tests/{codex => storage}/testnat.nim (94%) rename tests/{codex => storage}/testnode.nim (100%) rename tests/{codex => storage}/testrng.nim (97%) rename tests/{codex => storage}/teststores.nim (100%) rename tests/{codex => storage}/teststorestream.nim (97%) rename tests/{codex => storage}/testsystemclock.nim (88%) rename tests/{codex => storage}/testutils.nim (100%) rename tests/{codex => storage}/utils/testasynciter.nim (99%) rename tests/{codex => storage}/utils/testasyncstatemachine.nim (98%) rename tests/{codex => storage}/utils/testiter.nim (98%) rename tests/{codex => storage}/utils/testkeyutils.nim (91%) rename tests/{codex => storage}/utils/testoptions.nim (97%) rename tests/{codex => storage}/utils/testsafeasynciter.nim (99%) rename tests/{codex => storage}/utils/testtimer.nim (98%) rename tests/{codex => storage}/utils/testtrackedfutures.nim (98%) rename tests/{codex => storage}/utils/testutils.nim (97%) rename tests/{testCodex.nim => testStorage.nim} (52%) diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index 7121786a..83f1dcac 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -52,7 +52,7 @@ jobs: - name: Integration tests if: matrix.tests == 'integration' || matrix.tests == 'all' env: - CODEX_INTEGRATION_TEST_INCLUDES: ${{ matrix.includes }} + STORAGE_INTEGRATION_TEST_INCLUDES: ${{ matrix.includes }} run: make -j${ncpu} DEBUG=${{ runner.debug }} testIntegration - name: Upload integration tests log files diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e64d05b3..2860b58d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: uses: arnetheduck/nph-action@v1 with: version: latest - options: "codex/ tests/" + options: "storage/ tests/" fail: true suggest: true diff --git a/.gitignore b/.gitignore index 78030176..93b39e42 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,7 @@ nimble.paths .env .update.timestamp -codex.nims +storage.nims nimbus-build-system.paths docker/hostdatadir docker/prometheus-data @@ -50,3 +50,5 @@ data/ tests/cbindings/data-dir tests/cbindings/downloaded_hello.txt + +nimbledeps diff --git a/.nph.toml b/.nph.toml new file mode 100644 index 00000000..ac1b9524 --- /dev/null +++ b/.nph.toml @@ -0,0 +1,5 @@ +# Add to default exclusions (recommended - doesn't lose the defaults) +extend-exclude = [ +# "tests/fixtures", + "vendor", +] \ No newline at end of file diff --git a/Makefile b/Makefile index 80cd5e50..2046e2bf 100644 --- a/Makefile +++ b/Makefile @@ -181,11 +181,11 @@ endif coverage: $(MAKE) NIMFLAGS="$(NIMFLAGS) --lineDir:on --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage" test - cd nimcache/release/testCodex && rm -f *.c + cd nimcache/release/testStorage && rm -f *.c mkdir -p coverage - lcov --capture --keep-going --directory nimcache/release/testCodex --output-file coverage/coverage.info - shopt -s globstar && ls $$(pwd)/codex/{*,**/*}.nim - shopt -s globstar && lcov --extract coverage/coverage.info --keep-going $$(pwd)/codex/{*,**/*}.nim --output-file coverage/coverage.f.info + lcov --capture --keep-going --directory nimcache/release/testStorage --output-file coverage/coverage.info + shopt -s globstar && ls $$(pwd)/storage/{*,**/*}.nim + shopt -s globstar && lcov --extract coverage/coverage.info --keep-going $$(pwd)/storage/{*,**/*}.nim --output-file coverage/coverage.f.info echo -e $(BUILD_MSG) "coverage/report/index.html" genhtml coverage/coverage.f.info --keep-going --output-directory coverage/report @@ -237,7 +237,7 @@ nph/%: build-nph format: $(NPH) *.nim - $(NPH) codex/ + $(NPH) storage/ $(NPH) tests/ $(NPH) library/ @@ -267,15 +267,15 @@ libstorage: ifeq ($(STATIC), 1) echo -e $(BUILD_MSG) "build/$@.a" && \ - $(ENV_SCRIPT) nim libstorageStatic $(NIM_PARAMS) $(LIBSTORAGE_PARAMS) codex.nims + $(ENV_SCRIPT) nim libstorageStatic $(NIM_PARAMS) $(LIBSTORAGE_PARAMS) storage.nims else ifeq ($(detected_OS),Windows) echo -e $(BUILD_MSG) "build/$@.dll" && \ - $(ENV_SCRIPT) nim libstorageDynamic $(NIM_PARAMS) $(LIBSTORAGE_PARAMS) codex.nims + $(ENV_SCRIPT) nim libstorageDynamic $(NIM_PARAMS) $(LIBSTORAGE_PARAMS) storage.nims else ifeq ($(detected_OS),macOS) echo -e $(BUILD_MSG) "build/$@.dylib" && \ - $(ENV_SCRIPT) nim libstorageDynamic $(NIM_PARAMS) $(LIBSTORAGE_PARAMS) codex.nims + $(ENV_SCRIPT) nim libstorageDynamic $(NIM_PARAMS) $(LIBSTORAGE_PARAMS) storage.nims else echo -e $(BUILD_MSG) "build/$@.so" && \ - $(ENV_SCRIPT) nim libstorageDynamic $(NIM_PARAMS) $(LIBSTORAGE_PARAMS) codex.nims + $(ENV_SCRIPT) nim libstorageDynamic $(NIM_PARAMS) $(LIBSTORAGE_PARAMS) storage.nims endif endif # "variables.mk" was not included diff --git a/README.md b/README.md index 9575e2eb..280a22b9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Logos Storage Decentralized Engine +# Logos Storage Filesharing Client -> The Logos Storage project aims to create a decentralized engine that allows persisting data in p2p networks. +> The Logos Storage project aims to create a filesharing client that allows sharing data privately in p2p networks. > WARNING: This project is under active development and is considered pre-alpha. @@ -11,17 +11,17 @@ [![Docker](https://github.com/logos-storage/logos-storage-nim/actions/workflows/docker.yml/badge.svg?branch=master)](https://github.com/logos-storage/logos-storage-nim/actions/workflows/docker.yml?query=branch%3Amaster) [![Codecov](https://codecov.io/gh/logos-storage/logos-storage-nim/branch/master/graph/badge.svg?token=XFmCyPSNzW)](https://codecov.io/gh/logos-storage/logos-storage-nim) [![Discord](https://img.shields.io/discord/895609329053474826)](https://discord.gg/CaJTh24ddQ) -![Docker Pulls](https://img.shields.io/docker/pulls/codexstorage/nim-codex) +![Docker Pulls](https://img.shields.io/docker/pulls/logosstorage/logos-storage-nim) ## Build and Run -For detailed instructions on preparing to build logos-storagenim see [*Build Logos Storage*](https://docs.codex.storage/learn/build). - To build the project, clone it and run: ```bash make update && make +# Tip: use -j{ncpu} to for parallel execution, eg: +# make -j12 update && make -j12 ``` The executable will be placed under the `build` directory under the project root. @@ -41,12 +41,7 @@ It is possible to configure a Logos Storage node in several ways: The order of priority is the same as above: CLI options --> Environment variables --> Configuration file. -Please check [documentation](https://docs.codex.storage/learn/run#configuration) for more information. - -## Guides - -To get acquainted with Logos Storage, consider: -* running the simple [Logos Storage Two-Client Test](https://docs.codex.storage/learn/local-two-client-test) for a start, and; +Please check `build/storage --help` for more information. ## API @@ -54,8 +49,9 @@ The client exposes a REST API that can be used to interact with the clients. Ove ## Bindings -Logos Storage provides a C API that can be wrapped by other languages. The bindings is located in the `library` folder. -Currently, only a Go binding is included. +Logos Storage provides a C API that can be wrapped by other languages. The C API bindings are located in the `library` folder. + +Currently, only Go bindings are provided in this repo. However, Rust bindings for Logos Storage can be found at https://github.com/nipsysdev/storage-rust-bindings. ### Build the C library @@ -71,7 +67,8 @@ See https://github.com/logos-storage/logos-storage-go-bindings-example. ### Static vs Dynamic build -By default, Logos Storage builds a dynamic library (`libstorage.so`), which you can load at runtime. +By default, Logos Storage builds a dynamic library (`libstorage.so`/`libstorage.dylib`/`libstroage.dll`), which you can load at runtime. + If you prefer a static library (`libstorage.a`), set the `STATIC` flag: ```bash diff --git a/build.nims b/build.nims index 612c5725..b74a931f 100644 --- a/build.nims +++ b/build.nims @@ -62,15 +62,15 @@ proc test(name: string, outName = name, srcDir = "tests/", params = "", lang = " exec "build/" & outName task storage, "build logos storage binary": - buildBinary "codex", + buildBinary "storage", outname = "storage", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" task testStorage, "Build & run Logos Storage tests": - test "testCodex", outName = "testStorage" + test "testStorage", outName = "testStorage" task testIntegration, "Run integration tests": - buildBinary "codex", + buildBinary "storage", outName = "storage", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" test "testIntegration" @@ -110,7 +110,7 @@ task coverage, "generates code coverage report": echo " *****************************************************************" var nimSrcs = " " - for f in walkDirRec("codex", {pcFile}): + for f in walkDirRec("storage", {pcFile}): if f.endswith(".nim"): nimSrcs.add " " & f.absolutePath.quoteShell() diff --git a/codex/contentids_exts.nim b/codex/contentids_exts.nim deleted file mode 100644 index 28bcf2d1..00000000 --- a/codex/contentids_exts.nim +++ /dev/null @@ -1,2 +0,0 @@ -const ContentIdsExts = - [multiCodec("codex-root"), multiCodec("codex-manifest"), multiCodec("codex-block")] diff --git a/codex/multicodec_exts.nim b/codex/multicodec_exts.nim deleted file mode 100644 index ec629476..00000000 --- a/codex/multicodec_exts.nim +++ /dev/null @@ -1,2 +0,0 @@ -const CodecExts = - [("codex-manifest", 0xCD01), ("codex-block", 0xCD02), ("codex-root", 0xCD03)] diff --git a/codex/namespaces.nim b/codex/namespaces.nim deleted file mode 100644 index 169be184..00000000 --- a/codex/namespaces.nim +++ /dev/null @@ -1,25 +0,0 @@ -## Logos Storage -## Copyright (c) 2022 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -const - # Namespaces - CodexMetaNamespace* = "meta" # meta info stored here - CodexRepoNamespace* = "repo" # repository namespace, blocks and manifests are subkeys - CodexBlockTotalNamespace* = CodexMetaNamespace & "/total" - # number of blocks in the repo - CodexBlocksNamespace* = CodexRepoNamespace & "/blocks" # blocks namespace - CodexManifestNamespace* = CodexRepoNamespace & "/manifests" # manifest namespace - CodexBlocksTtlNamespace* = # Cid TTL - CodexMetaNamespace & "/ttl" - CodexBlockProofNamespace* = # Cid and Proof - CodexMetaNamespace & "/proof" - CodexDhtNamespace* = "dht" # Dht namespace - CodexDhtProvidersNamespace* = # Dht providers namespace - CodexDhtNamespace & "/providers" - CodexQuotaNamespace* = CodexMetaNamespace & "/quota" # quota's namespace diff --git a/config.nims b/config.nims index 4b24ba54..dd912d06 100644 --- a/config.nims +++ b/config.nims @@ -93,11 +93,11 @@ else: --warningAsError: "ProveField:on" --define: - "libp2p_multicodec_exts:../../../codex/multicodec_exts.nim" + "libp2p_multicodec_exts:../../../storage/multicodec_exts.nim" --define: - "libp2p_multihash_exts:../../../codex/multihash_exts.nim" + "libp2p_multihash_exts:../../../storage/multihash_exts.nim" --define: - "libp2p_contentids_exts:../../../codex/contentids_exts.nim" + "libp2p_contentids_exts:../../../storage/contentids_exts.nim" when (NimMajor, NimMinor) >= (1, 4): --warning: @@ -142,8 +142,8 @@ switch("warning", "LockLevel:off") switch("define", "libp2p_pki_schemes=secp256k1") #TODO this infects everything in this folder, ideally it would only -# apply to codex.nim, but since codex.nims is used for other purpose -# we can't use it. And codex.cfg doesn't work +# apply to storage.nim, but since storage.nims is used for other purpose +# we can't use it. And storage.cfg doesn't work switch("define", "chronicles_sinks=textlines[dynamic],json[dynamic],textlines[dynamic]") # Workaround for assembler incompatibility between constantine and secp256k1 diff --git a/docker/README.md b/docker/README.md index 76af6cd8..2c337bed 100644 --- a/docker/README.md +++ b/docker/README.md @@ -37,7 +37,7 @@ ## Environment variables - We can configure Codex using [Environment variables](../README#environment-variables) and [docker-compose.yaml](docker-compose.yaml) file can be useful as an example. + We can configure Logos Storage using [Environment variables](../README#environment-variables) and [docker-compose.yaml](docker-compose.yaml) file can be useful as an example. We also added a temporary environment variable `NAT_IP_AUTO` to the entrypoint which is set as `false` for releases and ` true` for regular builds. That approach is useful for Dist-Tests. ```shell @@ -51,7 +51,7 @@ 2. The docker image can then be minified using [slim](https://github.com/slimtoolkit/slim). Install slim on your path and then run: ```shell slim # brings up interactive prompt - >>> build --target status-im/codexsetup --http-probe-off true + >>> build --target status-im/logosstoragesetup --http-probe-off true ``` - 3. This should output an image with name `status-im/codexsetup.slim` + 3. This should output an image with name `status-im/logosstoragesetup.slim` 4. We can then bring up the image using `docker-compose up -d`. diff --git a/library/README.md b/library/README.md index b9fd0567..4e78808a 100644 --- a/library/README.md +++ b/library/README.md @@ -16,7 +16,7 @@ sequenceDiagram participant C as C API (libstorage.h) participant Ctx as StorageContext participant Thr as Worker Thread - participant Eng as CodexServer + participant Eng as StorageServer App->>Go: Start Go->>C: storage_start_node diff --git a/library/libstorage.nim b/library/libstorage.nim index 46f0749c..e41e52dd 100644 --- a/library/libstorage.nim +++ b/library/libstorage.nim @@ -39,7 +39,7 @@ import ./storage_thread_requests/requests/node_download_request import ./storage_thread_requests/requests/node_storage_request import ./ffi_types -from ../codex/conf import codexVersion +from ../storage/conf import storageVersion logScope: topics = "libstorage" @@ -125,12 +125,12 @@ proc storage_new( proc storage_version(ctx: ptr StorageContext): ptr cchar {.dynlib, exportc.} = initializeLibrary() - return asNewCString(conf.codexVersion) + return asNewCString(conf.storageVersion) proc storage_revision(ctx: ptr StorageContext): ptr cchar {.dynlib, exportc.} = initializeLibrary() - return asNewCString(conf.codexRevision) + return asNewCString(conf.storageVersion) proc storage_repo( ctx: ptr StorageContext, callback: StorageCallback, userData: pointer diff --git a/library/storage_context.nim b/library/storage_context.nim index 260b6a98..cbb140af 100644 --- a/library/storage_context.nim +++ b/library/storage_context.nim @@ -16,7 +16,7 @@ import taskpools/channels_spsc_single import ./ffi_types import ./storage_thread_requests/[storage_thread_request] -from ../codex/codex import CodexServer +from ../storage/storage import StorageServer logScope: topics = "libstorage" @@ -124,7 +124,7 @@ proc sendRequestToStorageThread*( ok() proc runStorage(ctx: ptr StorageContext) {.async: (raises: []).} = - var storage: CodexServer + var storage: StorageServer while true: try: diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index c8dab2c4..8bf3106c 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -10,11 +10,11 @@ import chronos import chronicles import codexdht/discv5/spr import ../../alloc -import ../../../codex/conf -import ../../../codex/rest/json -import ../../../codex/node +import ../../../storage/conf +import ../../../storage/rest/json +import ../../../storage/node -from ../../../codex/codex import CodexServer, node +from ../../../storage/storage import StorageServer, node logScope: topics = "libstorage libstoragedebug" @@ -47,7 +47,7 @@ proc destroyShared(self: ptr NodeDebugRequest) = deallocShared(self) proc getDebug( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = let node = storage[].node let table = RestRoutingTable.init(node.discovery.protocol.routingTable) @@ -64,7 +64,7 @@ proc getDebug( return ok($json) proc getPeer( - storage: ptr CodexServer, peerId: cstring + storage: ptr StorageServer, peerId: cstring ): Future[Result[string, string]] {.async: (raises: []).} = when storage_enable_api_debug_peers: let node = storage[].node @@ -88,7 +88,7 @@ proc getPeer( return err("Failed to get peer: peer debug API is disabled") proc updateLogLevel( - storage: ptr CodexServer, logLevel: cstring + storage: ptr StorageServer, logLevel: cstring ): Future[Result[string, string]] {.async: (raises: []).} = try: {.gcsafe.}: @@ -99,7 +99,7 @@ proc updateLogLevel( return ok("") proc process*( - self: ptr NodeDebugRequest, storage: ptr CodexServer + self: ptr NodeDebugRequest, storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_download_request.nim b/library/storage_thread_requests/requests/node_download_request.nim index 1d8ef59c..44b38482 100644 --- a/library/storage_thread_requests/requests/node_download_request.nim +++ b/library/storage_thread_requests/requests/node_download_request.nim @@ -21,12 +21,12 @@ import chronicles import libp2p/stream/[lpstream] import serde/json as serde import ../../alloc -import ../../../codex/units -import ../../../codex/codextypes +import ../../../storage/units +import ../../../storage/storagetypes -from ../../../codex/codex import CodexServer, node -from ../../../codex/node import retrieve, fetchManifest -from ../../../codex/rest/json import `%`, RestContent +from ../../../storage/storage import StorageServer, node +from ../../../storage/node import retrieve, fetchManifest +from ../../../storage/rest/json import `%`, RestContent from libp2p import Cid, init, `$` logScope: @@ -80,7 +80,7 @@ proc destroyShared(self: ptr NodeDownloadRequest) = deallocShared(self) proc init( - storage: ptr CodexServer, cCid: cstring = "", chunkSize: csize_t = 0, local: bool + storage: ptr StorageServer, cCid: cstring = "", chunkSize: csize_t = 0, local: bool ): Future[Result[string, string]] {.async: (raises: []).} = ## Init a new session to download the file identified by cid. ## @@ -114,7 +114,7 @@ proc init( return ok("") proc chunk( - storage: ptr CodexServer, cCid: cstring = "", onChunk: OnChunkHandler + storage: ptr StorageServer, cCid: cstring = "", onChunk: OnChunkHandler ): Future[Result[string, string]] {.async: (raises: []).} = ## Download the next chunk of the file identified by cid. ## The chunk is passed to the onChunk handler. @@ -164,7 +164,7 @@ proc chunk( return ok("") proc streamData( - storage: ptr CodexServer, + storage: ptr StorageServer, stream: LPStream, onChunk: OnChunkHandler, chunkSize: csize_t, @@ -207,7 +207,7 @@ proc streamData( return ok("") proc stream( - storage: ptr CodexServer, + storage: ptr StorageServer, cCid: cstring, chunkSize: csize_t, local: bool, @@ -251,7 +251,7 @@ proc stream( return ok("") proc cancel( - storage: ptr CodexServer, cCid: cstring + storage: ptr StorageServer, cCid: cstring ): Future[Result[string, string]] {.raises: [], async: (raises: []).} = ## Cancel the download session identified by cid. ## This operation is not supported when using the stream mode, @@ -279,7 +279,7 @@ proc cancel( return ok("") proc manifest( - storage: ptr CodexServer, cCid: cstring + storage: ptr StorageServer, cCid: cstring ): Future[Result[string, string]] {.raises: [], async: (raises: []).} = let cid = Cid.init($cCid) if cid.isErr: @@ -296,7 +296,7 @@ proc manifest( return err("Failed to fetch manifest: download cancelled.") proc process*( - self: ptr NodeDownloadRequest, storage: ptr CodexServer, onChunk: OnChunkHandler + self: ptr NodeDownloadRequest, storage: ptr StorageServer, onChunk: OnChunkHandler ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_info_request.nim b/library/storage_thread_requests/requests/node_info_request.nim index cbf364e8..7e755a3a 100644 --- a/library/storage_thread_requests/requests/node_info_request.nim +++ b/library/storage_thread_requests/requests/node_info_request.nim @@ -5,11 +5,11 @@ import chronos import chronicles import confutils import codexdht/discv5/spr -import ../../../codex/conf -import ../../../codex/rest/json -import ../../../codex/node +import ../../../storage/conf +import ../../../storage/rest/json +import ../../../storage/node -from ../../../codex/codex import CodexServer, config, node +from ../../../storage/storage import StorageServer, config, node logScope: topics = "libstorage libstorageinfo" @@ -31,12 +31,12 @@ proc destroyShared(self: ptr NodeInfoRequest) = deallocShared(self) proc getRepo( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = return ok($(storage[].config.dataDir)) proc getSpr( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = let spr = storage[].node.discovery.dhtRecord if spr.isNone: @@ -45,12 +45,12 @@ proc getSpr( return ok(spr.get.toURI) proc getPeerId( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = return ok($storage[].node.switch.peerInfo.peerId) proc process*( - self: ptr NodeInfoRequest, storage: ptr CodexServer + self: ptr NodeInfoRequest, storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_lifecycle_request.nim b/library/storage_thread_requests/requests/node_lifecycle_request.nim index 491847aa..31da9416 100644 --- a/library/storage_thread_requests/requests/node_lifecycle_request.nim +++ b/library/storage_thread_requests/requests/node_lifecycle_request.nim @@ -17,12 +17,12 @@ import libp2p import json_serialization import json_serialization/std/[options, net] import ../../alloc -import ../../../codex/conf -import ../../../codex/utils -import ../../../codex/utils/[keyutils, fileutils] -import ../../../codex/units +import ../../../storage/conf +import ../../../storage/utils +import ../../../storage/utils/[keyutils, fileutils] +import ../../../storage/units -from ../../../codex/codex import CodexServer, new, start, stop, close +from ../../../storage/storage import StorageServer, new, start, stop, close logScope: topics = "libstorage libstoragelifecycle" @@ -89,16 +89,16 @@ proc destroyShared(self: ptr NodeLifecycleRequest) = proc createStorage( configJson: cstring -): Future[Result[CodexServer, string]] {.async: (raises: []).} = - var conf: CodexConf +): Future[Result[StorageServer, string]] {.async: (raises: []).} = + var conf: StorageConf try: - conf = CodexConf.load( - version = codexFullVersion, + conf = StorageConf.load( + version = storageFullVersion, envVarsPrefix = "storage", cmdLine = @[], secondarySources = proc( - config: CodexConf, sources: auto + config: StorageConf, sources: auto ) {.gcsafe, raises: [ConfigurationError].} = if configJson.len > 0: sources.addConfigFileContent(Json, $(configJson)) @@ -149,14 +149,14 @@ proc createStorage( let server = try: - CodexServer.new(conf, pk) + StorageServer.new(conf, pk) except Exception as exc: return err("Failed to create Storage: " & exc.msg) return ok(server) proc process*( - self: ptr NodeLifecycleRequest, storage: ptr CodexServer + self: ptr NodeLifecycleRequest, storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_p2p_request.nim b/library/storage_thread_requests/requests/node_p2p_request.nim index dd06eb39..bfc29b88 100644 --- a/library/storage_thread_requests/requests/node_p2p_request.nim +++ b/library/storage_thread_requests/requests/node_p2p_request.nim @@ -8,9 +8,9 @@ import chronos import chronicles import libp2p import ../../alloc -import ../../../codex/node +import ../../../storage/node -from ../../../codex/codex import CodexServer, node +from ../../../storage/storage import StorageServer, node logScope: topics = "libstorage libstoragep2p" @@ -40,7 +40,7 @@ proc destroyShared(self: ptr NodeP2PRequest) = deallocShared(self) proc connect( - storage: ptr CodexServer, peerId: cstring, peerAddresses: seq[cstring] = @[] + storage: ptr StorageServer, peerId: cstring, peerAddresses: seq[cstring] = @[] ): Future[Result[string, string]] {.async: (raises: []).} = let node = storage[].node let res = PeerId.init($peerId) @@ -81,7 +81,7 @@ proc connect( return ok("") proc process*( - self: ptr NodeP2PRequest, storage: ptr CodexServer + self: ptr NodeP2PRequest, storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_storage_request.nim b/library/storage_thread_requests/requests/node_storage_request.nim index 72ee14f9..32808b1a 100644 --- a/library/storage_thread_requests/requests/node_storage_request.nim +++ b/library/storage_thread_requests/requests/node_storage_request.nim @@ -14,12 +14,12 @@ import chronicles import libp2p/stream/[lpstream] import serde/json as serde import ../../alloc -import ../../../codex/units -import ../../../codex/manifest -import ../../../codex/stores/repostore +import ../../../storage/units +import ../../../storage/manifest +import ../../../storage/stores/repostore -from ../../../codex/codex import CodexServer, node, repoStore -from ../../../codex/node import +from ../../../storage/storage import StorageServer, node, repoStore +from ../../../storage/node import iterateManifests, fetchManifest, fetchDatasetAsyncTask, delete, hasLocalBlock from libp2p import Cid, init, `$` @@ -61,7 +61,7 @@ type ManifestWithCid = object manifest {.serialize.}: Manifest proc list( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = var manifests = newSeq[ManifestWithCid]() proc onManifest(cid: Cid, manifest: Manifest) {.raises: [], gcsafe.} = @@ -78,7 +78,7 @@ proc list( return ok(serde.toJson(manifests)) proc delete( - storage: ptr CodexServer, cCid: cstring + storage: ptr StorageServer, cCid: cstring ): Future[Result[string, string]] {.async: (raises: []).} = let cid = Cid.init($cCid) if cid.isErr: @@ -97,7 +97,7 @@ proc delete( return ok("") proc fetch( - storage: ptr CodexServer, cCid: cstring + storage: ptr StorageServer, cCid: cstring ): Future[Result[string, string]] {.async: (raises: []).} = let cid = Cid.init($cCid) if cid.isErr: @@ -116,7 +116,7 @@ proc fetch( return err("Failed to fetch the data: download cancelled.") proc space( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = let repoStore = storage[].repoStore let space = StorageSpace( @@ -128,7 +128,7 @@ proc space( return ok(serde.toJson(space)) proc exists( - storage: ptr CodexServer, cCid: cstring + storage: ptr StorageServer, cCid: cstring ): Future[Result[string, string]] {.async: (raises: []).} = let cid = Cid.init($cCid) if cid.isErr: @@ -142,7 +142,7 @@ proc exists( return err("Failed to check the data existence: operation cancelled.") proc process*( - self: ptr NodeStorageRequest, storage: ptr CodexServer + self: ptr NodeStorageRequest, storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_upload_request.nim b/library/storage_thread_requests/requests/node_upload_request.nim index 17ef1e8e..064439b2 100644 --- a/library/storage_thread_requests/requests/node_upload_request.nim +++ b/library/storage_thread_requests/requests/node_upload_request.nim @@ -24,11 +24,11 @@ import questionable/results import faststreams/inputs import libp2p/stream/[bufferstream, lpstream] import ../../alloc -import ../../../codex/units -import ../../../codex/codextypes +import ../../../storage/units +import ../../../storage/storagetypes -from ../../../codex/codex import CodexServer, node -from ../../../codex/node import store +from ../../../storage/storage import StorageServer, node +from ../../../storage/node import store from libp2p import Cid, `$` logScope: @@ -86,7 +86,7 @@ proc destroyShared(self: ptr NodeUploadRequest) = deallocShared(self) proc init( - storage: ptr CodexServer, filepath: cstring = "", chunkSize: csize_t = 0 + storage: ptr StorageServer, filepath: cstring = "", chunkSize: csize_t = 0 ): Future[Result[string, string]] {.async: (raises: []).} = ## Init a new session upload and return its ID. ## The session contains the future corresponding to the @@ -156,7 +156,7 @@ proc init( return ok(sessionId) proc chunk( - storage: ptr CodexServer, sessionId: cstring, chunk: seq[byte] + storage: ptr StorageServer, sessionId: cstring, chunk: seq[byte] ): Future[Result[string, string]] {.async: (raises: []).} = ## Upload a chunk of data to the session identified by sessionId. ## The chunk is pushed to the BufferStream of the session. @@ -205,7 +205,7 @@ proc chunk( return ok("") proc finalize( - storage: ptr CodexServer, sessionId: cstring + storage: ptr StorageServer, sessionId: cstring ): Future[Result[string, string]] {.async: (raises: []).} = ## Finalize the upload session identified by sessionId. ## This closes the BufferStream and waits for the `node.store` future @@ -242,7 +242,7 @@ proc finalize( session.fut.cancelSoon() proc cancel( - storage: ptr CodexServer, sessionId: cstring + storage: ptr StorageServer, sessionId: cstring ): Future[Result[string, string]] {.async: (raises: []).} = ## Cancel the upload session identified by sessionId. ## This cancels the `node.store` future and removes the session @@ -293,7 +293,7 @@ proc streamFile( return err("Failed to stream the file: " & $e.msg) proc file( - storage: ptr CodexServer, sessionId: cstring, onProgress: OnProgressHandler + storage: ptr StorageServer, sessionId: cstring, onProgress: OnProgressHandler ): Future[Result[string, string]] {.async: (raises: []).} = ## Starts the file upload for the session identified by sessionId. ## Will call finalize when done and return the CID of the uploaded file. @@ -333,7 +333,7 @@ proc file( proc process*( self: ptr NodeUploadRequest, - storage: ptr CodexServer, + storage: ptr StorageServer, onUploadProgress: OnProgressHandler = nil, ): Future[Result[string, string]] {.async: (raises: []).} = defer: diff --git a/library/storage_thread_requests/storage_thread_request.nim b/library/storage_thread_requests/storage_thread_request.nim index 375bf1b8..946ef4f8 100644 --- a/library/storage_thread_requests/storage_thread_request.nim +++ b/library/storage_thread_requests/storage_thread_request.nim @@ -14,7 +14,7 @@ import ./requests/node_upload_request import ./requests/node_download_request import ./requests/node_storage_request -from ../../codex/codex import CodexServer +from ../../storage/storage import StorageServer type RequestType* {.pure.} = enum LIFECYCLE @@ -89,7 +89,7 @@ proc handleRes[T: string | void | seq[byte]]( proc process*( T: type StorageThreadRequest, request: ptr StorageThreadRequest, - storage: ptr CodexServer, + storage: ptr StorageServer, ) {.async: (raises: []).} = ## Processes the request in the Logos Storage thread. ## Dispatch to the appropriate request handler based on reqType. diff --git a/metrics/README.md b/metrics/README.md index f21d74e9..aa90207c 100644 --- a/metrics/README.md +++ b/metrics/README.md @@ -12,7 +12,7 @@ Use the `--metrics-address` and `--metrics-port` flags to to adjust the address Metrics are useful to monitor the health of the process and should aid in identifying and debugging potential issues that would be hard to notice otherwise. -All Logos Storage metrics should be prefixed with the `codex_` prefix to be able to differentiate from metrics exposed by other subsystems. For example libp2p generally prefixed with the `libp2p_` prefix. +All Logos Storage metrics should be prefixed with the `storage_` prefix to be able to differentiate from metrics exposed by other subsystems. For example libp2p generally prefixed with the `libp2p_` prefix. Metrics can be added on an as needed basis, however, keep in mind the potential overhead they might introduce. In particular, be careful with labels as they will generate as many metrics as there are labels for a specific collector. If a metrics or a set of metrics are expensive, it is usually advisable to put them behind a compile time flag. @@ -20,7 +20,7 @@ Metrics can be added on an as needed basis, however, keep in mind the potential The exposed metrics can be aggregate by the [Prometheus](https://prometheus.io/) monitoring systems and additionally graphed through [Grafana](https://grafana.com/). -This directory contains both the default `prometheus.yml` config file as well as a basic `codex-grafana-dashboard.json` file that can be augmented with additional panels and metrics on an as needed basis. +This directory contains both the default `prometheus.yml` config file as well as a basic `storage-grafana-dashboard.json` file that can be augmented with additional panels and metrics on an as needed basis. Additionally, please consider installing the [node_exporter](https://github.com/prometheus/node_exporter) agent to collect machine level metrics such as overall memory, process, networking, disc IO, etc... @@ -34,7 +34,7 @@ This will take you to the import page. ![](assets/import.png) -Use either one of the presented methods (upload json, load from a url or copy paste the json into the text-box), to upload the `codex-grafana-dashboard.json` file. +Use either one of the presented methods (upload json, load from a url or copy paste the json into the text-box), to upload the `storage-grafana-dashboard.json` file. Finally, you'll be presented with the following screen where you can change the name and the `UID` of the imported dashboard. This is only necessary if there is already a dashboard with the same name or `UID`. diff --git a/metrics/prometheus.yml b/metrics/prometheus.yml index 79b99e1c..2bddf402 100644 --- a/metrics/prometheus.yml +++ b/metrics/prometheus.yml @@ -2,7 +2,7 @@ global: scrape_interval: 12s scrape_configs: - - job_name: "codex" + - job_name: "storage" static_configs: - targets: ['127.0.0.1:8008'] - job_name: "node_exporter" diff --git a/metrics/codex-grafana-dashboard.json b/metrics/storage-grafana-dashboard.json similarity index 100% rename from metrics/codex-grafana-dashboard.json rename to metrics/storage-grafana-dashboard.json diff --git a/nix/default.nix b/nix/default.nix index 3ec55f7d..d7cd15ce 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -30,7 +30,7 @@ let in stdenv.mkDerivation rec { pname = "storage"; - version = "${tools.findKeyValue "version = \"([0-9]+\.[0-9]+\.[0-9]+)\"" ../codex.nimble}-${revision}"; + version = "${tools.findKeyValue "version = \"([0-9]+\.[0-9]+\.[0-9]+)\"" ../storage.nimble}-${revision}"; inherit src; diff --git a/nix/service.nix b/nix/service.nix index 345feb9b..228d8740 100644 --- a/nix/service.nix +++ b/nix/service.nix @@ -17,7 +17,7 @@ in package = mkOption { type = types.package; default = pkgs.callPackage ./default.nix { src = self; inherit circomCompatPkg; }; - defaultText = literalExpression "pkgs.codex"; + defaultText = literalExpression "pkgs.storage"; description = mdDoc "Package to use as Nim Logos Storage node."; }; diff --git a/openapi.yaml b/openapi.yaml index 04d8960d..150986d8 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -171,7 +171,7 @@ components: type: string nullable: true description: "The original name of the uploaded content (optional)" - example: codex.png + example: storage.png mimetype: type: string nullable: true @@ -287,7 +287,7 @@ paths: description: The content disposition used to send the filename. schema: type: string - example: 'attachment; filename="codex.png"' + example: 'attachment; filename="storage.png"' requestBody: content: application/octet-stream: diff --git a/codex.nim b/storage.nim similarity index 87% rename from codex.nim rename to storage.nim index aad998b0..d8d2bfa0 100644 --- a/codex.nim +++ b/storage.nim @@ -18,36 +18,36 @@ import pkg/confutils/toml/std/uri as confTomlUri import pkg/toml_serialization import pkg/libp2p -import ./codex/conf -import ./codex/codex -import ./codex/logutils -import ./codex/units -import ./codex/utils/keyutils -import ./codex/codextypes +import ./storage/conf +import ./storage/storage +import ./storage/logutils +import ./storage/units +import ./storage/utils/keyutils +import ./storage/storagetypes -export codex, conf, libp2p, chronos, logutils +export storage, conf, libp2p, chronos, logutils when isMainModule: import std/os import pkg/confutils/defs - import ./codex/utils/fileutils + import ./storage/utils/fileutils logScope: - topics = "codex" + topics = "storage" when defined(posix): import system/ansi_c - type CodexStatus {.pure.} = enum + type StorageStatus {.pure.} = enum Stopped Stopping Running - let config = CodexConf.load( - version = codexFullVersion, + let config = StorageConf.load( + version = storageFullVersion, envVarsPrefix = "storage", secondarySources = proc( - config: CodexConf, sources: auto + config: StorageConf, sources: auto ) {.gcsafe, raises: [ConfigurationError].} = if configFile =? config.configFile: sources.addConfigFile(Toml, configFile) @@ -81,7 +81,7 @@ when isMainModule: trace "Repo dir initialized", dir = config.dataDir / "repo" var - state: CodexStatus + state: StorageStatus shutdown: Future[void] let @@ -94,7 +94,7 @@ when isMainModule: privateKey = setupKey(keyPath).expect("Should setup private key!") server = try: - CodexServer.new(config, privateKey) + StorageServer.new(config, privateKey) except Exception as exc: error "Failed to start Logos Storage", msg = exc.msg quit QuitFailure @@ -102,7 +102,7 @@ when isMainModule: ## Ctrl+C handling proc doShutdown() = shutdown = server.shutdown() - state = CodexStatus.Stopping + state = StorageStatus.Stopping notice "Stopping Logos Storage" @@ -142,8 +142,8 @@ when isMainModule: # had a chance to start (currently you'll get a SISGSEV if you try to). quit QuitFailure - state = CodexStatus.Running - while state == CodexStatus.Running: + state = StorageStatus.Running + while state == StorageStatus.Running: try: # poll chronos chronos.poll() diff --git a/codex.nimble b/storage.nimble similarity index 100% rename from codex.nimble rename to storage.nimble diff --git a/codex/blockexchange.nim b/storage/blockexchange.nim similarity index 100% rename from codex/blockexchange.nim rename to storage/blockexchange.nim diff --git a/codex/blockexchange/engine.nim b/storage/blockexchange/engine.nim similarity index 100% rename from codex/blockexchange/engine.nim rename to storage/blockexchange/engine.nim diff --git a/codex/blockexchange/engine/advertiser.nim b/storage/blockexchange/engine/advertiser.nim similarity index 95% rename from codex/blockexchange/engine/advertiser.nim rename to storage/blockexchange/engine/advertiser.nim index c038db38..5b538572 100644 --- a/codex/blockexchange/engine/advertiser.nim +++ b/storage/blockexchange/engine/advertiser.nim @@ -28,9 +28,9 @@ import ../../logutils import ../../manifest logScope: - topics = "codex discoveryengine advertiser" + topics = "storage discoveryengine advertiser" -declareGauge(codex_inflight_advertise, "inflight advertise requests") +declareGauge(storage_inflight_advertise, "inflight advertise requests") const DefaultConcurrentAdvertRequests = 10 @@ -106,11 +106,11 @@ proc processQueueLoop(b: Advertiser) {.async: (raises: []).} = let request = b.discovery.provide(cid) b.inFlightAdvReqs[cid] = request - codex_inflight_advertise.set(b.inFlightAdvReqs.len.int64) + storage_inflight_advertise.set(b.inFlightAdvReqs.len.int64) defer: b.inFlightAdvReqs.del(cid) - codex_inflight_advertise.set(b.inFlightAdvReqs.len.int64) + storage_inflight_advertise.set(b.inFlightAdvReqs.len.int64) await request except CancelledError: diff --git a/codex/blockexchange/engine/discovery.nim b/storage/blockexchange/engine/discovery.nim similarity index 96% rename from codex/blockexchange/engine/discovery.nim rename to storage/blockexchange/engine/discovery.nim index 6a1b808d..9d5d8c08 100644 --- a/codex/blockexchange/engine/discovery.nim +++ b/storage/blockexchange/engine/discovery.nim @@ -31,9 +31,9 @@ import ../../logutils import ../../manifest logScope: - topics = "codex discoveryengine" + topics = "storage discoveryengine" -declareGauge(codex_inflight_discovery, "inflight discovery requests") +declareGauge(storage_inflight_discovery, "inflight discovery requests") const DefaultConcurrentDiscRequests = 10 @@ -114,11 +114,11 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} = if haves.len < b.minPeersPerBlock: let request = b.discovery.find(cid) b.inFlightDiscReqs[cid] = request - codex_inflight_discovery.set(b.inFlightDiscReqs.len.int64) + storage_inflight_discovery.set(b.inFlightDiscReqs.len.int64) defer: b.inFlightDiscReqs.del(cid) - codex_inflight_discovery.set(b.inFlightDiscReqs.len.int64) + storage_inflight_discovery.set(b.inFlightDiscReqs.len.int64) if (await request.withTimeout(DefaultDiscoveryTimeout)) and peers =? (await request).catch: diff --git a/codex/blockexchange/engine/engine.nim b/storage/blockexchange/engine/engine.nim similarity index 95% rename from codex/blockexchange/engine/engine.nim rename to storage/blockexchange/engine/engine.nim index 2da068d9..595f5541 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/storage/blockexchange/engine/engine.nim @@ -44,39 +44,41 @@ import ./pendingblocks export peers, pendingblocks, discovery logScope: - topics = "codex blockexcengine" + topics = "storage blockexcengine" declareCounter( - codex_block_exchange_want_have_lists_sent, "codex blockexchange wantHave lists sent" + storage_block_exchange_want_have_lists_sent, + "storage blockexchange wantHave lists sent", ) declareCounter( - codex_block_exchange_want_have_lists_received, - "codex blockexchange wantHave lists received", + storage_block_exchange_want_have_lists_received, + "storage blockexchange wantHave lists received", ) declareCounter( - codex_block_exchange_want_block_lists_sent, "codex blockexchange wantBlock lists sent" + storage_block_exchange_want_block_lists_sent, + "storage blockexchange wantBlock lists sent", ) declareCounter( - codex_block_exchange_want_block_lists_received, - "codex blockexchange wantBlock lists received", + storage_block_exchange_want_block_lists_received, + "storage blockexchange wantBlock lists received", ) -declareCounter(codex_block_exchange_blocks_sent, "codex blockexchange blocks sent") +declareCounter(storage_block_exchange_blocks_sent, "storage blockexchange blocks sent") declareCounter( - codex_block_exchange_blocks_received, "codex blockexchange blocks received" + storage_block_exchange_blocks_received, "storage blockexchange blocks received" ) declareCounter( - codex_block_exchange_spurious_blocks_received, - "codex blockexchange unrequested/duplicate blocks received", + storage_block_exchange_spurious_blocks_received, + "storage blockexchange unrequested/duplicate blocks received", ) declareCounter( - codex_block_exchange_discovery_requests_total, + storage_block_exchange_discovery_requests_total, "Total number of peer discovery requests sent", ) declareCounter( - codex_block_exchange_peer_timeouts_total, "Total number of peer activity timeouts" + storage_block_exchange_peer_timeouts_total, "Total number of peer activity timeouts" ) declareCounter( - codex_block_exchange_requests_failed_total, + storage_block_exchange_requests_failed_total, "Total number of block requests that failed after exhausting retries", ) @@ -166,7 +168,7 @@ proc sendWantHave( let toAsk = addresses.filterIt(it notin p.peerHave) trace "Sending wantHave request", toAsk, peer = p.id await self.network.request.sendWantList(p.id, toAsk, wantType = WantType.WantHave) - codex_block_exchange_want_have_lists_sent.inc() + storage_block_exchange_want_have_lists_sent.inc() proc sendWantBlock( self: BlockExcEngine, addresses: seq[BlockAddress], blockPeer: BlockExcPeerCtx @@ -175,7 +177,7 @@ proc sendWantBlock( await self.network.request.sendWantList( blockPeer.id, addresses, wantType = WantType.WantBlock ) # we want this remote to send us a block - codex_block_exchange_want_block_lists_sent.inc() + storage_block_exchange_want_block_lists_sent.inc() proc sendBatchedWantList( self: BlockExcEngine, @@ -297,7 +299,7 @@ proc refreshBlockKnowledge(self: BlockExcEngine) {.async: (raises: [CancelledErr proc searchForNewPeers(self: BlockExcEngine, cid: Cid) = if self.lastDiscRequest + DiscoveryRateLimit < Moment.now(): trace "Searching for new peers for", cid = cid - codex_block_exchange_discovery_requests_total.inc() + storage_block_exchange_discovery_requests_total.inc() self.lastDiscRequest = Moment.now() # always refresh before calling await! self.discovery.queueFindBlocksReq(@[cid]) else: @@ -333,7 +335,7 @@ proc downloadInternal( if self.pendingBlocks.retriesExhausted(address): trace "Error retries exhausted" - codex_block_exchange_requests_failed_total.inc() + storage_block_exchange_requests_failed_total.inc() handle.fail(newException(RetriesExhaustedError, "Error retries exhausted")) break @@ -415,7 +417,7 @@ proc downloadInternal( else: # If the peer timed out, retries immediately. trace "Peer timed out during block request", peer = scheduledPeer.id - codex_block_exchange_peer_timeouts_total.inc() + storage_block_exchange_peer_timeouts_total.inc() await self.network.dropPeer(scheduledPeer.id) # Evicts peer immediately or we may end up picking it again in the # next retry. @@ -426,7 +428,7 @@ proc downloadInternal( await handle.cancelAndWait() except RetriesExhaustedError as exc: warn "Retries exhausted for block", address, exc = exc.msg - codex_block_exchange_requests_failed_total.inc() + storage_block_exchange_requests_failed_total.inc() if not handle.finished: handle.fail(exc) finally: @@ -690,7 +692,7 @@ proc blocksDeliveryHandler*( # Unknown peers and unrequested blocks are dropped with a warning. if not allowSpurious and (peerCtx == nil or not peerCtx.blockReceived(bd.address)): warn "Dropping unrequested or duplicate block received from peer" - codex_block_exchange_spurious_blocks_received.inc() + storage_block_exchange_spurious_blocks_received.inc() continue if err =? self.validateBlockDelivery(bd).errorOption: @@ -729,7 +731,7 @@ proc blocksDeliveryHandler*( discard lastIdle = Moment.now() - codex_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64) + storage_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64) if err =? catch(await self.resolveBlocks(validatedBlocksDelivery)).errorOption: warn "Error resolving blocks", err = err.msg @@ -789,11 +791,11 @@ proc wantListHandler*( BlockPresence(address: e.address, `type`: BlockPresenceType.DontHave) ) - codex_block_exchange_want_have_lists_received.inc() + storage_block_exchange_want_have_lists_received.inc() of WantType.WantBlock: peerCtx.wantedBlocks.incl(e.address) schedulePeer = true - codex_block_exchange_want_block_lists_received.inc() + storage_block_exchange_want_block_lists_received.inc() else: # Updating existing entry in peer wants # peer doesn't want this block anymore if e.cancel: @@ -902,7 +904,7 @@ proc taskHandler*( continue await self.network.request.sendBlocksDelivery(peerCtx.id, blockDeliveries) - codex_block_exchange_blocks_sent.inc(blockDeliveries.len.int64) + storage_block_exchange_blocks_sent.inc(blockDeliveries.len.int64) # Drops the batch from the peer's set of wanted blocks; i.e. assumes that after # we send the blocks, then the peer no longer wants them, so we don't need to # re-send them. Note that the send might still fail down the line and we will diff --git a/codex/blockexchange/engine/pendingblocks.nim b/storage/blockexchange/engine/pendingblocks.nim similarity index 94% rename from codex/blockexchange/engine/pendingblocks.nim rename to storage/blockexchange/engine/pendingblocks.nim index f843870f..a6308cf7 100644 --- a/codex/blockexchange/engine/pendingblocks.nim +++ b/storage/blockexchange/engine/pendingblocks.nim @@ -22,14 +22,15 @@ import ../../blocktype import ../../logutils logScope: - topics = "codex pendingblocks" + topics = "storage pendingblocks" declareGauge( - codex_block_exchange_pending_block_requests, - "codex blockexchange pending block requests", + storage_block_exchange_pending_block_requests, + "storage blockexchange pending block requests", ) declareGauge( - codex_block_exchange_retrieval_time_us, "codex blockexchange block retrieval time us" + storage_block_exchange_retrieval_time_us, + "storage blockexchange block retrieval time us", ) const @@ -53,7 +54,7 @@ type lastInclusion*: Moment # time at which we last included a block into our wantlist proc updatePendingBlockGauge(p: PendingBlocksManager) = - codex_block_exchange_pending_block_requests.set(p.blocks.len.int64) + storage_block_exchange_pending_block_requests.set(p.blocks.len.int64) proc getWantHandle*( self: PendingBlocksManager, address: BlockAddress, requested: ?PeerId = PeerId.none @@ -123,7 +124,7 @@ proc resolve*( blockReq.handle.complete(bd.blk) - codex_block_exchange_retrieval_time_us.set(retrievalDurationUs) + storage_block_exchange_retrieval_time_us.set(retrievalDurationUs) else: trace "Block handle already finished", address = bd.address diff --git a/codex/blockexchange/network.nim b/storage/blockexchange/network.nim similarity index 100% rename from codex/blockexchange/network.nim rename to storage/blockexchange/network.nim diff --git a/codex/blockexchange/network/network.nim b/storage/blockexchange/network/network.nim similarity index 99% rename from codex/blockexchange/network/network.nim rename to storage/blockexchange/network/network.nim index 91617d0f..5f62d4a2 100644 --- a/codex/blockexchange/network/network.nim +++ b/storage/blockexchange/network/network.nim @@ -27,10 +27,10 @@ import ./networkpeer export networkpeer logScope: - topics = "codex blockexcnetwork" + topics = "storage blockexcnetwork" const - Codec* = "/codex/blockexc/1.0.0" + Codec* = "/storage/blockexc/1.0.0" DefaultMaxInflight* = 100 type diff --git a/codex/blockexchange/network/networkpeer.nim b/storage/blockexchange/network/networkpeer.nim similarity index 98% rename from codex/blockexchange/network/networkpeer.nim rename to storage/blockexchange/network/networkpeer.nim index 927303a6..f61d3286 100644 --- a/codex/blockexchange/network/networkpeer.nim +++ b/storage/blockexchange/network/networkpeer.nim @@ -19,7 +19,7 @@ import ../../logutils import ../../utils/trackedfutures logScope: - topics = "codex blockexcnetworkpeer" + topics = "storage blockexcnetworkpeer" const DefaultYieldInterval = 50.millis diff --git a/codex/blockexchange/peers.nim b/storage/blockexchange/peers.nim similarity index 100% rename from codex/blockexchange/peers.nim rename to storage/blockexchange/peers.nim diff --git a/codex/blockexchange/peers/peercontext.nim b/storage/blockexchange/peers/peercontext.nim similarity index 100% rename from codex/blockexchange/peers/peercontext.nim rename to storage/blockexchange/peers/peercontext.nim diff --git a/codex/blockexchange/peers/peerctxstore.nim b/storage/blockexchange/peers/peerctxstore.nim similarity index 98% rename from codex/blockexchange/peers/peerctxstore.nim rename to storage/blockexchange/peers/peerctxstore.nim index 853bd33f..ae935809 100644 --- a/codex/blockexchange/peers/peerctxstore.nim +++ b/storage/blockexchange/peers/peerctxstore.nim @@ -25,7 +25,7 @@ import ./peercontext export peercontext logScope: - topics = "codex peerctxstore" + topics = "storage peerctxstore" type PeerCtxStore* = ref object of RootObj diff --git a/codex/blockexchange/protobuf/blockexc.nim b/storage/blockexchange/protobuf/blockexc.nim similarity index 100% rename from codex/blockexchange/protobuf/blockexc.nim rename to storage/blockexchange/protobuf/blockexc.nim diff --git a/codex/blockexchange/protobuf/message.nim b/storage/blockexchange/protobuf/message.nim similarity index 100% rename from codex/blockexchange/protobuf/message.nim rename to storage/blockexchange/protobuf/message.nim diff --git a/codex/blockexchange/protobuf/message.proto b/storage/blockexchange/protobuf/message.proto similarity index 100% rename from codex/blockexchange/protobuf/message.proto rename to storage/blockexchange/protobuf/message.proto diff --git a/codex/blockexchange/protobuf/message.proto.license b/storage/blockexchange/protobuf/message.proto.license similarity index 100% rename from codex/blockexchange/protobuf/message.proto.license rename to storage/blockexchange/protobuf/message.proto.license diff --git a/codex/blockexchange/protobuf/presence.nim b/storage/blockexchange/protobuf/presence.nim similarity index 100% rename from codex/blockexchange/protobuf/presence.nim rename to storage/blockexchange/protobuf/presence.nim diff --git a/codex/blocktype.nim b/storage/blocktype.nim similarity index 98% rename from codex/blocktype.nim rename to storage/blocktype.nim index b5a1019c..d056c789 100644 --- a/codex/blocktype.nim +++ b/storage/blocktype.nim @@ -25,9 +25,9 @@ import ./utils import ./errors import ./logutils import ./utils/json -import ./codextypes +import ./storagetypes -export errors, logutils, units, codextypes +export errors, logutils, units, storagetypes type Block* = ref object of RootObj diff --git a/codex/chunker.nim b/storage/chunker.nim similarity index 100% rename from codex/chunker.nim rename to storage/chunker.nim diff --git a/codex/clock.nim b/storage/clock.nim similarity index 100% rename from codex/clock.nim rename to storage/clock.nim diff --git a/codex/conf.nim b/storage/conf.nim similarity index 96% rename from codex/conf.nim rename to storage/conf.nim index f9a80175..50d191cf 100644 --- a/codex/conf.nim +++ b/storage/conf.nim @@ -35,7 +35,7 @@ import pkg/questionable import pkg/questionable/results import pkg/stew/base64 -import ./codextypes +import ./storagetypes import ./discovery import ./logutils import ./stores @@ -46,7 +46,7 @@ import ./utils/natutils from ./blockexchange/engine/pendingblocks import DefaultBlockRetries -export units, net, codextypes, logutils, completeCmdArg, parseCmdArg, NatConfig +export units, net, storagetypes, logutils, completeCmdArg, parseCmdArg, NatConfig export DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, @@ -90,7 +90,7 @@ type repoSQLite = "sqlite" repoLevelDb = "leveldb" - CodexConf* = object + StorageConf* = object configFile* {. desc: "Loads the configuration from a TOML file", defaultValueDesc: "none", @@ -277,19 +277,19 @@ type desc: "Logs to file", defaultValue: string.none, name: "log-file", hidden .}: Option[string] -func defaultAddress*(conf: CodexConf): IpAddress = +func defaultAddress*(conf: StorageConf): IpAddress = result = static parseIpAddress("127.0.0.1") func defaultNatConfig*(): NatConfig = result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAny) -proc getCodexVersion(): string = +proc getStorageVersion(): string = let tag = strip(staticExec("git describe --tags --abbrev=0")) if tag.isEmptyOrWhitespace: return "untagged build" return tag -proc getCodexRevision(): string = +proc getStorageRevision(): string = # using a slice in a static context breaks nimsuggest for some reason var res = strip(staticExec("git rev-parse --short HEAD")) return res @@ -298,12 +298,12 @@ proc getNimBanner(): string = staticExec("nim --version | grep Version") const - codexVersion* = getCodexVersion() - codexRevision* = getCodexRevision() + storageVersion* = getStorageVersion() + storageRevision* = getStorageRevision() nimBanner* = getNimBanner() - codexFullVersion* = - "Storage version: " & codexVersion & "\p" & "Storage revision: " & codexRevision & + storageFullVersion* = + "Storage version: " & storageVersion & "\p" & "Storage revision: " & storageRevision & "\p" proc parseCmdArg*( @@ -533,7 +533,7 @@ proc updateLogLevel*(logLevel: string) {.raises: [ValueError].} = if not setTopicState(topicName, settings.state, settings.logLevel): warn "Unrecognized logging topic", topic = topicName -proc setupLogging*(conf: CodexConf) = +proc setupLogging*(conf: StorageConf) = when defaultChroniclesStream.outputs.type.arity != 3: warn "Logging configuration options not enabled in the current build" else: @@ -597,7 +597,7 @@ proc setupLogging*(conf: CodexConf) = else: defaultChroniclesStream.outputs[0].writer = writer -proc setupMetrics*(config: CodexConf) = +proc setupMetrics*(config: StorageConf) = if config.metricsEnabled: let metricsAddress = config.metricsAddress notice "Starting metrics HTTP server", diff --git a/storage/contentids_exts.nim b/storage/contentids_exts.nim new file mode 100644 index 00000000..3efeb0ab --- /dev/null +++ b/storage/contentids_exts.nim @@ -0,0 +1,5 @@ +const ContentIdsExts = [ + multiCodec("storage-root"), + multiCodec("storage-manifest"), + multiCodec("storage-block"), +] diff --git a/codex/discovery.nim b/storage/discovery.nim similarity index 99% rename from codex/discovery.nim rename to storage/discovery.nim index 1cded8e6..73be4ee3 100644 --- a/codex/discovery.nim +++ b/storage/discovery.nim @@ -32,7 +32,7 @@ export discv5 # much more elegantly. logScope: - topics = "codex discovery" + topics = "storage discovery" type Discovery* = ref object of RootObj protocol*: discv5.Protocol # dht protocol diff --git a/codex/errors.nim b/storage/errors.nim similarity index 93% rename from codex/errors.nim rename to storage/errors.nim index 655aec87..84be6f77 100644 --- a/codex/errors.nim +++ b/storage/errors.nim @@ -20,8 +20,8 @@ import pkg/questionable/results export results type - CodexError* = object of CatchableError # base codex error - CodexResult*[T] = Result[T, ref CodexError] + StorageError* = object of CatchableError # base Storage error + StorageResult*[T] = Result[T, ref StorageError] FinishedFailed*[T] = tuple[success: seq[Future[T]], failure: seq[Future[T]]] @@ -37,7 +37,7 @@ template mapFailure*[T, V, E]( ) template mapFailure*[T, V](exp: Result[T, V]): Result[T, ref CatchableError] = - mapFailure(exp, CodexError) + mapFailure(exp, StorageError) # TODO: using a template here, causes bad codegen func toFailure*[T](exp: Option[T]): Result[T, ref CatchableError] {.inline.} = diff --git a/codex/logutils.nim b/storage/logutils.nim similarity index 96% rename from codex/logutils.nim rename to storage/logutils.nim index f3b98548..d45bc141 100644 --- a/codex/logutils.nim +++ b/storage/logutils.nim @@ -13,24 +13,24 @@ ## 6. Remove need to [avoid importing or exporting `toJson`, `%`, `%*` to prevent ## conflicts](https://github.com/logos-storage/logos-storage-nim/pull/645#issuecomment-1838834467) ## -## When declaring a new type, one should consider importing the `codex/logutils` +## When declaring a new type, one should consider importing the `storage/logutils` ## module, and specifying `formatIt`. If textlines log output and json log output ## need to be different, overload `formatIt` and specify a `LogFormat`. If json ## serialization is needed, it can be declared with a `%` proc. `logutils` ## imports and exports `nim-serde` which handles the de/serialization, examples -## below. **Only `codex/logutils` needs to be imported.** +## below. **Only `storage/logutils` needs to be imported.** ## -## Using `logutils` in the Codex codebase: -## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils` +## Using `logutils` in the Storage codebase: +## - Instead of importing `pkg/chronicles`, import `pkg/storage/logutils` ## - most of `chronicles` is exported by `logutils` ## - Instead of importing `std/json`, import `pkg/serde/json` ## - `std/json` is exported by `serde` which is exported by `logutils` ## - Instead of importing `pkg/nim-json-serialization`, import -## `pkg/serde/json` or use codex-specific overloads by importing `utils/json` +## `pkg/serde/json` or use storage-specific overloads by importing `utils/json` ## - one of the goals is to remove the use of `nim-json-serialization` ## ## ```nim -## import pkg/codex/logutils +## import pkg/storage/logutils ## ## type ## BlockAddress* = object diff --git a/codex/manifest.nim b/storage/manifest.nim similarity index 100% rename from codex/manifest.nim rename to storage/manifest.nim diff --git a/codex/manifest/coders.nim b/storage/manifest/coders.nim similarity index 100% rename from codex/manifest/coders.nim rename to storage/manifest/coders.nim diff --git a/codex/manifest/manifest.nim b/storage/manifest/manifest.nim similarity index 98% rename from codex/manifest/manifest.nim rename to storage/manifest/manifest.nim index 7eed1088..90c105f2 100644 --- a/codex/manifest/manifest.nim +++ b/storage/manifest/manifest.nim @@ -71,7 +71,7 @@ func mimetype*(self: Manifest): ?string = ############################################################ func isManifest*(cid: Cid): ?!bool = - success (ManifestCodec == ?cid.contentType().mapFailure(CodexError)) + success (ManifestCodec == ?cid.contentType().mapFailure(StorageError)) func isManifest*(mc: MultiCodec): ?!bool = success mc == ManifestCodec diff --git a/codex/merkletree.nim b/storage/merkletree.nim similarity index 75% rename from codex/merkletree.nim rename to storage/merkletree.nim index 366af992..36d50505 100644 --- a/codex/merkletree.nim +++ b/storage/merkletree.nim @@ -1,4 +1,5 @@ import ./merkletree/merkletree import ./merkletree/coders +import ./merkletree/coders export merkletree, coders diff --git a/codex/merkletree/coders.nim b/storage/merkletree/coders.nim similarity index 100% rename from codex/merkletree/coders.nim rename to storage/merkletree/coders.nim diff --git a/codex/merkletree/merkletree.nim b/storage/merkletree/merkletree.nim similarity index 99% rename from codex/merkletree/merkletree.nim rename to storage/merkletree/merkletree.nim index 0b5de28d..67912b62 100644 --- a/codex/merkletree/merkletree.nim +++ b/storage/merkletree/merkletree.nim @@ -22,14 +22,14 @@ import pkg/merkletree import ../utils import ../rng import ../errors -import ../codextypes +import ../storagetypes from ../utils/digest import digestBytes export merkletree logScope: - topics = "codex merkletree" + topics = "storage merkletree" type ByteTreeKey* {.pure.} = enum @@ -50,7 +50,6 @@ type func getProof*(self: StorageMerkleTree, index: int): ?!StorageMerkleProof = var proof = StorageMerkleProof(mcodec: self.mcodec) - ?self.getProof(index, proof) success proof diff --git a/storage/multicodec_exts.nim b/storage/multicodec_exts.nim new file mode 100644 index 00000000..7171219a --- /dev/null +++ b/storage/multicodec_exts.nim @@ -0,0 +1,2 @@ +const CodecExts = + [("storage-manifest", 0xCD01), ("storage-block", 0xCD02), ("storage-root", 0xCD03)] diff --git a/codex/multihash_exts.nim b/storage/multihash_exts.nim similarity index 100% rename from codex/multihash_exts.nim rename to storage/multihash_exts.nim diff --git a/storage/namespaces.nim b/storage/namespaces.nim new file mode 100644 index 00000000..289cc9a3 --- /dev/null +++ b/storage/namespaces.nim @@ -0,0 +1,25 @@ +## Logos Storage +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +const + # Namespaces + StorageMetaNamespace* = "meta" # meta info stored here + StorageRepoNamespace* = "repo" # repository namespace, blocks and manifests are subkeys + StorageBlockTotalNamespace* = StorageMetaNamespace & "/total" + # number of blocks in the repo + StorageBlocksNamespace* = StorageRepoNamespace & "/blocks" # blocks namespace + StorageManifestNamespace* = StorageRepoNamespace & "/manifests" # manifest namespace + StorageBlocksTtlNamespace* = # Cid TTL + StorageMetaNamespace & "/ttl" + StorageBlockProofNamespace* = # Cid and Proof + StorageMetaNamespace & "/proof" + StorageDhtNamespace* = "dht" # Dht namespace + StorageDhtProvidersNamespace* = # Dht providers namespace + StorageDhtNamespace & "/providers" + StorageQuotaNamespace* = StorageMetaNamespace & "/quota" # quota's namespace diff --git a/codex/nat.nim b/storage/nat.nim similarity index 99% rename from codex/nat.nim rename to storage/nat.nim index f3919437..0a711df5 100644 --- a/codex/nat.nim +++ b/storage/nat.nim @@ -412,7 +412,7 @@ proc nattedAddress*( if ipPart.isSome and port.isSome: # Try to setup NAT mapping for the address let (newIP, tcp, udp) = - setupAddress(natConfig, ipPart.get, port.get, udpPort, "codex") + setupAddress(natConfig, ipPart.get, port.get, udpPort, "storage") if newIP.isSome: # NAT mapping successful - add discovery address with mapped UDP port discoveryAddrs.add(getMultiAddrWithIPAndUDPPort(newIP.get, udp.get)) diff --git a/codex/node.nim b/storage/node.nim similarity index 90% rename from codex/node.nim rename to storage/node.nim index 1e2e0b80..ac04cb58 100644 --- a/codex/node.nim +++ b/storage/node.nim @@ -45,7 +45,7 @@ import ./utils/trackedfutures export logutils logScope: - topics = "codex node" + topics = "storage node" const DefaultFetchBatch = 1024 @@ -53,7 +53,7 @@ const BatchRefillThreshold = 0.75 # Refill when 75% of window completes type - CodexNode* = object + StorageNode* = object switch: Switch networkId: PeerId networkStore: NetworkStore @@ -63,27 +63,27 @@ type taskPool: Taskpool trackedFutures: TrackedFutures - CodexNodeRef* = ref CodexNode + StorageNodeRef* = ref StorageNode OnManifest* = proc(cid: Cid, manifest: Manifest): void {.gcsafe, raises: [].} BatchProc* = proc(blocks: seq[bt.Block]): Future[?!void] {.async: (raises: [CancelledError]).} OnBlockStoredProc = proc(chunk: seq[byte]): void {.gcsafe, raises: [].} -func switch*(self: CodexNodeRef): Switch = +func switch*(self: StorageNodeRef): Switch = return self.switch -func blockStore*(self: CodexNodeRef): BlockStore = +func blockStore*(self: StorageNodeRef): BlockStore = return self.networkStore -func engine*(self: CodexNodeRef): BlockExcEngine = +func engine*(self: StorageNodeRef): BlockExcEngine = return self.engine -func discovery*(self: CodexNodeRef): Discovery = +func discovery*(self: StorageNodeRef): Discovery = return self.discovery proc storeManifest*( - self: CodexNodeRef, manifest: Manifest + self: StorageNodeRef, manifest: Manifest ): Future[?!bt.Block] {.async.} = without encodedVerifiable =? manifest.encode(), err: trace "Unable to encode manifest" @@ -100,7 +100,7 @@ proc storeManifest*( success blk proc fetchManifest*( - self: CodexNodeRef, cid: Cid + self: StorageNodeRef, cid: Cid ): Future[?!Manifest] {.async: (raises: [CancelledError]).} = ## Fetch and decode a manifest block ## @@ -124,18 +124,18 @@ proc fetchManifest*( return manifest.success -proc findPeer*(self: CodexNodeRef, peerId: PeerId): Future[?PeerRecord] {.async.} = - ## Find peer using the discovery service from the given CodexNode +proc findPeer*(self: StorageNodeRef, peerId: PeerId): Future[?PeerRecord] {.async.} = + ## Find peer using the discovery service from the given StorageNode ## return await self.discovery.findPeer(peerId) proc connect*( - self: CodexNodeRef, peerId: PeerId, addrs: seq[MultiAddress] + self: StorageNodeRef, peerId: PeerId, addrs: seq[MultiAddress] ): Future[void] = self.switch.connect(peerId, addrs) proc updateExpiry*( - self: CodexNodeRef, manifestCid: Cid, expiry: SecondsSince1970 + self: StorageNodeRef, manifestCid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async: (raises: [CancelledError]).} = without manifest =? await self.fetchManifest(manifestCid), error: trace "Unable to fetch manifest for cid", manifestCid @@ -158,7 +158,7 @@ proc updateExpiry*( return success() proc fetchBatched*( - self: CodexNodeRef, + self: StorageNodeRef, cid: Cid, iter: Iter[int], batchSize = DefaultFetchBatch, @@ -233,7 +233,7 @@ proc fetchBatched*( success() proc fetchBatched*( - self: CodexNodeRef, + self: StorageNodeRef, manifest: Manifest, batchSize = DefaultFetchBatch, onBatch: BatchProc = nil, @@ -249,7 +249,7 @@ proc fetchBatched*( self.fetchBatched(manifest.treeCid, iter, batchSize, onBatch, fetchLocal) proc fetchDatasetAsync*( - self: CodexNodeRef, manifest: Manifest, fetchLocal = true + self: StorageNodeRef, manifest: Manifest, fetchLocal = true ): Future[void] {.async: (raises: []).} = ## Asynchronously fetch a dataset in the background. ## This task will be tracked and cleaned up on node shutdown. @@ -264,14 +264,14 @@ proc fetchDatasetAsync*( except CancelledError as exc: trace "Cancelled fetching blocks", exc = exc.msg -proc fetchDatasetAsyncTask*(self: CodexNodeRef, manifest: Manifest) = +proc fetchDatasetAsyncTask*(self: StorageNodeRef, manifest: Manifest) = ## Start fetching a dataset in the background. ## The task will be tracked and cleaned up on node shutdown. ## self.trackedFutures.track(self.fetchDatasetAsync(manifest, fetchLocal = false)) proc streamSingleBlock( - self: CodexNodeRef, cid: Cid + self: StorageNodeRef, cid: Cid ): Future[?!LPStream] {.async: (raises: [CancelledError]).} = ## Streams the contents of a single block. ## @@ -296,7 +296,7 @@ proc streamSingleBlock( LPStream(stream).success proc streamEntireDataset( - self: CodexNodeRef, manifest: Manifest, manifestCid: Cid + self: StorageNodeRef, manifest: Manifest, manifestCid: Cid ): Future[?!LPStream] {.async: (raises: [CancelledError]).} = ## Streams the contents of the entire dataset described by the manifest. ## @@ -324,7 +324,7 @@ proc streamEntireDataset( stream.success proc retrieve*( - self: CodexNodeRef, cid: Cid, local: bool = true + self: StorageNodeRef, cid: Cid, local: bool = true ): Future[?!LPStream] {.async: (raises: [CancelledError]).} = ## Retrieve by Cid a single block or an entire dataset described by manifest ## @@ -340,7 +340,7 @@ proc retrieve*( await self.streamEntireDataset(manifest, cid) -proc deleteSingleBlock(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} = +proc deleteSingleBlock(self: StorageNodeRef, cid: Cid): Future[?!void] {.async.} = if err =? (await self.networkStore.delBlock(cid)).errorOption: error "Error deleting block", cid, err = err.msg return failure(err) @@ -348,7 +348,7 @@ proc deleteSingleBlock(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} = trace "Deleted block", cid return success() -proc deleteEntireDataset(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} = +proc deleteEntireDataset(self: StorageNodeRef, cid: Cid): Future[?!void] {.async.} = # Deletion is a strictly local operation var store = self.networkStore.localStore @@ -382,7 +382,7 @@ proc deleteEntireDataset(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} success() proc delete*( - self: CodexNodeRef, cid: Cid + self: StorageNodeRef, cid: Cid ): Future[?!void] {.async: (raises: [CatchableError]).} = ## Deletes a whole dataset, if Cid is a Manifest Cid, or a single block, if Cid a block Cid, ## from the underlying block store. This is a strictly local operation. @@ -400,7 +400,7 @@ proc delete*( await self.deleteEntireDataset(cid) proc store*( - self: CodexNodeRef, + self: StorageNodeRef, stream: LPStream, filename: ?string = string.none, mimetype: ?string = string.none, @@ -484,7 +484,7 @@ proc store*( return manifestBlk.cid.success -proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} = +proc iterateManifests*(self: StorageNodeRef, onManifest: OnManifest) {.async.} = without cidsIter =? await self.networkStore.listBlocks(BlockType.Manifest): warn "Failed to listBlocks" return @@ -502,11 +502,11 @@ proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} = onManifest(cid, manifest) proc onExpiryUpdate( - self: CodexNodeRef, rootCid: Cid, expiry: SecondsSince1970 + self: StorageNodeRef, rootCid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async: (raises: [CancelledError]).} = return await self.updateExpiry(rootCid, expiry) -proc start*(self: CodexNodeRef) {.async.} = +proc start*(self: StorageNodeRef) {.async.} = if not self.engine.isNil: await self.engine.start() @@ -519,7 +519,7 @@ proc start*(self: CodexNodeRef) {.async.} = self.networkId = self.switch.peerInfo.peerId notice "Started Storage node", id = self.networkId, addrs = self.switch.peerInfo.addrs -proc stop*(self: CodexNodeRef) {.async.} = +proc stop*(self: StorageNodeRef) {.async.} = trace "Stopping node" await self.trackedFutures.cancelTracked() @@ -533,22 +533,22 @@ proc stop*(self: CodexNodeRef) {.async.} = if not self.clock.isNil: await self.clock.stop() -proc close*(self: CodexNodeRef) {.async.} = +proc close*(self: StorageNodeRef) {.async.} = if not self.networkStore.isNil: await self.networkStore.close proc new*( - T: type CodexNodeRef, + T: type StorageNodeRef, switch: Switch, networkStore: NetworkStore, engine: BlockExcEngine, discovery: Discovery, taskpool: Taskpool, -): CodexNodeRef = - ## Create new instance of a Codex self, call `start` to run it +): StorageNodeRef = + ## Create new instance of a Storage self, call `start` to run it ## - CodexNodeRef( + StorageNodeRef( switch: switch, networkStore: networkStore, engine: engine, @@ -558,7 +558,7 @@ proc new*( ) proc hasLocalBlock*( - self: CodexNodeRef, cid: Cid + self: StorageNodeRef, cid: Cid ): Future[bool] {.async: (raises: [CancelledError]).} = ## Returns true if the given Cid is present in the local store diff --git a/codex/rest/api.nim b/storage/rest/api.nim similarity index 96% rename from codex/rest/api.nim rename to storage/rest/api.nim index bf08a2e7..0c4a9c06 100644 --- a/codex/rest/api.nim +++ b/storage/rest/api.nim @@ -39,10 +39,10 @@ import ./coders import ./json logScope: - topics = "codex restapi" + topics = "storage restapi" -declareCounter(codex_api_uploads, "codex API uploads") -declareCounter(codex_api_downloads, "codex API downloads") +declareCounter(storage_api_uploads, "storage API uploads") +declareCounter(storage_api_downloads, "storage API downloads") proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].} = 0 @@ -50,7 +50,7 @@ proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].} proc formatManifest(cid: Cid, manifest: Manifest): RestContent = return RestContent.init(cid, manifest) -proc formatManifestBlocks(node: CodexNodeRef): Future[JsonNode] {.async.} = +proc formatManifestBlocks(node: StorageNodeRef): Future[JsonNode] {.async.} = var content: seq[RestContent] proc addManifest(cid: Cid, manifest: Manifest) = @@ -67,7 +67,7 @@ proc isPending(resp: HttpResponseRef): bool = return resp.getResponseState() == HttpResponseState.Empty proc retrieveCid( - node: CodexNodeRef, cid: Cid, local: bool = true, resp: HttpResponseRef + node: StorageNodeRef, cid: Cid, local: bool = true, resp: HttpResponseRef ): Future[void] {.async: (raises: [CancelledError, HttpWriteError]).} = ## Download a file from the node in a streaming ## manner @@ -131,7 +131,7 @@ proc retrieveCid( await resp.send(addr buff[0], buff.len) await resp.finish() - codex_api_downloads.inc() + storage_api_downloads.inc() except CancelledError as exc: raise exc except LPStreamError as exc: @@ -173,7 +173,7 @@ proc getFilenameFromContentDisposition(contentDisposition: string): ?string = let filename = parts[1].strip() return filename[0 ..^ 2].some -proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) = +proc initDataApi(node: StorageNodeRef, repoStore: RepoStore, router: var RestRouter) = let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion router.api(MethodOptions, "/api/storage/v1/data") do( @@ -238,7 +238,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute error "Error uploading file", exc = error.msg return RestApiResponse.error(Http500, error.msg) - codex_api_uploads.inc() + storage_api_uploads.inc() trace "Uploaded file", cid return RestApiResponse.response($cid) except CancelledError: @@ -383,7 +383,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute ) return RestApiResponse.response($json, contentType = "application/json") -proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = +proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter) = let allowedOrigin = router.allowedOrigin ## various node management api's @@ -465,7 +465,7 @@ proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = return RestApiResponse.error(Http500, "Unknown error dialling peer", headers = headers) -proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = +proc initDebugApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter) = let allowedOrigin = router.allowedOrigin router.api(MethodGet, "/api/storage/v1/debug/info") do() -> RestApiResponse: @@ -484,7 +484,7 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "", "announceAddresses": node.discovery.announceAddrs, "table": table, - "storage": {"version": $codexVersion, "revision": $codexRevision}, + "storage": {"version": $storageVersion, "revision": $storageRevision}, } # return pretty json for human readability @@ -542,8 +542,8 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = return RestApiResponse.error(Http500, headers = headers) proc initRestApi*( - node: CodexNodeRef, - conf: CodexConf, + node: StorageNodeRef, + conf: StorageConf, repoStore: RepoStore, corsAllowedOrigin: ?string, ): RestRouter = diff --git a/codex/rest/coders.nim b/storage/rest/coders.nim similarity index 100% rename from codex/rest/coders.nim rename to storage/rest/coders.nim diff --git a/codex/rest/json.nim b/storage/rest/json.nim similarity index 100% rename from codex/rest/json.nim rename to storage/rest/json.nim diff --git a/codex/rng.nim b/storage/rng.nim similarity index 100% rename from codex/rng.nim rename to storage/rng.nim diff --git a/codex/codex.nim b/storage/storage.nim similarity index 80% rename from codex/codex.nim rename to storage/storage.nim index 001bae89..e85b3959 100644 --- a/codex/codex.nim +++ b/storage/storage.nim @@ -35,35 +35,35 @@ import ./discovery import ./systemclock import ./utils/addrutils import ./namespaces -import ./codextypes +import ./storagetypes import ./logutils import ./nat logScope: - topics = "codex node" + topics = "storage node" type - CodexServer* = ref object - config: CodexConf + StorageServer* = ref object + config: StorageConf restServer: RestServerRef - codexNode: CodexNodeRef + storageNode: StorageNodeRef repoStore: RepoStore maintenance: BlockMaintainer taskpool: Taskpool isStarted: bool - CodexPrivateKey* = libp2p.PrivateKey # alias + StoragePrivateKey* = libp2p.PrivateKey # alias -func config*(self: CodexServer): CodexConf = +func config*(self: StorageServer): StorageConf = return self.config -func node*(self: CodexServer): CodexNodeRef = - return self.codexNode +func node*(self: StorageServer): StorageNodeRef = + return self.storageNode -func repoStore*(self: CodexServer): RepoStore = +func repoStore*(self: StorageServer): RepoStore = return self.repoStore -proc start*(s: CodexServer) {.async.} = +proc start*(s: StorageServer) {.async.} = if s.isStarted: warn "Storage server already started, skipping" return @@ -73,23 +73,23 @@ proc start*(s: CodexServer) {.async.} = s.maintenance.start() - await s.codexNode.switch.start() + await s.storageNode.switch.start() let (announceAddrs, discoveryAddrs) = nattedAddress( - s.config.nat, s.codexNode.switch.peerInfo.addrs, s.config.discoveryPort + s.config.nat, s.storageNode.switch.peerInfo.addrs, s.config.discoveryPort ) - s.codexNode.discovery.updateAnnounceRecord(announceAddrs) - s.codexNode.discovery.updateDhtRecord(discoveryAddrs) + s.storageNode.discovery.updateAnnounceRecord(announceAddrs) + s.storageNode.discovery.updateDhtRecord(discoveryAddrs) - await s.codexNode.start() + await s.storageNode.start() if s.restServer != nil: s.restServer.start() s.isStarted = true -proc stop*(s: CodexServer) {.async.} = +proc stop*(s: StorageServer) {.async.} = if not s.isStarted: warn "Storage is not started" return @@ -97,8 +97,8 @@ proc stop*(s: CodexServer) {.async.} = notice "Stopping Storage node" var futures = @[ - s.codexNode.switch.stop(), - s.codexNode.stop(), + s.storageNode.switch.stop(), + s.storageNode.stop(), s.repoStore.stop(), s.maintenance.stop(), ] @@ -114,9 +114,9 @@ proc stop*(s: CodexServer) {.async.} = error "Failed to stop Storage node", failures = res.failure.len raiseAssert "Failed to stop Storage node" -proc close*(s: CodexServer) {.async.} = +proc close*(s: StorageServer) {.async.} = var futures = - @[s.codexNode.close(), s.repoStore.close(), s.codexNode.discovery.close()] + @[s.storageNode.close(), s.repoStore.close(), s.storageNode.discovery.close()] let res = await noCancel allFinishedFailed[void](futures) @@ -131,14 +131,14 @@ proc close*(s: CodexServer) {.async.} = error "Failed to close Storage node", failures = res.failure.len raiseAssert "Failed to close Storage node" -proc shutdown*(server: CodexServer) {.async.} = +proc shutdown*(server: StorageServer) {.async.} = await server.stop() await server.close() proc new*( - T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey -): CodexServer = - ## create CodexServer including setting up datastore, repostore, etc + T: type StorageServer, config: StorageConf, privateKey: StoragePrivateKey +): StorageServer = + ## create StorageServer including setting up datastore, repostore, etc let switch = SwitchBuilder .new() .withPrivateKey(privateKey) @@ -169,7 +169,7 @@ proc new*( cache = CacheStore.new(cacheSize = config.cacheSize) ## Is unused? - let discoveryDir = config.dataDir / CodexDhtNamespace + let discoveryDir = config.dataDir / StorageDhtNamespace if io2.createPath(discoveryDir).isErr: trace "Unable to create discovery directory for block store", @@ -178,7 +178,7 @@ proc new*( msg: "Unable to create discovery directory for block store: " & discoveryDir ) - let providersPath = config.dataDir / CodexDhtProvidersNamespace + let providersPath = config.dataDir / StorageDhtProvidersNamespace let discoveryStoreRes = LevelDbDatastore.new(providersPath) if discoveryStoreRes.isErr: error "Failed to initialize discovery datastore", @@ -221,7 +221,7 @@ proc new*( repoStore = RepoStore.new( repoDs = repoData, - metaDs = LevelDbDatastore.new(config.dataDir / CodexMetaNamespace).expect( + metaDs = LevelDbDatastore.new(config.dataDir / StorageMetaNamespace).expect( "Should create metadata store!" ), quotaMaxBytes = config.storageQuota, @@ -244,7 +244,7 @@ proc new*( ) store = NetworkStore.new(engine, repoStore) - codexNode = CodexNodeRef.new( + storageNode = StorageNodeRef.new( switch = switch, networkStore = store, engine = engine, @@ -257,7 +257,7 @@ proc new*( if config.apiBindAddress.isSome: restServer = RestServerRef .new( - codexNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin), + storageNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin), initTAddress(config.apiBindAddress.get(), config.apiPort), bufferSize = (1024 * 64), maxRequestBodySize = int.high, @@ -266,9 +266,9 @@ proc new*( switch.mount(network) - CodexServer( + StorageServer( config: config, - codexNode: codexNode, + storageNode: storageNode, restServer: restServer, repoStore: repoStore, maintenance: maintenance, diff --git a/codex/codextypes.nim b/storage/storagetypes.nim similarity index 90% rename from codex/codextypes.nim rename to storage/storagetypes.nim index bd4dfcf8..d9bc790e 100644 --- a/codex/codextypes.nim +++ b/storage/storagetypes.nim @@ -30,11 +30,11 @@ const # hashes Sha256HashCodec* = multiCodec("sha2-256") - ManifestCodec* = multiCodec("codex-manifest") - DatasetRootCodec* = multiCodec("codex-root") - BlockCodec* = multiCodec("codex-block") + ManifestCodec* = multiCodec("storage-manifest") + DatasetRootCodec* = multiCodec("storage-root") + BlockCodec* = multiCodec("storage-block") - CodexPrimitivesCodecs* = [ManifestCodec, DatasetRootCodec, BlockCodec] + StoragePrimitivesCodecs* = [ManifestCodec, DatasetRootCodec, BlockCodec] proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] = ## Initialize padding blocks table diff --git a/codex/stores.nim b/storage/stores.nim similarity index 100% rename from codex/stores.nim rename to storage/stores.nim diff --git a/codex/stores/blockstore.nim b/storage/stores/blockstore.nim similarity index 99% rename from codex/stores/blockstore.nim rename to storage/stores/blockstore.nim index 7edabc4d..5abed158 100644 --- a/codex/stores/blockstore.nim +++ b/storage/stores/blockstore.nim @@ -22,7 +22,7 @@ import ../utils export blocktype type - BlockNotFoundError* = object of CodexError + BlockNotFoundError* = object of StorageError BlockType* {.pure.} = enum Manifest diff --git a/codex/stores/cachestore.nim b/storage/stores/cachestore.nim similarity index 99% rename from codex/stores/cachestore.nim rename to storage/stores/cachestore.nim index 98fa3deb..0c48b9f9 100644 --- a/codex/stores/cachestore.nim +++ b/storage/stores/cachestore.nim @@ -30,7 +30,7 @@ import ../clock export blockstore logScope: - topics = "codex cachestore" + topics = "storage cachestore" type CacheStore* = ref object of BlockStore @@ -39,7 +39,7 @@ type cache: LruCache[Cid, Block] cidAndProofCache: LruCache[(Cid, Natural), (Cid, StorageMerkleProof)] - InvalidBlockSize* = object of CodexError + InvalidBlockSize* = object of StorageError const DefaultCacheSize*: NBytes = 5.MiBs diff --git a/codex/stores/keyutils.nim b/storage/stores/keyutils.nim similarity index 65% rename from codex/stores/keyutils.nim rename to storage/stores/keyutils.nim index 78efa732..67e14f86 100644 --- a/codex/stores/keyutils.nim +++ b/storage/stores/keyutils.nim @@ -17,14 +17,14 @@ import ../namespaces import ../manifest const - CodexMetaKey* = Key.init(CodexMetaNamespace).tryGet - CodexRepoKey* = Key.init(CodexRepoNamespace).tryGet - CodexBlocksKey* = Key.init(CodexBlocksNamespace).tryGet - CodexTotalBlocksKey* = Key.init(CodexBlockTotalNamespace).tryGet - CodexManifestKey* = Key.init(CodexManifestNamespace).tryGet - BlocksTtlKey* = Key.init(CodexBlocksTtlNamespace).tryGet - BlockProofKey* = Key.init(CodexBlockProofNamespace).tryGet - QuotaKey* = Key.init(CodexQuotaNamespace).tryGet + StorageMetaKey* = Key.init(StorageMetaNamespace).tryGet + StorageRepoKey* = Key.init(StorageRepoNamespace).tryGet + StorageBlocksKey* = Key.init(StorageBlocksNamespace).tryGet + StorageTotalBlocksKey* = Key.init(StorageBlockTotalNamespace).tryGet + StorageManifestKey* = Key.init(StorageManifestNamespace).tryGet + BlocksTtlKey* = Key.init(StorageBlocksTtlNamespace).tryGet + BlockProofKey* = Key.init(StorageBlockProofNamespace).tryGet + QuotaKey* = Key.init(StorageQuotaNamespace).tryGet QuotaUsedKey* = (QuotaKey / "used").tryGet QuotaReservedKey* = (QuotaKey / "reserved").tryGet @@ -32,9 +32,9 @@ func makePrefixKey*(postFixLen: int, cid: Cid): ?!Key = let cidKey = ?Key.init(($cid)[^postFixLen ..^ 1] & "/" & $cid) if ?cid.isManifest: - success CodexManifestKey / cidKey + success StorageManifestKey / cidKey else: - success CodexBlocksKey / cidKey + success StorageBlocksKey / cidKey proc createBlockExpirationMetadataKey*(cid: Cid): ?!Key = BlocksTtlKey / $cid diff --git a/codex/stores/maintenance.nim b/storage/stores/maintenance.nim similarity index 99% rename from codex/stores/maintenance.nim rename to storage/stores/maintenance.nim index b2fece4d..ffdcafbc 100644 --- a/codex/stores/maintenance.nim +++ b/storage/stores/maintenance.nim @@ -24,7 +24,7 @@ import ../logutils import ../systemclock logScope: - topics = "codex maintenance" + topics = "storage maintenance" const DefaultBlockInterval* = 10.minutes diff --git a/codex/stores/networkstore.nim b/storage/stores/networkstore.nim similarity index 99% rename from codex/stores/networkstore.nim rename to storage/stores/networkstore.nim index 6aa76d8c..be0b0862 100644 --- a/codex/stores/networkstore.nim +++ b/storage/stores/networkstore.nim @@ -25,7 +25,7 @@ import ./blockstore export blockstore, blockexchange, asyncheapqueue logScope: - topics = "codex networkstore" + topics = "storage networkstore" type NetworkStore* = ref object of BlockStore engine*: BlockExcEngine # blockexc decision engine diff --git a/codex/stores/queryiterhelper.nim b/storage/stores/queryiterhelper.nim similarity index 100% rename from codex/stores/queryiterhelper.nim rename to storage/stores/queryiterhelper.nim diff --git a/codex/stores/repostore.nim b/storage/stores/repostore.nim similarity index 100% rename from codex/stores/repostore.nim rename to storage/stores/repostore.nim diff --git a/codex/stores/repostore/coders.nim b/storage/stores/repostore/coders.nim similarity index 100% rename from codex/stores/repostore/coders.nim rename to storage/stores/repostore/coders.nim diff --git a/codex/stores/repostore/operations.nim b/storage/stores/repostore/operations.nim similarity index 94% rename from codex/stores/repostore/operations.nim rename to storage/stores/repostore/operations.nim index efd7f56f..f2e67094 100644 --- a/codex/stores/repostore/operations.nim +++ b/storage/stores/repostore/operations.nim @@ -26,11 +26,11 @@ import ../../logutils import ../../merkletree logScope: - topics = "codex repostore" + topics = "storage repostore" -declareGauge(codex_repostore_blocks, "codex repostore blocks") -declareGauge(codex_repostore_bytes_used, "codex repostore bytes used") -declareGauge(codex_repostore_bytes_reserved, "codex repostore bytes reserved") +declareGauge(storage_repostore_blocks, "storage repostore blocks") +declareGauge(storage_repostore_bytes_used, "storage repostore bytes used") +declareGauge(storage_repostore_bytes_reserved, "storage repostore bytes reserved") proc putLeafMetadata*( self: RepoStore, @@ -90,7 +90,7 @@ proc updateTotalBlocksCount*( self: RepoStore, plusCount: Natural = 0, minusCount: Natural = 0 ): Future[?!void] {.async: (raises: [CancelledError]).} = await self.metaDs.modify( - CodexTotalBlocksKey, + StorageTotalBlocksKey, proc(maybeCurrCount: ?Natural): Future[?Natural] {.async.} = let count: Natural = if currCount =? maybeCurrCount: @@ -99,7 +99,7 @@ proc updateTotalBlocksCount*( plusCount - minusCount self.totalBlocks = count - codex_repostore_blocks.set(count.int64) + storage_repostore_blocks.set(count.int64) count.some, ) @@ -132,8 +132,8 @@ proc updateQuotaUsage*( ) else: self.quotaUsage = usage - codex_repostore_bytes_used.set(usage.used.int64) - codex_repostore_bytes_reserved.set(usage.reserved.int64) + storage_repostore_bytes_used.set(usage.used.int64) + storage_repostore_bytes_reserved.set(usage.reserved.int64) return usage.some, ) diff --git a/codex/stores/repostore/store.nim b/storage/stores/repostore/store.nim similarity index 97% rename from codex/stores/repostore/store.nim rename to storage/stores/repostore/store.nim index 554bd62c..feec3807 100644 --- a/codex/stores/repostore/store.nim +++ b/storage/stores/repostore/store.nim @@ -32,7 +32,7 @@ import ../../utils export blocktype, cid logScope: - topics = "codex repostore" + topics = "storage repostore" ########################################################### # BlockStore API @@ -323,9 +323,9 @@ method listBlocks*( let key = case blockType - of BlockType.Manifest: CodexManifestKey - of BlockType.Block: CodexBlocksKey - of BlockType.Both: CodexRepoKey + of BlockType.Manifest: StorageManifestKey + of BlockType.Block: StorageBlocksKey + of BlockType.Both: StorageRepoKey let query = Query.init(key, value = false) without queryIter =? (await self.repoDs.query(query)), err: @@ -444,7 +444,7 @@ proc release*( proc start*( self: RepoStore -): Future[void] {.async: (raises: [CancelledError, CodexError]).} = +): Future[void] {.async: (raises: [CancelledError, StorageError]).} = ## Start repo ## if self.started: @@ -453,10 +453,10 @@ proc start*( trace "Starting rep" if err =? (await self.updateTotalBlocksCount()).errorOption: - raise newException(CodexError, err.msg) + raise newException(StorageError, err.msg) if err =? (await self.updateQuotaUsage()).errorOption: - raise newException(CodexError, err.msg) + raise newException(StorageError, err.msg) self.started = true diff --git a/codex/stores/repostore/types.nim b/storage/stores/repostore/types.nim similarity index 98% rename from codex/stores/repostore/types.nim rename to storage/stores/repostore/types.nim index 6936b92d..1d6c743a 100644 --- a/codex/stores/repostore/types.nim +++ b/storage/stores/repostore/types.nim @@ -25,7 +25,7 @@ const DefaultQuotaBytes* = 20.GiBs type - QuotaNotEnoughError* = object of CodexError + QuotaNotEnoughError* = object of StorageError RepoStore* = ref object of BlockStore postFixLen*: int diff --git a/codex/stores/treehelper.nim b/storage/stores/treehelper.nim similarity index 100% rename from codex/stores/treehelper.nim rename to storage/stores/treehelper.nim diff --git a/codex/streams.nim b/storage/streams.nim similarity index 100% rename from codex/streams.nim rename to storage/streams.nim diff --git a/codex/streams/asyncstreamwrapper.nim b/storage/streams/asyncstreamwrapper.nim similarity index 100% rename from codex/streams/asyncstreamwrapper.nim rename to storage/streams/asyncstreamwrapper.nim diff --git a/codex/streams/seekablestream.nim b/storage/streams/seekablestream.nim similarity index 94% rename from codex/streams/seekablestream.nim rename to storage/streams/seekablestream.nim index 926fa70d..fc467b2e 100644 --- a/codex/streams/seekablestream.nim +++ b/storage/streams/seekablestream.nim @@ -15,7 +15,7 @@ import ../logutils export lpstream, chronos, logutils logScope: - topics = "codex seekablestream" + topics = "storage seekablestream" type SeekableStream* = ref object of LPStream offset*: int diff --git a/codex/streams/storestream.nim b/storage/streams/storestream.nim similarity index 99% rename from codex/streams/storestream.nim rename to storage/streams/storestream.nim index d2fc51e9..b3297baf 100644 --- a/codex/streams/storestream.nim +++ b/storage/streams/storestream.nim @@ -25,7 +25,7 @@ import ./seekablestream export stores, blocktype, manifest, chronos logScope: - topics = "codex storestream" + topics = "storage storestream" const StoreStreamTrackerName* = "StoreStream" diff --git a/codex/systemclock.nim b/storage/systemclock.nim similarity index 100% rename from codex/systemclock.nim rename to storage/systemclock.nim diff --git a/codex/units.nim b/storage/units.nim similarity index 100% rename from codex/units.nim rename to storage/units.nim diff --git a/codex/utils.nim b/storage/utils.nim similarity index 100% rename from codex/utils.nim rename to storage/utils.nim diff --git a/codex/utils/addrutils.nim b/storage/utils/addrutils.nim similarity index 100% rename from codex/utils/addrutils.nim rename to storage/utils/addrutils.nim diff --git a/codex/utils/arrayutils.nim b/storage/utils/arrayutils.nim similarity index 100% rename from codex/utils/arrayutils.nim rename to storage/utils/arrayutils.nim diff --git a/codex/utils/asyncheapqueue.nim b/storage/utils/asyncheapqueue.nim similarity index 100% rename from codex/utils/asyncheapqueue.nim rename to storage/utils/asyncheapqueue.nim diff --git a/codex/utils/asynciter.nim b/storage/utils/asynciter.nim similarity index 100% rename from codex/utils/asynciter.nim rename to storage/utils/asynciter.nim diff --git a/codex/utils/asyncstatemachine.nim b/storage/utils/asyncstatemachine.nim similarity index 100% rename from codex/utils/asyncstatemachine.nim rename to storage/utils/asyncstatemachine.nim diff --git a/codex/utils/digest.nim b/storage/utils/digest.nim similarity index 100% rename from codex/utils/digest.nim rename to storage/utils/digest.nim diff --git a/codex/utils/exceptions.nim b/storage/utils/exceptions.nim similarity index 100% rename from codex/utils/exceptions.nim rename to storage/utils/exceptions.nim diff --git a/codex/utils/fileutils.nim b/storage/utils/fileutils.nim similarity index 100% rename from codex/utils/fileutils.nim rename to storage/utils/fileutils.nim diff --git a/codex/utils/iter.nim b/storage/utils/iter.nim similarity index 100% rename from codex/utils/iter.nim rename to storage/utils/iter.nim diff --git a/codex/utils/json.nim b/storage/utils/json.nim similarity index 100% rename from codex/utils/json.nim rename to storage/utils/json.nim diff --git a/codex/utils/keyutils.nim b/storage/utils/keyutils.nim similarity index 57% rename from codex/utils/keyutils.nim rename to storage/utils/keyutils.nim index dcdc17bb..372777c2 100644 --- a/codex/utils/keyutils.nim +++ b/storage/utils/keyutils.nim @@ -20,25 +20,25 @@ import ../rng export crypto type - CodexKeyError = object of CodexError - CodexKeyUnsafeError = object of CodexKeyError + StorageKeyError = object of StorageError + StorageKeyUnsafeError = object of StorageKeyError proc setupKey*(path: string): ?!PrivateKey = if not path.fileAccessible({AccessFlags.Find}): info "Creating a private key and saving it" let - res = ?PrivateKey.random(Rng.instance()[]).mapFailure(CodexKeyError) - bytes = ?res.getBytes().mapFailure(CodexKeyError) + res = ?PrivateKey.random(Rng.instance()[]).mapFailure(StorageKeyError) + bytes = ?res.getBytes().mapFailure(StorageKeyError) - ?path.secureWriteFile(bytes).mapFailure(CodexKeyError) - return PrivateKey.init(bytes).mapFailure(CodexKeyError) + ?path.secureWriteFile(bytes).mapFailure(StorageKeyError) + return PrivateKey.init(bytes).mapFailure(StorageKeyError) info "Found a network private key" - if not ?checkSecureFile(path).mapFailure(CodexKeyError): + if not ?checkSecureFile(path).mapFailure(StorageKeyError): warn "The network private key file is not safe, aborting" return failure newException( - CodexKeyUnsafeError, "The network private key file is not safe" + StorageKeyUnsafeError, "The network private key file is not safe" ) - let kb = ?path.readAllBytes().mapFailure(CodexKeyError) - return PrivateKey.init(kb).mapFailure(CodexKeyError) + let kb = ?path.readAllBytes().mapFailure(StorageKeyError) + return PrivateKey.init(kb).mapFailure(StorageKeyError) diff --git a/codex/utils/natutils.nim b/storage/utils/natutils.nim similarity index 100% rename from codex/utils/natutils.nim rename to storage/utils/natutils.nim diff --git a/codex/utils/options.nim b/storage/utils/options.nim similarity index 100% rename from codex/utils/options.nim rename to storage/utils/options.nim diff --git a/codex/utils/safeasynciter.nim b/storage/utils/safeasynciter.nim similarity index 100% rename from codex/utils/safeasynciter.nim rename to storage/utils/safeasynciter.nim diff --git a/codex/utils/sharedbuf.nim b/storage/utils/sharedbuf.nim similarity index 100% rename from codex/utils/sharedbuf.nim rename to storage/utils/sharedbuf.nim diff --git a/codex/utils/stintutils.nim b/storage/utils/stintutils.nim similarity index 100% rename from codex/utils/stintutils.nim rename to storage/utils/stintutils.nim diff --git a/codex/utils/timer.nim b/storage/utils/timer.nim similarity index 100% rename from codex/utils/timer.nim rename to storage/utils/timer.nim diff --git a/codex/utils/trackedfutures.nim b/storage/utils/trackedfutures.nim similarity index 100% rename from codex/utils/trackedfutures.nim rename to storage/utils/trackedfutures.nim diff --git a/codex/utils/uniqueptr.nim b/storage/utils/uniqueptr.nim similarity index 100% rename from codex/utils/uniqueptr.nim rename to storage/utils/uniqueptr.nim diff --git a/tests/codex/testmerkletree.nim b/tests/codex/testmerkletree.nim deleted file mode 100644 index 47ba59cb..00000000 --- a/tests/codex/testmerkletree.nim +++ /dev/null @@ -1,4 +0,0 @@ -import ./merkletree/testcodextree -import ./merkletree/testcodexcoders - -{.warning[UnusedImport]: off.} diff --git a/tests/coverage.nim b/tests/coverage.nim index b40b8f32..0c36e6a5 100644 --- a/tests/coverage.nim +++ b/tests/coverage.nim @@ -1 +1 @@ -include ./testCodex +include ./testStorage diff --git a/tests/examples.nim b/tests/examples.nim index 4d413c3c..710bbcfb 100644 --- a/tests/examples.nim +++ b/tests/examples.nim @@ -3,15 +3,15 @@ import std/sequtils import std/times import std/typetraits -import pkg/codex/rng -import pkg/codex/stores -import pkg/codex/units +import pkg/storage/rng +import pkg/storage/stores +import pkg/storage/units import pkg/chronos import pkg/stew/byteutils import pkg/stint -import ./codex/helpers/randomchunker +import ./storage/helpers/randomchunker export randomchunker export units diff --git a/tests/helpers/trackers.nim b/tests/helpers/trackers.nim index 898053c2..e0b72d09 100644 --- a/tests/helpers/trackers.nim +++ b/tests/helpers/trackers.nim @@ -1,4 +1,4 @@ -import pkg/codex/streams/storestream +import pkg/storage/streams/storestream import pkg/unittest2 # From lip2p/tests/helpers diff --git a/tests/integration/1_minute/testblockexpiration.nim b/tests/integration/1_minute/testblockexpiration.nim index be66c388..cc518afd 100644 --- a/tests/integration/1_minute/testblockexpiration.nim +++ b/tests/integration/1_minute/testblockexpiration.nim @@ -9,7 +9,7 @@ multinodesuite "Node block expiration tests": test "node retains not-expired file", NodeConfigs( - clients: CodexConfigs + clients: StorageConfigs .init(nodes = 1) .withBlockTtl(0, 10) .withBlockMaintenanceInterval(0, 1).some @@ -29,7 +29,7 @@ multinodesuite "Node block expiration tests": test "node deletes expired file", NodeConfigs( - clients: CodexConfigs + clients: StorageConfigs .init(nodes = 1) .withBlockTtl(0, 1) .withBlockMaintenanceInterval(0, 1).some diff --git a/tests/integration/5_minutes/testrestapi.nim b/tests/integration/5_minutes/testrestapi.nim index 4bbf6fa9..fde9a8ce 100644 --- a/tests/integration/5_minutes/testrestapi.nim +++ b/tests/integration/5_minutes/testrestapi.nim @@ -3,12 +3,12 @@ import std/net import std/sequtils import std/strformat from pkg/libp2p import `==`, `$`, Cid -import pkg/codex/units -import pkg/codex/manifest +import pkg/storage/units +import pkg/storage/manifest import ../twonodes import ../../examples -import ../../codex/examples -import ../../codex/slots/helpers +import ../../storage/examples +import ../../storage/slots/helpers import json twonodessuite "REST API": @@ -16,7 +16,7 @@ twonodessuite "REST API": check !(await client1.info()) != !(await client2.info()) test "nodes can set chronicles log level", twoNodesConfig: - await client1.setLogLevel("DEBUG;TRACE:codex") + await client1.setLogLevel("DEBUG;TRACE:storage") test "node accepts file uploads", twoNodesConfig: let cid1 = (await client1.upload("some file contents")).get diff --git a/tests/integration/5_minutes/testrestapivalidation.nim b/tests/integration/5_minutes/testrestapivalidation.nim index e11a0d7b..e13f9d1f 100644 --- a/tests/integration/5_minutes/testrestapivalidation.nim +++ b/tests/integration/5_minutes/testrestapivalidation.nim @@ -1,18 +1,18 @@ import std/times -import pkg/codex/conf +import pkg/storage/conf import pkg/stint from pkg/libp2p import Cid, `$` import ../../asynctest import ../../checktest import ../../examples -import ../../codex/examples -import ../codexconfig -import ../codexclient +import ../../storage/examples +import ../storageconfig +import ../storageclient import ../multinodes multinodesuite "Rest API validation": - let config = NodeConfigs(clients: CodexConfigs.init(nodes = 1).some) - var client: CodexClient + let config = NodeConfigs(clients: StorageConfigs.init(nodes = 1).some) + var client: StorageClient setup: client = clients()[0].client diff --git a/tests/integration/5_minutes/testupdownload.nim b/tests/integration/5_minutes/testupdownload.nim index ec5054c8..7545f948 100644 --- a/tests/integration/5_minutes/testupdownload.nim +++ b/tests/integration/5_minutes/testupdownload.nim @@ -1,6 +1,6 @@ -import pkg/codex/rest/json +import pkg/storage/rest/json import ../twonodes -import ../../codex/examples +import ../../storage/examples import json from pkg/libp2p import Cid, `$` @@ -84,7 +84,7 @@ twonodessuite "Uploads and downloads": content1 == resp2 test "reliable transfer test", twoNodesConfig: - proc transferTest(a: CodexClient, b: CodexClient) {.async.} = + proc transferTest(a: StorageClient, b: StorageClient) {.async.} = let data = await RandomChunker.example(blocks = 8) let cid = (await a.upload(data)).get let response = (await b.download(cid)).get diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 19c5b5b9..f6f8135f 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -4,20 +4,20 @@ import std/sequtils import std/strutils import std/sugar import std/times -import pkg/codex/conf -import pkg/codex/logutils +import pkg/storage/conf +import pkg/storage/logutils import pkg/chronos/transports/stream import pkg/questionable -import ./codexconfig -import ./codexprocess +import ./storageconfig +import ./storageprocess import ./nodeconfigs import ./utils import ../asynctest import ../checktest export asynctest -export codexprocess -export codexconfig +export storageprocess +export storageconfig export nodeconfigs {.push raises: [].} @@ -34,12 +34,12 @@ type SuiteTimeoutError = object of MultiNodeSuiteError const HardhatPort {.intdefine.}: int = 8545 -const CodexApiPort {.intdefine.}: int = 8080 -const CodexDiscPort {.intdefine.}: int = 8090 +const StorageApiPort {.intdefine.}: int = 8080 +const StorageDiscPort {.intdefine.}: int = 8090 const TestId {.strdefine.}: string = "TestId" -const CodexLogToFile {.booldefine.}: bool = false -const CodexLogLevel {.strdefine.}: string = "" -const CodexLogsDir {.strdefine.}: string = "" +const StorageLogToFile {.booldefine.}: bool = false +const StorageLogLevel {.strdefine.}: string = "" +const StorageLogsDir {.strdefine.}: string = "" proc raiseMultiNodeSuiteError( msg: string, parent: ref CatchableError = nil @@ -77,9 +77,9 @@ template multinodesuite*(suiteName: string, body: untyped) = var nodeConfigs: NodeConfigs var snapshot: JsonNode var lastUsedHardhatPort = HardhatPort - var lastUsedCodexApiPort = CodexApiPort - var lastUsedCodexDiscPort = CodexDiscPort - var codexPortLock: AsyncLock + var lastUsedStorageApiPort = StorageApiPort + var lastUsedStorageDiscPort = StorageDiscPort + var storagePortLock: AsyncLock template test(tname, startNodeConfigs, tbody) = currentTestName = tname @@ -91,18 +91,18 @@ template multinodesuite*(suiteName: string, body: untyped) = let parts = url.split(':') url = @[parts[0], parts[1], $port].join(":") - proc newCodexProcess( - roleIdx: int, conf: CodexConfig, role: Role + proc newStorageProcess( + roleIdx: int, conf: StorageConfig, role: Role ): Future[NodeProcess] {.async: (raises: [MultiNodeSuiteError, CancelledError]).} = let nodeIdx = running.len var config = conf let datadir = getDataDir(TestId, currentTestName, $starttime, $role, some roleIdx) try: - if config.logFile.isSome or CodexLogToFile: + if config.logFile.isSome or StorageLogToFile: try: let updatedLogFile = getLogFile( - CodexLogsDir, starttime, suiteName, currentTestName, $role, some roleIdx + StorageLogsDir, starttime, suiteName, currentTestName, $role, some roleIdx ) config.withLogFile(updatedLogFile) except IOError as e: @@ -118,17 +118,17 @@ template multinodesuite*(suiteName: string, body: untyped) = e, ) - when CodexLogLevel != "": - config.addCliOption("--log-level", CodexLogLevel) + when StorageLogLevel != "": + config.addCliOption("--log-level", StorageLogLevel) var apiPort, discPort: int - withLock(codexPortLock): - apiPort = await nextFreePort(lastUsedCodexApiPort + nodeIdx) - discPort = await nextFreePort(lastUsedCodexDiscPort + nodeIdx) + withLock(storagePortLock): + apiPort = await nextFreePort(lastUsedStorageApiPort + nodeIdx) + discPort = await nextFreePort(lastUsedStorageDiscPort + nodeIdx) config.addCliOption("--api-port", $apiPort) config.addCliOption("--disc-port", $discPort) - lastUsedCodexApiPort = apiPort - lastUsedCodexDiscPort = discPort + lastUsedStorageApiPort = apiPort + lastUsedStorageDiscPort = discPort for bootstrapNode in bootstrapNodes: config.addCliOption("--bootstrap-node", bootstrapNode) @@ -136,30 +136,30 @@ template multinodesuite*(suiteName: string, body: untyped) = config.addCliOption("--data-dir", datadir) config.addCliOption("--nat", "none") config.addCliOption("--listen-addrs", "/ip4/127.0.0.1/tcp/0") - except CodexConfigError as e: + except StorageConfigError as e: raiseMultiNodeSuiteError "invalid cli option, error: " & e.msg try: - let node = await CodexProcess.startNode( + let node = await StorageProcess.startNode( config.cliArgs, config.debugEnabled, $role & $roleIdx ) await node.waitUntilStarted() trace "node started", nodeName = $role & $roleIdx return node - except CodexConfigError as e: + except StorageConfigError as e: raiseMultiNodeSuiteError "failed to get cli args from config: " & e.msg, e except NodeProcessError as e: raiseMultiNodeSuiteError "node not started, error: " & e.msg, e - proc clients(): seq[CodexProcess] {.used.} = + proc clients(): seq[StorageProcess] {.used.} = return collect: for r in running: if r.role == Role.Client: - CodexProcess(r.node) + StorageProcess(r.node) - proc startClientNode(conf: CodexConfig): Future[NodeProcess] {.async.} = + proc startClientNode(conf: StorageConfig): Future[NodeProcess] {.async.} = let clientIdx = clients().len - return await newCodexProcess(clientIdx, conf, Role.Client) + return await newStorageProcess(clientIdx, conf, Role.Client) proc teardownImpl() {.async.} = for nodes in @[clients()]: @@ -167,7 +167,7 @@ template multinodesuite*(suiteName: string, body: untyped) = await node.stop() # also stops rest client try: node.removeDataDir() - except CodexProcessError as e: + except StorageProcessError as e: error "Failed to remove data dir during teardown", error = e.msg running = @[] @@ -191,7 +191,7 @@ template multinodesuite*(suiteName: string, body: untyped) = quit(1) proc updateBootstrapNodes( - node: CodexProcess + node: StorageProcess ): Future[void] {.async: (raises: [MultiNodeSuiteError]).} = try: without ninfo =? await node.client.info(): @@ -206,9 +206,11 @@ template multinodesuite*(suiteName: string, body: untyped) = # When this file is run with `-d:chronicles_sinks=textlines[file]`, we # need to set the log file path at runtime, otherwise chronicles didn't seem to # create a log file even when using an absolute path - when defaultChroniclesStream.outputs is (FileOutput,) and CodexLogsDir.len > 0: + when defaultChroniclesStream.outputs is (FileOutput,) and StorageLogsDir.len > 0: let logFile = - CodexLogsDir / sanitize(getAppFilename().extractFilename & ".chronicles.log") + StorageLogsDir / sanitize( + getAppFilename().extractFilename & ".chronicles.log" + ) let success = defaultChroniclesStream.outputs[0].open(logFile, fmAppend) doAssert success, "Failed to open log file: " & logFile @@ -219,7 +221,7 @@ template multinodesuite*(suiteName: string, body: untyped) = for config in clients.configs: let node = await startClientNode(config) running.add RunningNode(role: Role.Client, node: node) - await CodexProcess(node).updateBootstrapNodes() + await StorageProcess(node).updateBootstrapNodes() teardown: await teardownImpl() diff --git a/tests/integration/nodeconfigs.nim b/tests/integration/nodeconfigs.nim index 5f11bf55..c8012392 100644 --- a/tests/integration/nodeconfigs.nim +++ b/tests/integration/nodeconfigs.nim @@ -1,5 +1,5 @@ import pkg/questionable -import ./codexconfig +import ./storageconfig type NodeConfigs* = object - clients*: ?CodexConfigs + clients*: ?StorageConfigs diff --git a/tests/integration/nodeprocess.nim b/tests/integration/nodeprocess.nim index a45e7806..907eeba5 100644 --- a/tests/integration/nodeprocess.nim +++ b/tests/integration/nodeprocess.nim @@ -7,12 +7,12 @@ import pkg/libp2p import std/os import std/strformat import std/strutils -import codex/conf -import codex/utils/exceptions -import codex/utils/trackedfutures -import ./codexclient +import storage/conf +import storage/utils/exceptions +import storage/utils/trackedfutures +import ./storageclient -export codexclient +export storageclient export chronicles {.push raises: [].} diff --git a/tests/integration/codexclient.nim b/tests/integration/storageclient.nim similarity index 81% rename from tests/integration/codexclient.nim rename to tests/integration/storageclient.nim index c406a97b..15c78c29 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/storageclient.nim @@ -4,27 +4,27 @@ from pkg/libp2p import Cid, `$`, init import pkg/stint import pkg/questionable/results import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient, httptable] -import pkg/codex/logutils -import pkg/codex/rest/json -import pkg/codex/errors +import pkg/storage/logutils +import pkg/storage/rest/json +import pkg/storage/errors export httptable, httpclient -type CodexClient* = ref object +type StorageClient* = ref object baseurl: string session: HttpSessionRef type HasBlockResponse = object has: bool -proc new*(_: type CodexClient, baseurl: string): CodexClient = - CodexClient(session: HttpSessionRef.new(), baseurl: baseurl) +proc new*(_: type StorageClient, baseurl: string): StorageClient = + StorageClient(session: HttpSessionRef.new(), baseurl: baseurl) -proc close*(self: CodexClient): Future[void] {.async: (raises: []).} = +proc close*(self: StorageClient): Future[void] {.async: (raises: []).} = await self.session.closeWait() proc request( - self: CodexClient, + self: StorageClient, httpMethod: httputils.HttpMethod, url: string, body: openArray[char] = [], @@ -46,7 +46,7 @@ proc request( .send() proc post*( - self: CodexClient, + self: StorageClient, url: string, body: string = "", headers: seq[HttpHeaderTuple] = @[], @@ -56,21 +56,21 @@ proc post*( return self.request(MethodPost, url, headers = headers, body = body) proc get( - self: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[] + self: StorageClient, url: string, headers: seq[HttpHeaderTuple] = @[] ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = return self.request(MethodGet, url, headers = headers) proc delete( - self: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[] + self: StorageClient, url: string, headers: seq[HttpHeaderTuple] = @[] ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = return self.request(MethodDelete, url, headers = headers) proc patch*( - self: CodexClient, + self: StorageClient, url: string, body: string = "", headers: seq[HttpHeaderTuple] = @[], @@ -85,19 +85,19 @@ proc body*( return bytesToString (await response.getBodyBytes()) proc getContent( - client: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[] + client: StorageClient, url: string, headers: seq[HttpHeaderTuple] = @[] ): Future[string] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.get(url, headers) return await response.body proc info*( - client: CodexClient + client: StorageClient ): Future[?!JsonNode] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.get(client.baseurl & "/debug/info") return JsonNode.parse(await response.body) proc setLogLevel*( - client: CodexClient, level: string + client: StorageClient, level: string ): Future[void] {.async: (raises: [CancelledError, HttpError]).} = let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level @@ -106,26 +106,26 @@ proc setLogLevel*( assert response.status == 200 proc uploadRaw*( - client: CodexClient, contents: string, headers: seq[HttpHeaderTuple] = @[] + client: StorageClient, contents: string, headers: seq[HttpHeaderTuple] = @[] ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = return client.post(client.baseurl & "/data", body = contents, headers = headers) proc upload*( - client: CodexClient, contents: string + client: StorageClient, contents: string ): Future[?!Cid] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.uploadRaw(contents) assert response.status == 200 Cid.init(await response.body).mapFailure proc upload*( - client: CodexClient, bytes: seq[byte] + client: StorageClient, bytes: seq[byte] ): Future[?!Cid] {.async: (raw: true).} = return client.upload(string.fromBytes(bytes)) proc downloadRaw*( - client: CodexClient, cid: string, local = false + client: StorageClient, cid: string, local = false ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = @@ -133,7 +133,7 @@ proc downloadRaw*( client.get(client.baseurl & "/data/" & cid & (if local: "" else: "/network/stream")) proc downloadBytes*( - client: CodexClient, cid: Cid, local = false + client: StorageClient, cid: Cid, local = false ): Future[?!seq[byte]] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.downloadRaw($cid, local = local) @@ -143,14 +143,14 @@ proc downloadBytes*( success await response.getBodyBytes() proc download*( - client: CodexClient, cid: Cid, local = false + client: StorageClient, cid: Cid, local = false ): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = without response =? await client.downloadBytes(cid, local = local), err: return failure(err) return success bytesToString(response) proc downloadNoStream*( - client: CodexClient, cid: Cid + client: StorageClient, cid: Cid ): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.post(client.baseurl & "/data/" & $cid & "/network") @@ -160,7 +160,7 @@ proc downloadNoStream*( success await response.body proc downloadManifestOnly*( - client: CodexClient, cid: Cid + client: StorageClient, cid: Cid ): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.get(client.baseurl & "/data/" & $cid & "/network/manifest") @@ -171,14 +171,14 @@ proc downloadManifestOnly*( success await response.body proc deleteRaw*( - client: CodexClient, cid: string + client: StorageClient, cid: string ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = return client.delete(client.baseurl & "/data/" & cid) proc delete*( - client: CodexClient, cid: Cid + client: StorageClient, cid: Cid ): Future[?!void] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.deleteRaw($cid) @@ -188,14 +188,14 @@ proc delete*( success() proc listRaw*( - client: CodexClient + client: StorageClient ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = return client.get(client.baseurl & "/data") proc list*( - client: CodexClient + client: StorageClient ): Future[?!RestContentList] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.listRaw() @@ -205,7 +205,7 @@ proc list*( RestContentList.fromJson(await response.body) proc space*( - client: CodexClient + client: StorageClient ): Future[?!RestRepoStore] {.async: (raises: [CancelledError, HttpError]).} = let url = client.baseurl & "/space" let response = await client.get(url) @@ -215,11 +215,11 @@ proc space*( RestRepoStore.fromJson(await response.body) -proc buildUrl*(client: CodexClient, path: string): string = +proc buildUrl*(client: StorageClient, path: string): string = return client.baseurl & path proc hasBlock*( - client: CodexClient, cid: Cid + client: StorageClient, cid: Cid ): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} = let url = client.baseurl & "/data/" & $cid & "/exists" let body = await client.getContent(url) @@ -229,7 +229,7 @@ proc hasBlock*( return response.get.has.success proc hasBlockRaw*( - client: CodexClient, cid: string + client: StorageClient, cid: string ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = diff --git a/tests/integration/codexconfig.nim b/tests/integration/storageconfig.nim similarity index 56% rename from tests/integration/codexconfig.nim rename to tests/integration/storageconfig.nim index 01e6d1c9..4aeb6d60 100644 --- a/tests/integration/codexconfig.nim +++ b/tests/integration/storageconfig.nim @@ -5,8 +5,8 @@ import std/strutils import std/sugar import std/tables from pkg/chronicles import LogLevel -import pkg/codex/conf -import pkg/codex/units +import pkg/storage/conf +import pkg/storage/units import pkg/confutils import pkg/confutils/defs import libp2p except setup @@ -17,39 +17,41 @@ export clioption export confutils type - CodexConfigs* = object - configs*: seq[CodexConfig] + StorageConfigs* = object + configs*: seq[StorageConfig] - CodexConfig* = object + StorageConfig* = object cliOptions: Table[StartUpCmd, Table[string, CliOption]] debugEnabled*: bool - CodexConfigError* = object of CatchableError + StorageConfigError* = object of CatchableError -proc cliArgs*(config: CodexConfig): seq[string] {.gcsafe, raises: [CodexConfigError].} +proc cliArgs*( + config: StorageConfig +): seq[string] {.gcsafe, raises: [StorageConfigError].} -proc raiseCodexConfigError(msg: string) {.raises: [CodexConfigError].} = - raise newException(CodexConfigError, msg) +proc raiseStorageConfigError(msg: string) {.raises: [StorageConfigError].} = + raise newException(StorageConfigError, msg) template convertError(body) = try: body except CatchableError as e: - raiseCodexConfigError e.msg + raiseStorageConfigError e.msg -proc init*(_: type CodexConfigs, nodes = 1): CodexConfigs {.raises: [].} = - CodexConfigs(configs: newSeq[CodexConfig](nodes)) +proc init*(_: type StorageConfigs, nodes = 1): StorageConfigs {.raises: [].} = + StorageConfigs(configs: newSeq[StorageConfig](nodes)) -func nodes*(self: CodexConfigs): int = +func nodes*(self: StorageConfigs): int = self.configs.len -proc checkBounds(self: CodexConfigs, idx: int) {.raises: [CodexConfigError].} = +proc checkBounds(self: StorageConfigs, idx: int) {.raises: [StorageConfigError].} = if idx notin 0 ..< self.configs.len: - raiseCodexConfigError "index must be in bounds of the number of nodes" + raiseStorageConfigError "index must be in bounds of the number of nodes" proc buildConfig( - config: CodexConfig, msg: string -): CodexConf {.raises: [CodexConfigError].} = + config: StorageConfig, msg: string +): StorageConf {.raises: [StorageConfigError].} = proc postFix(msg: string): string = if msg.len > 0: ": " & msg @@ -57,38 +59,40 @@ proc buildConfig( "" try: - return CodexConf.load(cmdLine = config.cliArgs, quitOnFailure = false) + return StorageConf.load(cmdLine = config.cliArgs, quitOnFailure = false) except ConfigurationError as e: - raiseCodexConfigError msg & e.msg.postFix + raiseStorageConfigError msg & e.msg.postFix except Exception as e: ## TODO: remove once proper exception handling added to nim-confutils - raiseCodexConfigError msg & e.msg.postFix + raiseStorageConfigError msg & e.msg.postFix proc addCliOption*( - config: var CodexConfig, group = StartUpCmd.noCmd, cliOption: CliOption -) {.raises: [CodexConfigError].} = + config: var StorageConfig, group = StartUpCmd.noCmd, cliOption: CliOption +) {.raises: [StorageConfigError].} = var options = config.cliOptions.getOrDefault(group) options[cliOption.key] = cliOption # overwrite if already exists config.cliOptions[group] = options discard config.buildConfig("Invalid cli arg " & $cliOption) proc addCliOption*( - config: var CodexConfig, group = StartUpCmd.noCmd, key: string, value = "" -) {.raises: [CodexConfigError].} = + config: var StorageConfig, group = StartUpCmd.noCmd, key: string, value = "" +) {.raises: [StorageConfigError].} = config.addCliOption(group, CliOption(key: key, value: value)) proc addCliOption*( - config: var CodexConfig, cliOption: CliOption -) {.raises: [CodexConfigError].} = + config: var StorageConfig, cliOption: CliOption +) {.raises: [StorageConfigError].} = config.addCliOption(StartUpCmd.noCmd, cliOption) proc addCliOption*( - config: var CodexConfig, key: string, value = "" -) {.raises: [CodexConfigError].} = + config: var StorageConfig, key: string, value = "" +) {.raises: [StorageConfigError].} = config.addCliOption(StartUpCmd.noCmd, CliOption(key: key, value: value)) -proc cliArgs*(config: CodexConfig): seq[string] {.gcsafe, raises: [CodexConfigError].} = - ## converts CodexConfig cli options and command groups in a sequence of args +proc cliArgs*( + config: StorageConfig +): seq[string] {.gcsafe, raises: [StorageConfigError].} = + ## converts StorageConfig cli options and command groups in a sequence of args ## and filters out cli options by node index if provided in the CliOption var args: seq[string] = @[] @@ -102,18 +106,18 @@ proc cliArgs*(config: CodexConfig): seq[string] {.gcsafe, raises: [CodexConfigEr return args -proc logFile*(config: CodexConfig): ?string {.raises: [CodexConfigError].} = - let built = config.buildConfig("Invalid codex config cli params") +proc logFile*(config: StorageConfig): ?string {.raises: [StorageConfigError].} = + let built = config.buildConfig("Invalid storage config cli params") built.logFile -proc logLevel*(config: CodexConfig): LogLevel {.raises: [CodexConfigError].} = +proc logLevel*(config: StorageConfig): LogLevel {.raises: [StorageConfigError].} = convertError: - let built = config.buildConfig("Invalid codex config cli params") + let built = config.buildConfig("Invalid storage config cli params") return parseEnum[LogLevel](built.logLevel.toUpperAscii) proc debug*( - self: CodexConfigs, idx: int, enabled = true -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, enabled = true +): StorageConfigs {.raises: [StorageConfigError].} = ## output log in stdout for a specific node in the group self.checkBounds idx @@ -122,7 +126,7 @@ proc debug*( startConfig.configs[idx].debugEnabled = enabled return startConfig -proc debug*(self: CodexConfigs, enabled = true): CodexConfigs {.raises: [].} = +proc debug*(self: StorageConfigs, enabled = true): StorageConfigs {.raises: [].} = ## output log in stdout for all nodes in group var startConfig = self for config in startConfig.configs.mitems: @@ -130,15 +134,17 @@ proc debug*(self: CodexConfigs, enabled = true): CodexConfigs {.raises: [].} = return startConfig proc withLogFile*( - self: CodexConfigs, idx: int -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self startConfig.configs[idx].addCliOption("--log-file", "") return startConfig -proc withLogFile*(self: CodexConfigs): CodexConfigs {.raises: [CodexConfigError].} = +proc withLogFile*( + self: StorageConfigs +): StorageConfigs {.raises: [StorageConfigError].} = ## typically called from test, sets config such that a log file should be ## created var startConfig = self @@ -147,9 +153,9 @@ proc withLogFile*(self: CodexConfigs): CodexConfigs {.raises: [CodexConfigError] return startConfig proc withLogFile*( - self: var CodexConfig, logFile: string -) {.raises: [CodexConfigError].} = - #: CodexConfigs = + self: var StorageConfig, logFile: string +) {.raises: [StorageConfigError].} = + #: StorageConfigs = ## typically called internally from the test suite, sets a log file path to ## be created during the test run, for a specified node in the group # var config = self @@ -157,15 +163,15 @@ proc withLogFile*( # return startConfig proc withLogLevel*( - self: CodexConfig, level: LogLevel | string -): CodexConfig {.raises: [CodexConfigError].} = + self: StorageConfig, level: LogLevel | string +): StorageConfig {.raises: [StorageConfigError].} = var config = self config.addCliOption("--log-level", $level) return config proc withLogLevel*( - self: CodexConfigs, idx: int, level: LogLevel | string -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, level: LogLevel | string +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self @@ -173,23 +179,23 @@ proc withLogLevel*( return startConfig proc withLogLevel*( - self: CodexConfigs, level: LogLevel | string -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, level: LogLevel | string +): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: config.addCliOption("--log-level", $level) return startConfig proc withBlockTtl*( - self: CodexConfig, ttl: int -): CodexConfig {.raises: [CodexConfigError].} = + self: StorageConfig, ttl: int +): StorageConfig {.raises: [StorageConfigError].} = var config = self config.addCliOption("--block-ttl", $ttl) return config proc withBlockTtl*( - self: CodexConfigs, idx: int, ttl: int -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, ttl: int +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self @@ -197,23 +203,23 @@ proc withBlockTtl*( return startConfig proc withBlockTtl*( - self: CodexConfigs, ttl: int -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, ttl: int +): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: config.addCliOption("--block-ttl", $ttl) return startConfig proc withBlockMaintenanceInterval*( - self: CodexConfig, interval: int -): CodexConfig {.raises: [CodexConfigError].} = + self: StorageConfig, interval: int +): StorageConfig {.raises: [StorageConfigError].} = var config = self config.addCliOption("--block-mi", $interval) return config proc withBlockMaintenanceInterval*( - self: CodexConfigs, idx: int, interval: int -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, interval: int +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self @@ -221,26 +227,26 @@ proc withBlockMaintenanceInterval*( return startConfig proc withBlockMaintenanceInterval*( - self: CodexConfigs, interval: int -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, interval: int +): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: config.addCliOption("--block-mi", $interval) return startConfig proc logLevelWithTopics( - config: CodexConfig, topics: varargs[string] -): string {.raises: [CodexConfigError].} = + config: StorageConfig, topics: varargs[string] +): string {.raises: [StorageConfigError].} = convertError: var logLevel = LogLevel.INFO - let built = config.buildConfig("Invalid codex config cli params") + let built = config.buildConfig("Invalid storage config cli params") logLevel = parseEnum[LogLevel](built.logLevel.toUpperAscii) let level = $logLevel & ";TRACE: " & topics.join(",") return level proc withLogTopics*( - self: CodexConfigs, idx: int, topics: varargs[string] -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, topics: varargs[string] +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx convertError: @@ -250,8 +256,8 @@ proc withLogTopics*( return startConfig.withLogLevel(idx, level) proc withLogTopics*( - self: CodexConfigs, topics: varargs[string] -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, topics: varargs[string] +): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: let level = config.logLevelWithTopics(topics) @@ -259,8 +265,8 @@ proc withLogTopics*( return startConfig proc withStorageQuota*( - self: CodexConfigs, idx: int, quota: NBytes -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, quota: NBytes +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self @@ -268,8 +274,8 @@ proc withStorageQuota*( return startConfig proc withStorageQuota*( - self: CodexConfigs, quota: NBytes -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, quota: NBytes +): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: config.addCliOption("--storage-quota", $quota) diff --git a/tests/integration/codexprocess.nim b/tests/integration/storageprocess.nim similarity index 53% rename from tests/integration/codexprocess.nim rename to tests/integration/storageprocess.nim index 0d2df752..ec58d273 100644 --- a/tests/integration/codexprocess.nim +++ b/tests/integration/storageprocess.nim @@ -7,94 +7,94 @@ import pkg/libp2p import std/os import std/strutils import std/times -import codex/conf -import ./codexclient +import storage/conf +import ./storageclient import ./nodeprocess -export codexclient +export storageclient export chronicles export nodeprocess {.push raises: [].} logScope: - topics = "integration testing codex process" + topics = "integration testing storage process" type - CodexProcess* = ref object of NodeProcess - client: ?CodexClient + StorageProcess* = ref object of NodeProcess + client: ?StorageClient - CodexProcessError* = object of NodeProcessError + StorageProcessError* = object of NodeProcessError -proc raiseCodexProcessError( +proc raiseStorageProcessError( msg: string, parent: ref CatchableError -) {.raises: [CodexProcessError].} = - raise newException(CodexProcessError, msg & ": " & parent.msg, parent) +) {.raises: [StorageProcessError].} = + raise newException(StorageProcessError, msg & ": " & parent.msg, parent) template convertError(msg, body: typed) = # Don't use this in an async proc, unless body does not raise CancelledError try: body except CatchableError as parent: - raiseCodexProcessError(msg, parent) + raiseStorageProcessError(msg, parent) -method workingDir(node: CodexProcess): string = +method workingDir(node: StorageProcess): string = return currentSourcePath() / ".." / ".." / ".." -method executable(node: CodexProcess): string = +method executable(node: StorageProcess): string = return "build" / "storage" -method startedOutput(node: CodexProcess): string = +method startedOutput(node: StorageProcess): string = return "REST service started" -method processOptions(node: CodexProcess): set[AsyncProcessOption] = +method processOptions(node: StorageProcess): set[AsyncProcessOption] = return {AsyncProcessOption.StdErrToStdOut} -method outputLineEndings(node: CodexProcess): string = +method outputLineEndings(node: StorageProcess): string = return "\n" -method onOutputLineCaptured(node: CodexProcess, line: string) = +method onOutputLineCaptured(node: StorageProcess, line: string) = discard -proc config(node: CodexProcess): CodexConf {.raises: [CodexProcessError].} = +proc config(node: StorageProcess): StorageConf {.raises: [StorageProcessError].} = # cannot use convertError here as it uses typed parameters which forces type # resolution, while confutils.load uses untyped parameters and expects type # resolution not to happen yet. In other words, it won't compile. try: - return CodexConf.load( + return StorageConf.load( cmdLine = node.arguments, quitOnFailure = false, secondarySources = nil ) except ConfigurationError as parent: - raiseCodexProcessError "Failed to load node arguments into CodexConf", parent + raiseStorageProcessError "Failed to load node arguments into StorageConf", parent -proc dataDir(node: CodexProcess): string {.raises: [CodexProcessError].} = +proc dataDir(node: StorageProcess): string {.raises: [StorageProcessError].} = return node.config.dataDir.string -proc apiUrl*(node: CodexProcess): string {.raises: [CodexProcessError].} = +proc apiUrl*(node: StorageProcess): string {.raises: [StorageProcessError].} = let config = node.config without apiBindAddress =? config.apiBindAddress: raise - newException(CodexProcessError, "REST API not started: --api-bindaddr not set") + newException(StorageProcessError, "REST API not started: --api-bindaddr not set") return "http://" & apiBindAddress & ":" & $config.apiPort & "/api/storage/v1" -proc logFile*(node: CodexProcess): ?string {.raises: [CodexProcessError].} = +proc logFile*(node: StorageProcess): ?string {.raises: [StorageProcessError].} = node.config.logFile -proc client*(node: CodexProcess): CodexClient {.raises: [CodexProcessError].} = +proc client*(node: StorageProcess): StorageClient {.raises: [StorageProcessError].} = if client =? node.client: return client - let client = CodexClient.new(node.apiUrl) + let client = StorageClient.new(node.apiUrl) node.client = some client return client -proc updateLogFile(node: CodexProcess, newLogFile: string) = +proc updateLogFile(node: StorageProcess, newLogFile: string) = for arg in node.arguments.mitems: if arg.startsWith("--log-file="): arg = "--log-file=" & newLogFile break -method restart*(node: CodexProcess) {.async.} = - trace "restarting codex" +method restart*(node: StorageProcess) {.async.} = + trace "restarting storage" await node.stop() if logFile =? node.logFile: # chronicles truncates the existing log file on start, so changed the log @@ -104,13 +104,13 @@ method restart*(node: CodexProcess) {.async.} = ) await node.start() await node.waitUntilStarted() - trace "codex process restarted" + trace "storage process restarted" -method stop*(node: CodexProcess) {.async: (raises: []).} = +method stop*(node: StorageProcess) {.async: (raises: []).} = logScope: nodeName = node.name - trace "stopping codex client" + trace "stopping storage client" await procCall NodeProcess(node).stop() if not node.process.isNil: @@ -120,8 +120,8 @@ method stop*(node: CodexProcess) {.async: (raises: []).} = if client =? node.client: await client.close() - node.client = none CodexClient + node.client = none StorageClient -method removeDataDir*(node: CodexProcess) {.raises: [CodexProcessError].} = - convertError("failed to remove codex node data directory"): +method removeDataDir*(node: StorageProcess) {.raises: [StorageProcessError].} = + convertError("failed to remove storage node data directory"): removeDir(node.dataDir) diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index f9eb5d37..7fa55244 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -1,23 +1,23 @@ import std/macros import pkg/questionable import ./multinodes -import ./codexconfig -import ./codexprocess -import ./codexclient +import ./storageconfig +import ./storageprocess +import ./storageclient import ./nodeconfigs -export codexclient +export storageclient export multinodes template twonodessuite*(name: string, body: untyped) = multinodesuite name: let twoNodesConfig {.inject, used.} = - NodeConfigs(clients: CodexConfigs.init(nodes = 2).some) + NodeConfigs(clients: StorageConfigs.init(nodes = 2).some) - var node1 {.inject, used.}: CodexProcess - var node2 {.inject, used.}: CodexProcess - var client1 {.inject, used.}: CodexClient - var client2 {.inject, used.}: CodexClient + var node1 {.inject, used.}: StorageProcess + var node2 {.inject, used.}: StorageProcess + var client1 {.inject, used.}: StorageClient + var client2 {.inject, used.}: StorageClient setup: node1 = clients()[0] diff --git a/tests/integration/utils.nim b/tests/integration/utils.nim index 3e522a04..7d48d5d5 100644 --- a/tests/integration/utils.nim +++ b/tests/integration/utils.nim @@ -2,7 +2,7 @@ import std/os import std/strformat import pkg/chronos import pkg/chronos/asyncproc -import pkg/codex/logutils +import pkg/storage/logutils {.push raises: [].} @@ -88,5 +88,5 @@ proc getDataDir*(testId, testName, startTime, role: string, index = int.none): s if idx =? index: suffix &= "_" & $idx - getTempDir() / "Codex" / sanitize(testId) / sanitize(testName) / sanitize(startTime) / + getTempDir() / "Storage" / sanitize(testId) / sanitize(testName) / sanitize(startTime) / sanitize(suffix) diff --git a/tests/logging.nim b/tests/logging.nim index d165fe48..30f91e6e 100644 --- a/tests/logging.nim +++ b/tests/logging.nim @@ -1,5 +1,5 @@ when not defined(nimscript): - import pkg/codex/logutils + import pkg/storage/logutils proc ignoreLogging(level: LogLevel, message: LogOutputStr) = discard diff --git a/tests/codex/blockexchange/discovery/testdiscovery.nim b/tests/storage/blockexchange/discovery/testdiscovery.nim similarity index 98% rename from tests/codex/blockexchange/discovery/testdiscovery.nim rename to tests/storage/blockexchange/discovery/testdiscovery.nim index b8aec57b..ab237542 100644 --- a/tests/codex/blockexchange/discovery/testdiscovery.nim +++ b/tests/storage/blockexchange/discovery/testdiscovery.nim @@ -6,13 +6,13 @@ import pkg/chronos import pkg/libp2p/errors -import pkg/codex/rng -import pkg/codex/stores -import pkg/codex/blockexchange -import pkg/codex/chunker -import pkg/codex/manifest -import pkg/codex/merkletree -import pkg/codex/blocktype as bt +import pkg/storage/rng +import pkg/storage/stores +import pkg/storage/blockexchange +import pkg/storage/chunker +import pkg/storage/manifest +import pkg/storage/merkletree +import pkg/storage/blocktype as bt import ../../../asynctest import ../../helpers diff --git a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim b/tests/storage/blockexchange/discovery/testdiscoveryengine.nim similarity index 95% rename from tests/codex/blockexchange/discovery/testdiscoveryengine.nim rename to tests/storage/blockexchange/discovery/testdiscoveryengine.nim index 978fe690..3bd940a0 100644 --- a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/storage/blockexchange/discovery/testdiscoveryengine.nim @@ -3,14 +3,14 @@ import std/tables import pkg/chronos -import pkg/codex/rng -import pkg/codex/stores -import pkg/codex/blockexchange -import pkg/codex/chunker -import pkg/codex/blocktype as bt -import pkg/codex/blockexchange/engine -import pkg/codex/manifest -import pkg/codex/merkletree +import pkg/storage/rng +import pkg/storage/stores +import pkg/storage/blockexchange +import pkg/storage/chunker +import pkg/storage/blocktype as bt +import pkg/storage/blockexchange/engine +import pkg/storage/manifest +import pkg/storage/merkletree import ../../../asynctest import ../../helpers diff --git a/tests/codex/blockexchange/engine/testadvertiser.nim b/tests/storage/blockexchange/engine/testadvertiser.nim similarity index 93% rename from tests/codex/blockexchange/engine/testadvertiser.nim rename to tests/storage/blockexchange/engine/testadvertiser.nim index 83a70f65..c39d35c1 100644 --- a/tests/codex/blockexchange/engine/testadvertiser.nim +++ b/tests/storage/blockexchange/engine/testadvertiser.nim @@ -2,12 +2,12 @@ import pkg/chronos import pkg/libp2p/routing_record import pkg/codexdht/discv5/protocol as discv5 -import pkg/codex/blockexchange -import pkg/codex/stores -import pkg/codex/chunker -import pkg/codex/discovery -import pkg/codex/blocktype as bt -import pkg/codex/manifest +import pkg/storage/blockexchange +import pkg/storage/stores +import pkg/storage/chunker +import pkg/storage/discovery +import pkg/storage/blocktype as bt +import pkg/storage/manifest import ../../../asynctest import ../../helpers diff --git a/tests/codex/blockexchange/engine/testblockexc.nim b/tests/storage/blockexchange/engine/testblockexc.nim similarity index 97% rename from tests/codex/blockexchange/engine/testblockexc.nim rename to tests/storage/blockexchange/engine/testblockexc.nim index 4b0fe173..515236bf 100644 --- a/tests/codex/blockexchange/engine/testblockexc.nim +++ b/tests/storage/blockexchange/engine/testblockexc.nim @@ -5,11 +5,11 @@ import std/importutils import pkg/chronos import pkg/stew/byteutils -import pkg/codex/stores -import pkg/codex/blockexchange -import pkg/codex/chunker -import pkg/codex/discovery -import pkg/codex/blocktype as bt +import pkg/storage/stores +import pkg/storage/blockexchange +import pkg/storage/chunker +import pkg/storage/discovery +import pkg/storage/blocktype as bt import ../../../asynctest import ../../examples diff --git a/tests/codex/blockexchange/engine/testengine.nim b/tests/storage/blockexchange/engine/testengine.nim similarity index 98% rename from tests/codex/blockexchange/engine/testengine.nim rename to tests/storage/blockexchange/engine/testengine.nim index 54f16c8d..b04fb29d 100644 --- a/tests/codex/blockexchange/engine/testengine.nim +++ b/tests/storage/blockexchange/engine/testengine.nim @@ -8,13 +8,13 @@ import pkg/libp2p/errors import pkg/libp2p/routing_record import pkg/codexdht/discv5/protocol as discv5 -import pkg/codex/rng -import pkg/codex/blockexchange -import pkg/codex/stores -import pkg/codex/chunker -import pkg/codex/discovery -import pkg/codex/blocktype -import pkg/codex/utils/asyncheapqueue +import pkg/storage/rng +import pkg/storage/blockexchange +import pkg/storage/stores +import pkg/storage/chunker +import pkg/storage/discovery +import pkg/storage/blocktype +import pkg/storage/utils/asyncheapqueue import ../../../asynctest import ../../helpers diff --git a/tests/codex/blockexchange/protobuf/testpresence.nim b/tests/storage/blockexchange/protobuf/testpresence.nim similarity index 94% rename from tests/codex/blockexchange/protobuf/testpresence.nim rename to tests/storage/blockexchange/protobuf/testpresence.nim index 2f449e15..4bef0448 100644 --- a/tests/codex/blockexchange/protobuf/testpresence.nim +++ b/tests/storage/blockexchange/protobuf/testpresence.nim @@ -1,6 +1,6 @@ import pkg/chronos -import pkg/codex/blockexchange/protobuf/presence +import pkg/storage/blockexchange/protobuf/presence import ../../../asynctest import ../../examples diff --git a/tests/codex/blockexchange/testdiscovery.nim b/tests/storage/blockexchange/testdiscovery.nim similarity index 100% rename from tests/codex/blockexchange/testdiscovery.nim rename to tests/storage/blockexchange/testdiscovery.nim diff --git a/tests/codex/blockexchange/testengine.nim b/tests/storage/blockexchange/testengine.nim similarity index 100% rename from tests/codex/blockexchange/testengine.nim rename to tests/storage/blockexchange/testengine.nim diff --git a/tests/codex/blockexchange/testnetwork.nim b/tests/storage/blockexchange/testnetwork.nim similarity index 98% rename from tests/codex/blockexchange/testnetwork.nim rename to tests/storage/blockexchange/testnetwork.nim index 6af156a9..0ed8b41a 100644 --- a/tests/codex/blockexchange/testnetwork.nim +++ b/tests/storage/blockexchange/testnetwork.nim @@ -3,10 +3,10 @@ import std/tables import pkg/chronos -import pkg/codex/rng -import pkg/codex/chunker -import pkg/codex/blocktype as bt -import pkg/codex/blockexchange +import pkg/storage/rng +import pkg/storage/chunker +import pkg/storage/blocktype as bt +import pkg/storage/blockexchange import ../../asynctest import ../examples diff --git a/tests/codex/blockexchange/testpeerctxstore.nim b/tests/storage/blockexchange/testpeerctxstore.nim similarity index 94% rename from tests/codex/blockexchange/testpeerctxstore.nim rename to tests/storage/blockexchange/testpeerctxstore.nim index 880e08c3..6150bb9f 100644 --- a/tests/codex/blockexchange/testpeerctxstore.nim +++ b/tests/storage/blockexchange/testpeerctxstore.nim @@ -4,9 +4,9 @@ import std/sequtils import pkg/unittest2 import pkg/libp2p -import pkg/codex/blockexchange/peers -import pkg/codex/blockexchange/protobuf/blockexc -import pkg/codex/blockexchange/protobuf/presence +import pkg/storage/blockexchange/peers +import pkg/storage/blockexchange/protobuf/blockexc +import pkg/storage/blockexchange/protobuf/presence import ../helpers import ../examples diff --git a/tests/codex/blockexchange/testpendingblocks.nim b/tests/storage/blockexchange/testpendingblocks.nim similarity index 97% rename from tests/codex/blockexchange/testpendingblocks.nim rename to tests/storage/blockexchange/testpendingblocks.nim index af1e6728..eba4ead1 100644 --- a/tests/codex/blockexchange/testpendingblocks.nim +++ b/tests/storage/blockexchange/testpendingblocks.nim @@ -4,8 +4,8 @@ import std/algorithm import pkg/chronos import pkg/stew/byteutils -import pkg/codex/blocktype as bt -import pkg/codex/blockexchange +import pkg/storage/blocktype as bt +import pkg/storage/blockexchange import ../helpers import ../../asynctest diff --git a/tests/codex/blockexchange/testprotobuf.nim b/tests/storage/blockexchange/testprotobuf.nim similarity index 100% rename from tests/codex/blockexchange/testprotobuf.nim rename to tests/storage/blockexchange/testprotobuf.nim diff --git a/tests/codex/examples.nim b/tests/storage/examples.nim similarity index 88% rename from tests/codex/examples.nim rename to tests/storage/examples.nim index 4c9535cb..43512f8d 100644 --- a/tests/codex/examples.nim +++ b/tests/storage/examples.nim @@ -2,11 +2,11 @@ import std/random import std/sequtils import pkg/libp2p import pkg/stint -import pkg/codex/rng -import pkg/codex/stores -import pkg/codex/blocktype as bt -import pkg/codex/merkletree -import pkg/codex/manifest +import pkg/storage/rng +import pkg/storage/stores +import pkg/storage/blocktype as bt +import pkg/storage/merkletree +import pkg/storage/manifest import ../examples export examples diff --git a/tests/codex/helpers.nim b/tests/storage/helpers.nim similarity index 93% rename from tests/codex/helpers.nim rename to tests/storage/helpers.nim index b855f412..28932fd8 100644 --- a/tests/codex/helpers.nim +++ b/tests/storage/helpers.nim @@ -3,13 +3,13 @@ import std/sequtils import pkg/chronos import pkg/libp2p import pkg/libp2p/varint -import pkg/codex/blocktype -import pkg/codex/stores -import pkg/codex/manifest -import pkg/codex/merkletree -import pkg/codex/blockexchange -import pkg/codex/rng -import pkg/codex/utils +import pkg/storage/blocktype +import pkg/storage/stores +import pkg/storage/manifest +import pkg/storage/merkletree +import pkg/storage/blockexchange +import pkg/storage/rng +import pkg/storage/utils import ./helpers/nodeutils import ./helpers/datasetutils diff --git a/tests/codex/helpers/always.nim b/tests/storage/helpers/always.nim similarity index 100% rename from tests/codex/helpers/always.nim rename to tests/storage/helpers/always.nim diff --git a/tests/codex/helpers/datasetutils.nim b/tests/storage/helpers/datasetutils.nim similarity index 90% rename from tests/codex/helpers/datasetutils.nim rename to tests/storage/helpers/datasetutils.nim index d4aedc41..078a08d9 100644 --- a/tests/codex/helpers/datasetutils.nim +++ b/tests/storage/helpers/datasetutils.nim @@ -1,10 +1,10 @@ import std/random import pkg/chronos -import pkg/codex/blocktype as bt -import pkg/codex/merkletree -import pkg/codex/manifest -import pkg/codex/rng +import pkg/storage/blocktype as bt +import pkg/storage/merkletree +import pkg/storage/manifest +import pkg/storage/rng import ./randomchunker diff --git a/tests/codex/helpers/mockchunker.nim b/tests/storage/helpers/mockchunker.nim similarity index 96% rename from tests/codex/helpers/mockchunker.nim rename to tests/storage/helpers/mockchunker.nim index 1ecd8a21..cba8d0e9 100644 --- a/tests/codex/helpers/mockchunker.nim +++ b/tests/storage/helpers/mockchunker.nim @@ -1,5 +1,5 @@ import pkg/chronos -import pkg/codex/chunker +import pkg/storage/chunker export chunker diff --git a/tests/codex/helpers/mockclock.nim b/tests/storage/helpers/mockclock.nim similarity index 98% rename from tests/codex/helpers/mockclock.nim rename to tests/storage/helpers/mockclock.nim index 76446041..20a71918 100644 --- a/tests/codex/helpers/mockclock.nim +++ b/tests/storage/helpers/mockclock.nim @@ -1,6 +1,6 @@ import std/times import pkg/chronos -import codex/clock +import storage/clock export clock diff --git a/tests/codex/helpers/mockdiscovery.nim b/tests/storage/helpers/mockdiscovery.nim similarity index 99% rename from tests/codex/helpers/mockdiscovery.nim rename to tests/storage/helpers/mockdiscovery.nim index f0142650..8ed61979 100644 --- a/tests/codex/helpers/mockdiscovery.nim +++ b/tests/storage/helpers/mockdiscovery.nim @@ -10,7 +10,7 @@ import pkg/chronos import pkg/libp2p import pkg/questionable -import pkg/codex/discovery +import pkg/storage/discovery import pkg/contractabi/address as ca type MockDiscovery* = ref object of Discovery diff --git a/tests/codex/helpers/mockrepostore.nim b/tests/storage/helpers/mockrepostore.nim similarity index 93% rename from tests/codex/helpers/mockrepostore.nim rename to tests/storage/helpers/mockrepostore.nim index 3d29f4a3..cc38d55d 100644 --- a/tests/codex/helpers/mockrepostore.nim +++ b/tests/storage/helpers/mockrepostore.nim @@ -13,9 +13,9 @@ import pkg/libp2p import pkg/questionable import pkg/questionable/results -import pkg/codex/stores/repostore -import pkg/codex/utils/asynciter -import pkg/codex/utils/safeasynciter +import pkg/storage/stores/repostore +import pkg/storage/utils/asynciter +import pkg/storage/utils/safeasynciter type MockRepoStore* = ref object of RepoStore delBlockCids*: seq[Cid] diff --git a/tests/codex/helpers/mocktimer.nim b/tests/storage/helpers/mocktimer.nim similarity index 97% rename from tests/codex/helpers/mocktimer.nim rename to tests/storage/helpers/mocktimer.nim index 162a6e81..63a22a86 100644 --- a/tests/codex/helpers/mocktimer.nim +++ b/tests/storage/helpers/mocktimer.nim @@ -9,7 +9,7 @@ import pkg/chronos -import codex/utils/timer +import storage/utils/timer type MockTimer* = ref object of Timer startCalled*: int diff --git a/tests/codex/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim similarity index 94% rename from tests/codex/helpers/nodeutils.nim rename to tests/storage/helpers/nodeutils.nim index 55ab984b..5ab198cd 100644 --- a/tests/codex/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -6,18 +6,18 @@ import pkg/taskpools import pkg/libp2p import pkg/libp2p/errors -import pkg/codex/discovery -import pkg/codex/stores -import pkg/codex/blocktype as bt -import pkg/codex/blockexchange -import pkg/codex/systemclock -import pkg/codex/nat -import pkg/codex/utils/natutils -import pkg/codex/utils/safeasynciter -import pkg/codex/merkletree -import pkg/codex/manifest +import pkg/storage/discovery +import pkg/storage/stores +import pkg/storage/blocktype as bt +import pkg/storage/blockexchange +import pkg/storage/systemclock +import pkg/storage/nat +import pkg/storage/utils/natutils +import pkg/storage/utils/safeasynciter +import pkg/storage/merkletree +import pkg/storage/manifest -import pkg/codex/node +import pkg/storage/node import ./datasetutils import ./mockdiscovery @@ -49,7 +49,7 @@ type discovery*: DiscoveryEngine engine*: BlockExcEngine networkStore*: NetworkStore - node*: CodexNodeRef = nil + node*: StorageNodeRef = nil tempDbs*: seq[TempLevelDb] = @[] NodesCluster* = ref object @@ -85,7 +85,7 @@ converter toTuple*( converter toComponents*(cluster: NodesCluster): seq[NodesComponents] = cluster.components -proc nodes*(cluster: NodesCluster): seq[CodexNodeRef] = +proc nodes*(cluster: NodesCluster): seq[StorageNodeRef] = cluster.components.filterIt(it.node != nil).mapIt(it.node) proc localStores*(cluster: NodesCluster): seq[BlockStore] = @@ -209,7 +209,7 @@ proc generateNodes*( let node = if config.createFullNode: - let fullNode = CodexNodeRef.new( + let fullNode = StorageNodeRef.new( switch = switch, networkStore = networkStore, engine = engine, @@ -321,7 +321,7 @@ proc linearTopology*(nodes: seq[NodesComponents]) {.async.} = proc downloadDataset*( node: NodesComponents, dataset: TestDataset ): Future[void] {.async.} = - # This is the same as fetchBatched, but we don't construct CodexNodes so I can't use + # This is the same as fetchBatched, but we don't construct StorageNodes so I can't use # it here. let requestAddresses = collect: for i in 0 ..< dataset.manifest.blocksCount: diff --git a/tests/codex/helpers/randomchunker.nim b/tests/storage/helpers/randomchunker.nim similarity index 94% rename from tests/codex/helpers/randomchunker.nim rename to tests/storage/helpers/randomchunker.nim index d1383e84..8ee1ba45 100644 --- a/tests/codex/helpers/randomchunker.nim +++ b/tests/storage/helpers/randomchunker.nim @@ -2,8 +2,8 @@ import std/sequtils import pkg/chronos -import pkg/codex/chunker -import pkg/codex/rng +import pkg/storage/chunker +import pkg/storage/rng export chunker diff --git a/tests/codex/merkletree/helpers.nim b/tests/storage/merkletree/helpers.nim similarity index 91% rename from tests/codex/merkletree/helpers.nim rename to tests/storage/merkletree/helpers.nim index c2361b9d..c8fef2f0 100644 --- a/tests/codex/merkletree/helpers.nim +++ b/tests/storage/merkletree/helpers.nim @@ -1,4 +1,4 @@ -import pkg/codex/merkletree +import pkg/storage/merkletree import ../helpers export merkletree, helpers diff --git a/tests/codex/merkletree/testcodexcoders.nim b/tests/storage/merkletree/teststoragecoders.nim similarity index 97% rename from tests/codex/merkletree/testcodexcoders.nim rename to tests/storage/merkletree/teststoragecoders.nim index eddafc9e..44921ef7 100644 --- a/tests/codex/merkletree/testcodexcoders.nim +++ b/tests/storage/merkletree/teststoragecoders.nim @@ -3,7 +3,7 @@ import pkg/unittest2 import pkg/questionable/results import pkg/stew/byteutils -import pkg/codex/merkletree +import pkg/storage/merkletree import ./helpers const data = [ diff --git a/tests/codex/merkletree/testcodextree.nim b/tests/storage/merkletree/teststoragetree.nim similarity index 94% rename from tests/codex/merkletree/testcodextree.nim rename to tests/storage/merkletree/teststoragetree.nim index 15ec6ce9..ae9e035e 100644 --- a/tests/codex/merkletree/testcodextree.nim +++ b/tests/storage/merkletree/teststoragetree.nim @@ -5,9 +5,9 @@ import pkg/questionable/results import pkg/stew/byteutils import pkg/libp2p -import pkg/codex/codextypes -import pkg/codex/merkletree -import pkg/codex/utils/digest +import pkg/storage/storagetypes +import pkg/storage/merkletree +import pkg/storage/utils/digest import pkg/taskpools @@ -35,14 +35,17 @@ suite "Test StorageMerkleTree": test "Cannot init tree without any multihash leaves": check: StorageMerkleTree.init(leaves = newSeq[MultiHash]()).isErr + StorageMerkleTree.init(leaves = newSeq[MultiHash]()).isErr test "Cannot init tree without any cid leaves": check: StorageMerkleTree.init(leaves = newSeq[Cid]()).isErr + StorageMerkleTree.init(leaves = newSeq[Cid]()).isErr test "Cannot init tree without any byte leaves": check: StorageMerkleTree.init(sha256, leaves = newSeq[ByteHash]()).isErr + StorageMerkleTree.init(sha256, leaves = newSeq[ByteHash]()).isErr test "Should build tree from multihash leaves": var diff --git a/tests/codex/node/helpers.nim b/tests/storage/node/helpers.nim similarity index 95% rename from tests/codex/node/helpers.nim rename to tests/storage/node/helpers.nim index 72bb1c51..00a87c97 100644 --- a/tests/codex/node/helpers.nim +++ b/tests/storage/node/helpers.nim @@ -3,9 +3,9 @@ import std/times import pkg/libp2p import pkg/chronos -import pkg/codex/codextypes -import pkg/codex/chunker -import pkg/codex/stores +import pkg/storage/storagetypes +import pkg/storage/chunker +import pkg/storage/stores import pkg/taskpools import ../../asynctest @@ -71,7 +71,7 @@ template setupAndTearDown*() {.dirty.} = localStoreMetaDs: Datastore engine: BlockExcEngine store: NetworkStore - node: CodexNodeRef + node: StorageNodeRef blockDiscovery: Discovery peerStore: PeerCtxStore pendingBlocks: PendingBlocksManager @@ -110,7 +110,7 @@ template setupAndTearDown*() {.dirty.} = localStore, network, discovery, advertiser, peerStore, pendingBlocks ) store = NetworkStore.new(engine, localStore) - node = CodexNodeRef.new( + node = StorageNodeRef.new( switch = switch, networkStore = store, engine = engine, diff --git a/tests/codex/node/testnode.nim b/tests/storage/node/testnode.nim similarity index 93% rename from tests/codex/node/testnode.nim rename to tests/storage/node/testnode.nim index cc5690d7..389b4f8f 100644 --- a/tests/codex/node/testnode.nim +++ b/tests/storage/node/testnode.nim @@ -14,19 +14,19 @@ import pkg/taskpools import pkg/codexdht/discv5/protocol as discv5 -import pkg/codex/logutils -import pkg/codex/stores -import pkg/codex/clock -import pkg/codex/systemclock -import pkg/codex/blockexchange -import pkg/codex/chunker -import pkg/codex/manifest -import pkg/codex/discovery -import pkg/codex/merkletree -import pkg/codex/blocktype as bt -import pkg/codex/rng +import pkg/storage/logutils +import pkg/storage/stores +import pkg/storage/clock +import pkg/storage/systemclock +import pkg/storage/blockexchange +import pkg/storage/chunker +import pkg/storage/manifest +import pkg/storage/discovery +import pkg/storage/merkletree +import pkg/storage/blocktype as bt +import pkg/storage/rng -import pkg/codex/node {.all.} +import pkg/storage/node {.all.} import ../../asynctest import ../examples @@ -36,7 +36,7 @@ import ../slots/helpers import ./helpers -privateAccess(CodexNodeRef) # enable access to private fields +privateAccess(StorageNodeRef) # enable access to private fields asyncchecksuite "Test Node - Basic": setupAndTearDown() diff --git a/tests/codex/slots/helpers.nim b/tests/storage/slots/helpers.nim similarity index 91% rename from tests/codex/slots/helpers.nim rename to tests/storage/slots/helpers.nim index 17c4d401..e476cfbd 100644 --- a/tests/codex/slots/helpers.nim +++ b/tests/storage/slots/helpers.nim @@ -3,13 +3,13 @@ import std/sugar import pkg/chronos import pkg/libp2p/cid -import pkg/codex/codextypes -import pkg/codex/stores -import pkg/codex/merkletree -import pkg/codex/manifest -import pkg/codex/blocktype as bt -import pkg/codex/chunker -import pkg/codex/rng +import pkg/storage/storagetypes +import pkg/storage/stores +import pkg/storage/merkletree +import pkg/storage/manifest +import pkg/storage/blocktype as bt +import pkg/storage/chunker +import pkg/storage/rng import pkg/taskpools import ../helpers diff --git a/tests/codex/stores/commonstoretests.nim b/tests/storage/stores/commonstoretests.nim similarity index 96% rename from tests/codex/stores/commonstoretests.nim rename to tests/storage/stores/commonstoretests.nim index 03627f87..9292a9af 100644 --- a/tests/codex/stores/commonstoretests.nim +++ b/tests/storage/stores/commonstoretests.nim @@ -7,11 +7,11 @@ import pkg/libp2p/multicodec import pkg/stew/byteutils import pkg/questionable import pkg/questionable/results -import pkg/codex/stores/cachestore -import pkg/codex/chunker -import pkg/codex/manifest -import pkg/codex/merkletree -import pkg/codex/utils +import pkg/storage/stores/cachestore +import pkg/storage/chunker +import pkg/storage/manifest +import pkg/storage/merkletree +import pkg/storage/utils import ../../asynctest import ../helpers diff --git a/tests/codex/stores/repostore/testcoders.nim b/tests/storage/stores/repostore/testcoders.nim similarity index 93% rename from tests/codex/stores/repostore/testcoders.nim rename to tests/storage/stores/repostore/testcoders.nim index 26a8642d..0ac8bbd8 100644 --- a/tests/codex/stores/repostore/testcoders.nim +++ b/tests/storage/stores/repostore/testcoders.nim @@ -5,9 +5,9 @@ import pkg/stew/objects import pkg/questionable import pkg/questionable/results -import pkg/codex/clock -import pkg/codex/stores/repostore/types -import pkg/codex/stores/repostore/coders +import pkg/storage/clock +import pkg/storage/stores/repostore/types +import pkg/storage/stores/repostore/coders import ../../helpers diff --git a/tests/codex/stores/testcachestore.nim b/tests/storage/stores/testcachestore.nim similarity index 97% rename from tests/codex/stores/testcachestore.nim rename to tests/storage/stores/testcachestore.nim index 03075e1a..949272ff 100644 --- a/tests/codex/stores/testcachestore.nim +++ b/tests/storage/stores/testcachestore.nim @@ -3,8 +3,8 @@ import std/strutils import pkg/chronos import pkg/stew/byteutils import pkg/questionable/results -import pkg/codex/stores/cachestore -import pkg/codex/chunker +import pkg/storage/stores/cachestore +import pkg/storage/chunker import ./commonstoretests diff --git a/tests/codex/stores/testkeyutils.nim b/tests/storage/stores/testkeyutils.nim similarity index 87% rename from tests/codex/stores/testkeyutils.nim rename to tests/storage/stores/testkeyutils.nim index 799e1884..0f75a2ed 100644 --- a/tests/codex/stores/testkeyutils.nim +++ b/tests/storage/stores/testkeyutils.nim @@ -12,9 +12,9 @@ import std/sequtils import pkg/chronos import pkg/questionable/results import pkg/libp2p -import pkg/codex/blocktype as bt -import pkg/codex/stores/repostore -import pkg/codex/clock +import pkg/storage/blocktype as bt +import pkg/storage/stores/repostore +import pkg/storage/clock import ../../asynctest import ../helpers/mocktimer @@ -22,8 +22,8 @@ import ../helpers/mockrepostore import ../helpers/mockclock import ../examples -import codex/namespaces -import codex/stores/keyutils +import storage/namespaces +import storage/stores/keyutils proc createManifestCid(): ?!Cid = let @@ -49,7 +49,7 @@ suite "KeyUtils": check: namespaces.len == 4 - namespaces[0].value == CodexRepoNamespace + namespaces[0].value == StorageRepoNamespace namespaces[1].value == "blocks" namespaces[2].value == expectedPrefix namespaces[3].value == expectedPostfix @@ -65,7 +65,7 @@ suite "KeyUtils": check: namespaces.len == 4 - namespaces[0].value == CodexRepoNamespace + namespaces[0].value == StorageRepoNamespace namespaces[1].value == "manifests" namespaces[2].value == expectedPrefix namespaces[3].value == expectedPostfix @@ -78,7 +78,7 @@ suite "KeyUtils": check: namespaces.len == 3 - namespaces[0].value == CodexMetaNamespace + namespaces[0].value == StorageMetaNamespace namespaces[1].value == "ttl" namespaces[2].value == $cid @@ -88,6 +88,6 @@ suite "KeyUtils": check: namespaces.len == 3 - namespaces[0].value == CodexMetaNamespace + namespaces[0].value == StorageMetaNamespace namespaces[1].value == "ttl" namespaces[2].value == "*" diff --git a/tests/codex/stores/testmaintenance.nim b/tests/storage/stores/testmaintenance.nim similarity index 97% rename from tests/codex/stores/testmaintenance.nim rename to tests/storage/stores/testmaintenance.nim index 7c979d4c..520f89e5 100644 --- a/tests/codex/stores/testmaintenance.nim +++ b/tests/storage/stores/testmaintenance.nim @@ -8,9 +8,9 @@ ## those terms. import pkg/chronos -import pkg/codex/blocktype as bt -import pkg/codex/stores/repostore -import pkg/codex/clock +import pkg/storage/blocktype as bt +import pkg/storage/stores/repostore +import pkg/storage/clock import ../../asynctest import ../helpers @@ -19,7 +19,7 @@ import ../helpers/mockrepostore import ../helpers/mockclock import ../examples -import codex/stores/maintenance +import storage/stores/maintenance suite "BlockMaintainer": var mockRepoStore: MockRepoStore diff --git a/tests/codex/stores/testqueryiterhelper.nim b/tests/storage/stores/testqueryiterhelper.nim similarity index 94% rename from tests/codex/stores/testqueryiterhelper.nim rename to tests/storage/stores/testqueryiterhelper.nim index 4e83dad4..8e38849f 100644 --- a/tests/codex/stores/testqueryiterhelper.nim +++ b/tests/storage/stores/testqueryiterhelper.nim @@ -5,8 +5,8 @@ import pkg/questionable import pkg/chronos import pkg/datastore/typedds import pkg/datastore/sql/sqliteds -import pkg/codex/stores/queryiterhelper -import pkg/codex/utils/asynciter +import pkg/storage/stores/queryiterhelper +import pkg/storage/utils/asynciter import ../../asynctest import ../helpers diff --git a/tests/codex/stores/testrepostore.nim b/tests/storage/stores/testrepostore.nim similarity index 98% rename from tests/codex/stores/testrepostore.nim rename to tests/storage/stores/testrepostore.nim index b524c860..0af4e742 100644 --- a/tests/codex/stores/testrepostore.nim +++ b/tests/storage/stores/testrepostore.nim @@ -9,14 +9,14 @@ import pkg/chronos import pkg/stew/byteutils import pkg/datastore -import pkg/codex/stores/cachestore -import pkg/codex/chunker -import pkg/codex/stores -import pkg/codex/stores/repostore/operations -import pkg/codex/blocktype as bt -import pkg/codex/clock -import pkg/codex/utils/safeasynciter -import pkg/codex/merkletree +import pkg/storage/stores/cachestore +import pkg/storage/chunker +import pkg/storage/stores +import pkg/storage/stores/repostore/operations +import pkg/storage/blocktype as bt +import pkg/storage/clock +import pkg/storage/utils/safeasynciter +import pkg/storage/merkletree import ../../asynctest import ../helpers diff --git a/tests/codex/testasyncheapqueue.nim b/tests/storage/testasyncheapqueue.nim similarity index 98% rename from tests/codex/testasyncheapqueue.nim rename to tests/storage/testasyncheapqueue.nim index 2d2cfb0c..8a6459e6 100644 --- a/tests/codex/testasyncheapqueue.nim +++ b/tests/storage/testasyncheapqueue.nim @@ -1,8 +1,8 @@ import pkg/chronos import pkg/results -import pkg/codex/utils/asyncheapqueue -import pkg/codex/rng +import pkg/storage/utils/asyncheapqueue +import pkg/storage/rng import ../asynctest import ./helpers diff --git a/tests/codex/testasyncstreamwrapper.nim b/tests/storage/testasyncstreamwrapper.nim similarity index 98% rename from tests/codex/testasyncstreamwrapper.nim rename to tests/storage/testasyncstreamwrapper.nim index 2aa23039..63ff0b67 100644 --- a/tests/codex/testasyncstreamwrapper.nim +++ b/tests/storage/testasyncstreamwrapper.nim @@ -2,7 +2,7 @@ import pkg/chronos import pkg/chronos/transports/stream import pkg/chronos/transports/common import pkg/chronos/streams/asyncstream -import pkg/codex/streams +import pkg/storage/streams import pkg/stew/byteutils import ../asynctest diff --git a/tests/codex/testblockexchange.nim b/tests/storage/testblockexchange.nim similarity index 100% rename from tests/codex/testblockexchange.nim rename to tests/storage/testblockexchange.nim diff --git a/tests/codex/testblocktype.nim b/tests/storage/testblocktype.nim similarity index 97% rename from tests/codex/testblocktype.nim rename to tests/storage/testblocktype.nim index b0ea2732..d987b16e 100644 --- a/tests/codex/testblocktype.nim +++ b/tests/storage/testblocktype.nim @@ -1,7 +1,7 @@ import pkg/unittest2 import pkg/libp2p/cid -import pkg/codex/blocktype +import pkg/storage/blocktype import ./examples diff --git a/tests/codex/testchunking.nim b/tests/storage/testchunking.nim similarity index 98% rename from tests/codex/testchunking.nim rename to tests/storage/testchunking.nim index 531d92bf..9feabc49 100644 --- a/tests/codex/testchunking.nim +++ b/tests/storage/testchunking.nim @@ -1,6 +1,6 @@ import pkg/stew/byteutils -import pkg/codex/chunker -import pkg/codex/logutils +import pkg/storage/chunker +import pkg/storage/logutils import pkg/chronos import ../asynctest diff --git a/tests/codex/testclock.nim b/tests/storage/testclock.nim similarity index 95% rename from tests/codex/testclock.nim rename to tests/storage/testclock.nim index 967de672..fdba653e 100644 --- a/tests/codex/testclock.nim +++ b/tests/storage/testclock.nim @@ -1,6 +1,6 @@ import pkg/unittest2 -import codex/clock +import storage/clock import ./helpers suite "Clock": diff --git a/tests/codex/testlogutils.nim b/tests/storage/testlogutils.nim similarity index 97% rename from tests/codex/testlogutils.nim rename to tests/storage/testlogutils.nim index ba3a00f3..1918137f 100644 --- a/tests/codex/testlogutils.nim +++ b/tests/storage/testlogutils.nim @@ -2,11 +2,11 @@ import std/options import std/strutils import pkg/unittest2 -import pkg/codex/blocktype -import pkg/codex/conf -import pkg/codex/logutils -import pkg/codex/units -import pkg/codex/utils/json +import pkg/storage/blocktype +import pkg/storage/conf +import pkg/storage/logutils +import pkg/storage/units +import pkg/storage/utils/json import pkg/libp2p/cid import pkg/libp2p/multiaddress import pkg/questionable diff --git a/tests/codex/testmanifest.nim b/tests/storage/testmanifest.nim similarity index 85% rename from tests/codex/testmanifest.nim rename to tests/storage/testmanifest.nim index 6e773b0e..01e5f859 100644 --- a/tests/codex/testmanifest.nim +++ b/tests/storage/testmanifest.nim @@ -1,10 +1,10 @@ import pkg/chronos import pkg/questionable/results -import pkg/codex/chunker -import pkg/codex/blocktype as bt -import pkg/codex/manifest +import pkg/storage/chunker +import pkg/storage/blocktype as bt +import pkg/storage/manifest -import pkg/codex/merkletree +import pkg/storage/merkletree import ../asynctest import ./helpers diff --git a/tests/storage/testmerkletree.nim b/tests/storage/testmerkletree.nim new file mode 100644 index 00000000..9f06acf2 --- /dev/null +++ b/tests/storage/testmerkletree.nim @@ -0,0 +1,4 @@ +import ./merkletree/teststoragetree +import ./merkletree/teststoragecoders + +{.warning[UnusedImport]: off.} diff --git a/tests/codex/testnat.nim b/tests/storage/testnat.nim similarity index 94% rename from tests/codex/testnat.nim rename to tests/storage/testnat.nim index c47b71de..d8539cb5 100644 --- a/tests/codex/testnat.nim +++ b/tests/storage/testnat.nim @@ -3,9 +3,9 @@ import pkg/chronos import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/results -import ../../codex/nat -import ../../codex/utils/natutils -import ../../codex/utils +import ../../storage/nat +import ../../storage/utils/natutils +import ../../storage/utils suite "NAT Address Tests": test "nattedAddress with local addresses": diff --git a/tests/codex/testnode.nim b/tests/storage/testnode.nim similarity index 100% rename from tests/codex/testnode.nim rename to tests/storage/testnode.nim diff --git a/tests/codex/testrng.nim b/tests/storage/testrng.nim similarity index 97% rename from tests/codex/testrng.nim rename to tests/storage/testrng.nim index f97a253b..d086f551 100644 --- a/tests/codex/testrng.nim +++ b/tests/storage/testrng.nim @@ -2,7 +2,7 @@ import std/unittest import std/sequtils import std/sets -import ../../codex/rng +import ../../storage/rng suite "Random Number Generator (RNG)": let rng = Rng.instance() diff --git a/tests/codex/teststores.nim b/tests/storage/teststores.nim similarity index 100% rename from tests/codex/teststores.nim rename to tests/storage/teststores.nim diff --git a/tests/codex/teststorestream.nim b/tests/storage/teststorestream.nim similarity index 97% rename from tests/codex/teststorestream.nim rename to tests/storage/teststorestream.nim index c1bd32dd..ad779808 100644 --- a/tests/codex/teststorestream.nim +++ b/tests/storage/teststorestream.nim @@ -1,6 +1,6 @@ import pkg/chronos -import pkg/codex/[streams, stores, manifest, blocktype as bt] +import pkg/storage/[streams, stores, manifest, blocktype as bt] import ../asynctest import ./examples diff --git a/tests/codex/testsystemclock.nim b/tests/storage/testsystemclock.nim similarity index 88% rename from tests/codex/testsystemclock.nim rename to tests/storage/testsystemclock.nim index 3f179260..4f701b98 100644 --- a/tests/codex/testsystemclock.nim +++ b/tests/storage/testsystemclock.nim @@ -1,7 +1,7 @@ import std/times import pkg/unittest2 -import pkg/codex/systemclock +import pkg/storage/systemclock import ./helpers suite "SystemClock": diff --git a/tests/codex/testutils.nim b/tests/storage/testutils.nim similarity index 100% rename from tests/codex/testutils.nim rename to tests/storage/testutils.nim diff --git a/tests/codex/utils/testasynciter.nim b/tests/storage/utils/testasynciter.nim similarity index 99% rename from tests/codex/utils/testasynciter.nim rename to tests/storage/utils/testasynciter.nim index 984cbca7..17d909b3 100644 --- a/tests/codex/utils/testasynciter.nim +++ b/tests/storage/utils/testasynciter.nim @@ -1,6 +1,6 @@ import pkg/questionable import pkg/chronos -import pkg/codex/utils/asynciter +import pkg/storage/utils/asynciter import ../../asynctest import ../helpers diff --git a/tests/codex/utils/testasyncstatemachine.nim b/tests/storage/utils/testasyncstatemachine.nim similarity index 98% rename from tests/codex/utils/testasyncstatemachine.nim rename to tests/storage/utils/testasyncstatemachine.nim index c46bb605..98d376ed 100644 --- a/tests/codex/utils/testasyncstatemachine.nim +++ b/tests/storage/utils/testasyncstatemachine.nim @@ -1,6 +1,6 @@ import pkg/questionable import pkg/chronos -import codex/utils/asyncstatemachine +import storage/utils/asyncstatemachine import ../../asynctest import ../helpers diff --git a/tests/codex/utils/testiter.nim b/tests/storage/utils/testiter.nim similarity index 98% rename from tests/codex/utils/testiter.nim rename to tests/storage/utils/testiter.nim index ec19c484..39f8e67d 100644 --- a/tests/codex/utils/testiter.nim +++ b/tests/storage/utils/testiter.nim @@ -2,7 +2,7 @@ import std/sugar import pkg/questionable import pkg/chronos -import pkg/codex/utils/iter +import pkg/storage/utils/iter import ../../asynctest import ../helpers diff --git a/tests/codex/utils/testkeyutils.nim b/tests/storage/utils/testkeyutils.nim similarity index 91% rename from tests/codex/utils/testkeyutils.nim rename to tests/storage/utils/testkeyutils.nim index 104258f3..b351bbce 100644 --- a/tests/codex/utils/testkeyutils.nim +++ b/tests/storage/utils/testkeyutils.nim @@ -1,7 +1,7 @@ import std/os import pkg/unittest2 -import pkg/codex/utils/keyutils +import pkg/storage/utils/keyutils import ../helpers @@ -9,7 +9,7 @@ when defined(windows): import stew/windows/acl suite "keyutils": - let path = getTempDir() / "CodexTest" + let path = getTempDir() / "StorageTest" setup: os.createDir(path) diff --git a/tests/codex/utils/testoptions.nim b/tests/storage/utils/testoptions.nim similarity index 97% rename from tests/codex/utils/testoptions.nim rename to tests/storage/utils/testoptions.nim index 650715bc..1cc749fa 100644 --- a/tests/codex/utils/testoptions.nim +++ b/tests/storage/utils/testoptions.nim @@ -1,5 +1,5 @@ import pkg/unittest2 -import pkg/codex/utils/options +import pkg/storage/utils/options import ../helpers diff --git a/tests/codex/utils/testsafeasynciter.nim b/tests/storage/utils/testsafeasynciter.nim similarity index 99% rename from tests/codex/utils/testsafeasynciter.nim rename to tests/storage/utils/testsafeasynciter.nim index 87b0d84a..c3e1e12d 100644 --- a/tests/codex/utils/testsafeasynciter.nim +++ b/tests/storage/utils/testsafeasynciter.nim @@ -1,8 +1,8 @@ import std/sugar import pkg/questionable import pkg/chronos -import pkg/codex/utils/iter -import pkg/codex/utils/safeasynciter +import pkg/storage/utils/iter +import pkg/storage/utils/safeasynciter import ../../asynctest import ../helpers diff --git a/tests/codex/utils/testtimer.nim b/tests/storage/utils/testtimer.nim similarity index 98% rename from tests/codex/utils/testtimer.nim rename to tests/storage/utils/testtimer.nim index 79b95e42..6a349c6f 100644 --- a/tests/codex/utils/testtimer.nim +++ b/tests/storage/utils/testtimer.nim @@ -9,7 +9,7 @@ import pkg/chronos -import codex/utils/timer +import storage/utils/timer import ../../asynctest import ../helpers diff --git a/tests/codex/utils/testtrackedfutures.nim b/tests/storage/utils/testtrackedfutures.nim similarity index 98% rename from tests/codex/utils/testtrackedfutures.nim rename to tests/storage/utils/testtrackedfutures.nim index 993d5b43..af2c5f00 100644 --- a/tests/codex/utils/testtrackedfutures.nim +++ b/tests/storage/utils/testtrackedfutures.nim @@ -1,5 +1,5 @@ import pkg/chronos -import codex/utils/trackedfutures +import storage/utils/trackedfutures import ../../asynctest import ../helpers diff --git a/tests/codex/utils/testutils.nim b/tests/storage/utils/testutils.nim similarity index 97% rename from tests/codex/utils/testutils.nim rename to tests/storage/utils/testutils.nim index b2150deb..306dccbc 100644 --- a/tests/codex/utils/testutils.nim +++ b/tests/storage/utils/testutils.nim @@ -1,6 +1,6 @@ import pkg/unittest2 -import pkg/codex/utils +import pkg/storage/utils suite "findIt": setup: diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index eef2f79e..5e289ff5 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -4,8 +4,8 @@ import ./imports ## Limit which integration tests to run by setting the ## environment variable during compilation. For example: -## CODEX_INTEGRATION_TEST_INCLUDES="testFoo.nim,testBar.nim" -const includes = getEnv("CODEX_INTEGRATION_TEST_INCLUDES") +## STORAGE_INTEGRATION_TEST_INCLUDES="testFoo.nim,testBar.nim" +const includes = getEnv("STORAGE_INTEGRATION_TEST_INCLUDES") when includes != "": # import only the specified tests diff --git a/tests/testCodex.nim b/tests/testStorage.nim similarity index 52% rename from tests/testCodex.nim rename to tests/testStorage.nim index 5f6b78bf..609a50ac 100644 --- a/tests/testCodex.nim +++ b/tests/testStorage.nim @@ -1,6 +1,6 @@ import std/os import ./imports -importTests(currentSourcePath().parentDir() / "codex") +importTests(currentSourcePath().parentDir() / "storage") {.warning[UnusedImport]: off.}