diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e64d05b3..2860b58d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: uses: arnetheduck/nph-action@v1 with: version: latest - options: "codex/ tests/" + options: "storage/ tests/" fail: true suggest: true diff --git a/Makefile b/Makefile index 80cd5e50..b030340e 100644 --- a/Makefile +++ b/Makefile @@ -181,11 +181,11 @@ endif coverage: $(MAKE) NIMFLAGS="$(NIMFLAGS) --lineDir:on --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage" test - cd nimcache/release/testCodex && rm -f *.c + cd nimcache/release/testStorage && rm -f *.c mkdir -p coverage - lcov --capture --keep-going --directory nimcache/release/testCodex --output-file coverage/coverage.info - shopt -s globstar && ls $$(pwd)/codex/{*,**/*}.nim - shopt -s globstar && lcov --extract coverage/coverage.info --keep-going $$(pwd)/codex/{*,**/*}.nim --output-file coverage/coverage.f.info + lcov --capture --keep-going --directory nimcache/release/testStorage --output-file coverage/coverage.info + shopt -s globstar && ls $$(pwd)/storage/{*,**/*}.nim + shopt -s globstar && lcov --extract coverage/coverage.info --keep-going $$(pwd)/storage/{*,**/*}.nim --output-file coverage/coverage.f.info echo -e $(BUILD_MSG) "coverage/report/index.html" genhtml coverage/coverage.f.info --keep-going --output-directory coverage/report @@ -237,7 +237,7 @@ nph/%: build-nph format: $(NPH) *.nim - $(NPH) codex/ + $(NPH) storage/ $(NPH) tests/ $(NPH) library/ diff --git a/README.md b/README.md index 9575e2eb..d21e42b6 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,12 @@ [![Docker](https://github.com/logos-storage/logos-storage-nim/actions/workflows/docker.yml/badge.svg?branch=master)](https://github.com/logos-storage/logos-storage-nim/actions/workflows/docker.yml?query=branch%3Amaster) [![Codecov](https://codecov.io/gh/logos-storage/logos-storage-nim/branch/master/graph/badge.svg?token=XFmCyPSNzW)](https://codecov.io/gh/logos-storage/logos-storage-nim) [![Discord](https://img.shields.io/discord/895609329053474826)](https://discord.gg/CaJTh24ddQ) -![Docker Pulls](https://img.shields.io/docker/pulls/codexstorage/nim-codex) +![Docker Pulls](https://img.shields.io/docker/pulls/logosstorage/logos-storage-nim) ## Build and Run -For detailed instructions on preparing to build logos-storagenim see [*Build Logos Storage*](https://docs.codex.storage/learn/build). +For detailed instructions on preparing to build logos-storage-nim see [*Build Logos Storage*](https://docs.codex.storage/learn/build). To build the project, clone it and run: diff --git a/library/libstorage.nim b/library/libstorage.nim index 46f0749c..e41e52dd 100644 --- a/library/libstorage.nim +++ b/library/libstorage.nim @@ -39,7 +39,7 @@ import ./storage_thread_requests/requests/node_download_request import ./storage_thread_requests/requests/node_storage_request import ./ffi_types -from ../codex/conf import codexVersion +from ../storage/conf import storageVersion logScope: topics = "libstorage" @@ -125,12 +125,12 @@ proc storage_new( proc storage_version(ctx: ptr StorageContext): ptr cchar {.dynlib, exportc.} = initializeLibrary() - return asNewCString(conf.codexVersion) + return asNewCString(conf.storageVersion) proc storage_revision(ctx: ptr StorageContext): ptr cchar {.dynlib, exportc.} = initializeLibrary() - return asNewCString(conf.codexRevision) + return asNewCString(conf.storageVersion) proc storage_repo( ctx: ptr StorageContext, callback: StorageCallback, userData: pointer diff --git a/library/storage_context.nim b/library/storage_context.nim index 260b6a98..cbb140af 100644 --- a/library/storage_context.nim +++ b/library/storage_context.nim @@ -16,7 +16,7 @@ import taskpools/channels_spsc_single import ./ffi_types import ./storage_thread_requests/[storage_thread_request] -from ../codex/codex import CodexServer +from ../storage/storage import StorageServer logScope: topics = "libstorage" @@ -124,7 +124,7 @@ proc sendRequestToStorageThread*( ok() proc runStorage(ctx: ptr StorageContext) {.async: (raises: []).} = - var storage: CodexServer + var storage: StorageServer while true: try: diff --git a/library/storage_thread_requests/requests/node_debug_request.nim b/library/storage_thread_requests/requests/node_debug_request.nim index c8dab2c4..8bf3106c 100644 --- a/library/storage_thread_requests/requests/node_debug_request.nim +++ b/library/storage_thread_requests/requests/node_debug_request.nim @@ -10,11 +10,11 @@ import chronos import chronicles import codexdht/discv5/spr import ../../alloc -import ../../../codex/conf -import ../../../codex/rest/json -import ../../../codex/node +import ../../../storage/conf +import ../../../storage/rest/json +import ../../../storage/node -from ../../../codex/codex import CodexServer, node +from ../../../storage/storage import StorageServer, node logScope: topics = "libstorage libstoragedebug" @@ -47,7 +47,7 @@ proc destroyShared(self: ptr NodeDebugRequest) = deallocShared(self) proc getDebug( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = let node = storage[].node let table = RestRoutingTable.init(node.discovery.protocol.routingTable) @@ -64,7 +64,7 @@ proc getDebug( return ok($json) proc getPeer( - storage: ptr CodexServer, peerId: cstring + storage: ptr StorageServer, peerId: cstring ): Future[Result[string, string]] {.async: (raises: []).} = when storage_enable_api_debug_peers: let node = storage[].node @@ -88,7 +88,7 @@ proc getPeer( return err("Failed to get peer: peer debug API is disabled") proc updateLogLevel( - storage: ptr CodexServer, logLevel: cstring + storage: ptr StorageServer, logLevel: cstring ): Future[Result[string, string]] {.async: (raises: []).} = try: {.gcsafe.}: @@ -99,7 +99,7 @@ proc updateLogLevel( return ok("") proc process*( - self: ptr NodeDebugRequest, storage: ptr CodexServer + self: ptr NodeDebugRequest, storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_download_request.nim b/library/storage_thread_requests/requests/node_download_request.nim index 1d8ef59c..44b38482 100644 --- a/library/storage_thread_requests/requests/node_download_request.nim +++ b/library/storage_thread_requests/requests/node_download_request.nim @@ -21,12 +21,12 @@ import chronicles import libp2p/stream/[lpstream] import serde/json as serde import ../../alloc -import ../../../codex/units -import ../../../codex/codextypes +import ../../../storage/units +import ../../../storage/storagetypes -from ../../../codex/codex import CodexServer, node -from ../../../codex/node import retrieve, fetchManifest -from ../../../codex/rest/json import `%`, RestContent +from ../../../storage/storage import StorageServer, node +from ../../../storage/node import retrieve, fetchManifest +from ../../../storage/rest/json import `%`, RestContent from libp2p import Cid, init, `$` logScope: @@ -80,7 +80,7 @@ proc destroyShared(self: ptr NodeDownloadRequest) = deallocShared(self) proc init( - storage: ptr CodexServer, cCid: cstring = "", chunkSize: csize_t = 0, local: bool + storage: ptr StorageServer, cCid: cstring = "", chunkSize: csize_t = 0, local: bool ): Future[Result[string, string]] {.async: (raises: []).} = ## Init a new session to download the file identified by cid. ## @@ -114,7 +114,7 @@ proc init( return ok("") proc chunk( - storage: ptr CodexServer, cCid: cstring = "", onChunk: OnChunkHandler + storage: ptr StorageServer, cCid: cstring = "", onChunk: OnChunkHandler ): Future[Result[string, string]] {.async: (raises: []).} = ## Download the next chunk of the file identified by cid. ## The chunk is passed to the onChunk handler. @@ -164,7 +164,7 @@ proc chunk( return ok("") proc streamData( - storage: ptr CodexServer, + storage: ptr StorageServer, stream: LPStream, onChunk: OnChunkHandler, chunkSize: csize_t, @@ -207,7 +207,7 @@ proc streamData( return ok("") proc stream( - storage: ptr CodexServer, + storage: ptr StorageServer, cCid: cstring, chunkSize: csize_t, local: bool, @@ -251,7 +251,7 @@ proc stream( return ok("") proc cancel( - storage: ptr CodexServer, cCid: cstring + storage: ptr StorageServer, cCid: cstring ): Future[Result[string, string]] {.raises: [], async: (raises: []).} = ## Cancel the download session identified by cid. ## This operation is not supported when using the stream mode, @@ -279,7 +279,7 @@ proc cancel( return ok("") proc manifest( - storage: ptr CodexServer, cCid: cstring + storage: ptr StorageServer, cCid: cstring ): Future[Result[string, string]] {.raises: [], async: (raises: []).} = let cid = Cid.init($cCid) if cid.isErr: @@ -296,7 +296,7 @@ proc manifest( return err("Failed to fetch manifest: download cancelled.") proc process*( - self: ptr NodeDownloadRequest, storage: ptr CodexServer, onChunk: OnChunkHandler + self: ptr NodeDownloadRequest, storage: ptr StorageServer, onChunk: OnChunkHandler ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_info_request.nim b/library/storage_thread_requests/requests/node_info_request.nim index cbf364e8..7e755a3a 100644 --- a/library/storage_thread_requests/requests/node_info_request.nim +++ b/library/storage_thread_requests/requests/node_info_request.nim @@ -5,11 +5,11 @@ import chronos import chronicles import confutils import codexdht/discv5/spr -import ../../../codex/conf -import ../../../codex/rest/json -import ../../../codex/node +import ../../../storage/conf +import ../../../storage/rest/json +import ../../../storage/node -from ../../../codex/codex import CodexServer, config, node +from ../../../storage/storage import StorageServer, config, node logScope: topics = "libstorage libstorageinfo" @@ -31,12 +31,12 @@ proc destroyShared(self: ptr NodeInfoRequest) = deallocShared(self) proc getRepo( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = return ok($(storage[].config.dataDir)) proc getSpr( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = let spr = storage[].node.discovery.dhtRecord if spr.isNone: @@ -45,12 +45,12 @@ proc getSpr( return ok(spr.get.toURI) proc getPeerId( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = return ok($storage[].node.switch.peerInfo.peerId) proc process*( - self: ptr NodeInfoRequest, storage: ptr CodexServer + self: ptr NodeInfoRequest, storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_lifecycle_request.nim b/library/storage_thread_requests/requests/node_lifecycle_request.nim index 491847aa..31da9416 100644 --- a/library/storage_thread_requests/requests/node_lifecycle_request.nim +++ b/library/storage_thread_requests/requests/node_lifecycle_request.nim @@ -17,12 +17,12 @@ import libp2p import json_serialization import json_serialization/std/[options, net] import ../../alloc -import ../../../codex/conf -import ../../../codex/utils -import ../../../codex/utils/[keyutils, fileutils] -import ../../../codex/units +import ../../../storage/conf +import ../../../storage/utils +import ../../../storage/utils/[keyutils, fileutils] +import ../../../storage/units -from ../../../codex/codex import CodexServer, new, start, stop, close +from ../../../storage/storage import StorageServer, new, start, stop, close logScope: topics = "libstorage libstoragelifecycle" @@ -89,16 +89,16 @@ proc destroyShared(self: ptr NodeLifecycleRequest) = proc createStorage( configJson: cstring -): Future[Result[CodexServer, string]] {.async: (raises: []).} = - var conf: CodexConf +): Future[Result[StorageServer, string]] {.async: (raises: []).} = + var conf: StorageConf try: - conf = CodexConf.load( - version = codexFullVersion, + conf = StorageConf.load( + version = storageFullVersion, envVarsPrefix = "storage", cmdLine = @[], secondarySources = proc( - config: CodexConf, sources: auto + config: StorageConf, sources: auto ) {.gcsafe, raises: [ConfigurationError].} = if configJson.len > 0: sources.addConfigFileContent(Json, $(configJson)) @@ -149,14 +149,14 @@ proc createStorage( let server = try: - CodexServer.new(conf, pk) + StorageServer.new(conf, pk) except Exception as exc: return err("Failed to create Storage: " & exc.msg) return ok(server) proc process*( - self: ptr NodeLifecycleRequest, storage: ptr CodexServer + self: ptr NodeLifecycleRequest, storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_p2p_request.nim b/library/storage_thread_requests/requests/node_p2p_request.nim index dd06eb39..bfc29b88 100644 --- a/library/storage_thread_requests/requests/node_p2p_request.nim +++ b/library/storage_thread_requests/requests/node_p2p_request.nim @@ -8,9 +8,9 @@ import chronos import chronicles import libp2p import ../../alloc -import ../../../codex/node +import ../../../storage/node -from ../../../codex/codex import CodexServer, node +from ../../../storage/storage import StorageServer, node logScope: topics = "libstorage libstoragep2p" @@ -40,7 +40,7 @@ proc destroyShared(self: ptr NodeP2PRequest) = deallocShared(self) proc connect( - storage: ptr CodexServer, peerId: cstring, peerAddresses: seq[cstring] = @[] + storage: ptr StorageServer, peerId: cstring, peerAddresses: seq[cstring] = @[] ): Future[Result[string, string]] {.async: (raises: []).} = let node = storage[].node let res = PeerId.init($peerId) @@ -81,7 +81,7 @@ proc connect( return ok("") proc process*( - self: ptr NodeP2PRequest, storage: ptr CodexServer + self: ptr NodeP2PRequest, storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_storage_request.nim b/library/storage_thread_requests/requests/node_storage_request.nim index 72ee14f9..32808b1a 100644 --- a/library/storage_thread_requests/requests/node_storage_request.nim +++ b/library/storage_thread_requests/requests/node_storage_request.nim @@ -14,12 +14,12 @@ import chronicles import libp2p/stream/[lpstream] import serde/json as serde import ../../alloc -import ../../../codex/units -import ../../../codex/manifest -import ../../../codex/stores/repostore +import ../../../storage/units +import ../../../storage/manifest +import ../../../storage/stores/repostore -from ../../../codex/codex import CodexServer, node, repoStore -from ../../../codex/node import +from ../../../storage/storage import StorageServer, node, repoStore +from ../../../storage/node import iterateManifests, fetchManifest, fetchDatasetAsyncTask, delete, hasLocalBlock from libp2p import Cid, init, `$` @@ -61,7 +61,7 @@ type ManifestWithCid = object manifest {.serialize.}: Manifest proc list( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = var manifests = newSeq[ManifestWithCid]() proc onManifest(cid: Cid, manifest: Manifest) {.raises: [], gcsafe.} = @@ -78,7 +78,7 @@ proc list( return ok(serde.toJson(manifests)) proc delete( - storage: ptr CodexServer, cCid: cstring + storage: ptr StorageServer, cCid: cstring ): Future[Result[string, string]] {.async: (raises: []).} = let cid = Cid.init($cCid) if cid.isErr: @@ -97,7 +97,7 @@ proc delete( return ok("") proc fetch( - storage: ptr CodexServer, cCid: cstring + storage: ptr StorageServer, cCid: cstring ): Future[Result[string, string]] {.async: (raises: []).} = let cid = Cid.init($cCid) if cid.isErr: @@ -116,7 +116,7 @@ proc fetch( return err("Failed to fetch the data: download cancelled.") proc space( - storage: ptr CodexServer + storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = let repoStore = storage[].repoStore let space = StorageSpace( @@ -128,7 +128,7 @@ proc space( return ok(serde.toJson(space)) proc exists( - storage: ptr CodexServer, cCid: cstring + storage: ptr StorageServer, cCid: cstring ): Future[Result[string, string]] {.async: (raises: []).} = let cid = Cid.init($cCid) if cid.isErr: @@ -142,7 +142,7 @@ proc exists( return err("Failed to check the data existence: operation cancelled.") proc process*( - self: ptr NodeStorageRequest, storage: ptr CodexServer + self: ptr NodeStorageRequest, storage: ptr StorageServer ): Future[Result[string, string]] {.async: (raises: []).} = defer: destroyShared(self) diff --git a/library/storage_thread_requests/requests/node_upload_request.nim b/library/storage_thread_requests/requests/node_upload_request.nim index 17ef1e8e..064439b2 100644 --- a/library/storage_thread_requests/requests/node_upload_request.nim +++ b/library/storage_thread_requests/requests/node_upload_request.nim @@ -24,11 +24,11 @@ import questionable/results import faststreams/inputs import libp2p/stream/[bufferstream, lpstream] import ../../alloc -import ../../../codex/units -import ../../../codex/codextypes +import ../../../storage/units +import ../../../storage/storagetypes -from ../../../codex/codex import CodexServer, node -from ../../../codex/node import store +from ../../../storage/storage import StorageServer, node +from ../../../storage/node import store from libp2p import Cid, `$` logScope: @@ -86,7 +86,7 @@ proc destroyShared(self: ptr NodeUploadRequest) = deallocShared(self) proc init( - storage: ptr CodexServer, filepath: cstring = "", chunkSize: csize_t = 0 + storage: ptr StorageServer, filepath: cstring = "", chunkSize: csize_t = 0 ): Future[Result[string, string]] {.async: (raises: []).} = ## Init a new session upload and return its ID. ## The session contains the future corresponding to the @@ -156,7 +156,7 @@ proc init( return ok(sessionId) proc chunk( - storage: ptr CodexServer, sessionId: cstring, chunk: seq[byte] + storage: ptr StorageServer, sessionId: cstring, chunk: seq[byte] ): Future[Result[string, string]] {.async: (raises: []).} = ## Upload a chunk of data to the session identified by sessionId. ## The chunk is pushed to the BufferStream of the session. @@ -205,7 +205,7 @@ proc chunk( return ok("") proc finalize( - storage: ptr CodexServer, sessionId: cstring + storage: ptr StorageServer, sessionId: cstring ): Future[Result[string, string]] {.async: (raises: []).} = ## Finalize the upload session identified by sessionId. ## This closes the BufferStream and waits for the `node.store` future @@ -242,7 +242,7 @@ proc finalize( session.fut.cancelSoon() proc cancel( - storage: ptr CodexServer, sessionId: cstring + storage: ptr StorageServer, sessionId: cstring ): Future[Result[string, string]] {.async: (raises: []).} = ## Cancel the upload session identified by sessionId. ## This cancels the `node.store` future and removes the session @@ -293,7 +293,7 @@ proc streamFile( return err("Failed to stream the file: " & $e.msg) proc file( - storage: ptr CodexServer, sessionId: cstring, onProgress: OnProgressHandler + storage: ptr StorageServer, sessionId: cstring, onProgress: OnProgressHandler ): Future[Result[string, string]] {.async: (raises: []).} = ## Starts the file upload for the session identified by sessionId. ## Will call finalize when done and return the CID of the uploaded file. @@ -333,7 +333,7 @@ proc file( proc process*( self: ptr NodeUploadRequest, - storage: ptr CodexServer, + storage: ptr StorageServer, onUploadProgress: OnProgressHandler = nil, ): Future[Result[string, string]] {.async: (raises: []).} = defer: diff --git a/library/storage_thread_requests/storage_thread_request.nim b/library/storage_thread_requests/storage_thread_request.nim index 375bf1b8..946ef4f8 100644 --- a/library/storage_thread_requests/storage_thread_request.nim +++ b/library/storage_thread_requests/storage_thread_request.nim @@ -14,7 +14,7 @@ import ./requests/node_upload_request import ./requests/node_download_request import ./requests/node_storage_request -from ../../codex/codex import CodexServer +from ../../storage/storage import StorageServer type RequestType* {.pure.} = enum LIFECYCLE @@ -89,7 +89,7 @@ proc handleRes[T: string | void | seq[byte]]( proc process*( T: type StorageThreadRequest, request: ptr StorageThreadRequest, - storage: ptr CodexServer, + storage: ptr StorageServer, ) {.async: (raises: []).} = ## Processes the request in the Logos Storage thread. ## Dispatch to the appropriate request handler based on reqType. diff --git a/nimbledeps/nimbledata2.json b/nimbledeps/nimbledata2.json new file mode 100644 index 00000000..6ffe1cc4 --- /dev/null +++ b/nimbledeps/nimbledata2.json @@ -0,0 +1,4 @@ +{ + "version": 1, + "reverseDeps": {} +} \ No newline at end of file diff --git a/storage.nim b/storage.nim index aad998b0..d8d2bfa0 100644 --- a/storage.nim +++ b/storage.nim @@ -18,36 +18,36 @@ import pkg/confutils/toml/std/uri as confTomlUri import pkg/toml_serialization import pkg/libp2p -import ./codex/conf -import ./codex/codex -import ./codex/logutils -import ./codex/units -import ./codex/utils/keyutils -import ./codex/codextypes +import ./storage/conf +import ./storage/storage +import ./storage/logutils +import ./storage/units +import ./storage/utils/keyutils +import ./storage/storagetypes -export codex, conf, libp2p, chronos, logutils +export storage, conf, libp2p, chronos, logutils when isMainModule: import std/os import pkg/confutils/defs - import ./codex/utils/fileutils + import ./storage/utils/fileutils logScope: - topics = "codex" + topics = "storage" when defined(posix): import system/ansi_c - type CodexStatus {.pure.} = enum + type StorageStatus {.pure.} = enum Stopped Stopping Running - let config = CodexConf.load( - version = codexFullVersion, + let config = StorageConf.load( + version = storageFullVersion, envVarsPrefix = "storage", secondarySources = proc( - config: CodexConf, sources: auto + config: StorageConf, sources: auto ) {.gcsafe, raises: [ConfigurationError].} = if configFile =? config.configFile: sources.addConfigFile(Toml, configFile) @@ -81,7 +81,7 @@ when isMainModule: trace "Repo dir initialized", dir = config.dataDir / "repo" var - state: CodexStatus + state: StorageStatus shutdown: Future[void] let @@ -94,7 +94,7 @@ when isMainModule: privateKey = setupKey(keyPath).expect("Should setup private key!") server = try: - CodexServer.new(config, privateKey) + StorageServer.new(config, privateKey) except Exception as exc: error "Failed to start Logos Storage", msg = exc.msg quit QuitFailure @@ -102,7 +102,7 @@ when isMainModule: ## Ctrl+C handling proc doShutdown() = shutdown = server.shutdown() - state = CodexStatus.Stopping + state = StorageStatus.Stopping notice "Stopping Logos Storage" @@ -142,8 +142,8 @@ when isMainModule: # had a chance to start (currently you'll get a SISGSEV if you try to). quit QuitFailure - state = CodexStatus.Running - while state == CodexStatus.Running: + state = StorageStatus.Running + while state == StorageStatus.Running: try: # poll chronos chronos.poll() diff --git a/storage/blockexchange/engine/advertiser.nim b/storage/blockexchange/engine/advertiser.nim index c038db38..5b538572 100644 --- a/storage/blockexchange/engine/advertiser.nim +++ b/storage/blockexchange/engine/advertiser.nim @@ -28,9 +28,9 @@ import ../../logutils import ../../manifest logScope: - topics = "codex discoveryengine advertiser" + topics = "storage discoveryengine advertiser" -declareGauge(codex_inflight_advertise, "inflight advertise requests") +declareGauge(storage_inflight_advertise, "inflight advertise requests") const DefaultConcurrentAdvertRequests = 10 @@ -106,11 +106,11 @@ proc processQueueLoop(b: Advertiser) {.async: (raises: []).} = let request = b.discovery.provide(cid) b.inFlightAdvReqs[cid] = request - codex_inflight_advertise.set(b.inFlightAdvReqs.len.int64) + storage_inflight_advertise.set(b.inFlightAdvReqs.len.int64) defer: b.inFlightAdvReqs.del(cid) - codex_inflight_advertise.set(b.inFlightAdvReqs.len.int64) + storage_inflight_advertise.set(b.inFlightAdvReqs.len.int64) await request except CancelledError: diff --git a/storage/blockexchange/engine/discovery.nim b/storage/blockexchange/engine/discovery.nim index 6a1b808d..9d5d8c08 100644 --- a/storage/blockexchange/engine/discovery.nim +++ b/storage/blockexchange/engine/discovery.nim @@ -31,9 +31,9 @@ import ../../logutils import ../../manifest logScope: - topics = "codex discoveryengine" + topics = "storage discoveryengine" -declareGauge(codex_inflight_discovery, "inflight discovery requests") +declareGauge(storage_inflight_discovery, "inflight discovery requests") const DefaultConcurrentDiscRequests = 10 @@ -114,11 +114,11 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} = if haves.len < b.minPeersPerBlock: let request = b.discovery.find(cid) b.inFlightDiscReqs[cid] = request - codex_inflight_discovery.set(b.inFlightDiscReqs.len.int64) + storage_inflight_discovery.set(b.inFlightDiscReqs.len.int64) defer: b.inFlightDiscReqs.del(cid) - codex_inflight_discovery.set(b.inFlightDiscReqs.len.int64) + storage_inflight_discovery.set(b.inFlightDiscReqs.len.int64) if (await request.withTimeout(DefaultDiscoveryTimeout)) and peers =? (await request).catch: diff --git a/storage/blockexchange/engine/engine.nim b/storage/blockexchange/engine/engine.nim index 519659ee..f0a29e47 100644 --- a/storage/blockexchange/engine/engine.nim +++ b/storage/blockexchange/engine/engine.nim @@ -44,39 +44,39 @@ import ./pendingblocks export peers, pendingblocks, discovery logScope: - topics = "codex blockexcengine" + topics = "storage blockexcengine" declareCounter( - codex_block_exchange_want_have_lists_sent, "codex blockexchange wantHave lists sent" + storage_block_exchange_want_have_lists_sent, "storage blockexchange wantHave lists sent" ) declareCounter( - codex_block_exchange_want_have_lists_received, - "codex blockexchange wantHave lists received", + storage_block_exchange_want_have_lists_received, + "storage blockexchange wantHave lists received", ) declareCounter( - codex_block_exchange_want_block_lists_sent, "codex blockexchange wantBlock lists sent" + storage_block_exchange_want_block_lists_sent, "storage blockexchange wantBlock lists sent" ) declareCounter( - codex_block_exchange_want_block_lists_received, - "codex blockexchange wantBlock lists received", + storage_block_exchange_want_block_lists_received, + "storage blockexchange wantBlock lists received", ) -declareCounter(codex_block_exchange_blocks_sent, "codex blockexchange blocks sent") +declareCounter(storage_block_exchange_blocks_sent, "storage blockexchange blocks sent") declareCounter( - codex_block_exchange_blocks_received, "codex blockexchange blocks received" + storage_block_exchange_blocks_received, "storage blockexchange blocks received" ) declareCounter( - codex_block_exchange_spurious_blocks_received, - "codex blockexchange unrequested/duplicate blocks received", + storage_block_exchange_spurious_blocks_received, + "storage blockexchange unrequested/duplicate blocks received", ) declareCounter( - codex_block_exchange_discovery_requests_total, + storage_block_exchange_discovery_requests_total, "Total number of peer discovery requests sent", ) declareCounter( - codex_block_exchange_peer_timeouts_total, "Total number of peer activity timeouts" + storage_block_exchange_peer_timeouts_total, "Total number of peer activity timeouts" ) declareCounter( - codex_block_exchange_requests_failed_total, + storage_block_exchange_requests_failed_total, "Total number of block requests that failed after exhausting retries", ) @@ -166,7 +166,7 @@ proc sendWantHave( let toAsk = addresses.filterIt(it notin p.peerHave) trace "Sending wantHave request", toAsk, peer = p.id await self.network.request.sendWantList(p.id, toAsk, wantType = WantType.WantHave) - codex_block_exchange_want_have_lists_sent.inc() + storage_block_exchange_want_have_lists_sent.inc() proc sendWantBlock( self: BlockExcEngine, addresses: seq[BlockAddress], blockPeer: BlockExcPeerCtx @@ -175,7 +175,7 @@ proc sendWantBlock( await self.network.request.sendWantList( blockPeer.id, addresses, wantType = WantType.WantBlock ) # we want this remote to send us a block - codex_block_exchange_want_block_lists_sent.inc() + storage_block_exchange_want_block_lists_sent.inc() proc sendBatchedWantList( self: BlockExcEngine, @@ -297,7 +297,7 @@ proc refreshBlockKnowledge(self: BlockExcEngine) {.async: (raises: [CancelledErr proc searchForNewPeers(self: BlockExcEngine, cid: Cid) = if self.lastDiscRequest + DiscoveryRateLimit < Moment.now(): trace "Searching for new peers for", cid = cid - codex_block_exchange_discovery_requests_total.inc() + storage_block_exchange_discovery_requests_total.inc() self.lastDiscRequest = Moment.now() # always refresh before calling await! self.discovery.queueFindBlocksReq(@[cid]) else: @@ -333,7 +333,7 @@ proc downloadInternal( if self.pendingBlocks.retriesExhausted(address): trace "Error retries exhausted" - codex_block_exchange_requests_failed_total.inc() + storage_block_exchange_requests_failed_total.inc() handle.fail(newException(RetriesExhaustedError, "Error retries exhausted")) break @@ -415,7 +415,7 @@ proc downloadInternal( else: # If the peer timed out, retries immediately. trace "Peer timed out during block request", peer = scheduledPeer.id - codex_block_exchange_peer_timeouts_total.inc() + storage_block_exchange_peer_timeouts_total.inc() await self.network.dropPeer(scheduledPeer.id) # Evicts peer immediately or we may end up picking it again in the # next retry. @@ -426,7 +426,7 @@ proc downloadInternal( await handle.cancelAndWait() except RetriesExhaustedError as exc: warn "Retries exhausted for block", address, exc = exc.msg - codex_block_exchange_requests_failed_total.inc() + storage_block_exchange_requests_failed_total.inc() if not handle.finished: handle.fail(exc) finally: @@ -690,7 +690,7 @@ proc blocksDeliveryHandler*( # Unknown peers and unrequested blocks are dropped with a warning. if not allowSpurious and (peerCtx == nil or not peerCtx.blockReceived(bd.address)): warn "Dropping unrequested or duplicate block received from peer" - codex_block_exchange_spurious_blocks_received.inc() + storage_block_exchange_spurious_blocks_received.inc() continue if err =? self.validateBlockDelivery(bd).errorOption: @@ -729,7 +729,7 @@ proc blocksDeliveryHandler*( discard lastIdle = Moment.now() - codex_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64) + storage_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64) if err =? catch(await self.resolveBlocks(validatedBlocksDelivery)).errorOption: warn "Error resolving blocks", err = err.msg @@ -789,11 +789,11 @@ proc wantListHandler*( BlockPresence(address: e.address, `type`: BlockPresenceType.DontHave) ) - codex_block_exchange_want_have_lists_received.inc() + storage_block_exchange_want_have_lists_received.inc() of WantType.WantBlock: peerCtx.wantedBlocks.incl(e.address) schedulePeer = true - codex_block_exchange_want_block_lists_received.inc() + storage_block_exchange_want_block_lists_received.inc() else: # Updating existing entry in peer wants # peer doesn't want this block anymore if e.cancel: @@ -903,7 +903,7 @@ proc taskHandler*( continue await self.network.request.sendBlocksDelivery(peerCtx.id, blockDeliveries) - codex_block_exchange_blocks_sent.inc(blockDeliveries.len.int64) + storage_block_exchange_blocks_sent.inc(blockDeliveries.len.int64) # Drops the batch from the peer's set of wanted blocks; i.e. assumes that after # we send the blocks, then the peer no longer wants them, so we don't need to # re-send them. Note that the send might still fail down the line and we will diff --git a/storage/blockexchange/engine/pendingblocks.nim b/storage/blockexchange/engine/pendingblocks.nim index f843870f..770093dc 100644 --- a/storage/blockexchange/engine/pendingblocks.nim +++ b/storage/blockexchange/engine/pendingblocks.nim @@ -22,14 +22,14 @@ import ../../blocktype import ../../logutils logScope: - topics = "codex pendingblocks" + topics = "storage pendingblocks" declareGauge( - codex_block_exchange_pending_block_requests, - "codex blockexchange pending block requests", + storage_block_exchange_pending_block_requests, + "storage blockexchange pending block requests", ) declareGauge( - codex_block_exchange_retrieval_time_us, "codex blockexchange block retrieval time us" + storage_block_exchange_retrieval_time_us, "storage blockexchange block retrieval time us" ) const @@ -53,7 +53,7 @@ type lastInclusion*: Moment # time at which we last included a block into our wantlist proc updatePendingBlockGauge(p: PendingBlocksManager) = - codex_block_exchange_pending_block_requests.set(p.blocks.len.int64) + storage_block_exchange_pending_block_requests.set(p.blocks.len.int64) proc getWantHandle*( self: PendingBlocksManager, address: BlockAddress, requested: ?PeerId = PeerId.none @@ -123,7 +123,7 @@ proc resolve*( blockReq.handle.complete(bd.blk) - codex_block_exchange_retrieval_time_us.set(retrievalDurationUs) + storage_block_exchange_retrieval_time_us.set(retrievalDurationUs) else: trace "Block handle already finished", address = bd.address diff --git a/storage/blockexchange/network/network.nim b/storage/blockexchange/network/network.nim index 91617d0f..5f62d4a2 100644 --- a/storage/blockexchange/network/network.nim +++ b/storage/blockexchange/network/network.nim @@ -27,10 +27,10 @@ import ./networkpeer export networkpeer logScope: - topics = "codex blockexcnetwork" + topics = "storage blockexcnetwork" const - Codec* = "/codex/blockexc/1.0.0" + Codec* = "/storage/blockexc/1.0.0" DefaultMaxInflight* = 100 type diff --git a/storage/blockexchange/network/networkpeer.nim b/storage/blockexchange/network/networkpeer.nim index 927303a6..f61d3286 100644 --- a/storage/blockexchange/network/networkpeer.nim +++ b/storage/blockexchange/network/networkpeer.nim @@ -19,7 +19,7 @@ import ../../logutils import ../../utils/trackedfutures logScope: - topics = "codex blockexcnetworkpeer" + topics = "storage blockexcnetworkpeer" const DefaultYieldInterval = 50.millis diff --git a/storage/blockexchange/peers/peerctxstore.nim b/storage/blockexchange/peers/peerctxstore.nim index 853bd33f..ae935809 100644 --- a/storage/blockexchange/peers/peerctxstore.nim +++ b/storage/blockexchange/peers/peerctxstore.nim @@ -25,7 +25,7 @@ import ./peercontext export peercontext logScope: - topics = "codex peerctxstore" + topics = "storage peerctxstore" type PeerCtxStore* = ref object of RootObj diff --git a/storage/blocktype.nim b/storage/blocktype.nim index b5a1019c..d056c789 100644 --- a/storage/blocktype.nim +++ b/storage/blocktype.nim @@ -25,9 +25,9 @@ import ./utils import ./errors import ./logutils import ./utils/json -import ./codextypes +import ./storagetypes -export errors, logutils, units, codextypes +export errors, logutils, units, storagetypes type Block* = ref object of RootObj diff --git a/storage/conf.nim b/storage/conf.nim index f9a80175..50d191cf 100644 --- a/storage/conf.nim +++ b/storage/conf.nim @@ -35,7 +35,7 @@ import pkg/questionable import pkg/questionable/results import pkg/stew/base64 -import ./codextypes +import ./storagetypes import ./discovery import ./logutils import ./stores @@ -46,7 +46,7 @@ import ./utils/natutils from ./blockexchange/engine/pendingblocks import DefaultBlockRetries -export units, net, codextypes, logutils, completeCmdArg, parseCmdArg, NatConfig +export units, net, storagetypes, logutils, completeCmdArg, parseCmdArg, NatConfig export DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, @@ -90,7 +90,7 @@ type repoSQLite = "sqlite" repoLevelDb = "leveldb" - CodexConf* = object + StorageConf* = object configFile* {. desc: "Loads the configuration from a TOML file", defaultValueDesc: "none", @@ -277,19 +277,19 @@ type desc: "Logs to file", defaultValue: string.none, name: "log-file", hidden .}: Option[string] -func defaultAddress*(conf: CodexConf): IpAddress = +func defaultAddress*(conf: StorageConf): IpAddress = result = static parseIpAddress("127.0.0.1") func defaultNatConfig*(): NatConfig = result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAny) -proc getCodexVersion(): string = +proc getStorageVersion(): string = let tag = strip(staticExec("git describe --tags --abbrev=0")) if tag.isEmptyOrWhitespace: return "untagged build" return tag -proc getCodexRevision(): string = +proc getStorageRevision(): string = # using a slice in a static context breaks nimsuggest for some reason var res = strip(staticExec("git rev-parse --short HEAD")) return res @@ -298,12 +298,12 @@ proc getNimBanner(): string = staticExec("nim --version | grep Version") const - codexVersion* = getCodexVersion() - codexRevision* = getCodexRevision() + storageVersion* = getStorageVersion() + storageRevision* = getStorageRevision() nimBanner* = getNimBanner() - codexFullVersion* = - "Storage version: " & codexVersion & "\p" & "Storage revision: " & codexRevision & + storageFullVersion* = + "Storage version: " & storageVersion & "\p" & "Storage revision: " & storageRevision & "\p" proc parseCmdArg*( @@ -533,7 +533,7 @@ proc updateLogLevel*(logLevel: string) {.raises: [ValueError].} = if not setTopicState(topicName, settings.state, settings.logLevel): warn "Unrecognized logging topic", topic = topicName -proc setupLogging*(conf: CodexConf) = +proc setupLogging*(conf: StorageConf) = when defaultChroniclesStream.outputs.type.arity != 3: warn "Logging configuration options not enabled in the current build" else: @@ -597,7 +597,7 @@ proc setupLogging*(conf: CodexConf) = else: defaultChroniclesStream.outputs[0].writer = writer -proc setupMetrics*(config: CodexConf) = +proc setupMetrics*(config: StorageConf) = if config.metricsEnabled: let metricsAddress = config.metricsAddress notice "Starting metrics HTTP server", diff --git a/storage/contentids_exts.nim b/storage/contentids_exts.nim index 28bcf2d1..fb12f72a 100644 --- a/storage/contentids_exts.nim +++ b/storage/contentids_exts.nim @@ -1,2 +1,2 @@ const ContentIdsExts = - [multiCodec("codex-root"), multiCodec("codex-manifest"), multiCodec("codex-block")] + [multiCodec("storage-root"), multiCodec("storage-manifest"), multiCodec("storage-block")] diff --git a/storage/discovery.nim b/storage/discovery.nim index 1cded8e6..73be4ee3 100644 --- a/storage/discovery.nim +++ b/storage/discovery.nim @@ -32,7 +32,7 @@ export discv5 # much more elegantly. logScope: - topics = "codex discovery" + topics = "storage discovery" type Discovery* = ref object of RootObj protocol*: discv5.Protocol # dht protocol diff --git a/storage/errors.nim b/storage/errors.nim index 655aec87..84be6f77 100644 --- a/storage/errors.nim +++ b/storage/errors.nim @@ -20,8 +20,8 @@ import pkg/questionable/results export results type - CodexError* = object of CatchableError # base codex error - CodexResult*[T] = Result[T, ref CodexError] + StorageError* = object of CatchableError # base Storage error + StorageResult*[T] = Result[T, ref StorageError] FinishedFailed*[T] = tuple[success: seq[Future[T]], failure: seq[Future[T]]] @@ -37,7 +37,7 @@ template mapFailure*[T, V, E]( ) template mapFailure*[T, V](exp: Result[T, V]): Result[T, ref CatchableError] = - mapFailure(exp, CodexError) + mapFailure(exp, StorageError) # TODO: using a template here, causes bad codegen func toFailure*[T](exp: Option[T]): Result[T, ref CatchableError] {.inline.} = diff --git a/storage/logutils.nim b/storage/logutils.nim index f3b98548..d45bc141 100644 --- a/storage/logutils.nim +++ b/storage/logutils.nim @@ -13,24 +13,24 @@ ## 6. Remove need to [avoid importing or exporting `toJson`, `%`, `%*` to prevent ## conflicts](https://github.com/logos-storage/logos-storage-nim/pull/645#issuecomment-1838834467) ## -## When declaring a new type, one should consider importing the `codex/logutils` +## When declaring a new type, one should consider importing the `storage/logutils` ## module, and specifying `formatIt`. If textlines log output and json log output ## need to be different, overload `formatIt` and specify a `LogFormat`. If json ## serialization is needed, it can be declared with a `%` proc. `logutils` ## imports and exports `nim-serde` which handles the de/serialization, examples -## below. **Only `codex/logutils` needs to be imported.** +## below. **Only `storage/logutils` needs to be imported.** ## -## Using `logutils` in the Codex codebase: -## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils` +## Using `logutils` in the Storage codebase: +## - Instead of importing `pkg/chronicles`, import `pkg/storage/logutils` ## - most of `chronicles` is exported by `logutils` ## - Instead of importing `std/json`, import `pkg/serde/json` ## - `std/json` is exported by `serde` which is exported by `logutils` ## - Instead of importing `pkg/nim-json-serialization`, import -## `pkg/serde/json` or use codex-specific overloads by importing `utils/json` +## `pkg/serde/json` or use storage-specific overloads by importing `utils/json` ## - one of the goals is to remove the use of `nim-json-serialization` ## ## ```nim -## import pkg/codex/logutils +## import pkg/storage/logutils ## ## type ## BlockAddress* = object diff --git a/storage/manifest/manifest.nim b/storage/manifest/manifest.nim index 7eed1088..90c105f2 100644 --- a/storage/manifest/manifest.nim +++ b/storage/manifest/manifest.nim @@ -71,7 +71,7 @@ func mimetype*(self: Manifest): ?string = ############################################################ func isManifest*(cid: Cid): ?!bool = - success (ManifestCodec == ?cid.contentType().mapFailure(CodexError)) + success (ManifestCodec == ?cid.contentType().mapFailure(StorageError)) func isManifest*(mc: MultiCodec): ?!bool = success mc == ManifestCodec diff --git a/storage/merkletree/merkletree.nim b/storage/merkletree/merkletree.nim index 33831227..4343dc73 100644 --- a/storage/merkletree/merkletree.nim +++ b/storage/merkletree/merkletree.nim @@ -22,14 +22,14 @@ import pkg/merkletree import ../utils import ../rng import ../errors -import ../codextypes +import ../storagetypes from ../utils/digest import digestBytes export merkletree logScope: - topics = "codex merkletree" + topics = "storage merkletree" type ByteTreeKey* {.pure.} = enum diff --git a/storage/multicodec_exts.nim b/storage/multicodec_exts.nim index ec629476..7171219a 100644 --- a/storage/multicodec_exts.nim +++ b/storage/multicodec_exts.nim @@ -1,2 +1,2 @@ const CodecExts = - [("codex-manifest", 0xCD01), ("codex-block", 0xCD02), ("codex-root", 0xCD03)] + [("storage-manifest", 0xCD01), ("storage-block", 0xCD02), ("storage-root", 0xCD03)] diff --git a/storage/namespaces.nim b/storage/namespaces.nim index 169be184..289cc9a3 100644 --- a/storage/namespaces.nim +++ b/storage/namespaces.nim @@ -9,17 +9,17 @@ const # Namespaces - CodexMetaNamespace* = "meta" # meta info stored here - CodexRepoNamespace* = "repo" # repository namespace, blocks and manifests are subkeys - CodexBlockTotalNamespace* = CodexMetaNamespace & "/total" + StorageMetaNamespace* = "meta" # meta info stored here + StorageRepoNamespace* = "repo" # repository namespace, blocks and manifests are subkeys + StorageBlockTotalNamespace* = StorageMetaNamespace & "/total" # number of blocks in the repo - CodexBlocksNamespace* = CodexRepoNamespace & "/blocks" # blocks namespace - CodexManifestNamespace* = CodexRepoNamespace & "/manifests" # manifest namespace - CodexBlocksTtlNamespace* = # Cid TTL - CodexMetaNamespace & "/ttl" - CodexBlockProofNamespace* = # Cid and Proof - CodexMetaNamespace & "/proof" - CodexDhtNamespace* = "dht" # Dht namespace - CodexDhtProvidersNamespace* = # Dht providers namespace - CodexDhtNamespace & "/providers" - CodexQuotaNamespace* = CodexMetaNamespace & "/quota" # quota's namespace + StorageBlocksNamespace* = StorageRepoNamespace & "/blocks" # blocks namespace + StorageManifestNamespace* = StorageRepoNamespace & "/manifests" # manifest namespace + StorageBlocksTtlNamespace* = # Cid TTL + StorageMetaNamespace & "/ttl" + StorageBlockProofNamespace* = # Cid and Proof + StorageMetaNamespace & "/proof" + StorageDhtNamespace* = "dht" # Dht namespace + StorageDhtProvidersNamespace* = # Dht providers namespace + StorageDhtNamespace & "/providers" + StorageQuotaNamespace* = StorageMetaNamespace & "/quota" # quota's namespace diff --git a/storage/nat.nim b/storage/nat.nim index f3919437..0a711df5 100644 --- a/storage/nat.nim +++ b/storage/nat.nim @@ -412,7 +412,7 @@ proc nattedAddress*( if ipPart.isSome and port.isSome: # Try to setup NAT mapping for the address let (newIP, tcp, udp) = - setupAddress(natConfig, ipPart.get, port.get, udpPort, "codex") + setupAddress(natConfig, ipPart.get, port.get, udpPort, "storage") if newIP.isSome: # NAT mapping successful - add discovery address with mapped UDP port discoveryAddrs.add(getMultiAddrWithIPAndUDPPort(newIP.get, udp.get)) diff --git a/storage/node.nim b/storage/node.nim index 1e2e0b80..ac04cb58 100644 --- a/storage/node.nim +++ b/storage/node.nim @@ -45,7 +45,7 @@ import ./utils/trackedfutures export logutils logScope: - topics = "codex node" + topics = "storage node" const DefaultFetchBatch = 1024 @@ -53,7 +53,7 @@ const BatchRefillThreshold = 0.75 # Refill when 75% of window completes type - CodexNode* = object + StorageNode* = object switch: Switch networkId: PeerId networkStore: NetworkStore @@ -63,27 +63,27 @@ type taskPool: Taskpool trackedFutures: TrackedFutures - CodexNodeRef* = ref CodexNode + StorageNodeRef* = ref StorageNode OnManifest* = proc(cid: Cid, manifest: Manifest): void {.gcsafe, raises: [].} BatchProc* = proc(blocks: seq[bt.Block]): Future[?!void] {.async: (raises: [CancelledError]).} OnBlockStoredProc = proc(chunk: seq[byte]): void {.gcsafe, raises: [].} -func switch*(self: CodexNodeRef): Switch = +func switch*(self: StorageNodeRef): Switch = return self.switch -func blockStore*(self: CodexNodeRef): BlockStore = +func blockStore*(self: StorageNodeRef): BlockStore = return self.networkStore -func engine*(self: CodexNodeRef): BlockExcEngine = +func engine*(self: StorageNodeRef): BlockExcEngine = return self.engine -func discovery*(self: CodexNodeRef): Discovery = +func discovery*(self: StorageNodeRef): Discovery = return self.discovery proc storeManifest*( - self: CodexNodeRef, manifest: Manifest + self: StorageNodeRef, manifest: Manifest ): Future[?!bt.Block] {.async.} = without encodedVerifiable =? manifest.encode(), err: trace "Unable to encode manifest" @@ -100,7 +100,7 @@ proc storeManifest*( success blk proc fetchManifest*( - self: CodexNodeRef, cid: Cid + self: StorageNodeRef, cid: Cid ): Future[?!Manifest] {.async: (raises: [CancelledError]).} = ## Fetch and decode a manifest block ## @@ -124,18 +124,18 @@ proc fetchManifest*( return manifest.success -proc findPeer*(self: CodexNodeRef, peerId: PeerId): Future[?PeerRecord] {.async.} = - ## Find peer using the discovery service from the given CodexNode +proc findPeer*(self: StorageNodeRef, peerId: PeerId): Future[?PeerRecord] {.async.} = + ## Find peer using the discovery service from the given StorageNode ## return await self.discovery.findPeer(peerId) proc connect*( - self: CodexNodeRef, peerId: PeerId, addrs: seq[MultiAddress] + self: StorageNodeRef, peerId: PeerId, addrs: seq[MultiAddress] ): Future[void] = self.switch.connect(peerId, addrs) proc updateExpiry*( - self: CodexNodeRef, manifestCid: Cid, expiry: SecondsSince1970 + self: StorageNodeRef, manifestCid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async: (raises: [CancelledError]).} = without manifest =? await self.fetchManifest(manifestCid), error: trace "Unable to fetch manifest for cid", manifestCid @@ -158,7 +158,7 @@ proc updateExpiry*( return success() proc fetchBatched*( - self: CodexNodeRef, + self: StorageNodeRef, cid: Cid, iter: Iter[int], batchSize = DefaultFetchBatch, @@ -233,7 +233,7 @@ proc fetchBatched*( success() proc fetchBatched*( - self: CodexNodeRef, + self: StorageNodeRef, manifest: Manifest, batchSize = DefaultFetchBatch, onBatch: BatchProc = nil, @@ -249,7 +249,7 @@ proc fetchBatched*( self.fetchBatched(manifest.treeCid, iter, batchSize, onBatch, fetchLocal) proc fetchDatasetAsync*( - self: CodexNodeRef, manifest: Manifest, fetchLocal = true + self: StorageNodeRef, manifest: Manifest, fetchLocal = true ): Future[void] {.async: (raises: []).} = ## Asynchronously fetch a dataset in the background. ## This task will be tracked and cleaned up on node shutdown. @@ -264,14 +264,14 @@ proc fetchDatasetAsync*( except CancelledError as exc: trace "Cancelled fetching blocks", exc = exc.msg -proc fetchDatasetAsyncTask*(self: CodexNodeRef, manifest: Manifest) = +proc fetchDatasetAsyncTask*(self: StorageNodeRef, manifest: Manifest) = ## Start fetching a dataset in the background. ## The task will be tracked and cleaned up on node shutdown. ## self.trackedFutures.track(self.fetchDatasetAsync(manifest, fetchLocal = false)) proc streamSingleBlock( - self: CodexNodeRef, cid: Cid + self: StorageNodeRef, cid: Cid ): Future[?!LPStream] {.async: (raises: [CancelledError]).} = ## Streams the contents of a single block. ## @@ -296,7 +296,7 @@ proc streamSingleBlock( LPStream(stream).success proc streamEntireDataset( - self: CodexNodeRef, manifest: Manifest, manifestCid: Cid + self: StorageNodeRef, manifest: Manifest, manifestCid: Cid ): Future[?!LPStream] {.async: (raises: [CancelledError]).} = ## Streams the contents of the entire dataset described by the manifest. ## @@ -324,7 +324,7 @@ proc streamEntireDataset( stream.success proc retrieve*( - self: CodexNodeRef, cid: Cid, local: bool = true + self: StorageNodeRef, cid: Cid, local: bool = true ): Future[?!LPStream] {.async: (raises: [CancelledError]).} = ## Retrieve by Cid a single block or an entire dataset described by manifest ## @@ -340,7 +340,7 @@ proc retrieve*( await self.streamEntireDataset(manifest, cid) -proc deleteSingleBlock(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} = +proc deleteSingleBlock(self: StorageNodeRef, cid: Cid): Future[?!void] {.async.} = if err =? (await self.networkStore.delBlock(cid)).errorOption: error "Error deleting block", cid, err = err.msg return failure(err) @@ -348,7 +348,7 @@ proc deleteSingleBlock(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} = trace "Deleted block", cid return success() -proc deleteEntireDataset(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} = +proc deleteEntireDataset(self: StorageNodeRef, cid: Cid): Future[?!void] {.async.} = # Deletion is a strictly local operation var store = self.networkStore.localStore @@ -382,7 +382,7 @@ proc deleteEntireDataset(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} success() proc delete*( - self: CodexNodeRef, cid: Cid + self: StorageNodeRef, cid: Cid ): Future[?!void] {.async: (raises: [CatchableError]).} = ## Deletes a whole dataset, if Cid is a Manifest Cid, or a single block, if Cid a block Cid, ## from the underlying block store. This is a strictly local operation. @@ -400,7 +400,7 @@ proc delete*( await self.deleteEntireDataset(cid) proc store*( - self: CodexNodeRef, + self: StorageNodeRef, stream: LPStream, filename: ?string = string.none, mimetype: ?string = string.none, @@ -484,7 +484,7 @@ proc store*( return manifestBlk.cid.success -proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} = +proc iterateManifests*(self: StorageNodeRef, onManifest: OnManifest) {.async.} = without cidsIter =? await self.networkStore.listBlocks(BlockType.Manifest): warn "Failed to listBlocks" return @@ -502,11 +502,11 @@ proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} = onManifest(cid, manifest) proc onExpiryUpdate( - self: CodexNodeRef, rootCid: Cid, expiry: SecondsSince1970 + self: StorageNodeRef, rootCid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async: (raises: [CancelledError]).} = return await self.updateExpiry(rootCid, expiry) -proc start*(self: CodexNodeRef) {.async.} = +proc start*(self: StorageNodeRef) {.async.} = if not self.engine.isNil: await self.engine.start() @@ -519,7 +519,7 @@ proc start*(self: CodexNodeRef) {.async.} = self.networkId = self.switch.peerInfo.peerId notice "Started Storage node", id = self.networkId, addrs = self.switch.peerInfo.addrs -proc stop*(self: CodexNodeRef) {.async.} = +proc stop*(self: StorageNodeRef) {.async.} = trace "Stopping node" await self.trackedFutures.cancelTracked() @@ -533,22 +533,22 @@ proc stop*(self: CodexNodeRef) {.async.} = if not self.clock.isNil: await self.clock.stop() -proc close*(self: CodexNodeRef) {.async.} = +proc close*(self: StorageNodeRef) {.async.} = if not self.networkStore.isNil: await self.networkStore.close proc new*( - T: type CodexNodeRef, + T: type StorageNodeRef, switch: Switch, networkStore: NetworkStore, engine: BlockExcEngine, discovery: Discovery, taskpool: Taskpool, -): CodexNodeRef = - ## Create new instance of a Codex self, call `start` to run it +): StorageNodeRef = + ## Create new instance of a Storage self, call `start` to run it ## - CodexNodeRef( + StorageNodeRef( switch: switch, networkStore: networkStore, engine: engine, @@ -558,7 +558,7 @@ proc new*( ) proc hasLocalBlock*( - self: CodexNodeRef, cid: Cid + self: StorageNodeRef, cid: Cid ): Future[bool] {.async: (raises: [CancelledError]).} = ## Returns true if the given Cid is present in the local store diff --git a/storage/rest/api.nim b/storage/rest/api.nim index bf08a2e7..efe87964 100644 --- a/storage/rest/api.nim +++ b/storage/rest/api.nim @@ -39,10 +39,10 @@ import ./coders import ./json logScope: - topics = "codex restapi" + topics = "storage restapi" -declareCounter(codex_api_uploads, "codex API uploads") -declareCounter(codex_api_downloads, "codex API downloads") +declareCounter(storage_api_uploads, "storage API uploads") +declareCounter(storage_api_downloads, "storage API downloads") proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].} = 0 @@ -50,7 +50,7 @@ proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].} proc formatManifest(cid: Cid, manifest: Manifest): RestContent = return RestContent.init(cid, manifest) -proc formatManifestBlocks(node: CodexNodeRef): Future[JsonNode] {.async.} = +proc formatManifestBlocks(node: StorageNodeRef): Future[JsonNode] {.async.} = var content: seq[RestContent] proc addManifest(cid: Cid, manifest: Manifest) = @@ -67,7 +67,7 @@ proc isPending(resp: HttpResponseRef): bool = return resp.getResponseState() == HttpResponseState.Empty proc retrieveCid( - node: CodexNodeRef, cid: Cid, local: bool = true, resp: HttpResponseRef + node: StorageNodeRef, cid: Cid, local: bool = true, resp: HttpResponseRef ): Future[void] {.async: (raises: [CancelledError, HttpWriteError]).} = ## Download a file from the node in a streaming ## manner @@ -131,7 +131,7 @@ proc retrieveCid( await resp.send(addr buff[0], buff.len) await resp.finish() - codex_api_downloads.inc() + storage_api_downloads.inc() except CancelledError as exc: raise exc except LPStreamError as exc: @@ -173,7 +173,7 @@ proc getFilenameFromContentDisposition(contentDisposition: string): ?string = let filename = parts[1].strip() return filename[0 ..^ 2].some -proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) = +proc initDataApi(node: StorageNodeRef, repoStore: RepoStore, router: var RestRouter) = let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion router.api(MethodOptions, "/api/storage/v1/data") do( @@ -238,7 +238,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute error "Error uploading file", exc = error.msg return RestApiResponse.error(Http500, error.msg) - codex_api_uploads.inc() + storage_api_uploads.inc() trace "Uploaded file", cid return RestApiResponse.response($cid) except CancelledError: @@ -383,7 +383,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute ) return RestApiResponse.response($json, contentType = "application/json") -proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = +proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter) = let allowedOrigin = router.allowedOrigin ## various node management api's @@ -465,7 +465,7 @@ proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = return RestApiResponse.error(Http500, "Unknown error dialling peer", headers = headers) -proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = +proc initDebugApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter) = let allowedOrigin = router.allowedOrigin router.api(MethodGet, "/api/storage/v1/debug/info") do() -> RestApiResponse: @@ -476,16 +476,20 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = try: let table = RestRoutingTable.init(node.discovery.protocol.routingTable) - let json = %*{ - "id": $node.switch.peerInfo.peerId, - "addrs": node.switch.peerInfo.addrs.mapIt($it), - "repo": $conf.dataDir, - "spr": - if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "", - "announceAddresses": node.discovery.announceAddrs, - "table": table, - "storage": {"version": $codexVersion, "revision": $codexRevision}, - } + let json = + %*{ + "id": $node.switch.peerInfo.peerId, + "addrs": node.switch.peerInfo.addrs.mapIt($it), + "repo": $conf.dataDir, + "spr": + if node.discovery.dhtRecord.isSome: + node.discovery.dhtRecord.get.toURI + else: + "", + "announceAddresses": node.discovery.announceAddrs, + "table": table, + "storage": {"version": $storageVersion, "revision": $storageRevision}, + } # return pretty json for human readability return RestApiResponse.response( @@ -542,8 +546,8 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = return RestApiResponse.error(Http500, headers = headers) proc initRestApi*( - node: CodexNodeRef, - conf: CodexConf, + node: StorageNodeRef, + conf: StorageConf, repoStore: RepoStore, corsAllowedOrigin: ?string, ): RestRouter = diff --git a/storage/storage.nim b/storage/storage.nim index 001bae89..6460a8b3 100644 --- a/storage/storage.nim +++ b/storage/storage.nim @@ -35,35 +35,35 @@ import ./discovery import ./systemclock import ./utils/addrutils import ./namespaces -import ./codextypes +import ./storagetypes import ./logutils import ./nat logScope: - topics = "codex node" + topics = "storage node" type - CodexServer* = ref object - config: CodexConf + StorageServer* = ref object + config: StorageConf restServer: RestServerRef - codexNode: CodexNodeRef + storageNode: StorageNodeRef repoStore: RepoStore maintenance: BlockMaintainer taskpool: Taskpool isStarted: bool - CodexPrivateKey* = libp2p.PrivateKey # alias + StoragePrivateKey* = libp2p.PrivateKey # alias -func config*(self: CodexServer): CodexConf = +func config*(self: StorageServer): StorageConf = return self.config -func node*(self: CodexServer): CodexNodeRef = - return self.codexNode +func node*(self: StorageServer): StorageNodeRef = + return self.storageNode -func repoStore*(self: CodexServer): RepoStore = +func repoStore*(self: StorageServer): RepoStore = return self.repoStore -proc start*(s: CodexServer) {.async.} = +proc start*(s: StorageServer) {.async.} = if s.isStarted: warn "Storage server already started, skipping" return @@ -73,35 +73,36 @@ proc start*(s: CodexServer) {.async.} = s.maintenance.start() - await s.codexNode.switch.start() + await s.storageNode.switch.start() let (announceAddrs, discoveryAddrs) = nattedAddress( - s.config.nat, s.codexNode.switch.peerInfo.addrs, s.config.discoveryPort + s.config.nat, s.storageNode.switch.peerInfo.addrs, s.config.discoveryPort ) - s.codexNode.discovery.updateAnnounceRecord(announceAddrs) - s.codexNode.discovery.updateDhtRecord(discoveryAddrs) + s.storageNode.discovery.updateAnnounceRecord(announceAddrs) + s.storageNode.discovery.updateDhtRecord(discoveryAddrs) - await s.codexNode.start() + await s.storageNode.start() if s.restServer != nil: s.restServer.start() s.isStarted = true -proc stop*(s: CodexServer) {.async.} = +proc stop*(s: StorageServer) {.async.} = if not s.isStarted: warn "Storage is not started" return notice "Stopping Storage node" - var futures = @[ - s.codexNode.switch.stop(), - s.codexNode.stop(), - s.repoStore.stop(), - s.maintenance.stop(), - ] + var futures = + @[ + s.storageNode.switch.stop(), + s.storageNode.stop(), + s.repoStore.stop(), + s.maintenance.stop(), + ] if s.restServer != nil: futures.add(s.restServer.stop()) @@ -114,9 +115,9 @@ proc stop*(s: CodexServer) {.async.} = error "Failed to stop Storage node", failures = res.failure.len raiseAssert "Failed to stop Storage node" -proc close*(s: CodexServer) {.async.} = +proc close*(s: StorageServer) {.async.} = var futures = - @[s.codexNode.close(), s.repoStore.close(), s.codexNode.discovery.close()] + @[s.storageNode.close(), s.repoStore.close(), s.storageNode.discovery.close()] let res = await noCancel allFinishedFailed[void](futures) @@ -131,14 +132,14 @@ proc close*(s: CodexServer) {.async.} = error "Failed to close Storage node", failures = res.failure.len raiseAssert "Failed to close Storage node" -proc shutdown*(server: CodexServer) {.async.} = +proc shutdown*(server: StorageServer) {.async.} = await server.stop() await server.close() proc new*( - T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey -): CodexServer = - ## create CodexServer including setting up datastore, repostore, etc + T: type StorageServer, config: StorageConf, privateKey: StoragePrivateKey +): StorageServer = + ## create StorageServer including setting up datastore, repostore, etc let switch = SwitchBuilder .new() .withPrivateKey(privateKey) @@ -169,7 +170,7 @@ proc new*( cache = CacheStore.new(cacheSize = config.cacheSize) ## Is unused? - let discoveryDir = config.dataDir / CodexDhtNamespace + let discoveryDir = config.dataDir / StorageDhtNamespace if io2.createPath(discoveryDir).isErr: trace "Unable to create discovery directory for block store", @@ -178,7 +179,7 @@ proc new*( msg: "Unable to create discovery directory for block store: " & discoveryDir ) - let providersPath = config.dataDir / CodexDhtProvidersNamespace + let providersPath = config.dataDir / StorageDhtProvidersNamespace let discoveryStoreRes = LevelDbDatastore.new(providersPath) if discoveryStoreRes.isErr: error "Failed to initialize discovery datastore", @@ -221,7 +222,7 @@ proc new*( repoStore = RepoStore.new( repoDs = repoData, - metaDs = LevelDbDatastore.new(config.dataDir / CodexMetaNamespace).expect( + metaDs = LevelDbDatastore.new(config.dataDir / StorageMetaNamespace).expect( "Should create metadata store!" ), quotaMaxBytes = config.storageQuota, @@ -244,7 +245,7 @@ proc new*( ) store = NetworkStore.new(engine, repoStore) - codexNode = CodexNodeRef.new( + storageNode = StorageNodeRef.new( switch = switch, networkStore = store, engine = engine, @@ -257,7 +258,7 @@ proc new*( if config.apiBindAddress.isSome: restServer = RestServerRef .new( - codexNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin), + storageNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin), initTAddress(config.apiBindAddress.get(), config.apiPort), bufferSize = (1024 * 64), maxRequestBodySize = int.high, @@ -266,9 +267,9 @@ proc new*( switch.mount(network) - CodexServer( + StorageServer( config: config, - codexNode: codexNode, + storageNode: storageNode, restServer: restServer, repoStore: repoStore, maintenance: maintenance, diff --git a/storage/codextypes.nim b/storage/storagetypes.nim similarity index 90% rename from storage/codextypes.nim rename to storage/storagetypes.nim index bd4dfcf8..d9bc790e 100644 --- a/storage/codextypes.nim +++ b/storage/storagetypes.nim @@ -30,11 +30,11 @@ const # hashes Sha256HashCodec* = multiCodec("sha2-256") - ManifestCodec* = multiCodec("codex-manifest") - DatasetRootCodec* = multiCodec("codex-root") - BlockCodec* = multiCodec("codex-block") + ManifestCodec* = multiCodec("storage-manifest") + DatasetRootCodec* = multiCodec("storage-root") + BlockCodec* = multiCodec("storage-block") - CodexPrimitivesCodecs* = [ManifestCodec, DatasetRootCodec, BlockCodec] + StoragePrimitivesCodecs* = [ManifestCodec, DatasetRootCodec, BlockCodec] proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] = ## Initialize padding blocks table diff --git a/storage/stores/blockstore.nim b/storage/stores/blockstore.nim index 7edabc4d..5abed158 100644 --- a/storage/stores/blockstore.nim +++ b/storage/stores/blockstore.nim @@ -22,7 +22,7 @@ import ../utils export blocktype type - BlockNotFoundError* = object of CodexError + BlockNotFoundError* = object of StorageError BlockType* {.pure.} = enum Manifest diff --git a/storage/stores/cachestore.nim b/storage/stores/cachestore.nim index 15e1bc74..62a5a4f0 100644 --- a/storage/stores/cachestore.nim +++ b/storage/stores/cachestore.nim @@ -30,7 +30,7 @@ import ../clock export blockstore logScope: - topics = "codex cachestore" + topics = "storage cachestore" type CacheStore* = ref object of BlockStore @@ -40,7 +40,7 @@ type cidAndProofCache: LruCache[(Cid, Natural), (Cid, StorageMerkleProof)] cidAndProofCache: LruCache[(Cid, Natural), (Cid, StorageMerkleProof)] - InvalidBlockSize* = object of CodexError + InvalidBlockSize* = object of StorageError const DefaultCacheSize*: NBytes = 5.MiBs diff --git a/storage/stores/keyutils.nim b/storage/stores/keyutils.nim index 78efa732..67e14f86 100644 --- a/storage/stores/keyutils.nim +++ b/storage/stores/keyutils.nim @@ -17,14 +17,14 @@ import ../namespaces import ../manifest const - CodexMetaKey* = Key.init(CodexMetaNamespace).tryGet - CodexRepoKey* = Key.init(CodexRepoNamespace).tryGet - CodexBlocksKey* = Key.init(CodexBlocksNamespace).tryGet - CodexTotalBlocksKey* = Key.init(CodexBlockTotalNamespace).tryGet - CodexManifestKey* = Key.init(CodexManifestNamespace).tryGet - BlocksTtlKey* = Key.init(CodexBlocksTtlNamespace).tryGet - BlockProofKey* = Key.init(CodexBlockProofNamespace).tryGet - QuotaKey* = Key.init(CodexQuotaNamespace).tryGet + StorageMetaKey* = Key.init(StorageMetaNamespace).tryGet + StorageRepoKey* = Key.init(StorageRepoNamespace).tryGet + StorageBlocksKey* = Key.init(StorageBlocksNamespace).tryGet + StorageTotalBlocksKey* = Key.init(StorageBlockTotalNamespace).tryGet + StorageManifestKey* = Key.init(StorageManifestNamespace).tryGet + BlocksTtlKey* = Key.init(StorageBlocksTtlNamespace).tryGet + BlockProofKey* = Key.init(StorageBlockProofNamespace).tryGet + QuotaKey* = Key.init(StorageQuotaNamespace).tryGet QuotaUsedKey* = (QuotaKey / "used").tryGet QuotaReservedKey* = (QuotaKey / "reserved").tryGet @@ -32,9 +32,9 @@ func makePrefixKey*(postFixLen: int, cid: Cid): ?!Key = let cidKey = ?Key.init(($cid)[^postFixLen ..^ 1] & "/" & $cid) if ?cid.isManifest: - success CodexManifestKey / cidKey + success StorageManifestKey / cidKey else: - success CodexBlocksKey / cidKey + success StorageBlocksKey / cidKey proc createBlockExpirationMetadataKey*(cid: Cid): ?!Key = BlocksTtlKey / $cid diff --git a/storage/stores/maintenance.nim b/storage/stores/maintenance.nim index b2fece4d..ffdcafbc 100644 --- a/storage/stores/maintenance.nim +++ b/storage/stores/maintenance.nim @@ -24,7 +24,7 @@ import ../logutils import ../systemclock logScope: - topics = "codex maintenance" + topics = "storage maintenance" const DefaultBlockInterval* = 10.minutes diff --git a/storage/stores/networkstore.nim b/storage/stores/networkstore.nim index 6aa76d8c..be0b0862 100644 --- a/storage/stores/networkstore.nim +++ b/storage/stores/networkstore.nim @@ -25,7 +25,7 @@ import ./blockstore export blockstore, blockexchange, asyncheapqueue logScope: - topics = "codex networkstore" + topics = "storage networkstore" type NetworkStore* = ref object of BlockStore engine*: BlockExcEngine # blockexc decision engine diff --git a/storage/stores/repostore/operations.nim b/storage/stores/repostore/operations.nim index efd7f56f..f2e67094 100644 --- a/storage/stores/repostore/operations.nim +++ b/storage/stores/repostore/operations.nim @@ -26,11 +26,11 @@ import ../../logutils import ../../merkletree logScope: - topics = "codex repostore" + topics = "storage repostore" -declareGauge(codex_repostore_blocks, "codex repostore blocks") -declareGauge(codex_repostore_bytes_used, "codex repostore bytes used") -declareGauge(codex_repostore_bytes_reserved, "codex repostore bytes reserved") +declareGauge(storage_repostore_blocks, "storage repostore blocks") +declareGauge(storage_repostore_bytes_used, "storage repostore bytes used") +declareGauge(storage_repostore_bytes_reserved, "storage repostore bytes reserved") proc putLeafMetadata*( self: RepoStore, @@ -90,7 +90,7 @@ proc updateTotalBlocksCount*( self: RepoStore, plusCount: Natural = 0, minusCount: Natural = 0 ): Future[?!void] {.async: (raises: [CancelledError]).} = await self.metaDs.modify( - CodexTotalBlocksKey, + StorageTotalBlocksKey, proc(maybeCurrCount: ?Natural): Future[?Natural] {.async.} = let count: Natural = if currCount =? maybeCurrCount: @@ -99,7 +99,7 @@ proc updateTotalBlocksCount*( plusCount - minusCount self.totalBlocks = count - codex_repostore_blocks.set(count.int64) + storage_repostore_blocks.set(count.int64) count.some, ) @@ -132,8 +132,8 @@ proc updateQuotaUsage*( ) else: self.quotaUsage = usage - codex_repostore_bytes_used.set(usage.used.int64) - codex_repostore_bytes_reserved.set(usage.reserved.int64) + storage_repostore_bytes_used.set(usage.used.int64) + storage_repostore_bytes_reserved.set(usage.reserved.int64) return usage.some, ) diff --git a/storage/stores/repostore/store.nim b/storage/stores/repostore/store.nim index 554bd62c..feec3807 100644 --- a/storage/stores/repostore/store.nim +++ b/storage/stores/repostore/store.nim @@ -32,7 +32,7 @@ import ../../utils export blocktype, cid logScope: - topics = "codex repostore" + topics = "storage repostore" ########################################################### # BlockStore API @@ -323,9 +323,9 @@ method listBlocks*( let key = case blockType - of BlockType.Manifest: CodexManifestKey - of BlockType.Block: CodexBlocksKey - of BlockType.Both: CodexRepoKey + of BlockType.Manifest: StorageManifestKey + of BlockType.Block: StorageBlocksKey + of BlockType.Both: StorageRepoKey let query = Query.init(key, value = false) without queryIter =? (await self.repoDs.query(query)), err: @@ -444,7 +444,7 @@ proc release*( proc start*( self: RepoStore -): Future[void] {.async: (raises: [CancelledError, CodexError]).} = +): Future[void] {.async: (raises: [CancelledError, StorageError]).} = ## Start repo ## if self.started: @@ -453,10 +453,10 @@ proc start*( trace "Starting rep" if err =? (await self.updateTotalBlocksCount()).errorOption: - raise newException(CodexError, err.msg) + raise newException(StorageError, err.msg) if err =? (await self.updateQuotaUsage()).errorOption: - raise newException(CodexError, err.msg) + raise newException(StorageError, err.msg) self.started = true diff --git a/storage/stores/repostore/types.nim b/storage/stores/repostore/types.nim index 6936b92d..1d6c743a 100644 --- a/storage/stores/repostore/types.nim +++ b/storage/stores/repostore/types.nim @@ -25,7 +25,7 @@ const DefaultQuotaBytes* = 20.GiBs type - QuotaNotEnoughError* = object of CodexError + QuotaNotEnoughError* = object of StorageError RepoStore* = ref object of BlockStore postFixLen*: int diff --git a/storage/streams/seekablestream.nim b/storage/streams/seekablestream.nim index 926fa70d..fc467b2e 100644 --- a/storage/streams/seekablestream.nim +++ b/storage/streams/seekablestream.nim @@ -15,7 +15,7 @@ import ../logutils export lpstream, chronos, logutils logScope: - topics = "codex seekablestream" + topics = "storage seekablestream" type SeekableStream* = ref object of LPStream offset*: int diff --git a/storage/streams/storestream.nim b/storage/streams/storestream.nim index d2fc51e9..b3297baf 100644 --- a/storage/streams/storestream.nim +++ b/storage/streams/storestream.nim @@ -25,7 +25,7 @@ import ./seekablestream export stores, blocktype, manifest, chronos logScope: - topics = "codex storestream" + topics = "storage storestream" const StoreStreamTrackerName* = "StoreStream" diff --git a/storage/utils/keyutils.nim b/storage/utils/keyutils.nim index dcdc17bb..372777c2 100644 --- a/storage/utils/keyutils.nim +++ b/storage/utils/keyutils.nim @@ -20,25 +20,25 @@ import ../rng export crypto type - CodexKeyError = object of CodexError - CodexKeyUnsafeError = object of CodexKeyError + StorageKeyError = object of StorageError + StorageKeyUnsafeError = object of StorageKeyError proc setupKey*(path: string): ?!PrivateKey = if not path.fileAccessible({AccessFlags.Find}): info "Creating a private key and saving it" let - res = ?PrivateKey.random(Rng.instance()[]).mapFailure(CodexKeyError) - bytes = ?res.getBytes().mapFailure(CodexKeyError) + res = ?PrivateKey.random(Rng.instance()[]).mapFailure(StorageKeyError) + bytes = ?res.getBytes().mapFailure(StorageKeyError) - ?path.secureWriteFile(bytes).mapFailure(CodexKeyError) - return PrivateKey.init(bytes).mapFailure(CodexKeyError) + ?path.secureWriteFile(bytes).mapFailure(StorageKeyError) + return PrivateKey.init(bytes).mapFailure(StorageKeyError) info "Found a network private key" - if not ?checkSecureFile(path).mapFailure(CodexKeyError): + if not ?checkSecureFile(path).mapFailure(StorageKeyError): warn "The network private key file is not safe, aborting" return failure newException( - CodexKeyUnsafeError, "The network private key file is not safe" + StorageKeyUnsafeError, "The network private key file is not safe" ) - let kb = ?path.readAllBytes().mapFailure(CodexKeyError) - return PrivateKey.init(kb).mapFailure(CodexKeyError) + let kb = ?path.readAllBytes().mapFailure(StorageKeyError) + return PrivateKey.init(kb).mapFailure(StorageKeyError) diff --git a/tests/coverage.nim b/tests/coverage.nim index b40b8f32..0c36e6a5 100644 --- a/tests/coverage.nim +++ b/tests/coverage.nim @@ -1 +1 @@ -include ./testCodex +include ./testStorage diff --git a/tests/examples.nim b/tests/examples.nim index 4d413c3c..710bbcfb 100644 --- a/tests/examples.nim +++ b/tests/examples.nim @@ -3,15 +3,15 @@ import std/sequtils import std/times import std/typetraits -import pkg/codex/rng -import pkg/codex/stores -import pkg/codex/units +import pkg/storage/rng +import pkg/storage/stores +import pkg/storage/units import pkg/chronos import pkg/stew/byteutils import pkg/stint -import ./codex/helpers/randomchunker +import ./storage/helpers/randomchunker export randomchunker export units diff --git a/tests/helpers/trackers.nim b/tests/helpers/trackers.nim index 898053c2..e0b72d09 100644 --- a/tests/helpers/trackers.nim +++ b/tests/helpers/trackers.nim @@ -1,4 +1,4 @@ -import pkg/codex/streams/storestream +import pkg/storage/streams/storestream import pkg/unittest2 # From lip2p/tests/helpers diff --git a/tests/integration/1_minute/testblockexpiration.nim b/tests/integration/1_minute/testblockexpiration.nim index be66c388..cc518afd 100644 --- a/tests/integration/1_minute/testblockexpiration.nim +++ b/tests/integration/1_minute/testblockexpiration.nim @@ -9,7 +9,7 @@ multinodesuite "Node block expiration tests": test "node retains not-expired file", NodeConfigs( - clients: CodexConfigs + clients: StorageConfigs .init(nodes = 1) .withBlockTtl(0, 10) .withBlockMaintenanceInterval(0, 1).some @@ -29,7 +29,7 @@ multinodesuite "Node block expiration tests": test "node deletes expired file", NodeConfigs( - clients: CodexConfigs + clients: StorageConfigs .init(nodes = 1) .withBlockTtl(0, 1) .withBlockMaintenanceInterval(0, 1).some diff --git a/tests/integration/5_minutes/testrestapi.nim b/tests/integration/5_minutes/testrestapi.nim index 4bbf6fa9..fde9a8ce 100644 --- a/tests/integration/5_minutes/testrestapi.nim +++ b/tests/integration/5_minutes/testrestapi.nim @@ -3,12 +3,12 @@ import std/net import std/sequtils import std/strformat from pkg/libp2p import `==`, `$`, Cid -import pkg/codex/units -import pkg/codex/manifest +import pkg/storage/units +import pkg/storage/manifest import ../twonodes import ../../examples -import ../../codex/examples -import ../../codex/slots/helpers +import ../../storage/examples +import ../../storage/slots/helpers import json twonodessuite "REST API": @@ -16,7 +16,7 @@ twonodessuite "REST API": check !(await client1.info()) != !(await client2.info()) test "nodes can set chronicles log level", twoNodesConfig: - await client1.setLogLevel("DEBUG;TRACE:codex") + await client1.setLogLevel("DEBUG;TRACE:storage") test "node accepts file uploads", twoNodesConfig: let cid1 = (await client1.upload("some file contents")).get diff --git a/tests/integration/5_minutes/testrestapivalidation.nim b/tests/integration/5_minutes/testrestapivalidation.nim index e11a0d7b..e13f9d1f 100644 --- a/tests/integration/5_minutes/testrestapivalidation.nim +++ b/tests/integration/5_minutes/testrestapivalidation.nim @@ -1,18 +1,18 @@ import std/times -import pkg/codex/conf +import pkg/storage/conf import pkg/stint from pkg/libp2p import Cid, `$` import ../../asynctest import ../../checktest import ../../examples -import ../../codex/examples -import ../codexconfig -import ../codexclient +import ../../storage/examples +import ../storageconfig +import ../storageclient import ../multinodes multinodesuite "Rest API validation": - let config = NodeConfigs(clients: CodexConfigs.init(nodes = 1).some) - var client: CodexClient + let config = NodeConfigs(clients: StorageConfigs.init(nodes = 1).some) + var client: StorageClient setup: client = clients()[0].client diff --git a/tests/integration/5_minutes/testupdownload.nim b/tests/integration/5_minutes/testupdownload.nim index ec5054c8..7545f948 100644 --- a/tests/integration/5_minutes/testupdownload.nim +++ b/tests/integration/5_minutes/testupdownload.nim @@ -1,6 +1,6 @@ -import pkg/codex/rest/json +import pkg/storage/rest/json import ../twonodes -import ../../codex/examples +import ../../storage/examples import json from pkg/libp2p import Cid, `$` @@ -84,7 +84,7 @@ twonodessuite "Uploads and downloads": content1 == resp2 test "reliable transfer test", twoNodesConfig: - proc transferTest(a: CodexClient, b: CodexClient) {.async.} = + proc transferTest(a: StorageClient, b: StorageClient) {.async.} = let data = await RandomChunker.example(blocks = 8) let cid = (await a.upload(data)).get let response = (await b.download(cid)).get diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 19c5b5b9..8d408b59 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -4,20 +4,20 @@ import std/sequtils import std/strutils import std/sugar import std/times -import pkg/codex/conf -import pkg/codex/logutils +import pkg/storage/conf +import pkg/storage/logutils import pkg/chronos/transports/stream import pkg/questionable -import ./codexconfig -import ./codexprocess +import ./storageconfig +import ./storageprocess import ./nodeconfigs import ./utils import ../asynctest import ../checktest export asynctest -export codexprocess -export codexconfig +export storageprocess +export storageconfig export nodeconfigs {.push raises: [].} @@ -34,12 +34,12 @@ type SuiteTimeoutError = object of MultiNodeSuiteError const HardhatPort {.intdefine.}: int = 8545 -const CodexApiPort {.intdefine.}: int = 8080 -const CodexDiscPort {.intdefine.}: int = 8090 +const StorageApiPort {.intdefine.}: int = 8080 +const StorageDiscPort {.intdefine.}: int = 8090 const TestId {.strdefine.}: string = "TestId" -const CodexLogToFile {.booldefine.}: bool = false -const CodexLogLevel {.strdefine.}: string = "" -const CodexLogsDir {.strdefine.}: string = "" +const StorageLogToFile {.booldefine.}: bool = false +const StorageLogLevel {.strdefine.}: string = "" +const StorageLogsDir {.strdefine.}: string = "" proc raiseMultiNodeSuiteError( msg: string, parent: ref CatchableError = nil @@ -77,9 +77,9 @@ template multinodesuite*(suiteName: string, body: untyped) = var nodeConfigs: NodeConfigs var snapshot: JsonNode var lastUsedHardhatPort = HardhatPort - var lastUsedCodexApiPort = CodexApiPort - var lastUsedCodexDiscPort = CodexDiscPort - var codexPortLock: AsyncLock + var lastUsedStorageApiPort = StorageApiPort + var lastUsedStorageDiscPort = StorageDiscPort + var storagePortLock: AsyncLock template test(tname, startNodeConfigs, tbody) = currentTestName = tname @@ -91,18 +91,18 @@ template multinodesuite*(suiteName: string, body: untyped) = let parts = url.split(':') url = @[parts[0], parts[1], $port].join(":") - proc newCodexProcess( - roleIdx: int, conf: CodexConfig, role: Role + proc newStorageProcess( + roleIdx: int, conf: StorageConfig, role: Role ): Future[NodeProcess] {.async: (raises: [MultiNodeSuiteError, CancelledError]).} = let nodeIdx = running.len var config = conf let datadir = getDataDir(TestId, currentTestName, $starttime, $role, some roleIdx) try: - if config.logFile.isSome or CodexLogToFile: + if config.logFile.isSome or StorageLogToFile: try: let updatedLogFile = getLogFile( - CodexLogsDir, starttime, suiteName, currentTestName, $role, some roleIdx + StorageLogsDir, starttime, suiteName, currentTestName, $role, some roleIdx ) config.withLogFile(updatedLogFile) except IOError as e: @@ -118,17 +118,17 @@ template multinodesuite*(suiteName: string, body: untyped) = e, ) - when CodexLogLevel != "": - config.addCliOption("--log-level", CodexLogLevel) + when StorageLogLevel != "": + config.addCliOption("--log-level", StorageLogLevel) var apiPort, discPort: int - withLock(codexPortLock): - apiPort = await nextFreePort(lastUsedCodexApiPort + nodeIdx) - discPort = await nextFreePort(lastUsedCodexDiscPort + nodeIdx) + withLock(storagePortLock): + apiPort = await nextFreePort(lastUsedStorageApiPort + nodeIdx) + discPort = await nextFreePort(lastUsedStorageDiscPort + nodeIdx) config.addCliOption("--api-port", $apiPort) config.addCliOption("--disc-port", $discPort) - lastUsedCodexApiPort = apiPort - lastUsedCodexDiscPort = discPort + lastUsedStorageApiPort = apiPort + lastUsedStorageDiscPort = discPort for bootstrapNode in bootstrapNodes: config.addCliOption("--bootstrap-node", bootstrapNode) @@ -136,30 +136,30 @@ template multinodesuite*(suiteName: string, body: untyped) = config.addCliOption("--data-dir", datadir) config.addCliOption("--nat", "none") config.addCliOption("--listen-addrs", "/ip4/127.0.0.1/tcp/0") - except CodexConfigError as e: + except StorageConfigError as e: raiseMultiNodeSuiteError "invalid cli option, error: " & e.msg try: - let node = await CodexProcess.startNode( + let node = await StorageProcess.startNode( config.cliArgs, config.debugEnabled, $role & $roleIdx ) await node.waitUntilStarted() trace "node started", nodeName = $role & $roleIdx return node - except CodexConfigError as e: + except StorageConfigError as e: raiseMultiNodeSuiteError "failed to get cli args from config: " & e.msg, e except NodeProcessError as e: raiseMultiNodeSuiteError "node not started, error: " & e.msg, e - proc clients(): seq[CodexProcess] {.used.} = + proc clients(): seq[StorageProcess] {.used.} = return collect: for r in running: if r.role == Role.Client: - CodexProcess(r.node) + StorageProcess(r.node) - proc startClientNode(conf: CodexConfig): Future[NodeProcess] {.async.} = + proc startClientNode(conf: StorageConfig): Future[NodeProcess] {.async.} = let clientIdx = clients().len - return await newCodexProcess(clientIdx, conf, Role.Client) + return await newStorageProcess(clientIdx, conf, Role.Client) proc teardownImpl() {.async.} = for nodes in @[clients()]: @@ -167,7 +167,7 @@ template multinodesuite*(suiteName: string, body: untyped) = await node.stop() # also stops rest client try: node.removeDataDir() - except CodexProcessError as e: + except StorageProcessError as e: error "Failed to remove data dir during teardown", error = e.msg running = @[] @@ -191,7 +191,7 @@ template multinodesuite*(suiteName: string, body: untyped) = quit(1) proc updateBootstrapNodes( - node: CodexProcess + node: StorageProcess ): Future[void] {.async: (raises: [MultiNodeSuiteError]).} = try: without ninfo =? await node.client.info(): @@ -206,9 +206,9 @@ template multinodesuite*(suiteName: string, body: untyped) = # When this file is run with `-d:chronicles_sinks=textlines[file]`, we # need to set the log file path at runtime, otherwise chronicles didn't seem to # create a log file even when using an absolute path - when defaultChroniclesStream.outputs is (FileOutput,) and CodexLogsDir.len > 0: + when defaultChroniclesStream.outputs is (FileOutput,) and StorageLogsDir.len > 0: let logFile = - CodexLogsDir / sanitize(getAppFilename().extractFilename & ".chronicles.log") + StorageLogsDir / sanitize(getAppFilename().extractFilename & ".chronicles.log") let success = defaultChroniclesStream.outputs[0].open(logFile, fmAppend) doAssert success, "Failed to open log file: " & logFile @@ -219,7 +219,7 @@ template multinodesuite*(suiteName: string, body: untyped) = for config in clients.configs: let node = await startClientNode(config) running.add RunningNode(role: Role.Client, node: node) - await CodexProcess(node).updateBootstrapNodes() + await StorageProcess(node).updateBootstrapNodes() teardown: await teardownImpl() diff --git a/tests/integration/nodeconfigs.nim b/tests/integration/nodeconfigs.nim index 5f11bf55..c8012392 100644 --- a/tests/integration/nodeconfigs.nim +++ b/tests/integration/nodeconfigs.nim @@ -1,5 +1,5 @@ import pkg/questionable -import ./codexconfig +import ./storageconfig type NodeConfigs* = object - clients*: ?CodexConfigs + clients*: ?StorageConfigs diff --git a/tests/integration/nodeprocess.nim b/tests/integration/nodeprocess.nim index a45e7806..907eeba5 100644 --- a/tests/integration/nodeprocess.nim +++ b/tests/integration/nodeprocess.nim @@ -7,12 +7,12 @@ import pkg/libp2p import std/os import std/strformat import std/strutils -import codex/conf -import codex/utils/exceptions -import codex/utils/trackedfutures -import ./codexclient +import storage/conf +import storage/utils/exceptions +import storage/utils/trackedfutures +import ./storageclient -export codexclient +export storageclient export chronicles {.push raises: [].} diff --git a/tests/integration/storageclient.nim b/tests/integration/storageclient.nim index c406a97b..15c78c29 100644 --- a/tests/integration/storageclient.nim +++ b/tests/integration/storageclient.nim @@ -4,27 +4,27 @@ from pkg/libp2p import Cid, `$`, init import pkg/stint import pkg/questionable/results import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient, httptable] -import pkg/codex/logutils -import pkg/codex/rest/json -import pkg/codex/errors +import pkg/storage/logutils +import pkg/storage/rest/json +import pkg/storage/errors export httptable, httpclient -type CodexClient* = ref object +type StorageClient* = ref object baseurl: string session: HttpSessionRef type HasBlockResponse = object has: bool -proc new*(_: type CodexClient, baseurl: string): CodexClient = - CodexClient(session: HttpSessionRef.new(), baseurl: baseurl) +proc new*(_: type StorageClient, baseurl: string): StorageClient = + StorageClient(session: HttpSessionRef.new(), baseurl: baseurl) -proc close*(self: CodexClient): Future[void] {.async: (raises: []).} = +proc close*(self: StorageClient): Future[void] {.async: (raises: []).} = await self.session.closeWait() proc request( - self: CodexClient, + self: StorageClient, httpMethod: httputils.HttpMethod, url: string, body: openArray[char] = [], @@ -46,7 +46,7 @@ proc request( .send() proc post*( - self: CodexClient, + self: StorageClient, url: string, body: string = "", headers: seq[HttpHeaderTuple] = @[], @@ -56,21 +56,21 @@ proc post*( return self.request(MethodPost, url, headers = headers, body = body) proc get( - self: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[] + self: StorageClient, url: string, headers: seq[HttpHeaderTuple] = @[] ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = return self.request(MethodGet, url, headers = headers) proc delete( - self: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[] + self: StorageClient, url: string, headers: seq[HttpHeaderTuple] = @[] ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = return self.request(MethodDelete, url, headers = headers) proc patch*( - self: CodexClient, + self: StorageClient, url: string, body: string = "", headers: seq[HttpHeaderTuple] = @[], @@ -85,19 +85,19 @@ proc body*( return bytesToString (await response.getBodyBytes()) proc getContent( - client: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[] + client: StorageClient, url: string, headers: seq[HttpHeaderTuple] = @[] ): Future[string] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.get(url, headers) return await response.body proc info*( - client: CodexClient + client: StorageClient ): Future[?!JsonNode] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.get(client.baseurl & "/debug/info") return JsonNode.parse(await response.body) proc setLogLevel*( - client: CodexClient, level: string + client: StorageClient, level: string ): Future[void] {.async: (raises: [CancelledError, HttpError]).} = let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level @@ -106,26 +106,26 @@ proc setLogLevel*( assert response.status == 200 proc uploadRaw*( - client: CodexClient, contents: string, headers: seq[HttpHeaderTuple] = @[] + client: StorageClient, contents: string, headers: seq[HttpHeaderTuple] = @[] ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = return client.post(client.baseurl & "/data", body = contents, headers = headers) proc upload*( - client: CodexClient, contents: string + client: StorageClient, contents: string ): Future[?!Cid] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.uploadRaw(contents) assert response.status == 200 Cid.init(await response.body).mapFailure proc upload*( - client: CodexClient, bytes: seq[byte] + client: StorageClient, bytes: seq[byte] ): Future[?!Cid] {.async: (raw: true).} = return client.upload(string.fromBytes(bytes)) proc downloadRaw*( - client: CodexClient, cid: string, local = false + client: StorageClient, cid: string, local = false ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = @@ -133,7 +133,7 @@ proc downloadRaw*( client.get(client.baseurl & "/data/" & cid & (if local: "" else: "/network/stream")) proc downloadBytes*( - client: CodexClient, cid: Cid, local = false + client: StorageClient, cid: Cid, local = false ): Future[?!seq[byte]] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.downloadRaw($cid, local = local) @@ -143,14 +143,14 @@ proc downloadBytes*( success await response.getBodyBytes() proc download*( - client: CodexClient, cid: Cid, local = false + client: StorageClient, cid: Cid, local = false ): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = without response =? await client.downloadBytes(cid, local = local), err: return failure(err) return success bytesToString(response) proc downloadNoStream*( - client: CodexClient, cid: Cid + client: StorageClient, cid: Cid ): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.post(client.baseurl & "/data/" & $cid & "/network") @@ -160,7 +160,7 @@ proc downloadNoStream*( success await response.body proc downloadManifestOnly*( - client: CodexClient, cid: Cid + client: StorageClient, cid: Cid ): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.get(client.baseurl & "/data/" & $cid & "/network/manifest") @@ -171,14 +171,14 @@ proc downloadManifestOnly*( success await response.body proc deleteRaw*( - client: CodexClient, cid: string + client: StorageClient, cid: string ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = return client.delete(client.baseurl & "/data/" & cid) proc delete*( - client: CodexClient, cid: Cid + client: StorageClient, cid: Cid ): Future[?!void] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.deleteRaw($cid) @@ -188,14 +188,14 @@ proc delete*( success() proc listRaw*( - client: CodexClient + client: StorageClient ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = return client.get(client.baseurl & "/data") proc list*( - client: CodexClient + client: StorageClient ): Future[?!RestContentList] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.listRaw() @@ -205,7 +205,7 @@ proc list*( RestContentList.fromJson(await response.body) proc space*( - client: CodexClient + client: StorageClient ): Future[?!RestRepoStore] {.async: (raises: [CancelledError, HttpError]).} = let url = client.baseurl & "/space" let response = await client.get(url) @@ -215,11 +215,11 @@ proc space*( RestRepoStore.fromJson(await response.body) -proc buildUrl*(client: CodexClient, path: string): string = +proc buildUrl*(client: StorageClient, path: string): string = return client.baseurl & path proc hasBlock*( - client: CodexClient, cid: Cid + client: StorageClient, cid: Cid ): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} = let url = client.baseurl & "/data/" & $cid & "/exists" let body = await client.getContent(url) @@ -229,7 +229,7 @@ proc hasBlock*( return response.get.has.success proc hasBlockRaw*( - client: CodexClient, cid: string + client: StorageClient, cid: string ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = diff --git a/tests/integration/storageconfig.nim b/tests/integration/storageconfig.nim index 01e6d1c9..d0d7d813 100644 --- a/tests/integration/storageconfig.nim +++ b/tests/integration/storageconfig.nim @@ -5,8 +5,8 @@ import std/strutils import std/sugar import std/tables from pkg/chronicles import LogLevel -import pkg/codex/conf -import pkg/codex/units +import pkg/storage/conf +import pkg/storage/units import pkg/confutils import pkg/confutils/defs import libp2p except setup @@ -17,39 +17,39 @@ export clioption export confutils type - CodexConfigs* = object - configs*: seq[CodexConfig] + StorageConfigs* = object + configs*: seq[StorageConfig] - CodexConfig* = object + StorageConfig* = object cliOptions: Table[StartUpCmd, Table[string, CliOption]] debugEnabled*: bool - CodexConfigError* = object of CatchableError + StorageConfigError* = object of CatchableError -proc cliArgs*(config: CodexConfig): seq[string] {.gcsafe, raises: [CodexConfigError].} +proc cliArgs*(config: StorageConfig): seq[string] {.gcsafe, raises: [StorageConfigError].} -proc raiseCodexConfigError(msg: string) {.raises: [CodexConfigError].} = - raise newException(CodexConfigError, msg) +proc raiseStorageConfigError(msg: string) {.raises: [StorageConfigError].} = + raise newException(StorageConfigError, msg) template convertError(body) = try: body except CatchableError as e: - raiseCodexConfigError e.msg + raiseStorageConfigError e.msg -proc init*(_: type CodexConfigs, nodes = 1): CodexConfigs {.raises: [].} = - CodexConfigs(configs: newSeq[CodexConfig](nodes)) +proc init*(_: type StorageConfigs, nodes = 1): StorageConfigs {.raises: [].} = + StorageConfigs(configs: newSeq[StorageConfig](nodes)) -func nodes*(self: CodexConfigs): int = +func nodes*(self: StorageConfigs): int = self.configs.len -proc checkBounds(self: CodexConfigs, idx: int) {.raises: [CodexConfigError].} = +proc checkBounds(self: StorageConfigs, idx: int) {.raises: [StorageConfigError].} = if idx notin 0 ..< self.configs.len: - raiseCodexConfigError "index must be in bounds of the number of nodes" + raiseStorageConfigError "index must be in bounds of the number of nodes" proc buildConfig( - config: CodexConfig, msg: string -): CodexConf {.raises: [CodexConfigError].} = + config: StorageConfig, msg: string +): StorageConf {.raises: [StorageConfigError].} = proc postFix(msg: string): string = if msg.len > 0: ": " & msg @@ -57,38 +57,38 @@ proc buildConfig( "" try: - return CodexConf.load(cmdLine = config.cliArgs, quitOnFailure = false) + return StorageConf.load(cmdLine = config.cliArgs, quitOnFailure = false) except ConfigurationError as e: - raiseCodexConfigError msg & e.msg.postFix + raiseStorageConfigError msg & e.msg.postFix except Exception as e: ## TODO: remove once proper exception handling added to nim-confutils - raiseCodexConfigError msg & e.msg.postFix + raiseStorageConfigError msg & e.msg.postFix proc addCliOption*( - config: var CodexConfig, group = StartUpCmd.noCmd, cliOption: CliOption -) {.raises: [CodexConfigError].} = + config: var StorageConfig, group = StartUpCmd.noCmd, cliOption: CliOption +) {.raises: [StorageConfigError].} = var options = config.cliOptions.getOrDefault(group) options[cliOption.key] = cliOption # overwrite if already exists config.cliOptions[group] = options discard config.buildConfig("Invalid cli arg " & $cliOption) proc addCliOption*( - config: var CodexConfig, group = StartUpCmd.noCmd, key: string, value = "" -) {.raises: [CodexConfigError].} = + config: var StorageConfig, group = StartUpCmd.noCmd, key: string, value = "" +) {.raises: [StorageConfigError].} = config.addCliOption(group, CliOption(key: key, value: value)) proc addCliOption*( - config: var CodexConfig, cliOption: CliOption -) {.raises: [CodexConfigError].} = + config: var StorageConfig, cliOption: CliOption +) {.raises: [StorageConfigError].} = config.addCliOption(StartUpCmd.noCmd, cliOption) proc addCliOption*( - config: var CodexConfig, key: string, value = "" -) {.raises: [CodexConfigError].} = + config: var StorageConfig, key: string, value = "" +) {.raises: [StorageConfigError].} = config.addCliOption(StartUpCmd.noCmd, CliOption(key: key, value: value)) -proc cliArgs*(config: CodexConfig): seq[string] {.gcsafe, raises: [CodexConfigError].} = - ## converts CodexConfig cli options and command groups in a sequence of args +proc cliArgs*(config: StorageConfig): seq[string] {.gcsafe, raises: [StorageConfigError].} = + ## converts StorageConfig cli options and command groups in a sequence of args ## and filters out cli options by node index if provided in the CliOption var args: seq[string] = @[] @@ -102,18 +102,18 @@ proc cliArgs*(config: CodexConfig): seq[string] {.gcsafe, raises: [CodexConfigEr return args -proc logFile*(config: CodexConfig): ?string {.raises: [CodexConfigError].} = - let built = config.buildConfig("Invalid codex config cli params") +proc logFile*(config: StorageConfig): ?string {.raises: [StorageConfigError].} = + let built = config.buildConfig("Invalid storage config cli params") built.logFile -proc logLevel*(config: CodexConfig): LogLevel {.raises: [CodexConfigError].} = +proc logLevel*(config: StorageConfig): LogLevel {.raises: [StorageConfigError].} = convertError: - let built = config.buildConfig("Invalid codex config cli params") + let built = config.buildConfig("Invalid storage config cli params") return parseEnum[LogLevel](built.logLevel.toUpperAscii) proc debug*( - self: CodexConfigs, idx: int, enabled = true -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, enabled = true +): StorageConfigs {.raises: [StorageConfigError].} = ## output log in stdout for a specific node in the group self.checkBounds idx @@ -122,7 +122,7 @@ proc debug*( startConfig.configs[idx].debugEnabled = enabled return startConfig -proc debug*(self: CodexConfigs, enabled = true): CodexConfigs {.raises: [].} = +proc debug*(self: StorageConfigs, enabled = true): StorageConfigs {.raises: [].} = ## output log in stdout for all nodes in group var startConfig = self for config in startConfig.configs.mitems: @@ -130,15 +130,15 @@ proc debug*(self: CodexConfigs, enabled = true): CodexConfigs {.raises: [].} = return startConfig proc withLogFile*( - self: CodexConfigs, idx: int -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self startConfig.configs[idx].addCliOption("--log-file", "") return startConfig -proc withLogFile*(self: CodexConfigs): CodexConfigs {.raises: [CodexConfigError].} = +proc withLogFile*(self: StorageConfigs): StorageConfigs {.raises: [StorageConfigError].} = ## typically called from test, sets config such that a log file should be ## created var startConfig = self @@ -147,9 +147,9 @@ proc withLogFile*(self: CodexConfigs): CodexConfigs {.raises: [CodexConfigError] return startConfig proc withLogFile*( - self: var CodexConfig, logFile: string -) {.raises: [CodexConfigError].} = - #: CodexConfigs = + self: var StorageConfig, logFile: string +) {.raises: [StorageConfigError].} = + #: StorageConfigs = ## typically called internally from the test suite, sets a log file path to ## be created during the test run, for a specified node in the group # var config = self @@ -157,15 +157,15 @@ proc withLogFile*( # return startConfig proc withLogLevel*( - self: CodexConfig, level: LogLevel | string -): CodexConfig {.raises: [CodexConfigError].} = + self: StorageConfig, level: LogLevel | string +): StorageConfig {.raises: [StorageConfigError].} = var config = self config.addCliOption("--log-level", $level) return config proc withLogLevel*( - self: CodexConfigs, idx: int, level: LogLevel | string -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, level: LogLevel | string +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self @@ -173,23 +173,23 @@ proc withLogLevel*( return startConfig proc withLogLevel*( - self: CodexConfigs, level: LogLevel | string -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, level: LogLevel | string +): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: config.addCliOption("--log-level", $level) return startConfig proc withBlockTtl*( - self: CodexConfig, ttl: int -): CodexConfig {.raises: [CodexConfigError].} = + self: StorageConfig, ttl: int +): StorageConfig {.raises: [StorageConfigError].} = var config = self config.addCliOption("--block-ttl", $ttl) return config proc withBlockTtl*( - self: CodexConfigs, idx: int, ttl: int -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, ttl: int +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self @@ -197,23 +197,23 @@ proc withBlockTtl*( return startConfig proc withBlockTtl*( - self: CodexConfigs, ttl: int -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, ttl: int +): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: config.addCliOption("--block-ttl", $ttl) return startConfig proc withBlockMaintenanceInterval*( - self: CodexConfig, interval: int -): CodexConfig {.raises: [CodexConfigError].} = + self: StorageConfig, interval: int +): StorageConfig {.raises: [StorageConfigError].} = var config = self config.addCliOption("--block-mi", $interval) return config proc withBlockMaintenanceInterval*( - self: CodexConfigs, idx: int, interval: int -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, interval: int +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self @@ -221,26 +221,26 @@ proc withBlockMaintenanceInterval*( return startConfig proc withBlockMaintenanceInterval*( - self: CodexConfigs, interval: int -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, interval: int +): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: config.addCliOption("--block-mi", $interval) return startConfig proc logLevelWithTopics( - config: CodexConfig, topics: varargs[string] -): string {.raises: [CodexConfigError].} = + config: StorageConfig, topics: varargs[string] +): string {.raises: [StorageConfigError].} = convertError: var logLevel = LogLevel.INFO - let built = config.buildConfig("Invalid codex config cli params") + let built = config.buildConfig("Invalid storage config cli params") logLevel = parseEnum[LogLevel](built.logLevel.toUpperAscii) let level = $logLevel & ";TRACE: " & topics.join(",") return level proc withLogTopics*( - self: CodexConfigs, idx: int, topics: varargs[string] -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, topics: varargs[string] +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx convertError: @@ -250,8 +250,8 @@ proc withLogTopics*( return startConfig.withLogLevel(idx, level) proc withLogTopics*( - self: CodexConfigs, topics: varargs[string] -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, topics: varargs[string] +): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: let level = config.logLevelWithTopics(topics) @@ -259,8 +259,8 @@ proc withLogTopics*( return startConfig proc withStorageQuota*( - self: CodexConfigs, idx: int, quota: NBytes -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, idx: int, quota: NBytes +): StorageConfigs {.raises: [StorageConfigError].} = self.checkBounds idx var startConfig = self @@ -268,8 +268,8 @@ proc withStorageQuota*( return startConfig proc withStorageQuota*( - self: CodexConfigs, quota: NBytes -): CodexConfigs {.raises: [CodexConfigError].} = + self: StorageConfigs, quota: NBytes +): StorageConfigs {.raises: [StorageConfigError].} = var startConfig = self for config in startConfig.configs.mitems: config.addCliOption("--storage-quota", $quota) diff --git a/tests/integration/storageprocess.nim b/tests/integration/storageprocess.nim index 0d2df752..ec58d273 100644 --- a/tests/integration/storageprocess.nim +++ b/tests/integration/storageprocess.nim @@ -7,94 +7,94 @@ import pkg/libp2p import std/os import std/strutils import std/times -import codex/conf -import ./codexclient +import storage/conf +import ./storageclient import ./nodeprocess -export codexclient +export storageclient export chronicles export nodeprocess {.push raises: [].} logScope: - topics = "integration testing codex process" + topics = "integration testing storage process" type - CodexProcess* = ref object of NodeProcess - client: ?CodexClient + StorageProcess* = ref object of NodeProcess + client: ?StorageClient - CodexProcessError* = object of NodeProcessError + StorageProcessError* = object of NodeProcessError -proc raiseCodexProcessError( +proc raiseStorageProcessError( msg: string, parent: ref CatchableError -) {.raises: [CodexProcessError].} = - raise newException(CodexProcessError, msg & ": " & parent.msg, parent) +) {.raises: [StorageProcessError].} = + raise newException(StorageProcessError, msg & ": " & parent.msg, parent) template convertError(msg, body: typed) = # Don't use this in an async proc, unless body does not raise CancelledError try: body except CatchableError as parent: - raiseCodexProcessError(msg, parent) + raiseStorageProcessError(msg, parent) -method workingDir(node: CodexProcess): string = +method workingDir(node: StorageProcess): string = return currentSourcePath() / ".." / ".." / ".." -method executable(node: CodexProcess): string = +method executable(node: StorageProcess): string = return "build" / "storage" -method startedOutput(node: CodexProcess): string = +method startedOutput(node: StorageProcess): string = return "REST service started" -method processOptions(node: CodexProcess): set[AsyncProcessOption] = +method processOptions(node: StorageProcess): set[AsyncProcessOption] = return {AsyncProcessOption.StdErrToStdOut} -method outputLineEndings(node: CodexProcess): string = +method outputLineEndings(node: StorageProcess): string = return "\n" -method onOutputLineCaptured(node: CodexProcess, line: string) = +method onOutputLineCaptured(node: StorageProcess, line: string) = discard -proc config(node: CodexProcess): CodexConf {.raises: [CodexProcessError].} = +proc config(node: StorageProcess): StorageConf {.raises: [StorageProcessError].} = # cannot use convertError here as it uses typed parameters which forces type # resolution, while confutils.load uses untyped parameters and expects type # resolution not to happen yet. In other words, it won't compile. try: - return CodexConf.load( + return StorageConf.load( cmdLine = node.arguments, quitOnFailure = false, secondarySources = nil ) except ConfigurationError as parent: - raiseCodexProcessError "Failed to load node arguments into CodexConf", parent + raiseStorageProcessError "Failed to load node arguments into StorageConf", parent -proc dataDir(node: CodexProcess): string {.raises: [CodexProcessError].} = +proc dataDir(node: StorageProcess): string {.raises: [StorageProcessError].} = return node.config.dataDir.string -proc apiUrl*(node: CodexProcess): string {.raises: [CodexProcessError].} = +proc apiUrl*(node: StorageProcess): string {.raises: [StorageProcessError].} = let config = node.config without apiBindAddress =? config.apiBindAddress: raise - newException(CodexProcessError, "REST API not started: --api-bindaddr not set") + newException(StorageProcessError, "REST API not started: --api-bindaddr not set") return "http://" & apiBindAddress & ":" & $config.apiPort & "/api/storage/v1" -proc logFile*(node: CodexProcess): ?string {.raises: [CodexProcessError].} = +proc logFile*(node: StorageProcess): ?string {.raises: [StorageProcessError].} = node.config.logFile -proc client*(node: CodexProcess): CodexClient {.raises: [CodexProcessError].} = +proc client*(node: StorageProcess): StorageClient {.raises: [StorageProcessError].} = if client =? node.client: return client - let client = CodexClient.new(node.apiUrl) + let client = StorageClient.new(node.apiUrl) node.client = some client return client -proc updateLogFile(node: CodexProcess, newLogFile: string) = +proc updateLogFile(node: StorageProcess, newLogFile: string) = for arg in node.arguments.mitems: if arg.startsWith("--log-file="): arg = "--log-file=" & newLogFile break -method restart*(node: CodexProcess) {.async.} = - trace "restarting codex" +method restart*(node: StorageProcess) {.async.} = + trace "restarting storage" await node.stop() if logFile =? node.logFile: # chronicles truncates the existing log file on start, so changed the log @@ -104,13 +104,13 @@ method restart*(node: CodexProcess) {.async.} = ) await node.start() await node.waitUntilStarted() - trace "codex process restarted" + trace "storage process restarted" -method stop*(node: CodexProcess) {.async: (raises: []).} = +method stop*(node: StorageProcess) {.async: (raises: []).} = logScope: nodeName = node.name - trace "stopping codex client" + trace "stopping storage client" await procCall NodeProcess(node).stop() if not node.process.isNil: @@ -120,8 +120,8 @@ method stop*(node: CodexProcess) {.async: (raises: []).} = if client =? node.client: await client.close() - node.client = none CodexClient + node.client = none StorageClient -method removeDataDir*(node: CodexProcess) {.raises: [CodexProcessError].} = - convertError("failed to remove codex node data directory"): +method removeDataDir*(node: StorageProcess) {.raises: [StorageProcessError].} = + convertError("failed to remove storage node data directory"): removeDir(node.dataDir) diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index f9eb5d37..7fa55244 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -1,23 +1,23 @@ import std/macros import pkg/questionable import ./multinodes -import ./codexconfig -import ./codexprocess -import ./codexclient +import ./storageconfig +import ./storageprocess +import ./storageclient import ./nodeconfigs -export codexclient +export storageclient export multinodes template twonodessuite*(name: string, body: untyped) = multinodesuite name: let twoNodesConfig {.inject, used.} = - NodeConfigs(clients: CodexConfigs.init(nodes = 2).some) + NodeConfigs(clients: StorageConfigs.init(nodes = 2).some) - var node1 {.inject, used.}: CodexProcess - var node2 {.inject, used.}: CodexProcess - var client1 {.inject, used.}: CodexClient - var client2 {.inject, used.}: CodexClient + var node1 {.inject, used.}: StorageProcess + var node2 {.inject, used.}: StorageProcess + var client1 {.inject, used.}: StorageClient + var client2 {.inject, used.}: StorageClient setup: node1 = clients()[0] diff --git a/tests/integration/utils.nim b/tests/integration/utils.nim index 3e522a04..7d48d5d5 100644 --- a/tests/integration/utils.nim +++ b/tests/integration/utils.nim @@ -2,7 +2,7 @@ import std/os import std/strformat import pkg/chronos import pkg/chronos/asyncproc -import pkg/codex/logutils +import pkg/storage/logutils {.push raises: [].} @@ -88,5 +88,5 @@ proc getDataDir*(testId, testName, startTime, role: string, index = int.none): s if idx =? index: suffix &= "_" & $idx - getTempDir() / "Codex" / sanitize(testId) / sanitize(testName) / sanitize(startTime) / + getTempDir() / "Storage" / sanitize(testId) / sanitize(testName) / sanitize(startTime) / sanitize(suffix) diff --git a/tests/logging.nim b/tests/logging.nim index d165fe48..30f91e6e 100644 --- a/tests/logging.nim +++ b/tests/logging.nim @@ -1,5 +1,5 @@ when not defined(nimscript): - import pkg/codex/logutils + import pkg/storage/logutils proc ignoreLogging(level: LogLevel, message: LogOutputStr) = discard diff --git a/tests/storage/blockexchange/discovery/testdiscovery.nim b/tests/storage/blockexchange/discovery/testdiscovery.nim index b8aec57b..ab237542 100644 --- a/tests/storage/blockexchange/discovery/testdiscovery.nim +++ b/tests/storage/blockexchange/discovery/testdiscovery.nim @@ -6,13 +6,13 @@ import pkg/chronos import pkg/libp2p/errors -import pkg/codex/rng -import pkg/codex/stores -import pkg/codex/blockexchange -import pkg/codex/chunker -import pkg/codex/manifest -import pkg/codex/merkletree -import pkg/codex/blocktype as bt +import pkg/storage/rng +import pkg/storage/stores +import pkg/storage/blockexchange +import pkg/storage/chunker +import pkg/storage/manifest +import pkg/storage/merkletree +import pkg/storage/blocktype as bt import ../../../asynctest import ../../helpers diff --git a/tests/storage/blockexchange/discovery/testdiscoveryengine.nim b/tests/storage/blockexchange/discovery/testdiscoveryengine.nim index 978fe690..3bd940a0 100644 --- a/tests/storage/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/storage/blockexchange/discovery/testdiscoveryengine.nim @@ -3,14 +3,14 @@ import std/tables import pkg/chronos -import pkg/codex/rng -import pkg/codex/stores -import pkg/codex/blockexchange -import pkg/codex/chunker -import pkg/codex/blocktype as bt -import pkg/codex/blockexchange/engine -import pkg/codex/manifest -import pkg/codex/merkletree +import pkg/storage/rng +import pkg/storage/stores +import pkg/storage/blockexchange +import pkg/storage/chunker +import pkg/storage/blocktype as bt +import pkg/storage/blockexchange/engine +import pkg/storage/manifest +import pkg/storage/merkletree import ../../../asynctest import ../../helpers diff --git a/tests/storage/blockexchange/engine/testadvertiser.nim b/tests/storage/blockexchange/engine/testadvertiser.nim index 83a70f65..c39d35c1 100644 --- a/tests/storage/blockexchange/engine/testadvertiser.nim +++ b/tests/storage/blockexchange/engine/testadvertiser.nim @@ -2,12 +2,12 @@ import pkg/chronos import pkg/libp2p/routing_record import pkg/codexdht/discv5/protocol as discv5 -import pkg/codex/blockexchange -import pkg/codex/stores -import pkg/codex/chunker -import pkg/codex/discovery -import pkg/codex/blocktype as bt -import pkg/codex/manifest +import pkg/storage/blockexchange +import pkg/storage/stores +import pkg/storage/chunker +import pkg/storage/discovery +import pkg/storage/blocktype as bt +import pkg/storage/manifest import ../../../asynctest import ../../helpers diff --git a/tests/storage/blockexchange/engine/testblockexc.nim b/tests/storage/blockexchange/engine/testblockexc.nim index 4b0fe173..515236bf 100644 --- a/tests/storage/blockexchange/engine/testblockexc.nim +++ b/tests/storage/blockexchange/engine/testblockexc.nim @@ -5,11 +5,11 @@ import std/importutils import pkg/chronos import pkg/stew/byteutils -import pkg/codex/stores -import pkg/codex/blockexchange -import pkg/codex/chunker -import pkg/codex/discovery -import pkg/codex/blocktype as bt +import pkg/storage/stores +import pkg/storage/blockexchange +import pkg/storage/chunker +import pkg/storage/discovery +import pkg/storage/blocktype as bt import ../../../asynctest import ../../examples diff --git a/tests/storage/blockexchange/engine/testengine.nim b/tests/storage/blockexchange/engine/testengine.nim index 54f16c8d..b04fb29d 100644 --- a/tests/storage/blockexchange/engine/testengine.nim +++ b/tests/storage/blockexchange/engine/testengine.nim @@ -8,13 +8,13 @@ import pkg/libp2p/errors import pkg/libp2p/routing_record import pkg/codexdht/discv5/protocol as discv5 -import pkg/codex/rng -import pkg/codex/blockexchange -import pkg/codex/stores -import pkg/codex/chunker -import pkg/codex/discovery -import pkg/codex/blocktype -import pkg/codex/utils/asyncheapqueue +import pkg/storage/rng +import pkg/storage/blockexchange +import pkg/storage/stores +import pkg/storage/chunker +import pkg/storage/discovery +import pkg/storage/blocktype +import pkg/storage/utils/asyncheapqueue import ../../../asynctest import ../../helpers diff --git a/tests/storage/blockexchange/protobuf/testpresence.nim b/tests/storage/blockexchange/protobuf/testpresence.nim index 2f449e15..4bef0448 100644 --- a/tests/storage/blockexchange/protobuf/testpresence.nim +++ b/tests/storage/blockexchange/protobuf/testpresence.nim @@ -1,6 +1,6 @@ import pkg/chronos -import pkg/codex/blockexchange/protobuf/presence +import pkg/storage/blockexchange/protobuf/presence import ../../../asynctest import ../../examples diff --git a/tests/storage/blockexchange/testnetwork.nim b/tests/storage/blockexchange/testnetwork.nim index 6af156a9..0ed8b41a 100644 --- a/tests/storage/blockexchange/testnetwork.nim +++ b/tests/storage/blockexchange/testnetwork.nim @@ -3,10 +3,10 @@ import std/tables import pkg/chronos -import pkg/codex/rng -import pkg/codex/chunker -import pkg/codex/blocktype as bt -import pkg/codex/blockexchange +import pkg/storage/rng +import pkg/storage/chunker +import pkg/storage/blocktype as bt +import pkg/storage/blockexchange import ../../asynctest import ../examples diff --git a/tests/storage/blockexchange/testpeerctxstore.nim b/tests/storage/blockexchange/testpeerctxstore.nim index 880e08c3..6150bb9f 100644 --- a/tests/storage/blockexchange/testpeerctxstore.nim +++ b/tests/storage/blockexchange/testpeerctxstore.nim @@ -4,9 +4,9 @@ import std/sequtils import pkg/unittest2 import pkg/libp2p -import pkg/codex/blockexchange/peers -import pkg/codex/blockexchange/protobuf/blockexc -import pkg/codex/blockexchange/protobuf/presence +import pkg/storage/blockexchange/peers +import pkg/storage/blockexchange/protobuf/blockexc +import pkg/storage/blockexchange/protobuf/presence import ../helpers import ../examples diff --git a/tests/storage/blockexchange/testpendingblocks.nim b/tests/storage/blockexchange/testpendingblocks.nim index af1e6728..eba4ead1 100644 --- a/tests/storage/blockexchange/testpendingblocks.nim +++ b/tests/storage/blockexchange/testpendingblocks.nim @@ -4,8 +4,8 @@ import std/algorithm import pkg/chronos import pkg/stew/byteutils -import pkg/codex/blocktype as bt -import pkg/codex/blockexchange +import pkg/storage/blocktype as bt +import pkg/storage/blockexchange import ../helpers import ../../asynctest diff --git a/tests/storage/examples.nim b/tests/storage/examples.nim index 4c9535cb..43512f8d 100644 --- a/tests/storage/examples.nim +++ b/tests/storage/examples.nim @@ -2,11 +2,11 @@ import std/random import std/sequtils import pkg/libp2p import pkg/stint -import pkg/codex/rng -import pkg/codex/stores -import pkg/codex/blocktype as bt -import pkg/codex/merkletree -import pkg/codex/manifest +import pkg/storage/rng +import pkg/storage/stores +import pkg/storage/blocktype as bt +import pkg/storage/merkletree +import pkg/storage/manifest import ../examples export examples diff --git a/tests/storage/helpers.nim b/tests/storage/helpers.nim index b855f412..28932fd8 100644 --- a/tests/storage/helpers.nim +++ b/tests/storage/helpers.nim @@ -3,13 +3,13 @@ import std/sequtils import pkg/chronos import pkg/libp2p import pkg/libp2p/varint -import pkg/codex/blocktype -import pkg/codex/stores -import pkg/codex/manifest -import pkg/codex/merkletree -import pkg/codex/blockexchange -import pkg/codex/rng -import pkg/codex/utils +import pkg/storage/blocktype +import pkg/storage/stores +import pkg/storage/manifest +import pkg/storage/merkletree +import pkg/storage/blockexchange +import pkg/storage/rng +import pkg/storage/utils import ./helpers/nodeutils import ./helpers/datasetutils diff --git a/tests/storage/helpers/datasetutils.nim b/tests/storage/helpers/datasetutils.nim index d4aedc41..078a08d9 100644 --- a/tests/storage/helpers/datasetutils.nim +++ b/tests/storage/helpers/datasetutils.nim @@ -1,10 +1,10 @@ import std/random import pkg/chronos -import pkg/codex/blocktype as bt -import pkg/codex/merkletree -import pkg/codex/manifest -import pkg/codex/rng +import pkg/storage/blocktype as bt +import pkg/storage/merkletree +import pkg/storage/manifest +import pkg/storage/rng import ./randomchunker diff --git a/tests/storage/helpers/mockchunker.nim b/tests/storage/helpers/mockchunker.nim index 1ecd8a21..cba8d0e9 100644 --- a/tests/storage/helpers/mockchunker.nim +++ b/tests/storage/helpers/mockchunker.nim @@ -1,5 +1,5 @@ import pkg/chronos -import pkg/codex/chunker +import pkg/storage/chunker export chunker diff --git a/tests/storage/helpers/mockclock.nim b/tests/storage/helpers/mockclock.nim index 76446041..20a71918 100644 --- a/tests/storage/helpers/mockclock.nim +++ b/tests/storage/helpers/mockclock.nim @@ -1,6 +1,6 @@ import std/times import pkg/chronos -import codex/clock +import storage/clock export clock diff --git a/tests/storage/helpers/mockdiscovery.nim b/tests/storage/helpers/mockdiscovery.nim index f0142650..8ed61979 100644 --- a/tests/storage/helpers/mockdiscovery.nim +++ b/tests/storage/helpers/mockdiscovery.nim @@ -10,7 +10,7 @@ import pkg/chronos import pkg/libp2p import pkg/questionable -import pkg/codex/discovery +import pkg/storage/discovery import pkg/contractabi/address as ca type MockDiscovery* = ref object of Discovery diff --git a/tests/storage/helpers/mockrepostore.nim b/tests/storage/helpers/mockrepostore.nim index 3d29f4a3..cc38d55d 100644 --- a/tests/storage/helpers/mockrepostore.nim +++ b/tests/storage/helpers/mockrepostore.nim @@ -13,9 +13,9 @@ import pkg/libp2p import pkg/questionable import pkg/questionable/results -import pkg/codex/stores/repostore -import pkg/codex/utils/asynciter -import pkg/codex/utils/safeasynciter +import pkg/storage/stores/repostore +import pkg/storage/utils/asynciter +import pkg/storage/utils/safeasynciter type MockRepoStore* = ref object of RepoStore delBlockCids*: seq[Cid] diff --git a/tests/storage/helpers/mocktimer.nim b/tests/storage/helpers/mocktimer.nim index 162a6e81..63a22a86 100644 --- a/tests/storage/helpers/mocktimer.nim +++ b/tests/storage/helpers/mocktimer.nim @@ -9,7 +9,7 @@ import pkg/chronos -import codex/utils/timer +import storage/utils/timer type MockTimer* = ref object of Timer startCalled*: int diff --git a/tests/storage/helpers/nodeutils.nim b/tests/storage/helpers/nodeutils.nim index 55ab984b..5ab198cd 100644 --- a/tests/storage/helpers/nodeutils.nim +++ b/tests/storage/helpers/nodeutils.nim @@ -6,18 +6,18 @@ import pkg/taskpools import pkg/libp2p import pkg/libp2p/errors -import pkg/codex/discovery -import pkg/codex/stores -import pkg/codex/blocktype as bt -import pkg/codex/blockexchange -import pkg/codex/systemclock -import pkg/codex/nat -import pkg/codex/utils/natutils -import pkg/codex/utils/safeasynciter -import pkg/codex/merkletree -import pkg/codex/manifest +import pkg/storage/discovery +import pkg/storage/stores +import pkg/storage/blocktype as bt +import pkg/storage/blockexchange +import pkg/storage/systemclock +import pkg/storage/nat +import pkg/storage/utils/natutils +import pkg/storage/utils/safeasynciter +import pkg/storage/merkletree +import pkg/storage/manifest -import pkg/codex/node +import pkg/storage/node import ./datasetutils import ./mockdiscovery @@ -49,7 +49,7 @@ type discovery*: DiscoveryEngine engine*: BlockExcEngine networkStore*: NetworkStore - node*: CodexNodeRef = nil + node*: StorageNodeRef = nil tempDbs*: seq[TempLevelDb] = @[] NodesCluster* = ref object @@ -85,7 +85,7 @@ converter toTuple*( converter toComponents*(cluster: NodesCluster): seq[NodesComponents] = cluster.components -proc nodes*(cluster: NodesCluster): seq[CodexNodeRef] = +proc nodes*(cluster: NodesCluster): seq[StorageNodeRef] = cluster.components.filterIt(it.node != nil).mapIt(it.node) proc localStores*(cluster: NodesCluster): seq[BlockStore] = @@ -209,7 +209,7 @@ proc generateNodes*( let node = if config.createFullNode: - let fullNode = CodexNodeRef.new( + let fullNode = StorageNodeRef.new( switch = switch, networkStore = networkStore, engine = engine, @@ -321,7 +321,7 @@ proc linearTopology*(nodes: seq[NodesComponents]) {.async.} = proc downloadDataset*( node: NodesComponents, dataset: TestDataset ): Future[void] {.async.} = - # This is the same as fetchBatched, but we don't construct CodexNodes so I can't use + # This is the same as fetchBatched, but we don't construct StorageNodes so I can't use # it here. let requestAddresses = collect: for i in 0 ..< dataset.manifest.blocksCount: diff --git a/tests/storage/helpers/randomchunker.nim b/tests/storage/helpers/randomchunker.nim index d1383e84..8ee1ba45 100644 --- a/tests/storage/helpers/randomchunker.nim +++ b/tests/storage/helpers/randomchunker.nim @@ -2,8 +2,8 @@ import std/sequtils import pkg/chronos -import pkg/codex/chunker -import pkg/codex/rng +import pkg/storage/chunker +import pkg/storage/rng export chunker diff --git a/tests/storage/merkletree/helpers.nim b/tests/storage/merkletree/helpers.nim index c2361b9d..c8fef2f0 100644 --- a/tests/storage/merkletree/helpers.nim +++ b/tests/storage/merkletree/helpers.nim @@ -1,4 +1,4 @@ -import pkg/codex/merkletree +import pkg/storage/merkletree import ../helpers export merkletree, helpers diff --git a/tests/storage/merkletree/teststoragecoders.nim b/tests/storage/merkletree/teststoragecoders.nim index eddafc9e..44921ef7 100644 --- a/tests/storage/merkletree/teststoragecoders.nim +++ b/tests/storage/merkletree/teststoragecoders.nim @@ -3,7 +3,7 @@ import pkg/unittest2 import pkg/questionable/results import pkg/stew/byteutils -import pkg/codex/merkletree +import pkg/storage/merkletree import ./helpers const data = [ diff --git a/tests/storage/merkletree/teststoragetree.nim b/tests/storage/merkletree/teststoragetree.nim index 877a7fa5..ae9e035e 100644 --- a/tests/storage/merkletree/teststoragetree.nim +++ b/tests/storage/merkletree/teststoragetree.nim @@ -5,9 +5,9 @@ import pkg/questionable/results import pkg/stew/byteutils import pkg/libp2p -import pkg/codex/codextypes -import pkg/codex/merkletree -import pkg/codex/utils/digest +import pkg/storage/storagetypes +import pkg/storage/merkletree +import pkg/storage/utils/digest import pkg/taskpools diff --git a/tests/storage/node/helpers.nim b/tests/storage/node/helpers.nim index 72bb1c51..00a87c97 100644 --- a/tests/storage/node/helpers.nim +++ b/tests/storage/node/helpers.nim @@ -3,9 +3,9 @@ import std/times import pkg/libp2p import pkg/chronos -import pkg/codex/codextypes -import pkg/codex/chunker -import pkg/codex/stores +import pkg/storage/storagetypes +import pkg/storage/chunker +import pkg/storage/stores import pkg/taskpools import ../../asynctest @@ -71,7 +71,7 @@ template setupAndTearDown*() {.dirty.} = localStoreMetaDs: Datastore engine: BlockExcEngine store: NetworkStore - node: CodexNodeRef + node: StorageNodeRef blockDiscovery: Discovery peerStore: PeerCtxStore pendingBlocks: PendingBlocksManager @@ -110,7 +110,7 @@ template setupAndTearDown*() {.dirty.} = localStore, network, discovery, advertiser, peerStore, pendingBlocks ) store = NetworkStore.new(engine, localStore) - node = CodexNodeRef.new( + node = StorageNodeRef.new( switch = switch, networkStore = store, engine = engine, diff --git a/tests/storage/node/testnode.nim b/tests/storage/node/testnode.nim index cc5690d7..389b4f8f 100644 --- a/tests/storage/node/testnode.nim +++ b/tests/storage/node/testnode.nim @@ -14,19 +14,19 @@ import pkg/taskpools import pkg/codexdht/discv5/protocol as discv5 -import pkg/codex/logutils -import pkg/codex/stores -import pkg/codex/clock -import pkg/codex/systemclock -import pkg/codex/blockexchange -import pkg/codex/chunker -import pkg/codex/manifest -import pkg/codex/discovery -import pkg/codex/merkletree -import pkg/codex/blocktype as bt -import pkg/codex/rng +import pkg/storage/logutils +import pkg/storage/stores +import pkg/storage/clock +import pkg/storage/systemclock +import pkg/storage/blockexchange +import pkg/storage/chunker +import pkg/storage/manifest +import pkg/storage/discovery +import pkg/storage/merkletree +import pkg/storage/blocktype as bt +import pkg/storage/rng -import pkg/codex/node {.all.} +import pkg/storage/node {.all.} import ../../asynctest import ../examples @@ -36,7 +36,7 @@ import ../slots/helpers import ./helpers -privateAccess(CodexNodeRef) # enable access to private fields +privateAccess(StorageNodeRef) # enable access to private fields asyncchecksuite "Test Node - Basic": setupAndTearDown() diff --git a/tests/storage/slots/helpers.nim b/tests/storage/slots/helpers.nim index 17c4d401..e476cfbd 100644 --- a/tests/storage/slots/helpers.nim +++ b/tests/storage/slots/helpers.nim @@ -3,13 +3,13 @@ import std/sugar import pkg/chronos import pkg/libp2p/cid -import pkg/codex/codextypes -import pkg/codex/stores -import pkg/codex/merkletree -import pkg/codex/manifest -import pkg/codex/blocktype as bt -import pkg/codex/chunker -import pkg/codex/rng +import pkg/storage/storagetypes +import pkg/storage/stores +import pkg/storage/merkletree +import pkg/storage/manifest +import pkg/storage/blocktype as bt +import pkg/storage/chunker +import pkg/storage/rng import pkg/taskpools import ../helpers diff --git a/tests/storage/stores/commonstoretests.nim b/tests/storage/stores/commonstoretests.nim index 03627f87..9292a9af 100644 --- a/tests/storage/stores/commonstoretests.nim +++ b/tests/storage/stores/commonstoretests.nim @@ -7,11 +7,11 @@ import pkg/libp2p/multicodec import pkg/stew/byteutils import pkg/questionable import pkg/questionable/results -import pkg/codex/stores/cachestore -import pkg/codex/chunker -import pkg/codex/manifest -import pkg/codex/merkletree -import pkg/codex/utils +import pkg/storage/stores/cachestore +import pkg/storage/chunker +import pkg/storage/manifest +import pkg/storage/merkletree +import pkg/storage/utils import ../../asynctest import ../helpers diff --git a/tests/storage/stores/repostore/testcoders.nim b/tests/storage/stores/repostore/testcoders.nim index 26a8642d..0ac8bbd8 100644 --- a/tests/storage/stores/repostore/testcoders.nim +++ b/tests/storage/stores/repostore/testcoders.nim @@ -5,9 +5,9 @@ import pkg/stew/objects import pkg/questionable import pkg/questionable/results -import pkg/codex/clock -import pkg/codex/stores/repostore/types -import pkg/codex/stores/repostore/coders +import pkg/storage/clock +import pkg/storage/stores/repostore/types +import pkg/storage/stores/repostore/coders import ../../helpers diff --git a/tests/storage/stores/testcachestore.nim b/tests/storage/stores/testcachestore.nim index 03075e1a..949272ff 100644 --- a/tests/storage/stores/testcachestore.nim +++ b/tests/storage/stores/testcachestore.nim @@ -3,8 +3,8 @@ import std/strutils import pkg/chronos import pkg/stew/byteutils import pkg/questionable/results -import pkg/codex/stores/cachestore -import pkg/codex/chunker +import pkg/storage/stores/cachestore +import pkg/storage/chunker import ./commonstoretests diff --git a/tests/storage/stores/testkeyutils.nim b/tests/storage/stores/testkeyutils.nim index 799e1884..0f75a2ed 100644 --- a/tests/storage/stores/testkeyutils.nim +++ b/tests/storage/stores/testkeyutils.nim @@ -12,9 +12,9 @@ import std/sequtils import pkg/chronos import pkg/questionable/results import pkg/libp2p -import pkg/codex/blocktype as bt -import pkg/codex/stores/repostore -import pkg/codex/clock +import pkg/storage/blocktype as bt +import pkg/storage/stores/repostore +import pkg/storage/clock import ../../asynctest import ../helpers/mocktimer @@ -22,8 +22,8 @@ import ../helpers/mockrepostore import ../helpers/mockclock import ../examples -import codex/namespaces -import codex/stores/keyutils +import storage/namespaces +import storage/stores/keyutils proc createManifestCid(): ?!Cid = let @@ -49,7 +49,7 @@ suite "KeyUtils": check: namespaces.len == 4 - namespaces[0].value == CodexRepoNamespace + namespaces[0].value == StorageRepoNamespace namespaces[1].value == "blocks" namespaces[2].value == expectedPrefix namespaces[3].value == expectedPostfix @@ -65,7 +65,7 @@ suite "KeyUtils": check: namespaces.len == 4 - namespaces[0].value == CodexRepoNamespace + namespaces[0].value == StorageRepoNamespace namespaces[1].value == "manifests" namespaces[2].value == expectedPrefix namespaces[3].value == expectedPostfix @@ -78,7 +78,7 @@ suite "KeyUtils": check: namespaces.len == 3 - namespaces[0].value == CodexMetaNamespace + namespaces[0].value == StorageMetaNamespace namespaces[1].value == "ttl" namespaces[2].value == $cid @@ -88,6 +88,6 @@ suite "KeyUtils": check: namespaces.len == 3 - namespaces[0].value == CodexMetaNamespace + namespaces[0].value == StorageMetaNamespace namespaces[1].value == "ttl" namespaces[2].value == "*" diff --git a/tests/storage/stores/testmaintenance.nim b/tests/storage/stores/testmaintenance.nim index 7c979d4c..520f89e5 100644 --- a/tests/storage/stores/testmaintenance.nim +++ b/tests/storage/stores/testmaintenance.nim @@ -8,9 +8,9 @@ ## those terms. import pkg/chronos -import pkg/codex/blocktype as bt -import pkg/codex/stores/repostore -import pkg/codex/clock +import pkg/storage/blocktype as bt +import pkg/storage/stores/repostore +import pkg/storage/clock import ../../asynctest import ../helpers @@ -19,7 +19,7 @@ import ../helpers/mockrepostore import ../helpers/mockclock import ../examples -import codex/stores/maintenance +import storage/stores/maintenance suite "BlockMaintainer": var mockRepoStore: MockRepoStore diff --git a/tests/storage/stores/testqueryiterhelper.nim b/tests/storage/stores/testqueryiterhelper.nim index 4e83dad4..8e38849f 100644 --- a/tests/storage/stores/testqueryiterhelper.nim +++ b/tests/storage/stores/testqueryiterhelper.nim @@ -5,8 +5,8 @@ import pkg/questionable import pkg/chronos import pkg/datastore/typedds import pkg/datastore/sql/sqliteds -import pkg/codex/stores/queryiterhelper -import pkg/codex/utils/asynciter +import pkg/storage/stores/queryiterhelper +import pkg/storage/utils/asynciter import ../../asynctest import ../helpers diff --git a/tests/storage/stores/testrepostore.nim b/tests/storage/stores/testrepostore.nim index b524c860..0af4e742 100644 --- a/tests/storage/stores/testrepostore.nim +++ b/tests/storage/stores/testrepostore.nim @@ -9,14 +9,14 @@ import pkg/chronos import pkg/stew/byteutils import pkg/datastore -import pkg/codex/stores/cachestore -import pkg/codex/chunker -import pkg/codex/stores -import pkg/codex/stores/repostore/operations -import pkg/codex/blocktype as bt -import pkg/codex/clock -import pkg/codex/utils/safeasynciter -import pkg/codex/merkletree +import pkg/storage/stores/cachestore +import pkg/storage/chunker +import pkg/storage/stores +import pkg/storage/stores/repostore/operations +import pkg/storage/blocktype as bt +import pkg/storage/clock +import pkg/storage/utils/safeasynciter +import pkg/storage/merkletree import ../../asynctest import ../helpers diff --git a/tests/storage/testasyncheapqueue.nim b/tests/storage/testasyncheapqueue.nim index 2d2cfb0c..8a6459e6 100644 --- a/tests/storage/testasyncheapqueue.nim +++ b/tests/storage/testasyncheapqueue.nim @@ -1,8 +1,8 @@ import pkg/chronos import pkg/results -import pkg/codex/utils/asyncheapqueue -import pkg/codex/rng +import pkg/storage/utils/asyncheapqueue +import pkg/storage/rng import ../asynctest import ./helpers diff --git a/tests/storage/testasyncstreamwrapper.nim b/tests/storage/testasyncstreamwrapper.nim index 2aa23039..63ff0b67 100644 --- a/tests/storage/testasyncstreamwrapper.nim +++ b/tests/storage/testasyncstreamwrapper.nim @@ -2,7 +2,7 @@ import pkg/chronos import pkg/chronos/transports/stream import pkg/chronos/transports/common import pkg/chronos/streams/asyncstream -import pkg/codex/streams +import pkg/storage/streams import pkg/stew/byteutils import ../asynctest diff --git a/tests/storage/testblocktype.nim b/tests/storage/testblocktype.nim index b0ea2732..d987b16e 100644 --- a/tests/storage/testblocktype.nim +++ b/tests/storage/testblocktype.nim @@ -1,7 +1,7 @@ import pkg/unittest2 import pkg/libp2p/cid -import pkg/codex/blocktype +import pkg/storage/blocktype import ./examples diff --git a/tests/storage/testchunking.nim b/tests/storage/testchunking.nim index 531d92bf..9feabc49 100644 --- a/tests/storage/testchunking.nim +++ b/tests/storage/testchunking.nim @@ -1,6 +1,6 @@ import pkg/stew/byteutils -import pkg/codex/chunker -import pkg/codex/logutils +import pkg/storage/chunker +import pkg/storage/logutils import pkg/chronos import ../asynctest diff --git a/tests/storage/testclock.nim b/tests/storage/testclock.nim index 967de672..fdba653e 100644 --- a/tests/storage/testclock.nim +++ b/tests/storage/testclock.nim @@ -1,6 +1,6 @@ import pkg/unittest2 -import codex/clock +import storage/clock import ./helpers suite "Clock": diff --git a/tests/storage/testlogutils.nim b/tests/storage/testlogutils.nim index ba3a00f3..1918137f 100644 --- a/tests/storage/testlogutils.nim +++ b/tests/storage/testlogutils.nim @@ -2,11 +2,11 @@ import std/options import std/strutils import pkg/unittest2 -import pkg/codex/blocktype -import pkg/codex/conf -import pkg/codex/logutils -import pkg/codex/units -import pkg/codex/utils/json +import pkg/storage/blocktype +import pkg/storage/conf +import pkg/storage/logutils +import pkg/storage/units +import pkg/storage/utils/json import pkg/libp2p/cid import pkg/libp2p/multiaddress import pkg/questionable diff --git a/tests/storage/testmanifest.nim b/tests/storage/testmanifest.nim index 6e773b0e..01e5f859 100644 --- a/tests/storage/testmanifest.nim +++ b/tests/storage/testmanifest.nim @@ -1,10 +1,10 @@ import pkg/chronos import pkg/questionable/results -import pkg/codex/chunker -import pkg/codex/blocktype as bt -import pkg/codex/manifest +import pkg/storage/chunker +import pkg/storage/blocktype as bt +import pkg/storage/manifest -import pkg/codex/merkletree +import pkg/storage/merkletree import ../asynctest import ./helpers diff --git a/tests/storage/testmerkletree.nim b/tests/storage/testmerkletree.nim index 47ba59cb..9f06acf2 100644 --- a/tests/storage/testmerkletree.nim +++ b/tests/storage/testmerkletree.nim @@ -1,4 +1,4 @@ -import ./merkletree/testcodextree -import ./merkletree/testcodexcoders +import ./merkletree/teststoragetree +import ./merkletree/teststoragecoders {.warning[UnusedImport]: off.} diff --git a/tests/storage/testnat.nim b/tests/storage/testnat.nim index c47b71de..d8539cb5 100644 --- a/tests/storage/testnat.nim +++ b/tests/storage/testnat.nim @@ -3,9 +3,9 @@ import pkg/chronos import pkg/libp2p/[multiaddress, multihash, multicodec] import pkg/results -import ../../codex/nat -import ../../codex/utils/natutils -import ../../codex/utils +import ../../storage/nat +import ../../storage/utils/natutils +import ../../storage/utils suite "NAT Address Tests": test "nattedAddress with local addresses": diff --git a/tests/storage/testrng.nim b/tests/storage/testrng.nim index f97a253b..d086f551 100644 --- a/tests/storage/testrng.nim +++ b/tests/storage/testrng.nim @@ -2,7 +2,7 @@ import std/unittest import std/sequtils import std/sets -import ../../codex/rng +import ../../storage/rng suite "Random Number Generator (RNG)": let rng = Rng.instance() diff --git a/tests/storage/teststorestream.nim b/tests/storage/teststorestream.nim index c1bd32dd..ad779808 100644 --- a/tests/storage/teststorestream.nim +++ b/tests/storage/teststorestream.nim @@ -1,6 +1,6 @@ import pkg/chronos -import pkg/codex/[streams, stores, manifest, blocktype as bt] +import pkg/storage/[streams, stores, manifest, blocktype as bt] import ../asynctest import ./examples diff --git a/tests/storage/testsystemclock.nim b/tests/storage/testsystemclock.nim index 3f179260..4f701b98 100644 --- a/tests/storage/testsystemclock.nim +++ b/tests/storage/testsystemclock.nim @@ -1,7 +1,7 @@ import std/times import pkg/unittest2 -import pkg/codex/systemclock +import pkg/storage/systemclock import ./helpers suite "SystemClock": diff --git a/tests/storage/utils/testasynciter.nim b/tests/storage/utils/testasynciter.nim index 984cbca7..17d909b3 100644 --- a/tests/storage/utils/testasynciter.nim +++ b/tests/storage/utils/testasynciter.nim @@ -1,6 +1,6 @@ import pkg/questionable import pkg/chronos -import pkg/codex/utils/asynciter +import pkg/storage/utils/asynciter import ../../asynctest import ../helpers diff --git a/tests/storage/utils/testasyncstatemachine.nim b/tests/storage/utils/testasyncstatemachine.nim index c46bb605..98d376ed 100644 --- a/tests/storage/utils/testasyncstatemachine.nim +++ b/tests/storage/utils/testasyncstatemachine.nim @@ -1,6 +1,6 @@ import pkg/questionable import pkg/chronos -import codex/utils/asyncstatemachine +import storage/utils/asyncstatemachine import ../../asynctest import ../helpers diff --git a/tests/storage/utils/testiter.nim b/tests/storage/utils/testiter.nim index ec19c484..39f8e67d 100644 --- a/tests/storage/utils/testiter.nim +++ b/tests/storage/utils/testiter.nim @@ -2,7 +2,7 @@ import std/sugar import pkg/questionable import pkg/chronos -import pkg/codex/utils/iter +import pkg/storage/utils/iter import ../../asynctest import ../helpers diff --git a/tests/storage/utils/testkeyutils.nim b/tests/storage/utils/testkeyutils.nim index 104258f3..b351bbce 100644 --- a/tests/storage/utils/testkeyutils.nim +++ b/tests/storage/utils/testkeyutils.nim @@ -1,7 +1,7 @@ import std/os import pkg/unittest2 -import pkg/codex/utils/keyutils +import pkg/storage/utils/keyutils import ../helpers @@ -9,7 +9,7 @@ when defined(windows): import stew/windows/acl suite "keyutils": - let path = getTempDir() / "CodexTest" + let path = getTempDir() / "StorageTest" setup: os.createDir(path) diff --git a/tests/storage/utils/testoptions.nim b/tests/storage/utils/testoptions.nim index 650715bc..1cc749fa 100644 --- a/tests/storage/utils/testoptions.nim +++ b/tests/storage/utils/testoptions.nim @@ -1,5 +1,5 @@ import pkg/unittest2 -import pkg/codex/utils/options +import pkg/storage/utils/options import ../helpers diff --git a/tests/storage/utils/testsafeasynciter.nim b/tests/storage/utils/testsafeasynciter.nim index 87b0d84a..c3e1e12d 100644 --- a/tests/storage/utils/testsafeasynciter.nim +++ b/tests/storage/utils/testsafeasynciter.nim @@ -1,8 +1,8 @@ import std/sugar import pkg/questionable import pkg/chronos -import pkg/codex/utils/iter -import pkg/codex/utils/safeasynciter +import pkg/storage/utils/iter +import pkg/storage/utils/safeasynciter import ../../asynctest import ../helpers diff --git a/tests/storage/utils/testtimer.nim b/tests/storage/utils/testtimer.nim index 79b95e42..6a349c6f 100644 --- a/tests/storage/utils/testtimer.nim +++ b/tests/storage/utils/testtimer.nim @@ -9,7 +9,7 @@ import pkg/chronos -import codex/utils/timer +import storage/utils/timer import ../../asynctest import ../helpers diff --git a/tests/storage/utils/testtrackedfutures.nim b/tests/storage/utils/testtrackedfutures.nim index 993d5b43..af2c5f00 100644 --- a/tests/storage/utils/testtrackedfutures.nim +++ b/tests/storage/utils/testtrackedfutures.nim @@ -1,5 +1,5 @@ import pkg/chronos -import codex/utils/trackedfutures +import storage/utils/trackedfutures import ../../asynctest import ../helpers diff --git a/tests/storage/utils/testutils.nim b/tests/storage/utils/testutils.nim index b2150deb..306dccbc 100644 --- a/tests/storage/utils/testutils.nim +++ b/tests/storage/utils/testutils.nim @@ -1,6 +1,6 @@ import pkg/unittest2 -import pkg/codex/utils +import pkg/storage/utils suite "findIt": setup: diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index eef2f79e..5e289ff5 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -4,8 +4,8 @@ import ./imports ## Limit which integration tests to run by setting the ## environment variable during compilation. For example: -## CODEX_INTEGRATION_TEST_INCLUDES="testFoo.nim,testBar.nim" -const includes = getEnv("CODEX_INTEGRATION_TEST_INCLUDES") +## STORAGE_INTEGRATION_TEST_INCLUDES="testFoo.nim,testBar.nim" +const includes = getEnv("STORAGE_INTEGRATION_TEST_INCLUDES") when includes != "": # import only the specified tests diff --git a/tests/testCodex.nim b/tests/testStorage.nim similarity index 100% rename from tests/testCodex.nim rename to tests/testStorage.nim