mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-03-16 09:13:16 +00:00
WIP renaming
# Conflicts: # library/libstorage.nim # Conflicts: # storage/rest/api.nim # storage/storage.nim
This commit is contained in:
E M
parent
172ce8fe0b
commit
d62fbf5d52
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
@ -49,7 +49,7 @@ jobs:
|
||||
uses: arnetheduck/nph-action@v1
|
||||
with:
|
||||
version: latest
|
||||
options: "codex/ tests/"
|
||||
options: "storage/ tests/"
|
||||
fail: true
|
||||
suggest: true
|
||||
|
||||
|
||||
10
Makefile
10
Makefile
@ -181,11 +181,11 @@ endif
|
||||
|
||||
coverage:
|
||||
$(MAKE) NIMFLAGS="$(NIMFLAGS) --lineDir:on --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage" test
|
||||
cd nimcache/release/testCodex && rm -f *.c
|
||||
cd nimcache/release/testStorage && rm -f *.c
|
||||
mkdir -p coverage
|
||||
lcov --capture --keep-going --directory nimcache/release/testCodex --output-file coverage/coverage.info
|
||||
shopt -s globstar && ls $$(pwd)/codex/{*,**/*}.nim
|
||||
shopt -s globstar && lcov --extract coverage/coverage.info --keep-going $$(pwd)/codex/{*,**/*}.nim --output-file coverage/coverage.f.info
|
||||
lcov --capture --keep-going --directory nimcache/release/testStorage --output-file coverage/coverage.info
|
||||
shopt -s globstar && ls $$(pwd)/storage/{*,**/*}.nim
|
||||
shopt -s globstar && lcov --extract coverage/coverage.info --keep-going $$(pwd)/storage/{*,**/*}.nim --output-file coverage/coverage.f.info
|
||||
echo -e $(BUILD_MSG) "coverage/report/index.html"
|
||||
genhtml coverage/coverage.f.info --keep-going --output-directory coverage/report
|
||||
|
||||
@ -237,7 +237,7 @@ nph/%: build-nph
|
||||
|
||||
format:
|
||||
$(NPH) *.nim
|
||||
$(NPH) codex/
|
||||
$(NPH) storage/
|
||||
$(NPH) tests/
|
||||
$(NPH) library/
|
||||
|
||||
|
||||
@ -11,12 +11,12 @@
|
||||
[](https://github.com/logos-storage/logos-storage-nim/actions/workflows/docker.yml?query=branch%3Amaster)
|
||||
[](https://codecov.io/gh/logos-storage/logos-storage-nim)
|
||||
[](https://discord.gg/CaJTh24ddQ)
|
||||
|
||||
|
||||
|
||||
|
||||
## Build and Run
|
||||
|
||||
For detailed instructions on preparing to build logos-storagenim see [*Build Logos Storage*](https://docs.codex.storage/learn/build).
|
||||
For detailed instructions on preparing to build logos-storage-nim see [*Build Logos Storage*](https://docs.codex.storage/learn/build).
|
||||
|
||||
To build the project, clone it and run:
|
||||
|
||||
|
||||
@ -39,7 +39,7 @@ import ./storage_thread_requests/requests/node_download_request
|
||||
import ./storage_thread_requests/requests/node_storage_request
|
||||
import ./ffi_types
|
||||
|
||||
from ../codex/conf import codexVersion
|
||||
from ../storage/conf import storageVersion
|
||||
|
||||
logScope:
|
||||
topics = "libstorage"
|
||||
@ -125,12 +125,12 @@ proc storage_new(
|
||||
proc storage_version(ctx: ptr StorageContext): ptr cchar {.dynlib, exportc.} =
|
||||
initializeLibrary()
|
||||
|
||||
return asNewCString(conf.codexVersion)
|
||||
return asNewCString(conf.storageVersion)
|
||||
|
||||
proc storage_revision(ctx: ptr StorageContext): ptr cchar {.dynlib, exportc.} =
|
||||
initializeLibrary()
|
||||
|
||||
return asNewCString(conf.codexRevision)
|
||||
return asNewCString(conf.storageVersion)
|
||||
|
||||
proc storage_repo(
|
||||
ctx: ptr StorageContext, callback: StorageCallback, userData: pointer
|
||||
|
||||
@ -16,7 +16,7 @@ import taskpools/channels_spsc_single
|
||||
import ./ffi_types
|
||||
import ./storage_thread_requests/[storage_thread_request]
|
||||
|
||||
from ../codex/codex import CodexServer
|
||||
from ../storage/storage import StorageServer
|
||||
|
||||
logScope:
|
||||
topics = "libstorage"
|
||||
@ -124,7 +124,7 @@ proc sendRequestToStorageThread*(
|
||||
ok()
|
||||
|
||||
proc runStorage(ctx: ptr StorageContext) {.async: (raises: []).} =
|
||||
var storage: CodexServer
|
||||
var storage: StorageServer
|
||||
|
||||
while true:
|
||||
try:
|
||||
|
||||
@ -10,11 +10,11 @@ import chronos
|
||||
import chronicles
|
||||
import codexdht/discv5/spr
|
||||
import ../../alloc
|
||||
import ../../../codex/conf
|
||||
import ../../../codex/rest/json
|
||||
import ../../../codex/node
|
||||
import ../../../storage/conf
|
||||
import ../../../storage/rest/json
|
||||
import ../../../storage/node
|
||||
|
||||
from ../../../codex/codex import CodexServer, node
|
||||
from ../../../storage/storage import StorageServer, node
|
||||
|
||||
logScope:
|
||||
topics = "libstorage libstoragedebug"
|
||||
@ -47,7 +47,7 @@ proc destroyShared(self: ptr NodeDebugRequest) =
|
||||
deallocShared(self)
|
||||
|
||||
proc getDebug(
|
||||
storage: ptr CodexServer
|
||||
storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
let node = storage[].node
|
||||
let table = RestRoutingTable.init(node.discovery.protocol.routingTable)
|
||||
@ -64,7 +64,7 @@ proc getDebug(
|
||||
return ok($json)
|
||||
|
||||
proc getPeer(
|
||||
storage: ptr CodexServer, peerId: cstring
|
||||
storage: ptr StorageServer, peerId: cstring
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
when storage_enable_api_debug_peers:
|
||||
let node = storage[].node
|
||||
@ -88,7 +88,7 @@ proc getPeer(
|
||||
return err("Failed to get peer: peer debug API is disabled")
|
||||
|
||||
proc updateLogLevel(
|
||||
storage: ptr CodexServer, logLevel: cstring
|
||||
storage: ptr StorageServer, logLevel: cstring
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
try:
|
||||
{.gcsafe.}:
|
||||
@ -99,7 +99,7 @@ proc updateLogLevel(
|
||||
return ok("")
|
||||
|
||||
proc process*(
|
||||
self: ptr NodeDebugRequest, storage: ptr CodexServer
|
||||
self: ptr NodeDebugRequest, storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
defer:
|
||||
destroyShared(self)
|
||||
|
||||
@ -21,12 +21,12 @@ import chronicles
|
||||
import libp2p/stream/[lpstream]
|
||||
import serde/json as serde
|
||||
import ../../alloc
|
||||
import ../../../codex/units
|
||||
import ../../../codex/codextypes
|
||||
import ../../../storage/units
|
||||
import ../../../storage/storagetypes
|
||||
|
||||
from ../../../codex/codex import CodexServer, node
|
||||
from ../../../codex/node import retrieve, fetchManifest
|
||||
from ../../../codex/rest/json import `%`, RestContent
|
||||
from ../../../storage/storage import StorageServer, node
|
||||
from ../../../storage/node import retrieve, fetchManifest
|
||||
from ../../../storage/rest/json import `%`, RestContent
|
||||
from libp2p import Cid, init, `$`
|
||||
|
||||
logScope:
|
||||
@ -80,7 +80,7 @@ proc destroyShared(self: ptr NodeDownloadRequest) =
|
||||
deallocShared(self)
|
||||
|
||||
proc init(
|
||||
storage: ptr CodexServer, cCid: cstring = "", chunkSize: csize_t = 0, local: bool
|
||||
storage: ptr StorageServer, cCid: cstring = "", chunkSize: csize_t = 0, local: bool
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
## Init a new session to download the file identified by cid.
|
||||
##
|
||||
@ -114,7 +114,7 @@ proc init(
|
||||
return ok("")
|
||||
|
||||
proc chunk(
|
||||
storage: ptr CodexServer, cCid: cstring = "", onChunk: OnChunkHandler
|
||||
storage: ptr StorageServer, cCid: cstring = "", onChunk: OnChunkHandler
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
## Download the next chunk of the file identified by cid.
|
||||
## The chunk is passed to the onChunk handler.
|
||||
@ -164,7 +164,7 @@ proc chunk(
|
||||
return ok("")
|
||||
|
||||
proc streamData(
|
||||
storage: ptr CodexServer,
|
||||
storage: ptr StorageServer,
|
||||
stream: LPStream,
|
||||
onChunk: OnChunkHandler,
|
||||
chunkSize: csize_t,
|
||||
@ -207,7 +207,7 @@ proc streamData(
|
||||
return ok("")
|
||||
|
||||
proc stream(
|
||||
storage: ptr CodexServer,
|
||||
storage: ptr StorageServer,
|
||||
cCid: cstring,
|
||||
chunkSize: csize_t,
|
||||
local: bool,
|
||||
@ -251,7 +251,7 @@ proc stream(
|
||||
return ok("")
|
||||
|
||||
proc cancel(
|
||||
storage: ptr CodexServer, cCid: cstring
|
||||
storage: ptr StorageServer, cCid: cstring
|
||||
): Future[Result[string, string]] {.raises: [], async: (raises: []).} =
|
||||
## Cancel the download session identified by cid.
|
||||
## This operation is not supported when using the stream mode,
|
||||
@ -279,7 +279,7 @@ proc cancel(
|
||||
return ok("")
|
||||
|
||||
proc manifest(
|
||||
storage: ptr CodexServer, cCid: cstring
|
||||
storage: ptr StorageServer, cCid: cstring
|
||||
): Future[Result[string, string]] {.raises: [], async: (raises: []).} =
|
||||
let cid = Cid.init($cCid)
|
||||
if cid.isErr:
|
||||
@ -296,7 +296,7 @@ proc manifest(
|
||||
return err("Failed to fetch manifest: download cancelled.")
|
||||
|
||||
proc process*(
|
||||
self: ptr NodeDownloadRequest, storage: ptr CodexServer, onChunk: OnChunkHandler
|
||||
self: ptr NodeDownloadRequest, storage: ptr StorageServer, onChunk: OnChunkHandler
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
defer:
|
||||
destroyShared(self)
|
||||
|
||||
@ -5,11 +5,11 @@ import chronos
|
||||
import chronicles
|
||||
import confutils
|
||||
import codexdht/discv5/spr
|
||||
import ../../../codex/conf
|
||||
import ../../../codex/rest/json
|
||||
import ../../../codex/node
|
||||
import ../../../storage/conf
|
||||
import ../../../storage/rest/json
|
||||
import ../../../storage/node
|
||||
|
||||
from ../../../codex/codex import CodexServer, config, node
|
||||
from ../../../storage/storage import StorageServer, config, node
|
||||
|
||||
logScope:
|
||||
topics = "libstorage libstorageinfo"
|
||||
@ -31,12 +31,12 @@ proc destroyShared(self: ptr NodeInfoRequest) =
|
||||
deallocShared(self)
|
||||
|
||||
proc getRepo(
|
||||
storage: ptr CodexServer
|
||||
storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
return ok($(storage[].config.dataDir))
|
||||
|
||||
proc getSpr(
|
||||
storage: ptr CodexServer
|
||||
storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
let spr = storage[].node.discovery.dhtRecord
|
||||
if spr.isNone:
|
||||
@ -45,12 +45,12 @@ proc getSpr(
|
||||
return ok(spr.get.toURI)
|
||||
|
||||
proc getPeerId(
|
||||
storage: ptr CodexServer
|
||||
storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
return ok($storage[].node.switch.peerInfo.peerId)
|
||||
|
||||
proc process*(
|
||||
self: ptr NodeInfoRequest, storage: ptr CodexServer
|
||||
self: ptr NodeInfoRequest, storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
defer:
|
||||
destroyShared(self)
|
||||
|
||||
@ -17,12 +17,12 @@ import libp2p
|
||||
import json_serialization
|
||||
import json_serialization/std/[options, net]
|
||||
import ../../alloc
|
||||
import ../../../codex/conf
|
||||
import ../../../codex/utils
|
||||
import ../../../codex/utils/[keyutils, fileutils]
|
||||
import ../../../codex/units
|
||||
import ../../../storage/conf
|
||||
import ../../../storage/utils
|
||||
import ../../../storage/utils/[keyutils, fileutils]
|
||||
import ../../../storage/units
|
||||
|
||||
from ../../../codex/codex import CodexServer, new, start, stop, close
|
||||
from ../../../storage/storage import StorageServer, new, start, stop, close
|
||||
|
||||
logScope:
|
||||
topics = "libstorage libstoragelifecycle"
|
||||
@ -89,16 +89,16 @@ proc destroyShared(self: ptr NodeLifecycleRequest) =
|
||||
|
||||
proc createStorage(
|
||||
configJson: cstring
|
||||
): Future[Result[CodexServer, string]] {.async: (raises: []).} =
|
||||
var conf: CodexConf
|
||||
): Future[Result[StorageServer, string]] {.async: (raises: []).} =
|
||||
var conf: StorageConf
|
||||
|
||||
try:
|
||||
conf = CodexConf.load(
|
||||
version = codexFullVersion,
|
||||
conf = StorageConf.load(
|
||||
version = storageFullVersion,
|
||||
envVarsPrefix = "storage",
|
||||
cmdLine = @[],
|
||||
secondarySources = proc(
|
||||
config: CodexConf, sources: auto
|
||||
config: StorageConf, sources: auto
|
||||
) {.gcsafe, raises: [ConfigurationError].} =
|
||||
if configJson.len > 0:
|
||||
sources.addConfigFileContent(Json, $(configJson))
|
||||
@ -149,14 +149,14 @@ proc createStorage(
|
||||
|
||||
let server =
|
||||
try:
|
||||
CodexServer.new(conf, pk)
|
||||
StorageServer.new(conf, pk)
|
||||
except Exception as exc:
|
||||
return err("Failed to create Storage: " & exc.msg)
|
||||
|
||||
return ok(server)
|
||||
|
||||
proc process*(
|
||||
self: ptr NodeLifecycleRequest, storage: ptr CodexServer
|
||||
self: ptr NodeLifecycleRequest, storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
defer:
|
||||
destroyShared(self)
|
||||
|
||||
@ -8,9 +8,9 @@ import chronos
|
||||
import chronicles
|
||||
import libp2p
|
||||
import ../../alloc
|
||||
import ../../../codex/node
|
||||
import ../../../storage/node
|
||||
|
||||
from ../../../codex/codex import CodexServer, node
|
||||
from ../../../storage/storage import StorageServer, node
|
||||
|
||||
logScope:
|
||||
topics = "libstorage libstoragep2p"
|
||||
@ -40,7 +40,7 @@ proc destroyShared(self: ptr NodeP2PRequest) =
|
||||
deallocShared(self)
|
||||
|
||||
proc connect(
|
||||
storage: ptr CodexServer, peerId: cstring, peerAddresses: seq[cstring] = @[]
|
||||
storage: ptr StorageServer, peerId: cstring, peerAddresses: seq[cstring] = @[]
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
let node = storage[].node
|
||||
let res = PeerId.init($peerId)
|
||||
@ -81,7 +81,7 @@ proc connect(
|
||||
return ok("")
|
||||
|
||||
proc process*(
|
||||
self: ptr NodeP2PRequest, storage: ptr CodexServer
|
||||
self: ptr NodeP2PRequest, storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
defer:
|
||||
destroyShared(self)
|
||||
|
||||
@ -14,12 +14,12 @@ import chronicles
|
||||
import libp2p/stream/[lpstream]
|
||||
import serde/json as serde
|
||||
import ../../alloc
|
||||
import ../../../codex/units
|
||||
import ../../../codex/manifest
|
||||
import ../../../codex/stores/repostore
|
||||
import ../../../storage/units
|
||||
import ../../../storage/manifest
|
||||
import ../../../storage/stores/repostore
|
||||
|
||||
from ../../../codex/codex import CodexServer, node, repoStore
|
||||
from ../../../codex/node import
|
||||
from ../../../storage/storage import StorageServer, node, repoStore
|
||||
from ../../../storage/node import
|
||||
iterateManifests, fetchManifest, fetchDatasetAsyncTask, delete, hasLocalBlock
|
||||
from libp2p import Cid, init, `$`
|
||||
|
||||
@ -61,7 +61,7 @@ type ManifestWithCid = object
|
||||
manifest {.serialize.}: Manifest
|
||||
|
||||
proc list(
|
||||
storage: ptr CodexServer
|
||||
storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
var manifests = newSeq[ManifestWithCid]()
|
||||
proc onManifest(cid: Cid, manifest: Manifest) {.raises: [], gcsafe.} =
|
||||
@ -78,7 +78,7 @@ proc list(
|
||||
return ok(serde.toJson(manifests))
|
||||
|
||||
proc delete(
|
||||
storage: ptr CodexServer, cCid: cstring
|
||||
storage: ptr StorageServer, cCid: cstring
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
let cid = Cid.init($cCid)
|
||||
if cid.isErr:
|
||||
@ -97,7 +97,7 @@ proc delete(
|
||||
return ok("")
|
||||
|
||||
proc fetch(
|
||||
storage: ptr CodexServer, cCid: cstring
|
||||
storage: ptr StorageServer, cCid: cstring
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
let cid = Cid.init($cCid)
|
||||
if cid.isErr:
|
||||
@ -116,7 +116,7 @@ proc fetch(
|
||||
return err("Failed to fetch the data: download cancelled.")
|
||||
|
||||
proc space(
|
||||
storage: ptr CodexServer
|
||||
storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
let repoStore = storage[].repoStore
|
||||
let space = StorageSpace(
|
||||
@ -128,7 +128,7 @@ proc space(
|
||||
return ok(serde.toJson(space))
|
||||
|
||||
proc exists(
|
||||
storage: ptr CodexServer, cCid: cstring
|
||||
storage: ptr StorageServer, cCid: cstring
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
let cid = Cid.init($cCid)
|
||||
if cid.isErr:
|
||||
@ -142,7 +142,7 @@ proc exists(
|
||||
return err("Failed to check the data existence: operation cancelled.")
|
||||
|
||||
proc process*(
|
||||
self: ptr NodeStorageRequest, storage: ptr CodexServer
|
||||
self: ptr NodeStorageRequest, storage: ptr StorageServer
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
defer:
|
||||
destroyShared(self)
|
||||
|
||||
@ -24,11 +24,11 @@ import questionable/results
|
||||
import faststreams/inputs
|
||||
import libp2p/stream/[bufferstream, lpstream]
|
||||
import ../../alloc
|
||||
import ../../../codex/units
|
||||
import ../../../codex/codextypes
|
||||
import ../../../storage/units
|
||||
import ../../../storage/storagetypes
|
||||
|
||||
from ../../../codex/codex import CodexServer, node
|
||||
from ../../../codex/node import store
|
||||
from ../../../storage/storage import StorageServer, node
|
||||
from ../../../storage/node import store
|
||||
from libp2p import Cid, `$`
|
||||
|
||||
logScope:
|
||||
@ -86,7 +86,7 @@ proc destroyShared(self: ptr NodeUploadRequest) =
|
||||
deallocShared(self)
|
||||
|
||||
proc init(
|
||||
storage: ptr CodexServer, filepath: cstring = "", chunkSize: csize_t = 0
|
||||
storage: ptr StorageServer, filepath: cstring = "", chunkSize: csize_t = 0
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
## Init a new session upload and return its ID.
|
||||
## The session contains the future corresponding to the
|
||||
@ -156,7 +156,7 @@ proc init(
|
||||
return ok(sessionId)
|
||||
|
||||
proc chunk(
|
||||
storage: ptr CodexServer, sessionId: cstring, chunk: seq[byte]
|
||||
storage: ptr StorageServer, sessionId: cstring, chunk: seq[byte]
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
## Upload a chunk of data to the session identified by sessionId.
|
||||
## The chunk is pushed to the BufferStream of the session.
|
||||
@ -205,7 +205,7 @@ proc chunk(
|
||||
return ok("")
|
||||
|
||||
proc finalize(
|
||||
storage: ptr CodexServer, sessionId: cstring
|
||||
storage: ptr StorageServer, sessionId: cstring
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
## Finalize the upload session identified by sessionId.
|
||||
## This closes the BufferStream and waits for the `node.store` future
|
||||
@ -242,7 +242,7 @@ proc finalize(
|
||||
session.fut.cancelSoon()
|
||||
|
||||
proc cancel(
|
||||
storage: ptr CodexServer, sessionId: cstring
|
||||
storage: ptr StorageServer, sessionId: cstring
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
## Cancel the upload session identified by sessionId.
|
||||
## This cancels the `node.store` future and removes the session
|
||||
@ -293,7 +293,7 @@ proc streamFile(
|
||||
return err("Failed to stream the file: " & $e.msg)
|
||||
|
||||
proc file(
|
||||
storage: ptr CodexServer, sessionId: cstring, onProgress: OnProgressHandler
|
||||
storage: ptr StorageServer, sessionId: cstring, onProgress: OnProgressHandler
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
## Starts the file upload for the session identified by sessionId.
|
||||
## Will call finalize when done and return the CID of the uploaded file.
|
||||
@ -333,7 +333,7 @@ proc file(
|
||||
|
||||
proc process*(
|
||||
self: ptr NodeUploadRequest,
|
||||
storage: ptr CodexServer,
|
||||
storage: ptr StorageServer,
|
||||
onUploadProgress: OnProgressHandler = nil,
|
||||
): Future[Result[string, string]] {.async: (raises: []).} =
|
||||
defer:
|
||||
|
||||
@ -14,7 +14,7 @@ import ./requests/node_upload_request
|
||||
import ./requests/node_download_request
|
||||
import ./requests/node_storage_request
|
||||
|
||||
from ../../codex/codex import CodexServer
|
||||
from ../../storage/storage import StorageServer
|
||||
|
||||
type RequestType* {.pure.} = enum
|
||||
LIFECYCLE
|
||||
@ -89,7 +89,7 @@ proc handleRes[T: string | void | seq[byte]](
|
||||
proc process*(
|
||||
T: type StorageThreadRequest,
|
||||
request: ptr StorageThreadRequest,
|
||||
storage: ptr CodexServer,
|
||||
storage: ptr StorageServer,
|
||||
) {.async: (raises: []).} =
|
||||
## Processes the request in the Logos Storage thread.
|
||||
## Dispatch to the appropriate request handler based on reqType.
|
||||
|
||||
4
nimbledeps/nimbledata2.json
Normal file
4
nimbledeps/nimbledata2.json
Normal file
@ -0,0 +1,4 @@
|
||||
{
|
||||
"version": 1,
|
||||
"reverseDeps": {}
|
||||
}
|
||||
36
storage.nim
36
storage.nim
@ -18,36 +18,36 @@ import pkg/confutils/toml/std/uri as confTomlUri
|
||||
import pkg/toml_serialization
|
||||
import pkg/libp2p
|
||||
|
||||
import ./codex/conf
|
||||
import ./codex/codex
|
||||
import ./codex/logutils
|
||||
import ./codex/units
|
||||
import ./codex/utils/keyutils
|
||||
import ./codex/codextypes
|
||||
import ./storage/conf
|
||||
import ./storage/storage
|
||||
import ./storage/logutils
|
||||
import ./storage/units
|
||||
import ./storage/utils/keyutils
|
||||
import ./storage/storagetypes
|
||||
|
||||
export codex, conf, libp2p, chronos, logutils
|
||||
export storage, conf, libp2p, chronos, logutils
|
||||
|
||||
when isMainModule:
|
||||
import std/os
|
||||
import pkg/confutils/defs
|
||||
import ./codex/utils/fileutils
|
||||
import ./storage/utils/fileutils
|
||||
|
||||
logScope:
|
||||
topics = "codex"
|
||||
topics = "storage"
|
||||
|
||||
when defined(posix):
|
||||
import system/ansi_c
|
||||
|
||||
type CodexStatus {.pure.} = enum
|
||||
type StorageStatus {.pure.} = enum
|
||||
Stopped
|
||||
Stopping
|
||||
Running
|
||||
|
||||
let config = CodexConf.load(
|
||||
version = codexFullVersion,
|
||||
let config = StorageConf.load(
|
||||
version = storageFullVersion,
|
||||
envVarsPrefix = "storage",
|
||||
secondarySources = proc(
|
||||
config: CodexConf, sources: auto
|
||||
config: StorageConf, sources: auto
|
||||
) {.gcsafe, raises: [ConfigurationError].} =
|
||||
if configFile =? config.configFile:
|
||||
sources.addConfigFile(Toml, configFile)
|
||||
@ -81,7 +81,7 @@ when isMainModule:
|
||||
trace "Repo dir initialized", dir = config.dataDir / "repo"
|
||||
|
||||
var
|
||||
state: CodexStatus
|
||||
state: StorageStatus
|
||||
shutdown: Future[void]
|
||||
|
||||
let
|
||||
@ -94,7 +94,7 @@ when isMainModule:
|
||||
privateKey = setupKey(keyPath).expect("Should setup private key!")
|
||||
server =
|
||||
try:
|
||||
CodexServer.new(config, privateKey)
|
||||
StorageServer.new(config, privateKey)
|
||||
except Exception as exc:
|
||||
error "Failed to start Logos Storage", msg = exc.msg
|
||||
quit QuitFailure
|
||||
@ -102,7 +102,7 @@ when isMainModule:
|
||||
## Ctrl+C handling
|
||||
proc doShutdown() =
|
||||
shutdown = server.shutdown()
|
||||
state = CodexStatus.Stopping
|
||||
state = StorageStatus.Stopping
|
||||
|
||||
notice "Stopping Logos Storage"
|
||||
|
||||
@ -142,8 +142,8 @@ when isMainModule:
|
||||
# had a chance to start (currently you'll get a SISGSEV if you try to).
|
||||
quit QuitFailure
|
||||
|
||||
state = CodexStatus.Running
|
||||
while state == CodexStatus.Running:
|
||||
state = StorageStatus.Running
|
||||
while state == StorageStatus.Running:
|
||||
try:
|
||||
# poll chronos
|
||||
chronos.poll()
|
||||
|
||||
@ -28,9 +28,9 @@ import ../../logutils
|
||||
import ../../manifest
|
||||
|
||||
logScope:
|
||||
topics = "codex discoveryengine advertiser"
|
||||
topics = "storage discoveryengine advertiser"
|
||||
|
||||
declareGauge(codex_inflight_advertise, "inflight advertise requests")
|
||||
declareGauge(storage_inflight_advertise, "inflight advertise requests")
|
||||
|
||||
const
|
||||
DefaultConcurrentAdvertRequests = 10
|
||||
@ -106,11 +106,11 @@ proc processQueueLoop(b: Advertiser) {.async: (raises: []).} =
|
||||
|
||||
let request = b.discovery.provide(cid)
|
||||
b.inFlightAdvReqs[cid] = request
|
||||
codex_inflight_advertise.set(b.inFlightAdvReqs.len.int64)
|
||||
storage_inflight_advertise.set(b.inFlightAdvReqs.len.int64)
|
||||
|
||||
defer:
|
||||
b.inFlightAdvReqs.del(cid)
|
||||
codex_inflight_advertise.set(b.inFlightAdvReqs.len.int64)
|
||||
storage_inflight_advertise.set(b.inFlightAdvReqs.len.int64)
|
||||
|
||||
await request
|
||||
except CancelledError:
|
||||
|
||||
@ -31,9 +31,9 @@ import ../../logutils
|
||||
import ../../manifest
|
||||
|
||||
logScope:
|
||||
topics = "codex discoveryengine"
|
||||
topics = "storage discoveryengine"
|
||||
|
||||
declareGauge(codex_inflight_discovery, "inflight discovery requests")
|
||||
declareGauge(storage_inflight_discovery, "inflight discovery requests")
|
||||
|
||||
const
|
||||
DefaultConcurrentDiscRequests = 10
|
||||
@ -114,11 +114,11 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async: (raises: []).} =
|
||||
if haves.len < b.minPeersPerBlock:
|
||||
let request = b.discovery.find(cid)
|
||||
b.inFlightDiscReqs[cid] = request
|
||||
codex_inflight_discovery.set(b.inFlightDiscReqs.len.int64)
|
||||
storage_inflight_discovery.set(b.inFlightDiscReqs.len.int64)
|
||||
|
||||
defer:
|
||||
b.inFlightDiscReqs.del(cid)
|
||||
codex_inflight_discovery.set(b.inFlightDiscReqs.len.int64)
|
||||
storage_inflight_discovery.set(b.inFlightDiscReqs.len.int64)
|
||||
|
||||
if (await request.withTimeout(DefaultDiscoveryTimeout)) and
|
||||
peers =? (await request).catch:
|
||||
|
||||
@ -44,39 +44,39 @@ import ./pendingblocks
|
||||
export peers, pendingblocks, discovery
|
||||
|
||||
logScope:
|
||||
topics = "codex blockexcengine"
|
||||
topics = "storage blockexcengine"
|
||||
|
||||
declareCounter(
|
||||
codex_block_exchange_want_have_lists_sent, "codex blockexchange wantHave lists sent"
|
||||
storage_block_exchange_want_have_lists_sent, "storage blockexchange wantHave lists sent"
|
||||
)
|
||||
declareCounter(
|
||||
codex_block_exchange_want_have_lists_received,
|
||||
"codex blockexchange wantHave lists received",
|
||||
storage_block_exchange_want_have_lists_received,
|
||||
"storage blockexchange wantHave lists received",
|
||||
)
|
||||
declareCounter(
|
||||
codex_block_exchange_want_block_lists_sent, "codex blockexchange wantBlock lists sent"
|
||||
storage_block_exchange_want_block_lists_sent, "storage blockexchange wantBlock lists sent"
|
||||
)
|
||||
declareCounter(
|
||||
codex_block_exchange_want_block_lists_received,
|
||||
"codex blockexchange wantBlock lists received",
|
||||
storage_block_exchange_want_block_lists_received,
|
||||
"storage blockexchange wantBlock lists received",
|
||||
)
|
||||
declareCounter(codex_block_exchange_blocks_sent, "codex blockexchange blocks sent")
|
||||
declareCounter(storage_block_exchange_blocks_sent, "storage blockexchange blocks sent")
|
||||
declareCounter(
|
||||
codex_block_exchange_blocks_received, "codex blockexchange blocks received"
|
||||
storage_block_exchange_blocks_received, "storage blockexchange blocks received"
|
||||
)
|
||||
declareCounter(
|
||||
codex_block_exchange_spurious_blocks_received,
|
||||
"codex blockexchange unrequested/duplicate blocks received",
|
||||
storage_block_exchange_spurious_blocks_received,
|
||||
"storage blockexchange unrequested/duplicate blocks received",
|
||||
)
|
||||
declareCounter(
|
||||
codex_block_exchange_discovery_requests_total,
|
||||
storage_block_exchange_discovery_requests_total,
|
||||
"Total number of peer discovery requests sent",
|
||||
)
|
||||
declareCounter(
|
||||
codex_block_exchange_peer_timeouts_total, "Total number of peer activity timeouts"
|
||||
storage_block_exchange_peer_timeouts_total, "Total number of peer activity timeouts"
|
||||
)
|
||||
declareCounter(
|
||||
codex_block_exchange_requests_failed_total,
|
||||
storage_block_exchange_requests_failed_total,
|
||||
"Total number of block requests that failed after exhausting retries",
|
||||
)
|
||||
|
||||
@ -166,7 +166,7 @@ proc sendWantHave(
|
||||
let toAsk = addresses.filterIt(it notin p.peerHave)
|
||||
trace "Sending wantHave request", toAsk, peer = p.id
|
||||
await self.network.request.sendWantList(p.id, toAsk, wantType = WantType.WantHave)
|
||||
codex_block_exchange_want_have_lists_sent.inc()
|
||||
storage_block_exchange_want_have_lists_sent.inc()
|
||||
|
||||
proc sendWantBlock(
|
||||
self: BlockExcEngine, addresses: seq[BlockAddress], blockPeer: BlockExcPeerCtx
|
||||
@ -175,7 +175,7 @@ proc sendWantBlock(
|
||||
await self.network.request.sendWantList(
|
||||
blockPeer.id, addresses, wantType = WantType.WantBlock
|
||||
) # we want this remote to send us a block
|
||||
codex_block_exchange_want_block_lists_sent.inc()
|
||||
storage_block_exchange_want_block_lists_sent.inc()
|
||||
|
||||
proc sendBatchedWantList(
|
||||
self: BlockExcEngine,
|
||||
@ -297,7 +297,7 @@ proc refreshBlockKnowledge(self: BlockExcEngine) {.async: (raises: [CancelledErr
|
||||
proc searchForNewPeers(self: BlockExcEngine, cid: Cid) =
|
||||
if self.lastDiscRequest + DiscoveryRateLimit < Moment.now():
|
||||
trace "Searching for new peers for", cid = cid
|
||||
codex_block_exchange_discovery_requests_total.inc()
|
||||
storage_block_exchange_discovery_requests_total.inc()
|
||||
self.lastDiscRequest = Moment.now() # always refresh before calling await!
|
||||
self.discovery.queueFindBlocksReq(@[cid])
|
||||
else:
|
||||
@ -333,7 +333,7 @@ proc downloadInternal(
|
||||
|
||||
if self.pendingBlocks.retriesExhausted(address):
|
||||
trace "Error retries exhausted"
|
||||
codex_block_exchange_requests_failed_total.inc()
|
||||
storage_block_exchange_requests_failed_total.inc()
|
||||
handle.fail(newException(RetriesExhaustedError, "Error retries exhausted"))
|
||||
break
|
||||
|
||||
@ -415,7 +415,7 @@ proc downloadInternal(
|
||||
else:
|
||||
# If the peer timed out, retries immediately.
|
||||
trace "Peer timed out during block request", peer = scheduledPeer.id
|
||||
codex_block_exchange_peer_timeouts_total.inc()
|
||||
storage_block_exchange_peer_timeouts_total.inc()
|
||||
await self.network.dropPeer(scheduledPeer.id)
|
||||
# Evicts peer immediately or we may end up picking it again in the
|
||||
# next retry.
|
||||
@ -426,7 +426,7 @@ proc downloadInternal(
|
||||
await handle.cancelAndWait()
|
||||
except RetriesExhaustedError as exc:
|
||||
warn "Retries exhausted for block", address, exc = exc.msg
|
||||
codex_block_exchange_requests_failed_total.inc()
|
||||
storage_block_exchange_requests_failed_total.inc()
|
||||
if not handle.finished:
|
||||
handle.fail(exc)
|
||||
finally:
|
||||
@ -690,7 +690,7 @@ proc blocksDeliveryHandler*(
|
||||
# Unknown peers and unrequested blocks are dropped with a warning.
|
||||
if not allowSpurious and (peerCtx == nil or not peerCtx.blockReceived(bd.address)):
|
||||
warn "Dropping unrequested or duplicate block received from peer"
|
||||
codex_block_exchange_spurious_blocks_received.inc()
|
||||
storage_block_exchange_spurious_blocks_received.inc()
|
||||
continue
|
||||
|
||||
if err =? self.validateBlockDelivery(bd).errorOption:
|
||||
@ -729,7 +729,7 @@ proc blocksDeliveryHandler*(
|
||||
discard
|
||||
lastIdle = Moment.now()
|
||||
|
||||
codex_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64)
|
||||
storage_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64)
|
||||
|
||||
if err =? catch(await self.resolveBlocks(validatedBlocksDelivery)).errorOption:
|
||||
warn "Error resolving blocks", err = err.msg
|
||||
@ -789,11 +789,11 @@ proc wantListHandler*(
|
||||
BlockPresence(address: e.address, `type`: BlockPresenceType.DontHave)
|
||||
)
|
||||
|
||||
codex_block_exchange_want_have_lists_received.inc()
|
||||
storage_block_exchange_want_have_lists_received.inc()
|
||||
of WantType.WantBlock:
|
||||
peerCtx.wantedBlocks.incl(e.address)
|
||||
schedulePeer = true
|
||||
codex_block_exchange_want_block_lists_received.inc()
|
||||
storage_block_exchange_want_block_lists_received.inc()
|
||||
else: # Updating existing entry in peer wants
|
||||
# peer doesn't want this block anymore
|
||||
if e.cancel:
|
||||
@ -903,7 +903,7 @@ proc taskHandler*(
|
||||
continue
|
||||
|
||||
await self.network.request.sendBlocksDelivery(peerCtx.id, blockDeliveries)
|
||||
codex_block_exchange_blocks_sent.inc(blockDeliveries.len.int64)
|
||||
storage_block_exchange_blocks_sent.inc(blockDeliveries.len.int64)
|
||||
# Drops the batch from the peer's set of wanted blocks; i.e. assumes that after
|
||||
# we send the blocks, then the peer no longer wants them, so we don't need to
|
||||
# re-send them. Note that the send might still fail down the line and we will
|
||||
|
||||
@ -22,14 +22,14 @@ import ../../blocktype
|
||||
import ../../logutils
|
||||
|
||||
logScope:
|
||||
topics = "codex pendingblocks"
|
||||
topics = "storage pendingblocks"
|
||||
|
||||
declareGauge(
|
||||
codex_block_exchange_pending_block_requests,
|
||||
"codex blockexchange pending block requests",
|
||||
storage_block_exchange_pending_block_requests,
|
||||
"storage blockexchange pending block requests",
|
||||
)
|
||||
declareGauge(
|
||||
codex_block_exchange_retrieval_time_us, "codex blockexchange block retrieval time us"
|
||||
storage_block_exchange_retrieval_time_us, "storage blockexchange block retrieval time us"
|
||||
)
|
||||
|
||||
const
|
||||
@ -53,7 +53,7 @@ type
|
||||
lastInclusion*: Moment # time at which we last included a block into our wantlist
|
||||
|
||||
proc updatePendingBlockGauge(p: PendingBlocksManager) =
|
||||
codex_block_exchange_pending_block_requests.set(p.blocks.len.int64)
|
||||
storage_block_exchange_pending_block_requests.set(p.blocks.len.int64)
|
||||
|
||||
proc getWantHandle*(
|
||||
self: PendingBlocksManager, address: BlockAddress, requested: ?PeerId = PeerId.none
|
||||
@ -123,7 +123,7 @@ proc resolve*(
|
||||
|
||||
blockReq.handle.complete(bd.blk)
|
||||
|
||||
codex_block_exchange_retrieval_time_us.set(retrievalDurationUs)
|
||||
storage_block_exchange_retrieval_time_us.set(retrievalDurationUs)
|
||||
else:
|
||||
trace "Block handle already finished", address = bd.address
|
||||
|
||||
|
||||
@ -27,10 +27,10 @@ import ./networkpeer
|
||||
export networkpeer
|
||||
|
||||
logScope:
|
||||
topics = "codex blockexcnetwork"
|
||||
topics = "storage blockexcnetwork"
|
||||
|
||||
const
|
||||
Codec* = "/codex/blockexc/1.0.0"
|
||||
Codec* = "/storage/blockexc/1.0.0"
|
||||
DefaultMaxInflight* = 100
|
||||
|
||||
type
|
||||
|
||||
@ -19,7 +19,7 @@ import ../../logutils
|
||||
import ../../utils/trackedfutures
|
||||
|
||||
logScope:
|
||||
topics = "codex blockexcnetworkpeer"
|
||||
topics = "storage blockexcnetworkpeer"
|
||||
|
||||
const DefaultYieldInterval = 50.millis
|
||||
|
||||
|
||||
@ -25,7 +25,7 @@ import ./peercontext
|
||||
export peercontext
|
||||
|
||||
logScope:
|
||||
topics = "codex peerctxstore"
|
||||
topics = "storage peerctxstore"
|
||||
|
||||
type
|
||||
PeerCtxStore* = ref object of RootObj
|
||||
|
||||
@ -25,9 +25,9 @@ import ./utils
|
||||
import ./errors
|
||||
import ./logutils
|
||||
import ./utils/json
|
||||
import ./codextypes
|
||||
import ./storagetypes
|
||||
|
||||
export errors, logutils, units, codextypes
|
||||
export errors, logutils, units, storagetypes
|
||||
|
||||
type
|
||||
Block* = ref object of RootObj
|
||||
|
||||
@ -35,7 +35,7 @@ import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/stew/base64
|
||||
|
||||
import ./codextypes
|
||||
import ./storagetypes
|
||||
import ./discovery
|
||||
import ./logutils
|
||||
import ./stores
|
||||
@ -46,7 +46,7 @@ import ./utils/natutils
|
||||
|
||||
from ./blockexchange/engine/pendingblocks import DefaultBlockRetries
|
||||
|
||||
export units, net, codextypes, logutils, completeCmdArg, parseCmdArg, NatConfig
|
||||
export units, net, storagetypes, logutils, completeCmdArg, parseCmdArg, NatConfig
|
||||
|
||||
export
|
||||
DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval,
|
||||
@ -90,7 +90,7 @@ type
|
||||
repoSQLite = "sqlite"
|
||||
repoLevelDb = "leveldb"
|
||||
|
||||
CodexConf* = object
|
||||
StorageConf* = object
|
||||
configFile* {.
|
||||
desc: "Loads the configuration from a TOML file",
|
||||
defaultValueDesc: "none",
|
||||
@ -277,19 +277,19 @@ type
|
||||
desc: "Logs to file", defaultValue: string.none, name: "log-file", hidden
|
||||
.}: Option[string]
|
||||
|
||||
func defaultAddress*(conf: CodexConf): IpAddress =
|
||||
func defaultAddress*(conf: StorageConf): IpAddress =
|
||||
result = static parseIpAddress("127.0.0.1")
|
||||
|
||||
func defaultNatConfig*(): NatConfig =
|
||||
result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAny)
|
||||
|
||||
proc getCodexVersion(): string =
|
||||
proc getStorageVersion(): string =
|
||||
let tag = strip(staticExec("git describe --tags --abbrev=0"))
|
||||
if tag.isEmptyOrWhitespace:
|
||||
return "untagged build"
|
||||
return tag
|
||||
|
||||
proc getCodexRevision(): string =
|
||||
proc getStorageRevision(): string =
|
||||
# using a slice in a static context breaks nimsuggest for some reason
|
||||
var res = strip(staticExec("git rev-parse --short HEAD"))
|
||||
return res
|
||||
@ -298,12 +298,12 @@ proc getNimBanner(): string =
|
||||
staticExec("nim --version | grep Version")
|
||||
|
||||
const
|
||||
codexVersion* = getCodexVersion()
|
||||
codexRevision* = getCodexRevision()
|
||||
storageVersion* = getStorageVersion()
|
||||
storageRevision* = getStorageRevision()
|
||||
nimBanner* = getNimBanner()
|
||||
|
||||
codexFullVersion* =
|
||||
"Storage version: " & codexVersion & "\p" & "Storage revision: " & codexRevision &
|
||||
storageFullVersion* =
|
||||
"Storage version: " & storageVersion & "\p" & "Storage revision: " & storageRevision &
|
||||
"\p"
|
||||
|
||||
proc parseCmdArg*(
|
||||
@ -533,7 +533,7 @@ proc updateLogLevel*(logLevel: string) {.raises: [ValueError].} =
|
||||
if not setTopicState(topicName, settings.state, settings.logLevel):
|
||||
warn "Unrecognized logging topic", topic = topicName
|
||||
|
||||
proc setupLogging*(conf: CodexConf) =
|
||||
proc setupLogging*(conf: StorageConf) =
|
||||
when defaultChroniclesStream.outputs.type.arity != 3:
|
||||
warn "Logging configuration options not enabled in the current build"
|
||||
else:
|
||||
@ -597,7 +597,7 @@ proc setupLogging*(conf: CodexConf) =
|
||||
else:
|
||||
defaultChroniclesStream.outputs[0].writer = writer
|
||||
|
||||
proc setupMetrics*(config: CodexConf) =
|
||||
proc setupMetrics*(config: StorageConf) =
|
||||
if config.metricsEnabled:
|
||||
let metricsAddress = config.metricsAddress
|
||||
notice "Starting metrics HTTP server",
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
const ContentIdsExts =
|
||||
[multiCodec("codex-root"), multiCodec("codex-manifest"), multiCodec("codex-block")]
|
||||
[multiCodec("storage-root"), multiCodec("storage-manifest"), multiCodec("storage-block")]
|
||||
|
||||
@ -32,7 +32,7 @@ export discv5
|
||||
# much more elegantly.
|
||||
|
||||
logScope:
|
||||
topics = "codex discovery"
|
||||
topics = "storage discovery"
|
||||
|
||||
type Discovery* = ref object of RootObj
|
||||
protocol*: discv5.Protocol # dht protocol
|
||||
|
||||
@ -20,8 +20,8 @@ import pkg/questionable/results
|
||||
export results
|
||||
|
||||
type
|
||||
CodexError* = object of CatchableError # base codex error
|
||||
CodexResult*[T] = Result[T, ref CodexError]
|
||||
StorageError* = object of CatchableError # base Storage error
|
||||
StorageResult*[T] = Result[T, ref StorageError]
|
||||
|
||||
FinishedFailed*[T] = tuple[success: seq[Future[T]], failure: seq[Future[T]]]
|
||||
|
||||
@ -37,7 +37,7 @@ template mapFailure*[T, V, E](
|
||||
)
|
||||
|
||||
template mapFailure*[T, V](exp: Result[T, V]): Result[T, ref CatchableError] =
|
||||
mapFailure(exp, CodexError)
|
||||
mapFailure(exp, StorageError)
|
||||
|
||||
# TODO: using a template here, causes bad codegen
|
||||
func toFailure*[T](exp: Option[T]): Result[T, ref CatchableError] {.inline.} =
|
||||
|
||||
@ -13,24 +13,24 @@
|
||||
## 6. Remove need to [avoid importing or exporting `toJson`, `%`, `%*` to prevent
|
||||
## conflicts](https://github.com/logos-storage/logos-storage-nim/pull/645#issuecomment-1838834467)
|
||||
##
|
||||
## When declaring a new type, one should consider importing the `codex/logutils`
|
||||
## When declaring a new type, one should consider importing the `storage/logutils`
|
||||
## module, and specifying `formatIt`. If textlines log output and json log output
|
||||
## need to be different, overload `formatIt` and specify a `LogFormat`. If json
|
||||
## serialization is needed, it can be declared with a `%` proc. `logutils`
|
||||
## imports and exports `nim-serde` which handles the de/serialization, examples
|
||||
## below. **Only `codex/logutils` needs to be imported.**
|
||||
## below. **Only `storage/logutils` needs to be imported.**
|
||||
##
|
||||
## Using `logutils` in the Codex codebase:
|
||||
## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils`
|
||||
## Using `logutils` in the Storage codebase:
|
||||
## - Instead of importing `pkg/chronicles`, import `pkg/storage/logutils`
|
||||
## - most of `chronicles` is exported by `logutils`
|
||||
## - Instead of importing `std/json`, import `pkg/serde/json`
|
||||
## - `std/json` is exported by `serde` which is exported by `logutils`
|
||||
## - Instead of importing `pkg/nim-json-serialization`, import
|
||||
## `pkg/serde/json` or use codex-specific overloads by importing `utils/json`
|
||||
## `pkg/serde/json` or use storage-specific overloads by importing `utils/json`
|
||||
## - one of the goals is to remove the use of `nim-json-serialization`
|
||||
##
|
||||
## ```nim
|
||||
## import pkg/codex/logutils
|
||||
## import pkg/storage/logutils
|
||||
##
|
||||
## type
|
||||
## BlockAddress* = object
|
||||
|
||||
@ -71,7 +71,7 @@ func mimetype*(self: Manifest): ?string =
|
||||
############################################################
|
||||
|
||||
func isManifest*(cid: Cid): ?!bool =
|
||||
success (ManifestCodec == ?cid.contentType().mapFailure(CodexError))
|
||||
success (ManifestCodec == ?cid.contentType().mapFailure(StorageError))
|
||||
|
||||
func isManifest*(mc: MultiCodec): ?!bool =
|
||||
success mc == ManifestCodec
|
||||
|
||||
@ -22,14 +22,14 @@ import pkg/merkletree
|
||||
import ../utils
|
||||
import ../rng
|
||||
import ../errors
|
||||
import ../codextypes
|
||||
import ../storagetypes
|
||||
|
||||
from ../utils/digest import digestBytes
|
||||
|
||||
export merkletree
|
||||
|
||||
logScope:
|
||||
topics = "codex merkletree"
|
||||
topics = "storage merkletree"
|
||||
|
||||
type
|
||||
ByteTreeKey* {.pure.} = enum
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
const CodecExts =
|
||||
[("codex-manifest", 0xCD01), ("codex-block", 0xCD02), ("codex-root", 0xCD03)]
|
||||
[("storage-manifest", 0xCD01), ("storage-block", 0xCD02), ("storage-root", 0xCD03)]
|
||||
|
||||
@ -9,17 +9,17 @@
|
||||
|
||||
const
|
||||
# Namespaces
|
||||
CodexMetaNamespace* = "meta" # meta info stored here
|
||||
CodexRepoNamespace* = "repo" # repository namespace, blocks and manifests are subkeys
|
||||
CodexBlockTotalNamespace* = CodexMetaNamespace & "/total"
|
||||
StorageMetaNamespace* = "meta" # meta info stored here
|
||||
StorageRepoNamespace* = "repo" # repository namespace, blocks and manifests are subkeys
|
||||
StorageBlockTotalNamespace* = StorageMetaNamespace & "/total"
|
||||
# number of blocks in the repo
|
||||
CodexBlocksNamespace* = CodexRepoNamespace & "/blocks" # blocks namespace
|
||||
CodexManifestNamespace* = CodexRepoNamespace & "/manifests" # manifest namespace
|
||||
CodexBlocksTtlNamespace* = # Cid TTL
|
||||
CodexMetaNamespace & "/ttl"
|
||||
CodexBlockProofNamespace* = # Cid and Proof
|
||||
CodexMetaNamespace & "/proof"
|
||||
CodexDhtNamespace* = "dht" # Dht namespace
|
||||
CodexDhtProvidersNamespace* = # Dht providers namespace
|
||||
CodexDhtNamespace & "/providers"
|
||||
CodexQuotaNamespace* = CodexMetaNamespace & "/quota" # quota's namespace
|
||||
StorageBlocksNamespace* = StorageRepoNamespace & "/blocks" # blocks namespace
|
||||
StorageManifestNamespace* = StorageRepoNamespace & "/manifests" # manifest namespace
|
||||
StorageBlocksTtlNamespace* = # Cid TTL
|
||||
StorageMetaNamespace & "/ttl"
|
||||
StorageBlockProofNamespace* = # Cid and Proof
|
||||
StorageMetaNamespace & "/proof"
|
||||
StorageDhtNamespace* = "dht" # Dht namespace
|
||||
StorageDhtProvidersNamespace* = # Dht providers namespace
|
||||
StorageDhtNamespace & "/providers"
|
||||
StorageQuotaNamespace* = StorageMetaNamespace & "/quota" # quota's namespace
|
||||
|
||||
@ -412,7 +412,7 @@ proc nattedAddress*(
|
||||
if ipPart.isSome and port.isSome:
|
||||
# Try to setup NAT mapping for the address
|
||||
let (newIP, tcp, udp) =
|
||||
setupAddress(natConfig, ipPart.get, port.get, udpPort, "codex")
|
||||
setupAddress(natConfig, ipPart.get, port.get, udpPort, "storage")
|
||||
if newIP.isSome:
|
||||
# NAT mapping successful - add discovery address with mapped UDP port
|
||||
discoveryAddrs.add(getMultiAddrWithIPAndUDPPort(newIP.get, udp.get))
|
||||
|
||||
@ -45,7 +45,7 @@ import ./utils/trackedfutures
|
||||
export logutils
|
||||
|
||||
logScope:
|
||||
topics = "codex node"
|
||||
topics = "storage node"
|
||||
|
||||
const
|
||||
DefaultFetchBatch = 1024
|
||||
@ -53,7 +53,7 @@ const
|
||||
BatchRefillThreshold = 0.75 # Refill when 75% of window completes
|
||||
|
||||
type
|
||||
CodexNode* = object
|
||||
StorageNode* = object
|
||||
switch: Switch
|
||||
networkId: PeerId
|
||||
networkStore: NetworkStore
|
||||
@ -63,27 +63,27 @@ type
|
||||
taskPool: Taskpool
|
||||
trackedFutures: TrackedFutures
|
||||
|
||||
CodexNodeRef* = ref CodexNode
|
||||
StorageNodeRef* = ref StorageNode
|
||||
|
||||
OnManifest* = proc(cid: Cid, manifest: Manifest): void {.gcsafe, raises: [].}
|
||||
BatchProc* =
|
||||
proc(blocks: seq[bt.Block]): Future[?!void] {.async: (raises: [CancelledError]).}
|
||||
OnBlockStoredProc = proc(chunk: seq[byte]): void {.gcsafe, raises: [].}
|
||||
|
||||
func switch*(self: CodexNodeRef): Switch =
|
||||
func switch*(self: StorageNodeRef): Switch =
|
||||
return self.switch
|
||||
|
||||
func blockStore*(self: CodexNodeRef): BlockStore =
|
||||
func blockStore*(self: StorageNodeRef): BlockStore =
|
||||
return self.networkStore
|
||||
|
||||
func engine*(self: CodexNodeRef): BlockExcEngine =
|
||||
func engine*(self: StorageNodeRef): BlockExcEngine =
|
||||
return self.engine
|
||||
|
||||
func discovery*(self: CodexNodeRef): Discovery =
|
||||
func discovery*(self: StorageNodeRef): Discovery =
|
||||
return self.discovery
|
||||
|
||||
proc storeManifest*(
|
||||
self: CodexNodeRef, manifest: Manifest
|
||||
self: StorageNodeRef, manifest: Manifest
|
||||
): Future[?!bt.Block] {.async.} =
|
||||
without encodedVerifiable =? manifest.encode(), err:
|
||||
trace "Unable to encode manifest"
|
||||
@ -100,7 +100,7 @@ proc storeManifest*(
|
||||
success blk
|
||||
|
||||
proc fetchManifest*(
|
||||
self: CodexNodeRef, cid: Cid
|
||||
self: StorageNodeRef, cid: Cid
|
||||
): Future[?!Manifest] {.async: (raises: [CancelledError]).} =
|
||||
## Fetch and decode a manifest block
|
||||
##
|
||||
@ -124,18 +124,18 @@ proc fetchManifest*(
|
||||
|
||||
return manifest.success
|
||||
|
||||
proc findPeer*(self: CodexNodeRef, peerId: PeerId): Future[?PeerRecord] {.async.} =
|
||||
## Find peer using the discovery service from the given CodexNode
|
||||
proc findPeer*(self: StorageNodeRef, peerId: PeerId): Future[?PeerRecord] {.async.} =
|
||||
## Find peer using the discovery service from the given StorageNode
|
||||
##
|
||||
return await self.discovery.findPeer(peerId)
|
||||
|
||||
proc connect*(
|
||||
self: CodexNodeRef, peerId: PeerId, addrs: seq[MultiAddress]
|
||||
self: StorageNodeRef, peerId: PeerId, addrs: seq[MultiAddress]
|
||||
): Future[void] =
|
||||
self.switch.connect(peerId, addrs)
|
||||
|
||||
proc updateExpiry*(
|
||||
self: CodexNodeRef, manifestCid: Cid, expiry: SecondsSince1970
|
||||
self: StorageNodeRef, manifestCid: Cid, expiry: SecondsSince1970
|
||||
): Future[?!void] {.async: (raises: [CancelledError]).} =
|
||||
without manifest =? await self.fetchManifest(manifestCid), error:
|
||||
trace "Unable to fetch manifest for cid", manifestCid
|
||||
@ -158,7 +158,7 @@ proc updateExpiry*(
|
||||
return success()
|
||||
|
||||
proc fetchBatched*(
|
||||
self: CodexNodeRef,
|
||||
self: StorageNodeRef,
|
||||
cid: Cid,
|
||||
iter: Iter[int],
|
||||
batchSize = DefaultFetchBatch,
|
||||
@ -233,7 +233,7 @@ proc fetchBatched*(
|
||||
success()
|
||||
|
||||
proc fetchBatched*(
|
||||
self: CodexNodeRef,
|
||||
self: StorageNodeRef,
|
||||
manifest: Manifest,
|
||||
batchSize = DefaultFetchBatch,
|
||||
onBatch: BatchProc = nil,
|
||||
@ -249,7 +249,7 @@ proc fetchBatched*(
|
||||
self.fetchBatched(manifest.treeCid, iter, batchSize, onBatch, fetchLocal)
|
||||
|
||||
proc fetchDatasetAsync*(
|
||||
self: CodexNodeRef, manifest: Manifest, fetchLocal = true
|
||||
self: StorageNodeRef, manifest: Manifest, fetchLocal = true
|
||||
): Future[void] {.async: (raises: []).} =
|
||||
## Asynchronously fetch a dataset in the background.
|
||||
## This task will be tracked and cleaned up on node shutdown.
|
||||
@ -264,14 +264,14 @@ proc fetchDatasetAsync*(
|
||||
except CancelledError as exc:
|
||||
trace "Cancelled fetching blocks", exc = exc.msg
|
||||
|
||||
proc fetchDatasetAsyncTask*(self: CodexNodeRef, manifest: Manifest) =
|
||||
proc fetchDatasetAsyncTask*(self: StorageNodeRef, manifest: Manifest) =
|
||||
## Start fetching a dataset in the background.
|
||||
## The task will be tracked and cleaned up on node shutdown.
|
||||
##
|
||||
self.trackedFutures.track(self.fetchDatasetAsync(manifest, fetchLocal = false))
|
||||
|
||||
proc streamSingleBlock(
|
||||
self: CodexNodeRef, cid: Cid
|
||||
self: StorageNodeRef, cid: Cid
|
||||
): Future[?!LPStream] {.async: (raises: [CancelledError]).} =
|
||||
## Streams the contents of a single block.
|
||||
##
|
||||
@ -296,7 +296,7 @@ proc streamSingleBlock(
|
||||
LPStream(stream).success
|
||||
|
||||
proc streamEntireDataset(
|
||||
self: CodexNodeRef, manifest: Manifest, manifestCid: Cid
|
||||
self: StorageNodeRef, manifest: Manifest, manifestCid: Cid
|
||||
): Future[?!LPStream] {.async: (raises: [CancelledError]).} =
|
||||
## Streams the contents of the entire dataset described by the manifest.
|
||||
##
|
||||
@ -324,7 +324,7 @@ proc streamEntireDataset(
|
||||
stream.success
|
||||
|
||||
proc retrieve*(
|
||||
self: CodexNodeRef, cid: Cid, local: bool = true
|
||||
self: StorageNodeRef, cid: Cid, local: bool = true
|
||||
): Future[?!LPStream] {.async: (raises: [CancelledError]).} =
|
||||
## Retrieve by Cid a single block or an entire dataset described by manifest
|
||||
##
|
||||
@ -340,7 +340,7 @@ proc retrieve*(
|
||||
|
||||
await self.streamEntireDataset(manifest, cid)
|
||||
|
||||
proc deleteSingleBlock(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} =
|
||||
proc deleteSingleBlock(self: StorageNodeRef, cid: Cid): Future[?!void] {.async.} =
|
||||
if err =? (await self.networkStore.delBlock(cid)).errorOption:
|
||||
error "Error deleting block", cid, err = err.msg
|
||||
return failure(err)
|
||||
@ -348,7 +348,7 @@ proc deleteSingleBlock(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} =
|
||||
trace "Deleted block", cid
|
||||
return success()
|
||||
|
||||
proc deleteEntireDataset(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} =
|
||||
proc deleteEntireDataset(self: StorageNodeRef, cid: Cid): Future[?!void] {.async.} =
|
||||
# Deletion is a strictly local operation
|
||||
var store = self.networkStore.localStore
|
||||
|
||||
@ -382,7 +382,7 @@ proc deleteEntireDataset(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.}
|
||||
success()
|
||||
|
||||
proc delete*(
|
||||
self: CodexNodeRef, cid: Cid
|
||||
self: StorageNodeRef, cid: Cid
|
||||
): Future[?!void] {.async: (raises: [CatchableError]).} =
|
||||
## Deletes a whole dataset, if Cid is a Manifest Cid, or a single block, if Cid a block Cid,
|
||||
## from the underlying block store. This is a strictly local operation.
|
||||
@ -400,7 +400,7 @@ proc delete*(
|
||||
await self.deleteEntireDataset(cid)
|
||||
|
||||
proc store*(
|
||||
self: CodexNodeRef,
|
||||
self: StorageNodeRef,
|
||||
stream: LPStream,
|
||||
filename: ?string = string.none,
|
||||
mimetype: ?string = string.none,
|
||||
@ -484,7 +484,7 @@ proc store*(
|
||||
|
||||
return manifestBlk.cid.success
|
||||
|
||||
proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} =
|
||||
proc iterateManifests*(self: StorageNodeRef, onManifest: OnManifest) {.async.} =
|
||||
without cidsIter =? await self.networkStore.listBlocks(BlockType.Manifest):
|
||||
warn "Failed to listBlocks"
|
||||
return
|
||||
@ -502,11 +502,11 @@ proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} =
|
||||
onManifest(cid, manifest)
|
||||
|
||||
proc onExpiryUpdate(
|
||||
self: CodexNodeRef, rootCid: Cid, expiry: SecondsSince1970
|
||||
self: StorageNodeRef, rootCid: Cid, expiry: SecondsSince1970
|
||||
): Future[?!void] {.async: (raises: [CancelledError]).} =
|
||||
return await self.updateExpiry(rootCid, expiry)
|
||||
|
||||
proc start*(self: CodexNodeRef) {.async.} =
|
||||
proc start*(self: StorageNodeRef) {.async.} =
|
||||
if not self.engine.isNil:
|
||||
await self.engine.start()
|
||||
|
||||
@ -519,7 +519,7 @@ proc start*(self: CodexNodeRef) {.async.} =
|
||||
self.networkId = self.switch.peerInfo.peerId
|
||||
notice "Started Storage node", id = self.networkId, addrs = self.switch.peerInfo.addrs
|
||||
|
||||
proc stop*(self: CodexNodeRef) {.async.} =
|
||||
proc stop*(self: StorageNodeRef) {.async.} =
|
||||
trace "Stopping node"
|
||||
|
||||
await self.trackedFutures.cancelTracked()
|
||||
@ -533,22 +533,22 @@ proc stop*(self: CodexNodeRef) {.async.} =
|
||||
if not self.clock.isNil:
|
||||
await self.clock.stop()
|
||||
|
||||
proc close*(self: CodexNodeRef) {.async.} =
|
||||
proc close*(self: StorageNodeRef) {.async.} =
|
||||
if not self.networkStore.isNil:
|
||||
await self.networkStore.close
|
||||
|
||||
proc new*(
|
||||
T: type CodexNodeRef,
|
||||
T: type StorageNodeRef,
|
||||
switch: Switch,
|
||||
networkStore: NetworkStore,
|
||||
engine: BlockExcEngine,
|
||||
discovery: Discovery,
|
||||
taskpool: Taskpool,
|
||||
): CodexNodeRef =
|
||||
## Create new instance of a Codex self, call `start` to run it
|
||||
): StorageNodeRef =
|
||||
## Create new instance of a Storage self, call `start` to run it
|
||||
##
|
||||
|
||||
CodexNodeRef(
|
||||
StorageNodeRef(
|
||||
switch: switch,
|
||||
networkStore: networkStore,
|
||||
engine: engine,
|
||||
@ -558,7 +558,7 @@ proc new*(
|
||||
)
|
||||
|
||||
proc hasLocalBlock*(
|
||||
self: CodexNodeRef, cid: Cid
|
||||
self: StorageNodeRef, cid: Cid
|
||||
): Future[bool] {.async: (raises: [CancelledError]).} =
|
||||
## Returns true if the given Cid is present in the local store
|
||||
|
||||
|
||||
@ -39,10 +39,10 @@ import ./coders
|
||||
import ./json
|
||||
|
||||
logScope:
|
||||
topics = "codex restapi"
|
||||
topics = "storage restapi"
|
||||
|
||||
declareCounter(codex_api_uploads, "codex API uploads")
|
||||
declareCounter(codex_api_downloads, "codex API downloads")
|
||||
declareCounter(storage_api_uploads, "storage API uploads")
|
||||
declareCounter(storage_api_downloads, "storage API downloads")
|
||||
|
||||
proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].} =
|
||||
0
|
||||
@ -50,7 +50,7 @@ proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].}
|
||||
proc formatManifest(cid: Cid, manifest: Manifest): RestContent =
|
||||
return RestContent.init(cid, manifest)
|
||||
|
||||
proc formatManifestBlocks(node: CodexNodeRef): Future[JsonNode] {.async.} =
|
||||
proc formatManifestBlocks(node: StorageNodeRef): Future[JsonNode] {.async.} =
|
||||
var content: seq[RestContent]
|
||||
|
||||
proc addManifest(cid: Cid, manifest: Manifest) =
|
||||
@ -67,7 +67,7 @@ proc isPending(resp: HttpResponseRef): bool =
|
||||
return resp.getResponseState() == HttpResponseState.Empty
|
||||
|
||||
proc retrieveCid(
|
||||
node: CodexNodeRef, cid: Cid, local: bool = true, resp: HttpResponseRef
|
||||
node: StorageNodeRef, cid: Cid, local: bool = true, resp: HttpResponseRef
|
||||
): Future[void] {.async: (raises: [CancelledError, HttpWriteError]).} =
|
||||
## Download a file from the node in a streaming
|
||||
## manner
|
||||
@ -131,7 +131,7 @@ proc retrieveCid(
|
||||
|
||||
await resp.send(addr buff[0], buff.len)
|
||||
await resp.finish()
|
||||
codex_api_downloads.inc()
|
||||
storage_api_downloads.inc()
|
||||
except CancelledError as exc:
|
||||
raise exc
|
||||
except LPStreamError as exc:
|
||||
@ -173,7 +173,7 @@ proc getFilenameFromContentDisposition(contentDisposition: string): ?string =
|
||||
let filename = parts[1].strip()
|
||||
return filename[0 ..^ 2].some
|
||||
|
||||
proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) =
|
||||
proc initDataApi(node: StorageNodeRef, repoStore: RepoStore, router: var RestRouter) =
|
||||
let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion
|
||||
|
||||
router.api(MethodOptions, "/api/storage/v1/data") do(
|
||||
@ -238,7 +238,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
|
||||
error "Error uploading file", exc = error.msg
|
||||
return RestApiResponse.error(Http500, error.msg)
|
||||
|
||||
codex_api_uploads.inc()
|
||||
storage_api_uploads.inc()
|
||||
trace "Uploaded file", cid
|
||||
return RestApiResponse.response($cid)
|
||||
except CancelledError:
|
||||
@ -383,7 +383,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
|
||||
)
|
||||
return RestApiResponse.response($json, contentType = "application/json")
|
||||
|
||||
proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
|
||||
proc initNodeApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter) =
|
||||
let allowedOrigin = router.allowedOrigin
|
||||
|
||||
## various node management api's
|
||||
@ -465,7 +465,7 @@ proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
|
||||
return
|
||||
RestApiResponse.error(Http500, "Unknown error dialling peer", headers = headers)
|
||||
|
||||
proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
|
||||
proc initDebugApi(node: StorageNodeRef, conf: StorageConf, router: var RestRouter) =
|
||||
let allowedOrigin = router.allowedOrigin
|
||||
|
||||
router.api(MethodGet, "/api/storage/v1/debug/info") do() -> RestApiResponse:
|
||||
@ -476,16 +476,20 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
|
||||
try:
|
||||
let table = RestRoutingTable.init(node.discovery.protocol.routingTable)
|
||||
|
||||
let json = %*{
|
||||
"id": $node.switch.peerInfo.peerId,
|
||||
"addrs": node.switch.peerInfo.addrs.mapIt($it),
|
||||
"repo": $conf.dataDir,
|
||||
"spr":
|
||||
if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: "",
|
||||
"announceAddresses": node.discovery.announceAddrs,
|
||||
"table": table,
|
||||
"storage": {"version": $codexVersion, "revision": $codexRevision},
|
||||
}
|
||||
let json =
|
||||
%*{
|
||||
"id": $node.switch.peerInfo.peerId,
|
||||
"addrs": node.switch.peerInfo.addrs.mapIt($it),
|
||||
"repo": $conf.dataDir,
|
||||
"spr":
|
||||
if node.discovery.dhtRecord.isSome:
|
||||
node.discovery.dhtRecord.get.toURI
|
||||
else:
|
||||
"",
|
||||
"announceAddresses": node.discovery.announceAddrs,
|
||||
"table": table,
|
||||
"storage": {"version": $storageVersion, "revision": $storageRevision},
|
||||
}
|
||||
|
||||
# return pretty json for human readability
|
||||
return RestApiResponse.response(
|
||||
@ -542,8 +546,8 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
|
||||
return RestApiResponse.error(Http500, headers = headers)
|
||||
|
||||
proc initRestApi*(
|
||||
node: CodexNodeRef,
|
||||
conf: CodexConf,
|
||||
node: StorageNodeRef,
|
||||
conf: StorageConf,
|
||||
repoStore: RepoStore,
|
||||
corsAllowedOrigin: ?string,
|
||||
): RestRouter =
|
||||
|
||||
@ -35,35 +35,35 @@ import ./discovery
|
||||
import ./systemclock
|
||||
import ./utils/addrutils
|
||||
import ./namespaces
|
||||
import ./codextypes
|
||||
import ./storagetypes
|
||||
import ./logutils
|
||||
import ./nat
|
||||
|
||||
logScope:
|
||||
topics = "codex node"
|
||||
topics = "storage node"
|
||||
|
||||
type
|
||||
CodexServer* = ref object
|
||||
config: CodexConf
|
||||
StorageServer* = ref object
|
||||
config: StorageConf
|
||||
restServer: RestServerRef
|
||||
codexNode: CodexNodeRef
|
||||
storageNode: StorageNodeRef
|
||||
repoStore: RepoStore
|
||||
maintenance: BlockMaintainer
|
||||
taskpool: Taskpool
|
||||
isStarted: bool
|
||||
|
||||
CodexPrivateKey* = libp2p.PrivateKey # alias
|
||||
StoragePrivateKey* = libp2p.PrivateKey # alias
|
||||
|
||||
func config*(self: CodexServer): CodexConf =
|
||||
func config*(self: StorageServer): StorageConf =
|
||||
return self.config
|
||||
|
||||
func node*(self: CodexServer): CodexNodeRef =
|
||||
return self.codexNode
|
||||
func node*(self: StorageServer): StorageNodeRef =
|
||||
return self.storageNode
|
||||
|
||||
func repoStore*(self: CodexServer): RepoStore =
|
||||
func repoStore*(self: StorageServer): RepoStore =
|
||||
return self.repoStore
|
||||
|
||||
proc start*(s: CodexServer) {.async.} =
|
||||
proc start*(s: StorageServer) {.async.} =
|
||||
if s.isStarted:
|
||||
warn "Storage server already started, skipping"
|
||||
return
|
||||
@ -73,35 +73,36 @@ proc start*(s: CodexServer) {.async.} =
|
||||
|
||||
s.maintenance.start()
|
||||
|
||||
await s.codexNode.switch.start()
|
||||
await s.storageNode.switch.start()
|
||||
|
||||
let (announceAddrs, discoveryAddrs) = nattedAddress(
|
||||
s.config.nat, s.codexNode.switch.peerInfo.addrs, s.config.discoveryPort
|
||||
s.config.nat, s.storageNode.switch.peerInfo.addrs, s.config.discoveryPort
|
||||
)
|
||||
|
||||
s.codexNode.discovery.updateAnnounceRecord(announceAddrs)
|
||||
s.codexNode.discovery.updateDhtRecord(discoveryAddrs)
|
||||
s.storageNode.discovery.updateAnnounceRecord(announceAddrs)
|
||||
s.storageNode.discovery.updateDhtRecord(discoveryAddrs)
|
||||
|
||||
await s.codexNode.start()
|
||||
await s.storageNode.start()
|
||||
|
||||
if s.restServer != nil:
|
||||
s.restServer.start()
|
||||
|
||||
s.isStarted = true
|
||||
|
||||
proc stop*(s: CodexServer) {.async.} =
|
||||
proc stop*(s: StorageServer) {.async.} =
|
||||
if not s.isStarted:
|
||||
warn "Storage is not started"
|
||||
return
|
||||
|
||||
notice "Stopping Storage node"
|
||||
|
||||
var futures = @[
|
||||
s.codexNode.switch.stop(),
|
||||
s.codexNode.stop(),
|
||||
s.repoStore.stop(),
|
||||
s.maintenance.stop(),
|
||||
]
|
||||
var futures =
|
||||
@[
|
||||
s.storageNode.switch.stop(),
|
||||
s.storageNode.stop(),
|
||||
s.repoStore.stop(),
|
||||
s.maintenance.stop(),
|
||||
]
|
||||
|
||||
if s.restServer != nil:
|
||||
futures.add(s.restServer.stop())
|
||||
@ -114,9 +115,9 @@ proc stop*(s: CodexServer) {.async.} =
|
||||
error "Failed to stop Storage node", failures = res.failure.len
|
||||
raiseAssert "Failed to stop Storage node"
|
||||
|
||||
proc close*(s: CodexServer) {.async.} =
|
||||
proc close*(s: StorageServer) {.async.} =
|
||||
var futures =
|
||||
@[s.codexNode.close(), s.repoStore.close(), s.codexNode.discovery.close()]
|
||||
@[s.storageNode.close(), s.repoStore.close(), s.storageNode.discovery.close()]
|
||||
|
||||
let res = await noCancel allFinishedFailed[void](futures)
|
||||
|
||||
@ -131,14 +132,14 @@ proc close*(s: CodexServer) {.async.} =
|
||||
error "Failed to close Storage node", failures = res.failure.len
|
||||
raiseAssert "Failed to close Storage node"
|
||||
|
||||
proc shutdown*(server: CodexServer) {.async.} =
|
||||
proc shutdown*(server: StorageServer) {.async.} =
|
||||
await server.stop()
|
||||
await server.close()
|
||||
|
||||
proc new*(
|
||||
T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey
|
||||
): CodexServer =
|
||||
## create CodexServer including setting up datastore, repostore, etc
|
||||
T: type StorageServer, config: StorageConf, privateKey: StoragePrivateKey
|
||||
): StorageServer =
|
||||
## create StorageServer including setting up datastore, repostore, etc
|
||||
let switch = SwitchBuilder
|
||||
.new()
|
||||
.withPrivateKey(privateKey)
|
||||
@ -169,7 +170,7 @@ proc new*(
|
||||
cache = CacheStore.new(cacheSize = config.cacheSize)
|
||||
## Is unused?
|
||||
|
||||
let discoveryDir = config.dataDir / CodexDhtNamespace
|
||||
let discoveryDir = config.dataDir / StorageDhtNamespace
|
||||
|
||||
if io2.createPath(discoveryDir).isErr:
|
||||
trace "Unable to create discovery directory for block store",
|
||||
@ -178,7 +179,7 @@ proc new*(
|
||||
msg: "Unable to create discovery directory for block store: " & discoveryDir
|
||||
)
|
||||
|
||||
let providersPath = config.dataDir / CodexDhtProvidersNamespace
|
||||
let providersPath = config.dataDir / StorageDhtProvidersNamespace
|
||||
let discoveryStoreRes = LevelDbDatastore.new(providersPath)
|
||||
if discoveryStoreRes.isErr:
|
||||
error "Failed to initialize discovery datastore",
|
||||
@ -221,7 +222,7 @@ proc new*(
|
||||
|
||||
repoStore = RepoStore.new(
|
||||
repoDs = repoData,
|
||||
metaDs = LevelDbDatastore.new(config.dataDir / CodexMetaNamespace).expect(
|
||||
metaDs = LevelDbDatastore.new(config.dataDir / StorageMetaNamespace).expect(
|
||||
"Should create metadata store!"
|
||||
),
|
||||
quotaMaxBytes = config.storageQuota,
|
||||
@ -244,7 +245,7 @@ proc new*(
|
||||
)
|
||||
store = NetworkStore.new(engine, repoStore)
|
||||
|
||||
codexNode = CodexNodeRef.new(
|
||||
storageNode = StorageNodeRef.new(
|
||||
switch = switch,
|
||||
networkStore = store,
|
||||
engine = engine,
|
||||
@ -257,7 +258,7 @@ proc new*(
|
||||
if config.apiBindAddress.isSome:
|
||||
restServer = RestServerRef
|
||||
.new(
|
||||
codexNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin),
|
||||
storageNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin),
|
||||
initTAddress(config.apiBindAddress.get(), config.apiPort),
|
||||
bufferSize = (1024 * 64),
|
||||
maxRequestBodySize = int.high,
|
||||
@ -266,9 +267,9 @@ proc new*(
|
||||
|
||||
switch.mount(network)
|
||||
|
||||
CodexServer(
|
||||
StorageServer(
|
||||
config: config,
|
||||
codexNode: codexNode,
|
||||
storageNode: storageNode,
|
||||
restServer: restServer,
|
||||
repoStore: repoStore,
|
||||
maintenance: maintenance,
|
||||
|
||||
@ -30,11 +30,11 @@ const
|
||||
# hashes
|
||||
Sha256HashCodec* = multiCodec("sha2-256")
|
||||
|
||||
ManifestCodec* = multiCodec("codex-manifest")
|
||||
DatasetRootCodec* = multiCodec("codex-root")
|
||||
BlockCodec* = multiCodec("codex-block")
|
||||
ManifestCodec* = multiCodec("storage-manifest")
|
||||
DatasetRootCodec* = multiCodec("storage-root")
|
||||
BlockCodec* = multiCodec("storage-block")
|
||||
|
||||
CodexPrimitivesCodecs* = [ManifestCodec, DatasetRootCodec, BlockCodec]
|
||||
StoragePrimitivesCodecs* = [ManifestCodec, DatasetRootCodec, BlockCodec]
|
||||
|
||||
proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] =
|
||||
## Initialize padding blocks table
|
||||
@ -22,7 +22,7 @@ import ../utils
|
||||
export blocktype
|
||||
|
||||
type
|
||||
BlockNotFoundError* = object of CodexError
|
||||
BlockNotFoundError* = object of StorageError
|
||||
|
||||
BlockType* {.pure.} = enum
|
||||
Manifest
|
||||
|
||||
@ -30,7 +30,7 @@ import ../clock
|
||||
export blockstore
|
||||
|
||||
logScope:
|
||||
topics = "codex cachestore"
|
||||
topics = "storage cachestore"
|
||||
|
||||
type
|
||||
CacheStore* = ref object of BlockStore
|
||||
@ -40,7 +40,7 @@ type
|
||||
cidAndProofCache: LruCache[(Cid, Natural), (Cid, StorageMerkleProof)]
|
||||
cidAndProofCache: LruCache[(Cid, Natural), (Cid, StorageMerkleProof)]
|
||||
|
||||
InvalidBlockSize* = object of CodexError
|
||||
InvalidBlockSize* = object of StorageError
|
||||
|
||||
const DefaultCacheSize*: NBytes = 5.MiBs
|
||||
|
||||
|
||||
@ -17,14 +17,14 @@ import ../namespaces
|
||||
import ../manifest
|
||||
|
||||
const
|
||||
CodexMetaKey* = Key.init(CodexMetaNamespace).tryGet
|
||||
CodexRepoKey* = Key.init(CodexRepoNamespace).tryGet
|
||||
CodexBlocksKey* = Key.init(CodexBlocksNamespace).tryGet
|
||||
CodexTotalBlocksKey* = Key.init(CodexBlockTotalNamespace).tryGet
|
||||
CodexManifestKey* = Key.init(CodexManifestNamespace).tryGet
|
||||
BlocksTtlKey* = Key.init(CodexBlocksTtlNamespace).tryGet
|
||||
BlockProofKey* = Key.init(CodexBlockProofNamespace).tryGet
|
||||
QuotaKey* = Key.init(CodexQuotaNamespace).tryGet
|
||||
StorageMetaKey* = Key.init(StorageMetaNamespace).tryGet
|
||||
StorageRepoKey* = Key.init(StorageRepoNamespace).tryGet
|
||||
StorageBlocksKey* = Key.init(StorageBlocksNamespace).tryGet
|
||||
StorageTotalBlocksKey* = Key.init(StorageBlockTotalNamespace).tryGet
|
||||
StorageManifestKey* = Key.init(StorageManifestNamespace).tryGet
|
||||
BlocksTtlKey* = Key.init(StorageBlocksTtlNamespace).tryGet
|
||||
BlockProofKey* = Key.init(StorageBlockProofNamespace).tryGet
|
||||
QuotaKey* = Key.init(StorageQuotaNamespace).tryGet
|
||||
QuotaUsedKey* = (QuotaKey / "used").tryGet
|
||||
QuotaReservedKey* = (QuotaKey / "reserved").tryGet
|
||||
|
||||
@ -32,9 +32,9 @@ func makePrefixKey*(postFixLen: int, cid: Cid): ?!Key =
|
||||
let cidKey = ?Key.init(($cid)[^postFixLen ..^ 1] & "/" & $cid)
|
||||
|
||||
if ?cid.isManifest:
|
||||
success CodexManifestKey / cidKey
|
||||
success StorageManifestKey / cidKey
|
||||
else:
|
||||
success CodexBlocksKey / cidKey
|
||||
success StorageBlocksKey / cidKey
|
||||
|
||||
proc createBlockExpirationMetadataKey*(cid: Cid): ?!Key =
|
||||
BlocksTtlKey / $cid
|
||||
|
||||
@ -24,7 +24,7 @@ import ../logutils
|
||||
import ../systemclock
|
||||
|
||||
logScope:
|
||||
topics = "codex maintenance"
|
||||
topics = "storage maintenance"
|
||||
|
||||
const
|
||||
DefaultBlockInterval* = 10.minutes
|
||||
|
||||
@ -25,7 +25,7 @@ import ./blockstore
|
||||
export blockstore, blockexchange, asyncheapqueue
|
||||
|
||||
logScope:
|
||||
topics = "codex networkstore"
|
||||
topics = "storage networkstore"
|
||||
|
||||
type NetworkStore* = ref object of BlockStore
|
||||
engine*: BlockExcEngine # blockexc decision engine
|
||||
|
||||
@ -26,11 +26,11 @@ import ../../logutils
|
||||
import ../../merkletree
|
||||
|
||||
logScope:
|
||||
topics = "codex repostore"
|
||||
topics = "storage repostore"
|
||||
|
||||
declareGauge(codex_repostore_blocks, "codex repostore blocks")
|
||||
declareGauge(codex_repostore_bytes_used, "codex repostore bytes used")
|
||||
declareGauge(codex_repostore_bytes_reserved, "codex repostore bytes reserved")
|
||||
declareGauge(storage_repostore_blocks, "storage repostore blocks")
|
||||
declareGauge(storage_repostore_bytes_used, "storage repostore bytes used")
|
||||
declareGauge(storage_repostore_bytes_reserved, "storage repostore bytes reserved")
|
||||
|
||||
proc putLeafMetadata*(
|
||||
self: RepoStore,
|
||||
@ -90,7 +90,7 @@ proc updateTotalBlocksCount*(
|
||||
self: RepoStore, plusCount: Natural = 0, minusCount: Natural = 0
|
||||
): Future[?!void] {.async: (raises: [CancelledError]).} =
|
||||
await self.metaDs.modify(
|
||||
CodexTotalBlocksKey,
|
||||
StorageTotalBlocksKey,
|
||||
proc(maybeCurrCount: ?Natural): Future[?Natural] {.async.} =
|
||||
let count: Natural =
|
||||
if currCount =? maybeCurrCount:
|
||||
@ -99,7 +99,7 @@ proc updateTotalBlocksCount*(
|
||||
plusCount - minusCount
|
||||
|
||||
self.totalBlocks = count
|
||||
codex_repostore_blocks.set(count.int64)
|
||||
storage_repostore_blocks.set(count.int64)
|
||||
count.some,
|
||||
)
|
||||
|
||||
@ -132,8 +132,8 @@ proc updateQuotaUsage*(
|
||||
)
|
||||
else:
|
||||
self.quotaUsage = usage
|
||||
codex_repostore_bytes_used.set(usage.used.int64)
|
||||
codex_repostore_bytes_reserved.set(usage.reserved.int64)
|
||||
storage_repostore_bytes_used.set(usage.used.int64)
|
||||
storage_repostore_bytes_reserved.set(usage.reserved.int64)
|
||||
return usage.some,
|
||||
)
|
||||
|
||||
|
||||
@ -32,7 +32,7 @@ import ../../utils
|
||||
export blocktype, cid
|
||||
|
||||
logScope:
|
||||
topics = "codex repostore"
|
||||
topics = "storage repostore"
|
||||
|
||||
###########################################################
|
||||
# BlockStore API
|
||||
@ -323,9 +323,9 @@ method listBlocks*(
|
||||
|
||||
let key =
|
||||
case blockType
|
||||
of BlockType.Manifest: CodexManifestKey
|
||||
of BlockType.Block: CodexBlocksKey
|
||||
of BlockType.Both: CodexRepoKey
|
||||
of BlockType.Manifest: StorageManifestKey
|
||||
of BlockType.Block: StorageBlocksKey
|
||||
of BlockType.Both: StorageRepoKey
|
||||
|
||||
let query = Query.init(key, value = false)
|
||||
without queryIter =? (await self.repoDs.query(query)), err:
|
||||
@ -444,7 +444,7 @@ proc release*(
|
||||
|
||||
proc start*(
|
||||
self: RepoStore
|
||||
): Future[void] {.async: (raises: [CancelledError, CodexError]).} =
|
||||
): Future[void] {.async: (raises: [CancelledError, StorageError]).} =
|
||||
## Start repo
|
||||
##
|
||||
if self.started:
|
||||
@ -453,10 +453,10 @@ proc start*(
|
||||
|
||||
trace "Starting rep"
|
||||
if err =? (await self.updateTotalBlocksCount()).errorOption:
|
||||
raise newException(CodexError, err.msg)
|
||||
raise newException(StorageError, err.msg)
|
||||
|
||||
if err =? (await self.updateQuotaUsage()).errorOption:
|
||||
raise newException(CodexError, err.msg)
|
||||
raise newException(StorageError, err.msg)
|
||||
|
||||
self.started = true
|
||||
|
||||
|
||||
@ -25,7 +25,7 @@ const
|
||||
DefaultQuotaBytes* = 20.GiBs
|
||||
|
||||
type
|
||||
QuotaNotEnoughError* = object of CodexError
|
||||
QuotaNotEnoughError* = object of StorageError
|
||||
|
||||
RepoStore* = ref object of BlockStore
|
||||
postFixLen*: int
|
||||
|
||||
@ -15,7 +15,7 @@ import ../logutils
|
||||
export lpstream, chronos, logutils
|
||||
|
||||
logScope:
|
||||
topics = "codex seekablestream"
|
||||
topics = "storage seekablestream"
|
||||
|
||||
type SeekableStream* = ref object of LPStream
|
||||
offset*: int
|
||||
|
||||
@ -25,7 +25,7 @@ import ./seekablestream
|
||||
export stores, blocktype, manifest, chronos
|
||||
|
||||
logScope:
|
||||
topics = "codex storestream"
|
||||
topics = "storage storestream"
|
||||
|
||||
const StoreStreamTrackerName* = "StoreStream"
|
||||
|
||||
|
||||
@ -20,25 +20,25 @@ import ../rng
|
||||
export crypto
|
||||
|
||||
type
|
||||
CodexKeyError = object of CodexError
|
||||
CodexKeyUnsafeError = object of CodexKeyError
|
||||
StorageKeyError = object of StorageError
|
||||
StorageKeyUnsafeError = object of StorageKeyError
|
||||
|
||||
proc setupKey*(path: string): ?!PrivateKey =
|
||||
if not path.fileAccessible({AccessFlags.Find}):
|
||||
info "Creating a private key and saving it"
|
||||
let
|
||||
res = ?PrivateKey.random(Rng.instance()[]).mapFailure(CodexKeyError)
|
||||
bytes = ?res.getBytes().mapFailure(CodexKeyError)
|
||||
res = ?PrivateKey.random(Rng.instance()[]).mapFailure(StorageKeyError)
|
||||
bytes = ?res.getBytes().mapFailure(StorageKeyError)
|
||||
|
||||
?path.secureWriteFile(bytes).mapFailure(CodexKeyError)
|
||||
return PrivateKey.init(bytes).mapFailure(CodexKeyError)
|
||||
?path.secureWriteFile(bytes).mapFailure(StorageKeyError)
|
||||
return PrivateKey.init(bytes).mapFailure(StorageKeyError)
|
||||
|
||||
info "Found a network private key"
|
||||
if not ?checkSecureFile(path).mapFailure(CodexKeyError):
|
||||
if not ?checkSecureFile(path).mapFailure(StorageKeyError):
|
||||
warn "The network private key file is not safe, aborting"
|
||||
return failure newException(
|
||||
CodexKeyUnsafeError, "The network private key file is not safe"
|
||||
StorageKeyUnsafeError, "The network private key file is not safe"
|
||||
)
|
||||
|
||||
let kb = ?path.readAllBytes().mapFailure(CodexKeyError)
|
||||
return PrivateKey.init(kb).mapFailure(CodexKeyError)
|
||||
let kb = ?path.readAllBytes().mapFailure(StorageKeyError)
|
||||
return PrivateKey.init(kb).mapFailure(StorageKeyError)
|
||||
|
||||
@ -1 +1 @@
|
||||
include ./testCodex
|
||||
include ./testStorage
|
||||
|
||||
@ -3,15 +3,15 @@ import std/sequtils
|
||||
import std/times
|
||||
import std/typetraits
|
||||
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/units
|
||||
import pkg/storage/rng
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/units
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/stew/byteutils
|
||||
import pkg/stint
|
||||
|
||||
import ./codex/helpers/randomchunker
|
||||
import ./storage/helpers/randomchunker
|
||||
|
||||
export randomchunker
|
||||
export units
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import pkg/codex/streams/storestream
|
||||
import pkg/storage/streams/storestream
|
||||
import pkg/unittest2
|
||||
|
||||
# From lip2p/tests/helpers
|
||||
|
||||
@ -9,7 +9,7 @@ multinodesuite "Node block expiration tests":
|
||||
|
||||
test "node retains not-expired file",
|
||||
NodeConfigs(
|
||||
clients: CodexConfigs
|
||||
clients: StorageConfigs
|
||||
.init(nodes = 1)
|
||||
.withBlockTtl(0, 10)
|
||||
.withBlockMaintenanceInterval(0, 1).some
|
||||
@ -29,7 +29,7 @@ multinodesuite "Node block expiration tests":
|
||||
|
||||
test "node deletes expired file",
|
||||
NodeConfigs(
|
||||
clients: CodexConfigs
|
||||
clients: StorageConfigs
|
||||
.init(nodes = 1)
|
||||
.withBlockTtl(0, 1)
|
||||
.withBlockMaintenanceInterval(0, 1).some
|
||||
|
||||
@ -3,12 +3,12 @@ import std/net
|
||||
import std/sequtils
|
||||
import std/strformat
|
||||
from pkg/libp2p import `==`, `$`, Cid
|
||||
import pkg/codex/units
|
||||
import pkg/codex/manifest
|
||||
import pkg/storage/units
|
||||
import pkg/storage/manifest
|
||||
import ../twonodes
|
||||
import ../../examples
|
||||
import ../../codex/examples
|
||||
import ../../codex/slots/helpers
|
||||
import ../../storage/examples
|
||||
import ../../storage/slots/helpers
|
||||
import json
|
||||
|
||||
twonodessuite "REST API":
|
||||
@ -16,7 +16,7 @@ twonodessuite "REST API":
|
||||
check !(await client1.info()) != !(await client2.info())
|
||||
|
||||
test "nodes can set chronicles log level", twoNodesConfig:
|
||||
await client1.setLogLevel("DEBUG;TRACE:codex")
|
||||
await client1.setLogLevel("DEBUG;TRACE:storage")
|
||||
|
||||
test "node accepts file uploads", twoNodesConfig:
|
||||
let cid1 = (await client1.upload("some file contents")).get
|
||||
|
||||
@ -1,18 +1,18 @@
|
||||
import std/times
|
||||
import pkg/codex/conf
|
||||
import pkg/storage/conf
|
||||
import pkg/stint
|
||||
from pkg/libp2p import Cid, `$`
|
||||
import ../../asynctest
|
||||
import ../../checktest
|
||||
import ../../examples
|
||||
import ../../codex/examples
|
||||
import ../codexconfig
|
||||
import ../codexclient
|
||||
import ../../storage/examples
|
||||
import ../storageconfig
|
||||
import ../storageclient
|
||||
import ../multinodes
|
||||
|
||||
multinodesuite "Rest API validation":
|
||||
let config = NodeConfigs(clients: CodexConfigs.init(nodes = 1).some)
|
||||
var client: CodexClient
|
||||
let config = NodeConfigs(clients: StorageConfigs.init(nodes = 1).some)
|
||||
var client: StorageClient
|
||||
|
||||
setup:
|
||||
client = clients()[0].client
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import pkg/codex/rest/json
|
||||
import pkg/storage/rest/json
|
||||
import ../twonodes
|
||||
import ../../codex/examples
|
||||
import ../../storage/examples
|
||||
import json
|
||||
from pkg/libp2p import Cid, `$`
|
||||
|
||||
@ -84,7 +84,7 @@ twonodessuite "Uploads and downloads":
|
||||
content1 == resp2
|
||||
|
||||
test "reliable transfer test", twoNodesConfig:
|
||||
proc transferTest(a: CodexClient, b: CodexClient) {.async.} =
|
||||
proc transferTest(a: StorageClient, b: StorageClient) {.async.} =
|
||||
let data = await RandomChunker.example(blocks = 8)
|
||||
let cid = (await a.upload(data)).get
|
||||
let response = (await b.download(cid)).get
|
||||
|
||||
@ -4,20 +4,20 @@ import std/sequtils
|
||||
import std/strutils
|
||||
import std/sugar
|
||||
import std/times
|
||||
import pkg/codex/conf
|
||||
import pkg/codex/logutils
|
||||
import pkg/storage/conf
|
||||
import pkg/storage/logutils
|
||||
import pkg/chronos/transports/stream
|
||||
import pkg/questionable
|
||||
import ./codexconfig
|
||||
import ./codexprocess
|
||||
import ./storageconfig
|
||||
import ./storageprocess
|
||||
import ./nodeconfigs
|
||||
import ./utils
|
||||
import ../asynctest
|
||||
import ../checktest
|
||||
|
||||
export asynctest
|
||||
export codexprocess
|
||||
export codexconfig
|
||||
export storageprocess
|
||||
export storageconfig
|
||||
export nodeconfigs
|
||||
|
||||
{.push raises: [].}
|
||||
@ -34,12 +34,12 @@ type
|
||||
SuiteTimeoutError = object of MultiNodeSuiteError
|
||||
|
||||
const HardhatPort {.intdefine.}: int = 8545
|
||||
const CodexApiPort {.intdefine.}: int = 8080
|
||||
const CodexDiscPort {.intdefine.}: int = 8090
|
||||
const StorageApiPort {.intdefine.}: int = 8080
|
||||
const StorageDiscPort {.intdefine.}: int = 8090
|
||||
const TestId {.strdefine.}: string = "TestId"
|
||||
const CodexLogToFile {.booldefine.}: bool = false
|
||||
const CodexLogLevel {.strdefine.}: string = ""
|
||||
const CodexLogsDir {.strdefine.}: string = ""
|
||||
const StorageLogToFile {.booldefine.}: bool = false
|
||||
const StorageLogLevel {.strdefine.}: string = ""
|
||||
const StorageLogsDir {.strdefine.}: string = ""
|
||||
|
||||
proc raiseMultiNodeSuiteError(
|
||||
msg: string, parent: ref CatchableError = nil
|
||||
@ -77,9 +77,9 @@ template multinodesuite*(suiteName: string, body: untyped) =
|
||||
var nodeConfigs: NodeConfigs
|
||||
var snapshot: JsonNode
|
||||
var lastUsedHardhatPort = HardhatPort
|
||||
var lastUsedCodexApiPort = CodexApiPort
|
||||
var lastUsedCodexDiscPort = CodexDiscPort
|
||||
var codexPortLock: AsyncLock
|
||||
var lastUsedStorageApiPort = StorageApiPort
|
||||
var lastUsedStorageDiscPort = StorageDiscPort
|
||||
var storagePortLock: AsyncLock
|
||||
|
||||
template test(tname, startNodeConfigs, tbody) =
|
||||
currentTestName = tname
|
||||
@ -91,18 +91,18 @@ template multinodesuite*(suiteName: string, body: untyped) =
|
||||
let parts = url.split(':')
|
||||
url = @[parts[0], parts[1], $port].join(":")
|
||||
|
||||
proc newCodexProcess(
|
||||
roleIdx: int, conf: CodexConfig, role: Role
|
||||
proc newStorageProcess(
|
||||
roleIdx: int, conf: StorageConfig, role: Role
|
||||
): Future[NodeProcess] {.async: (raises: [MultiNodeSuiteError, CancelledError]).} =
|
||||
let nodeIdx = running.len
|
||||
var config = conf
|
||||
let datadir = getDataDir(TestId, currentTestName, $starttime, $role, some roleIdx)
|
||||
|
||||
try:
|
||||
if config.logFile.isSome or CodexLogToFile:
|
||||
if config.logFile.isSome or StorageLogToFile:
|
||||
try:
|
||||
let updatedLogFile = getLogFile(
|
||||
CodexLogsDir, starttime, suiteName, currentTestName, $role, some roleIdx
|
||||
StorageLogsDir, starttime, suiteName, currentTestName, $role, some roleIdx
|
||||
)
|
||||
config.withLogFile(updatedLogFile)
|
||||
except IOError as e:
|
||||
@ -118,17 +118,17 @@ template multinodesuite*(suiteName: string, body: untyped) =
|
||||
e,
|
||||
)
|
||||
|
||||
when CodexLogLevel != "":
|
||||
config.addCliOption("--log-level", CodexLogLevel)
|
||||
when StorageLogLevel != "":
|
||||
config.addCliOption("--log-level", StorageLogLevel)
|
||||
|
||||
var apiPort, discPort: int
|
||||
withLock(codexPortLock):
|
||||
apiPort = await nextFreePort(lastUsedCodexApiPort + nodeIdx)
|
||||
discPort = await nextFreePort(lastUsedCodexDiscPort + nodeIdx)
|
||||
withLock(storagePortLock):
|
||||
apiPort = await nextFreePort(lastUsedStorageApiPort + nodeIdx)
|
||||
discPort = await nextFreePort(lastUsedStorageDiscPort + nodeIdx)
|
||||
config.addCliOption("--api-port", $apiPort)
|
||||
config.addCliOption("--disc-port", $discPort)
|
||||
lastUsedCodexApiPort = apiPort
|
||||
lastUsedCodexDiscPort = discPort
|
||||
lastUsedStorageApiPort = apiPort
|
||||
lastUsedStorageDiscPort = discPort
|
||||
|
||||
for bootstrapNode in bootstrapNodes:
|
||||
config.addCliOption("--bootstrap-node", bootstrapNode)
|
||||
@ -136,30 +136,30 @@ template multinodesuite*(suiteName: string, body: untyped) =
|
||||
config.addCliOption("--data-dir", datadir)
|
||||
config.addCliOption("--nat", "none")
|
||||
config.addCliOption("--listen-addrs", "/ip4/127.0.0.1/tcp/0")
|
||||
except CodexConfigError as e:
|
||||
except StorageConfigError as e:
|
||||
raiseMultiNodeSuiteError "invalid cli option, error: " & e.msg
|
||||
|
||||
try:
|
||||
let node = await CodexProcess.startNode(
|
||||
let node = await StorageProcess.startNode(
|
||||
config.cliArgs, config.debugEnabled, $role & $roleIdx
|
||||
)
|
||||
await node.waitUntilStarted()
|
||||
trace "node started", nodeName = $role & $roleIdx
|
||||
return node
|
||||
except CodexConfigError as e:
|
||||
except StorageConfigError as e:
|
||||
raiseMultiNodeSuiteError "failed to get cli args from config: " & e.msg, e
|
||||
except NodeProcessError as e:
|
||||
raiseMultiNodeSuiteError "node not started, error: " & e.msg, e
|
||||
|
||||
proc clients(): seq[CodexProcess] {.used.} =
|
||||
proc clients(): seq[StorageProcess] {.used.} =
|
||||
return collect:
|
||||
for r in running:
|
||||
if r.role == Role.Client:
|
||||
CodexProcess(r.node)
|
||||
StorageProcess(r.node)
|
||||
|
||||
proc startClientNode(conf: CodexConfig): Future[NodeProcess] {.async.} =
|
||||
proc startClientNode(conf: StorageConfig): Future[NodeProcess] {.async.} =
|
||||
let clientIdx = clients().len
|
||||
return await newCodexProcess(clientIdx, conf, Role.Client)
|
||||
return await newStorageProcess(clientIdx, conf, Role.Client)
|
||||
|
||||
proc teardownImpl() {.async.} =
|
||||
for nodes in @[clients()]:
|
||||
@ -167,7 +167,7 @@ template multinodesuite*(suiteName: string, body: untyped) =
|
||||
await node.stop() # also stops rest client
|
||||
try:
|
||||
node.removeDataDir()
|
||||
except CodexProcessError as e:
|
||||
except StorageProcessError as e:
|
||||
error "Failed to remove data dir during teardown", error = e.msg
|
||||
|
||||
running = @[]
|
||||
@ -191,7 +191,7 @@ template multinodesuite*(suiteName: string, body: untyped) =
|
||||
quit(1)
|
||||
|
||||
proc updateBootstrapNodes(
|
||||
node: CodexProcess
|
||||
node: StorageProcess
|
||||
): Future[void] {.async: (raises: [MultiNodeSuiteError]).} =
|
||||
try:
|
||||
without ninfo =? await node.client.info():
|
||||
@ -206,9 +206,9 @@ template multinodesuite*(suiteName: string, body: untyped) =
|
||||
# When this file is run with `-d:chronicles_sinks=textlines[file]`, we
|
||||
# need to set the log file path at runtime, otherwise chronicles didn't seem to
|
||||
# create a log file even when using an absolute path
|
||||
when defaultChroniclesStream.outputs is (FileOutput,) and CodexLogsDir.len > 0:
|
||||
when defaultChroniclesStream.outputs is (FileOutput,) and StorageLogsDir.len > 0:
|
||||
let logFile =
|
||||
CodexLogsDir / sanitize(getAppFilename().extractFilename & ".chronicles.log")
|
||||
StorageLogsDir / sanitize(getAppFilename().extractFilename & ".chronicles.log")
|
||||
let success = defaultChroniclesStream.outputs[0].open(logFile, fmAppend)
|
||||
doAssert success, "Failed to open log file: " & logFile
|
||||
|
||||
@ -219,7 +219,7 @@ template multinodesuite*(suiteName: string, body: untyped) =
|
||||
for config in clients.configs:
|
||||
let node = await startClientNode(config)
|
||||
running.add RunningNode(role: Role.Client, node: node)
|
||||
await CodexProcess(node).updateBootstrapNodes()
|
||||
await StorageProcess(node).updateBootstrapNodes()
|
||||
|
||||
teardown:
|
||||
await teardownImpl()
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import pkg/questionable
|
||||
import ./codexconfig
|
||||
import ./storageconfig
|
||||
|
||||
type NodeConfigs* = object
|
||||
clients*: ?CodexConfigs
|
||||
clients*: ?StorageConfigs
|
||||
|
||||
@ -7,12 +7,12 @@ import pkg/libp2p
|
||||
import std/os
|
||||
import std/strformat
|
||||
import std/strutils
|
||||
import codex/conf
|
||||
import codex/utils/exceptions
|
||||
import codex/utils/trackedfutures
|
||||
import ./codexclient
|
||||
import storage/conf
|
||||
import storage/utils/exceptions
|
||||
import storage/utils/trackedfutures
|
||||
import ./storageclient
|
||||
|
||||
export codexclient
|
||||
export storageclient
|
||||
export chronicles
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
@ -4,27 +4,27 @@ from pkg/libp2p import Cid, `$`, init
|
||||
import pkg/stint
|
||||
import pkg/questionable/results
|
||||
import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient, httptable]
|
||||
import pkg/codex/logutils
|
||||
import pkg/codex/rest/json
|
||||
import pkg/codex/errors
|
||||
import pkg/storage/logutils
|
||||
import pkg/storage/rest/json
|
||||
import pkg/storage/errors
|
||||
|
||||
export httptable, httpclient
|
||||
|
||||
type CodexClient* = ref object
|
||||
type StorageClient* = ref object
|
||||
baseurl: string
|
||||
session: HttpSessionRef
|
||||
|
||||
type HasBlockResponse = object
|
||||
has: bool
|
||||
|
||||
proc new*(_: type CodexClient, baseurl: string): CodexClient =
|
||||
CodexClient(session: HttpSessionRef.new(), baseurl: baseurl)
|
||||
proc new*(_: type StorageClient, baseurl: string): StorageClient =
|
||||
StorageClient(session: HttpSessionRef.new(), baseurl: baseurl)
|
||||
|
||||
proc close*(self: CodexClient): Future[void] {.async: (raises: []).} =
|
||||
proc close*(self: StorageClient): Future[void] {.async: (raises: []).} =
|
||||
await self.session.closeWait()
|
||||
|
||||
proc request(
|
||||
self: CodexClient,
|
||||
self: StorageClient,
|
||||
httpMethod: httputils.HttpMethod,
|
||||
url: string,
|
||||
body: openArray[char] = [],
|
||||
@ -46,7 +46,7 @@ proc request(
|
||||
.send()
|
||||
|
||||
proc post*(
|
||||
self: CodexClient,
|
||||
self: StorageClient,
|
||||
url: string,
|
||||
body: string = "",
|
||||
headers: seq[HttpHeaderTuple] = @[],
|
||||
@ -56,21 +56,21 @@ proc post*(
|
||||
return self.request(MethodPost, url, headers = headers, body = body)
|
||||
|
||||
proc get(
|
||||
self: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[]
|
||||
self: StorageClient, url: string, headers: seq[HttpHeaderTuple] = @[]
|
||||
): Future[HttpClientResponseRef] {.
|
||||
async: (raw: true, raises: [CancelledError, HttpError])
|
||||
.} =
|
||||
return self.request(MethodGet, url, headers = headers)
|
||||
|
||||
proc delete(
|
||||
self: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[]
|
||||
self: StorageClient, url: string, headers: seq[HttpHeaderTuple] = @[]
|
||||
): Future[HttpClientResponseRef] {.
|
||||
async: (raw: true, raises: [CancelledError, HttpError])
|
||||
.} =
|
||||
return self.request(MethodDelete, url, headers = headers)
|
||||
|
||||
proc patch*(
|
||||
self: CodexClient,
|
||||
self: StorageClient,
|
||||
url: string,
|
||||
body: string = "",
|
||||
headers: seq[HttpHeaderTuple] = @[],
|
||||
@ -85,19 +85,19 @@ proc body*(
|
||||
return bytesToString (await response.getBodyBytes())
|
||||
|
||||
proc getContent(
|
||||
client: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[]
|
||||
client: StorageClient, url: string, headers: seq[HttpHeaderTuple] = @[]
|
||||
): Future[string] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let response = await client.get(url, headers)
|
||||
return await response.body
|
||||
|
||||
proc info*(
|
||||
client: CodexClient
|
||||
client: StorageClient
|
||||
): Future[?!JsonNode] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let response = await client.get(client.baseurl & "/debug/info")
|
||||
return JsonNode.parse(await response.body)
|
||||
|
||||
proc setLogLevel*(
|
||||
client: CodexClient, level: string
|
||||
client: StorageClient, level: string
|
||||
): Future[void] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let
|
||||
url = client.baseurl & "/debug/chronicles/loglevel?level=" & level
|
||||
@ -106,26 +106,26 @@ proc setLogLevel*(
|
||||
assert response.status == 200
|
||||
|
||||
proc uploadRaw*(
|
||||
client: CodexClient, contents: string, headers: seq[HttpHeaderTuple] = @[]
|
||||
client: StorageClient, contents: string, headers: seq[HttpHeaderTuple] = @[]
|
||||
): Future[HttpClientResponseRef] {.
|
||||
async: (raw: true, raises: [CancelledError, HttpError])
|
||||
.} =
|
||||
return client.post(client.baseurl & "/data", body = contents, headers = headers)
|
||||
|
||||
proc upload*(
|
||||
client: CodexClient, contents: string
|
||||
client: StorageClient, contents: string
|
||||
): Future[?!Cid] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let response = await client.uploadRaw(contents)
|
||||
assert response.status == 200
|
||||
Cid.init(await response.body).mapFailure
|
||||
|
||||
proc upload*(
|
||||
client: CodexClient, bytes: seq[byte]
|
||||
client: StorageClient, bytes: seq[byte]
|
||||
): Future[?!Cid] {.async: (raw: true).} =
|
||||
return client.upload(string.fromBytes(bytes))
|
||||
|
||||
proc downloadRaw*(
|
||||
client: CodexClient, cid: string, local = false
|
||||
client: StorageClient, cid: string, local = false
|
||||
): Future[HttpClientResponseRef] {.
|
||||
async: (raw: true, raises: [CancelledError, HttpError])
|
||||
.} =
|
||||
@ -133,7 +133,7 @@ proc downloadRaw*(
|
||||
client.get(client.baseurl & "/data/" & cid & (if local: "" else: "/network/stream"))
|
||||
|
||||
proc downloadBytes*(
|
||||
client: CodexClient, cid: Cid, local = false
|
||||
client: StorageClient, cid: Cid, local = false
|
||||
): Future[?!seq[byte]] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let response = await client.downloadRaw($cid, local = local)
|
||||
|
||||
@ -143,14 +143,14 @@ proc downloadBytes*(
|
||||
success await response.getBodyBytes()
|
||||
|
||||
proc download*(
|
||||
client: CodexClient, cid: Cid, local = false
|
||||
client: StorageClient, cid: Cid, local = false
|
||||
): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
without response =? await client.downloadBytes(cid, local = local), err:
|
||||
return failure(err)
|
||||
return success bytesToString(response)
|
||||
|
||||
proc downloadNoStream*(
|
||||
client: CodexClient, cid: Cid
|
||||
client: StorageClient, cid: Cid
|
||||
): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let response = await client.post(client.baseurl & "/data/" & $cid & "/network")
|
||||
|
||||
@ -160,7 +160,7 @@ proc downloadNoStream*(
|
||||
success await response.body
|
||||
|
||||
proc downloadManifestOnly*(
|
||||
client: CodexClient, cid: Cid
|
||||
client: StorageClient, cid: Cid
|
||||
): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let response =
|
||||
await client.get(client.baseurl & "/data/" & $cid & "/network/manifest")
|
||||
@ -171,14 +171,14 @@ proc downloadManifestOnly*(
|
||||
success await response.body
|
||||
|
||||
proc deleteRaw*(
|
||||
client: CodexClient, cid: string
|
||||
client: StorageClient, cid: string
|
||||
): Future[HttpClientResponseRef] {.
|
||||
async: (raw: true, raises: [CancelledError, HttpError])
|
||||
.} =
|
||||
return client.delete(client.baseurl & "/data/" & cid)
|
||||
|
||||
proc delete*(
|
||||
client: CodexClient, cid: Cid
|
||||
client: StorageClient, cid: Cid
|
||||
): Future[?!void] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let response = await client.deleteRaw($cid)
|
||||
|
||||
@ -188,14 +188,14 @@ proc delete*(
|
||||
success()
|
||||
|
||||
proc listRaw*(
|
||||
client: CodexClient
|
||||
client: StorageClient
|
||||
): Future[HttpClientResponseRef] {.
|
||||
async: (raw: true, raises: [CancelledError, HttpError])
|
||||
.} =
|
||||
return client.get(client.baseurl & "/data")
|
||||
|
||||
proc list*(
|
||||
client: CodexClient
|
||||
client: StorageClient
|
||||
): Future[?!RestContentList] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let response = await client.listRaw()
|
||||
|
||||
@ -205,7 +205,7 @@ proc list*(
|
||||
RestContentList.fromJson(await response.body)
|
||||
|
||||
proc space*(
|
||||
client: CodexClient
|
||||
client: StorageClient
|
||||
): Future[?!RestRepoStore] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let url = client.baseurl & "/space"
|
||||
let response = await client.get(url)
|
||||
@ -215,11 +215,11 @@ proc space*(
|
||||
|
||||
RestRepoStore.fromJson(await response.body)
|
||||
|
||||
proc buildUrl*(client: CodexClient, path: string): string =
|
||||
proc buildUrl*(client: StorageClient, path: string): string =
|
||||
return client.baseurl & path
|
||||
|
||||
proc hasBlock*(
|
||||
client: CodexClient, cid: Cid
|
||||
client: StorageClient, cid: Cid
|
||||
): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} =
|
||||
let url = client.baseurl & "/data/" & $cid & "/exists"
|
||||
let body = await client.getContent(url)
|
||||
@ -229,7 +229,7 @@ proc hasBlock*(
|
||||
return response.get.has.success
|
||||
|
||||
proc hasBlockRaw*(
|
||||
client: CodexClient, cid: string
|
||||
client: StorageClient, cid: string
|
||||
): Future[HttpClientResponseRef] {.
|
||||
async: (raw: true, raises: [CancelledError, HttpError])
|
||||
.} =
|
||||
|
||||
@ -5,8 +5,8 @@ import std/strutils
|
||||
import std/sugar
|
||||
import std/tables
|
||||
from pkg/chronicles import LogLevel
|
||||
import pkg/codex/conf
|
||||
import pkg/codex/units
|
||||
import pkg/storage/conf
|
||||
import pkg/storage/units
|
||||
import pkg/confutils
|
||||
import pkg/confutils/defs
|
||||
import libp2p except setup
|
||||
@ -17,39 +17,39 @@ export clioption
|
||||
export confutils
|
||||
|
||||
type
|
||||
CodexConfigs* = object
|
||||
configs*: seq[CodexConfig]
|
||||
StorageConfigs* = object
|
||||
configs*: seq[StorageConfig]
|
||||
|
||||
CodexConfig* = object
|
||||
StorageConfig* = object
|
||||
cliOptions: Table[StartUpCmd, Table[string, CliOption]]
|
||||
debugEnabled*: bool
|
||||
|
||||
CodexConfigError* = object of CatchableError
|
||||
StorageConfigError* = object of CatchableError
|
||||
|
||||
proc cliArgs*(config: CodexConfig): seq[string] {.gcsafe, raises: [CodexConfigError].}
|
||||
proc cliArgs*(config: StorageConfig): seq[string] {.gcsafe, raises: [StorageConfigError].}
|
||||
|
||||
proc raiseCodexConfigError(msg: string) {.raises: [CodexConfigError].} =
|
||||
raise newException(CodexConfigError, msg)
|
||||
proc raiseStorageConfigError(msg: string) {.raises: [StorageConfigError].} =
|
||||
raise newException(StorageConfigError, msg)
|
||||
|
||||
template convertError(body) =
|
||||
try:
|
||||
body
|
||||
except CatchableError as e:
|
||||
raiseCodexConfigError e.msg
|
||||
raiseStorageConfigError e.msg
|
||||
|
||||
proc init*(_: type CodexConfigs, nodes = 1): CodexConfigs {.raises: [].} =
|
||||
CodexConfigs(configs: newSeq[CodexConfig](nodes))
|
||||
proc init*(_: type StorageConfigs, nodes = 1): StorageConfigs {.raises: [].} =
|
||||
StorageConfigs(configs: newSeq[StorageConfig](nodes))
|
||||
|
||||
func nodes*(self: CodexConfigs): int =
|
||||
func nodes*(self: StorageConfigs): int =
|
||||
self.configs.len
|
||||
|
||||
proc checkBounds(self: CodexConfigs, idx: int) {.raises: [CodexConfigError].} =
|
||||
proc checkBounds(self: StorageConfigs, idx: int) {.raises: [StorageConfigError].} =
|
||||
if idx notin 0 ..< self.configs.len:
|
||||
raiseCodexConfigError "index must be in bounds of the number of nodes"
|
||||
raiseStorageConfigError "index must be in bounds of the number of nodes"
|
||||
|
||||
proc buildConfig(
|
||||
config: CodexConfig, msg: string
|
||||
): CodexConf {.raises: [CodexConfigError].} =
|
||||
config: StorageConfig, msg: string
|
||||
): StorageConf {.raises: [StorageConfigError].} =
|
||||
proc postFix(msg: string): string =
|
||||
if msg.len > 0:
|
||||
": " & msg
|
||||
@ -57,38 +57,38 @@ proc buildConfig(
|
||||
""
|
||||
|
||||
try:
|
||||
return CodexConf.load(cmdLine = config.cliArgs, quitOnFailure = false)
|
||||
return StorageConf.load(cmdLine = config.cliArgs, quitOnFailure = false)
|
||||
except ConfigurationError as e:
|
||||
raiseCodexConfigError msg & e.msg.postFix
|
||||
raiseStorageConfigError msg & e.msg.postFix
|
||||
except Exception as e:
|
||||
## TODO: remove once proper exception handling added to nim-confutils
|
||||
raiseCodexConfigError msg & e.msg.postFix
|
||||
raiseStorageConfigError msg & e.msg.postFix
|
||||
|
||||
proc addCliOption*(
|
||||
config: var CodexConfig, group = StartUpCmd.noCmd, cliOption: CliOption
|
||||
) {.raises: [CodexConfigError].} =
|
||||
config: var StorageConfig, group = StartUpCmd.noCmd, cliOption: CliOption
|
||||
) {.raises: [StorageConfigError].} =
|
||||
var options = config.cliOptions.getOrDefault(group)
|
||||
options[cliOption.key] = cliOption # overwrite if already exists
|
||||
config.cliOptions[group] = options
|
||||
discard config.buildConfig("Invalid cli arg " & $cliOption)
|
||||
|
||||
proc addCliOption*(
|
||||
config: var CodexConfig, group = StartUpCmd.noCmd, key: string, value = ""
|
||||
) {.raises: [CodexConfigError].} =
|
||||
config: var StorageConfig, group = StartUpCmd.noCmd, key: string, value = ""
|
||||
) {.raises: [StorageConfigError].} =
|
||||
config.addCliOption(group, CliOption(key: key, value: value))
|
||||
|
||||
proc addCliOption*(
|
||||
config: var CodexConfig, cliOption: CliOption
|
||||
) {.raises: [CodexConfigError].} =
|
||||
config: var StorageConfig, cliOption: CliOption
|
||||
) {.raises: [StorageConfigError].} =
|
||||
config.addCliOption(StartUpCmd.noCmd, cliOption)
|
||||
|
||||
proc addCliOption*(
|
||||
config: var CodexConfig, key: string, value = ""
|
||||
) {.raises: [CodexConfigError].} =
|
||||
config: var StorageConfig, key: string, value = ""
|
||||
) {.raises: [StorageConfigError].} =
|
||||
config.addCliOption(StartUpCmd.noCmd, CliOption(key: key, value: value))
|
||||
|
||||
proc cliArgs*(config: CodexConfig): seq[string] {.gcsafe, raises: [CodexConfigError].} =
|
||||
## converts CodexConfig cli options and command groups in a sequence of args
|
||||
proc cliArgs*(config: StorageConfig): seq[string] {.gcsafe, raises: [StorageConfigError].} =
|
||||
## converts StorageConfig cli options and command groups in a sequence of args
|
||||
## and filters out cli options by node index if provided in the CliOption
|
||||
var args: seq[string] = @[]
|
||||
|
||||
@ -102,18 +102,18 @@ proc cliArgs*(config: CodexConfig): seq[string] {.gcsafe, raises: [CodexConfigEr
|
||||
|
||||
return args
|
||||
|
||||
proc logFile*(config: CodexConfig): ?string {.raises: [CodexConfigError].} =
|
||||
let built = config.buildConfig("Invalid codex config cli params")
|
||||
proc logFile*(config: StorageConfig): ?string {.raises: [StorageConfigError].} =
|
||||
let built = config.buildConfig("Invalid storage config cli params")
|
||||
built.logFile
|
||||
|
||||
proc logLevel*(config: CodexConfig): LogLevel {.raises: [CodexConfigError].} =
|
||||
proc logLevel*(config: StorageConfig): LogLevel {.raises: [StorageConfigError].} =
|
||||
convertError:
|
||||
let built = config.buildConfig("Invalid codex config cli params")
|
||||
let built = config.buildConfig("Invalid storage config cli params")
|
||||
return parseEnum[LogLevel](built.logLevel.toUpperAscii)
|
||||
|
||||
proc debug*(
|
||||
self: CodexConfigs, idx: int, enabled = true
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, idx: int, enabled = true
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
## output log in stdout for a specific node in the group
|
||||
|
||||
self.checkBounds idx
|
||||
@ -122,7 +122,7 @@ proc debug*(
|
||||
startConfig.configs[idx].debugEnabled = enabled
|
||||
return startConfig
|
||||
|
||||
proc debug*(self: CodexConfigs, enabled = true): CodexConfigs {.raises: [].} =
|
||||
proc debug*(self: StorageConfigs, enabled = true): StorageConfigs {.raises: [].} =
|
||||
## output log in stdout for all nodes in group
|
||||
var startConfig = self
|
||||
for config in startConfig.configs.mitems:
|
||||
@ -130,15 +130,15 @@ proc debug*(self: CodexConfigs, enabled = true): CodexConfigs {.raises: [].} =
|
||||
return startConfig
|
||||
|
||||
proc withLogFile*(
|
||||
self: CodexConfigs, idx: int
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, idx: int
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
self.checkBounds idx
|
||||
|
||||
var startConfig = self
|
||||
startConfig.configs[idx].addCliOption("--log-file", "<updated_in_test>")
|
||||
return startConfig
|
||||
|
||||
proc withLogFile*(self: CodexConfigs): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
proc withLogFile*(self: StorageConfigs): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
## typically called from test, sets config such that a log file should be
|
||||
## created
|
||||
var startConfig = self
|
||||
@ -147,9 +147,9 @@ proc withLogFile*(self: CodexConfigs): CodexConfigs {.raises: [CodexConfigError]
|
||||
return startConfig
|
||||
|
||||
proc withLogFile*(
|
||||
self: var CodexConfig, logFile: string
|
||||
) {.raises: [CodexConfigError].} =
|
||||
#: CodexConfigs =
|
||||
self: var StorageConfig, logFile: string
|
||||
) {.raises: [StorageConfigError].} =
|
||||
#: StorageConfigs =
|
||||
## typically called internally from the test suite, sets a log file path to
|
||||
## be created during the test run, for a specified node in the group
|
||||
# var config = self
|
||||
@ -157,15 +157,15 @@ proc withLogFile*(
|
||||
# return startConfig
|
||||
|
||||
proc withLogLevel*(
|
||||
self: CodexConfig, level: LogLevel | string
|
||||
): CodexConfig {.raises: [CodexConfigError].} =
|
||||
self: StorageConfig, level: LogLevel | string
|
||||
): StorageConfig {.raises: [StorageConfigError].} =
|
||||
var config = self
|
||||
config.addCliOption("--log-level", $level)
|
||||
return config
|
||||
|
||||
proc withLogLevel*(
|
||||
self: CodexConfigs, idx: int, level: LogLevel | string
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, idx: int, level: LogLevel | string
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
self.checkBounds idx
|
||||
|
||||
var startConfig = self
|
||||
@ -173,23 +173,23 @@ proc withLogLevel*(
|
||||
return startConfig
|
||||
|
||||
proc withLogLevel*(
|
||||
self: CodexConfigs, level: LogLevel | string
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, level: LogLevel | string
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
var startConfig = self
|
||||
for config in startConfig.configs.mitems:
|
||||
config.addCliOption("--log-level", $level)
|
||||
return startConfig
|
||||
|
||||
proc withBlockTtl*(
|
||||
self: CodexConfig, ttl: int
|
||||
): CodexConfig {.raises: [CodexConfigError].} =
|
||||
self: StorageConfig, ttl: int
|
||||
): StorageConfig {.raises: [StorageConfigError].} =
|
||||
var config = self
|
||||
config.addCliOption("--block-ttl", $ttl)
|
||||
return config
|
||||
|
||||
proc withBlockTtl*(
|
||||
self: CodexConfigs, idx: int, ttl: int
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, idx: int, ttl: int
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
self.checkBounds idx
|
||||
|
||||
var startConfig = self
|
||||
@ -197,23 +197,23 @@ proc withBlockTtl*(
|
||||
return startConfig
|
||||
|
||||
proc withBlockTtl*(
|
||||
self: CodexConfigs, ttl: int
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, ttl: int
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
var startConfig = self
|
||||
for config in startConfig.configs.mitems:
|
||||
config.addCliOption("--block-ttl", $ttl)
|
||||
return startConfig
|
||||
|
||||
proc withBlockMaintenanceInterval*(
|
||||
self: CodexConfig, interval: int
|
||||
): CodexConfig {.raises: [CodexConfigError].} =
|
||||
self: StorageConfig, interval: int
|
||||
): StorageConfig {.raises: [StorageConfigError].} =
|
||||
var config = self
|
||||
config.addCliOption("--block-mi", $interval)
|
||||
return config
|
||||
|
||||
proc withBlockMaintenanceInterval*(
|
||||
self: CodexConfigs, idx: int, interval: int
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, idx: int, interval: int
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
self.checkBounds idx
|
||||
|
||||
var startConfig = self
|
||||
@ -221,26 +221,26 @@ proc withBlockMaintenanceInterval*(
|
||||
return startConfig
|
||||
|
||||
proc withBlockMaintenanceInterval*(
|
||||
self: CodexConfigs, interval: int
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, interval: int
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
var startConfig = self
|
||||
for config in startConfig.configs.mitems:
|
||||
config.addCliOption("--block-mi", $interval)
|
||||
return startConfig
|
||||
|
||||
proc logLevelWithTopics(
|
||||
config: CodexConfig, topics: varargs[string]
|
||||
): string {.raises: [CodexConfigError].} =
|
||||
config: StorageConfig, topics: varargs[string]
|
||||
): string {.raises: [StorageConfigError].} =
|
||||
convertError:
|
||||
var logLevel = LogLevel.INFO
|
||||
let built = config.buildConfig("Invalid codex config cli params")
|
||||
let built = config.buildConfig("Invalid storage config cli params")
|
||||
logLevel = parseEnum[LogLevel](built.logLevel.toUpperAscii)
|
||||
let level = $logLevel & ";TRACE: " & topics.join(",")
|
||||
return level
|
||||
|
||||
proc withLogTopics*(
|
||||
self: CodexConfigs, idx: int, topics: varargs[string]
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, idx: int, topics: varargs[string]
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
self.checkBounds idx
|
||||
|
||||
convertError:
|
||||
@ -250,8 +250,8 @@ proc withLogTopics*(
|
||||
return startConfig.withLogLevel(idx, level)
|
||||
|
||||
proc withLogTopics*(
|
||||
self: CodexConfigs, topics: varargs[string]
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, topics: varargs[string]
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
var startConfig = self
|
||||
for config in startConfig.configs.mitems:
|
||||
let level = config.logLevelWithTopics(topics)
|
||||
@ -259,8 +259,8 @@ proc withLogTopics*(
|
||||
return startConfig
|
||||
|
||||
proc withStorageQuota*(
|
||||
self: CodexConfigs, idx: int, quota: NBytes
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, idx: int, quota: NBytes
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
self.checkBounds idx
|
||||
|
||||
var startConfig = self
|
||||
@ -268,8 +268,8 @@ proc withStorageQuota*(
|
||||
return startConfig
|
||||
|
||||
proc withStorageQuota*(
|
||||
self: CodexConfigs, quota: NBytes
|
||||
): CodexConfigs {.raises: [CodexConfigError].} =
|
||||
self: StorageConfigs, quota: NBytes
|
||||
): StorageConfigs {.raises: [StorageConfigError].} =
|
||||
var startConfig = self
|
||||
for config in startConfig.configs.mitems:
|
||||
config.addCliOption("--storage-quota", $quota)
|
||||
|
||||
@ -7,94 +7,94 @@ import pkg/libp2p
|
||||
import std/os
|
||||
import std/strutils
|
||||
import std/times
|
||||
import codex/conf
|
||||
import ./codexclient
|
||||
import storage/conf
|
||||
import ./storageclient
|
||||
import ./nodeprocess
|
||||
|
||||
export codexclient
|
||||
export storageclient
|
||||
export chronicles
|
||||
export nodeprocess
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
logScope:
|
||||
topics = "integration testing codex process"
|
||||
topics = "integration testing storage process"
|
||||
|
||||
type
|
||||
CodexProcess* = ref object of NodeProcess
|
||||
client: ?CodexClient
|
||||
StorageProcess* = ref object of NodeProcess
|
||||
client: ?StorageClient
|
||||
|
||||
CodexProcessError* = object of NodeProcessError
|
||||
StorageProcessError* = object of NodeProcessError
|
||||
|
||||
proc raiseCodexProcessError(
|
||||
proc raiseStorageProcessError(
|
||||
msg: string, parent: ref CatchableError
|
||||
) {.raises: [CodexProcessError].} =
|
||||
raise newException(CodexProcessError, msg & ": " & parent.msg, parent)
|
||||
) {.raises: [StorageProcessError].} =
|
||||
raise newException(StorageProcessError, msg & ": " & parent.msg, parent)
|
||||
|
||||
template convertError(msg, body: typed) =
|
||||
# Don't use this in an async proc, unless body does not raise CancelledError
|
||||
try:
|
||||
body
|
||||
except CatchableError as parent:
|
||||
raiseCodexProcessError(msg, parent)
|
||||
raiseStorageProcessError(msg, parent)
|
||||
|
||||
method workingDir(node: CodexProcess): string =
|
||||
method workingDir(node: StorageProcess): string =
|
||||
return currentSourcePath() / ".." / ".." / ".."
|
||||
|
||||
method executable(node: CodexProcess): string =
|
||||
method executable(node: StorageProcess): string =
|
||||
return "build" / "storage"
|
||||
|
||||
method startedOutput(node: CodexProcess): string =
|
||||
method startedOutput(node: StorageProcess): string =
|
||||
return "REST service started"
|
||||
|
||||
method processOptions(node: CodexProcess): set[AsyncProcessOption] =
|
||||
method processOptions(node: StorageProcess): set[AsyncProcessOption] =
|
||||
return {AsyncProcessOption.StdErrToStdOut}
|
||||
|
||||
method outputLineEndings(node: CodexProcess): string =
|
||||
method outputLineEndings(node: StorageProcess): string =
|
||||
return "\n"
|
||||
|
||||
method onOutputLineCaptured(node: CodexProcess, line: string) =
|
||||
method onOutputLineCaptured(node: StorageProcess, line: string) =
|
||||
discard
|
||||
|
||||
proc config(node: CodexProcess): CodexConf {.raises: [CodexProcessError].} =
|
||||
proc config(node: StorageProcess): StorageConf {.raises: [StorageProcessError].} =
|
||||
# cannot use convertError here as it uses typed parameters which forces type
|
||||
# resolution, while confutils.load uses untyped parameters and expects type
|
||||
# resolution not to happen yet. In other words, it won't compile.
|
||||
try:
|
||||
return CodexConf.load(
|
||||
return StorageConf.load(
|
||||
cmdLine = node.arguments, quitOnFailure = false, secondarySources = nil
|
||||
)
|
||||
except ConfigurationError as parent:
|
||||
raiseCodexProcessError "Failed to load node arguments into CodexConf", parent
|
||||
raiseStorageProcessError "Failed to load node arguments into StorageConf", parent
|
||||
|
||||
proc dataDir(node: CodexProcess): string {.raises: [CodexProcessError].} =
|
||||
proc dataDir(node: StorageProcess): string {.raises: [StorageProcessError].} =
|
||||
return node.config.dataDir.string
|
||||
|
||||
proc apiUrl*(node: CodexProcess): string {.raises: [CodexProcessError].} =
|
||||
proc apiUrl*(node: StorageProcess): string {.raises: [StorageProcessError].} =
|
||||
let config = node.config
|
||||
without apiBindAddress =? config.apiBindAddress:
|
||||
raise
|
||||
newException(CodexProcessError, "REST API not started: --api-bindaddr not set")
|
||||
newException(StorageProcessError, "REST API not started: --api-bindaddr not set")
|
||||
return "http://" & apiBindAddress & ":" & $config.apiPort & "/api/storage/v1"
|
||||
|
||||
proc logFile*(node: CodexProcess): ?string {.raises: [CodexProcessError].} =
|
||||
proc logFile*(node: StorageProcess): ?string {.raises: [StorageProcessError].} =
|
||||
node.config.logFile
|
||||
|
||||
proc client*(node: CodexProcess): CodexClient {.raises: [CodexProcessError].} =
|
||||
proc client*(node: StorageProcess): StorageClient {.raises: [StorageProcessError].} =
|
||||
if client =? node.client:
|
||||
return client
|
||||
let client = CodexClient.new(node.apiUrl)
|
||||
let client = StorageClient.new(node.apiUrl)
|
||||
node.client = some client
|
||||
return client
|
||||
|
||||
proc updateLogFile(node: CodexProcess, newLogFile: string) =
|
||||
proc updateLogFile(node: StorageProcess, newLogFile: string) =
|
||||
for arg in node.arguments.mitems:
|
||||
if arg.startsWith("--log-file="):
|
||||
arg = "--log-file=" & newLogFile
|
||||
break
|
||||
|
||||
method restart*(node: CodexProcess) {.async.} =
|
||||
trace "restarting codex"
|
||||
method restart*(node: StorageProcess) {.async.} =
|
||||
trace "restarting storage"
|
||||
await node.stop()
|
||||
if logFile =? node.logFile:
|
||||
# chronicles truncates the existing log file on start, so changed the log
|
||||
@ -104,13 +104,13 @@ method restart*(node: CodexProcess) {.async.} =
|
||||
)
|
||||
await node.start()
|
||||
await node.waitUntilStarted()
|
||||
trace "codex process restarted"
|
||||
trace "storage process restarted"
|
||||
|
||||
method stop*(node: CodexProcess) {.async: (raises: []).} =
|
||||
method stop*(node: StorageProcess) {.async: (raises: []).} =
|
||||
logScope:
|
||||
nodeName = node.name
|
||||
|
||||
trace "stopping codex client"
|
||||
trace "stopping storage client"
|
||||
await procCall NodeProcess(node).stop()
|
||||
|
||||
if not node.process.isNil:
|
||||
@ -120,8 +120,8 @@ method stop*(node: CodexProcess) {.async: (raises: []).} =
|
||||
|
||||
if client =? node.client:
|
||||
await client.close()
|
||||
node.client = none CodexClient
|
||||
node.client = none StorageClient
|
||||
|
||||
method removeDataDir*(node: CodexProcess) {.raises: [CodexProcessError].} =
|
||||
convertError("failed to remove codex node data directory"):
|
||||
method removeDataDir*(node: StorageProcess) {.raises: [StorageProcessError].} =
|
||||
convertError("failed to remove storage node data directory"):
|
||||
removeDir(node.dataDir)
|
||||
|
||||
@ -1,23 +1,23 @@
|
||||
import std/macros
|
||||
import pkg/questionable
|
||||
import ./multinodes
|
||||
import ./codexconfig
|
||||
import ./codexprocess
|
||||
import ./codexclient
|
||||
import ./storageconfig
|
||||
import ./storageprocess
|
||||
import ./storageclient
|
||||
import ./nodeconfigs
|
||||
|
||||
export codexclient
|
||||
export storageclient
|
||||
export multinodes
|
||||
|
||||
template twonodessuite*(name: string, body: untyped) =
|
||||
multinodesuite name:
|
||||
let twoNodesConfig {.inject, used.} =
|
||||
NodeConfigs(clients: CodexConfigs.init(nodes = 2).some)
|
||||
NodeConfigs(clients: StorageConfigs.init(nodes = 2).some)
|
||||
|
||||
var node1 {.inject, used.}: CodexProcess
|
||||
var node2 {.inject, used.}: CodexProcess
|
||||
var client1 {.inject, used.}: CodexClient
|
||||
var client2 {.inject, used.}: CodexClient
|
||||
var node1 {.inject, used.}: StorageProcess
|
||||
var node2 {.inject, used.}: StorageProcess
|
||||
var client1 {.inject, used.}: StorageClient
|
||||
var client2 {.inject, used.}: StorageClient
|
||||
|
||||
setup:
|
||||
node1 = clients()[0]
|
||||
|
||||
@ -2,7 +2,7 @@ import std/os
|
||||
import std/strformat
|
||||
import pkg/chronos
|
||||
import pkg/chronos/asyncproc
|
||||
import pkg/codex/logutils
|
||||
import pkg/storage/logutils
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
@ -88,5 +88,5 @@ proc getDataDir*(testId, testName, startTime, role: string, index = int.none): s
|
||||
if idx =? index:
|
||||
suffix &= "_" & $idx
|
||||
|
||||
getTempDir() / "Codex" / sanitize(testId) / sanitize(testName) / sanitize(startTime) /
|
||||
getTempDir() / "Storage" / sanitize(testId) / sanitize(testName) / sanitize(startTime) /
|
||||
sanitize(suffix)
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
when not defined(nimscript):
|
||||
import pkg/codex/logutils
|
||||
import pkg/storage/logutils
|
||||
|
||||
proc ignoreLogging(level: LogLevel, message: LogOutputStr) =
|
||||
discard
|
||||
|
||||
@ -6,13 +6,13 @@ import pkg/chronos
|
||||
|
||||
import pkg/libp2p/errors
|
||||
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/storage/rng
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/blockexchange
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/manifest
|
||||
import pkg/storage/merkletree
|
||||
import pkg/storage/blocktype as bt
|
||||
|
||||
import ../../../asynctest
|
||||
import ../../helpers
|
||||
|
||||
@ -3,14 +3,14 @@ import std/tables
|
||||
|
||||
import pkg/chronos
|
||||
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/blockexchange/engine
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/merkletree
|
||||
import pkg/storage/rng
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/blockexchange
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/blockexchange/engine
|
||||
import pkg/storage/manifest
|
||||
import pkg/storage/merkletree
|
||||
|
||||
import ../../../asynctest
|
||||
import ../../helpers
|
||||
|
||||
@ -2,12 +2,12 @@ import pkg/chronos
|
||||
import pkg/libp2p/routing_record
|
||||
import pkg/codexdht/discv5/protocol as discv5
|
||||
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/discovery
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/manifest
|
||||
import pkg/storage/blockexchange
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/discovery
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/manifest
|
||||
|
||||
import ../../../asynctest
|
||||
import ../../helpers
|
||||
|
||||
@ -5,11 +5,11 @@ import std/importutils
|
||||
import pkg/chronos
|
||||
import pkg/stew/byteutils
|
||||
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/discovery
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/blockexchange
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/discovery
|
||||
import pkg/storage/blocktype as bt
|
||||
|
||||
import ../../../asynctest
|
||||
import ../../examples
|
||||
|
||||
@ -8,13 +8,13 @@ import pkg/libp2p/errors
|
||||
import pkg/libp2p/routing_record
|
||||
import pkg/codexdht/discv5/protocol as discv5
|
||||
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/discovery
|
||||
import pkg/codex/blocktype
|
||||
import pkg/codex/utils/asyncheapqueue
|
||||
import pkg/storage/rng
|
||||
import pkg/storage/blockexchange
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/discovery
|
||||
import pkg/storage/blocktype
|
||||
import pkg/storage/utils/asyncheapqueue
|
||||
|
||||
import ../../../asynctest
|
||||
import ../../helpers
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import pkg/chronos
|
||||
|
||||
import pkg/codex/blockexchange/protobuf/presence
|
||||
import pkg/storage/blockexchange/protobuf/presence
|
||||
|
||||
import ../../../asynctest
|
||||
import ../../examples
|
||||
|
||||
@ -3,10 +3,10 @@ import std/tables
|
||||
|
||||
import pkg/chronos
|
||||
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/storage/rng
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/blockexchange
|
||||
|
||||
import ../../asynctest
|
||||
import ../examples
|
||||
|
||||
@ -4,9 +4,9 @@ import std/sequtils
|
||||
import pkg/unittest2
|
||||
import pkg/libp2p
|
||||
|
||||
import pkg/codex/blockexchange/peers
|
||||
import pkg/codex/blockexchange/protobuf/blockexc
|
||||
import pkg/codex/blockexchange/protobuf/presence
|
||||
import pkg/storage/blockexchange/peers
|
||||
import pkg/storage/blockexchange/protobuf/blockexc
|
||||
import pkg/storage/blockexchange/protobuf/presence
|
||||
|
||||
import ../helpers
|
||||
import ../examples
|
||||
|
||||
@ -4,8 +4,8 @@ import std/algorithm
|
||||
import pkg/chronos
|
||||
import pkg/stew/byteutils
|
||||
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/blockexchange
|
||||
|
||||
import ../helpers
|
||||
import ../../asynctest
|
||||
|
||||
@ -2,11 +2,11 @@ import std/random
|
||||
import std/sequtils
|
||||
import pkg/libp2p
|
||||
import pkg/stint
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/manifest
|
||||
import pkg/storage/rng
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/merkletree
|
||||
import pkg/storage/manifest
|
||||
import ../examples
|
||||
|
||||
export examples
|
||||
|
||||
@ -3,13 +3,13 @@ import std/sequtils
|
||||
import pkg/chronos
|
||||
import pkg/libp2p
|
||||
import pkg/libp2p/varint
|
||||
import pkg/codex/blocktype
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/utils
|
||||
import pkg/storage/blocktype
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/manifest
|
||||
import pkg/storage/merkletree
|
||||
import pkg/storage/blockexchange
|
||||
import pkg/storage/rng
|
||||
import pkg/storage/utils
|
||||
|
||||
import ./helpers/nodeutils
|
||||
import ./helpers/datasetutils
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
import std/random
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/rng
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/merkletree
|
||||
import pkg/storage/manifest
|
||||
import pkg/storage/rng
|
||||
|
||||
import ./randomchunker
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import pkg/chronos
|
||||
import pkg/codex/chunker
|
||||
import pkg/storage/chunker
|
||||
|
||||
export chunker
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import std/times
|
||||
import pkg/chronos
|
||||
import codex/clock
|
||||
import storage/clock
|
||||
|
||||
export clock
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
import pkg/chronos
|
||||
import pkg/libp2p
|
||||
import pkg/questionable
|
||||
import pkg/codex/discovery
|
||||
import pkg/storage/discovery
|
||||
import pkg/contractabi/address as ca
|
||||
|
||||
type MockDiscovery* = ref object of Discovery
|
||||
|
||||
@ -13,9 +13,9 @@ import pkg/libp2p
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
|
||||
import pkg/codex/stores/repostore
|
||||
import pkg/codex/utils/asynciter
|
||||
import pkg/codex/utils/safeasynciter
|
||||
import pkg/storage/stores/repostore
|
||||
import pkg/storage/utils/asynciter
|
||||
import pkg/storage/utils/safeasynciter
|
||||
|
||||
type MockRepoStore* = ref object of RepoStore
|
||||
delBlockCids*: seq[Cid]
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
|
||||
import pkg/chronos
|
||||
|
||||
import codex/utils/timer
|
||||
import storage/utils/timer
|
||||
|
||||
type MockTimer* = ref object of Timer
|
||||
startCalled*: int
|
||||
|
||||
@ -6,18 +6,18 @@ import pkg/taskpools
|
||||
import pkg/libp2p
|
||||
import pkg/libp2p/errors
|
||||
|
||||
import pkg/codex/discovery
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/codex/systemclock
|
||||
import pkg/codex/nat
|
||||
import pkg/codex/utils/natutils
|
||||
import pkg/codex/utils/safeasynciter
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/manifest
|
||||
import pkg/storage/discovery
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/blockexchange
|
||||
import pkg/storage/systemclock
|
||||
import pkg/storage/nat
|
||||
import pkg/storage/utils/natutils
|
||||
import pkg/storage/utils/safeasynciter
|
||||
import pkg/storage/merkletree
|
||||
import pkg/storage/manifest
|
||||
|
||||
import pkg/codex/node
|
||||
import pkg/storage/node
|
||||
|
||||
import ./datasetutils
|
||||
import ./mockdiscovery
|
||||
@ -49,7 +49,7 @@ type
|
||||
discovery*: DiscoveryEngine
|
||||
engine*: BlockExcEngine
|
||||
networkStore*: NetworkStore
|
||||
node*: CodexNodeRef = nil
|
||||
node*: StorageNodeRef = nil
|
||||
tempDbs*: seq[TempLevelDb] = @[]
|
||||
|
||||
NodesCluster* = ref object
|
||||
@ -85,7 +85,7 @@ converter toTuple*(
|
||||
converter toComponents*(cluster: NodesCluster): seq[NodesComponents] =
|
||||
cluster.components
|
||||
|
||||
proc nodes*(cluster: NodesCluster): seq[CodexNodeRef] =
|
||||
proc nodes*(cluster: NodesCluster): seq[StorageNodeRef] =
|
||||
cluster.components.filterIt(it.node != nil).mapIt(it.node)
|
||||
|
||||
proc localStores*(cluster: NodesCluster): seq[BlockStore] =
|
||||
@ -209,7 +209,7 @@ proc generateNodes*(
|
||||
|
||||
let node =
|
||||
if config.createFullNode:
|
||||
let fullNode = CodexNodeRef.new(
|
||||
let fullNode = StorageNodeRef.new(
|
||||
switch = switch,
|
||||
networkStore = networkStore,
|
||||
engine = engine,
|
||||
@ -321,7 +321,7 @@ proc linearTopology*(nodes: seq[NodesComponents]) {.async.} =
|
||||
proc downloadDataset*(
|
||||
node: NodesComponents, dataset: TestDataset
|
||||
): Future[void] {.async.} =
|
||||
# This is the same as fetchBatched, but we don't construct CodexNodes so I can't use
|
||||
# This is the same as fetchBatched, but we don't construct StorageNodes so I can't use
|
||||
# it here.
|
||||
let requestAddresses = collect:
|
||||
for i in 0 ..< dataset.manifest.blocksCount:
|
||||
|
||||
@ -2,8 +2,8 @@ import std/sequtils
|
||||
|
||||
import pkg/chronos
|
||||
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/rng
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/rng
|
||||
|
||||
export chunker
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import pkg/codex/merkletree
|
||||
import pkg/storage/merkletree
|
||||
import ../helpers
|
||||
|
||||
export merkletree, helpers
|
||||
|
||||
@ -3,7 +3,7 @@ import pkg/unittest2
|
||||
import pkg/questionable/results
|
||||
import pkg/stew/byteutils
|
||||
|
||||
import pkg/codex/merkletree
|
||||
import pkg/storage/merkletree
|
||||
import ./helpers
|
||||
|
||||
const data = [
|
||||
|
||||
@ -5,9 +5,9 @@ import pkg/questionable/results
|
||||
import pkg/stew/byteutils
|
||||
import pkg/libp2p
|
||||
|
||||
import pkg/codex/codextypes
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/utils/digest
|
||||
import pkg/storage/storagetypes
|
||||
import pkg/storage/merkletree
|
||||
import pkg/storage/utils/digest
|
||||
|
||||
import pkg/taskpools
|
||||
|
||||
|
||||
@ -3,9 +3,9 @@ import std/times
|
||||
|
||||
import pkg/libp2p
|
||||
import pkg/chronos
|
||||
import pkg/codex/codextypes
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/stores
|
||||
import pkg/storage/storagetypes
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/stores
|
||||
import pkg/taskpools
|
||||
|
||||
import ../../asynctest
|
||||
@ -71,7 +71,7 @@ template setupAndTearDown*() {.dirty.} =
|
||||
localStoreMetaDs: Datastore
|
||||
engine: BlockExcEngine
|
||||
store: NetworkStore
|
||||
node: CodexNodeRef
|
||||
node: StorageNodeRef
|
||||
blockDiscovery: Discovery
|
||||
peerStore: PeerCtxStore
|
||||
pendingBlocks: PendingBlocksManager
|
||||
@ -110,7 +110,7 @@ template setupAndTearDown*() {.dirty.} =
|
||||
localStore, network, discovery, advertiser, peerStore, pendingBlocks
|
||||
)
|
||||
store = NetworkStore.new(engine, localStore)
|
||||
node = CodexNodeRef.new(
|
||||
node = StorageNodeRef.new(
|
||||
switch = switch,
|
||||
networkStore = store,
|
||||
engine = engine,
|
||||
|
||||
@ -14,19 +14,19 @@ import pkg/taskpools
|
||||
|
||||
import pkg/codexdht/discv5/protocol as discv5
|
||||
|
||||
import pkg/codex/logutils
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/clock
|
||||
import pkg/codex/systemclock
|
||||
import pkg/codex/blockexchange
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/discovery
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/rng
|
||||
import pkg/storage/logutils
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/clock
|
||||
import pkg/storage/systemclock
|
||||
import pkg/storage/blockexchange
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/manifest
|
||||
import pkg/storage/discovery
|
||||
import pkg/storage/merkletree
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/rng
|
||||
|
||||
import pkg/codex/node {.all.}
|
||||
import pkg/storage/node {.all.}
|
||||
|
||||
import ../../asynctest
|
||||
import ../examples
|
||||
@ -36,7 +36,7 @@ import ../slots/helpers
|
||||
|
||||
import ./helpers
|
||||
|
||||
privateAccess(CodexNodeRef) # enable access to private fields
|
||||
privateAccess(StorageNodeRef) # enable access to private fields
|
||||
|
||||
asyncchecksuite "Test Node - Basic":
|
||||
setupAndTearDown()
|
||||
|
||||
@ -3,13 +3,13 @@ import std/sugar
|
||||
import pkg/chronos
|
||||
import pkg/libp2p/cid
|
||||
|
||||
import pkg/codex/codextypes
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/rng
|
||||
import pkg/storage/storagetypes
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/merkletree
|
||||
import pkg/storage/manifest
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/rng
|
||||
import pkg/taskpools
|
||||
|
||||
import ../helpers
|
||||
|
||||
@ -7,11 +7,11 @@ import pkg/libp2p/multicodec
|
||||
import pkg/stew/byteutils
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/utils
|
||||
import pkg/storage/stores/cachestore
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/manifest
|
||||
import pkg/storage/merkletree
|
||||
import pkg/storage/utils
|
||||
|
||||
import ../../asynctest
|
||||
import ../helpers
|
||||
|
||||
@ -5,9 +5,9 @@ import pkg/stew/objects
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
|
||||
import pkg/codex/clock
|
||||
import pkg/codex/stores/repostore/types
|
||||
import pkg/codex/stores/repostore/coders
|
||||
import pkg/storage/clock
|
||||
import pkg/storage/stores/repostore/types
|
||||
import pkg/storage/stores/repostore/coders
|
||||
|
||||
import ../../helpers
|
||||
|
||||
|
||||
@ -3,8 +3,8 @@ import std/strutils
|
||||
import pkg/chronos
|
||||
import pkg/stew/byteutils
|
||||
import pkg/questionable/results
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/chunker
|
||||
import pkg/storage/stores/cachestore
|
||||
import pkg/storage/chunker
|
||||
|
||||
import ./commonstoretests
|
||||
|
||||
|
||||
@ -12,9 +12,9 @@ import std/sequtils
|
||||
import pkg/chronos
|
||||
import pkg/questionable/results
|
||||
import pkg/libp2p
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/stores/repostore
|
||||
import pkg/codex/clock
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/stores/repostore
|
||||
import pkg/storage/clock
|
||||
|
||||
import ../../asynctest
|
||||
import ../helpers/mocktimer
|
||||
@ -22,8 +22,8 @@ import ../helpers/mockrepostore
|
||||
import ../helpers/mockclock
|
||||
import ../examples
|
||||
|
||||
import codex/namespaces
|
||||
import codex/stores/keyutils
|
||||
import storage/namespaces
|
||||
import storage/stores/keyutils
|
||||
|
||||
proc createManifestCid(): ?!Cid =
|
||||
let
|
||||
@ -49,7 +49,7 @@ suite "KeyUtils":
|
||||
|
||||
check:
|
||||
namespaces.len == 4
|
||||
namespaces[0].value == CodexRepoNamespace
|
||||
namespaces[0].value == StorageRepoNamespace
|
||||
namespaces[1].value == "blocks"
|
||||
namespaces[2].value == expectedPrefix
|
||||
namespaces[3].value == expectedPostfix
|
||||
@ -65,7 +65,7 @@ suite "KeyUtils":
|
||||
|
||||
check:
|
||||
namespaces.len == 4
|
||||
namespaces[0].value == CodexRepoNamespace
|
||||
namespaces[0].value == StorageRepoNamespace
|
||||
namespaces[1].value == "manifests"
|
||||
namespaces[2].value == expectedPrefix
|
||||
namespaces[3].value == expectedPostfix
|
||||
@ -78,7 +78,7 @@ suite "KeyUtils":
|
||||
|
||||
check:
|
||||
namespaces.len == 3
|
||||
namespaces[0].value == CodexMetaNamespace
|
||||
namespaces[0].value == StorageMetaNamespace
|
||||
namespaces[1].value == "ttl"
|
||||
namespaces[2].value == $cid
|
||||
|
||||
@ -88,6 +88,6 @@ suite "KeyUtils":
|
||||
|
||||
check:
|
||||
namespaces.len == 3
|
||||
namespaces[0].value == CodexMetaNamespace
|
||||
namespaces[0].value == StorageMetaNamespace
|
||||
namespaces[1].value == "ttl"
|
||||
namespaces[2].value == "*"
|
||||
|
||||
@ -8,9 +8,9 @@
|
||||
## those terms.
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/stores/repostore
|
||||
import pkg/codex/clock
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/stores/repostore
|
||||
import pkg/storage/clock
|
||||
|
||||
import ../../asynctest
|
||||
import ../helpers
|
||||
@ -19,7 +19,7 @@ import ../helpers/mockrepostore
|
||||
import ../helpers/mockclock
|
||||
import ../examples
|
||||
|
||||
import codex/stores/maintenance
|
||||
import storage/stores/maintenance
|
||||
|
||||
suite "BlockMaintainer":
|
||||
var mockRepoStore: MockRepoStore
|
||||
|
||||
@ -5,8 +5,8 @@ import pkg/questionable
|
||||
import pkg/chronos
|
||||
import pkg/datastore/typedds
|
||||
import pkg/datastore/sql/sqliteds
|
||||
import pkg/codex/stores/queryiterhelper
|
||||
import pkg/codex/utils/asynciter
|
||||
import pkg/storage/stores/queryiterhelper
|
||||
import pkg/storage/utils/asynciter
|
||||
|
||||
import ../../asynctest
|
||||
import ../helpers
|
||||
|
||||
@ -9,14 +9,14 @@ import pkg/chronos
|
||||
import pkg/stew/byteutils
|
||||
import pkg/datastore
|
||||
|
||||
import pkg/codex/stores/cachestore
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/stores/repostore/operations
|
||||
import pkg/codex/blocktype as bt
|
||||
import pkg/codex/clock
|
||||
import pkg/codex/utils/safeasynciter
|
||||
import pkg/codex/merkletree
|
||||
import pkg/storage/stores/cachestore
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/stores
|
||||
import pkg/storage/stores/repostore/operations
|
||||
import pkg/storage/blocktype as bt
|
||||
import pkg/storage/clock
|
||||
import pkg/storage/utils/safeasynciter
|
||||
import pkg/storage/merkletree
|
||||
|
||||
import ../../asynctest
|
||||
import ../helpers
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
import pkg/chronos
|
||||
import pkg/results
|
||||
|
||||
import pkg/codex/utils/asyncheapqueue
|
||||
import pkg/codex/rng
|
||||
import pkg/storage/utils/asyncheapqueue
|
||||
import pkg/storage/rng
|
||||
|
||||
import ../asynctest
|
||||
import ./helpers
|
||||
|
||||
@ -2,7 +2,7 @@ import pkg/chronos
|
||||
import pkg/chronos/transports/stream
|
||||
import pkg/chronos/transports/common
|
||||
import pkg/chronos/streams/asyncstream
|
||||
import pkg/codex/streams
|
||||
import pkg/storage/streams
|
||||
import pkg/stew/byteutils
|
||||
|
||||
import ../asynctest
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import pkg/unittest2
|
||||
import pkg/libp2p/cid
|
||||
|
||||
import pkg/codex/blocktype
|
||||
import pkg/storage/blocktype
|
||||
|
||||
import ./examples
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import pkg/stew/byteutils
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/logutils
|
||||
import pkg/storage/chunker
|
||||
import pkg/storage/logutils
|
||||
import pkg/chronos
|
||||
|
||||
import ../asynctest
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user