From bd8fedaf281850b8883408dfb5a56508493991d0 Mon Sep 17 00:00:00 2001 From: Ben Bierens <39762930+benbierens@users.noreply.github.com> Date: Thu, 30 May 2024 08:57:10 +0200 Subject: [PATCH] Metadata in LevelDB (#806) * pulls in datastore-leveldb update * bump * Applies LevelDb as metadata store. Adds option for repostore. * Sets submodule to main branch * I can do syntax, me * Removes wildcard from metadata query key * Applies leveldb instead of sqlite-in-memory for tests * Restores query key wildcard. * Pins nim-datastore to latest master * bumps leveldb to 0.1.4 --------- Co-authored-by: Dmitriy Ryajov --- .gitmodules | 3 ++ codex/codex.nim | 8 +++-- codex/conf.nim | 3 +- codex/sales/reservations.nim | 13 ++++++++ tests/codex/node/helpers.nim | 8 +++-- tests/codex/sales/testreservations.nim | 13 ++++++-- tests/codex/sales/testsales.nim | 22 ++++++++++---- .../codex/slots/backends/testcircomcompat.nim | 12 ++++++-- tests/codex/slots/sampler/testsampler.nim | 8 +++-- tests/codex/slots/testprover.nim | 10 +++++-- tests/codex/slots/testslotbuilder.nim | 8 +++-- tests/codex/stores/testkeyutils.nim | 2 +- tests/codex/testerasure.nim | 10 +++++-- tests/helpers.nim | 3 +- tests/helpers/templeveldb.nim | 30 +++++++++++++++++++ vendor/nim-datastore | 2 +- vendor/nim-leveldbstatic | 1 + 17 files changed, 127 insertions(+), 29 deletions(-) create mode 100644 tests/helpers/templeveldb.nim create mode 160000 vendor/nim-leveldbstatic diff --git a/.gitmodules b/.gitmodules index c5162818..06d1b823 100644 --- a/.gitmodules +++ b/.gitmodules @@ -212,3 +212,6 @@ [submodule "vendor/nim-serde"] path = vendor/nim-serde url = https://github.com/codex-storage/nim-serde.git +[submodule "vendor/nim-leveldbstatic"] + path = vendor/nim-leveldbstatic + url = https://github.com/codex-storage/nim-leveldb.git diff --git a/codex/codex.nim b/codex/codex.nim index d8edc38e..4ca6d63d 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -232,7 +232,7 @@ proc new*( let discoveryStore = Datastore( - SQLiteDatastore.new(config.dataDir / CodexDhtProvidersNamespace) + LevelDbDatastore.new(config.dataDir / CodexDhtProvidersNamespace) .expect("Should create discovery datastore!")) discovery = Discovery.new( @@ -251,11 +251,13 @@ proc new*( .expect("Should create repo file data store!")) of repoSQLite: Datastore(SQLiteDatastore.new($config.dataDir) .expect("Should create repo SQLite data store!")) + of repoLevelDb: Datastore(LevelDbDatastore.new($config.dataDir) + .expect("Should create repo LevelDB data store!")) repoStore = RepoStore.new( repoDs = repoData, - metaDs = SQLiteDatastore.new(config.dataDir / CodexMetaNamespace) - .expect("Should create meta data store!"), + metaDs = LevelDbDatastore.new(config.dataDir / CodexMetaNamespace) + .expect("Should create metadata store!"), quotaMaxBytes = config.storageQuota.uint, blockTtl = config.blockTtl) diff --git a/codex/conf.nim b/codex/conf.nim index 8ec99041..dffe4cdd 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -82,6 +82,7 @@ type RepoKind* = enum repoFS = "fs" repoSQLite = "sqlite" + repoLevelDb = "leveldb" CodexConf* = object configFile* {. @@ -190,7 +191,7 @@ type abbr: "p" }: Port repoKind* {. - desc: "Backend for main repo store (fs, sqlite)" + desc: "Backend for main repo store (fs, sqlite, leveldb)" defaultValueDesc: "fs" defaultValue: repoFS name: "repo-kind" }: RepoKind diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index c64dd806..8ba762ea 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -75,10 +75,12 @@ type repo: RepoStore onAvailabilityAdded: ?OnAvailabilityAdded GetNext* = proc(): Future[?seq[byte]] {.upraises: [], gcsafe, closure.} + IterDispose* = proc(): Future[?!void] {.gcsafe, closure.} OnAvailabilityAdded* = proc(availability: Availability): Future[void] {.upraises: [], gcsafe.} StorableIter* = ref object finished*: bool next*: GetNext + dispose*: IterDispose ReservationsError* = object of CodexError ReserveFailedError* = object of ReservationsError ReleaseFailedError* = object of ReservationsError @@ -552,7 +554,11 @@ proc storables( return none seq[byte] + proc dispose(): Future[?!void] {.async.} = + return await results.dispose() + iter.next = next + iter.dispose = dispose return success iter proc allImpl( @@ -620,6 +626,12 @@ proc findAvailability*( minPrice, availMinPrice = availability.minPrice, collateral, availMaxCollateral = availability.maxCollateral + # TODO: As soon as we're on ARC-ORC, we can use destructors + # to automatically dispose our iterators when they fall out of scope. + # For now: + if err =? (await storables.dispose()).errorOption: + error "failed to dispose storables iter", error = err.msg + return none Availability return some availability trace "availability did not match", @@ -627,3 +639,4 @@ proc findAvailability*( duration, availDuration = availability.duration, minPrice, availMinPrice = availability.minPrice, collateral, availMaxCollateral = availability.maxCollateral + diff --git a/tests/codex/node/helpers.nim b/tests/codex/node/helpers.nim index 0eec414a..ffafe5fc 100644 --- a/tests/codex/node/helpers.nim +++ b/tests/codex/node/helpers.nim @@ -87,6 +87,8 @@ template setupAndTearDown*() {.dirty.} = let path = currentSourcePath().parentDir + repoTmp = TempLevelDb.new() + metaTmp = TempLevelDb.new() setup: file = open(path /../ "" /../ "fixtures" / "test.jpg") @@ -96,8 +98,8 @@ template setupAndTearDown*() {.dirty.} = network = BlockExcNetwork.new(switch) clock = SystemClock.new() - localStoreMetaDs = SQLiteDatastore.new(Memory).tryGet() - localStoreRepoDs = SQLiteDatastore.new(Memory).tryGet() + localStoreMetaDs = metaTmp.newDb() + localStoreRepoDs = repoTmp.newDb() localStore = RepoStore.new(localStoreRepoDs, localStoreMetaDs, clock = clock) await localStore.start() @@ -124,3 +126,5 @@ template setupAndTearDown*() {.dirty.} = teardown: close(file) await node.stop() + await metaTmp.destroyDb() + await repoTmp.destroyDb() diff --git a/tests/codex/sales/testreservations.nim b/tests/codex/sales/testreservations.nim index ae15ad2f..36256ec3 100644 --- a/tests/codex/sales/testreservations.nim +++ b/tests/codex/sales/testreservations.nim @@ -17,16 +17,23 @@ asyncchecksuite "Reservations module": var repo: RepoStore repoDs: Datastore - metaDs: SQLiteDatastore + metaDs: Datastore reservations: Reservations + let + repoTmp = TempLevelDb.new() + metaTmp = TempLevelDb.new() setup: randomize(1.int64) # create reproducible results - repoDs = SQLiteDatastore.new(Memory).tryGet() - metaDs = SQLiteDatastore.new(Memory).tryGet() + repoDs = repoTmp.newDb() + metaDs = metaTmp.newDb() repo = RepoStore.new(repoDs, metaDs) reservations = Reservations.new(repo) + teardown: + await repoTmp.destroyDb() + await metaTmp.destroyDb() + proc createAvailability(): Availability = let example = Availability.example let totalSize = rand(100000..200000) diff --git a/tests/codex/sales/testsales.nim b/tests/codex/sales/testsales.nim index 4aa83e25..c3352cfa 100644 --- a/tests/codex/sales/testsales.nim +++ b/tests/codex/sales/testsales.nim @@ -22,7 +22,10 @@ import ../examples import ./helpers/periods asyncchecksuite "Sales - start": - let proof = Groth16Proof.example + let + proof = Groth16Proof.example + repoTmp = TempLevelDb.new() + metaTmp = TempLevelDb.new() var request: StorageRequest var sales: Sales @@ -50,8 +53,8 @@ asyncchecksuite "Sales - start": market = MockMarket.new() clock = MockClock.new() - let repoDs = SQLiteDatastore.new(Memory).tryGet() - let metaDs = SQLiteDatastore.new(Memory).tryGet() + let repoDs = repoTmp.newDb() + let metaDs = metaTmp.newDb() repo = RepoStore.new(repoDs, metaDs) await repo.start() sales = Sales.new(market, clock, repo) @@ -73,6 +76,8 @@ asyncchecksuite "Sales - start": teardown: await sales.stop() await repo.stop() + await repoTmp.destroyDb() + await metaTmp.destroyDb() proc fillSlot(slotIdx: UInt256 = 0.u256) {.async.} = let address = await market.getSigner() @@ -113,7 +118,10 @@ asyncchecksuite "Sales - start": check sales.agents.any(agent => agent.data.requestId == request.id and agent.data.slotIndex == 1.u256) asyncchecksuite "Sales": - let proof = Groth16Proof.example + let + proof = Groth16Proof.example + repoTmp = TempLevelDb.new() + metaTmp = TempLevelDb.new() var availability: Availability var request: StorageRequest @@ -154,8 +162,8 @@ asyncchecksuite "Sales": market.requestEnds[request.id] = request.expiry.toSecondsSince1970 clock = MockClock.new() - let repoDs = SQLiteDatastore.new(Memory).tryGet() - let metaDs = SQLiteDatastore.new(Memory).tryGet() + let repoDs = repoTmp.newDb() + let metaDs = metaTmp.newDb() repo = RepoStore.new(repoDs, metaDs) await repo.start() sales = Sales.new(market, clock, repo) @@ -177,6 +185,8 @@ asyncchecksuite "Sales": teardown: await sales.stop() await repo.stop() + await repoTmp.destroyDb() + await metaTmp.destroyDb() proc allowRequestToStart {.async.} = # wait until we're in initialproving state diff --git a/tests/codex/slots/backends/testcircomcompat.nim b/tests/codex/slots/backends/testcircomcompat.nim index 08ac2d21..99097afd 100644 --- a/tests/codex/slots/backends/testcircomcompat.nim +++ b/tests/codex/slots/backends/testcircomcompat.nim @@ -1,4 +1,3 @@ - import std/sequtils import std/sugar import std/options @@ -19,6 +18,7 @@ import pkg/codex/stores import ./helpers import ../helpers +import ../../helpers suite "Test Circom Compat Backend - control inputs": let @@ -69,6 +69,9 @@ suite "Test Circom Compat Backend": wasm = "tests/circuits/fixtures/proof_main.wasm" zkey = "tests/circuits/fixtures/proof_main.zkey" + repoTmp = TempLevelDb.new() + metaTmp = TempLevelDb.new() + var store: BlockStore manifest: Manifest @@ -82,8 +85,8 @@ suite "Test Circom Compat Backend": setup: let - repoDs = SQLiteDatastore.new(Memory).tryGet() - metaDs = SQLiteDatastore.new(Memory).tryGet() + repoDs = repoTmp.newDb() + metaDs = metaTmp.newDb() store = RepoStore.new(repoDs, metaDs) @@ -105,6 +108,9 @@ suite "Test Circom Compat Backend": teardown: circom.release() # this comes from the rust FFI + await repoTmp.destroyDb() + await metaTmp.destroyDb() + test "Should verify with correct input": var diff --git a/tests/codex/slots/sampler/testsampler.nim b/tests/codex/slots/sampler/testsampler.nim index 2ed32011..a4089409 100644 --- a/tests/codex/slots/sampler/testsampler.nim +++ b/tests/codex/slots/sampler/testsampler.nim @@ -84,6 +84,8 @@ suite "Test Sampler": entropy = 1234567.toF blockSize = DefaultBlockSize cellSize = DefaultCellSize + repoTmp = TempLevelDb.new() + metaTmp = TempLevelDb.new() var store: RepoStore @@ -94,8 +96,8 @@ suite "Test Sampler": setup: let - repoDs = SQLiteDatastore.new(Memory).tryGet() - metaDs = SQLiteDatastore.new(Memory).tryGet() + repoDs = repoTmp.newDb() + metaDs = metaTmp.newDb() store = RepoStore.new(repoDs, metaDs) @@ -112,6 +114,8 @@ suite "Test Sampler": teardown: await store.close() + await repoTmp.destroyDb() + await metaTmp.destroyDb() test "Should fail instantiating for invalid slot index": let diff --git a/tests/codex/slots/testprover.nim b/tests/codex/slots/testprover.nim index 9deed96a..179047f3 100644 --- a/tests/codex/slots/testprover.nim +++ b/tests/codex/slots/testprover.nim @@ -31,6 +31,8 @@ suite "Test Prover": numDatasetBlocks = 8 blockSize = DefaultBlockSize cellSize = DefaultCellSize + repoTmp = TempLevelDb.new() + metaTmp = TempLevelDb.new() var datasetBlocks: seq[bt.Block] @@ -42,8 +44,8 @@ suite "Test Prover": setup: let - repoDs = SQLiteDatastore.new(Memory).tryGet() - metaDs = SQLiteDatastore.new(Memory).tryGet() + repoDs = repoTmp.newDb() + metaDs = metaTmp.newDb() store = RepoStore.new(repoDs, metaDs) @@ -55,6 +57,10 @@ suite "Test Prover": blockSize, cellSize) + teardown: + await repoTmp.destroyDb() + await metaTmp.destroyDb() + test "Should sample and prove a slot": let r1cs = "tests/circuits/fixtures/proof_main.r1cs" diff --git a/tests/codex/slots/testslotbuilder.nim b/tests/codex/slots/testslotbuilder.nim index 4b38ec1a..583e6d38 100644 --- a/tests/codex/slots/testslotbuilder.nim +++ b/tests/codex/slots/testslotbuilder.nim @@ -65,6 +65,8 @@ suite "Slot builder": # empty digest emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int) + repoTmp = TempLevelDb.new() + metaTmp = TempLevelDb.new() var datasetBlocks: seq[bt.Block] @@ -77,8 +79,8 @@ suite "Slot builder": setup: let - repoDs = SQLiteDatastore.new(Memory).tryGet() - metaDs = SQLiteDatastore.new(Memory).tryGet() + repoDs = repoTmp.newDb() + metaDs = metaTmp.newDb() localStore = RepoStore.new(repoDs, metaDs) chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize) @@ -96,6 +98,8 @@ suite "Slot builder": teardown: await localStore.close() + await repoTmp.destroyDb() + await metaTmp.destroyDb() # TODO: THIS IS A BUG IN asynctest, because it doesn't release the # objects after the test is done, so we need to do it manually diff --git a/tests/codex/stores/testkeyutils.nim b/tests/codex/stores/testkeyutils.nim index c750df05..b885220f 100644 --- a/tests/codex/stores/testkeyutils.nim +++ b/tests/codex/stores/testkeyutils.nim @@ -90,4 +90,4 @@ checksuite "KeyUtils": namespaces.len == 3 namespaces[0].value == CodexMetaNamespace namespaces[1].value == "ttl" - namespaces[2].value == "*" + namespaces[2].value == "*" \ No newline at end of file diff --git a/tests/codex/testerasure.nim b/tests/codex/testerasure.nim index 39ecca88..b92f4ae3 100644 --- a/tests/codex/testerasure.nim +++ b/tests/codex/testerasure.nim @@ -28,11 +28,13 @@ suite "Erasure encode/decode": var store: BlockStore var erasure: Erasure var taskpool: Taskpool + let repoTmp = TempLevelDb.new() + let metaTmp = TempLevelDb.new() setup: let - repoDs = SQLiteDatastore.new(Memory).tryGet() - metaDs = SQLiteDatastore.new(Memory).tryGet() + repoDs = repoTmp.newDb() + metaDs = metaTmp.newDb() rng = Rng.instance() chunker = RandomChunker.new(rng, size = dataSetSize, chunkSize = BlockSize) store = RepoStore.new(repoDs, metaDs) @@ -40,6 +42,10 @@ suite "Erasure encode/decode": erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, taskpool) manifest = await storeDataGetManifest(store, chunker) + teardown: + await repoTmp.destroyDb() + await metaTmp.destroyDb() + proc encode(buffers, parity: int): Future[Manifest] {.async.} = let encoded = (await erasure.encode( diff --git a/tests/helpers.nim b/tests/helpers.nim index 76275c25..a6a6ff44 100644 --- a/tests/helpers.nim +++ b/tests/helpers.nim @@ -1,4 +1,5 @@ import helpers/multisetup import helpers/trackers +import helpers/templeveldb -export multisetup, trackers +export multisetup, trackers, templeveldb diff --git a/tests/helpers/templeveldb.nim b/tests/helpers/templeveldb.nim new file mode 100644 index 00000000..97b40553 --- /dev/null +++ b/tests/helpers/templeveldb.nim @@ -0,0 +1,30 @@ +import os +import std/monotimes +import pkg/datastore +import pkg/chronos +import pkg/questionable/results + +type + TempLevelDb* = ref object + currentPath: string + ds: LevelDbDatastore + +var number = 0 + +proc newDb*(self: TempLevelDb): Datastore = + if self.currentPath.len > 0: + raiseAssert("TempLevelDb already active.") + self.currentPath = getTempDir() / "templeveldb" / $number / $getmonotime() + inc number + createdir(self.currentPath) + self.ds = LevelDbDatastore.new(self.currentPath).tryGet() + return self.ds + +proc destroyDb*(self: TempLevelDb): Future[void] {.async.} = + if self.currentPath.len == 0: + raiseAssert("TempLevelDb not active.") + try: + (await self.ds.close()).tryGet() + finally: + removedir(self.currentPath) + self.currentPath = "" diff --git a/vendor/nim-datastore b/vendor/nim-datastore index 8a95ed9c..f4989fcc 160000 --- a/vendor/nim-datastore +++ b/vendor/nim-datastore @@ -1 +1 @@ -Subproject commit 8a95ed9c90a9ea31fc1341b92c8a9c0935368cd9 +Subproject commit f4989fcce5d74a648e7e2598a72a7b21948f4a85 diff --git a/vendor/nim-leveldbstatic b/vendor/nim-leveldbstatic new file mode 160000 index 00000000..3cb21890 --- /dev/null +++ b/vendor/nim-leveldbstatic @@ -0,0 +1 @@ +Subproject commit 3cb21890d4dc29c579d309b94f60f51ee9633a6d