[repostore] Retrieve empty blocks (#513)

Add handling of empty blocks in the RepoStore.

* Add empty block handling to repostore for put, del, has
Also added tests for all empty block handling blockstore operations. This showed there was an ambiguous identifier present for `hasBlock`, so one of the two `hasBlock` definitions was removed in `repostore`.

* Change CacheStore to RepoStore in testerasure
As CacheStore is not used in the node, update the Datastore used in the erasure coding tests to be a RepoStore. This ensures that the K > 1 cases are being tested, where they will produce empty padding blocks in the erasure-coded manifests.
This commit is contained in:
Eric 2023-08-21 12:51:04 +10:00 committed by GitHub
parent ba41b5a232
commit 9cecb68520
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 45 additions and 12 deletions

View File

@ -85,6 +85,10 @@ method getBlock*(self: RepoStore, cid: Cid): Future[?!Block] {.async.} =
## Get a block from the blockstore
##
if cid.isEmpty:
trace "Empty block, ignoring"
return success cid.emptyBlock
without key =? makePrefixKey(self.postFixLen, cid), err:
trace "Error getting key from provider", err = err.msg
return failure(err)
@ -132,6 +136,10 @@ method putBlock*(
## Put a block to the blockstore
##
if blk.isEmpty:
trace "Empty block, ignoring"
return success()
without key =? makePrefixKey(self.postFixLen, blk.cid), err:
trace "Error getting key from provider", err = err.msg
return failure(err)
@ -205,6 +213,10 @@ method delBlock*(self: RepoStore, cid: Cid): Future[?!void] {.async.} =
trace "Deleting block", cid
if cid.isEmpty:
trace "Empty block, ignoring"
return success()
if blk =? (await self.getBlock(cid)):
if key =? makePrefixKey(self.postFixLen, cid) and
err =? (await self.repoDs.delete(key)).errorOption:
@ -233,6 +245,10 @@ method hasBlock*(self: RepoStore, cid: Cid): Future[?!bool] {.async.} =
## Check if the block exists in the blockstore
##
if cid.isEmpty:
trace "Empty block, ignoring"
return true.success
without key =? makePrefixKey(self.postFixLen, cid), err:
trace "Error getting key from provider", err = err.msg
return failure(err)
@ -320,17 +336,6 @@ method close*(self: RepoStore): Future[void] {.async.} =
(await self.repoDs.close()).expect("Should close datastore")
proc hasBlock*(self: RepoStore, cid: Cid): Future[?!bool] {.async.} =
## Check if the block exists in the blockstore.
## Return false if error encountered
##
without key =? makePrefixKey(self.postFixLen, cid), err:
trace "Error getting key from provider", err = err.msg
return failure(err.msg)
return await self.repoDs.has(key)
proc reserve*(self: RepoStore, bytes: uint): Future[?!void] {.async.} =
## Reserve bytes
##

View File

@ -19,6 +19,7 @@ import pkg/codex/clock
import ../helpers
import ../helpers/mockclock
import ../examples
import ./commonstoretests
checksuite "Test RepoStore start/stop":
@ -283,6 +284,28 @@ asyncchecksuite "RepoStore":
check blockExpirations2.len == 1
assertExpiration(blockExpirations2[0], blk3)
test "should put empty blocks":
let blk = Cid.example.emptyBlock
check (await repo.putBlock(blk)).isOk
test "should get empty blocks":
let blk = Cid.example.emptyBlock
let got = await repo.getBlock(blk.cid)
check got.isOk
check got.get.cid == blk.cid
test "should delete empty blocks":
let blk = Cid.example.emptyBlock
check (await repo.delBlock(blk.cid)).isOk
test "should have empty block":
let blk = Cid.example.emptyBlock
let has = await repo.hasBlock(blk.cid)
check has.isOk
check has.get
commonBlockStoreTests(
"RepoStore Sql backend", proc: BlockStore =
BlockStore(

View File

@ -2,6 +2,7 @@ import std/sequtils
import pkg/asynctest
import pkg/chronos
import pkg/datastore
import pkg/questionable/results
import pkg/codex/erasure
@ -21,12 +22,16 @@ asyncchecksuite "Erasure encode/decode":
var manifest: Manifest
var store: BlockStore
var erasure: Erasure
var repoDs: Datastore
var metaDs: SQLiteDatastore
setup:
rng = Rng.instance()
chunker = RandomChunker.new(rng, size = dataSetSize, chunkSize = BlockSize)
manifest = !Manifest.new(blockSize = BlockSize)
store = CacheStore.new(cacheSize = (dataSetSize * 2), chunkSize = BlockSize)
repoDs = SQLiteDatastore.new(Memory).tryGet()
metaDs = SQLiteDatastore.new(Memory).tryGet()
store = RepoStore.new(repoDs, metaDs)
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider)
while (