From 7023bf586baf55dc4fc607c79770b7a152455b3e Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 13 Mar 2023 16:16:45 +0100 Subject: [PATCH] Applies memory store in tests. --- codex/stores/memorystore.nim | 33 +++++++++++++------ .../blockexchange/discovery/testdiscovery.nim | 2 +- .../codex/blockexchange/engine/testengine.nim | 6 ++-- tests/codex/helpers/mockblockstore.nim | 2 -- tests/codex/storageproofs/testnetwork.nim | 2 +- tests/codex/storageproofs/testpor.nim | 4 +-- tests/codex/storageproofs/teststpstore.nim | 2 +- tests/codex/stores/testcachestore.nim | 2 -- tests/codex/stores/testmemorystore.nim | 13 +------- tests/codex/stores/testrepostore.nim | 1 - tests/codex/testerasure.nim | 2 +- tests/codex/testnode.nim | 4 +-- tests/codex/teststorestream.nim | 2 +- 13 files changed, 36 insertions(+), 39 deletions(-) diff --git a/codex/stores/memorystore.nim b/codex/stores/memorystore.nim index c3f40bea..f1b828bc 100644 --- a/codex/stores/memorystore.nim +++ b/codex/stores/memorystore.nim @@ -13,6 +13,7 @@ push: {.upraises: [].} import std/options import std/tables +import std/lists import pkg/chronicles import pkg/chronos @@ -31,10 +32,15 @@ logScope: topics = "codex memorystore" type + MemoryStoreNode = ref object + key: Cid + val: Block + MemoryStore* = ref object of BlockStore bytesUsed*: int capacity*: int - table: Table[Cid, Block] + table: Table[Cid, DoublyLinkedNode[MemoryStoreNode]] + list: DoublyLinkedList[MemoryStoreNode] const MiB* = 1024 * 1024 @@ -51,7 +57,7 @@ method getBlock*(self: MemoryStore, cid: Cid): Future[?!Block] {.async.} = return failure (ref BlockNotFoundError)(msg: "Block not in memory store") try: - return success self.table[cid] + return success self.table[cid].value.val except CatchableError as exc: trace "Error getting block from memory store", cid, error = exc.msg return failure exc @@ -65,9 +71,11 @@ method hasBlock*(self: MemoryStore, cid: Cid): Future[?!bool] {.async.} = return (cid in self.table).success func cids(self: MemoryStore): (iterator: Cid {.gcsafe.}) = + var it = self.list.head return iterator(): Cid = - for cid in self.table.keys: - yield cid + while not isNil(it): + yield it.value.key + it = it.next method listBlocks*(self: MemoryStore, blockType = BlockType.Manifest): Future[?!BlocksIter] {.async.} = var @@ -128,7 +136,9 @@ func putBlockSync(self: MemoryStore, blk: Block): ?!void = trace "Block size is larger than free capacity", blk = blkSize, freeCapacity return failure("Unable to store block: Insufficient free capacity.") - self.table[blk.cid] = blk + let node = newDoublyLinkedNode[MemoryStoreNode](MemoryStoreNode(key: blk.cid, val: blk)) + self.list.prepend(node) + self.table[blk.cid] = node self.bytesUsed += blkSize return success() @@ -146,12 +156,14 @@ method delBlock*(self: MemoryStore, cid: Cid): Future[?!void] {.async.} = trace "Empty block, ignoring" return success() - without toRemove =? await self.getBlock(cid), err: - trace "Unable to find block to remove" - return failure(err) + if cid notin self.table: + return failure (ref BlockNotFoundError)(msg: "Block not in memory store") + + let nodeToRemove = self.table[cid] self.table.del(cid) - self.bytesUsed -= toRemove.data.len + self.list.remove(nodeToRemove) + self.bytesUsed -= nodeToRemove.value.val.data.len return success() @@ -165,7 +177,8 @@ func new*( ): MemoryStore {.raises: [Defect, ValueError].} = let store = MemoryStore( - table: initTable[Cid, Block](), + table: initTable[Cid, DoublyLinkedNode[MemoryStoreNode]](), + list: initDoublyLinkedList[MemoryStoreNode](), bytesUsed: 0, capacity: capacity) diff --git a/tests/codex/blockexchange/discovery/testdiscovery.nim b/tests/codex/blockexchange/discovery/testdiscovery.nim index 5beed798..ee398fd5 100644 --- a/tests/codex/blockexchange/discovery/testdiscovery.nim +++ b/tests/codex/blockexchange/discovery/testdiscovery.nim @@ -194,7 +194,7 @@ suite "E2E - Multiple Nodes Discovery": blockDiscovery = MockDiscovery.new() wallet = WalletRef.example network = BlockExcNetwork.new(s) - localStore = CacheStore.new() + localStore = MemoryStore.new() peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() diff --git a/tests/codex/blockexchange/engine/testengine.nim b/tests/codex/blockexchange/engine/testengine.nim index 8e04d7a6..891b724b 100644 --- a/tests/codex/blockexchange/engine/testengine.nim +++ b/tests/codex/blockexchange/engine/testengine.nim @@ -104,7 +104,7 @@ suite "NetworkStore engine basic": sendAccount: sendAccount )) - localStore = CacheStore.new() + localStore = MemoryStore.new() discovery = DiscoveryEngine.new( localStore, peerStore, @@ -160,7 +160,7 @@ suite "NetworkStore engine handlers": peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() - localStore = CacheStore.new() + localStore = MemoryStore.new() network = BlockExcNetwork() discovery = DiscoveryEngine.new( @@ -388,7 +388,7 @@ suite "Task Handler": peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() - localStore = CacheStore.new() + localStore = MemoryStore.new() network = BlockExcNetwork() discovery = DiscoveryEngine.new( diff --git a/tests/codex/helpers/mockblockstore.nim b/tests/codex/helpers/mockblockstore.nim index db1143e9..474de2a5 100644 --- a/tests/codex/helpers/mockblockstore.nim +++ b/tests/codex/helpers/mockblockstore.nim @@ -1,5 +1,3 @@ -import std/strutils - import pkg/chronos import pkg/libp2p import pkg/questionable/results diff --git a/tests/codex/storageproofs/testnetwork.nim b/tests/codex/storageproofs/testnetwork.nim index 0fa040c3..76861077 100644 --- a/tests/codex/storageproofs/testnetwork.nim +++ b/tests/codex/storageproofs/testnetwork.nim @@ -48,7 +48,7 @@ suite "Storage Proofs Network": setupAll: chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) - store = MemoryStore.new(capacity = DataSetSize, chunkSize = BlockSize) + store = MemoryStore.new(capacity = DataSetSize) manifest = Manifest.new(blockSize = BlockSize).tryGet() (spk, ssk) = st.keyGen() diff --git a/tests/codex/storageproofs/testpor.nim b/tests/codex/storageproofs/testpor.nim index a0602978..f7f2e49c 100644 --- a/tests/codex/storageproofs/testpor.nim +++ b/tests/codex/storageproofs/testpor.nim @@ -29,7 +29,7 @@ suite "BLS PoR": setup: chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) - store = MemoryStore.new(capacity = DataSetSize, chunkSize = BlockSize) + store = MemoryStore.new(capacity = DataSetSize) manifest = Manifest.new(blockSize = BlockSize).tryGet() (spk, ssk) = st.keyGen() @@ -88,7 +88,7 @@ suite "Test Serialization": setupAll: chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) - store = MemoryStore.new(capacity = DataSetSize, chunkSize = BlockSize) + store = MemoryStore.new(capacity = DataSetSize) manifest = Manifest.new(blockSize = BlockSize).tryGet() while ( diff --git a/tests/codex/storageproofs/teststpstore.nim b/tests/codex/storageproofs/teststpstore.nim index a36d5ad6..6fa126d4 100644 --- a/tests/codex/storageproofs/teststpstore.nim +++ b/tests/codex/storageproofs/teststpstore.nim @@ -34,7 +34,7 @@ suite "Test PoR store": setupAll: chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize) - store = MemoryStore.new(capacity = DataSetSize, chunkSize = BlockSize) + store = MemoryStore.new(capacity = DataSetSize) manifest = Manifest.new(blockSize = BlockSize).tryGet() (spk, ssk) = st.keyGen() diff --git a/tests/codex/stores/testcachestore.nim b/tests/codex/stores/testcachestore.nim index f7a3e078..20fb5e41 100644 --- a/tests/codex/stores/testcachestore.nim +++ b/tests/codex/stores/testcachestore.nim @@ -1,5 +1,3 @@ -import std/strutils - import pkg/chronos import pkg/asynctest import pkg/libp2p diff --git a/tests/codex/stores/testmemorystore.nim b/tests/codex/stores/testmemorystore.nim index 7a32875f..cfb9f209 100644 --- a/tests/codex/stores/testmemorystore.nim +++ b/tests/codex/stores/testmemorystore.nim @@ -1,35 +1,24 @@ -import std/os -import std/strutils -import std/sequtils - -import pkg/questionable import pkg/questionable/results import pkg/chronos import pkg/asynctest import pkg/libp2p -import pkg/stew/byteutils -import pkg/stew/endians2 -import pkg/datastore import pkg/codex/stores/cachestore import pkg/codex/chunker import pkg/codex/stores import pkg/codex/blocktype as bt -import pkg/codex/clock import ../helpers -import ../helpers/mockclock import ./commonstoretests suite "MemoryStore": test "Should store initial blocks": let capacity = 100 - chunkSize = 10 blk = createTestBlock(10) - let store = MemoryStore.new([blk], capacity, chunkSize) + let store = MemoryStore.new([blk], capacity) let receivedBlk = await store.getBlock(blk.cid) diff --git a/tests/codex/stores/testrepostore.nim b/tests/codex/stores/testrepostore.nim index a3c97a84..49a4b892 100644 --- a/tests/codex/stores/testrepostore.nim +++ b/tests/codex/stores/testrepostore.nim @@ -8,7 +8,6 @@ import pkg/questionable/results import pkg/chronos import pkg/asynctest import pkg/libp2p -import pkg/stew/byteutils import pkg/stew/endians2 import pkg/datastore diff --git a/tests/codex/testerasure.nim b/tests/codex/testerasure.nim index 77c6e5f3..f1d4e002 100644 --- a/tests/codex/testerasure.nim +++ b/tests/codex/testerasure.nim @@ -28,7 +28,7 @@ suite "Erasure encode/decode": rng = Rng.instance() chunker = RandomChunker.new(rng, size = dataSetSize, chunkSize = BlockSize) manifest = !Manifest.new(blockSize = BlockSize) - store = MemoryStore.new(capacity = (dataSetSize * 2), chunkSize = BlockSize) + store = MemoryStore.new(capacity = (dataSetSize * 3)) erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider) while ( diff --git a/tests/codex/testnode.nim b/tests/codex/testnode.nim index 3c01db35..313a48fa 100644 --- a/tests/codex/testnode.nim +++ b/tests/codex/testnode.nim @@ -30,7 +30,7 @@ suite "Test Node": switch: Switch wallet: WalletRef network: BlockExcNetwork - localStore: CacheStore + localStore: MemoryStore engine: BlockExcEngine store: NetworkStore node: CodexNodeRef @@ -78,7 +78,7 @@ suite "Test Node": switch = newStandardSwitch() wallet = WalletRef.new(EthPrivateKey.random()) network = BlockExcNetwork.new(switch) - localStore = CacheStore.new() + localStore = MemoryStore.new() blockDiscovery = Discovery.new( switch.peerInfo.privateKey, announceAddrs = @[MultiAddress.init("/ip4/127.0.0.1/tcp/0") diff --git a/tests/codex/teststorestream.nim b/tests/codex/teststorestream.nim index 91b44acc..00e34c8e 100644 --- a/tests/codex/teststorestream.nim +++ b/tests/codex/teststorestream.nim @@ -38,7 +38,7 @@ suite "StoreStream": ] setup: - store = CacheStore.new() + store = MemoryStore.new() manifest = Manifest.new(blockSize = 10).tryGet() stream = StoreStream.new(store, manifest)