Applies memory store in tests.

codex/stores/memorystore.nim

@@ -13,6 +13,7 @@ push: {.upraises: [].}
 
 import std/options
 import std/tables
+import std/lists
 
 import pkg/chronicles
 import pkg/chronos
@@ -31,10 +32,15 @@ logScope:
   topics = "codex memorystore"
 
 type
+  MemoryStoreNode = ref object
+    key: Cid
+    val: Block
+
   MemoryStore* = ref object of BlockStore
     bytesUsed*: int
     capacity*: int
-    table: Table[Cid, Block]
+    table: Table[Cid, DoublyLinkedNode[MemoryStoreNode]]
+    list: DoublyLinkedList[MemoryStoreNode]
 
 const
   MiB* = 1024 * 1024
@@ -51,7 +57,7 @@ method getBlock*(self: MemoryStore, cid: Cid): Future[?!Block] {.async.} =
     return failure (ref BlockNotFoundError)(msg: "Block not in memory store")
 
   try:
-    return success self.table[cid]
+    return success self.table[cid].value.val
   except CatchableError as exc:
     trace "Error getting block from memory store", cid, error = exc.msg
     return failure exc
@@ -65,9 +71,11 @@ method hasBlock*(self: MemoryStore, cid: Cid): Future[?!bool] {.async.} =
   return (cid in self.table).success
 
 func cids(self: MemoryStore): (iterator: Cid {.gcsafe.}) =
+  var it = self.list.head
   return iterator(): Cid =
-    for cid in self.table.keys:
-      yield cid
+    while not isNil(it):
+      yield it.value.key
+      it = it.next
 
 method listBlocks*(self: MemoryStore, blockType = BlockType.Manifest): Future[?!BlocksIter] {.async.} =
   var
@@ -128,7 +136,9 @@ func putBlockSync(self: MemoryStore, blk: Block): ?!void =
     trace "Block size is larger than free capacity", blk = blkSize, freeCapacity
     return failure("Unable to store block: Insufficient free capacity.")
 
-  self.table[blk.cid] = blk
+  let node = newDoublyLinkedNode[MemoryStoreNode](MemoryStoreNode(key: blk.cid, val: blk))
+  self.list.prepend(node)
+  self.table[blk.cid] = node
   self.bytesUsed += blkSize
   return success()
 
@@ -146,12 +156,14 @@ method delBlock*(self: MemoryStore, cid: Cid): Future[?!void] {.async.} =
     trace "Empty block, ignoring"
     return success()
 
-  without toRemove =? await self.getBlock(cid), err:
-    trace "Unable to find block to remove"
-    return failure(err)
+  if cid notin self.table:
+    return failure (ref BlockNotFoundError)(msg: "Block not in memory store")
+
+  let nodeToRemove = self.table[cid]
 
   self.table.del(cid)
-  self.bytesUsed -= toRemove.data.len
+  self.list.remove(nodeToRemove)
+  self.bytesUsed -= nodeToRemove.value.val.data.len
 
   return success()
 
@@ -165,7 +177,8 @@ func new*(
   ): MemoryStore {.raises: [Defect, ValueError].} =
 
   let store = MemoryStore(
-      table: initTable[Cid, Block](),
+      table: initTable[Cid, DoublyLinkedNode[MemoryStoreNode]](),
+      list: initDoublyLinkedList[MemoryStoreNode](),
       bytesUsed: 0,
       capacity: capacity)
 

tests/codex/blockexchange/discovery/testdiscovery.nim

@@ -194,7 +194,7 @@ suite "E2E - Multiple Nodes Discovery":
         blockDiscovery = MockDiscovery.new()
         wallet = WalletRef.example
         network = BlockExcNetwork.new(s)
-        localStore = CacheStore.new()
+        localStore = MemoryStore.new()
         peerStore = PeerCtxStore.new()
         pendingBlocks = PendingBlocksManager.new()
 

tests/codex/blockexchange/engine/testengine.nim

@@ -104,7 +104,7 @@ suite "NetworkStore engine basic":
           sendAccount: sendAccount
       ))
 
-      localStore = CacheStore.new()
+      localStore = MemoryStore.new()
       discovery = DiscoveryEngine.new(
         localStore,
         peerStore,
@@ -160,7 +160,7 @@ suite "NetworkStore engine handlers":
     peerStore = PeerCtxStore.new()
     pendingBlocks = PendingBlocksManager.new()
 
-    localStore = CacheStore.new()
+    localStore = MemoryStore.new()
     network = BlockExcNetwork()
 
     discovery = DiscoveryEngine.new(
@@ -388,7 +388,7 @@ suite "Task Handler":
     peerStore = PeerCtxStore.new()
     pendingBlocks = PendingBlocksManager.new()
 
-    localStore = CacheStore.new()
+    localStore = MemoryStore.new()
     network = BlockExcNetwork()
 
     discovery = DiscoveryEngine.new(

tests/codex/helpers/mockblockstore.nim

@@ -1,5 +1,3 @@
-import std/strutils
-
 import pkg/chronos
 import pkg/libp2p
 import pkg/questionable/results

tests/codex/storageproofs/testnetwork.nim

@@ -48,7 +48,7 @@ suite "Storage Proofs Network":
 
   setupAll:
     chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize)
-    store = MemoryStore.new(capacity = DataSetSize, chunkSize = BlockSize)
+    store = MemoryStore.new(capacity = DataSetSize)
     manifest = Manifest.new(blockSize = BlockSize).tryGet()
     (spk, ssk) = st.keyGen()
 

tests/codex/storageproofs/testpor.nim

@@ -29,7 +29,7 @@ suite "BLS PoR":
 
   setup:
     chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize)
-    store = MemoryStore.new(capacity = DataSetSize, chunkSize = BlockSize)
+    store = MemoryStore.new(capacity = DataSetSize)
     manifest = Manifest.new(blockSize = BlockSize).tryGet()
     (spk, ssk) = st.keyGen()
 
@@ -88,7 +88,7 @@ suite "Test Serialization":
 
   setupAll:
     chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize)
-    store = MemoryStore.new(capacity = DataSetSize, chunkSize = BlockSize)
+    store = MemoryStore.new(capacity = DataSetSize)
     manifest = Manifest.new(blockSize = BlockSize).tryGet()
 
     while (

tests/codex/storageproofs/teststpstore.nim

@@ -34,7 +34,7 @@ suite "Test PoR store":
 
   setupAll:
     chunker = RandomChunker.new(Rng.instance(), size = DataSetSize, chunkSize = BlockSize)
-    store = MemoryStore.new(capacity = DataSetSize, chunkSize = BlockSize)
+    store = MemoryStore.new(capacity = DataSetSize)
     manifest = Manifest.new(blockSize = BlockSize).tryGet()
     (spk, ssk) = st.keyGen()
 

tests/codex/stores/testcachestore.nim

@@ -1,5 +1,3 @@
-import std/strutils
-
 import pkg/chronos
 import pkg/asynctest
 import pkg/libp2p

tests/codex/stores/testmemorystore.nim

@@ -1,35 +1,24 @@
-import std/os
-import std/strutils
-import std/sequtils
-
-import pkg/questionable
 import pkg/questionable/results
 
 import pkg/chronos
 import pkg/asynctest
 import pkg/libp2p
-import pkg/stew/byteutils
-import pkg/stew/endians2
-import pkg/datastore
 
 import pkg/codex/stores/cachestore
 import pkg/codex/chunker
 import pkg/codex/stores
 import pkg/codex/blocktype as bt
-import pkg/codex/clock
 
 import ../helpers
-import ../helpers/mockclock
 import ./commonstoretests
 
 suite "MemoryStore":
   test "Should store initial blocks":
     let
       capacity = 100
-      chunkSize = 10
       blk = createTestBlock(10)
 
-    let store = MemoryStore.new([blk], capacity, chunkSize)
+    let store = MemoryStore.new([blk], capacity)
 
     let receivedBlk = await store.getBlock(blk.cid)
 

tests/codex/stores/testrepostore.nim

@@ -8,7 +8,6 @@ import pkg/questionable/results
 import pkg/chronos
 import pkg/asynctest
 import pkg/libp2p
-import pkg/stew/byteutils
 import pkg/stew/endians2
 import pkg/datastore
 

tests/codex/testerasure.nim

@@ -28,7 +28,7 @@ suite "Erasure encode/decode":
     rng = Rng.instance()
     chunker = RandomChunker.new(rng, size = dataSetSize, chunkSize = BlockSize)
     manifest = !Manifest.new(blockSize = BlockSize)
-    store = MemoryStore.new(capacity = (dataSetSize * 2), chunkSize = BlockSize)
+    store = MemoryStore.new(capacity = (dataSetSize * 3))
     erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider)
 
     while (

tests/codex/testnode.nim

@@ -30,7 +30,7 @@ suite "Test Node":
     switch: Switch
     wallet: WalletRef
     network: BlockExcNetwork
-    localStore: CacheStore
+    localStore: MemoryStore
     engine: BlockExcEngine
     store: NetworkStore
     node: CodexNodeRef
@@ -78,7 +78,7 @@ suite "Test Node":
     switch = newStandardSwitch()
     wallet = WalletRef.new(EthPrivateKey.random())
     network = BlockExcNetwork.new(switch)
-    localStore = CacheStore.new()
+    localStore = MemoryStore.new()
     blockDiscovery = Discovery.new(
       switch.peerInfo.privateKey,
       announceAddrs = @[MultiAddress.init("/ip4/127.0.0.1/tcp/0")

tests/codex/teststorestream.nim

@@ -38,7 +38,7 @@ suite "StoreStream":
     ]
 
   setup:
-    store = CacheStore.new()
+    store = MemoryStore.new()
     manifest = Manifest.new(blockSize = 10).tryGet()
     stream = StoreStream.new(store, manifest)