Merge branch 'master' into feature/ceremony-files

codex/indexingstrategy.nim

@@ -49,8 +49,8 @@ func getLinearIndicies(
   self.checkIteration(iteration)
 
   let
-    first = self.firstIndex + iteration * (self.step + 1)
+    first = self.firstIndex + iteration * self.step
-    last = min(first + self.step, self.lastIndex)
+    last = min(first + self.step - 1, self.lastIndex)
 
   getIter(first, last, 1)
 
@@ -94,4 +94,4 @@ func init*(
     firstIndex: firstIndex,
     lastIndex: lastIndex,
     iterations: iterations,
-    step: divUp((lastIndex - firstIndex), iterations))
+    step: divUp((lastIndex - firstIndex + 1), iterations))

codex/manifest/manifest.nim

@@ -135,13 +135,6 @@ func isManifest*(mc: MultiCodec): ?!bool =
 # Various sizes and verification
 ############################################################
 
-func bytes*(self: Manifest, pad = true): NBytes =
-  ## Compute how many bytes corresponding StoreStream(Manifest, pad) will return
-  if pad or self.protected:
-    self.blocksCount.NBytes * self.blockSize
-  else:
-    self.datasetSize
-
 func rounded*(self: Manifest): int =
   ## Number of data blocks in *protected* manifest including padding at the end
   roundUp(self.originalBlocksCount, self.ecK)
@@ -238,7 +231,7 @@ func new*(
   treeCid: Cid,
   datasetSize: NBytes,
   ecK, ecM: int,
-  strategy: StrategyType): Manifest =
+  strategy = SteppedStrategy): Manifest =
   ## Create an erasure protected dataset from an
   ## unprotected one
   ##
@@ -284,7 +277,7 @@ func new*(
   ecM: int,
   originalTreeCid: Cid,
   originalDatasetSize: NBytes,
-  strategy: StrategyType): Manifest =
+  strategy = SteppedStrategy): Manifest =
 
   Manifest(
     treeCid: treeCid,
@@ -306,7 +299,7 @@ func new*(
   verifyRoot: Cid,
   slotRoots: openArray[Cid],
   cellSize = DefaultCellSize,
-  strategy = SteppedStrategy): ?!Manifest =
+  strategy = LinearStrategy): ?!Manifest =
   ## Create a verifiable dataset from an
   ## protected one
   ##
@@ -331,6 +324,7 @@ func new*(
     ecM: manifest.ecM,
     originalTreeCid: manifest.treeCid,
     originalDatasetSize: manifest.originalDatasetSize,
+    protectedStrategy: manifest.protectedStrategy,
     verifiable: true,
     verifyRoot: verifyRoot,
     slotRoots: @slotRoots,

codex/node.nim

@@ -534,7 +534,9 @@ proc onStore(
     trace "Unable to fetch manifest for cid", cid, err = err.msg
     return failure(err)
 
-  without builder =? Poseidon2Builder.new(self.networkStore, manifest), err:
+  without builder =? Poseidon2Builder.new(
+    self.networkStore, manifest, manifest.verifiableStrategy
+  ), err:
     trace "Unable to create slots builder", err = err.msg
     return failure(err)
 
@@ -559,8 +561,8 @@ proc onStore(
 
     return success()
 
-  without indexer =? manifest.protectedStrategy.init(
+  without indexer =? manifest.verifiableStrategy.init(
-    0, manifest.numSlotBlocks() - 1, manifest.numSlots).catch, err:
+    0, manifest.blocksCount - 1, manifest.numSlots).catch, err:
     trace "Unable to create indexing strategy from protected manifest", err = err.msg
     return failure(err)
 

codex/streams/storestream.nim

@@ -38,7 +38,6 @@ type
   StoreStream* = ref object of SeekableStream
     store*: BlockStore          # Store where to lookup block contents
     manifest*: Manifest         # List of block CIDs
-    pad*: bool                  # Pad last block to manifest.blockSize?
 
 method initStream*(s: StoreStream) =
   if s.objName.len == 0:
@@ -57,13 +56,15 @@ proc new*(
   result = StoreStream(
     store: store,
     manifest: manifest,
-    pad: pad,
     offset: 0)
 
   result.initStream()
 
 method `size`*(self: StoreStream): int =
-  bytes(self.manifest, self.pad).int
+  ## The size of a StoreStream is the size of the original dataset, without
+  ## padding or parity blocks.
+  let m = self.manifest
+  (if m.protected: m.originalDatasetSize else: m.datasetSize).int
 
 proc `size=`*(self: StoreStream, size: int)
   {.error: "Setting the size is forbidden".} =

tests/codex/node/testcontracts.nim

@@ -134,7 +134,7 @@ asyncchecksuite "Test Node - Host contracts":
       return success()
 
     (await onStore(request, 1.u256, onBlocks)).tryGet()
-    check fetchedBytes == 262144
+    check fetchedBytes == 12 * DefaultBlockSize.uint
 
     let indexer = verifiable.protectedStrategy.init(
       0, verifiable.numSlotBlocks() - 1, verifiable.numSlots)

tests/codex/testindexingstrategy.nim

@@ -58,6 +58,14 @@ suite "Indexing strategies":
     expect IndexingWrongIterationsError:
       discard LinearStrategy.init(0, 10, 0)
 
+  test "should split elements evenly when possible":
+    let
+      l = LinearStrategy.init(0, 11, 3)
+    check:
+      toSeq(l.getIndicies(0)) == @[0, 1, 2, 3].mapIt(it)
+      toSeq(l.getIndicies(1)) == @[4, 5, 6, 7].mapIt(it)
+      toSeq(l.getIndicies(2)) == @[8, 9, 10, 11].mapIt(it)
+
   test "linear - oob":
     expect IndexingError:
       discard linear.getIndicies(3)

tests/codex/testmanifest.nim

@@ -74,3 +74,36 @@ checksuite "Manifest":
   test "Should encode/decode to/from verifiable manifest":
     check:
       encodeDecode(verifiableManifest) == verifiableManifest
+
+
+suite "Manifest - Attribute Inheritance":
+  proc makeProtectedManifest(strategy: StrategyType): Manifest =
+    Manifest.new(
+      manifest = Manifest.new(
+        treeCid = Cid.example,
+        blockSize = 1.MiBs,
+        datasetSize = 100.MiBs,
+      ),
+      treeCid = Cid.example,
+      datasetSize = 200.MiBs,
+      ecK = 1,
+      ecM = 1,
+      strategy = strategy
+    )
+
+  test "Should preserve interleaving strategy for protected manifest in verifiable manifest":
+    var verifiable = Manifest.new(
+      manifest = makeProtectedManifest(SteppedStrategy),
+      verifyRoot = Cid.example,
+      slotRoots = @[Cid.example, Cid.example]
+    ).tryGet()
+
+    check verifiable.protectedStrategy == SteppedStrategy
+
+    verifiable = Manifest.new(
+      manifest = makeProtectedManifest(LinearStrategy),
+      verifyRoot = Cid.example,
+      slotRoots = @[Cid.example, Cid.example]
+    ).tryGet()
+
+    check verifiable.protectedStrategy == LinearStrategy

tests/codex/teststorestream.nim

@@ -1,12 +1,15 @@
 import pkg/chronos
 import pkg/questionable/results
 
-import pkg/codex/streams
+import pkg/codex/[
-import pkg/codex/stores
+  streams,
-import pkg/codex/manifest
+  stores,
-import pkg/codex/blocktype as bt
+  indexingstrategy,
+  manifest,
+  blocktype as bt]
 
 import ../asynctest
+import ./examples
 import ./helpers
 
 asyncchecksuite "StoreStream":
@@ -99,3 +102,40 @@ asyncchecksuite "StoreStream":
 
     await stream.readExactly(addr buf[0], 15)
     check sequentialBytes(buf,15,0)
+
+suite "StoreStream - Size Tests":
+
+  var stream: StoreStream
+
+  teardown:
+    await stream.close()
+
+  test "Should return dataset size as stream size":
+    let manifest = Manifest.new(
+      treeCid = Cid.example,
+      datasetSize = 80.NBytes,
+      blockSize = 10.NBytes
+    )
+
+    stream = StoreStream.new(CacheStore.new(), manifest)
+
+    check stream.size == 80
+
+  test "Should not count parity/padding bytes as part of stream size":
+    let protectedManifest = Manifest.new(
+      treeCid = Cid.example,
+      datasetSize = 120.NBytes, # size including parity bytes
+      blockSize = 10.NBytes,
+      version = CIDv1,
+      hcodec = Sha256HashCodec,
+      codec = BlockCodec,
+      ecK = 2,
+      ecM = 1,
+      originalTreeCid = Cid.example,
+      originalDatasetSize = 80.NBytes, # size without parity bytes
+      strategy = StrategyType.SteppedStrategy
+    )
+
+    stream = StoreStream.new(CacheStore.new(), protectedManifest)
+
+    check stream.size == 80