Make block a ref (#56)

* cleanup manifest

* make block a ref type

dagger/blockexchange/network.nim

@@ -145,7 +145,7 @@ proc handleBlocks(
     without cid =? Cid.init(blob.prefix):
       trace "Unable to initialize Cid from protobuf message"
 
-    without blk =? bt.Block.init(cid, blob.data, verify = true):
+    without blk =? bt.Block.new(cid, blob.data, verify = true):
       trace "Unable to initialize Block from data"
 
     blks.add(blk)

dagger/blocktype.nim

@@ -20,7 +20,7 @@ const
   BlockSize* = 4096 # file chunk read size
 
 type
-  Block* = object of RootObj
+  Block* = ref object of RootObj
     cid*: Cid
     data*: seq[byte]
 
@@ -28,7 +28,7 @@ proc `$`*(b: Block): string =
   result &= "cid: " & $b.cid
   result &= "\ndata: " & string.fromBytes(b.data)
 
-func init*(
+func new*(
   T: type Block,
   data: openArray[byte] = [],
   version = CIDv1,
@@ -45,7 +45,7 @@ func init*(
     cid: cid,
     data: @data).success
 
-func init*(
+func new*(
   T: type Block,
   cid: Cid,
   data: openArray[byte],
@@ -53,13 +53,13 @@ func init*(
 
   let
     mhash = ? cid.mhash.mapFailure
-    b = ? Block.init(
+    b = ? Block.new(
       data = @data,
       version = cid.cidver,
       codec = cid.mcodec,
       mcodec = mhash.mcodec)
 
   if verify and cid != b.cid:
-    return failure "Cid's don't match!"
+    return "Cid's don't match!".failure
 
   success b

dagger/node.nim

@@ -130,7 +130,7 @@ proc store*(
       chunk.len > 0):
 
       trace "Got data from stream", len = chunk.len
-      without blk =? bt.Block.init(chunk):
+      without blk =? bt.Block.new(chunk):
         return failure("Unable to init block from chunk!")
 
       blockManifest.add(blk.cid)
@@ -151,7 +151,7 @@ proc store*(
       newException(DaggerError, "Could not generate dataset manifest!"))
 
   # Store as a dag-pb block
-  without manifest =? bt.Block.init(data = data, codec = DagPBCodec):
+  without manifest =? bt.Block.new(data = data, codec = DagPBCodec):
     trace "Unable to init block from manifest data!"
     return failure("Unable to init block from manifest data!")
 

dagger/stores/fsstore.nim

@@ -54,7 +54,7 @@ method getBlock*(
     trace "Cannot read file from fs store", path , error
     return Block.failure("Cannot read file from fs store")
 
-  return Block.init(cid, data)
+  return Block.new(cid, data)
 
 method putBlock*(
   self: FSStore,

tests/dagger/blockexc/testblockexc.nim

@@ -41,14 +41,14 @@ suite "NetworkStore engine - 2 nodes":
       if chunk.len <= 0:
         break
 
-      blocks1.add(bt.Block.init(chunk).tryGet())
+      blocks1.add(bt.Block.new(chunk).tryGet())
 
     while true:
       let chunk = await chunker2.getBytes()
       if chunk.len <= 0:
         break
 
-      blocks2.add(bt.Block.init(chunk).tryGet())
+      blocks2.add(bt.Block.new(chunk).tryGet())
 
     switch1 = newStandardSwitch()
     switch2 = newStandardSwitch()
@@ -119,7 +119,7 @@ suite "NetworkStore engine - 2 nodes":
     check peerCtx2.account.?address == pricing2.address.some
 
   test "should send want-have for block":
-    let blk = bt.Block.init("Block 1".toBytes).tryGet()
+    let blk = bt.Block.new("Block 1".toBytes).tryGet()
     check await blockexc2.engine.localStore.putBlock(blk)
 
     let entry = Entry(
@@ -144,7 +144,7 @@ suite "NetworkStore engine - 2 nodes":
     check blocks.mapIt( !it.read ) == blocks2
 
   test "remote should send blocks when available":
-    let blk = bt.Block.init("Block 1".toBytes).tryGet()
+    let blk = bt.Block.new("Block 1".toBytes).tryGet()
 
     # should fail retrieving block from remote
     check not await blockexc1.getBlock(blk.cid)
@@ -183,7 +183,7 @@ suite "NetworkStore - multiple nodes":
       if chunk.len <= 0:
         break
 
-      blocks.add(bt.Block.init(chunk).tryGet())
+      blocks.add(bt.Block.new(chunk).tryGet())
 
     for e in generateNodes(5):
       switch.add(e.switch)

tests/dagger/blockexc/testengine.nim

@@ -34,7 +34,7 @@ suite "NetworkStore engine basic":
       if chunk.len <= 0:
         break
 
-      blocks.add(bt.Block.init(chunk).tryGet())
+      blocks.add(bt.Block.new(chunk).tryGet())
 
     done = newFuture[void]()
 
@@ -103,7 +103,7 @@ suite "NetworkStore engine handlers":
       if chunk.len <= 0:
         break
 
-      blocks.add(bt.Block.init(chunk).tryGet())
+      blocks.add(bt.Block.new(chunk).tryGet())
 
     done = newFuture[void]()
     engine = BlockExcEngine.new(CacheStore.new(), wallet, BlockExcNetwork())
@@ -227,7 +227,7 @@ suite "Task Handler":
       if chunk.len <= 0:
         break
 
-      blocks.add(bt.Block.init(chunk).tryGet())
+      blocks.add(bt.Block.new(chunk).tryGet())
 
     done = newFuture[void]()
     engine = BlockExcEngine.new(CacheStore.new(), wallet, BlockExcNetwork())
@@ -281,7 +281,7 @@ suite "Task Handler":
 
   test "Should send presence":
     let present = blocks
-    let missing = @[bt.Block.init("missing".toBytes).tryGet()]
+    let missing = @[bt.Block.new("missing".toBytes).tryGet()]
     let price = (!engine.pricing).price
 
     proc sendPresence(id: PeerID, presence: seq[BlockPresence]) =

tests/dagger/blockexc/testnetwork.nim

@@ -38,7 +38,7 @@ suite "NetworkStore network":
       if chunk.len <= 0:
         break
 
-      blocks.add(bt.Block.init(chunk).tryGet())
+      blocks.add(bt.Block.new(chunk).tryGet())
 
     done = newFuture[void]()
     buffer = BufferStream.new()
@@ -154,7 +154,7 @@ suite "NetworkStore Network - e2e":
       if chunk.len <= 0:
         break
 
-      blocks.add(bt.Block.init(chunk).tryGet())
+      blocks.add(bt.Block.new(chunk).tryGet())
 
     done = newFuture[void]()
     switch1 = newStandardSwitch()
@@ -215,7 +215,7 @@ suite "NetworkStore Network - e2e":
 
     await done.wait(500.millis)
 
-  test "broadcast precense":
+  test "broadcast presence":
     proc presenceHandler(
       peer: PeerID,
       precense: seq[BlockPresence]) {.gcsafe, async.} =

tests/dagger/examples.nim

@@ -42,7 +42,7 @@ proc example*(_: type Pricing): Pricing =
 proc example*(_: type Block): Block =
   let length = rand(4096)
   let bytes = newSeqWith(length, rand(uint8))
-  Block.init(bytes).tryGet()
+  Block.new(bytes).tryGet()
 
 proc example*(_: type PeerId): PeerID =
   let key = PrivateKey.random(Rng.instance[]).get

tests/dagger/helpers.nim

@@ -1,10 +1,18 @@
+import pkg/libp2p
 import pkg/libp2p/varint
+import pkg/dagger/blocktype
 
 import ./helpers/nodeutils
 import ./helpers/randomchunker
 
 export randomchunker, nodeutils
 
+# NOTE: The meaning of equality for blocks
+# is changed here, because blocks are now `ref`
+# types. This is only in tests!!!
+func `==`*(a, b: Block): bool =
+  (a.cid == b.cid) and (a.data == b.data)
+
 proc lenPrefix*(msg: openArray[byte]): seq[byte] =
   ## Write `msg` with a varint-encoded length prefix
   ##

tests/dagger/stores/testcachestore.nim

@@ -16,10 +16,10 @@ suite "Cache Store tests":
     store: CacheStore
 
   setup:
-    newBlock = Block.init("New Kids on the Block".toBytes()).tryGet()
-    newBlock1 = Block.init("1".repeat(100).toBytes()).tryGet()
-    newBlock2 = Block.init("2".repeat(100).toBytes()).tryGet()
-    newBlock3 = Block.init("3".repeat(100).toBytes()).tryGet()
+    newBlock = Block.new("New Kids on the Block".toBytes()).tryGet()
+    newBlock1 = Block.new("1".repeat(100).toBytes()).tryGet()
+    newBlock2 = Block.new("2".repeat(100).toBytes()).tryGet()
+    newBlock3 = Block.new("3".repeat(100).toBytes()).tryGet()
     store = CacheStore.new()
 
   test "constructor":

tests/dagger/stores/testfsstore.nim

@@ -21,7 +21,7 @@ suite "FS Store":
   var
     store: FSStore
     repoDir: string
-    newBlock = Block.init("New Block".toBytes()).tryGet()
+    newBlock = Block.new("New Block".toBytes()).tryGet()
 
   setup:
     repoDir = path.parentDir / "repo"

tests/dagger/testmanifest.nim

@@ -17,13 +17,13 @@ suite "Manifest":
   test "Should produce valid tree hash checksum":
     var manifest = Manifest.new(
         blocks = @[
-            Block.init("Block 1".toBytes).tryGet().cid,
-            Block.init("Block 2".toBytes).tryGet().cid,
-            Block.init("Block 3".toBytes).tryGet().cid,
-            Block.init("Block 4".toBytes).tryGet().cid,
-            Block.init("Block 5".toBytes).tryGet().cid,
-            Block.init("Block 6".toBytes).tryGet().cid,
-            Block.init("Block 7".toBytes).tryGet().cid,
+            Block.new("Block 1".toBytes).tryGet().cid,
+            Block.new("Block 2".toBytes).tryGet().cid,
+            Block.new("Block 3".toBytes).tryGet().cid,
+            Block.new("Block 4".toBytes).tryGet().cid,
+            Block.new("Block 5".toBytes).tryGet().cid,
+            Block.new("Block 6".toBytes).tryGet().cid,
+            Block.new("Block 7".toBytes).tryGet().cid,
           ]).tryGet()
 
     let
@@ -42,7 +42,7 @@ suite "Manifest":
   test "Should encode/decode to/from manifest":
     let
       blocks = (0..<1000).mapIt(
-        Block.init(("Block " & $it).toBytes).tryGet().cid
+        Block.new(("Block " & $it).toBytes).tryGet().cid
       )
 
     var

tests/dagger/testnode.nim

@@ -63,7 +63,7 @@ suite "Test Node":
         let chunk = await chunker.getBytes();
         chunk.len > 0):
         await stream.pushData(chunk)
-        manifest.add(bt.Block.init(chunk).tryGet().cid)
+        manifest.add(bt.Block.new(chunk).tryGet().cid)
     finally:
       await stream.pushEof()
       await stream.close()
@@ -92,14 +92,14 @@ suite "Test Node":
       chunk.len > 0):
 
       let
-        blk = bt.Block.init(chunk).tryGet()
+        blk = bt.Block.new(chunk).tryGet()
 
       original &= chunk
       check await localStore.putBlock(blk)
       manifest.add(blk.cid)
 
     let
-      manifestBlock = bt.Block.init(
+      manifestBlock = bt.Block.new(
         manifest.encode().tryGet(),
         codec = DagPBCodec)
         .tryGet()
@@ -126,7 +126,7 @@ suite "Test Node":
   test "Retrieve One Block":
     let
       testString = "Block 1"
-      blk = bt.Block.init(testString.toBytes).tryGet()
+      blk = bt.Block.new(testString.toBytes).tryGet()
 
     var
       stream = BufferStream.new()

tests/dagger/teststorestream.nim

@@ -38,7 +38,7 @@ suite "StoreStream":
 
     for d in data:
       let
-        blk = Block.init(d).tryGet()
+        blk = Block.new(d).tryGet()
 
       manifest.add(blk.cid)
       if not (await store.putBlock(blk)):