reorg files and fix tests

dagger.nimble

@@ -4,7 +4,7 @@ description = "The hardrive for Web3"
 license = "MIT"
 
 requires "nim >= 1.2.6",
-         "libp2p >= 0.0.2 & < 0.1.0",
+         "libp2p#unstable",
          "nimcrypto >= 0.4.1",
          "bearssl >= 0.1.4",
          "chronicles >= 0.7.2",

dagger/bitswap/network.nim

@@ -13,6 +13,8 @@ import pkg/chronicles
 import pkg/chronos
 
 import pkg/libp2p
+import pkg/questionable
+import pkg/questionable/results
 
 import ../blocktype as bt
 import ./protobuf/bitswap as pb
@@ -142,9 +144,11 @@ proc handleBlocks(
   var blks: seq[bt.Block]
   for blk in blocks:
     when blk is pb.Block:
-      blks.add(bt.Block.new(Cid.init(blk.prefix).get(), blk.data))
+      if b =? bt.Block.new(Cid.init(blk.prefix).get(), blk.data):
+        blks.add(b)
     elif blk is seq[byte]:
-      blks.add(bt.Block.new(Cid.init(blk).get(), blk))
+      if b =? bt.Block.new(Cid.init(blk).get(), blk):
+        blks.add(b)
     else:
       error("Invalid block type")
 

dagger/blockstream.nim

@@ -1,29 +0,0 @@
-## Nim-Dagger
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
-
-{.push raises: [Defect].}
-
-import pkg/questionable
-import pkg/questionable/results
-import ./blocktype
-
-export blocktype
-
-type
-  BlockStreamRef* = ref object of RootObj
-
-method nextBlock*(b: BlockStreamRef): ?!Block {.base.} =
-  doAssert(false, "Not implemented!")
-
-iterator items*(b: BlockStreamRef): Block =
-  while true:
-    without blk =? b.nextBlock():
-      break
-
-    yield blk

dagger/blocktype.nim

@@ -11,11 +11,10 @@
 
 import pkg/libp2p
 import pkg/questionable
+import pkg/questionable/results
 import pkg/stew/byteutils
 
 type
-  CidDontMatchError* = object of CatchableError
-
   Block* = object of RootObj
     cid*: Cid
     data*: seq[byte]
@@ -24,31 +23,31 @@ proc `$`*(b: Block): string =
   result &= "cid: " & $b.cid
   result &= "\ndata: " & string.fromBytes(b.data)
 
+proc new*(
+  T: type Block,
+  data: openArray[byte] = [],
+  version = CIDv1,
+  hcodec = multiCodec("sha2-256"),
+  codec = multiCodec("raw")): ?!T =
+  let hash =  MultiHash.digest($hcodec, data).get()
+  success Block(
+    cid: Cid.init(version, codec, hash).get(),
+    data: @data)
+
 proc new*(
   T: type Block,
   cid: Cid,
   data: openArray[byte] = [],
-  verify: bool = false): T =
-  let b = Block.new(
+  verify: bool = false): ?!T =
+  let res = Block.new(
     data,
     cid.cidver,
     cid.mhash.get().mcodec,
     cid.mcodec
   )
 
-  if verify and cid != b.cid:
-    raise newException(CidDontMatchError,
-      "The suplied Cid doesn't match the data!")
+  if b =? res:
+    if verify and cid != b.cid:
+      return failure("The suplied Cid doesn't match the data!")
 
-  return b
-
-proc new*(
-  T: type Block,
-  data: openArray[byte] = [],
-  version = CIDv1,
-  hcodec = multiCodec("sha2-256"),
-  codec = multiCodec("raw")): T =
-  let hash =  MultiHash.digest($hcodec, data).get()
-  Block(
-    cid: Cid.init(version, codec, hash).get(),
-    data: @data)
+  res

dagger/chunkedblockstream.nim

@@ -1,28 +0,0 @@
-## Nim-Dagger
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
-
-{.push raises: [Defect].}
-
-import pkg/questionable
-import pkg/questionable/results
-
-import ./blockstream
-import ./chunker
-
-type
-  ChunkedBlockStreamRef* = ref object of BlockStreamRef
-    chunker*: Chunker
-
-method nextBlock*(c: ChunkedBlockStreamRef): ?!Block =
-  let data: seq[byte] = c.chunker.getBytes()
-  if data.len > 0:
-    return success Block.new(data)
-
-proc new*(T: type ChunkedBlockStreamRef, chunker: Chunker): T =
-  T(chunker: chunker)

dagger/chunker.nim

@@ -16,7 +16,7 @@ import std/sequtils
 import pkg/questionable
 import pkg/questionable/results
 
-import ./p2p/rng
+import ./rng
 import ./blocktype
 
 export blocktype

dagger/dagger.nim

@@ -0,0 +1,9 @@
+## Nim-Dagger
+## Copyright (c) 2021 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+

dagger/dataset.nim

@@ -26,10 +26,8 @@ proc hashBytes*(mh: MultiHash): seq[byte] =
   mh.data.buffer[mh.dpos..(mh.dpos + mh.size - 1)]
 
 proc hashBytes*(b: Block): seq[byte] =
-  without mh =? b.cid.mhash:
-    return
-
-  mh.hashBytes()
+  if mh =? b.cid.mhash:
+    mh.hashBytes()
 
 method nextBlock*(d: BlockSetRef): ?!Block =
   d.stream.nextBlock()

dagger/stores/blockstore.nim

@@ -8,10 +8,13 @@
 ## those terms.
 
 import std/sequtils
-import chronos
+
+import pkg/chronos
+import pkg/libp2p
+
 import ../blocktype
 
-export blocktype
+export blocktype, libp2p
 
 type
   ChangeType* {.pure.} = enum

tests/dagger/bitswap/testbitswap.nim

@@ -7,7 +7,7 @@ import pkg/stew/byteutils
 import pkg/libp2p
 import pkg/libp2p/errors
 
-import pkg/dagger/p2p/rng
+import pkg/dagger/rng
 import pkg/dagger/bitswap
 import pkg/dagger/bitswap/engine/payments
 import pkg/dagger/stores/memorystore
@@ -23,9 +23,9 @@ suite "Bitswap engine - 2 nodes":
 
   let
     chunker1 = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256)
-    blocks1 = chunker1.mapIt( bt.Block.new(it) )
+    blocks1 = chunker1.mapIt( !bt.Block.new(it) )
     chunker2 = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256)
-    blocks2 = chunker2.mapIt( bt.Block.new(it) )
+    blocks2 = chunker2.mapIt( !bt.Block.new(it) )
 
   var
     switch1, switch2: Switch
@@ -108,7 +108,7 @@ suite "Bitswap engine - 2 nodes":
     check peerCtx2.account.?address == pricing2.address.some
 
   test "should send want-have for block":
-    let blk = bt.Block.new("Block 1".toBytes)
+    let blk = !bt.Block.new("Block 1".toBytes)
     bitswap2.engine.localStore.putBlocks(@[blk])
 
     let entry = Entry(
@@ -129,7 +129,7 @@ suite "Bitswap engine - 2 nodes":
     check blocks == blocks2
 
   test "remote should send blocks when available":
-    let blk = bt.Block.new("Block 1".toBytes)
+    let blk = !bt.Block.new("Block 1".toBytes)
 
     # should fail retrieving block from remote
     check not await bitswap1.getBlocks(@[blk.cid])
@@ -159,7 +159,7 @@ suite "Bitswap engine - 2 nodes":
 suite "Bitswap - multiple nodes":
   let
     chunker = newRandomChunker(Rng.instance(), size = 4096, chunkSize = 256)
-    blocks = chunker.mapIt( bt.Block.new(it) )
+    blocks = chunker.mapIt( !bt.Block.new(it) )
 
   var
     switch: seq[Switch]

tests/dagger/bitswap/testengine.nim

@@ -7,7 +7,7 @@ import pkg/chronos
 import pkg/libp2p
 import pkg/libp2p/errors
 
-import pkg/dagger/p2p/rng
+import pkg/dagger/rng
 import pkg/dagger/bitswap
 import pkg/dagger/bitswap/pendingblocks
 import pkg/dagger/bitswap/engine/payments
@@ -25,7 +25,7 @@ suite "Bitswap engine basic":
     seckey = PrivateKey.random(rng[]).tryGet()
     peerId = PeerID.init(seckey.getKey().tryGet()).tryGet()
     chunker = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256)
-    blocks = chunker.mapIt( bt.Block.new(it) )
+    blocks = chunker.mapIt( !bt.Block.new(it) )
     wallet = WalletRef.example
 
   var
@@ -69,7 +69,6 @@ suite "Bitswap engine basic":
     engine.pricing = pricing.some
 
     engine.setupPeer(peerId)
-
     await done.wait(100.millis)
 
 suite "Bitswap engine handlers":
@@ -78,7 +77,7 @@ suite "Bitswap engine handlers":
     seckey = PrivateKey.random(rng[]).tryGet()
     peerId = PeerID.init(seckey.getKey().tryGet()).tryGet()
     chunker = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256)
-    blocks = chunker.mapIt( bt.Block.new(it) )
+    blocks = chunker.mapIt( !bt.Block.new(it) )
     wallet = WalletRef.example
 
   var
@@ -190,7 +189,7 @@ suite "Task Handler":
   let
     rng = Rng.instance()
     chunker = newRandomChunker(Rng.instance(), size = 2048, chunkSize = 256)
-    blocks = chunker.mapIt( bt.Block.new(it) )
+    blocks = chunker.mapIt( !bt.Block.new(it) )
     wallet = WalletRef.example
 
   var
@@ -251,7 +250,7 @@ suite "Task Handler":
 
   test "Should send presence":
     let present = blocks
-    let missing = @[bt.Block.new("missing".toBytes)]
+    let missing = @[!bt.Block.new("missing".toBytes)]
     let price = (!engine.pricing).price
 
     proc sendPresence(id: PeerID, presence: seq[BlockPresence]) =

tests/dagger/bitswap/testnetwork.nim

@@ -10,7 +10,7 @@ import pkg/protobuf_serialization
 import pkg/dagger/stores/memorystore
 import pkg/dagger/bitswap/network
 import pkg/dagger/bitswap/protobuf/payments
-import pkg/dagger/p2p/rng
+import pkg/dagger/rng
 import pkg/dagger/chunker
 import pkg/dagger/blocktype as bt
 
@@ -23,7 +23,7 @@ suite "Bitswap network":
     seckey = PrivateKey.random(rng[]).tryGet()
     peerId = PeerID.init(seckey.getKey().tryGet()).tryGet()
     chunker = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256)
-    blocks = chunker.mapIt( bt.Block.new(it) )
+    blocks = chunker.mapIt( !bt.Block.new(it) )
 
   var
     network: BitswapNetwork
@@ -134,7 +134,7 @@ suite "Bitswap network":
 suite "Bitswap Network - e2e":
   let
     chunker = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256)
-    blocks = chunker.mapIt( bt.Block.new(it) )
+    blocks = chunker.mapIt( !bt.Block.new(it) )
 
   var
     switch1, switch2: Switch

tests/dagger/dataset.nim

@@ -0,0 +1,20 @@
+import std/sequtils
+
+import pkg/chronos
+import pkg/asynctest
+import pkg/stew/results
+import pkg/dagger/chunker
+import pkg/dagger/merkletree
+import pkg/stew/byteutils
+import pkg/dagger/p2p/rng
+import pkg/dagger/blocktype as bt
+
+suite "Data set":
+
+  test "Make from Blocks":
+    let
+      chunker = newRandomChunker(Rng.instance(), size = 256*3, chunkSize = 256)
+      blocks = chunker.mapIt( bt.Block.new(it) )
+
+    let merkle = MerkleTreeRef.fromBlocks(blocks)
+

tests/dagger/examples.nim

@@ -2,7 +2,7 @@ import std/random
 import std/sequtils
 import pkg/libp2p
 import pkg/nitro
-import pkg/dagger/p2p/rng
+import pkg/dagger/rng
 import pkg/dagger/bitswap/protobuf/payments
 import pkg/dagger/bitswap/peercontext
 import pkg/dagger/bitswap/engine
@@ -44,7 +44,7 @@ proc example*(_: type Pricing): Pricing =
 proc example*(_: type Block): Block =
   let length = rand(4096)
   let bytes = newSeqWith(length, rand(uint8))
-  Block.new(bytes)
+  !Block.new(bytes)
 
 proc example*(_: type PeerId): PeerID =
   let key = PrivateKey.random(Rng.instance[]).get

tests/dagger/testasyncheapqueue.nim

@@ -3,7 +3,7 @@ import pkg/asynctest
 import pkg/stew/results
 
 import pkg/dagger/utils/asyncheapqueue
-import pkg/dagger/p2p/rng
+import pkg/dagger/rng
 
 type
   Task* = tuple[name: string, priority: int]

tests/dagger/testblockstore.nim

@@ -1,10 +1,12 @@
 import std/sequtils
+
 import pkg/chronos
 import pkg/asynctest
 import pkg/libp2p
 import pkg/stew/byteutils
-
-import pkg/dagger/p2p/rng
+import pkg/questionable
+import pkg/questionable/results
+import pkg/dagger/rng
 import pkg/dagger/stores/memorystore
 import pkg/dagger/chunker
 
@@ -14,7 +16,7 @@ suite "Memory Store":
 
   var store: MemoryStore
   var chunker = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256)
-  var blocks = chunker.mapIt( Block.new(it) )
+  var blocks = chunker.mapIt( !Block.new(it) )
 
   setup:
     store = MemoryStore.new(blocks)
@@ -40,9 +42,9 @@ suite "Memory Store":
 
   test "add blocks change handler":
     let blocks = @[
-      Block.new("Block 1".toBytes),
-      Block.new("Block 2".toBytes),
-      Block.new("Block 3".toBytes),
+      !Block.new("Block 1".toBytes),
+      !Block.new("Block 2".toBytes),
+      !Block.new("Block 3".toBytes),
     ]
 
     var triggered = false
@@ -59,9 +61,9 @@ suite "Memory Store":
 
   test "add blocks change handler":
     let blocks = @[
-      Block.new("Block 1".toBytes),
-      Block.new("Block 2".toBytes),
-      Block.new("Block 3".toBytes),
+      !Block.new("Block 1".toBytes),
+      !Block.new("Block 2".toBytes),
+      !Block.new("Block 3".toBytes),
     ]
 
     var triggered = false