diff --git a/dagger.nimble b/dagger.nimble index c7d345ca..d794b933 100644 --- a/dagger.nimble +++ b/dagger.nimble @@ -4,7 +4,7 @@ description = "The hardrive for Web3" license = "MIT" requires "nim >= 1.2.6", - "libp2p >= 0.0.2 & < 0.1.0", + "libp2p#unstable", "nimcrypto >= 0.4.1", "bearssl >= 0.1.4", "chronicles >= 0.7.2", diff --git a/dagger/bitswap/network.nim b/dagger/bitswap/network.nim index 4867748e..f71b0098 100644 --- a/dagger/bitswap/network.nim +++ b/dagger/bitswap/network.nim @@ -13,6 +13,8 @@ import pkg/chronicles import pkg/chronos import pkg/libp2p +import pkg/questionable +import pkg/questionable/results import ../blocktype as bt import ./protobuf/bitswap as pb @@ -142,9 +144,11 @@ proc handleBlocks( var blks: seq[bt.Block] for blk in blocks: when blk is pb.Block: - blks.add(bt.Block.new(Cid.init(blk.prefix).get(), blk.data)) + if b =? bt.Block.new(Cid.init(blk.prefix).get(), blk.data): + blks.add(b) elif blk is seq[byte]: - blks.add(bt.Block.new(Cid.init(blk).get(), blk)) + if b =? bt.Block.new(Cid.init(blk).get(), blk): + blks.add(b) else: error("Invalid block type") diff --git a/dagger/blockstream.nim b/dagger/blockstream.nim deleted file mode 100644 index 96f0eac2..00000000 --- a/dagger/blockstream.nim +++ /dev/null @@ -1,29 +0,0 @@ -## Nim-Dagger -## Copyright (c) 2021 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [Defect].} - -import pkg/questionable -import pkg/questionable/results -import ./blocktype - -export blocktype - -type - BlockStreamRef* = ref object of RootObj - -method nextBlock*(b: BlockStreamRef): ?!Block {.base.} = - doAssert(false, "Not implemented!") - -iterator items*(b: BlockStreamRef): Block = - while true: - without blk =? b.nextBlock(): - break - - yield blk diff --git a/dagger/blocktype.nim b/dagger/blocktype.nim index 4e825b89..8d64fdab 100644 --- a/dagger/blocktype.nim +++ b/dagger/blocktype.nim @@ -11,11 +11,10 @@ import pkg/libp2p import pkg/questionable +import pkg/questionable/results import pkg/stew/byteutils type - CidDontMatchError* = object of CatchableError - Block* = object of RootObj cid*: Cid data*: seq[byte] @@ -24,31 +23,31 @@ proc `$`*(b: Block): string = result &= "cid: " & $b.cid result &= "\ndata: " & string.fromBytes(b.data) +proc new*( + T: type Block, + data: openArray[byte] = [], + version = CIDv1, + hcodec = multiCodec("sha2-256"), + codec = multiCodec("raw")): ?!T = + let hash = MultiHash.digest($hcodec, data).get() + success Block( + cid: Cid.init(version, codec, hash).get(), + data: @data) + proc new*( T: type Block, cid: Cid, data: openArray[byte] = [], - verify: bool = false): T = - let b = Block.new( + verify: bool = false): ?!T = + let res = Block.new( data, cid.cidver, cid.mhash.get().mcodec, cid.mcodec ) - if verify and cid != b.cid: - raise newException(CidDontMatchError, - "The suplied Cid doesn't match the data!") + if b =? res: + if verify and cid != b.cid: + return failure("The suplied Cid doesn't match the data!") - return b - -proc new*( - T: type Block, - data: openArray[byte] = [], - version = CIDv1, - hcodec = multiCodec("sha2-256"), - codec = multiCodec("raw")): T = - let hash = MultiHash.digest($hcodec, data).get() - Block( - cid: Cid.init(version, codec, hash).get(), - data: @data) + res diff --git a/dagger/chunkedblockstream.nim b/dagger/chunkedblockstream.nim deleted file mode 100644 index 49b6b6f5..00000000 --- a/dagger/chunkedblockstream.nim +++ /dev/null @@ -1,28 +0,0 @@ -## Nim-Dagger -## Copyright (c) 2021 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [Defect].} - -import pkg/questionable -import pkg/questionable/results - -import ./blockstream -import ./chunker - -type - ChunkedBlockStreamRef* = ref object of BlockStreamRef - chunker*: Chunker - -method nextBlock*(c: ChunkedBlockStreamRef): ?!Block = - let data: seq[byte] = c.chunker.getBytes() - if data.len > 0: - return success Block.new(data) - -proc new*(T: type ChunkedBlockStreamRef, chunker: Chunker): T = - T(chunker: chunker) diff --git a/dagger/chunker.nim b/dagger/chunker.nim index f9e97d66..e42fe0da 100644 --- a/dagger/chunker.nim +++ b/dagger/chunker.nim @@ -16,7 +16,7 @@ import std/sequtils import pkg/questionable import pkg/questionable/results -import ./p2p/rng +import ./rng import ./blocktype export blocktype diff --git a/dagger/dagger.nim b/dagger/dagger.nim new file mode 100644 index 00000000..ec0ba0b2 --- /dev/null +++ b/dagger/dagger.nim @@ -0,0 +1,9 @@ +## Nim-Dagger +## Copyright (c) 2021 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + diff --git a/dagger/dataset.nim b/dagger/dataset.nim index 7656bc06..48a23c03 100644 --- a/dagger/dataset.nim +++ b/dagger/dataset.nim @@ -26,10 +26,8 @@ proc hashBytes*(mh: MultiHash): seq[byte] = mh.data.buffer[mh.dpos..(mh.dpos + mh.size - 1)] proc hashBytes*(b: Block): seq[byte] = - without mh =? b.cid.mhash: - return - - mh.hashBytes() + if mh =? b.cid.mhash: + mh.hashBytes() method nextBlock*(d: BlockSetRef): ?!Block = d.stream.nextBlock() diff --git a/dagger/p2p/rng.nim b/dagger/rng.nim similarity index 100% rename from dagger/p2p/rng.nim rename to dagger/rng.nim diff --git a/dagger/stores/blockstore.nim b/dagger/stores/blockstore.nim index 8e887439..f6064976 100644 --- a/dagger/stores/blockstore.nim +++ b/dagger/stores/blockstore.nim @@ -8,10 +8,13 @@ ## those terms. import std/sequtils -import chronos + +import pkg/chronos +import pkg/libp2p + import ../blocktype -export blocktype +export blocktype, libp2p type ChangeType* {.pure.} = enum diff --git a/tests/dagger/bitswap/testbitswap.nim b/tests/dagger/bitswap/testbitswap.nim index b5ae1a4d..5822d1cd 100644 --- a/tests/dagger/bitswap/testbitswap.nim +++ b/tests/dagger/bitswap/testbitswap.nim @@ -7,7 +7,7 @@ import pkg/stew/byteutils import pkg/libp2p import pkg/libp2p/errors -import pkg/dagger/p2p/rng +import pkg/dagger/rng import pkg/dagger/bitswap import pkg/dagger/bitswap/engine/payments import pkg/dagger/stores/memorystore @@ -23,9 +23,9 @@ suite "Bitswap engine - 2 nodes": let chunker1 = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256) - blocks1 = chunker1.mapIt( bt.Block.new(it) ) + blocks1 = chunker1.mapIt( !bt.Block.new(it) ) chunker2 = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256) - blocks2 = chunker2.mapIt( bt.Block.new(it) ) + blocks2 = chunker2.mapIt( !bt.Block.new(it) ) var switch1, switch2: Switch @@ -108,7 +108,7 @@ suite "Bitswap engine - 2 nodes": check peerCtx2.account.?address == pricing2.address.some test "should send want-have for block": - let blk = bt.Block.new("Block 1".toBytes) + let blk = !bt.Block.new("Block 1".toBytes) bitswap2.engine.localStore.putBlocks(@[blk]) let entry = Entry( @@ -129,7 +129,7 @@ suite "Bitswap engine - 2 nodes": check blocks == blocks2 test "remote should send blocks when available": - let blk = bt.Block.new("Block 1".toBytes) + let blk = !bt.Block.new("Block 1".toBytes) # should fail retrieving block from remote check not await bitswap1.getBlocks(@[blk.cid]) @@ -159,7 +159,7 @@ suite "Bitswap engine - 2 nodes": suite "Bitswap - multiple nodes": let chunker = newRandomChunker(Rng.instance(), size = 4096, chunkSize = 256) - blocks = chunker.mapIt( bt.Block.new(it) ) + blocks = chunker.mapIt( !bt.Block.new(it) ) var switch: seq[Switch] diff --git a/tests/dagger/bitswap/testengine.nim b/tests/dagger/bitswap/testengine.nim index fabec292..477050af 100644 --- a/tests/dagger/bitswap/testengine.nim +++ b/tests/dagger/bitswap/testengine.nim @@ -7,7 +7,7 @@ import pkg/chronos import pkg/libp2p import pkg/libp2p/errors -import pkg/dagger/p2p/rng +import pkg/dagger/rng import pkg/dagger/bitswap import pkg/dagger/bitswap/pendingblocks import pkg/dagger/bitswap/engine/payments @@ -25,7 +25,7 @@ suite "Bitswap engine basic": seckey = PrivateKey.random(rng[]).tryGet() peerId = PeerID.init(seckey.getKey().tryGet()).tryGet() chunker = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256) - blocks = chunker.mapIt( bt.Block.new(it) ) + blocks = chunker.mapIt( !bt.Block.new(it) ) wallet = WalletRef.example var @@ -69,7 +69,6 @@ suite "Bitswap engine basic": engine.pricing = pricing.some engine.setupPeer(peerId) - await done.wait(100.millis) suite "Bitswap engine handlers": @@ -78,7 +77,7 @@ suite "Bitswap engine handlers": seckey = PrivateKey.random(rng[]).tryGet() peerId = PeerID.init(seckey.getKey().tryGet()).tryGet() chunker = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256) - blocks = chunker.mapIt( bt.Block.new(it) ) + blocks = chunker.mapIt( !bt.Block.new(it) ) wallet = WalletRef.example var @@ -190,7 +189,7 @@ suite "Task Handler": let rng = Rng.instance() chunker = newRandomChunker(Rng.instance(), size = 2048, chunkSize = 256) - blocks = chunker.mapIt( bt.Block.new(it) ) + blocks = chunker.mapIt( !bt.Block.new(it) ) wallet = WalletRef.example var @@ -251,7 +250,7 @@ suite "Task Handler": test "Should send presence": let present = blocks - let missing = @[bt.Block.new("missing".toBytes)] + let missing = @[!bt.Block.new("missing".toBytes)] let price = (!engine.pricing).price proc sendPresence(id: PeerID, presence: seq[BlockPresence]) = diff --git a/tests/dagger/bitswap/testnetwork.nim b/tests/dagger/bitswap/testnetwork.nim index 92077027..82daf3f4 100644 --- a/tests/dagger/bitswap/testnetwork.nim +++ b/tests/dagger/bitswap/testnetwork.nim @@ -10,7 +10,7 @@ import pkg/protobuf_serialization import pkg/dagger/stores/memorystore import pkg/dagger/bitswap/network import pkg/dagger/bitswap/protobuf/payments -import pkg/dagger/p2p/rng +import pkg/dagger/rng import pkg/dagger/chunker import pkg/dagger/blocktype as bt @@ -23,7 +23,7 @@ suite "Bitswap network": seckey = PrivateKey.random(rng[]).tryGet() peerId = PeerID.init(seckey.getKey().tryGet()).tryGet() chunker = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256) - blocks = chunker.mapIt( bt.Block.new(it) ) + blocks = chunker.mapIt( !bt.Block.new(it) ) var network: BitswapNetwork @@ -134,7 +134,7 @@ suite "Bitswap network": suite "Bitswap Network - e2e": let chunker = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256) - blocks = chunker.mapIt( bt.Block.new(it) ) + blocks = chunker.mapIt( !bt.Block.new(it) ) var switch1, switch2: Switch diff --git a/tests/dagger/dataset.nim b/tests/dagger/dataset.nim new file mode 100644 index 00000000..ab01ef03 --- /dev/null +++ b/tests/dagger/dataset.nim @@ -0,0 +1,20 @@ +import std/sequtils + +import pkg/chronos +import pkg/asynctest +import pkg/stew/results +import pkg/dagger/chunker +import pkg/dagger/merkletree +import pkg/stew/byteutils +import pkg/dagger/p2p/rng +import pkg/dagger/blocktype as bt + +suite "Data set": + + test "Make from Blocks": + let + chunker = newRandomChunker(Rng.instance(), size = 256*3, chunkSize = 256) + blocks = chunker.mapIt( bt.Block.new(it) ) + + let merkle = MerkleTreeRef.fromBlocks(blocks) + diff --git a/tests/dagger/examples.nim b/tests/dagger/examples.nim index d749e83f..87939e01 100644 --- a/tests/dagger/examples.nim +++ b/tests/dagger/examples.nim @@ -2,7 +2,7 @@ import std/random import std/sequtils import pkg/libp2p import pkg/nitro -import pkg/dagger/p2p/rng +import pkg/dagger/rng import pkg/dagger/bitswap/protobuf/payments import pkg/dagger/bitswap/peercontext import pkg/dagger/bitswap/engine @@ -44,7 +44,7 @@ proc example*(_: type Pricing): Pricing = proc example*(_: type Block): Block = let length = rand(4096) let bytes = newSeqWith(length, rand(uint8)) - Block.new(bytes) + !Block.new(bytes) proc example*(_: type PeerId): PeerID = let key = PrivateKey.random(Rng.instance[]).get diff --git a/tests/dagger/testasyncheapqueue.nim b/tests/dagger/testasyncheapqueue.nim index 9e2230e9..88295ffe 100644 --- a/tests/dagger/testasyncheapqueue.nim +++ b/tests/dagger/testasyncheapqueue.nim @@ -3,7 +3,7 @@ import pkg/asynctest import pkg/stew/results import pkg/dagger/utils/asyncheapqueue -import pkg/dagger/p2p/rng +import pkg/dagger/rng type Task* = tuple[name: string, priority: int] diff --git a/tests/dagger/testblockstore.nim b/tests/dagger/testblockstore.nim index 517b403b..bc5b2c04 100644 --- a/tests/dagger/testblockstore.nim +++ b/tests/dagger/testblockstore.nim @@ -1,10 +1,12 @@ import std/sequtils + import pkg/chronos import pkg/asynctest import pkg/libp2p import pkg/stew/byteutils - -import pkg/dagger/p2p/rng +import pkg/questionable +import pkg/questionable/results +import pkg/dagger/rng import pkg/dagger/stores/memorystore import pkg/dagger/chunker @@ -14,7 +16,7 @@ suite "Memory Store": var store: MemoryStore var chunker = newRandomChunker(Rng.instance(), size = 1024, chunkSize = 256) - var blocks = chunker.mapIt( Block.new(it) ) + var blocks = chunker.mapIt( !Block.new(it) ) setup: store = MemoryStore.new(blocks) @@ -40,9 +42,9 @@ suite "Memory Store": test "add blocks change handler": let blocks = @[ - Block.new("Block 1".toBytes), - Block.new("Block 2".toBytes), - Block.new("Block 3".toBytes), + !Block.new("Block 1".toBytes), + !Block.new("Block 2".toBytes), + !Block.new("Block 3".toBytes), ] var triggered = false @@ -59,9 +61,9 @@ suite "Memory Store": test "add blocks change handler": let blocks = @[ - Block.new("Block 1".toBytes), - Block.new("Block 2".toBytes), - Block.new("Block 3".toBytes), + !Block.new("Block 1".toBytes), + !Block.new("Block 2".toBytes), + !Block.new("Block 3".toBytes), ] var triggered = false