nim-codex/tests/dagger/blockexc/testnetwork.nim
Dmitriy Ryajov fbe161a073
Node setup (#32)
* settup basic nim node

* adding http utils

* adding confutils

* rough rest api proto

* adding missing deps

* turn tls emulation off

* adding toml serialization

* wip

* adding missing deps

* make sure to clean old state in teardown

* adding file upload rest endpoint

* renaming blockexchange to networkstore

* updating nim-presto

* updating libp2p

* wip adding streaming upload

* reworked chunking

* bump to latest unstable

* adding asyncfutures stream

* make streamable

* deleting unused files

* reworking stores api

* use new stores api

* rework blockset and remove blockstream

* don't return option from constructor

* rework chunker

* wip implement upload

* fix tests

* move unrelated logic to engine

* don't print entire message

* logging

* basic encode/decode to/from dag-pb

* add basic upload/download support

* fix tests

* renaming blockset to manifest

* don't pass config to node

* remove config and use new manifest

* wip: make endpoints more reliable

* wip: adding node tests

* include correct manifest test

* removing asyncfutures

* proper chunking of files

* simplify stream reading

* test with encoding/decoding with many blocks

* add block storing tests

* adding retrieval test

* add logging

* tidy up chunker

* tidy up manifest and node

* use default chunk size

* fix tests

* fix tests

* make sure Eof is set properly

* wip

* minor cleanup

* add file utils

* cleanup config

* splitout DaggerServer and "main"

* remove events since they are not used

* add broadcast method to network peer

* add and wire localstore

* use localstore in the node

* wip

* logging

* move file utils

* use the constant

* updating deps

* fix memstore

* use latest libp2p unstable

* fix tests

* rework block streaming

* don't fail storing if the block already exists

* add helper info endpoint

* correct comment

* rename localstore to fsstore

* fix tests

* remove unused tests

* add test to retrieve one block

* move some test files around

* consolidate setup

* Update dagger/blockexchange/engine.nim

Co-authored-by: Tanguy <tanguy@status.im>

* typo

* better block path handling

* don't inherit rootobj

* remove useless template

* Update tests/dagger/blockexc/testblockexc.nim

Co-authored-by: markspanbroek <mark@spanbroek.net>

* use isMainModule

* use proper flag for starter/stoped

* cleanup optional use

* wrap in isMainModule

* use `cancelAndAwait`

* remove unused imports

* wip

* don't use optional

* use functional error api

* rework store tests and add fs tests

* Block.new() to Block.init()

* don't use optional for engine blocks

* use result instead of optional for getBlock

* remove unused imports

* move stopping servers to `shutdown`

* use result instead of optional

* rework with results

* fix tests

* use waitFor in signal handlers

* error helper

* use `?` and mapFailure where possible

* remove unnecesary `=?`

* improve empty cid digest initialization

Co-authored-by: Tanguy <tanguy@status.im>
Co-authored-by: markspanbroek <mark@spanbroek.net>
2022-01-10 09:32:56 -06:00

267 lines
6.8 KiB
Nim

import std/sequtils
import std/tables
import pkg/asynctest
import pkg/chronos
import pkg/libp2p
import pkg/libp2p/errors
import questionable
import questionable/results
import pkg/protobuf_serialization
import pkg/dagger/rng
import pkg/dagger/chunker
import pkg/dagger/blocktype as bt
import pkg/dagger/blockexchange
import ../helpers
import ../examples
suite "NetworkStore network":
let
rng = Rng.instance()
seckey = PrivateKey.random(rng[]).tryGet()
peerId = PeerID.init(seckey.getPublicKey().tryGet()).tryGet()
chunker = RandomChunker.new(Rng.instance(), size = 1024, chunkSize = 256)
var
network: BlockExcNetwork
networkPeer: NetworkPeer
buffer: BufferStream
blocks: seq[bt.Block]
done: Future[void]
proc getConn(): Future[Connection] {.async.} =
return Connection(buffer)
setup:
while true:
let chunk = await chunker.getBytes()
if chunk.len <= 0:
break
blocks.add(bt.Block.init(chunk))
done = newFuture[void]()
buffer = BufferStream.new()
network = BlockExcNetwork.new(
switch = newStandardSwitch(),
connProvider = getConn)
network.setupPeer(peerId)
networkPeer = network.peers[peerId]
discard await networkPeer.connect()
test "Want List handler":
proc wantListHandler(peer: PeerID, wantList: WantList) {.gcsafe, async.} =
# check that we got the correct amount of entries
check wantList.entries.len == 4
for b in blocks:
check b.cid in wantList.entries
let entry = wantList.entries[wantList.entries.find(b.cid)]
check entry.wantType == WantType.wantHave
check entry.priority == 1
check entry.cancel == true
check entry.sendDontHave == true
done.complete()
network.handlers.onWantList = wantListHandler
let wantList = makeWantList(
blocks.mapIt( it.cid ),
1, true, WantType.wantHave,
true, true)
let msg = Message(wantlist: wantList)
await buffer.pushData(lenPrefix(Protobuf.encode(msg)))
await done.wait(500.millis)
test "Blocks Handler":
proc blocksHandler(peer: PeerID, blks: seq[bt.Block]) {.gcsafe, async.} =
check blks == blocks
done.complete()
network.handlers.onBlocks = blocksHandler
let msg = Message(payload: makeBlocks(blocks))
await buffer.pushData(lenPrefix(Protobuf.encode(msg)))
await done.wait(500.millis)
test "Presence Handler":
proc presenceHandler(
peer: PeerID,
precense: seq[BlockPresence]) {.gcsafe, async.} =
for b in blocks:
check:
b.cid in precense
done.complete()
network.handlers.onPresence = presenceHandler
let msg = Message(
blockPresences: blocks.mapIt(
BlockPresence(
cid: it.cid.data.buffer,
type: BlockPresenceType.presenceHave
)))
await buffer.pushData(lenPrefix(Protobuf.encode(msg)))
await done.wait(500.millis)
test "handles account messages":
let account = Account(address: EthAddress.example)
proc handleAccount(peer: PeerID, received: Account) {.gcsafe, async.} =
check received == account
done.complete()
network.handlers.onAccount = handleAccount
let message = Message(account: AccountMessage.init(account))
await buffer.pushData(lenPrefix(Protobuf.encode(message)))
await done.wait(100.millis)
test "handles payment messages":
let payment = SignedState.example
proc handlePayment(peer: PeerID, received: SignedState) {.gcsafe, async.} =
check received == payment
done.complete()
network.handlers.onPayment = handlePayment
let message = Message(payment: StateChannelUpdate.init(payment))
await buffer.pushData(lenPrefix(Protobuf.encode(message)))
await done.wait(100.millis)
suite "NetworkStore Network - e2e":
let
chunker = RandomChunker.new(Rng.instance(), size = 1024, chunkSize = 256)
var
switch1, switch2: Switch
network1, network2: BlockExcNetwork
blocks: seq[bt.Block]
done: Future[void]
setup:
while true:
let chunk = await chunker.getBytes()
if chunk.len <= 0:
break
blocks.add(bt.Block.init(chunk))
done = newFuture[void]()
switch1 = newStandardSwitch()
switch2 = newStandardSwitch()
await switch1.start()
await switch2.start()
network1 = BlockExcNetwork.new(
switch = switch1)
switch1.mount(network1)
network2 = BlockExcNetwork.new(
switch = switch2)
switch2.mount(network2)
await switch1.connect(
switch2.peerInfo.peerId,
switch2.peerInfo.addrs)
teardown:
await allFuturesThrowing(
switch1.stop(),
switch2.stop())
test "broadcast want list":
proc wantListHandler(peer: PeerID, wantList: WantList) {.gcsafe, async.} =
# check that we got the correct amount of entries
check wantList.entries.len == 4
for b in blocks:
check b.cid in wantList.entries
let entry = wantList.entries[wantList.entries.find(b.cid)]
check entry.wantType == WantType.wantHave
check entry.priority == 1
check entry.cancel == true
check entry.sendDontHave == true
done.complete()
network2.handlers.onWantList = wantListHandler
network1.broadcastWantList(
switch2.peerInfo.peerId,
blocks.mapIt( it.cid ),
1, true, WantType.wantHave,
true, true)
await done.wait(500.millis)
test "broadcast blocks":
proc blocksHandler(peer: PeerID, blks: seq[bt.Block]) {.gcsafe, async.} =
check blks == blocks
done.complete()
network2.handlers.onBlocks = blocksHandler
network1.broadcastBlocks(
switch2.peerInfo.peerId,
blocks)
await done.wait(500.millis)
test "broadcast precense":
proc presenceHandler(
peer: PeerID,
precense: seq[BlockPresence]) {.gcsafe, async.} =
for b in blocks:
check:
b.cid in precense
done.complete()
network2.handlers.onPresence = presenceHandler
network1.broadcastBlockPresence(
switch2.peerInfo.peerId,
blocks.mapIt(
BlockPresence(
cid: it.cid.data.buffer,
type: BlockPresenceType.presenceHave
)))
await done.wait(500.millis)
test "broadcasts account":
let account = Account(address: EthAddress.example)
proc handleAccount(peer: PeerID, received: Account) {.gcsafe, async.} =
check received == account
done.complete()
network2.handlers.onAccount = handleAccount
network1.broadcastAccount(switch2.peerInfo.peerId, account)
await done.wait(500.millis)
test "broadcasts payment":
let payment = SignedState.example
proc handlePayment(peer: PeerID, received: SignedState) {.gcsafe, async.} =
check received == payment
done.complete()
network2.handlers.onPayment = handlePayment
network1.broadcastPayment(switch2.peerInfo.peerId, payment)
await done.wait(500.millis)