mirror of
https://github.com/status-im/nim-codex.git
synced 2025-01-30 12:36:26 +00:00
fbe161a073
* settup basic nim node * adding http utils * adding confutils * rough rest api proto * adding missing deps * turn tls emulation off * adding toml serialization * wip * adding missing deps * make sure to clean old state in teardown * adding file upload rest endpoint * renaming blockexchange to networkstore * updating nim-presto * updating libp2p * wip adding streaming upload * reworked chunking * bump to latest unstable * adding asyncfutures stream * make streamable * deleting unused files * reworking stores api * use new stores api * rework blockset and remove blockstream * don't return option from constructor * rework chunker * wip implement upload * fix tests * move unrelated logic to engine * don't print entire message * logging * basic encode/decode to/from dag-pb * add basic upload/download support * fix tests * renaming blockset to manifest * don't pass config to node * remove config and use new manifest * wip: make endpoints more reliable * wip: adding node tests * include correct manifest test * removing asyncfutures * proper chunking of files * simplify stream reading * test with encoding/decoding with many blocks * add block storing tests * adding retrieval test * add logging * tidy up chunker * tidy up manifest and node * use default chunk size * fix tests * fix tests * make sure Eof is set properly * wip * minor cleanup * add file utils * cleanup config * splitout DaggerServer and "main" * remove events since they are not used * add broadcast method to network peer * add and wire localstore * use localstore in the node * wip * logging * move file utils * use the constant * updating deps * fix memstore * use latest libp2p unstable * fix tests * rework block streaming * don't fail storing if the block already exists * add helper info endpoint * correct comment * rename localstore to fsstore * fix tests * remove unused tests * add test to retrieve one block * move some test files around * consolidate setup * Update dagger/blockexchange/engine.nim Co-authored-by: Tanguy <tanguy@status.im> * typo * better block path handling * don't inherit rootobj * remove useless template * Update tests/dagger/blockexc/testblockexc.nim Co-authored-by: markspanbroek <mark@spanbroek.net> * use isMainModule * use proper flag for starter/stoped * cleanup optional use * wrap in isMainModule * use `cancelAndAwait` * remove unused imports * wip * don't use optional * use functional error api * rework store tests and add fs tests * Block.new() to Block.init() * don't use optional for engine blocks * use result instead of optional for getBlock * remove unused imports * move stopping servers to `shutdown` * use result instead of optional * rework with results * fix tests * use waitFor in signal handlers * error helper * use `?` and mapFailure where possible * remove unnecesary `=?` * improve empty cid digest initialization Co-authored-by: Tanguy <tanguy@status.im> Co-authored-by: markspanbroek <mark@spanbroek.net>
267 lines
6.8 KiB
Nim
267 lines
6.8 KiB
Nim
import std/sequtils
|
|
import std/tables
|
|
|
|
import pkg/asynctest
|
|
import pkg/chronos
|
|
import pkg/libp2p
|
|
import pkg/libp2p/errors
|
|
import questionable
|
|
import questionable/results
|
|
import pkg/protobuf_serialization
|
|
|
|
import pkg/dagger/rng
|
|
import pkg/dagger/chunker
|
|
import pkg/dagger/blocktype as bt
|
|
import pkg/dagger/blockexchange
|
|
|
|
import ../helpers
|
|
import ../examples
|
|
|
|
suite "NetworkStore network":
|
|
let
|
|
rng = Rng.instance()
|
|
seckey = PrivateKey.random(rng[]).tryGet()
|
|
peerId = PeerID.init(seckey.getPublicKey().tryGet()).tryGet()
|
|
chunker = RandomChunker.new(Rng.instance(), size = 1024, chunkSize = 256)
|
|
|
|
var
|
|
network: BlockExcNetwork
|
|
networkPeer: NetworkPeer
|
|
buffer: BufferStream
|
|
blocks: seq[bt.Block]
|
|
done: Future[void]
|
|
|
|
proc getConn(): Future[Connection] {.async.} =
|
|
return Connection(buffer)
|
|
|
|
setup:
|
|
while true:
|
|
let chunk = await chunker.getBytes()
|
|
if chunk.len <= 0:
|
|
break
|
|
|
|
blocks.add(bt.Block.init(chunk))
|
|
|
|
done = newFuture[void]()
|
|
buffer = BufferStream.new()
|
|
network = BlockExcNetwork.new(
|
|
switch = newStandardSwitch(),
|
|
connProvider = getConn)
|
|
network.setupPeer(peerId)
|
|
networkPeer = network.peers[peerId]
|
|
discard await networkPeer.connect()
|
|
|
|
test "Want List handler":
|
|
proc wantListHandler(peer: PeerID, wantList: WantList) {.gcsafe, async.} =
|
|
# check that we got the correct amount of entries
|
|
check wantList.entries.len == 4
|
|
|
|
for b in blocks:
|
|
check b.cid in wantList.entries
|
|
let entry = wantList.entries[wantList.entries.find(b.cid)]
|
|
check entry.wantType == WantType.wantHave
|
|
check entry.priority == 1
|
|
check entry.cancel == true
|
|
check entry.sendDontHave == true
|
|
|
|
done.complete()
|
|
|
|
network.handlers.onWantList = wantListHandler
|
|
|
|
let wantList = makeWantList(
|
|
blocks.mapIt( it.cid ),
|
|
1, true, WantType.wantHave,
|
|
true, true)
|
|
|
|
let msg = Message(wantlist: wantList)
|
|
await buffer.pushData(lenPrefix(Protobuf.encode(msg)))
|
|
|
|
await done.wait(500.millis)
|
|
|
|
test "Blocks Handler":
|
|
proc blocksHandler(peer: PeerID, blks: seq[bt.Block]) {.gcsafe, async.} =
|
|
check blks == blocks
|
|
done.complete()
|
|
|
|
network.handlers.onBlocks = blocksHandler
|
|
|
|
let msg = Message(payload: makeBlocks(blocks))
|
|
await buffer.pushData(lenPrefix(Protobuf.encode(msg)))
|
|
|
|
await done.wait(500.millis)
|
|
|
|
test "Presence Handler":
|
|
proc presenceHandler(
|
|
peer: PeerID,
|
|
precense: seq[BlockPresence]) {.gcsafe, async.} =
|
|
for b in blocks:
|
|
check:
|
|
b.cid in precense
|
|
|
|
done.complete()
|
|
|
|
network.handlers.onPresence = presenceHandler
|
|
|
|
let msg = Message(
|
|
blockPresences: blocks.mapIt(
|
|
BlockPresence(
|
|
cid: it.cid.data.buffer,
|
|
type: BlockPresenceType.presenceHave
|
|
)))
|
|
await buffer.pushData(lenPrefix(Protobuf.encode(msg)))
|
|
|
|
await done.wait(500.millis)
|
|
|
|
test "handles account messages":
|
|
let account = Account(address: EthAddress.example)
|
|
|
|
proc handleAccount(peer: PeerID, received: Account) {.gcsafe, async.} =
|
|
check received == account
|
|
done.complete()
|
|
|
|
network.handlers.onAccount = handleAccount
|
|
|
|
let message = Message(account: AccountMessage.init(account))
|
|
await buffer.pushData(lenPrefix(Protobuf.encode(message)))
|
|
|
|
await done.wait(100.millis)
|
|
|
|
test "handles payment messages":
|
|
let payment = SignedState.example
|
|
|
|
proc handlePayment(peer: PeerID, received: SignedState) {.gcsafe, async.} =
|
|
check received == payment
|
|
done.complete()
|
|
|
|
network.handlers.onPayment = handlePayment
|
|
|
|
let message = Message(payment: StateChannelUpdate.init(payment))
|
|
await buffer.pushData(lenPrefix(Protobuf.encode(message)))
|
|
|
|
await done.wait(100.millis)
|
|
|
|
suite "NetworkStore Network - e2e":
|
|
let
|
|
chunker = RandomChunker.new(Rng.instance(), size = 1024, chunkSize = 256)
|
|
|
|
var
|
|
switch1, switch2: Switch
|
|
network1, network2: BlockExcNetwork
|
|
blocks: seq[bt.Block]
|
|
done: Future[void]
|
|
|
|
setup:
|
|
while true:
|
|
let chunk = await chunker.getBytes()
|
|
if chunk.len <= 0:
|
|
break
|
|
|
|
blocks.add(bt.Block.init(chunk))
|
|
|
|
done = newFuture[void]()
|
|
switch1 = newStandardSwitch()
|
|
switch2 = newStandardSwitch()
|
|
await switch1.start()
|
|
await switch2.start()
|
|
|
|
network1 = BlockExcNetwork.new(
|
|
switch = switch1)
|
|
switch1.mount(network1)
|
|
|
|
network2 = BlockExcNetwork.new(
|
|
switch = switch2)
|
|
switch2.mount(network2)
|
|
|
|
await switch1.connect(
|
|
switch2.peerInfo.peerId,
|
|
switch2.peerInfo.addrs)
|
|
|
|
teardown:
|
|
await allFuturesThrowing(
|
|
switch1.stop(),
|
|
switch2.stop())
|
|
|
|
test "broadcast want list":
|
|
proc wantListHandler(peer: PeerID, wantList: WantList) {.gcsafe, async.} =
|
|
# check that we got the correct amount of entries
|
|
check wantList.entries.len == 4
|
|
|
|
for b in blocks:
|
|
check b.cid in wantList.entries
|
|
let entry = wantList.entries[wantList.entries.find(b.cid)]
|
|
check entry.wantType == WantType.wantHave
|
|
check entry.priority == 1
|
|
check entry.cancel == true
|
|
check entry.sendDontHave == true
|
|
|
|
done.complete()
|
|
|
|
network2.handlers.onWantList = wantListHandler
|
|
network1.broadcastWantList(
|
|
switch2.peerInfo.peerId,
|
|
blocks.mapIt( it.cid ),
|
|
1, true, WantType.wantHave,
|
|
true, true)
|
|
|
|
await done.wait(500.millis)
|
|
|
|
test "broadcast blocks":
|
|
proc blocksHandler(peer: PeerID, blks: seq[bt.Block]) {.gcsafe, async.} =
|
|
check blks == blocks
|
|
done.complete()
|
|
|
|
network2.handlers.onBlocks = blocksHandler
|
|
network1.broadcastBlocks(
|
|
switch2.peerInfo.peerId,
|
|
blocks)
|
|
|
|
await done.wait(500.millis)
|
|
|
|
test "broadcast precense":
|
|
proc presenceHandler(
|
|
peer: PeerID,
|
|
precense: seq[BlockPresence]) {.gcsafe, async.} =
|
|
for b in blocks:
|
|
check:
|
|
b.cid in precense
|
|
|
|
done.complete()
|
|
|
|
network2.handlers.onPresence = presenceHandler
|
|
|
|
network1.broadcastBlockPresence(
|
|
switch2.peerInfo.peerId,
|
|
blocks.mapIt(
|
|
BlockPresence(
|
|
cid: it.cid.data.buffer,
|
|
type: BlockPresenceType.presenceHave
|
|
)))
|
|
|
|
await done.wait(500.millis)
|
|
|
|
test "broadcasts account":
|
|
let account = Account(address: EthAddress.example)
|
|
|
|
proc handleAccount(peer: PeerID, received: Account) {.gcsafe, async.} =
|
|
check received == account
|
|
done.complete()
|
|
|
|
network2.handlers.onAccount = handleAccount
|
|
|
|
network1.broadcastAccount(switch2.peerInfo.peerId, account)
|
|
|
|
await done.wait(500.millis)
|
|
|
|
test "broadcasts payment":
|
|
let payment = SignedState.example
|
|
|
|
proc handlePayment(peer: PeerID, received: SignedState) {.gcsafe, async.} =
|
|
check received == payment
|
|
done.complete()
|
|
|
|
network2.handlers.onPayment = handlePayment
|
|
|
|
network1.broadcastPayment(switch2.peerInfo.peerId, payment)
|
|
|
|
await done.wait(500.millis)
|