2024-01-15 16:45:04 +00:00
|
|
|
import std/os
|
|
|
|
import std/options
|
|
|
|
import std/math
|
|
|
|
import std/times
|
|
|
|
import std/sequtils
|
|
|
|
import std/importutils
|
2024-03-23 09:56:35 +00:00
|
|
|
import std/cpuinfo
|
2024-01-15 16:45:04 +00:00
|
|
|
|
|
|
|
import pkg/chronos
|
|
|
|
import pkg/stew/byteutils
|
|
|
|
import pkg/datastore
|
2024-06-26 02:38:04 +00:00
|
|
|
import pkg/datastore/typedds
|
2024-01-15 16:45:04 +00:00
|
|
|
import pkg/questionable
|
|
|
|
import pkg/questionable/results
|
|
|
|
import pkg/stint
|
|
|
|
import pkg/poseidon2
|
|
|
|
import pkg/poseidon2/io
|
2024-03-23 09:56:35 +00:00
|
|
|
import pkg/taskpools
|
2024-01-15 16:45:04 +00:00
|
|
|
|
|
|
|
import pkg/nitro
|
|
|
|
import pkg/codexdht/discv5/protocol as discv5
|
|
|
|
|
feat: create logging proxy (#663)
* implement a logging proxy
The logging proxy:
- prevents the need to import chronicles (as well as export except toJson),
- prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization,
- and handles json formatting correctly in chronicles json sinks
* Rename logging -> logutils to avoid ambiguity with common names
* clean up
* add setProperty for JsonRecord, remove nim-json-serialization conflict
* Allow specifying textlines and json format separately
Not specifying a LogFormat will apply the formatting to both textlines and json sinks.
Specifying a LogFormat will apply the formatting to only that sink.
* remove unneeded usages of std/json
We only need to import utils/json instead of std/json
* move serialization from rest/json to utils/json so it can be shared
* fix NoColors ambiguity
Was causing unit tests to fail on Windows.
* Remove nre usage to fix Windows error
Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :)
* Add logutils module doc
* Shorten logutils.formatIt for `NBytes`
Both json and textlines formatIt were not needed, and could be combined into one formatIt
* remove debug integration test config
debug output and logformat of json for integration test logs
* Use ## module doc to support docgen
* bump nim-poseidon2 to export fromBytes
Before the changes in this branch, fromBytes was likely being resolved by nim-stew, or other dependency. With the changes in this branch, that dependency was removed and fromBytes could no longer be resolved. By exporting fromBytes from nim-poseidon, the correct resolution is now happening.
* fixes to get compiling after rebasing master
* Add support for Result types being logged using formatIt
2024-01-23 07:35:03 +00:00
|
|
|
import pkg/codex/logutils
|
2024-01-15 16:45:04 +00:00
|
|
|
import pkg/codex/stores
|
|
|
|
import pkg/codex/clock
|
|
|
|
import pkg/codex/contracts
|
|
|
|
import pkg/codex/systemclock
|
|
|
|
import pkg/codex/blockexchange
|
|
|
|
import pkg/codex/chunker
|
|
|
|
import pkg/codex/slots
|
|
|
|
import pkg/codex/manifest
|
|
|
|
import pkg/codex/discovery
|
|
|
|
import pkg/codex/erasure
|
|
|
|
import pkg/codex/merkletree
|
|
|
|
import pkg/codex/blocktype as bt
|
2024-06-26 02:38:04 +00:00
|
|
|
import pkg/codex/stores/repostore/coders
|
2024-01-15 16:45:04 +00:00
|
|
|
import pkg/codex/utils/asynciter
|
2024-02-08 02:27:11 +00:00
|
|
|
import pkg/codex/indexingstrategy
|
2024-01-15 16:45:04 +00:00
|
|
|
|
|
|
|
import pkg/codex/node {.all.}
|
|
|
|
|
2024-01-29 20:03:51 +00:00
|
|
|
import ../../asynctest
|
2024-01-15 16:45:04 +00:00
|
|
|
import ../../examples
|
|
|
|
import ../helpers
|
|
|
|
import ../helpers/mockmarket
|
|
|
|
import ../helpers/mockclock
|
|
|
|
|
|
|
|
import ./helpers
|
|
|
|
|
|
|
|
privateAccess(CodexNodeRef) # enable access to private fields
|
|
|
|
|
|
|
|
asyncchecksuite "Test Node - Host contracts":
|
|
|
|
setupAndTearDown()
|
|
|
|
|
|
|
|
var
|
|
|
|
sales: Sales
|
|
|
|
purchasing: Purchasing
|
|
|
|
manifest: Manifest
|
|
|
|
manifestCidStr: string
|
|
|
|
manifestCid: Cid
|
|
|
|
market: MockMarket
|
2024-02-08 02:27:11 +00:00
|
|
|
builder: Poseidon2Builder
|
2024-01-15 16:45:04 +00:00
|
|
|
verifiable: Manifest
|
|
|
|
verifiableBlock: bt.Block
|
|
|
|
protected: Manifest
|
|
|
|
|
|
|
|
setup:
|
|
|
|
# Setup Host Contracts and dependencies
|
|
|
|
market = MockMarket.new()
|
|
|
|
sales = Sales.new(market, clock, localStore)
|
|
|
|
|
|
|
|
node.contracts = (
|
|
|
|
none ClientInteractions,
|
|
|
|
some HostInteractions.new(clock, sales),
|
|
|
|
none ValidatorInteractions)
|
|
|
|
|
|
|
|
await node.start()
|
|
|
|
|
|
|
|
# Populate manifest in local store
|
|
|
|
manifest = await storeDataGetManifest(localStore, chunker)
|
|
|
|
let
|
|
|
|
manifestBlock = bt.Block.new(
|
|
|
|
manifest.encode().tryGet(),
|
|
|
|
codec = ManifestCodec).tryGet()
|
2024-03-23 09:56:35 +00:00
|
|
|
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, taskpool)
|
2024-01-15 16:45:04 +00:00
|
|
|
|
|
|
|
manifestCid = manifestBlock.cid
|
|
|
|
manifestCidStr = $(manifestCid)
|
|
|
|
|
|
|
|
(await localStore.putBlock(manifestBlock)).tryGet()
|
|
|
|
|
|
|
|
protected = (await erasure.encode(manifest, 3, 2)).tryGet()
|
2024-02-08 02:27:11 +00:00
|
|
|
builder = Poseidon2Builder.new(localStore, protected).tryGet()
|
2024-01-15 16:45:04 +00:00
|
|
|
verifiable = (await builder.buildManifest()).tryGet()
|
|
|
|
verifiableBlock = bt.Block.new(
|
|
|
|
verifiable.encode().tryGet(),
|
|
|
|
codec = ManifestCodec).tryGet()
|
|
|
|
|
|
|
|
(await localStore.putBlock(verifiableBlock)).tryGet()
|
|
|
|
|
|
|
|
test "onExpiryUpdate callback is set":
|
|
|
|
check sales.onExpiryUpdate.isSome
|
|
|
|
|
|
|
|
test "onExpiryUpdate callback":
|
|
|
|
let
|
|
|
|
# The blocks have set default TTL, so in order to update it we have to have larger TTL
|
2024-02-16 22:12:16 +00:00
|
|
|
expectedExpiry: SecondsSince1970 = clock.now + DefaultBlockTtl.seconds + 11123
|
2024-01-15 16:45:04 +00:00
|
|
|
expiryUpdateCallback = !sales.onExpiryUpdate
|
|
|
|
|
|
|
|
(await expiryUpdateCallback(manifestCidStr, expectedExpiry)).tryGet()
|
|
|
|
|
|
|
|
for index in 0..<manifest.blocksCount:
|
|
|
|
let
|
|
|
|
blk = (await localStore.getBlock(manifest.treeCid, index)).tryGet
|
2024-06-26 02:38:04 +00:00
|
|
|
key = (createBlockExpirationMetadataKey(blk.cid)).tryGet
|
|
|
|
bytes = (await localStoreMetaDs.get(key)).tryGet
|
|
|
|
blkMd = BlockMetadata.decode(bytes).tryGet
|
2024-01-15 16:45:04 +00:00
|
|
|
|
2024-06-26 02:38:04 +00:00
|
|
|
check blkMd.expiry == expectedExpiry
|
2024-01-15 16:45:04 +00:00
|
|
|
|
|
|
|
test "onStore callback is set":
|
|
|
|
check sales.onStore.isSome
|
|
|
|
|
|
|
|
test "onStore callback":
|
|
|
|
let onStore = !sales.onStore
|
|
|
|
var request = StorageRequest.example
|
|
|
|
request.content.cid = $verifiableBlock.cid
|
|
|
|
request.expiry = (getTime() + DefaultBlockTtl.toTimesDuration + 1.hours).toUnix.u256
|
|
|
|
var fetchedBytes: uint = 0
|
|
|
|
|
|
|
|
let onBlocks = proc(blocks: seq[bt.Block]): Future[?!void] {.async.} =
|
|
|
|
for blk in blocks:
|
|
|
|
fetchedBytes += blk.data.len.uint
|
|
|
|
return success()
|
|
|
|
|
|
|
|
(await onStore(request, 1.u256, onBlocks)).tryGet()
|
2024-06-26 02:38:04 +00:00
|
|
|
check fetchedBytes == 12 * DefaultBlockSize.uint
|
2024-01-15 16:45:04 +00:00
|
|
|
|
2024-02-08 02:27:11 +00:00
|
|
|
let indexer = verifiable.protectedStrategy.init(
|
2024-02-19 02:19:59 +00:00
|
|
|
0, verifiable.numSlotBlocks() - 1, verifiable.numSlots)
|
2024-02-08 02:27:11 +00:00
|
|
|
|
|
|
|
for index in indexer.getIndicies(1):
|
2024-01-15 16:45:04 +00:00
|
|
|
let
|
|
|
|
blk = (await localStore.getBlock(verifiable.treeCid, index)).tryGet
|
2024-06-26 02:38:04 +00:00
|
|
|
key = (createBlockExpirationMetadataKey(blk.cid)).tryGet
|
|
|
|
bytes = (await localStoreMetaDs.get(key)).tryGet
|
|
|
|
blkMd = BlockMetadata.decode(bytes).tryGet
|
2024-01-15 16:45:04 +00:00
|
|
|
|
2024-06-26 02:38:04 +00:00
|
|
|
check blkMd.expiry == request.expiry.toSecondsSince1970
|