Merge branch 'master' into feat/chronos-v4

This commit is contained in:
gmega 2024-01-29 20:57:32 -03:00
commit 92fc374fe3
No known key found for this signature in database
GPG Key ID: FFD8DAF00660270F
50 changed files with 93 additions and 75 deletions

View File

@ -76,30 +76,31 @@ The following options are available:
--metrics Enable the metrics server [=false].
--metrics-address Listening address of the metrics server [=127.0.0.1].
--metrics-port Listening HTTP port of the metrics server [=8008].
-d, --data-dir The directory where codex will store configuration and data..
-d, --data-dir The directory where codex will store configuration and data.
-i, --listen-addrs Multi Addresses to listen on [=/ip4/0.0.0.0/tcp/0].
-a, --nat IP Addresses to announce behind a NAT [=127.0.0.1].
-e, --disc-ip Discovery listen address [=0.0.0.0].
-u, --disc-port Discovery (UDP) port [=8090].
--net-privkey Source of network (secp256k1) private key file path or name [=key].
-b, --bootstrap-node Specifies one or more bootstrap nodes to use when connecting to the network..
-b, --bootstrap-node Specifies one or more bootstrap nodes to use when connecting to the network.
--max-peers The maximum number of peers to connect to [=160].
--agent-string Node agent string which is used as identifier in network [=Codex].
--api-bindaddr The REST API bind address [=127.0.0.1].
-p, --api-port The REST Api port [=8080].
--repo-kind backend for main repo store (fs, sqlite) [=fs].
--repo-kind Backend for main repo store (fs, sqlite) [=fs].
-q, --storage-quota The size of the total storage quota dedicated to the node [=8589934592].
-t, --block-ttl Default block timeout in seconds - 0 disables the ttl [=$DefaultBlockTtl].
--block-mi Time interval in seconds - determines frequency of block maintenance cycle: how
often blocks are checked for expiration and cleanup.
often blocks are checked for expiration and cleanup
[=$DefaultBlockMaintenanceInterval].
--block-mn Number of blocks to check every maintenance cycle. [=1000].
-c, --cache-size The size in MiB of the block cache, 0 disables the cache - might help on slow
hardrives [=0].
--block-mn Number of blocks to check every maintenance cycle [=1000].
-c, --cache-size The size of the block cache, 0 disables the cache - might help on slow hardrives
[=0].
--persistence Enables persistence mechanism, requires an Ethereum node [=false].
--eth-provider The URL of the JSON-RPC API of the Ethereum node [=ws://localhost:8545].
--eth-account The Ethereum account that is used for storage contracts [=EthAddress.none].
--eth-deployment The json file describing the contract deployment [=string.none].
--eth-private-key File containing Ethereum private key for storage contracts [=string.none].
--marketplace-address Address of deployed Marketplace contract [=EthAddress.none].
--validator Enables validator, requires an Ethereum node [=false].
--validator-max-slots Maximum number of slots that the validator monitors [=1000].

View File

@ -5,7 +5,7 @@
"hostOS": "macosx",
"items": {
"asynctest": {
"commit": "fe1a34caf572b05f8bdba3b650f1871af9fce31e",
"commit": "8e2f4e73b97123be0f0041c129942b32df23ecb1",
"dir": "vendor/asynctest",
"url": "https://github.com/codex-storage/asynctest"
},

View File

@ -7,7 +7,7 @@ srcDir = "."
installFiles = @["build.nims"]
requires "nim >= 1.2.0"
requires "asynctest >= 0.4.3 & < 0.5.0"
requires "asynctest >= 0.5.1 & < 0.6.0"
requires "bearssl >= 0.1.4"
requires "chronicles >= 0.7.2"
requires "chronos#head"

View File

@ -105,7 +105,7 @@ type
name: "metrics-port" }: Port
dataDir* {.
desc: "The directory where codex will store configuration and data."
desc: "The directory where codex will store configuration and data"
defaultValue: defaultDataDir()
defaultValueDesc: ""
abbr: "d"
@ -154,7 +154,7 @@ type
name: "net-privkey" }: string
bootstrapNodes* {.
desc: "Specifies one or more bootstrap nodes to use when connecting to the network."
desc: "Specifies one or more bootstrap nodes to use when connecting to the network"
abbr: "b"
name: "bootstrap-node" }: seq[SignedPeerRecord]
@ -182,7 +182,7 @@ type
abbr: "p" }: Port
repoKind* {.
desc: "backend for main repo store (fs, sqlite)"
desc: "Backend for main repo store (fs, sqlite)"
defaultValueDesc: "fs"
defaultValue: repoFS
name: "repo-kind" }: RepoKind
@ -202,13 +202,13 @@ type
abbr: "t" }: Duration
blockMaintenanceInterval* {.
desc: "Time interval in seconds - determines frequency of block maintenance cycle: how often blocks are checked for expiration and cleanup."
desc: "Time interval in seconds - determines frequency of block maintenance cycle: how often blocks are checked for expiration and cleanup"
defaultValue: DefaultBlockMaintenanceInterval
defaultValueDesc: $DefaultBlockMaintenanceInterval
name: "block-mi" }: Duration
blockMaintenanceNumberOfBlocks* {.
desc: "Number of blocks to check every maintenance cycle."
desc: "Number of blocks to check every maintenance cycle"
defaultValue: DefaultNumberOfBlocksToMaintainPerInterval
defaultValueDesc: $DefaultNumberOfBlocksToMaintainPerInterval
name: "block-mn" }: int

3
tests/asynctest.nim Normal file
View File

@ -0,0 +1,3 @@
import pkg/asynctest/chronos/unittest
export unittest

View File

@ -2,7 +2,6 @@ import std/sequtils
import std/sugar
import std/tables
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/libp2p/errors
@ -15,11 +14,12 @@ import pkg/codex/manifest
import pkg/codex/merkletree
import pkg/codex/blocktype as bt
import ../../helpers/mockdiscovery
import ../../../asynctest
import ../../helpers
import ../../helpers/mockdiscovery
import ../../examples
asyncchecksuite "Block Advertising and Discovery":
let chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256)

View File

@ -2,8 +2,6 @@ import std/sequtils
import std/sugar
import std/tables
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/codex/rng
@ -13,9 +11,9 @@ import pkg/codex/chunker
import pkg/codex/blocktype as bt
import pkg/codex/blockexchange/engine
import ../../helpers/mockdiscovery
import ../../../asynctest
import ../../helpers
import ../../helpers/mockdiscovery
import ../../examples
asyncchecksuite "Test Discovery Engine":

View File

@ -1,7 +1,6 @@
import std/sequtils
import std/algorithm
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/stew/byteutils
@ -12,6 +11,7 @@ import pkg/codex/chunker
import pkg/codex/discovery
import pkg/codex/blocktype as bt
import ../../../asynctest
import ../../examples
import ../../helpers

View File

@ -3,7 +3,6 @@ import std/random
import std/algorithm
import pkg/stew/byteutils
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/libp2p/errors
import pkg/libp2p/routing_record
@ -17,6 +16,7 @@ import pkg/codex/discovery
import pkg/codex/blocktype
import pkg/codex/utils/asyncheapqueue
import ../../../asynctest
import ../../helpers
import ../../examples

View File

@ -1,9 +1,11 @@
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/stew/byteutils
import pkg/codex/stores
import ../../../asynctest
import ../../examples
import ../../helpers
import pkg/codex/stores
checksuite "account protobuf messages":

View File

@ -1,7 +1,8 @@
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/codex/blockexchange/protobuf/presence
import ../../../asynctest
import ../../examples
import ../../helpers

View File

@ -1,7 +1,6 @@
import std/sequtils
import std/tables
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/codex/rng
@ -9,8 +8,9 @@ import pkg/codex/chunker
import pkg/codex/blocktype as bt
import pkg/codex/blockexchange
import ../helpers
import ../../asynctest
import ../examples
import ../helpers
asyncchecksuite "Network - Handlers":
let

View File

@ -2,13 +2,13 @@ import std/sequtils
import std/algorithm
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/stew/byteutils
import pkg/codex/blocktype as bt
import pkg/codex/blockexchange
import ../helpers
import ../../asynctest
checksuite "Pending Blocks":
test "Should add want handle":

View File

@ -3,11 +3,12 @@ import std/times
import pkg/libp2p
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/codex/codextypes
import pkg/codex/chunker
import ../../asynctest
proc toTimesDuration*(d: chronos.Duration): times.Duration =
initDuration(seconds = d.seconds)

View File

@ -5,7 +5,6 @@ import std/times
import std/sequtils
import std/importutils
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/stew/byteutils
import pkg/datastore
@ -35,6 +34,7 @@ import pkg/codex/utils/asynciter
import pkg/codex/node {.all.}
import ../../asynctest
import ../../examples
import ../helpers
import ../helpers/mockmarket

View File

@ -5,7 +5,6 @@ import std/times
import std/sequtils
import std/importutils
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/stew/byteutils
import pkg/datastore
@ -34,6 +33,7 @@ import pkg/codex/blocktype as bt
import pkg/codex/node {.all.}
import ../../asynctest
import ../examples
import ../helpers
import ../helpers/mockmarket

View File

@ -1,4 +1,3 @@
import pkg/asynctest/chronos/unittest
import pkg/questionable/results
import pkg/codex/clock
@ -11,6 +10,7 @@ import pkg/codex/sales/states/errored
import pkg/codex/sales/states/proving
import pkg/codex/sales/states/finished
import ../../../asynctest
import ../../helpers/mockmarket
import ../../examples
import ../../helpers

View File

@ -1,4 +1,3 @@
import pkg/asynctest/chronos/unittest
import pkg/questionable
import pkg/chronos
import pkg/codex/contracts/requests
@ -10,6 +9,8 @@ import pkg/codex/sales/states/errored
import pkg/codex/sales/salesagent
import pkg/codex/sales/salescontext
import pkg/codex/market
import ../../../asynctest
import ../../examples
import ../../helpers
import ../../helpers/mockmarket

View File

@ -1,4 +1,3 @@
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/questionable
import pkg/codex/contracts/requests
@ -8,6 +7,8 @@ import pkg/codex/sales/states/failed
import pkg/codex/sales/states/payout
import pkg/codex/sales/salesagent
import pkg/codex/sales/salescontext
import ../../../asynctest
import ../../examples
import ../../helpers
import ../../helpers/mockmarket

View File

@ -1,4 +1,3 @@
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/questionable
import pkg/codex/contracts/requests
@ -9,6 +8,8 @@ import pkg/codex/sales/states/failed
import pkg/codex/sales/states/payout
import pkg/codex/sales/salesagent
import pkg/codex/sales/salescontext
import ../../../asynctest
import ../../examples
import ../../helpers
import ../../helpers/mockmarket

View File

@ -1,4 +1,3 @@
import pkg/asynctest/chronos/unittest
import pkg/codex/contracts/requests
import pkg/codex/sales
import pkg/codex/sales/salesagent
@ -9,6 +8,8 @@ import pkg/codex/sales/states/filled
import pkg/codex/sales/states/finished
import pkg/codex/sales/states/failed
import pkg/codex/sales/states/payout
import ../../../asynctest
import ../../helpers/mockmarket
import ../../examples
import ../../helpers

View File

@ -3,13 +3,13 @@ import std/random
import pkg/questionable
import pkg/questionable/results
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/datastore
import pkg/codex/stores
import pkg/codex/sales
import pkg/codex/utils/json
import ../../asynctest
import ../examples
import ../helpers

View File

@ -1,7 +1,6 @@
import std/sequtils
import std/sugar
import std/times
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/datastore
import pkg/questionable
@ -14,7 +13,7 @@ import pkg/codex/sales/slotqueue
import pkg/codex/stores/repostore
import pkg/codex/blocktype as bt
import pkg/codex/node
import pkg/codex/utils/asyncspawn
import ../../asynctest
import ../helpers
import ../helpers/mockmarket
import ../helpers/mockclock
@ -289,7 +288,7 @@ asyncchecksuite "Sales":
slot: UInt256,
onBatch: BatchProc): Future[?!void] {.async.} =
let blk = bt.Block.new( @[1.byte] ).get
asyncSpawn onBatch( blk.repeat(request.ask.slotSize.truncate(int)) )
discard await onBatch( blk.repeat(request.ask.slotSize.truncate(int)) )
return success()
createAvailability()
@ -303,7 +302,7 @@ asyncchecksuite "Sales":
onBatch: BatchProc): Future[?!void] {.async.} =
slotIndex = slot
let blk = bt.Block.new( @[1.byte] ).get
asyncSpawn onBatch(@[ blk ])
discard await onBatch(@[ blk ])
return success()
let sold = newFuture[void]()
sales.onSale = proc(request: StorageRequest, slotIndex: UInt256) =

View File

@ -1,5 +1,4 @@
import std/times
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/codex/sales
import pkg/codex/sales/salesagent
@ -7,6 +6,8 @@ import pkg/codex/sales/salescontext
import pkg/codex/sales/statemachine
import pkg/codex/sales/states/errorhandling
import pkg/codex/proving
import ../../asynctest
import ../helpers/mockmarket
import ../helpers/mockclock
import ../helpers

View File

@ -1,5 +1,4 @@
import std/sequtils
import pkg/asynctest/chronos/unittest
import pkg/chronicles
import pkg/chronos
import pkg/datastore
@ -9,6 +8,7 @@ import pkg/questionable/results
import pkg/codex/logutils
import pkg/codex/sales/slotqueue
import ../../asynctest
import ../helpers
import ../helpers/mockmarket
import ../examples

View File

@ -1,12 +1,12 @@
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/poseidon2
import pkg/poseidon2/io
import pkg/constantine/math/io/io_fields
import pkg/questionable/results
import pkg/codex/merkletree
import pkg/codex/slots/converters
import ../../asynctest
import ../examples
import ../merkletree/helpers

View File

@ -26,7 +26,8 @@ import pkg/codex/slots/builder/builder
import ../helpers
import ../examples
import ../merkletree/helpers
import testsampler_expected
import ../../asynctest
import ./testsampler_expected
import ./provingtestenv
asyncchecksuite "Test DataSampler":

View File

@ -4,7 +4,6 @@ import std/importutils
import std/sugar
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/questionable/results
import pkg/codex/blocktype as bt
import pkg/codex/rng
@ -19,6 +18,7 @@ import pkg/poseidon2
import pkg/poseidon2/io
import constantine/math/io/io_fields
import ../../asynctest
import ../helpers
import ../examples
import ../merkletree/helpers

View File

@ -10,7 +10,6 @@ import pkg/constantine/math/io/io_fields
import pkg/poseidon2/io
import pkg/poseidon2
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/codex/stores/cachestore
import pkg/codex/chunker
import pkg/codex/stores
@ -22,6 +21,7 @@ import pkg/codex/stores/cachestore
import pkg/codex/slots/sampler/utils
import ../../asynctest
import ../helpers
import ../examples
import ../merkletree/helpers

View File

@ -3,7 +3,6 @@ import std/strutils
import std/options
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/libp2p/multicodec
import pkg/stew/byteutils
import pkg/questionable
@ -14,6 +13,7 @@ import pkg/codex/manifest
import pkg/codex/merkletree
import pkg/codex/utils
import ../../asynctest
import ../helpers
type

View File

@ -1,7 +1,6 @@
import std/strutils
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/stew/byteutils
import pkg/questionable/results
import pkg/codex/stores/cachestore
@ -9,6 +8,7 @@ import pkg/codex/chunker
import ./commonstoretests
import ../../asynctest
import ../helpers
checksuite "Cache Store":

View File

@ -10,13 +10,13 @@
import std/random
import std/sequtils
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/questionable
import pkg/questionable/results
import pkg/codex/blocktype as bt
import pkg/codex/stores/repostore
import pkg/codex/clock
import ../../asynctest
import ../helpers/mocktimer
import ../helpers/mockrepostore
import ../helpers/mockclock

View File

@ -8,12 +8,12 @@
## those terms.
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/questionable/results
import pkg/codex/blocktype as bt
import pkg/codex/stores/repostore
import pkg/codex/clock
import ../../asynctest
import ../helpers
import ../helpers/mocktimer
import ../helpers/mockrepostore

View File

@ -6,7 +6,6 @@ import pkg/questionable
import pkg/questionable/results
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/stew/byteutils
import pkg/stew/endians2
import pkg/datastore
@ -18,6 +17,7 @@ import pkg/codex/blocktype as bt
import pkg/codex/clock
import pkg/codex/utils/asynciter
import ../../asynctest
import ../helpers
import ../helpers/mockclock
import ../examples

View File

@ -1,10 +1,10 @@
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/stew/results
import pkg/codex/utils/asyncheapqueue
import pkg/codex/rng
import ../asynctest
import ./helpers
type

View File

@ -1,4 +1,3 @@
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/chronos/transports/stream
import pkg/chronos/transports/common
@ -6,6 +5,7 @@ import pkg/chronos/streams/asyncstream
import pkg/codex/streams
import pkg/stew/byteutils
import ../asynctest
import ./helpers
asyncchecksuite "AsyncStreamWrapper":

View File

@ -1,10 +1,10 @@
import pkg/asynctest/chronos/unittest
import pkg/stew/byteutils
import pkg/codex/chunker
import pkg/codex/logutils
import pkg/chronos
import ../asynctest
import ./helpers
asyncchecksuite "Chunking":

View File

@ -1,7 +1,6 @@
import std/sequtils
import std/sugar
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/datastore
import pkg/questionable/results
@ -13,6 +12,7 @@ import pkg/codex/blocktype as bt
import pkg/codex/rng
import pkg/codex/utils
import ../asynctest
import ./helpers
suite "Erasure encode/decode":

View File

@ -1,9 +1,9 @@
import std/sequtils
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/codex/utils/asynciter
import ../asynctest
import ./helpers
import pkg/codex/indexingstrategy

View File

@ -2,9 +2,6 @@ import std/sequtils
import pkg/chronos
import pkg/questionable/results
import pkg/asynctest/chronos/unittest
import pkg/stew/byteutils
import pkg/codex/chunker
import pkg/codex/blocktype as bt
import pkg/codex/manifest
@ -13,6 +10,7 @@ import pkg/poseidon2
import pkg/codex/slots
import pkg/codex/merkletree
import ../asynctest
import ./helpers
import ./examples

View File

@ -1,5 +1,4 @@
import std/times
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/stint
import pkg/codex/purchasing
@ -9,6 +8,8 @@ import pkg/codex/purchasing/states/submitted
import pkg/codex/purchasing/states/unknown
import pkg/codex/purchasing/states/cancelled
import pkg/codex/purchasing/states/failed
import ../asynctest
import ./helpers/mockmarket
import ./helpers/mockclock
import ./examples

View File

@ -1,14 +1,14 @@
import pkg/chronos
import pkg/asynctest/chronos/unittest
import pkg/questionable/results
import ./helpers
import pkg/codex/streams
import pkg/codex/stores
import pkg/codex/manifest
import pkg/codex/blocktype as bt
import ../asynctest
import ./helpers
asyncchecksuite "StoreStream":
var
manifest: Manifest

View File

@ -1,8 +1,9 @@
import pkg/asynctest/chronos/unittest
import pkg/chronos
import codex/validation
import codex/periods
import ../asynctest
import ./helpers/mockmarket
import ./helpers/mockclock
import ./examples

View File

@ -1,8 +1,9 @@
import pkg/asynctest/chronos/unittest
import pkg/questionable
import pkg/chronos
import pkg/upraises
import codex/utils/asyncstatemachine
import ../../asynctest
import ../helpers
type

View File

@ -1,8 +1,9 @@
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/questionable
import pkg/questionable/results
import codex/utils/then
import ../../asynctest
import ../helpers
proc newError(): ref CatchableError =

View File

@ -8,9 +8,10 @@
## those terms.
import pkg/chronos
import pkg/asynctest/chronos/unittest
import codex/utils/timer
import ../../asynctest
import ../helpers
asyncchecksuite "Timer":

View File

@ -1,6 +1,7 @@
import pkg/asynctest/chronos/unittest
import pkg/chronos
import codex/utils/trackedfutures
import ../../asynctest
import ../helpers
type Module = object

View File

@ -1,9 +1,9 @@
import pkg/asynctest/chronos/unittest
import pkg/ethers
import codex/contracts/deployment
import codex/conf
import codex/contracts
import ../asynctest
import ../checktest
type MockProvider = ref object of Provider

View File

@ -1,6 +1,7 @@
import pkg/asynctest/chronos/unittest
import std/json
import pkg/ethers
import ./asynctest
import ./checktest
## Unit testing suite that sets up an Ethereum testing environment.

View File

@ -2,13 +2,15 @@ import std/times
import std/os
import std/json
import std/tempfiles
import pkg/asynctest/chronos/unittest
import pkg/chronos
import pkg/stint
import pkg/questionable
import pkg/questionable/results
import ./asynctest
import ./integration/nodes
suite "Taiko L2 Integration Tests":
var node1, node2: NodeProcess