logos-storage/logos-storage-nim
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim.git synced 2026-01-09 08:53:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/blockexc-prevent-retransmit

Browse Source
This commit is contained in:
Ben Bierens 2025-02-20 15:16:49 +01:00 committed by GitHub
commit bc16c27edf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
128 changed files with 2338 additions and 1325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/docker-reusable.yml vendored
View File

@ -94,11 +94,11 @@ jobs:
- target:
os: linux
arch: amd64
builder: ubuntu-22.04
builder: ubuntu-24.04
- target:
os: linux
arch: arm64
builder: ubuntu-22.04-arm
builder: ubuntu-24.04-arm
name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
runs-on: ${{ matrix.builder }}

16
.github/workflows/docs.yml vendored
View File

@ -2,17 +2,17 @@ name: OpenAPI
on:
push:
branches:
- 'master'
tags:
- "v*.*.*"
paths:
- 'openapi.yaml'
- '.github/workflows/docs.yml'
- "openapi.yaml"
- ".github/workflows/docs.yml"
pull_request:
branches:
- '**'
- "**"
paths:
- 'openapi.yaml'
- '.github/workflows/docs.yml'
- "openapi.yaml"
- ".github/workflows/docs.yml"
# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
@ -40,7 +40,7 @@ jobs:
deploy:
name: Deploy
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/master'
if: startsWith(github.ref, 'refs/tags/')
steps:
- name: Checkout
uses: actions/checkout@v4

34
build.nims
View File

@ -4,7 +4,6 @@ import std/os except commandLineParams
### Helper functions
proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
if not dirExists "build":
mkDir "build"
@ -14,13 +13,15 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
for param in commandLineParams():
extra_params &= " " & param
else:
for i in 2..<paramCount():
for i in 2 ..< paramCount():
extra_params &= " " & paramStr(i)
let
# Place build output in 'build' folder, even if name includes a longer path.
outName = os.lastPathPart(name)
cmd = "nim " & lang & " --out:build/" & outName & " " & extra_params & " " & srcDir & name & ".nim"
cmd =
"nim " & lang & " --out:build/" & outName & " " & extra_params & " " & srcDir &
name & ".nim"
exec(cmd)
@ -29,7 +30,8 @@ proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
exec "build/" & name
task codex, "build codex binary":
buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
buildBinary "codex",
params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
task toolsCirdl, "build tools/cirdl binary":
buildBinary "tools/cirdl/cirdl"
@ -41,7 +43,9 @@ task testContracts, "Build & run Codex Contract tests":
test "testContracts"
task testIntegration, "Run integration tests":
buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true"
buildBinary "codex",
params =
"-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true"
test "testIntegration"
# use params to enable logging from the integration test executable
# test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " &
@ -90,15 +94,25 @@ task coverage, "generates code coverage report":
var nimSrcs = " "
for f in walkDirRec("codex", {pcFile}):
if f.endswith(".nim"): nimSrcs.add " " & f.absolutePath.quoteShell()
if f.endswith(".nim"):
nimSrcs.add " " & f.absolutePath.quoteShell()
echo "======== Running Tests ======== "
test "coverage", srcDir = "tests/", params = " --nimcache:nimcache/coverage -d:release -d:codex_enable_proof_failures=true"
test "coverage",
srcDir = "tests/",
params =
" --nimcache:nimcache/coverage -d:release -d:codex_enable_proof_failures=true"
exec("rm nimcache/coverage/*.c")
rmDir("coverage"); mkDir("coverage")
rmDir("coverage")
mkDir("coverage")
echo " ======== Running LCOV ======== "
exec("lcov --capture --directory nimcache/coverage --output-file coverage/coverage.info")
exec("lcov --extract coverage/coverage.info --output-file coverage/coverage.f.info " & nimSrcs)
exec(
"lcov --capture --directory nimcache/coverage --output-file coverage/coverage.info"
)
exec(
"lcov --extract coverage/coverage.info --output-file coverage/coverage.f.info " &
nimSrcs
)
echo " ======== Generating HTML coverage report ======== "
exec("genhtml coverage/coverage.f.info --output-directory coverage/report ")
echo " ======== Coverage report Done ======== "

39
codex.nim
View File

@ -38,33 +38,35 @@ when isMainModule:
when defined(posix):
import system/ansi_c
type
CodexStatus {.pure.} = enum
Stopped,
Stopping,
Running
type CodexStatus {.pure.} = enum
Stopped
Stopping
Running
let config = CodexConf.load(
version = codexFullVersion,
envVarsPrefix = "codex",
secondarySources = proc (config: CodexConf, sources: auto) {.gcsafe, raises: [ConfigurationError].} =
if configFile =? config.configFile:
sources.addConfigFile(Toml, configFile)
secondarySources = proc(
config: CodexConf, sources: auto
) {.gcsafe, raises: [ConfigurationError].} =
if configFile =? config.configFile:
sources.addConfigFile(Toml, configFile)
,
)
config.setupLogging()
config.setupMetrics()
if not(checkAndCreateDataDir((config.dataDir).string)):
if not (checkAndCreateDataDir((config.dataDir).string)):
# We are unable to access/create data folder or data folder's
# permissions are insecure.
quit QuitFailure
if config.prover() and not(checkAndCreateDataDir((config.circuitDir).string)):
if config.prover() and not (checkAndCreateDataDir((config.circuitDir).string)):
quit QuitFailure
trace "Data dir initialized", dir = $config.dataDir
if not(checkAndCreateDataDir((config.dataDir / "repo"))):
if not (checkAndCreateDataDir((config.dataDir / "repo"))):
# We are unable to access/create data folder or data folder's
# permissions are insecure.
quit QuitFailure
@ -83,11 +85,12 @@ when isMainModule:
config.dataDir / config.netPrivKeyFile
privateKey = setupKey(keyPath).expect("Should setup private key!")
server = try:
CodexServer.new(config, privateKey)
except Exception as exc:
error "Failed to start Codex", msg = exc.msg
quit QuitFailure
server =
try:
CodexServer.new(config, privateKey)
except Exception as exc:
error "Failed to start Codex", msg = exc.msg
quit QuitFailure
## Ctrl+C handling
proc doShutdown() =
@ -101,7 +104,9 @@ when isMainModule:
# workaround for https://github.com/nim-lang/Nim/issues/4057
try:
setupForeignThreadGc()
except Exception as exc: raiseAssert exc.msg # shouldn't happen
except Exception as exc:
raiseAssert exc.msg
# shouldn't happen
notice "Shutting down after having received SIGINT"
doShutdown()

2
codex/blockexchange/engine/engine.nim
View File

@ -402,7 +402,7 @@ proc wantListHandler*(b: BlockExcEngine, peer: PeerId, wantList: WantList) {.asy
have = await e.address in b.localStore
price = @(b.pricing.get(Pricing(price: 0.u256)).price.toBytesBE)
case e.wantType:
case e.wantType
of WantType.WantHave:
if have:
presence.add(

3
codex/clock.nim
View File

@ -40,5 +40,8 @@ proc toSecondsSince1970*(bytes: seq[byte]): SecondsSince1970 =
let asUint = uint64.fromBytes(bytes)
cast[int64](asUint)
proc toSecondsSince1970*(num: uint64): SecondsSince1970 =
cast[int64](num)
proc toSecondsSince1970*(bigint: UInt256): SecondsSince1970 =
bigint.truncate(int64)

20
codex/codex.nim
View File

@ -11,8 +11,10 @@ import std/sequtils
import std/strutils
import std/os
import std/tables
import std/cpuinfo
import pkg/chronos
import pkg/taskpools
import pkg/presto
import pkg/libp2p
import pkg/confutils
@ -107,7 +109,9 @@ proc bootstrapInteractions(s: CodexServer): Future[void] {.async.} =
quit QuitFailure
let marketplace = Marketplace.new(marketplaceAddress, signer)
let market = OnChainMarket.new(marketplace, config.rewardRecipient)
let market = OnChainMarket.new(
marketplace, config.rewardRecipient, config.marketplaceRequestCacheSize
)
let clock = OnChainClock.new(provider)
var client: ?ClientInteractions
@ -194,7 +198,18 @@ proc new*(
.withTcpTransport({ServerFlags.ReuseAddr})
.build()
var cache: CacheStore = nil
var
cache: CacheStore = nil
taskpool: Taskpool
try:
if config.numThreads == ThreadCount(0):
taskpool = Taskpool.new(numThreads = min(countProcessors(), 16))
else:
taskpool = Taskpool.new(numThreads = int(config.numThreads))
info "Threadpool started", numThreads = taskpool.numThreads
except CatchableError as exc:
raiseAssert("Failure in taskpool initialization:" & exc.msg)
if config.cacheSize > 0'nb:
cache = CacheStore.new(cacheSize = config.cacheSize)
@ -286,6 +301,7 @@ proc new*(
engine = engine,
discovery = discovery,
prover = prover,
taskPool = taskpool,
)
restServer = RestServerRef

44
codex/conf.nim
View File

@ -44,6 +44,7 @@ import ./utils
import ./nat
import ./utils/natutils
from ./contracts/config import DefaultRequestCacheSize
from ./validationconfig import MaxSlots, ValidationGroups
export units, net, codextypes, logutils, completeCmdArg, parseCmdArg, NatConfig
@ -51,7 +52,11 @@ export ValidationGroups, MaxSlots
export
DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockMaintenanceInterval,
DefaultNumberOfBlocksToMaintainPerInterval
DefaultNumberOfBlocksToMaintainPerInterval, DefaultRequestCacheSize
type ThreadCount* = distinct Natural
proc `==`*(a, b: ThreadCount): bool {.borrow.}
proc defaultDataDir*(): string =
let dataDir =
@ -71,6 +76,7 @@ const
DefaultDataDir* = defaultDataDir()
DefaultCircuitDir* = defaultDataDir() / "circuits"
DefaultThreadCount* = ThreadCount(0)
type
StartUpCmd* {.pure.} = enum
@ -184,6 +190,13 @@ type
name: "max-peers"
.}: int
numThreads* {.
desc:
"Number of worker threads (\"0\" = use as many threads as there are CPU cores available)",
defaultValue: DefaultThreadCount,
name: "num-threads"
.}: ThreadCount
agentString* {.
defaultValue: "Codex",
desc: "Node agent string which is used as identifier in network",
@ -347,6 +360,16 @@ type
name: "reward-recipient"
.}: Option[EthAddress]
marketplaceRequestCacheSize* {.
desc:
"Maximum number of StorageRequests kept in memory." &
"Reduces fetching of StorageRequest data from the contract.",
defaultValue: DefaultRequestCacheSize,
defaultValueDesc: $DefaultRequestCacheSize,
name: "request-cache-size",
hidden
.}: uint16
case persistenceCmd* {.defaultValue: noCmd, command.}: PersistenceCmd
of PersistenceCmd.prover:
circuitDir* {.
@ -482,6 +505,13 @@ proc parseCmdArg*(
quit QuitFailure
ma
proc parseCmdArg*(T: type ThreadCount, input: string): T {.upraises: [ValueError].} =
let count = parseInt(input)
if count != 0 and count < 2:
warn "Invalid number of threads", input = input
quit QuitFailure
ThreadCount(count)
proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T =
var res: SignedPeerRecord
try:
@ -579,6 +609,15 @@ proc readValue*(
quit QuitFailure
val = NBytes(value)
proc readValue*(
r: var TomlReader, val: var ThreadCount
) {.upraises: [SerializationError, IOError].} =
var str = r.readValue(string)
try:
val = parseCmdArg(ThreadCount, str)
except CatchableError as err:
raise newException(SerializationError, err.msg)
proc readValue*(
r: var TomlReader, val: var Duration
) {.upraises: [SerializationError, IOError].} =
@ -609,6 +648,9 @@ proc completeCmdArg*(T: type NBytes, val: string): seq[string] =
proc completeCmdArg*(T: type Duration, val: string): seq[string] =
discard
proc completeCmdArg*(T: type ThreadCount, val: string): seq[string] =
discard
# silly chronicles, colors is a compile-time property
proc stripAnsi*(v: string): string =
var

46
codex/contracts/config.nim
View File

@ -4,47 +4,66 @@ import pkg/questionable/results
export contractabi
const DefaultRequestCacheSize* = 128.uint16
type
MarketplaceConfig* = object
collateral*: CollateralConfig
proofs*: ProofConfig
reservations*: SlotReservationsConfig
requestDurationLimit*: uint64
CollateralConfig* = object
repairRewardPercentage*: uint8
# percentage of remaining collateral slot has after it has been freed
maxNumberOfSlashes*: uint8 # frees slot when the number of slashes reaches this value
slashCriterion*: uint16 # amount of proofs missed that lead to slashing
slashPercentage*: uint8 # percentage of the collateral that is slashed
validatorRewardPercentage*: uint8
# percentage of the slashed amount going to the validators
ProofConfig* = object
period*: UInt256 # proofs requirements are calculated per period (in seconds)
timeout*: UInt256 # mark proofs as missing before the timeout (in seconds)
period*: uint64 # proofs requirements are calculated per period (in seconds)
timeout*: uint64 # mark proofs as missing before the timeout (in seconds)
downtime*: uint8 # ignore this much recent blocks for proof requirements
downtimeProduct*: uint8
zkeyHash*: string # hash of the zkey file which is linked to the verifier
# Ensures the pointer does not remain in downtime for many consecutive
# periods. For each period increase, move the pointer `pointerProduct`
# blocks. Should be a prime number to ensure there are no cycles.
downtimeProduct*: uint8
SlotReservationsConfig* = object
maxReservations*: uint8
func fromTuple(_: type ProofConfig, tupl: tuple): ProofConfig =
ProofConfig(
period: tupl[0],
timeout: tupl[1],
downtime: tupl[2],
zkeyHash: tupl[3],
downtimeProduct: tupl[4],
downtimeProduct: tupl[3],
zkeyHash: tupl[4],
)
func fromTuple(_: type SlotReservationsConfig, tupl: tuple): SlotReservationsConfig =
SlotReservationsConfig(maxReservations: tupl[0])
func fromTuple(_: type CollateralConfig, tupl: tuple): CollateralConfig =
CollateralConfig(
repairRewardPercentage: tupl[0],
maxNumberOfSlashes: tupl[1],
slashCriterion: tupl[2],
slashPercentage: tupl[3],
slashPercentage: tupl[2],
validatorRewardPercentage: tupl[3],
)
func fromTuple(_: type MarketplaceConfig, tupl: tuple): MarketplaceConfig =
MarketplaceConfig(collateral: tupl[0], proofs: tupl[1])
MarketplaceConfig(
collateral: tupl[0],
proofs: tupl[1],
reservations: tupl[2],
requestDurationLimit: tupl[3],
)
func solidityType*(_: type SlotReservationsConfig): string =
solidityType(SlotReservationsConfig.fieldTypes)
func solidityType*(_: type ProofConfig): string =
solidityType(ProofConfig.fieldTypes)
@ -53,7 +72,10 @@ func solidityType*(_: type CollateralConfig): string =
solidityType(CollateralConfig.fieldTypes)
func solidityType*(_: type MarketplaceConfig): string =
solidityType(CollateralConfig.fieldTypes)
solidityType(MarketplaceConfig.fieldTypes)
func encode*(encoder: var AbiEncoder, slot: SlotReservationsConfig) =
encoder.write(slot.fieldValues)
func encode*(encoder: var AbiEncoder, slot: ProofConfig) =
encoder.write(slot.fieldValues)
@ -68,6 +90,10 @@ func decode*(decoder: var AbiDecoder, T: type ProofConfig): ?!T =
let tupl = ?decoder.read(ProofConfig.fieldTypes)
success ProofConfig.fromTuple(tupl)
func decode*(decoder: var AbiDecoder, T: type SlotReservationsConfig): ?!T =
let tupl = ?decoder.read(SlotReservationsConfig.fieldTypes)
success SlotReservationsConfig.fromTuple(tupl)
func decode*(decoder: var AbiDecoder, T: type CollateralConfig): ?!T =
let tupl = ?decoder.read(CollateralConfig.fieldTypes)
success CollateralConfig.fromTuple(tupl)

46
codex/contracts/market.nim
View File

@ -2,6 +2,7 @@ import std/strutils
import pkg/ethers
import pkg/upraises
import pkg/questionable
import pkg/lrucache
import ../utils/exceptions
import ../logutils
import ../market
@ -20,6 +21,7 @@ type
signer: Signer
rewardRecipient: ?Address
configuration: ?MarketplaceConfig
requestCache: LruCache[string, StorageRequest]
MarketSubscription = market.Subscription
EventSubscription = ethers.Subscription
@ -27,12 +29,22 @@ type
eventSubscription: EventSubscription
func new*(
_: type OnChainMarket, contract: Marketplace, rewardRecipient = Address.none
_: type OnChainMarket,
contract: Marketplace,
rewardRecipient = Address.none,
requestCacheSize: uint16 = DefaultRequestCacheSize,
): OnChainMarket =
without signer =? contract.signer:
raiseAssert("Marketplace contract should have a signer")
OnChainMarket(contract: contract, signer: signer, rewardRecipient: rewardRecipient)
var requestCache = newLruCache[string, StorageRequest](int(requestCacheSize))
OnChainMarket(
contract: contract,
signer: signer,
rewardRecipient: rewardRecipient,
requestCache: requestCache,
)
proc raiseMarketError(message: string) {.raises: [MarketError].} =
raise newException(MarketError, message)
@ -72,16 +84,21 @@ method periodicity*(market: OnChainMarket): Future[Periodicity] {.async.} =
let period = config.proofs.period
return Periodicity(seconds: period)
method proofTimeout*(market: OnChainMarket): Future[UInt256] {.async.} =
method proofTimeout*(market: OnChainMarket): Future[uint64] {.async.} =
convertEthersError:
let config = await market.config()
return config.proofs.timeout
method repairRewardPercentage*(market: OnChainMarket): Future[uint8] {.async.} =
convertEthersError:
let config = await market.contract.configuration()
let config = await market.config()
return config.collateral.repairRewardPercentage
method requestDurationLimit*(market: OnChainMarket): Future[uint64] {.async.} =
convertEthersError:
let config = await market.config()
return config.requestDurationLimit
method proofDowntime*(market: OnChainMarket): Future[uint8] {.async.} =
convertEthersError:
let config = await market.config()
@ -112,9 +129,16 @@ method requestStorage(market: OnChainMarket, request: StorageRequest) {.async.}
method getRequest*(
market: OnChainMarket, id: RequestId
): Future[?StorageRequest] {.async.} =
let key = $id
if market.requestCache.contains(key):
return some market.requestCache[key]
convertEthersError:
try:
return some await market.contract.getRequest(id)
let request = await market.contract.getRequest(id)
market.requestCache[key] = request
return some request
except Marketplace_UnknownRequest:
return none StorageRequest
@ -146,7 +170,7 @@ method requestExpiresAt*(
return await market.contract.requestExpiry(id)
method getHost(
market: OnChainMarket, requestId: RequestId, slotIndex: UInt256
market: OnChainMarket, requestId: RequestId, slotIndex: uint64
): Future[?Address] {.async.} =
convertEthersError:
let slotId = slotId(requestId, slotIndex)
@ -172,7 +196,7 @@ method getActiveSlot*(market: OnChainMarket, slotId: SlotId): Future[?Slot] {.as
method fillSlot(
market: OnChainMarket,
requestId: RequestId,
slotIndex: UInt256,
slotIndex: uint64,
proof: Groth16Proof,
collateral: UInt256,
) {.async.} =
@ -256,7 +280,7 @@ method canProofBeMarkedAsMissing*(
return false
method reserveSlot*(
market: OnChainMarket, requestId: RequestId, slotIndex: UInt256
market: OnChainMarket, requestId: RequestId, slotIndex: uint64
) {.async.} =
convertEthersError:
discard await market.contract
@ -269,7 +293,7 @@ method reserveSlot*(
.confirm(1)
method canReserveSlot*(
market: OnChainMarket, requestId: RequestId, slotIndex: UInt256
market: OnChainMarket, requestId: RequestId, slotIndex: uint64
): Future[bool] {.async.} =
convertEthersError:
return await market.contract.canReserveSlot(requestId, slotIndex)
@ -305,10 +329,10 @@ method subscribeSlotFilled*(
method subscribeSlotFilled*(
market: OnChainMarket,
requestId: RequestId,
slotIndex: UInt256,
slotIndex: uint64,
callback: OnSlotFilled,
): Future[MarketSubscription] {.async.} =
proc onSlotFilled(eventRequestId: RequestId, eventSlotIndex: UInt256) =
proc onSlotFilled(eventRequestId: RequestId, eventSlotIndex: uint64) =
if eventRequestId == requestId and eventSlotIndex == slotIndex:
callback(requestId, slotIndex)

19
codex/contracts/marketplace.nim
View File

@ -42,6 +42,7 @@ type
Marketplace_InsufficientCollateral* = object of SolidityError
Marketplace_InsufficientReward* = object of SolidityError
Marketplace_InvalidCid* = object of SolidityError
Marketplace_DurationExceedsLimit* = object of SolidityError
Proofs_InsufficientBlockHeight* = object of SolidityError
Proofs_InvalidProof* = object of SolidityError
Proofs_ProofAlreadySubmitted* = object of SolidityError
@ -59,10 +60,6 @@ proc currentCollateral*(
marketplace: Marketplace, id: SlotId
): UInt256 {.contract, view.}
proc slashMisses*(marketplace: Marketplace): UInt256 {.contract, view.}
proc slashPercentage*(marketplace: Marketplace): UInt256 {.contract, view.}
proc minCollateralThreshold*(marketplace: Marketplace): UInt256 {.contract, view.}
proc requestStorage*(
marketplace: Marketplace, request: StorageRequest
): Confirmable {.
@ -75,10 +72,7 @@ proc requestStorage*(
.}
proc fillSlot*(
marketplace: Marketplace,
requestId: RequestId,
slotIndex: UInt256,
proof: Groth16Proof,
marketplace: Marketplace, requestId: RequestId, slotIndex: uint64, proof: Groth16Proof
): Confirmable {.
contract,
errors: [
@ -154,9 +148,6 @@ proc requestExpiry*(
marketplace: Marketplace, requestId: RequestId
): SecondsSince1970 {.contract, view.}
proc proofTimeout*(marketplace: Marketplace): UInt256 {.contract, view.}
proc proofEnd*(marketplace: Marketplace, id: SlotId): UInt256 {.contract, view.}
proc missingProofs*(marketplace: Marketplace, id: SlotId): UInt256 {.contract, view.}
proc isProofRequired*(marketplace: Marketplace, id: SlotId): bool {.contract, view.}
proc willProofBeRequired*(marketplace: Marketplace, id: SlotId): bool {.contract, view.}
@ -175,7 +166,7 @@ proc submitProof*(
.}
proc markProofAsMissing*(
marketplace: Marketplace, id: SlotId, period: UInt256
marketplace: Marketplace, id: SlotId, period: uint64
): Confirmable {.
contract,
errors: [
@ -186,9 +177,9 @@ proc markProofAsMissing*(
.}
proc reserveSlot*(
marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256
marketplace: Marketplace, requestId: RequestId, slotIndex: uint64
): Confirmable {.contract.}
proc canReserveSlot*(
marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256
marketplace: Marketplace, requestId: RequestId, slotIndex: uint64
): bool {.contract, view.}

10
codex/contracts/provider.nim
View File

@ -14,7 +14,7 @@ proc raiseProviderError(message: string) {.raises: [ProviderError].} =
proc blockNumberAndTimestamp*(
provider: Provider, blockTag: BlockTag
): Future[(UInt256, UInt256)] {.async: (raises: [ProviderError]).} =
): Future[(UInt256, UInt256)] {.async: (raises: [ProviderError, CancelledError]).} =
without latestBlock =? await provider.getBlock(blockTag):
raiseProviderError("Could not get latest block")
@ -25,7 +25,7 @@ proc blockNumberAndTimestamp*(
proc binarySearchFindClosestBlock(
provider: Provider, epochTime: int, low: UInt256, high: UInt256
): Future[UInt256] {.async: (raises: [ProviderError]).} =
): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} =
let (_, lowTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(low))
let (_, highTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(high))
if abs(lowTimestamp.truncate(int) - epochTime) <
@ -39,7 +39,7 @@ proc binarySearchBlockNumberForEpoch(
epochTime: UInt256,
latestBlockNumber: UInt256,
earliestBlockNumber: UInt256,
): Future[UInt256] {.async: (raises: [ProviderError]).} =
): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} =
var low = earliestBlockNumber
var high = latestBlockNumber
@ -65,7 +65,7 @@ proc binarySearchBlockNumberForEpoch(
proc blockNumberForEpoch*(
provider: Provider, epochTime: SecondsSince1970
): Future[UInt256] {.async: (raises: [ProviderError]).} =
): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} =
let epochTimeUInt256 = epochTime.u256
let (latestBlockNumber, latestBlockTimestamp) =
await provider.blockNumberAndTimestamp(BlockTag.latest)
@ -118,6 +118,6 @@ proc blockNumberForEpoch*(
proc pastBlockTag*(
provider: Provider, blocksAgo: int
): Future[BlockTag] {.async: (raises: [ProviderError]).} =
): Future[BlockTag] {.async: (raises: [ProviderError, CancelledError]).} =
let head = await provider.getBlockNumber()
return BlockTag.init(head - blocksAgo.abs.u256)

56
codex/contracts/requests.nim
View File

@ -6,8 +6,11 @@ import pkg/nimcrypto
import pkg/ethers/fields
import pkg/questionable/results
import pkg/stew/byteutils
import pkg/libp2p/[cid, multicodec]
import ../logutils
import ../utils/json
import ../clock
from ../errors import mapFailure
export contractabi
@ -16,25 +19,25 @@ type
client* {.serialize.}: Address
ask* {.serialize.}: StorageAsk
content* {.serialize.}: StorageContent
expiry* {.serialize.}: UInt256
expiry* {.serialize.}: uint64
nonce*: Nonce
StorageAsk* = object
slots* {.serialize.}: uint64
slotSize* {.serialize.}: UInt256
duration* {.serialize.}: UInt256
proofProbability* {.serialize.}: UInt256
pricePerBytePerSecond* {.serialize.}: UInt256
collateralPerByte* {.serialize.}: UInt256
slots* {.serialize.}: uint64
slotSize* {.serialize.}: uint64
duration* {.serialize.}: uint64
maxSlotLoss* {.serialize.}: uint64
StorageContent* = object
cid* {.serialize.}: string
cid* {.serialize.}: Cid
merkleRoot*: array[32, byte]
Slot* = object
request* {.serialize.}: StorageRequest
slotIndex* {.serialize.}: UInt256
slotIndex* {.serialize.}: uint64
SlotId* = distinct array[32, byte]
RequestId* = distinct array[32, byte]
@ -108,18 +111,21 @@ func fromTuple(_: type Slot, tupl: tuple): Slot =
func fromTuple(_: type StorageAsk, tupl: tuple): StorageAsk =
StorageAsk(
slots: tupl[0],
slotSize: tupl[1],
duration: tupl[2],
proofProbability: tupl[3],
pricePerBytePerSecond: tupl[4],
collateralPerByte: tupl[5],
proofProbability: tupl[0],
pricePerBytePerSecond: tupl[1],
collateralPerByte: tupl[2],
slots: tupl[3],
slotSize: tupl[4],
duration: tupl[5],
maxSlotLoss: tupl[6],
)
func fromTuple(_: type StorageContent, tupl: tuple): StorageContent =
StorageContent(cid: tupl[0], merkleRoot: tupl[1])
func solidityType*(_: type Cid): string =
solidityType(seq[byte])
func solidityType*(_: type StorageContent): string =
solidityType(StorageContent.fieldTypes)
@ -129,6 +135,10 @@ func solidityType*(_: type StorageAsk): string =
func solidityType*(_: type StorageRequest): string =
solidityType(StorageRequest.fieldTypes)
# Note: it seems to be ok to ignore the vbuffer offset for now
func encode*(encoder: var AbiEncoder, cid: Cid) =
encoder.write(cid.data.buffer)
func encode*(encoder: var AbiEncoder, content: StorageContent) =
encoder.write(content.fieldValues)
@ -141,8 +151,12 @@ func encode*(encoder: var AbiEncoder, id: RequestId | SlotId | Nonce) =
func encode*(encoder: var AbiEncoder, request: StorageRequest) =
encoder.write(request.fieldValues)
func encode*(encoder: var AbiEncoder, request: Slot) =
encoder.write(request.fieldValues)
func encode*(encoder: var AbiEncoder, slot: Slot) =
encoder.write(slot.fieldValues)
func decode*(decoder: var AbiDecoder, T: type Cid): ?!T =
let data = ?decoder.read(seq[byte])
Cid.init(data).mapFailure
func decode*(decoder: var AbiDecoder, T: type StorageContent): ?!T =
let tupl = ?decoder.read(StorageContent.fieldTypes)
@ -164,21 +178,21 @@ func id*(request: StorageRequest): RequestId =
let encoding = AbiEncoder.encode((request,))
RequestId(keccak256.digest(encoding).data)
func slotId*(requestId: RequestId, slotIndex: UInt256): SlotId =
func slotId*(requestId: RequestId, slotIndex: uint64): SlotId =
let encoding = AbiEncoder.encode((requestId, slotIndex))
SlotId(keccak256.digest(encoding).data)
func slotId*(request: StorageRequest, slotIndex: UInt256): SlotId =
func slotId*(request: StorageRequest, slotIndex: uint64): SlotId =
slotId(request.id, slotIndex)
func id*(slot: Slot): SlotId =
slotId(slot.request, slot.slotIndex)
func pricePerSlotPerSecond*(ask: StorageAsk): UInt256 =
ask.pricePerBytePerSecond * ask.slotSize
ask.pricePerBytePerSecond * ask.slotSize.u256
func pricePerSlot*(ask: StorageAsk): UInt256 =
ask.duration * ask.pricePerSlotPerSecond
ask.duration.u256 * ask.pricePerSlotPerSecond
func totalPrice*(ask: StorageAsk): UInt256 =
ask.slots.u256 * ask.pricePerSlot
@ -187,7 +201,7 @@ func totalPrice*(request: StorageRequest): UInt256 =
request.ask.totalPrice
func collateralPerSlot*(ask: StorageAsk): UInt256 =
ask.collateralPerByte * ask.slotSize
ask.collateralPerByte * ask.slotSize.u256
func size*(ask: StorageAsk): UInt256 =
ask.slots.u256 * ask.slotSize
func size*(ask: StorageAsk): uint64 =
ask.slots * ask.slotSize

8
codex/erasure/backend.nim
View File

@ -29,14 +29,18 @@ method release*(self: ErasureBackend) {.base, gcsafe.} =
raiseAssert("not implemented!")
method encode*(
self: EncoderBackend, buffers, parity: var openArray[seq[byte]]
self: EncoderBackend,
buffers, parity: ptr UncheckedArray[ptr UncheckedArray[byte]],
dataLen, parityLen: int,
): Result[void, cstring] {.base, gcsafe.} =
## encode buffers using a backend
##
raiseAssert("not implemented!")
method decode*(
self: DecoderBackend, buffers, parity, recovered: var openArray[seq[byte]]
self: DecoderBackend,
buffers, parity, recovered: ptr UncheckedArray[ptr UncheckedArray[byte]],
dataLen, parityLen, recoveredLen: int,
): Result[void, cstring] {.base, gcsafe.} =
## decode buffers using a backend
##

16
codex/erasure/backends/leopard.nim
View File

@ -10,7 +10,7 @@
import std/options
import pkg/leopard
import pkg/stew/results
import pkg/results
import ../backend
@ -22,11 +22,13 @@ type
decoder*: Option[LeoDecoder]
method encode*(
self: LeoEncoderBackend, data, parity: var openArray[seq[byte]]
self: LeoEncoderBackend,
data, parity: ptr UncheckedArray[ptr UncheckedArray[byte]],
dataLen, parityLen: int,
): Result[void, cstring] =
## Encode data using Leopard backend
if parity.len == 0:
if parityLen == 0:
return ok()
var encoder =
@ -36,10 +38,12 @@ method encode*(
else:
self.encoder.get()
encoder.encode(data, parity)
encoder.encode(data, parity, dataLen, parityLen)
method decode*(
self: LeoDecoderBackend, data, parity, recovered: var openArray[seq[byte]]
self: LeoDecoderBackend,
data, parity, recovered: ptr UncheckedArray[ptr UncheckedArray[byte]],
dataLen, parityLen, recoveredLen: int,
): Result[void, cstring] =
## Decode data using given Leopard backend
@ -50,7 +54,7 @@ method decode*(
else:
self.decoder.get()
decoder.decode(data, parity, recovered)
decoder.decode(data, parity, recovered, dataLen, parityLen, recoveredLen)
method release*(self: LeoEncoderBackend) =
if self.encoder.isSome:

255
codex/erasure/erasure.nim
View File

@ -12,12 +12,14 @@ import pkg/upraises
push:
{.upraises: [].}
import std/sequtils
import std/sugar
import std/[sugar, atomics, sequtils]
import pkg/chronos
import pkg/chronos/threadsync
import pkg/chronicles
import pkg/libp2p/[multicodec, cid, multihash]
import pkg/libp2p/protobuf/minprotobuf
import pkg/taskpools
import ../logutils
import ../manifest
@ -28,6 +30,7 @@ import ../utils
import ../utils/asynciter
import ../indexingstrategy
import ../errors
import ../utils/arrayutils
import pkg/stew/byteutils
@ -68,6 +71,7 @@ type
proc(size, blocks, parity: int): DecoderBackend {.raises: [Defect], noSideEffect.}
Erasure* = ref object
taskPool: Taskpool
encoderProvider*: EncoderProvider
decoderProvider*: DecoderProvider
store*: BlockStore
@ -87,6 +91,24 @@ type
# provided.
minSize*: NBytes
EncodeTask = object
success: Atomic[bool]
erasure: ptr Erasure
blocks: ptr UncheckedArray[ptr UncheckedArray[byte]]
parity: ptr UncheckedArray[ptr UncheckedArray[byte]]
blockSize, blocksLen, parityLen: int
signal: ThreadSignalPtr
DecodeTask = object
success: Atomic[bool]
erasure: ptr Erasure
blocks: ptr UncheckedArray[ptr UncheckedArray[byte]]
parity: ptr UncheckedArray[ptr UncheckedArray[byte]]
recovered: ptr UncheckedArray[ptr UncheckedArray[byte]]
blockSize, blocksLen: int
parityLen, recoveredLen: int
signal: ThreadSignalPtr
func indexToPos(steps, idx, step: int): int {.inline.} =
## Convert an index to a position in the encoded
## dataset
@ -269,6 +291,81 @@ proc init*(
strategy: strategy,
)
proc leopardEncodeTask(tp: Taskpool, task: ptr EncodeTask) {.gcsafe.} =
# Task suitable for running in taskpools - look, no GC!
let encoder =
task[].erasure.encoderProvider(task[].blockSize, task[].blocksLen, task[].parityLen)
defer:
encoder.release()
discard task[].signal.fireSync()
if (
let res =
encoder.encode(task[].blocks, task[].parity, task[].blocksLen, task[].parityLen)
res.isErr
):
warn "Error from leopard encoder backend!", error = $res.error
task[].success.store(false)
else:
task[].success.store(true)
proc encodeAsync*(
self: Erasure,
blockSize, blocksLen, parityLen: int,
data: ref seq[seq[byte]],
parity: ptr UncheckedArray[ptr UncheckedArray[byte]],
): Future[?!void] {.async: (raises: [CancelledError]).} =
without threadPtr =? ThreadSignalPtr.new():
return failure("Unable to create thread signal")
defer:
threadPtr.close().expect("closing once works")
var blockData = createDoubleArray(blocksLen, blockSize)
for i in 0 ..< data[].len:
copyMem(blockData[i], addr data[i][0], blockSize)
defer:
freeDoubleArray(blockData, blocksLen)
## Create an ecode task with block data
var task = EncodeTask(
erasure: addr self,
blockSize: blockSize,
blocksLen: blocksLen,
parityLen: parityLen,
blocks: blockData,
parity: parity,
signal: threadPtr,
)
let t = addr task
doAssert self.taskPool.numThreads > 1,
"Must have at least one separate thread or signal will never be fired"
self.taskPool.spawn leopardEncodeTask(self.taskPool, t)
let threadFut = threadPtr.wait()
try:
await threadFut.join()
except CatchableError as exc:
try:
await threadFut
except AsyncError as asyncExc:
return failure(asyncExc.msg)
finally:
if exc of CancelledError:
raise (ref CancelledError) exc
else:
return failure(exc.msg)
if not t.success.load():
return failure("Leopard encoding failed")
success()
proc encodeData(
self: Erasure, manifest: Manifest, params: EncodingParams
): Future[?!Manifest] {.async.} =
@ -276,7 +373,6 @@ proc encodeData(
##
## `manifest` - the manifest to encode
##
logScope:
steps = params.steps
rounded_blocks = params.rounded
@ -286,7 +382,6 @@ proc encodeData(
var
cids = seq[Cid].new()
encoder = self.encoderProvider(manifest.blockSize.int, params.ecK, params.ecM)
emptyBlock = newSeq[byte](manifest.blockSize.int)
cids[].setLen(params.blocksCount)
@ -296,8 +391,7 @@ proc encodeData(
# TODO: Don't allocate a new seq every time, allocate once and zero out
var
data = seq[seq[byte]].new() # number of blocks to encode
parityData =
newSeqWith[seq[byte]](params.ecM, newSeq[byte](manifest.blockSize.int))
parity = createDoubleArray(params.ecM, manifest.blockSize.int)
data[].setLen(params.ecK)
# TODO: this is a tight blocking loop so we sleep here to allow
@ -311,15 +405,25 @@ proc encodeData(
trace "Unable to prepare data", error = err.msg
return failure(err)
trace "Erasure coding data", data = data[].len, parity = parityData.len
trace "Erasure coding data", data = data[].len
if (let res = encoder.encode(data[], parityData); res.isErr):
trace "Unable to encode manifest!", error = $res.error
return failure($res.error)
try:
if err =? (
await self.encodeAsync(
manifest.blockSize.int, params.ecK, params.ecM, data, parity
)
).errorOption:
return failure(err)
except CancelledError as exc:
raise exc
finally:
freeDoubleArray(parity, params.ecM)
var idx = params.rounded + step
for j in 0 ..< params.ecM:
without blk =? bt.Block.new(parityData[j]), error:
var innerPtr: ptr UncheckedArray[byte] = parity[][j]
without blk =? bt.Block.new(innerPtr.toOpenArray(0, manifest.blockSize.int - 1)),
error:
trace "Unable to create parity block", err = error.msg
return failure(error)
@ -356,8 +460,6 @@ proc encodeData(
except CatchableError as exc:
trace "Erasure coding encoding error", exc = exc.msg
return failure(exc)
finally:
encoder.release()
proc encode*(
self: Erasure,
@ -381,6 +483,101 @@ proc encode*(
return success encodedManifest
proc leopardDecodeTask(tp: Taskpool, task: ptr DecodeTask) {.gcsafe.} =
# Task suitable for running in taskpools - look, no GC!
let decoder =
task[].erasure.decoderProvider(task[].blockSize, task[].blocksLen, task[].parityLen)
defer:
decoder.release()
if (
let res = decoder.decode(
task[].blocks,
task[].parity,
task[].recovered,
task[].blocksLen,
task[].parityLen,
task[].recoveredLen,
)
res.isErr
):
warn "Error from leopard decoder backend!", error = $res.error
task[].success.store(false)
else:
task[].success.store(true)
discard task[].signal.fireSync()
proc decodeAsync*(
self: Erasure,
blockSize, blocksLen, parityLen: int,
blocks, parity: ref seq[seq[byte]],
recovered: ptr UncheckedArray[ptr UncheckedArray[byte]],
): Future[?!void] {.async: (raises: [CancelledError]).} =
without threadPtr =? ThreadSignalPtr.new():
return failure("Unable to create thread signal")
defer:
threadPtr.close().expect("closing once works")
var
blocksData = createDoubleArray(blocksLen, blockSize)
parityData = createDoubleArray(parityLen, blockSize)
for i in 0 ..< blocks[].len:
if blocks[i].len > 0:
copyMem(blocksData[i], addr blocks[i][0], blockSize)
else:
blocksData[i] = nil
for i in 0 ..< parity[].len:
if parity[i].len > 0:
copyMem(parityData[i], addr parity[i][0], blockSize)
else:
parityData[i] = nil
defer:
freeDoubleArray(blocksData, blocksLen)
freeDoubleArray(parityData, parityLen)
## Create an decode task with block data
var task = DecodeTask(
erasure: addr self,
blockSize: blockSize,
blocksLen: blocksLen,
parityLen: parityLen,
recoveredLen: blocksLen,
blocks: blocksData,
parity: parityData,
recovered: recovered,
signal: threadPtr,
)
# Hold the task pointer until the signal is received
let t = addr task
doAssert self.taskPool.numThreads > 1,
"Must have at least one separate thread or signal will never be fired"
self.taskPool.spawn leopardDecodeTask(self.taskPool, t)
let threadFut = threadPtr.wait()
try:
await threadFut.join()
except CatchableError as exc:
try:
await threadFut
except AsyncError as asyncExc:
return failure(asyncExc.msg)
finally:
if exc of CancelledError:
raise (ref CancelledError) exc
else:
return failure(exc.msg)
if not t.success.load():
return failure("Leopard encoding failed")
success()
proc decode*(self: Erasure, encoded: Manifest): Future[?!Manifest] {.async.} =
## Decode a protected manifest into it's original
## manifest
@ -388,7 +585,6 @@ proc decode*(self: Erasure, encoded: Manifest): Future[?!Manifest] {.async.} =
## `encoded` - the encoded (protected) manifest to
## be recovered
##
logScope:
steps = encoded.steps
rounded_blocks = encoded.rounded
@ -411,8 +607,7 @@ proc decode*(self: Erasure, encoded: Manifest): Future[?!Manifest] {.async.} =
var
data = seq[seq[byte]].new()
parityData = seq[seq[byte]].new()
recovered =
newSeqWith[seq[byte]](encoded.ecK, newSeq[byte](encoded.blockSize.int))
recovered = createDoubleArray(encoded.ecK, encoded.blockSize.int)
data[].setLen(encoded.ecK) # set len to K
parityData[].setLen(encoded.ecM) # set len to M
@ -430,15 +625,26 @@ proc decode*(self: Erasure, encoded: Manifest): Future[?!Manifest] {.async.} =
continue
trace "Erasure decoding data"
if (let err = decoder.decode(data[], parityData[], recovered); err.isErr):
trace "Unable to decode data!", err = $err.error
return failure($err.error)
try:
if err =? (
await self.decodeAsync(
encoded.blockSize.int, encoded.ecK, encoded.ecM, data, parityData, recovered
)
).errorOption:
return failure(err)
except CancelledError as exc:
raise exc
finally:
freeDoubleArray(recovered, encoded.ecK)
for i in 0 ..< encoded.ecK:
let idx = i * encoded.steps + step
if data[i].len <= 0 and not cids[idx].isEmpty:
without blk =? bt.Block.new(recovered[i]), error:
var innerPtr: ptr UncheckedArray[byte] = recovered[][i]
without blk =? bt.Block.new(
innerPtr.toOpenArray(0, encoded.blockSize.int - 1)
), error:
trace "Unable to create block!", exc = error.msg
return failure(error)
@ -490,10 +696,13 @@ proc new*(
store: BlockStore,
encoderProvider: EncoderProvider,
decoderProvider: DecoderProvider,
taskPool: Taskpool,
): Erasure =
## Create a new Erasure instance for encoding and decoding manifests
##
Erasure(
store: store, encoderProvider: encoderProvider, decoderProvider: decoderProvider
store: store,
encoderProvider: encoderProvider,
decoderProvider: decoderProvider,
taskPool: taskPool,
)

2
codex/errors.nim
View File

@ -9,7 +9,7 @@
import std/options
import pkg/stew/results
import pkg/results
import pkg/chronos
import pkg/questionable/results

11
codex/manifest/coders.nim
View File

@ -63,7 +63,6 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
# optional ErasureInfo erasure = 7; # erasure coding info
# optional filename: ?string = 8; # original filename
# optional mimetype: ?string = 9; # original mimetype
# optional uploadedAt: ?int64 = 10; # original uploadedAt
# }
# ```
#
@ -102,9 +101,6 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
if manifest.mimetype.isSome:
header.write(9, manifest.mimetype.get())
if manifest.uploadedAt.isSome:
header.write(10, manifest.uploadedAt.get().uint64)
pbNode.write(1, header) # set the treeCid as the data field
pbNode.finish()
@ -135,7 +131,6 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
verifiableStrategy: uint32
filename: string
mimetype: string
uploadedAt: uint64
# Decode `Header` message
if pbNode.getField(1, pbHeader).isErr:
@ -169,9 +164,6 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
if pbHeader.getField(9, mimetype).isErr:
return failure("Unable to decode `mimetype` from manifest!")
if pbHeader.getField(10, uploadedAt).isErr:
return failure("Unable to decode `uploadedAt` from manifest!")
let protected = pbErasureInfo.buffer.len > 0
var verifiable = false
if protected:
@ -211,7 +203,6 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
var filenameOption = if filename.len == 0: string.none else: filename.some
var mimetypeOption = if mimetype.len == 0: string.none else: mimetype.some
var uploadedAtOption = if uploadedAt == 0: int64.none else: uploadedAt.int64.some
let self =
if protected:
@ -229,7 +220,6 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
strategy = StrategyType(protectedStrategy),
filename = filenameOption,
mimetype = mimetypeOption,
uploadedAt = uploadedAtOption,
)
else:
Manifest.new(
@ -241,7 +231,6 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
codec = codec.MultiCodec,
filename = filenameOption,
mimetype = mimetypeOption,
uploadedAt = uploadedAtOption,
)
?self.verify()

18
codex/manifest/manifest.nim
View File

@ -38,7 +38,6 @@ type Manifest* = ref object of RootObj
version: CidVersion # Cid version
filename {.serialize.}: ?string # The filename of the content uploaded (optional)
mimetype {.serialize.}: ?string # The mimetype of the content uploaded (optional)
uploadedAt {.serialize.}: ?int64 # The UTC creation timestamp in seconds
case protected {.serialize.}: bool # Protected datasets have erasure coded info
of true:
ecK: int # Number of blocks to encode
@ -131,8 +130,6 @@ func filename*(self: Manifest): ?string =
func mimetype*(self: Manifest): ?string =
self.mimetype
func uploadedAt*(self: Manifest): ?int64 =
self.uploadedAt
############################################################
# Operations on block list
############################################################
@ -165,14 +162,11 @@ func verify*(self: Manifest): ?!void =
return success()
func cid*(self: Manifest): ?!Cid {.deprecated: "use treeCid instead".} =
self.treeCid.success
func `==`*(a, b: Manifest): bool =
(a.treeCid == b.treeCid) and (a.datasetSize == b.datasetSize) and
(a.blockSize == b.blockSize) and (a.version == b.version) and (a.hcodec == b.hcodec) and
(a.codec == b.codec) and (a.protected == b.protected) and (a.filename == b.filename) and
(a.mimetype == b.mimetype) and (a.uploadedAt == b.uploadedAt) and (
(a.mimetype == b.mimetype) and (
if a.protected:
(a.ecK == b.ecK) and (a.ecM == b.ecM) and (a.originalTreeCid == b.originalTreeCid) and
(a.originalDatasetSize == b.originalDatasetSize) and
@ -202,9 +196,6 @@ func `$`*(self: Manifest): string =
if self.mimetype.isSome:
result &= ", mimetype: " & $self.mimetype
if self.uploadedAt.isSome:
result &= ", uploadedAt: " & $self.uploadedAt
result &= (
if self.protected:
", ecK: " & $self.ecK & ", ecM: " & $self.ecM & ", originalTreeCid: " &
@ -236,7 +227,6 @@ func new*(
protected = false,
filename: ?string = string.none,
mimetype: ?string = string.none,
uploadedAt: ?int64 = int64.none,
): Manifest =
T(
treeCid: treeCid,
@ -248,7 +238,6 @@ func new*(
protected: protected,
filename: filename,
mimetype: mimetype,
uploadedAt: uploadedAt,
)
func new*(
@ -278,7 +267,6 @@ func new*(
protectedStrategy: strategy,
filename: manifest.filename,
mimetype: manifest.mimetype,
uploadedAt: manifest.uploadedAt,
)
func new*(T: type Manifest, manifest: Manifest): Manifest =
@ -296,7 +284,6 @@ func new*(T: type Manifest, manifest: Manifest): Manifest =
protected: false,
filename: manifest.filename,
mimetype: manifest.mimetype,
uploadedAt: manifest.uploadedAt,
)
func new*(
@ -314,7 +301,6 @@ func new*(
strategy = SteppedStrategy,
filename: ?string = string.none,
mimetype: ?string = string.none,
uploadedAt: ?int64 = int64.none,
): Manifest =
Manifest(
treeCid: treeCid,
@ -331,7 +317,6 @@ func new*(
protectedStrategy: strategy,
filename: filename,
mimetype: mimetype,
uploadedAt: uploadedAt,
)
func new*(
@ -374,7 +359,6 @@ func new*(
verifiableStrategy: strategy,
filename: manifest.filename,
mimetype: manifest.mimetype,
uploadedAt: manifest.uploadedAt,
)
func new*(T: type Manifest, data: openArray[byte]): ?!Manifest =

32
codex/market.nim
View File

@ -20,13 +20,12 @@ type
MarketError* = object of CodexError
Subscription* = ref object of RootObj
OnRequest* =
proc(id: RequestId, ask: StorageAsk, expiry: UInt256) {.gcsafe, upraises: [].}
proc(id: RequestId, ask: StorageAsk, expiry: uint64) {.gcsafe, upraises: [].}
OnFulfillment* = proc(requestId: RequestId) {.gcsafe, upraises: [].}
OnSlotFilled* =
proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises: [].}
OnSlotFreed* = proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises: [].}
OnSlotFilled* = proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, upraises: [].}
OnSlotFreed* = proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, upraises: [].}
OnSlotReservationsFull* =
proc(requestId: RequestId, slotIndex: UInt256) {.gcsafe, upraises: [].}
proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, upraises: [].}
OnRequestCancelled* = proc(requestId: RequestId) {.gcsafe, upraises: [].}
OnRequestFailed* = proc(requestId: RequestId) {.gcsafe, upraises: [].}
OnProofSubmitted* = proc(id: SlotId) {.gcsafe, upraises: [].}
@ -37,19 +36,19 @@ type
StorageRequested* = object of MarketplaceEvent
requestId*: RequestId
ask*: StorageAsk
expiry*: UInt256
expiry*: uint64
SlotFilled* = object of MarketplaceEvent
requestId* {.indexed.}: RequestId
slotIndex*: UInt256
slotIndex*: uint64
SlotFreed* = object of MarketplaceEvent
requestId* {.indexed.}: RequestId
slotIndex*: UInt256
slotIndex*: uint64
SlotReservationsFull* = object of MarketplaceEvent
requestId* {.indexed.}: RequestId
slotIndex*: UInt256
slotIndex*: uint64
RequestFulfilled* = object of MarketplaceEvent
requestId* {.indexed.}: RequestId
@ -72,12 +71,15 @@ method getSigner*(market: Market): Future[Address] {.base, async.} =
method periodicity*(market: Market): Future[Periodicity] {.base, async.} =
raiseAssert("not implemented")
method proofTimeout*(market: Market): Future[UInt256] {.base, async.} =
method proofTimeout*(market: Market): Future[uint64] {.base, async.} =
raiseAssert("not implemented")
method repairRewardPercentage*(market: Market): Future[uint8] {.base, async.} =
raiseAssert("not implemented")
method requestDurationLimit*(market: Market): Future[uint64] {.base, async.} =
raiseAssert("not implemented")
method proofDowntime*(market: Market): Future[uint8] {.base, async.} =
raiseAssert("not implemented")
@ -122,7 +124,7 @@ method requestExpiresAt*(
raiseAssert("not implemented")
method getHost*(
market: Market, requestId: RequestId, slotIndex: UInt256
market: Market, requestId: RequestId, slotIndex: uint64
): Future[?Address] {.base, async.} =
raiseAssert("not implemented")
@ -137,7 +139,7 @@ method getActiveSlot*(market: Market, slotId: SlotId): Future[?Slot] {.base, asy
method fillSlot*(
market: Market,
requestId: RequestId,
slotIndex: UInt256,
slotIndex: uint64,
proof: Groth16Proof,
collateral: UInt256,
) {.base, async.} =
@ -177,12 +179,12 @@ method canProofBeMarkedAsMissing*(
raiseAssert("not implemented")
method reserveSlot*(
market: Market, requestId: RequestId, slotIndex: UInt256
market: Market, requestId: RequestId, slotIndex: uint64
) {.base, async.} =
raiseAssert("not implemented")
method canReserveSlot*(
market: Market, requestId: RequestId, slotIndex: UInt256
market: Market, requestId: RequestId, slotIndex: uint64
): Future[bool] {.base, async.} =
raiseAssert("not implemented")
@ -202,7 +204,7 @@ method subscribeSlotFilled*(
raiseAssert("not implemented")
method subscribeSlotFilled*(
market: Market, requestId: RequestId, slotIndex: UInt256, callback: OnSlotFilled
market: Market, requestId: RequestId, slotIndex: uint64, callback: OnSlotFilled
): Future[Subscription] {.base, async.} =
raiseAssert("not implemented")

148
codex/node.nim
View File

@ -15,6 +15,7 @@ import std/strformat
import std/sugar
import times
import pkg/taskpools
import pkg/questionable
import pkg/questionable/results
import pkg/chronos
@ -70,6 +71,7 @@ type
contracts*: Contracts
clock*: Clock
storage*: Contracts
taskpool: Taskpool
CodexNodeRef* = ref CodexNode
@ -235,8 +237,9 @@ proc streamEntireDataset(
# Retrieve, decode and save to the local store all EС groups
proc erasureJob(): Future[?!void] {.async.} =
# Spawn an erasure decoding job
let erasure =
Erasure.new(self.networkStore, leoEncoderProvider, leoDecoderProvider)
let erasure = Erasure.new(
self.networkStore, leoEncoderProvider, leoDecoderProvider, self.taskpool
)
without _ =? (await erasure.decode(manifest)), error:
error "Unable to erasure decode manifest", manifestCid, exc = error.msg
return failure(error)
@ -267,6 +270,65 @@ proc retrieve*(
await self.streamEntireDataset(manifest, cid)
proc deleteSingleBlock(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} =
if err =? (await self.networkStore.delBlock(cid)).errorOption:
error "Error deleting block", cid, err = err.msg
return failure(err)
trace "Deleted block", cid
return success()
proc deleteEntireDataset(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} =
# Deletion is a strictly local operation
var store = self.networkStore.localStore
if not (await cid in store):
# As per the contract for delete*, an absent dataset is not an error.
return success()
without manifestBlock =? await store.getBlock(cid), err:
return failure(err)
without manifest =? Manifest.decode(manifestBlock), err:
return failure(err)
let runtimeQuota = initDuration(milliseconds = 100)
var lastIdle = getTime()
for i in 0 ..< manifest.blocksCount:
if (getTime() - lastIdle) >= runtimeQuota:
await idleAsync()
lastIdle = getTime()
if err =? (await store.delBlock(manifest.treeCid, i)).errorOption:
# The contract for delBlock is fuzzy, but we assume that if the block is
# simply missing we won't get an error. This is a best effort operation and
# can simply be retried.
error "Failed to delete block within dataset", index = i, err = err.msg
return failure(err)
if err =? (await store.delBlock(cid)).errorOption:
error "Error deleting manifest block", err = err.msg
success()
proc delete*(
self: CodexNodeRef, cid: Cid
): Future[?!void] {.async: (raises: [CatchableError]).} =
## Deletes a whole dataset, if Cid is a Manifest Cid, or a single block, if Cid a block Cid,
## from the underlying block store. This is a strictly local operation.
##
## Missing blocks in dataset deletes are ignored.
##
without isManifest =? cid.isManifest, err:
trace "Bad content type for CID:", cid = cid, err = err.msg
return failure(err)
if not isManifest:
return await self.deleteSingleBlock(cid)
await self.deleteEntireDataset(cid)
proc store*(
self: CodexNodeRef,
stream: LPStream,
@ -332,7 +394,6 @@ proc store*(
codec = dataCodec,
filename = filename,
mimetype = mimetype,
uploadedAt = now().utc.toTime.toUnix.some,
)
without manifestBlk =? await self.storeManifest(manifest), err:
@ -369,13 +430,13 @@ proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} =
proc setupRequest(
self: CodexNodeRef,
cid: Cid,
duration: UInt256,
duration: uint64,
proofProbability: UInt256,
nodes: uint,
tolerance: uint,
pricePerBytePerSecond: UInt256,
collateralPerByte: UInt256,
expiry: UInt256,
expiry: uint64,
): Future[?!StorageRequest] {.async.} =
## Setup slots for a given dataset
##
@ -403,8 +464,9 @@ proc setupRequest(
return failure error
# Erasure code the dataset according to provided parameters
let erasure =
Erasure.new(self.networkStore.localStore, leoEncoderProvider, leoDecoderProvider)
let erasure = Erasure.new(
self.networkStore.localStore, leoEncoderProvider, leoDecoderProvider, self.taskpool
)
without encoded =? (await erasure.encode(manifest, ecK, ecM)), error:
trace "Unable to erasure code dataset"
@ -432,17 +494,14 @@ proc setupRequest(
request = StorageRequest(
ask: StorageAsk(
slots: verifiable.numSlots.uint64,
slotSize: builder.slotBytes.uint.u256,
slotSize: builder.slotBytes.uint64,
duration: duration,
proofProbability: proofProbability,
pricePerBytePerSecond: pricePerBytePerSecond,
collateralPerByte: collateralPerByte,
maxSlotLoss: tolerance,
),
content: StorageContent(
cid: $manifestBlk.cid, # TODO: why string?
merkleRoot: verifyRoot,
),
content: StorageContent(cid: manifestBlk.cid, merkleRoot: verifyRoot),
expiry: expiry,
)
@ -452,13 +511,13 @@ proc setupRequest(
proc requestStorage*(
self: CodexNodeRef,
cid: Cid,
duration: UInt256,
duration: uint64,
proofProbability: UInt256,
nodes: uint,
tolerance: uint,
pricePerBytePerSecond: UInt256,
collateralPerByte: UInt256,
expiry: UInt256,
expiry: uint64,
): Future[?!PurchaseId] {.async.} =
## Initiate a request for storage sequence, this might
## be a multistep procedure.
@ -472,7 +531,7 @@ proc requestStorage*(
pricePerBytePerSecond = pricePerBytePerSecond
proofProbability = proofProbability
collateralPerByte = collateralPerByte
expiry = expiry.truncate(int64)
expiry = expiry
now = self.clock.now
trace "Received a request for storage!"
@ -494,21 +553,19 @@ proc requestStorage*(
success purchase.id
proc onStore(
self: CodexNodeRef, request: StorageRequest, slotIdx: UInt256, blocksCb: BlocksCb
self: CodexNodeRef, request: StorageRequest, slotIdx: uint64, blocksCb: BlocksCb
): Future[?!void] {.async.} =
## store data in local storage
##
let cid = request.content.cid
logScope:
cid = request.content.cid
cid = $cid
slotIdx = slotIdx
trace "Received a request to store a slot"
without cid =? Cid.init(request.content.cid).mapFailure, err:
trace "Unable to parse Cid", cid
return failure(err)
without manifest =? (await self.fetchManifest(cid)), err:
trace "Unable to fetch manifest for cid", cid, err = err.msg
return failure(err)
@ -518,11 +575,9 @@ proc onStore(
trace "Unable to create slots builder", err = err.msg
return failure(err)
let
slotIdx = slotIdx.truncate(int)
expiry = request.expiry.toSecondsSince1970
let expiry = request.expiry
if slotIdx > manifest.slotRoots.high:
if slotIdx > manifest.slotRoots.high.uint64:
trace "Slot index not in manifest", slotIdx
return failure(newException(CodexError, "Slot index not in manifest"))
@ -530,7 +585,7 @@ proc onStore(
trace "Updating expiry for blocks", blocks = blocks.len
let ensureExpiryFutures =
blocks.mapIt(self.networkStore.ensureExpiry(it.cid, expiry))
blocks.mapIt(self.networkStore.ensureExpiry(it.cid, expiry.toSecondsSince1970))
if updateExpiryErr =? (await allFutureResult(ensureExpiryFutures)).errorOption:
return failure(updateExpiryErr)
@ -546,7 +601,11 @@ proc onStore(
trace "Unable to create indexing strategy from protected manifest", err = err.msg
return failure(err)
without blksIter =? indexer.getIndicies(slotIdx).catch, err:
if slotIdx > int.high.uint64:
error "Cannot cast slot index to int", slotIndex = slotIdx
return
without blksIter =? indexer.getIndicies(slotIdx.int).catch, err:
trace "Unable to get indicies from strategy", err = err.msg
return failure(err)
@ -556,13 +615,13 @@ proc onStore(
trace "Unable to fetch blocks", err = err.msg
return failure(err)
without slotRoot =? (await builder.buildSlot(slotIdx.Natural)), err:
without slotRoot =? (await builder.buildSlot(slotIdx.int)), err:
trace "Unable to build slot", err = err.msg
return failure(err)
trace "Slot successfully retrieved and reconstructed"
if cid =? slotRoot.toSlotCid() and cid != manifest.slotRoots[slotIdx.int]:
if cid =? slotRoot.toSlotCid() and cid != manifest.slotRoots[slotIdx]:
trace "Slot root mismatch",
manifest = manifest.slotRoots[slotIdx.int], recovered = slotRoot.toSlotCid()
return failure(newException(CodexError, "Slot root mismatch"))
@ -578,8 +637,8 @@ proc onProve(
##
let
cidStr = slot.request.content.cid
slotIdx = slot.slotIndex.truncate(Natural)
cidStr = $slot.request.content.cid
slotIdx = slot.slotIndex
logScope:
cid = cidStr
@ -600,7 +659,8 @@ proc onProve(
return failure(err)
when defined(verify_circuit):
without (inputs, proof) =? await prover.prove(slotIdx, manifest, challenge), err:
without (inputs, proof) =? await prover.prove(slotIdx.int, manifest, challenge),
err:
error "Unable to generate proof", err = err.msg
return failure(err)
@ -614,7 +674,7 @@ proc onProve(
trace "Proof verified successfully"
else:
without (_, proof) =? await prover.prove(slotIdx, manifest, challenge), err:
without (_, proof) =? await prover.prove(slotIdx.int, manifest, challenge), err:
error "Unable to generate proof", err = err.msg
return failure(err)
@ -627,16 +687,11 @@ proc onProve(
failure "Prover not enabled"
proc onExpiryUpdate(
self: CodexNodeRef, rootCid: string, expiry: SecondsSince1970
self: CodexNodeRef, rootCid: Cid, expiry: SecondsSince1970
): Future[?!void] {.async.} =
without cid =? Cid.init(rootCid):
trace "Unable to parse Cid", cid
let error = newException(CodexError, "Unable to parse Cid")
return failure(error)
return await self.updateExpiry(rootCid, expiry)
return await self.updateExpiry(cid, expiry)
proc onClear(self: CodexNodeRef, request: StorageRequest, slotIndex: UInt256) =
proc onClear(self: CodexNodeRef, request: StorageRequest, slotIndex: uint64) =
# TODO: remove data from local storage
discard
@ -652,16 +707,16 @@ proc start*(self: CodexNodeRef) {.async.} =
if hostContracts =? self.contracts.host:
hostContracts.sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
request: StorageRequest, slot: uint64, onBatch: BatchProc
): Future[?!void] =
self.onStore(request, slot, onBatch)
hostContracts.sales.onExpiryUpdate = proc(
rootCid: string, expiry: SecondsSince1970
rootCid: Cid, expiry: SecondsSince1970
): Future[?!void] =
self.onExpiryUpdate(rootCid, expiry)
hostContracts.sales.onClear = proc(request: StorageRequest, slotIndex: UInt256) =
hostContracts.sales.onClear = proc(request: StorageRequest, slotIndex: uint64) =
# TODO: remove data from local storage
self.onClear(request, slotIndex)
@ -724,12 +779,16 @@ proc stop*(self: CodexNodeRef) {.async.} =
if not self.networkStore.isNil:
await self.networkStore.close
if not self.taskpool.isNil:
self.taskpool.shutdown()
proc new*(
T: type CodexNodeRef,
switch: Switch,
networkStore: NetworkStore,
engine: BlockExcEngine,
discovery: Discovery,
taskpool: Taskpool,
prover = Prover.none,
contracts = Contracts.default,
): CodexNodeRef =
@ -742,5 +801,6 @@ proc new*(
engine: engine,
prover: prover,
discovery: discovery,
taskPool: taskpool,
contracts: contracts,
)

6
codex/periods.nim
View File

@ -2,10 +2,10 @@ import pkg/stint
type
Periodicity* = object
seconds*: UInt256
seconds*: uint64
Period* = UInt256
Timestamp* = UInt256
Period* = uint64
Timestamp* = uint64
func periodOf*(periodicity: Periodicity, timestamp: Timestamp): Period =
timestamp div periodicity.seconds

2
codex/purchasing.nim
View File

@ -14,7 +14,7 @@ export purchase
type
Purchasing* = ref object
market: Market
market*: Market
clock: Clock
purchases: Table[PurchaseId, Purchase]
proofProbability*: UInt256

24
codex/purchasing/states/cancelled.nim
View File

@ -1,25 +1,35 @@
import pkg/metrics
import ../../logutils
import ../../utils/exceptions
import ../statemachine
import ./errorhandling
import ./error
declareCounter(codex_purchases_cancelled, "codex purchases cancelled")
logScope:
topics = "marketplace purchases cancelled"
type PurchaseCancelled* = ref object of ErrorHandlingState
type PurchaseCancelled* = ref object of PurchaseState
method `$`*(state: PurchaseCancelled): string =
"cancelled"
method run*(state: PurchaseCancelled, machine: Machine): Future[?State] {.async.} =
method run*(
state: PurchaseCancelled, machine: Machine
): Future[?State] {.async: (raises: []).} =
codex_purchases_cancelled.inc()
let purchase = Purchase(machine)
warn "Request cancelled, withdrawing remaining funds", requestId = purchase.requestId
await purchase.market.withdrawFunds(purchase.requestId)
try:
warn "Request cancelled, withdrawing remaining funds",
requestId = purchase.requestId
await purchase.market.withdrawFunds(purchase.requestId)
let error = newException(Timeout, "Purchase cancelled due to timeout")
purchase.future.fail(error)
let error = newException(Timeout, "Purchase cancelled due to timeout")
purchase.future.fail(error)
except CancelledError as e:
trace "PurchaseCancelled.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during PurchaseCancelled.run", error = e.msgDetail
return some State(PurchaseErrored(error: e))

4
codex/purchasing/states/error.nim
View File

@ -14,7 +14,9 @@ type PurchaseErrored* = ref object of PurchaseState
method `$`*(state: PurchaseErrored): string =
"errored"
method run*(state: PurchaseErrored, machine: Machine): Future[?State] {.async.} =
method run*(
state: PurchaseErrored, machine: Machine
): Future[?State] {.async: (raises: []).} =
codex_purchases_error.inc()
let purchase = Purchase(machine)

8
codex/purchasing/states/errorhandling.nim
View File

@ -1,8 +0,0 @@
import pkg/questionable
import ../statemachine
import ./error
type ErrorHandlingState* = ref object of PurchaseState
method onError*(state: ErrorHandlingState, error: ref CatchableError): ?State =
some State(PurchaseErrored(error: error))

16
codex/purchasing/states/failed.nim
View File

@ -1,6 +1,7 @@
import pkg/metrics
import ../statemachine
import ../../logutils
import ../../utils/exceptions
import ./error
declareCounter(codex_purchases_failed, "codex purchases failed")
@ -10,11 +11,20 @@ type PurchaseFailed* = ref object of PurchaseState
method `$`*(state: PurchaseFailed): string =
"failed"
method run*(state: PurchaseFailed, machine: Machine): Future[?State] {.async.} =
method run*(
state: PurchaseFailed, machine: Machine
): Future[?State] {.async: (raises: []).} =
codex_purchases_failed.inc()
let purchase = Purchase(machine)
warn "Request failed, withdrawing remaining funds", requestId = purchase.requestId
await purchase.market.withdrawFunds(purchase.requestId)
try:
warn "Request failed, withdrawing remaining funds", requestId = purchase.requestId
await purchase.market.withdrawFunds(purchase.requestId)
except CancelledError as e:
trace "PurchaseFailed.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during PurchaseFailed.run", error = e.msgDetail
return some State(PurchaseErrored(error: e))
let error = newException(PurchaseError, "Purchase failed")
return some State(PurchaseErrored(error: error))

19
codex/purchasing/states/finished.nim
View File

@ -1,7 +1,9 @@
import pkg/metrics
import ../statemachine
import ../../utils/exceptions
import ../../logutils
import ./error
declareCounter(codex_purchases_finished, "codex purchases finished")
@ -13,10 +15,19 @@ type PurchaseFinished* = ref object of PurchaseState
method `$`*(state: PurchaseFinished): string =
"finished"
method run*(state: PurchaseFinished, machine: Machine): Future[?State] {.async.} =
method run*(
state: PurchaseFinished, machine: Machine
): Future[?State] {.async: (raises: []).} =
codex_purchases_finished.inc()
let purchase = Purchase(machine)
info "Purchase finished, withdrawing remaining funds", requestId = purchase.requestId
await purchase.market.withdrawFunds(purchase.requestId)
try:
info "Purchase finished, withdrawing remaining funds",
requestId = purchase.requestId
await purchase.market.withdrawFunds(purchase.requestId)
purchase.future.complete()
purchase.future.complete()
except CancelledError as e:
trace "PurchaseFinished.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during PurchaseFinished.run", error = e.msgDetail
return some State(PurchaseErrored(error: e))

22
codex/purchasing/states/pending.nim
View File

@ -1,18 +1,28 @@
import pkg/metrics
import ../../logutils
import ../../utils/exceptions
import ../statemachine
import ./errorhandling
import ./submitted
import ./error
declareCounter(codex_purchases_pending, "codex purchases pending")
type PurchasePending* = ref object of ErrorHandlingState
type PurchasePending* = ref object of PurchaseState
method `$`*(state: PurchasePending): string =
"pending"
method run*(state: PurchasePending, machine: Machine): Future[?State] {.async.} =
method run*(
state: PurchasePending, machine: Machine
): Future[?State] {.async: (raises: []).} =
codex_purchases_pending.inc()
let purchase = Purchase(machine)
let request = !purchase.request
await purchase.market.requestStorage(request)
return some State(PurchaseSubmitted())
try:
let request = !purchase.request
await purchase.market.requestStorage(request)
return some State(PurchaseSubmitted())
except CancelledError as e:
trace "PurchasePending.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during PurchasePending.run", error = e.msgDetail
return some State(PurchaseErrored(error: e))

36
codex/purchasing/states/started.nim
View File

@ -1,22 +1,25 @@
import pkg/metrics
import ../../logutils
import ../../utils/exceptions
import ../statemachine
import ./errorhandling
import ./finished
import ./failed
import ./error
declareCounter(codex_purchases_started, "codex purchases started")
logScope:
topics = "marketplace purchases started"
type PurchaseStarted* = ref object of ErrorHandlingState
type PurchaseStarted* = ref object of PurchaseState
method `$`*(state: PurchaseStarted): string =
"started"
method run*(state: PurchaseStarted, machine: Machine): Future[?State] {.async.} =
method run*(
state: PurchaseStarted, machine: Machine
): Future[?State] {.async: (raises: []).} =
codex_purchases_started.inc()
let purchase = Purchase(machine)
@ -28,15 +31,24 @@ method run*(state: PurchaseStarted, machine: Machine): Future[?State] {.async.}
proc callback(_: RequestId) =
failed.complete()
let subscription = await market.subscribeRequestFailed(purchase.requestId, callback)
var ended: Future[void]
try:
let subscription = await market.subscribeRequestFailed(purchase.requestId, callback)
# Ensure that we're past the request end by waiting an additional second
let ended = clock.waitUntil((await market.getRequestEnd(purchase.requestId)) + 1)
let fut = await one(ended, failed)
await subscription.unsubscribe()
if fut.id == failed.id:
# Ensure that we're past the request end by waiting an additional second
ended = clock.waitUntil((await market.getRequestEnd(purchase.requestId)) + 1)
let fut = await one(ended, failed)
await subscription.unsubscribe()
if fut.id == failed.id:
ended.cancelSoon()
return some State(PurchaseFailed())
else:
failed.cancelSoon()
return some State(PurchaseFinished())
except CancelledError as e:
ended.cancelSoon()
return some State(PurchaseFailed())
else:
failed.cancelSoon()
return some State(PurchaseFinished())
trace "PurchaseStarted.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during PurchaseStarted.run", error = e.msgDetail
return some State(PurchaseErrored(error: e))

14
codex/purchasing/states/submitted.nim
View File

@ -1,22 +1,25 @@
import pkg/metrics
import ../../logutils
import ../../utils/exceptions
import ../statemachine
import ./errorhandling
import ./started
import ./cancelled
import ./error
logScope:
topics = "marketplace purchases submitted"
declareCounter(codex_purchases_submitted, "codex purchases submitted")
type PurchaseSubmitted* = ref object of ErrorHandlingState
type PurchaseSubmitted* = ref object of PurchaseState
method `$`*(state: PurchaseSubmitted): string =
"submitted"
method run*(state: PurchaseSubmitted, machine: Machine): Future[?State] {.async.} =
method run*(
state: PurchaseSubmitted, machine: Machine
): Future[?State] {.async: (raises: []).} =
codex_purchases_submitted.inc()
let purchase = Purchase(machine)
let request = !purchase.request
@ -44,5 +47,10 @@ method run*(state: PurchaseSubmitted, machine: Machine): Future[?State] {.async.
await wait().withTimeout()
except Timeout:
return some State(PurchaseCancelled())
except CancelledError as e:
trace "PurchaseSubmitted.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during PurchaseSubmitted.run", error = e.msgDetail
return some State(PurchaseErrored(error: e))
return some State(PurchaseStarted())

48
codex/purchasing/states/unknown.nim
View File

@ -1,34 +1,44 @@
import pkg/metrics
import ../../utils/exceptions
import ../../logutils
import ../statemachine
import ./errorhandling
import ./submitted
import ./started
import ./cancelled
import ./finished
import ./failed
import ./error
declareCounter(codex_purchases_unknown, "codex purchases unknown")
type PurchaseUnknown* = ref object of ErrorHandlingState
type PurchaseUnknown* = ref object of PurchaseState
method `$`*(state: PurchaseUnknown): string =
"unknown"
method run*(state: PurchaseUnknown, machine: Machine): Future[?State] {.async.} =
codex_purchases_unknown.inc()
let purchase = Purchase(machine)
if (request =? await purchase.market.getRequest(purchase.requestId)) and
(requestState =? await purchase.market.requestState(purchase.requestId)):
purchase.request = some request
method run*(
state: PurchaseUnknown, machine: Machine
): Future[?State] {.async: (raises: []).} =
try:
codex_purchases_unknown.inc()
let purchase = Purchase(machine)
if (request =? await purchase.market.getRequest(purchase.requestId)) and
(requestState =? await purchase.market.requestState(purchase.requestId)):
purchase.request = some request
case requestState
of RequestState.New:
return some State(PurchaseSubmitted())
of RequestState.Started:
return some State(PurchaseStarted())
of RequestState.Cancelled:
return some State(PurchaseCancelled())
of RequestState.Finished:
return some State(PurchaseFinished())
of RequestState.Failed:
return some State(PurchaseFailed())
case requestState
of RequestState.New:
return some State(PurchaseSubmitted())
of RequestState.Started:
return some State(PurchaseStarted())
of RequestState.Cancelled:
return some State(PurchaseCancelled())
of RequestState.Finished:
return some State(PurchaseFinished())
of RequestState.Failed:
return some State(PurchaseFailed())
except CancelledError as e:
trace "PurchaseUnknown.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during PurchaseUnknown.run", error = e.msgDetail
return some State(PurchaseErrored(error: e))

40
codex/rest/api.nim
View File

@ -238,6 +238,15 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
let json = await formatManifestBlocks(node)
return RestApiResponse.response($json, contentType = "application/json")
router.api(MethodOptions, "/api/codex/v1/data/{cid}") do(
cid: Cid, resp: HttpResponseRef
) -> RestApiResponse:
if corsOrigin =? allowedOrigin:
resp.setCorsHeaders("GET,DELETE", corsOrigin)
resp.status = Http204
await resp.sendBody("")
router.api(MethodGet, "/api/codex/v1/data/{cid}") do(
cid: Cid, resp: HttpResponseRef
) -> RestApiResponse:
@ -254,6 +263,27 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
await node.retrieveCid(cid.get(), local = true, resp = resp)
router.api(MethodDelete, "/api/codex/v1/data/{cid}") do(
cid: Cid, resp: HttpResponseRef
) -> RestApiResponse:
## Deletes either a single block or an entire dataset
## from the local node. Does nothing and returns 200
## if the dataset is not locally available.
##
var headers = buildCorsHeaders("DELETE", allowedOrigin)
if cid.isErr:
return RestApiResponse.error(Http400, $cid.error(), headers = headers)
if err =? (await node.delete(cid.get())).errorOption:
return RestApiResponse.error(Http500, err.msg, headers = headers)
if corsOrigin =? allowedOrigin:
resp.setCorsHeaders("DELETE", corsOrigin)
resp.status = Http204
await resp.sendBody("")
router.api(MethodPost, "/api/codex/v1/data/{cid}/network") do(
cid: Cid, resp: HttpResponseRef
) -> RestApiResponse:
@ -433,7 +463,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
Http400, "Total size must be larger then zero", headers = headers
)
if not reservations.hasAvailable(restAv.totalSize.truncate(uint)):
if not reservations.hasAvailable(restAv.totalSize):
return
RestApiResponse.error(Http422, "Not enough storage quota", headers = headers)
@ -607,6 +637,14 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
without params =? StorageRequestParams.fromJson(body), error:
return RestApiResponse.error(Http400, error.msg, headers = headers)
let requestDurationLimit = await contracts.purchasing.market.requestDurationLimit
if params.duration > requestDurationLimit:
return RestApiResponse.error(
Http400,
"Duration exceeds limit of " & $requestDurationLimit & " seconds",
headers = headers,
)
let nodes = params.nodes |? 3
let tolerance = params.tolerance |? 1

2
codex/rest/coders.nim
View File

@ -14,7 +14,7 @@ import pkg/chronos
import pkg/libp2p
import pkg/stew/base10
import pkg/stew/byteutils
import pkg/stew/results
import pkg/results
import pkg/stint
import ../sales

12
codex/rest/json.nim
View File

@ -13,11 +13,11 @@ export json
type
StorageRequestParams* = object
duration* {.serialize.}: UInt256
duration* {.serialize.}: uint64
proofProbability* {.serialize.}: UInt256
pricePerBytePerSecond* {.serialize.}: UInt256
collateralPerByte* {.serialize.}: UInt256
expiry* {.serialize.}: ?UInt256
expiry* {.serialize.}: ?uint64
nodes* {.serialize.}: ?uint
tolerance* {.serialize.}: ?uint
@ -28,16 +28,16 @@ type
error* {.serialize.}: ?string
RestAvailability* = object
totalSize* {.serialize.}: UInt256
duration* {.serialize.}: UInt256
totalSize* {.serialize.}: uint64
duration* {.serialize.}: uint64
minPricePerBytePerSecond* {.serialize.}: UInt256
totalCollateral* {.serialize.}: UInt256
freeSize* {.serialize.}: ?UInt256
freeSize* {.serialize.}: ?uint64
RestSalesAgent* = object
state* {.serialize.}: string
requestId* {.serialize.}: RequestId
slotIndex* {.serialize.}: UInt256
slotIndex* {.serialize.}: uint64
request* {.serialize.}: ?StorageRequest
reservation* {.serialize.}: ?Reservation

53
codex/sales.nim
View File

@ -150,16 +150,16 @@ proc cleanUp(
).errorOption:
error "failure deleting reservation", error = deleteErr.msg
if data.slotIndex > uint16.high.uint64:
error "Cannot cast slot index to uint16", slotIndex = data.slotIndex
return
# Re-add items back into the queue to prevent small availabilities from
# draining the queue. Seen items will be ordered last.
if reprocessSlot and request =? data.request:
let queue = sales.context.slotQueue
var seenItem = SlotQueueItem.init(
data.requestId,
data.slotIndex.truncate(uint16),
data.ask,
request.expiry,
seen = true,
data.requestId, data.slotIndex.uint16, data.ask, request.expiry, seen = true
)
trace "pushing ignored item to queue, marked as seen"
if err =? queue.push(seenItem).errorOption:
@ -172,7 +172,7 @@ proc cleanUp(
processing.complete()
proc filled(
sales: Sales, request: StorageRequest, slotIndex: UInt256, processing: Future[void]
sales: Sales, request: StorageRequest, slotIndex: uint64, processing: Future[void]
) =
if onSale =? sales.context.onSale:
onSale(request, slotIndex)
@ -184,16 +184,15 @@ proc filled(
proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) =
debug "Processing slot from queue", requestId = item.requestId, slot = item.slotIndex
let agent = newSalesAgent(
sales.context, item.requestId, item.slotIndex.u256, none StorageRequest
)
let agent =
newSalesAgent(sales.context, item.requestId, item.slotIndex, none StorageRequest)
agent.onCleanUp = proc(
returnBytes = false, reprocessSlot = false, returnedCollateral = UInt256.none
) {.async.} =
await sales.cleanUp(agent, returnBytes, reprocessSlot, returnedCollateral, done)
agent.onFilled = some proc(request: StorageRequest, slotIndex: UInt256) =
agent.onFilled = some proc(request: StorageRequest, slotIndex: uint64) =
sales.filled(request, slotIndex, done)
agent.start(SalePreparing())
@ -283,7 +282,7 @@ proc onAvailabilityAdded(sales: Sales, availability: Availability) {.async.} =
queue.unpause()
proc onStorageRequested(
sales: Sales, requestId: RequestId, ask: StorageAsk, expiry: UInt256
sales: Sales, requestId: RequestId, ask: StorageAsk, expiry: uint64
) =
logScope:
topics = "marketplace sales onStorageRequested"
@ -312,7 +311,7 @@ proc onStorageRequested(
else:
warn "Error adding request to SlotQueue", error = err.msg
proc onSlotFreed(sales: Sales, requestId: RequestId, slotIndex: UInt256) =
proc onSlotFreed(sales: Sales, requestId: RequestId, slotIndex: uint64) =
logScope:
topics = "marketplace sales onSlotFreed"
requestId
@ -325,8 +324,12 @@ proc onSlotFreed(sales: Sales, requestId: RequestId, slotIndex: UInt256) =
let market = context.market
let queue = context.slotQueue
# first attempt to populate request using existing slot metadata in queue
without var found =? queue.populateItem(requestId, slotIndex.truncate(uint16)):
if slotIndex > uint16.high.uint64:
error "Cannot cast slot index to uint16, value = ", slotIndex
return
# first attempt to populate request using existing metadata in queue
without var found =? queue.populateItem(requestId, slotIndex.uint16):
trace "no existing request metadata, getting request info from contract"
# if there's no existing slot for that request, retrieve the request
# from the contract.
@ -335,7 +338,7 @@ proc onSlotFreed(sales: Sales, requestId: RequestId, slotIndex: UInt256) =
error "unknown request in contract"
return
found = SlotQueueItem.init(request, slotIndex.truncate(uint16))
found = SlotQueueItem.init(request, slotIndex.uint16)
except CancelledError:
discard # do not propagate as addSlotToQueue was asyncSpawned
except CatchableError as e:
@ -353,7 +356,7 @@ proc subscribeRequested(sales: Sales) {.async.} =
let context = sales.context
let market = context.market
proc onStorageRequested(requestId: RequestId, ask: StorageAsk, expiry: UInt256) =
proc onStorageRequested(requestId: RequestId, ask: StorageAsk, expiry: uint64) =
sales.onStorageRequested(requestId, ask, expiry)
try:
@ -426,9 +429,13 @@ proc subscribeSlotFilled(sales: Sales) {.async.} =
let market = context.market
let queue = context.slotQueue
proc onSlotFilled(requestId: RequestId, slotIndex: UInt256) =
proc onSlotFilled(requestId: RequestId, slotIndex: uint64) =
if slotIndex > uint16.high.uint64:
error "Cannot cast slot index to uint16, value = ", slotIndex
return
trace "slot filled, removing from slot queue", requestId, slotIndex
queue.delete(requestId, slotIndex.truncate(uint16))
queue.delete(requestId, slotIndex.uint16)
for agent in sales.agents:
agent.onSlotFilled(requestId, slotIndex)
@ -445,7 +452,7 @@ proc subscribeSlotFreed(sales: Sales) {.async.} =
let context = sales.context
let market = context.market
proc onSlotFreed(requestId: RequestId, slotIndex: UInt256) =
proc onSlotFreed(requestId: RequestId, slotIndex: uint64) =
sales.onSlotFreed(requestId, slotIndex)
try:
@ -461,9 +468,13 @@ proc subscribeSlotReservationsFull(sales: Sales) {.async.} =
let market = context.market
let queue = context.slotQueue
proc onSlotReservationsFull(requestId: RequestId, slotIndex: UInt256) =
proc onSlotReservationsFull(requestId: RequestId, slotIndex: uint64) =
if slotIndex > uint16.high.uint64:
error "Cannot cast slot index to uint16, value = ", slotIndex
return
trace "reservations for slot full, removing from slot queue", requestId, slotIndex
queue.delete(requestId, slotIndex.truncate(uint16))
queue.delete(requestId, slotIndex.uint16)
try:
let sub = await market.subscribeSlotReservationsFull(onSlotReservationsFull)

59
codex/sales/reservations.nim
View File

@ -64,9 +64,9 @@ type
SomeStorableId = AvailabilityId | ReservationId
Availability* = ref object
id* {.serialize.}: AvailabilityId
totalSize* {.serialize.}: UInt256
freeSize* {.serialize.}: UInt256
duration* {.serialize.}: UInt256
totalSize* {.serialize.}: uint64
freeSize* {.serialize.}: uint64
duration* {.serialize.}: uint64
minPricePerBytePerSecond* {.serialize.}: UInt256
totalCollateral {.serialize.}: UInt256
totalRemainingCollateral* {.serialize.}: UInt256
@ -74,9 +74,9 @@ type
Reservation* = ref object
id* {.serialize.}: ReservationId
availabilityId* {.serialize.}: AvailabilityId
size* {.serialize.}: UInt256
size* {.serialize.}: uint64
requestId* {.serialize.}: RequestId
slotIndex* {.serialize.}: UInt256
slotIndex* {.serialize.}: uint64
Reservations* = ref object of RootObj
availabilityLock: AsyncLock
@ -123,9 +123,9 @@ proc new*(T: type Reservations, repo: RepoStore): Reservations =
proc init*(
_: type Availability,
totalSize: UInt256,
freeSize: UInt256,
duration: UInt256,
totalSize: uint64,
freeSize: uint64,
duration: uint64,
minPricePerBytePerSecond: UInt256,
totalCollateral: UInt256,
): Availability =
@ -151,9 +151,9 @@ proc `totalCollateral=`*(self: Availability, value: UInt256) {.inline.} =
proc init*(
_: type Reservation,
availabilityId: AvailabilityId,
size: UInt256,
size: uint64,
requestId: RequestId,
slotIndex: UInt256,
slotIndex: uint64,
): Reservation =
var id: array[32, byte]
doAssert randomBytes(id) == 32
@ -206,7 +206,7 @@ func key*(availability: Availability): ?!Key =
return availability.id.key
func maxCollateralPerByte*(availability: Availability): UInt256 =
return availability.totalRemainingCollateral div availability.freeSize
return availability.totalRemainingCollateral div availability.freeSize.stuint(256)
func key*(reservation: Reservation): ?!Key =
return key(reservation.id, reservation.availabilityId)
@ -289,16 +289,12 @@ proc updateAvailability(
trace "totalSize changed, updating repo reservation"
if oldAvailability.totalSize < obj.totalSize: # storage added
if reserveErr =? (
await self.repo.reserve(
(obj.totalSize - oldAvailability.totalSize).truncate(uint).NBytes
)
await self.repo.reserve((obj.totalSize - oldAvailability.totalSize).NBytes)
).errorOption:
return failure(reserveErr.toErr(ReserveFailedError))
elif oldAvailability.totalSize > obj.totalSize: # storage removed
if reserveErr =? (
await self.repo.release(
(oldAvailability.totalSize - obj.totalSize).truncate(uint).NBytes
)
await self.repo.release((oldAvailability.totalSize - obj.totalSize).NBytes)
).errorOption:
return failure(reserveErr.toErr(ReleaseFailedError))
@ -361,7 +357,7 @@ proc deleteReservation*(
else:
return failure(error)
if reservation.size > 0.u256:
if reservation.size > 0.uint64:
trace "returning remaining reservation bytes to availability",
size = reservation.size
@ -389,8 +385,8 @@ proc deleteReservation*(
proc createAvailability*(
self: Reservations,
size: UInt256,
duration: UInt256,
size: uint64,
duration: uint64,
minPricePerBytePerSecond: UInt256,
totalCollateral: UInt256,
): Future[?!Availability] {.async.} =
@ -399,7 +395,7 @@ proc createAvailability*(
let availability =
Availability.init(size, size, duration, minPricePerBytePerSecond, totalCollateral)
let bytes = availability.freeSize.truncate(uint)
let bytes = availability.freeSize
if reserveErr =? (await self.repo.reserve(bytes.NBytes)).errorOption:
return failure(reserveErr.toErr(ReserveFailedError))
@ -418,9 +414,9 @@ proc createAvailability*(
method createReservation*(
self: Reservations,
availabilityId: AvailabilityId,
slotSize: UInt256,
slotSize: uint64,
requestId: RequestId,
slotIndex: UInt256,
slotIndex: uint64,
collateralPerByte: UInt256,
): Future[?!Reservation] {.async, base.} =
withLock(self.availabilityLock):
@ -450,7 +446,7 @@ method createReservation*(
availability.freeSize -= slotSize
# adjust the remaining totalRemainingCollateral
availability.totalRemainingCollateral -= slotSize * collateralPerByte
availability.totalRemainingCollateral -= slotSize.stuint(256) * collateralPerByte
# update availability with reduced size
trace "Updating availability with reduced size"
@ -475,7 +471,7 @@ proc returnBytesToAvailability*(
self: Reservations,
availabilityId: AvailabilityId,
reservationId: ReservationId,
bytes: UInt256,
bytes: uint64,
): Future[?!void] {.async.} =
logScope:
reservationId
@ -502,8 +498,7 @@ proc returnBytesToAvailability*(
# First lets see if we can re-reserve the bytes, if the Repo's quota
# is depleted then we will fail-fast as there is nothing to be done atm.
if reserveErr =?
(await self.repo.reserve(bytesToBeReturned.truncate(uint).NBytes)).errorOption:
if reserveErr =? (await self.repo.reserve(bytesToBeReturned.NBytes)).errorOption:
return failure(reserveErr.toErr(ReserveFailedError))
without availabilityKey =? availabilityId.key, error:
@ -517,8 +512,7 @@ proc returnBytesToAvailability*(
# Update availability with returned size
if updateErr =? (await self.updateAvailability(availability)).errorOption:
trace "Rolling back returning bytes"
if rollbackErr =?
(await self.repo.release(bytesToBeReturned.truncate(uint).NBytes)).errorOption:
if rollbackErr =? (await self.repo.release(bytesToBeReturned.NBytes)).errorOption:
rollbackErr.parent = updateErr
return failure(rollbackErr)
@ -546,7 +540,7 @@ proc release*(
without var reservation =? (await self.get(key, Reservation)), error:
return failure(error)
if reservation.size < bytes.u256:
if reservation.size < bytes:
let error = newException(
BytesOutOfBoundsError,
"trying to release an amount of bytes that is greater than the total size of the Reservation",
@ -556,7 +550,7 @@ proc release*(
if releaseErr =? (await self.repo.release(bytes.NBytes)).errorOption:
return failure(releaseErr.toErr(ReleaseFailedError))
reservation.size -= bytes.u256
reservation.size -= bytes
# persist partially used Reservation with updated size
if err =? (await self.update(reservation)).errorOption:
@ -643,7 +637,8 @@ proc all*(
proc findAvailability*(
self: Reservations,
size, duration, pricePerBytePerSecond, collateralPerByte: UInt256,
size, duration: uint64,
pricePerBytePerSecond, collateralPerByte: UInt256,
): Future[?Availability] {.async.} =
without storables =? (await self.storables(Availability)), e:
error "failed to get all storables", error = e.msg

58
codex/sales/salesagent.nim
View File

@ -6,6 +6,7 @@ import pkg/upraises
import ../contracts/requests
import ../errors
import ../logutils
import ../utils/exceptions
import ./statemachine
import ./salescontext
import ./salesdata
@ -28,7 +29,7 @@ type
OnCleanUp* = proc(
returnBytes = false, reprocessSlot = false, returnedCollateral = UInt256.none
): Future[void] {.gcsafe, upraises: [].}
OnFilled* = proc(request: StorageRequest, slotIndex: UInt256) {.gcsafe, upraises: [].}
OnFilled* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, upraises: [].}
SalesAgentError = object of CodexError
AllSlotsFilledError* = object of SalesAgentError
@ -39,7 +40,7 @@ func `==`*(a, b: SalesAgent): bool =
proc newSalesAgent*(
context: SalesContext,
requestId: RequestId,
slotIndex: UInt256,
slotIndex: uint64,
request: ?StorageRequest,
): SalesAgent =
var agent = SalesAgent.new()
@ -68,41 +69,48 @@ proc subscribeCancellation(agent: SalesAgent) {.async.} =
let data = agent.data
let clock = agent.context.clock
proc onCancelled() {.async.} =
proc onCancelled() {.async: (raises: []).} =
without request =? data.request:
return
let market = agent.context.market
let expiry = await market.requestExpiresAt(data.requestId)
try:
let market = agent.context.market
let expiry = await market.requestExpiresAt(data.requestId)
while true:
let deadline = max(clock.now, expiry) + 1
trace "Waiting for request to be cancelled", now = clock.now, expiry = deadline
await clock.waitUntil(deadline)
while true:
let deadline = max(clock.now, expiry) + 1
trace "Waiting for request to be cancelled", now = clock.now, expiry = deadline
await clock.waitUntil(deadline)
without state =? await agent.retrieveRequestState():
error "Uknown request", requestId = data.requestId
return
without state =? await agent.retrieveRequestState():
error "Unknown request", requestId = data.requestId
return
case state
of New:
discard
of RequestState.Cancelled:
agent.schedule(cancelledEvent(request))
break
of RequestState.Started, RequestState.Finished, RequestState.Failed:
break
case state
of New:
discard
of RequestState.Cancelled:
agent.schedule(cancelledEvent(request))
break
of RequestState.Started, RequestState.Finished, RequestState.Failed:
break
debug "The request is not yet canceled, even though it should be. Waiting for some more time.",
currentState = state, now = clock.now
debug "The request is not yet canceled, even though it should be. Waiting for some more time.",
currentState = state, now = clock.now
except CancelledError:
trace "Waiting for expiry to lapse was cancelled", requestId = data.requestId
except CatchableError as e:
error "Error while waiting for expiry to lapse", error = e.msgDetail
data.cancelled = onCancelled()
asyncSpawn data.cancelled
method onFulfilled*(
agent: SalesAgent, requestId: RequestId
) {.base, gcsafe, upraises: [].} =
if agent.data.requestId == requestId and not agent.data.cancelled.isNil:
agent.data.cancelled.cancelSoon()
let cancelled = agent.data.cancelled
if agent.data.requestId == requestId and not cancelled.isNil and not cancelled.finished:
cancelled.cancelSoon()
method onFailed*(
agent: SalesAgent, requestId: RequestId
@ -113,7 +121,7 @@ method onFailed*(
agent.schedule(failedEvent(request))
method onSlotFilled*(
agent: SalesAgent, requestId: RequestId, slotIndex: UInt256
agent: SalesAgent, requestId: RequestId, slotIndex: uint64
) {.base, gcsafe, upraises: [].} =
if agent.data.requestId == requestId and agent.data.slotIndex == slotIndex:
agent.schedule(slotFilledEvent(requestId, slotIndex))

9
codex/sales/salescontext.nim
View File

@ -1,6 +1,7 @@
import pkg/questionable
import pkg/questionable/results
import pkg/upraises
import pkg/libp2p/cid
import ../market
import ../clock
@ -25,13 +26,13 @@ type
BlocksCb* = proc(blocks: seq[bt.Block]): Future[?!void] {.gcsafe, raises: [].}
OnStore* = proc(
request: StorageRequest, slot: UInt256, blocksCb: BlocksCb
request: StorageRequest, slot: uint64, blocksCb: BlocksCb
): Future[?!void] {.gcsafe, upraises: [].}
OnProve* = proc(slot: Slot, challenge: ProofChallenge): Future[?!Groth16Proof] {.
gcsafe, upraises: []
.}
OnExpiryUpdate* = proc(rootCid: string, expiry: SecondsSince1970): Future[?!void] {.
OnExpiryUpdate* = proc(rootCid: Cid, expiry: SecondsSince1970): Future[?!void] {.
gcsafe, upraises: []
.}
OnClear* = proc(request: StorageRequest, slotIndex: UInt256) {.gcsafe, upraises: [].}
OnSale* = proc(request: StorageRequest, slotIndex: UInt256) {.gcsafe, upraises: [].}
OnClear* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, upraises: [].}
OnSale* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, upraises: [].}

2
codex/sales/salesdata.nim
View File

@ -7,6 +7,6 @@ type SalesData* = ref object
requestId*: RequestId
ask*: StorageAsk
request*: ?StorageRequest
slotIndex*: UInt256
slotIndex*: uint64
cancelled*: Future[void]
reservation*: ?Reservation

15
codex/sales/slotqueue.nim
View File

@ -5,6 +5,7 @@ import pkg/questionable
import pkg/questionable/results
import pkg/upraises
import ../errors
import ../clock
import ../logutils
import ../rng
import ../utils
@ -30,11 +31,11 @@ type
SlotQueueItem* = object
requestId: RequestId
slotIndex: uint16
slotSize: UInt256
duration: UInt256
slotSize: uint64
duration: uint64
pricePerBytePerSecond: UInt256
collateralPerByte: UInt256
expiry: UInt256
expiry: uint64
seen: bool
# don't need to -1 to prevent overflow when adding 1 (to always allow push)
@ -135,7 +136,7 @@ proc init*(
requestId: RequestId,
slotIndex: uint16,
ask: StorageAsk,
expiry: UInt256,
expiry: uint64,
seen = false,
): SlotQueueItem =
SlotQueueItem(
@ -155,7 +156,7 @@ proc init*(
SlotQueueItem.init(request.id, slotIndex, request.ask, request.expiry)
proc init*(
_: type SlotQueueItem, requestId: RequestId, ask: StorageAsk, expiry: UInt256
_: type SlotQueueItem, requestId: RequestId, ask: StorageAsk, expiry: uint64
): seq[SlotQueueItem] =
if not ask.slots.inRange:
raise newException(SlotsOutOfRangeError, "Too many slots")
@ -182,10 +183,10 @@ proc requestId*(self: SlotQueueItem): RequestId =
proc slotIndex*(self: SlotQueueItem): uint16 =
self.slotIndex
proc slotSize*(self: SlotQueueItem): UInt256 =
proc slotSize*(self: SlotQueueItem): uint64 =
self.slotSize
proc duration*(self: SlotQueueItem): UInt256 =
proc duration*(self: SlotQueueItem): uint64 =
self.duration
proc pricePerBytePerSecond*(self: SlotQueueItem): UInt256 =

4
codex/sales/statemachine.nim
View File

@ -25,7 +25,7 @@ method onFailed*(
discard
method onSlotFilled*(
state: SaleState, requestId: RequestId, slotIndex: UInt256
state: SaleState, requestId: RequestId, slotIndex: uint64
): ?State {.base, upraises: [].} =
discard
@ -37,6 +37,6 @@ proc failedEvent*(request: StorageRequest): Event =
return proc(state: State): ?State =
SaleState(state).onFailed(request)
proc slotFilledEvent*(requestId: RequestId, slotIndex: UInt256): Event =
proc slotFilledEvent*(requestId: RequestId, slotIndex: uint64): Event =
return proc(state: State): ?State =
SaleState(state).onSlotFilled(requestId, slotIndex)

45
codex/sales/states/cancelled.nim
View File

@ -1,17 +1,20 @@
import ../../logutils
import ../../utils/exceptions
import ../salesagent
import ../statemachine
import ./errorhandling
import ./errored
logScope:
topics = "marketplace sales cancelled"
type SaleCancelled* = ref object of ErrorHandlingState
type SaleCancelled* = ref object of SaleState
method `$`*(state: SaleCancelled): string =
"SaleCancelled"
method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleCancelled, machine: Machine
): Future[?State] {.async: (raises: []).} =
let agent = SalesAgent(machine)
let data = agent.data
let market = agent.context.market
@ -19,21 +22,27 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} =
without request =? data.request:
raiseAssert "no sale request"
let slot = Slot(request: request, slotIndex: data.slotIndex)
debug "Collecting collateral and partial payout",
requestId = data.requestId, slotIndex = data.slotIndex
let currentCollateral = await market.currentCollateral(slot.id)
await market.freeSlot(slot.id)
try:
let slot = Slot(request: request, slotIndex: data.slotIndex)
debug "Collecting collateral and partial payout",
requestId = data.requestId, slotIndex = data.slotIndex
let currentCollateral = await market.currentCollateral(slot.id)
await market.freeSlot(slot.id)
if onClear =? agent.context.onClear and request =? data.request:
onClear(request, data.slotIndex)
if onClear =? agent.context.onClear and request =? data.request:
onClear(request, data.slotIndex)
if onCleanUp =? agent.onCleanUp:
await onCleanUp(
returnBytes = true,
reprocessSlot = false,
returnedCollateral = some currentCollateral,
)
if onCleanUp =? agent.onCleanUp:
await onCleanUp(
returnBytes = true,
reprocessSlot = false,
returnedCollateral = some currentCollateral,
)
warn "Sale cancelled due to timeout",
requestId = data.requestId, slotIndex = data.slotIndex
warn "Sale cancelled due to timeout",
requestId = data.requestId, slotIndex = data.slotIndex
except CancelledError as e:
trace "SaleCancelled.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleCancelled.run", error = e.msgDetail
return some State(SaleErrored(error: e))

26
codex/sales/states/downloading.nim
View File

@ -4,16 +4,16 @@ import pkg/questionable/results
import ../../blocktype as bt
import ../../logutils
import ../../market
import ../../utils/exceptions
import ../salesagent
import ../statemachine
import ./errorhandling
import ./cancelled
import ./failed
import ./filled
import ./initialproving
import ./errored
type SaleDownloading* = ref object of ErrorHandlingState
type SaleDownloading* = ref object of SaleState
logScope:
topics = "marketplace sales downloading"
@ -28,11 +28,13 @@ method onFailed*(state: SaleDownloading, request: StorageRequest): ?State =
return some State(SaleFailed())
method onSlotFilled*(
state: SaleDownloading, requestId: RequestId, slotIndex: UInt256
state: SaleDownloading, requestId: RequestId, slotIndex: uint64
): ?State =
return some State(SaleFilled())
method run*(state: SaleDownloading, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleDownloading, machine: Machine
): Future[?State] {.async: (raises: []).} =
let agent = SalesAgent(machine)
let data = agent.data
let context = agent.context
@ -64,9 +66,15 @@ method run*(state: SaleDownloading, machine: Machine): Future[?State] {.async.}
trace "Releasing batch of bytes written to disk", bytes
return await reservations.release(reservation.id, reservation.availabilityId, bytes)
trace "Starting download"
if err =? (await onStore(request, data.slotIndex, onBlocks)).errorOption:
return some State(SaleErrored(error: err, reprocessSlot: false))
try:
trace "Starting download"
if err =? (await onStore(request, data.slotIndex, onBlocks)).errorOption:
return some State(SaleErrored(error: err, reprocessSlot: false))
trace "Download complete"
return some State(SaleInitialProving())
trace "Download complete"
return some State(SaleInitialProving())
except CancelledError as e:
trace "SaleDownloading.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleDownloading.run", error = e.msgDetail
return some State(SaleErrored(error: e))

20
codex/sales/states/errored.nim
View File

@ -17,10 +17,9 @@ type SaleErrored* = ref object of SaleState
method `$`*(state: SaleErrored): string =
"SaleErrored"
method onError*(state: SaleState, err: ref CatchableError): ?State {.upraises: [].} =
error "error during SaleErrored run", error = err.msg
method run*(state: SaleErrored, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleErrored, machine: Machine
): Future[?State] {.async: (raises: []).} =
let agent = SalesAgent(machine)
let data = agent.data
let context = agent.context
@ -30,8 +29,13 @@ method run*(state: SaleErrored, machine: Machine): Future[?State] {.async.} =
requestId = data.requestId,
slotIndex = data.slotIndex
if onClear =? context.onClear and request =? data.request:
onClear(request, data.slotIndex)
try:
if onClear =? context.onClear and request =? data.request:
onClear(request, data.slotIndex)
if onCleanUp =? agent.onCleanUp:
await onCleanUp(returnBytes = true, reprocessSlot = state.reprocessSlot)
if onCleanUp =? agent.onCleanUp:
await onCleanUp(returnBytes = true, reprocessSlot = state.reprocessSlot)
except CancelledError as e:
trace "SaleErrored.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleErrored.run", error = e.msgDetail

8
codex/sales/states/errorhandling.nim
View File

@ -1,8 +0,0 @@
import pkg/questionable
import ../statemachine
import ./errored
type ErrorHandlingState* = ref object of SaleState
method onError*(state: ErrorHandlingState, error: ref CatchableError): ?State =
some State(SaleErrored(error: error))

27
codex/sales/states/failed.nim
View File

@ -1,30 +1,39 @@
import ../../logutils
import ../../utils/exceptions
import ../../utils/exceptions
import ../salesagent
import ../statemachine
import ./errorhandling
import ./errored
logScope:
topics = "marketplace sales failed"
type
SaleFailed* = ref object of ErrorHandlingState
SaleFailed* = ref object of SaleState
SaleFailedError* = object of SaleError
method `$`*(state: SaleFailed): string =
"SaleFailed"
method run*(state: SaleFailed, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleFailed, machine: Machine
): Future[?State] {.async: (raises: []).} =
let data = SalesAgent(machine).data
let market = SalesAgent(machine).context.market
without request =? data.request:
raiseAssert "no sale request"
let slot = Slot(request: request, slotIndex: data.slotIndex)
debug "Removing slot from mySlots",
requestId = data.requestId, slotIndex = data.slotIndex
await market.freeSlot(slot.id)
try:
let slot = Slot(request: request, slotIndex: data.slotIndex)
debug "Removing slot from mySlots",
requestId = data.requestId, slotIndex = data.slotIndex
await market.freeSlot(slot.id)
let error = newException(SaleFailedError, "Sale failed")
return some State(SaleErrored(error: error))
let error = newException(SaleFailedError, "Sale failed")
return some State(SaleErrored(error: error))
except CancelledError as e:
trace "SaleFailed.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleFailed.run", error = e.msgDetail
return some State(SaleErrored(error: e))

64
codex/sales/states/filled.nim
View File

@ -3,9 +3,9 @@ import pkg/questionable/results
import ../../conf
import ../../logutils
import ../../utils/exceptions
import ../statemachine
import ../salesagent
import ./errorhandling
import ./errored
import ./cancelled
import ./failed
@ -18,7 +18,7 @@ logScope:
topics = "marketplace sales filled"
type
SaleFilled* = ref object of ErrorHandlingState
SaleFilled* = ref object of SaleState
HostMismatchError* = object of CatchableError
method onCancelled*(state: SaleFilled, request: StorageRequest): ?State =
@ -30,40 +30,48 @@ method onFailed*(state: SaleFilled, request: StorageRequest): ?State =
method `$`*(state: SaleFilled): string =
"SaleFilled"
method run*(state: SaleFilled, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleFilled, machine: Machine
): Future[?State] {.async: (raises: []).} =
let agent = SalesAgent(machine)
let data = agent.data
let context = agent.context
let market = context.market
let host = await market.getHost(data.requestId, data.slotIndex)
let me = await market.getSigner()
if host == me.some:
info "Slot succesfully filled",
requestId = data.requestId, slotIndex = data.slotIndex
try:
let host = await market.getHost(data.requestId, data.slotIndex)
let me = await market.getSigner()
without request =? data.request:
raiseAssert "no sale request"
if host == me.some:
info "Slot succesfully filled",
requestId = data.requestId, slotIndex = data.slotIndex
if onFilled =? agent.onFilled:
onFilled(request, data.slotIndex)
without request =? data.request:
raiseAssert "no sale request"
without onExpiryUpdate =? context.onExpiryUpdate:
raiseAssert "onExpiryUpdate callback not set"
if onFilled =? agent.onFilled:
onFilled(request, data.slotIndex)
let requestEnd = await market.getRequestEnd(data.requestId)
if err =? (await onExpiryUpdate(request.content.cid, requestEnd)).errorOption:
return some State(SaleErrored(error: err))
without onExpiryUpdate =? context.onExpiryUpdate:
raiseAssert "onExpiryUpdate callback not set"
when codex_enable_proof_failures:
if context.simulateProofFailures > 0:
info "Proving with failure rate", rate = context.simulateProofFailures
return some State(
SaleProvingSimulated(failEveryNProofs: context.simulateProofFailures)
)
let requestEnd = await market.getRequestEnd(data.requestId)
if err =? (await onExpiryUpdate(request.content.cid, requestEnd)).errorOption:
return some State(SaleErrored(error: err))
return some State(SaleProving())
else:
let error = newException(HostMismatchError, "Slot filled by other host")
return some State(SaleErrored(error: error))
when codex_enable_proof_failures:
if context.simulateProofFailures > 0:
info "Proving with failure rate", rate = context.simulateProofFailures
return some State(
SaleProvingSimulated(failEveryNProofs: context.simulateProofFailures)
)
return some State(SaleProving())
else:
let error = newException(HostMismatchError, "Slot filled by other host")
return some State(SaleErrored(error: error))
except CancelledError as e:
trace "SaleFilled.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleFilled.run", error = e.msgDetail
return some State(SaleErrored(error: e))

60
codex/sales/states/filling.nim
View File

@ -1,9 +1,9 @@
import pkg/stint
import ../../logutils
import ../../market
import ../../utils/exceptions
import ../statemachine
import ../salesagent
import ./errorhandling
import ./filled
import ./cancelled
import ./failed
@ -13,7 +13,7 @@ import ./errored
logScope:
topics = "marketplace sales filling"
type SaleFilling* = ref object of ErrorHandlingState
type SaleFilling* = ref object of SaleState
proof*: Groth16Proof
method `$`*(state: SaleFilling): string =
@ -25,7 +25,9 @@ method onCancelled*(state: SaleFilling, request: StorageRequest): ?State =
method onFailed*(state: SaleFilling, request: StorageRequest): ?State =
return some State(SaleFailed())
method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleFilling, machine: Machine
): Future[?State] {.async: (raises: []).} =
let data = SalesAgent(machine).data
let market = SalesAgent(machine).context.market
without (request =? data.request):
@ -35,28 +37,34 @@ method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} =
requestId = data.requestId
slotIndex = data.slotIndex
let slotState = await market.slotState(slotId(data.requestId, data.slotIndex))
let requestedCollateral = request.ask.collateralPerSlot
var collateral: UInt256
if slotState == SlotState.Repair:
# When repairing the node gets "discount" on the collateral that it needs to
let repairRewardPercentage = (await market.repairRewardPercentage).u256
collateral =
requestedCollateral -
((requestedCollateral * repairRewardPercentage)).div(100.u256)
else:
collateral = requestedCollateral
debug "Filling slot"
try:
await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral)
except MarketError as e:
if e.msg.contains "Slot is not free":
debug "Slot is already filled, ignoring slot"
return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
else:
return some State(SaleErrored(error: e))
# other CatchableErrors are handled "automatically" by the ErrorHandlingState
let slotState = await market.slotState(slotId(data.requestId, data.slotIndex))
let requestedCollateral = request.ask.collateralPerSlot
var collateral: UInt256
return some State(SaleFilled())
if slotState == SlotState.Repair:
# When repairing the node gets "discount" on the collateral that it needs to
let repairRewardPercentage = (await market.repairRewardPercentage).u256
collateral =
requestedCollateral -
((requestedCollateral * repairRewardPercentage)).div(100.u256)
else:
collateral = requestedCollateral
debug "Filling slot"
try:
await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral)
except MarketError as e:
if e.msg.contains "Slot is not free":
debug "Slot is already filled, ignoring slot"
return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
else:
return some State(SaleErrored(error: e))
# other CatchableErrors are handled "automatically" by the SaleState
return some State(SaleFilled())
except CancelledError as e:
trace "SaleFilling.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleFilling.run", error = e.msgDetail
return some State(SaleErrored(error: e))

19
codex/sales/states/finished.nim
View File

@ -1,16 +1,17 @@
import pkg/chronos
import ../../logutils
import ../../utils/exceptions
import ../statemachine
import ../salesagent
import ./errorhandling
import ./cancelled
import ./failed
import ./errored
logScope:
topics = "marketplace sales finished"
type SaleFinished* = ref object of ErrorHandlingState
type SaleFinished* = ref object of SaleState
returnedCollateral*: ?UInt256
method `$`*(state: SaleFinished): string =
@ -22,7 +23,9 @@ method onCancelled*(state: SaleFinished, request: StorageRequest): ?State =
method onFailed*(state: SaleFinished, request: StorageRequest): ?State =
return some State(SaleFailed())
method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleFinished, machine: Machine
): Future[?State] {.async: (raises: []).} =
let agent = SalesAgent(machine)
let data = agent.data
@ -32,5 +35,11 @@ method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} =
info "Slot finished and paid out",
requestId = data.requestId, slotIndex = data.slotIndex
if onCleanUp =? agent.onCleanUp:
await onCleanUp(returnedCollateral = state.returnedCollateral)
try:
if onCleanUp =? agent.onCleanUp:
await onCleanUp(returnedCollateral = state.returnedCollateral)
except CancelledError as e:
trace "SaleFilled.run onCleanUp was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleFilled.run in onCleanUp callback", error = e.msgDetail
return some State(SaleErrored(error: e))

23
codex/sales/states/ignored.nim
View File

@ -1,9 +1,10 @@
import pkg/chronos
import ../../logutils
import ../../utils/exceptions
import ../statemachine
import ../salesagent
import ./errorhandling
import ./errored
logScope:
topics = "marketplace sales ignored"
@ -11,17 +12,25 @@ logScope:
# Ignored slots could mean there was no availability or that the slot could
# not be reserved.
type SaleIgnored* = ref object of ErrorHandlingState
type SaleIgnored* = ref object of SaleState
reprocessSlot*: bool # readd slot to queue with `seen` flag
returnBytes*: bool # return unreleased bytes from Reservation to Availability
method `$`*(state: SaleIgnored): string =
"SaleIgnored"
method run*(state: SaleIgnored, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleIgnored, machine: Machine
): Future[?State] {.async: (raises: []).} =
let agent = SalesAgent(machine)
if onCleanUp =? agent.onCleanUp:
await onCleanUp(
reprocessSlot = state.reprocessSlot, returnBytes = state.returnBytes
)
try:
if onCleanUp =? agent.onCleanUp:
await onCleanUp(
reprocessSlot = state.reprocessSlot, returnBytes = state.returnBytes
)
except CancelledError as e:
trace "SaleIgnored.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleIgnored.run in onCleanUp", error = e.msgDetail
return some State(SaleErrored(error: e))

40
codex/sales/states/initialproving.nim
View File

@ -1,9 +1,9 @@
import pkg/questionable/results
import ../../clock
import ../../logutils
import ../../utils/exceptions
import ../statemachine
import ../salesagent
import ./errorhandling
import ./filling
import ./cancelled
import ./errored
@ -12,7 +12,7 @@ import ./failed
logScope:
topics = "marketplace sales initial-proving"
type SaleInitialProving* = ref object of ErrorHandlingState
type SaleInitialProving* = ref object of SaleState
method `$`*(state: SaleInitialProving): string =
"SaleInitialProving"
@ -25,9 +25,9 @@ method onFailed*(state: SaleInitialProving, request: StorageRequest): ?State =
proc waitUntilNextPeriod(clock: Clock, periodicity: Periodicity) {.async.} =
trace "Waiting until next period"
let period = periodicity.periodOf(clock.now().u256)
let periodEnd = periodicity.periodEnd(period).truncate(int64)
await clock.waitUntil(periodEnd + 1)
let period = periodicity.periodOf(clock.now().Timestamp)
let periodEnd = periodicity.periodEnd(period)
await clock.waitUntil((periodEnd + 1).toSecondsSince1970)
proc waitForStableChallenge(market: Market, clock: Clock, slotId: SlotId) {.async.} =
let periodicity = await market.periodicity()
@ -36,7 +36,9 @@ proc waitForStableChallenge(market: Market, clock: Clock, slotId: SlotId) {.asyn
while (await market.getPointer(slotId)) > (256 - downtime):
await clock.waitUntilNextPeriod(periodicity)
method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleInitialProving, machine: Machine
): Future[?State] {.async: (raises: []).} =
let data = SalesAgent(machine).data
let context = SalesAgent(machine).context
let market = context.market
@ -48,16 +50,22 @@ method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async
without onProve =? context.onProve:
raiseAssert "onProve callback not set"
debug "Waiting for a proof challenge that is valid for the entire period"
let slot = Slot(request: request, slotIndex: data.slotIndex)
await waitForStableChallenge(market, clock, slot.id)
try:
debug "Waiting for a proof challenge that is valid for the entire period"
let slot = Slot(request: request, slotIndex: data.slotIndex)
await waitForStableChallenge(market, clock, slot.id)
debug "Generating initial proof", requestId = data.requestId
let challenge = await context.market.getChallenge(slot.id)
without proof =? (await onProve(slot, challenge)), err:
error "Failed to generate initial proof", error = err.msg
return some State(SaleErrored(error: err))
debug "Generating initial proof", requestId = data.requestId
let challenge = await context.market.getChallenge(slot.id)
without proof =? (await onProve(slot, challenge)), err:
error "Failed to generate initial proof", error = err.msg
return some State(SaleErrored(error: err))
debug "Finished proof calculation", requestId = data.requestId
debug "Finished proof calculation", requestId = data.requestId
return some State(SaleFilling(proof: proof))
return some State(SaleFilling(proof: proof))
except CancelledError as e:
trace "SaleInitialProving.run onCleanUp was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleInitialProving.run", error = e.msgDetail
return some State(SaleErrored(error: e))

27
codex/sales/states/payout.nim
View File

@ -1,16 +1,17 @@
import ../../logutils
import ../../market
import ../../utils/exceptions
import ../statemachine
import ../salesagent
import ./errorhandling
import ./cancelled
import ./failed
import ./finished
import ./errored
logScope:
topics = "marketplace sales payout"
type SalePayout* = ref object of ErrorHandlingState
type SalePayout* = ref object of SaleState
method `$`*(state: SalePayout): string =
"SalePayout"
@ -21,17 +22,25 @@ method onCancelled*(state: SalePayout, request: StorageRequest): ?State =
method onFailed*(state: SalePayout, request: StorageRequest): ?State =
return some State(SaleFailed())
method run*(state: SalePayout, machine: Machine): Future[?State] {.async.} =
method run*(
state: SalePayout, machine: Machine
): Future[?State] {.async: (raises: []).} =
let data = SalesAgent(machine).data
let market = SalesAgent(machine).context.market
without request =? data.request:
raiseAssert "no sale request"
let slot = Slot(request: request, slotIndex: data.slotIndex)
debug "Collecting finished slot's reward",
requestId = data.requestId, slotIndex = data.slotIndex
let currentCollateral = await market.currentCollateral(slot.id)
await market.freeSlot(slot.id)
try:
let slot = Slot(request: request, slotIndex: data.slotIndex)
debug "Collecting finished slot's reward",
requestId = data.requestId, slotIndex = data.slotIndex
let currentCollateral = await market.currentCollateral(slot.id)
await market.freeSlot(slot.id)
return some State(SaleFinished(returnedCollateral: some currentCollateral))
return some State(SaleFinished(returnedCollateral: some currentCollateral))
except CancelledError as e:
trace "SalePayout.run onCleanUp was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SalePayout.run", error = e.msgDetail
return some State(SaleErrored(error: e))

100
codex/sales/states/preparing.nim
View File

@ -4,9 +4,9 @@ import pkg/metrics
import ../../logutils
import ../../market
import ../../utils/exceptions
import ../salesagent
import ../statemachine
import ./errorhandling
import ./cancelled
import ./failed
import ./filled
@ -18,7 +18,7 @@ declareCounter(
codex_reservations_availability_mismatch, "codex reservations availability_mismatch"
)
type SalePreparing* = ref object of ErrorHandlingState
type SalePreparing* = ref object of SaleState
logScope:
topics = "marketplace sales preparing"
@ -33,66 +33,74 @@ method onFailed*(state: SalePreparing, request: StorageRequest): ?State =
return some State(SaleFailed())
method onSlotFilled*(
state: SalePreparing, requestId: RequestId, slotIndex: UInt256
state: SalePreparing, requestId: RequestId, slotIndex: uint64
): ?State =
return some State(SaleFilled())
method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} =
method run*(
state: SalePreparing, machine: Machine
): Future[?State] {.async: (raises: []).} =
let agent = SalesAgent(machine)
let data = agent.data
let context = agent.context
let market = context.market
let reservations = context.reservations
await agent.retrieveRequest()
await agent.subscribe()
try:
await agent.retrieveRequest()
await agent.subscribe()
without request =? data.request:
raiseAssert "no sale request"
without request =? data.request:
raiseAssert "no sale request"
let slotId = slotId(data.requestId, data.slotIndex)
let state = await market.slotState(slotId)
if state != SlotState.Free and state != SlotState.Repair:
return some State(SaleIgnored(reprocessSlot: false, returnBytes: false))
let slotId = slotId(data.requestId, data.slotIndex)
let state = await market.slotState(slotId)
if state != SlotState.Free and state != SlotState.Repair:
return some State(SaleIgnored(reprocessSlot: false, returnBytes: false))
# TODO: Once implemented, check to ensure the host is allowed to fill the slot,
# due to the [sliding window mechanism](https://github.com/codex-storage/codex-research/blob/master/design/marketplace.md#dispersal)
# TODO: Once implemented, check to ensure the host is allowed to fill the slot,
# due to the [sliding window mechanism](https://github.com/codex-storage/codex-research/blob/master/design/marketplace.md#dispersal)
logScope:
slotIndex = data.slotIndex
slotSize = request.ask.slotSize
duration = request.ask.duration
pricePerBytePerSecond = request.ask.pricePerBytePerSecond
collateralPerByte = request.ask.collateralPerByte
logScope:
slotIndex = data.slotIndex
slotSize = request.ask.slotSize
duration = request.ask.duration
pricePerBytePerSecond = request.ask.pricePerBytePerSecond
collateralPerByte = request.ask.collateralPerByte
without availability =?
await reservations.findAvailability(
request.ask.slotSize, request.ask.duration, request.ask.pricePerBytePerSecond,
request.ask.collateralPerByte,
):
debug "No availability found for request, ignoring"
without availability =?
await reservations.findAvailability(
request.ask.slotSize, request.ask.duration, request.ask.pricePerBytePerSecond,
request.ask.collateralPerByte,
):
debug "No availability found for request, ignoring"
return some State(SaleIgnored(reprocessSlot: true))
info "Availability found for request, creating reservation"
without reservation =?
await reservations.createReservation(
availability.id, request.ask.slotSize, request.id, data.slotIndex,
request.ask.collateralPerByte,
), error:
trace "Creation of reservation failed"
# Race condition:
# reservations.findAvailability (line 64) is no guarantee. You can never know for certain that the reservation can be created until after you have it.
# Should createReservation fail because there's no space, we proceed to SaleIgnored.
if error of BytesOutOfBoundsError:
# Lets monitor how often this happen and if it is often we can make it more inteligent to handle it
codex_reservations_availability_mismatch.inc()
return some State(SaleIgnored(reprocessSlot: true))
return some State(SaleErrored(error: error))
info "Availability found for request, creating reservation"
trace "Reservation created succesfully"
without reservation =?
await reservations.createReservation(
availability.id, request.ask.slotSize, request.id, data.slotIndex,
request.ask.collateralPerByte,
), error:
trace "Creation of reservation failed"
# Race condition:
# reservations.findAvailability (line 64) is no guarantee. You can never know for certain that the reservation can be created until after you have it.
# Should createReservation fail because there's no space, we proceed to SaleIgnored.
if error of BytesOutOfBoundsError:
# Lets monitor how often this happen and if it is often we can make it more inteligent to handle it
codex_reservations_availability_mismatch.inc()
return some State(SaleIgnored(reprocessSlot: true))
data.reservation = some reservation
return some State(SaleSlotReserving())
return some State(SaleErrored(error: error))
trace "Reservation created successfully"
data.reservation = some reservation
return some State(SaleSlotReserving())
except CancelledError as e:
trace "SalePreparing.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SalePreparing.run", error = e.msgDetail
return some State(SaleErrored(error: e))

65
codex/sales/states/proving.nim
View File

@ -6,7 +6,6 @@ import ../../utils/exceptions
import ../statemachine
import ../salesagent
import ../salescontext
import ./errorhandling
import ./cancelled
import ./failed
import ./errored
@ -18,7 +17,7 @@ logScope:
type
SlotFreedError* = object of CatchableError
SlotNotFilledError* = object of CatchableError
SaleProving* = ref object of ErrorHandlingState
SaleProving* = ref object of SaleState
loop: Future[void]
method prove*(
@ -47,7 +46,7 @@ proc proveLoop(
market: Market,
clock: Clock,
request: StorageRequest,
slotIndex: UInt256,
slotIndex: uint64,
onProve: OnProve,
) {.async.} =
let slot = Slot(request: request, slotIndex: slotIndex)
@ -61,12 +60,12 @@ proc proveLoop(
proc getCurrentPeriod(): Future[Period] {.async.} =
let periodicity = await market.periodicity()
return periodicity.periodOf(clock.now().u256)
return periodicity.periodOf(clock.now().Timestamp)
proc waitUntilPeriod(period: Period) {.async.} =
let periodicity = await market.periodicity()
# Ensure that we're past the period boundary by waiting an additional second
await clock.waitUntil(periodicity.periodStart(period).truncate(int64) + 1)
await clock.waitUntil((periodicity.periodStart(period) + 1).toSecondsSince1970)
while true:
let currentPeriod = await getCurrentPeriod()
@ -113,7 +112,9 @@ method onFailed*(state: SaleProving, request: StorageRequest): ?State =
# state change
return some State(SaleFailed())
method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleProving, machine: Machine
): Future[?State] {.async: (raises: []).} =
let data = SalesAgent(machine).data
let context = SalesAgent(machine).context
@ -129,27 +130,37 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} =
without clock =? context.clock:
raiseAssert("clock not set")
debug "Start proving", requestId = data.requestId, slotIndex = data.slotIndex
try:
let loop = state.proveLoop(market, clock, request, data.slotIndex, onProve)
state.loop = loop
await loop
except CancelledError:
discard
debug "Start proving", requestId = data.requestId, slotIndex = data.slotIndex
try:
let loop = state.proveLoop(market, clock, request, data.slotIndex, onProve)
state.loop = loop
await loop
except CancelledError as e:
trace "proving loop cancelled"
discard
except CatchableError as e:
error "Proving failed",
msg = e.msg, typ = $(type e), stack = e.getStackTrace(), error = e.msgDetail
return some State(SaleErrored(error: e))
finally:
# Cleanup of the proving loop
debug "Stopping proving.", requestId = data.requestId, slotIndex = data.slotIndex
if not state.loop.isNil:
if not state.loop.finished:
try:
await state.loop.cancelAndWait()
except CancelledError:
discard
except CatchableError as e:
error "Error during cancellation of proving loop", msg = e.msg
state.loop = nil
return some State(SalePayout())
except CancelledError as e:
trace "SaleProving.run onCleanUp was cancelled", error = e.msgDetail
except CatchableError as e:
error "Proving failed", msg = e.msg
error "Error during SaleProving.run", error = e.msgDetail
return some State(SaleErrored(error: e))
finally:
# Cleanup of the proving loop
debug "Stopping proving.", requestId = data.requestId, slotIndex = data.slotIndex
if not state.loop.isNil:
if not state.loop.finished:
try:
await state.loop.cancelAndWait()
except CatchableError as e:
error "Error during cancellation of proving loop", msg = e.msg
state.loop = nil
return some State(SalePayout())

43
codex/sales/states/provingsimulated.nim
View File

@ -4,12 +4,14 @@ when codex_enable_proof_failures:
import pkg/stint
import pkg/ethers
import ../../contracts/marketplace
import ../../contracts/requests
import ../../logutils
import ../../market
import ../../utils/exceptions
import ../salescontext
import ./proving
import ./errored
logScope:
topics = "marketplace sales simulated-proving"
@ -18,7 +20,7 @@ when codex_enable_proof_failures:
failEveryNProofs*: int
proofCount: int
proc onSubmitProofError(error: ref CatchableError, period: UInt256, slotId: SlotId) =
proc onSubmitProofError(error: ref CatchableError, period: Period, slotId: SlotId) =
error "Submitting invalid proof failed", period, slotId, msg = error.msgDetail
method prove*(
@ -29,22 +31,27 @@ when codex_enable_proof_failures:
market: Market,
currentPeriod: Period,
) {.async.} =
trace "Processing proving in simulated mode"
state.proofCount += 1
if state.failEveryNProofs > 0 and state.proofCount mod state.failEveryNProofs == 0:
state.proofCount = 0
try:
trace "Processing proving in simulated mode"
state.proofCount += 1
if state.failEveryNProofs > 0 and state.proofCount mod state.failEveryNProofs == 0:
state.proofCount = 0
try:
warn "Submitting INVALID proof", period = currentPeriod, slotId = slot.id
await market.submitProof(slot.id, Groth16Proof.default)
except MarketError as e:
if not e.msg.contains("Invalid proof"):
try:
warn "Submitting INVALID proof", period = currentPeriod, slotId = slot.id
await market.submitProof(slot.id, Groth16Proof.default)
except Proofs_InvalidProof as e:
discard # expected
except CancelledError as error:
raise error
except CatchableError as e:
onSubmitProofError(e, currentPeriod, slot.id)
except CancelledError as error:
raise error
except CatchableError as e:
onSubmitProofError(e, currentPeriod, slot.id)
else:
await procCall SaleProving(state).prove(
slot, challenge, onProve, market, currentPeriod
)
else:
await procCall SaleProving(state).prove(
slot, challenge, onProve, market, currentPeriod
)
except CancelledError as e:
trace "Submitting INVALID proof cancelled", error = e.msgDetail
raise e
except CatchableError as e:
error "Submitting INVALID proof failed", error = e.msgDetail

52
codex/sales/states/slotreserving.nim
View File

@ -3,16 +3,16 @@ import pkg/metrics
import ../../logutils
import ../../market
import ../../utils/exceptions
import ../salesagent
import ../statemachine
import ./errorhandling
import ./cancelled
import ./failed
import ./ignored
import ./downloading
import ./errored
type SaleSlotReserving* = ref object of ErrorHandlingState
type SaleSlotReserving* = ref object of SaleState
logScope:
topics = "marketplace sales reserving"
@ -26,7 +26,9 @@ method onCancelled*(state: SaleSlotReserving, request: StorageRequest): ?State =
method onFailed*(state: SaleSlotReserving, request: StorageRequest): ?State =
return some State(SaleFailed())
method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleSlotReserving, machine: Machine
): Future[?State] {.async: (raises: []).} =
let agent = SalesAgent(machine)
let data = agent.data
let context = agent.context
@ -36,23 +38,29 @@ method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.
requestId = data.requestId
slotIndex = data.slotIndex
let canReserve = await market.canReserveSlot(data.requestId, data.slotIndex)
if canReserve:
try:
trace "Reserving slot"
await market.reserveSlot(data.requestId, data.slotIndex)
except MarketError as e:
if e.msg.contains "SlotReservations_ReservationNotAllowed":
debug "Slot cannot be reserved, ignoring", error = e.msg
return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
else:
return some State(SaleErrored(error: e))
# other CatchableErrors are handled "automatically" by the ErrorHandlingState
try:
let canReserve = await market.canReserveSlot(data.requestId, data.slotIndex)
if canReserve:
try:
trace "Reserving slot"
await market.reserveSlot(data.requestId, data.slotIndex)
except MarketError as e:
if e.msg.contains "SlotReservations_ReservationNotAllowed":
debug "Slot cannot be reserved, ignoring", error = e.msg
return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
else:
return some State(SaleErrored(error: e))
# other CatchableErrors are handled "automatically" by the SaleState
trace "Slot successfully reserved"
return some State(SaleDownloading())
else:
# do not re-add this slot to the queue, and return bytes from Reservation to
# the Availability
debug "Slot cannot be reserved, ignoring"
return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
trace "Slot successfully reserved"
return some State(SaleDownloading())
else:
# do not re-add this slot to the queue, and return bytes from Reservation to
# the Availability
debug "Slot cannot be reserved, ignoring"
return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
except CancelledError as e:
trace "SaleSlotReserving.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleSlotReserving.run", error = e.msgDetail
return some State(SaleErrored(error: e))

59
codex/sales/states/unknown.nim
View File

@ -1,4 +1,5 @@
import ../../logutils
import ../../utils/exceptions
import ../statemachine
import ../salesagent
import ./filled
@ -26,34 +27,42 @@ method onCancelled*(state: SaleUnknown, request: StorageRequest): ?State =
method onFailed*(state: SaleUnknown, request: StorageRequest): ?State =
return some State(SaleFailed())
method run*(state: SaleUnknown, machine: Machine): Future[?State] {.async.} =
method run*(
state: SaleUnknown, machine: Machine
): Future[?State] {.async: (raises: []).} =
let agent = SalesAgent(machine)
let data = agent.data
let market = agent.context.market
await agent.retrieveRequest()
await agent.subscribe()
try:
await agent.retrieveRequest()
await agent.subscribe()
let slotId = slotId(data.requestId, data.slotIndex)
let slotState = await market.slotState(slotId)
let slotId = slotId(data.requestId, data.slotIndex)
let slotState = await market.slotState(slotId)
case slotState
of SlotState.Free:
let error =
newException(UnexpectedSlotError, "Slot state on chain should not be 'free'")
return some State(SaleErrored(error: error))
of SlotState.Filled:
return some State(SaleFilled())
of SlotState.Finished:
return some State(SalePayout())
of SlotState.Paid:
return some State(SaleFinished())
of SlotState.Failed:
return some State(SaleFailed())
of SlotState.Cancelled:
return some State(SaleCancelled())
of SlotState.Repair:
let error = newException(
SlotFreedError, "Slot was forcible freed and host was removed from its hosting"
)
return some State(SaleErrored(error: error))
case slotState
of SlotState.Free:
let error =
newException(UnexpectedSlotError, "Slot state on chain should not be 'free'")
return some State(SaleErrored(error: error))
of SlotState.Filled:
return some State(SaleFilled())
of SlotState.Finished:
return some State(SalePayout())
of SlotState.Paid:
return some State(SaleFinished())
of SlotState.Failed:
return some State(SaleFailed())
of SlotState.Cancelled:
return some State(SaleCancelled())
of SlotState.Repair:
let error = newException(
SlotFreedError, "Slot was forcible freed and host was removed from its hosting"
)
return some State(SaleErrored(error: error))
except CancelledError as e:
trace "SaleUnknown.run was cancelled", error = e.msgDetail
except CatchableError as e:
error "Error during SaleUnknown.run", error = e.msgDetail
return some State(SaleErrored(error: e))

2
codex/slots/builder/builder.nim
View File

@ -189,7 +189,7 @@ proc getCellHashes*[T, H](
blkIdx = blkIdx
pos = i
trace "Getting block CID for tree at index"
trace "Getting block CID for tree at index", index = blkIdx
without (_, tree) =? (await self.buildBlockTree(blkIdx, i)) and digest =? tree.root,
err:
error "Failed to get block CID for tree at index", err = err.msg

14
codex/stores/repostore/operations.nim
View File

@ -57,6 +57,17 @@ proc putLeafMetadata*(
(md.some, res),
)
proc delLeafMetadata*(
self: RepoStore, treeCid: Cid, index: Natural
): Future[?!void] {.async.} =
without key =? createBlockCidAndProofMetadataKey(treeCid, index), err:
return failure(err)
if err =? (await self.metaDs.delete(key)).errorOption:
return failure(err)
success()
proc getLeafMetadata*(
self: RepoStore, treeCid: Cid, index: Natural
): Future[?!LeafMetadata] {.async.} =
@ -205,9 +216,6 @@ proc storeBlock*(
proc tryDeleteBlock*(
self: RepoStore, cid: Cid, expiryLimit = SecondsSince1970.low
): Future[?!DeleteResult] {.async.} =
if cid.isEmpty:
return success(DeleteResult(kind: InUse))
without metaKey =? createBlockExpirationMetadataKey(cid), err:
return failure(err)

55
codex/stores/repostore/store.nim
View File

@ -186,13 +186,13 @@ method putBlock*(
return success()
method delBlock*(self: RepoStore, cid: Cid): Future[?!void] {.async.} =
## Delete a block from the blockstore when block refCount is 0 or block is expired
##
proc delBlockInternal(self: RepoStore, cid: Cid): Future[?!DeleteResultKind] {.async.} =
logScope:
cid = cid
if cid.isEmpty:
return success(Deleted)
trace "Attempting to delete a block"
without res =? await self.tryDeleteBlock(cid, self.clock.now()), err:
@ -205,12 +205,28 @@ method delBlock*(self: RepoStore, cid: Cid): Future[?!void] {.async.} =
if err =? (await self.updateQuotaUsage(minusUsed = res.released)).errorOption:
return failure(err)
elif res.kind == InUse:
trace "Block in use, refCount > 0 and not expired"
else:
trace "Block not found in store"
return success()
success(res.kind)
method delBlock*(self: RepoStore, cid: Cid): Future[?!void] {.async.} =
## Delete a block from the blockstore when block refCount is 0 or block is expired
##
logScope:
cid = cid
without outcome =? await self.delBlockInternal(cid), err:
return failure(err)
case outcome
of InUse:
failure("Directly deleting a block that is part of a dataset is not allowed.")
of NotFound:
trace "Block not found, ignoring"
success()
of Deleted:
trace "Block already deleted"
success()
method delBlock*(
self: RepoStore, treeCid: Cid, index: Natural
@ -221,12 +237,19 @@ method delBlock*(
else:
return failure(err)
if err =? (await self.delLeafMetadata(treeCid, index)).errorOption:
error "Failed to delete leaf metadata, block will remain on disk.", err = err.msg
return failure(err)
if err =?
(await self.updateBlockMetadata(leafMd.blkCid, minusRefCount = 1)).errorOption:
if not (err of BlockNotFoundError):
return failure(err)
await self.delBlock(leafMd.blkCid) # safe delete, only if refCount == 0
without _ =? await self.delBlockInternal(leafMd.blkCid), err:
return failure(err)
success()
method hasBlock*(self: RepoStore, cid: Cid): Future[?!bool] {.async.} =
## Check if the block exists in the blockstore
@ -295,6 +318,18 @@ proc createBlockExpirationQuery(maxNumber: int, offset: int): ?!Query =
let queryKey = ?createBlockExpirationMetadataQueryKey()
success Query.init(queryKey, offset = offset, limit = maxNumber)
proc blockRefCount*(self: RepoStore, cid: Cid): Future[?!Natural] {.async.} =
## Returns the reference count for a block. If the count is zero;
## this means the block is eligible for garbage collection.
##
without key =? createBlockExpirationMetadataKey(cid), err:
return failure(err)
without md =? await get[BlockMetadata](self.metaDs, key), err:
return failure(err)
return success(md.refCount)
method getBlockExpirations*(
self: RepoStore, maxNumber: int, offset: int
): Future[?!AsyncIter[BlockExpiration]] {.async, base.} =

2
codex/streams/storestream.nim
View File

@ -110,7 +110,7 @@ method readOnce*(
raise newLPStreamReadError(error)
trace "Reading bytes from store stream",
manifestCid = self.manifest.cid.get(),
manifestCid = self.manifest.treeCid,
numBlocks = self.manifest.blocksCount,
blockNum,
blkCid = blk.cid,

25
codex/utils/arrayutils.nim Normal file
View File

@ -0,0 +1,25 @@
import std/sequtils
proc createDoubleArray*(
outerLen, innerLen: int
): ptr UncheckedArray[ptr UncheckedArray[byte]] =
# Allocate outer array
result = cast[ptr UncheckedArray[ptr UncheckedArray[byte]]](allocShared0(
sizeof(ptr UncheckedArray[byte]) * outerLen
))
# Allocate each inner array
for i in 0 ..< outerLen:
result[i] = cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * innerLen))
proc freeDoubleArray*(
arr: ptr UncheckedArray[ptr UncheckedArray[byte]], outerLen: int
) =
# Free each inner array
for i in 0 ..< outerLen:
if not arr[i].isNil:
deallocShared(arr[i])
# Free outer array
if not arr.isNil:
deallocShared(arr)

2
codex/utils/asyncheapqueue.nim
View File

@ -9,7 +9,7 @@
import std/sequtils
import pkg/chronos
import pkg/stew/results
import pkg/results
# Based on chronos AsyncHeapQueue and std/heapqueue

21
codex/utils/asyncstatemachine.nim
View File

@ -2,6 +2,7 @@ import pkg/questionable
import pkg/chronos
import ../logutils
import ./trackedfutures
import ./exceptions
{.push raises: [].}
@ -46,24 +47,14 @@ proc schedule*(machine: Machine, event: Event) =
except AsyncQueueFullError:
raiseAssert "unlimited queue is full?!"
method run*(state: State, machine: Machine): Future[?State] {.base, async.} =
method run*(
state: State, machine: Machine
): Future[?State] {.base, async: (raises: []).} =
discard
method onError*(state: State, error: ref CatchableError): ?State {.base.} =
raise (ref Defect)(msg: "error in state machine: " & error.msg, parent: error)
proc onError(machine: Machine, error: ref CatchableError): Event =
return proc(state: State): ?State =
state.onError(error)
proc run(machine: Machine, state: State) {.async: (raises: []).} =
try:
if next =? await state.run(machine):
machine.schedule(Event.transition(state, next))
except CancelledError:
discard # do not propagate
except CatchableError as e:
machine.schedule(machine.onError(e))
if next =? await state.run(machine):
machine.schedule(Event.transition(state, next))
proc scheduler(machine: Machine) {.async: (raises: []).} =
var running: Future[void].Raising([])

3
codex/utils/natutils.nim
View File

@ -1,7 +1,6 @@
{.push raises: [].}
import
std/[tables, hashes], stew/results, stew/shims/net as stewNet, chronos, chronicles
import std/[tables, hashes], pkg/results, stew/shims/net as stewNet, chronos, chronicles
import pkg/libp2p

27
codex/validation.nim
View File

@ -2,6 +2,7 @@ import std/sets
import std/sequtils
import pkg/chronos
import pkg/questionable/results
import pkg/stew/endians2
import ./validationconfig
import ./market
@ -19,11 +20,9 @@ type Validation* = ref object
subscriptions: seq[Subscription]
running: Future[void]
periodicity: Periodicity
proofTimeout: UInt256
proofTimeout: uint64
config: ValidationConfig
const MaxStorageRequestDuration = 30.days
logScope:
topics = "codex validator"
@ -35,18 +34,19 @@ proc new*(
proc slots*(validation: Validation): seq[SlotId] =
validation.slots.toSeq
proc getCurrentPeriod(validation: Validation): UInt256 =
return validation.periodicity.periodOf(validation.clock.now().u256)
proc getCurrentPeriod(validation: Validation): Period =
return validation.periodicity.periodOf(validation.clock.now().Timestamp)
proc waitUntilNextPeriod(validation: Validation) {.async.} =
let period = validation.getCurrentPeriod()
let periodEnd = validation.periodicity.periodEnd(period)
trace "Waiting until next period", currentPeriod = period
await validation.clock.waitUntil(periodEnd.truncate(int64) + 1)
await validation.clock.waitUntil((periodEnd + 1).toSecondsSince1970)
func groupIndexForSlotId*(slotId: SlotId, validationGroups: ValidationGroups): uint16 =
let slotIdUInt256 = UInt256.fromBytesBE(slotId.toArray)
(slotIdUInt256 mod validationGroups.u256).truncate(uint16)
let a = slotId.toArray
let slotIdInt64 = uint64.fromBytesBE(a)
(slotIdInt64 mod uint64(validationGroups)).uint16
func maxSlotsConstraintRespected(validation: Validation): bool =
validation.config.maxSlots == 0 or validation.slots.len < validation.config.maxSlots
@ -57,7 +57,7 @@ func shouldValidateSlot(validation: Validation, slotId: SlotId): bool =
groupIndexForSlotId(slotId, validationGroups) == validation.config.groupIndex
proc subscribeSlotFilled(validation: Validation) {.async.} =
proc onSlotFilled(requestId: RequestId, slotIndex: UInt256) =
proc onSlotFilled(requestId: RequestId, slotIndex: uint64) =
if not validation.maxSlotsConstraintRespected:
return
let slotId = slotId(requestId, slotIndex)
@ -115,14 +115,13 @@ proc run(validation: Validation) {.async: (raises: []).} =
except CatchableError as e:
error "Validation failed", msg = e.msg
proc epochForDurationBackFromNow(
validation: Validation, duration: Duration
): SecondsSince1970 =
return validation.clock.now - duration.secs
proc findEpoch(validation: Validation, secondsAgo: uint64): SecondsSince1970 =
return validation.clock.now - secondsAgo.int64
proc restoreHistoricalState(validation: Validation) {.async.} =
trace "Restoring historical state..."
let startTimeEpoch = validation.epochForDurationBackFromNow(MaxStorageRequestDuration)
let requestDurationLimit = await validation.market.requestDurationLimit
let startTimeEpoch = validation.findEpoch(secondsAgo = requestDurationLimit)
let slotFilledEvents =
await validation.market.queryPastSlotFilledEvents(fromTime = startTimeEpoch)
for event in slotFilledEvents:

79
config.nims
View File

@ -1,21 +1,24 @@
include "build.nims"
import std/os
const currentDir = currentSourcePath()[0 .. ^(len("config.nims") + 1)]
when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and
# BEWARE
# In Nim 1.6, config files are evaluated with a working directory
# matching where the Nim command was invocated. This means that we
# must do all file existence checks with full absolute paths:
system.fileExists(currentDir & "nimbus-build-system.paths"):
# BEWARE
# In Nim 1.6, config files are evaluated with a working directory
# matching where the Nim command was invocated. This means that we
# must do all file existence checks with full absolute paths:
system.fileExists(currentDir & "nimbus-build-system.paths"):
include "nimbus-build-system.paths"
when defined(release):
switch("nimcache", joinPath(currentSourcePath.parentDir, "nimcache/release/$projectName"))
switch(
"nimcache", joinPath(currentSourcePath.parentDir, "nimcache/release/$projectName")
)
else:
switch("nimcache", joinPath(currentSourcePath.parentDir, "nimcache/debug/$projectName"))
switch(
"nimcache", joinPath(currentSourcePath.parentDir, "nimcache/debug/$projectName")
)
when defined(limitStackUsage):
# This limits stack usage of each individual function to 1MB - the option is
@ -34,7 +37,8 @@ when defined(windows):
# increase stack size
switch("passL", "-Wl,--stack,8388608")
# https://github.com/nim-lang/Nim/issues/4057
--tlsEmulation:off
--tlsEmulation:
off
if defined(i386):
# set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM
switch("passL", "-Wl,--large-address-aware")
@ -63,30 +67,47 @@ else:
# ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes)
switch("passC", "-mno-avx512vl")
--tlsEmulation:off
--threads:on
--opt:speed
--excessiveStackTrace:on
--tlsEmulation:
off
--threads:
on
--opt:
speed
--excessiveStackTrace:
on
# enable metric collection
--define:metrics
--define:
metrics
# for heap-usage-by-instance-type metrics and object base-type strings
--define:nimTypeNames
--styleCheck:usages
--styleCheck:error
--maxLoopIterationsVM:1000000000
--fieldChecks:on
--warningAsError:"ProveField:on"
--define:
nimTypeNames
--styleCheck:
usages
--styleCheck:
error
--maxLoopIterationsVM:
1000000000
--fieldChecks:
on
--warningAsError:
"ProveField:on"
when (NimMajor, NimMinor) >= (1, 4):
--warning:"ObservableStores:off"
--warning:"LockLevel:off"
--hint:"XCannotRaiseY:off"
--warning:
"ObservableStores:off"
--warning:
"LockLevel:off"
--hint:
"XCannotRaiseY:off"
when (NimMajor, NimMinor) >= (1, 6):
--warning:"DotLikeOps:off"
--warning:
"DotLikeOps:off"
when (NimMajor, NimMinor, NimPatch) >= (1, 6, 11):
--warning:"BareExcept:off"
--warning:
"BareExcept:off"
when (NimMajor, NimMinor) >= (2, 0):
--mm:refc
--mm:
refc
switch("define", "withoutPCRE")
@ -94,10 +115,12 @@ switch("define", "withoutPCRE")
# "--debugger:native" build. It can be increased with `ulimit -n 1024`.
if not defined(macosx):
# add debugging symbols and original files and line numbers
--debugger:native
--debugger:
native
if not (defined(windows) and defined(i386)) and not defined(disable_libbacktrace):
# light-weight stack traces using libbacktrace and libunwind
--define:nimStackTraceOverride
--define:
nimStackTraceOverride
switch("import", "libbacktrace")
# `switch("warning[CaseTransition]", "off")` fails with "Error: invalid command line option: '--warning[CaseTransition]'"

6
openapi.yaml
View File

@ -371,12 +371,6 @@ components:
nullable: true
description: "The original mimetype of the uploaded content (optional)"
example: image/png
uploadedAt:
type: integer
format: int64
nullable: true
description: "The UTC upload timestamp in seconds"
example: 1729244192
Space:
type: object

4
tests/codex/blockexchange/discovery/testdiscovery.nim
View File

@ -96,9 +96,9 @@ asyncchecksuite "Block Advertising and Discovery":
await engine.stop()
test "Should advertise both manifests and trees":
test "Should advertise trees":
let
cids = @[manifest.cid.tryGet, manifest.treeCid]
cids = @[manifest.treeCid]
advertised = initTable.collect:
for cid in cids:
{cid: newFuture[void]()}

24
tests/codex/examples.nim
View File

@ -8,6 +8,7 @@ import pkg/codex/stores
import pkg/codex/blocktype as bt
import pkg/codex/sales
import pkg/codex/merkletree
import pkg/codex/manifest
import ../examples
export examples
@ -36,8 +37,8 @@ proc example*(_: type SignedState): SignedState =
proc example*(_: type Pricing): Pricing =
Pricing(address: EthAddress.example, price: uint32.rand.u256)
proc example*(_: type bt.Block): bt.Block =
let length = rand(4096)
proc example*(_: type bt.Block, size: int = 4096): bt.Block =
let length = rand(size)
let bytes = newSeqWith(length, rand(uint8))
bt.Block.new(bytes).tryGet()
@ -51,6 +52,15 @@ proc example*(_: type BlockExcPeerCtx): BlockExcPeerCtx =
proc example*(_: type Cid): Cid =
bt.Block.example.cid
proc example*(_: type Manifest): Manifest =
Manifest.new(
treeCid = Cid.example,
blockSize = 256.NBytes,
datasetSize = 4096.NBytes,
filename = "example.txt".some,
mimetype = "text/plain".some,
)
proc example*(_: type MultiHash, mcodec = Sha256HashCodec): MultiHash =
let bytes = newSeqWith(256, rand(uint8))
MultiHash.digest($mcodec, bytes).tryGet()
@ -58,19 +68,19 @@ proc example*(_: type MultiHash, mcodec = Sha256HashCodec): MultiHash =
proc example*(
_: type Availability, collateralPerByte = uint8.example.u256
): Availability =
let totalSize = uint16.example.u256
let totalSize = uint16.example.uint64
Availability.init(
totalSize = totalSize,
freeSize = uint16.example.u256,
duration = uint16.example.u256,
freeSize = uint16.example.uint64,
duration = uint16.example.uint64,
minPricePerBytePerSecond = uint8.example.u256,
totalCollateral = totalSize * collateralPerByte,
totalCollateral = totalSize.u256 * collateralPerByte,
)
proc example*(_: type Reservation): Reservation =
Reservation.init(
availabilityId = AvailabilityId(array[32, byte].example),
size = uint16.example.u256,
size = uint16.example.uint64,
slotId = SlotId.example,
)

27
tests/codex/helpers.nim
View File

@ -85,30 +85,31 @@ proc makeWantList*(
)
proc storeDataGetManifest*(
store: BlockStore, chunker: Chunker
store: BlockStore, blocks: seq[Block]
): Future[Manifest] {.async.} =
var cids = newSeq[Cid]()
while (let chunk = await chunker.getBytes(); chunk.len > 0):
let blk = Block.new(chunk).tryGet()
cids.add(blk.cid)
for blk in blocks:
(await store.putBlock(blk)).tryGet()
let
tree = CodexTree.init(cids).tryGet()
(manifest, tree) = makeManifestAndTree(blocks).tryGet()
treeCid = tree.rootCid.tryGet()
manifest = Manifest.new(
treeCid = treeCid,
blockSize = NBytes(chunker.chunkSize),
datasetSize = NBytes(chunker.offset),
)
for i in 0 ..< tree.leavesCount:
let proof = tree.getProof(i).tryGet()
(await store.putCidAndProof(treeCid, i, cids[i], proof)).tryGet()
(await store.putCidAndProof(treeCid, i, blocks[i].cid, proof)).tryGet()
return manifest
proc storeDataGetManifest*(
store: BlockStore, chunker: Chunker
): Future[Manifest] {.async.} =
var blocks = newSeq[Block]()
while (let chunk = await chunker.getBytes(); chunk.len > 0):
blocks.add(Block.new(chunk).tryGet())
return await storeDataGetManifest(store, blocks)
proc makeRandomBlocks*(
datasetSize: int, blockSize: NBytes
): Future[seq[Block]] {.async.} =

40
tests/codex/helpers/mockmarket.nim
View File

@ -57,7 +57,7 @@ type
MockSlot* = object
requestId*: RequestId
host*: Address
slotIndex*: UInt256
slotIndex*: uint64
proof*: Groth16Proof
timestamp: ?SecondsSince1970
collateral*: UInt256
@ -84,7 +84,7 @@ type
SlotFilledSubscription* = ref object of Subscription
market: MockMarket
requestId: ?RequestId
slotIndex: ?UInt256
slotIndex: ?uint64
callback: OnSlotFilled
SlotFreedSubscription* = ref object of Subscription
@ -122,12 +122,17 @@ proc new*(_: type MockMarket, clock: ?Clock = Clock.none): MockMarket =
collateral: CollateralConfig(
repairRewardPercentage: 10,
maxNumberOfSlashes: 5,
slashCriterion: 3,
slashPercentage: 10,
validatorRewardPercentage: 20,
),
proofs: ProofConfig(
period: 10.u256, timeout: 5.u256, downtime: 64.uint8, downtimeProduct: 67.uint8
period: 10.Period,
timeout: 5.uint64,
downtime: 64.uint8,
downtimeProduct: 67.uint8,
),
reservations: SlotReservationsConfig(maxReservations: 3),
requestDurationLimit: (60 * 60 * 24 * 30).uint64,
)
MockMarket(
signer: Address.example, config: config, canReserveSlot: true, clock: clock
@ -139,9 +144,12 @@ method getSigner*(market: MockMarket): Future[Address] {.async.} =
method periodicity*(mock: MockMarket): Future[Periodicity] {.async.} =
return Periodicity(seconds: mock.config.proofs.period)
method proofTimeout*(market: MockMarket): Future[UInt256] {.async.} =
method proofTimeout*(market: MockMarket): Future[uint64] {.async.} =
return market.config.proofs.timeout
method requestDurationLimit*(market: MockMarket): Future[uint64] {.async.} =
return market.config.requestDurationLimit
method proofDowntime*(market: MockMarket): Future[uint8] {.async.} =
return market.config.proofs.downtime
@ -171,9 +179,9 @@ method getRequest*(
return some request
return none StorageRequest
method getActiveSlot*(market: MockMarket, slotId: SlotId): Future[?Slot] {.async.} =
method getActiveSlot*(market: MockMarket, id: SlotId): Future[?Slot] {.async.} =
for slot in market.filled:
if slotId(slot.requestId, slot.slotIndex) == slotId and
if slotId(slot.requestId, slot.slotIndex) == id and
request =? await market.getRequest(slot.requestId):
return some Slot(request: request, slotIndex: slot.slotIndex)
return none Slot
@ -199,7 +207,7 @@ method requestExpiresAt*(
return market.requestExpiry[id]
method getHost*(
market: MockMarket, requestId: RequestId, slotIndex: UInt256
market: MockMarket, requestId: RequestId, slotIndex: uint64
): Future[?Address] {.async.} =
for slot in market.filled:
if slot.requestId == requestId and slot.slotIndex == slotIndex:
@ -214,7 +222,7 @@ method currentCollateral*(
return slot.collateral
return 0.u256
proc emitSlotFilled*(market: MockMarket, requestId: RequestId, slotIndex: UInt256) =
proc emitSlotFilled*(market: MockMarket, requestId: RequestId, slotIndex: uint64) =
var subscriptions = market.subscriptions.onSlotFilled
for subscription in subscriptions:
let requestMatches =
@ -224,13 +232,13 @@ proc emitSlotFilled*(market: MockMarket, requestId: RequestId, slotIndex: UInt25
if requestMatches and slotMatches:
subscription.callback(requestId, slotIndex)
proc emitSlotFreed*(market: MockMarket, requestId: RequestId, slotIndex: UInt256) =
proc emitSlotFreed*(market: MockMarket, requestId: RequestId, slotIndex: uint64) =
var subscriptions = market.subscriptions.onSlotFreed
for subscription in subscriptions:
subscription.callback(requestId, slotIndex)
proc emitSlotReservationsFull*(
market: MockMarket, requestId: RequestId, slotIndex: UInt256
market: MockMarket, requestId: RequestId, slotIndex: uint64
) =
var subscriptions = market.subscriptions.onSlotReservationsFull
for subscription in subscriptions:
@ -257,7 +265,7 @@ proc emitRequestFailed*(market: MockMarket, requestId: RequestId) =
proc fillSlot*(
market: MockMarket,
requestId: RequestId,
slotIndex: UInt256,
slotIndex: uint64,
proof: Groth16Proof,
host: Address,
collateral = 0.u256,
@ -277,7 +285,7 @@ proc fillSlot*(
method fillSlot*(
market: MockMarket,
requestId: RequestId,
slotIndex: UInt256,
slotIndex: uint64,
proof: Groth16Proof,
collateral: UInt256,
) {.async.} =
@ -341,13 +349,13 @@ method canProofBeMarkedAsMissing*(
return market.canBeMarkedAsMissing.contains(id)
method reserveSlot*(
market: MockMarket, requestId: RequestId, slotIndex: UInt256
market: MockMarket, requestId: RequestId, slotIndex: uint64
) {.async.} =
if error =? market.reserveSlotThrowError:
raise error
method canReserveSlot*(
market: MockMarket, requestId: RequestId, slotIndex: UInt256
market: MockMarket, requestId: RequestId, slotIndex: uint64
): Future[bool] {.async.} =
return market.canReserveSlot
@ -390,7 +398,7 @@ method subscribeSlotFilled*(
return subscription
method subscribeSlotFilled*(
market: MockMarket, requestId: RequestId, slotIndex: UInt256, callback: OnSlotFilled
market: MockMarket, requestId: RequestId, slotIndex: uint64, callback: OnSlotFilled
): Future[Subscription] {.async.} =
let subscription = SlotFilledSubscription(
market: market,

4
tests/codex/helpers/mockreservations.nim
View File

@ -24,9 +24,9 @@ proc setCreateReservationThrowError*(
method createReservation*(
self: MockReservations,
availabilityId: AvailabilityId,
slotSize: UInt256,
slotSize: uint64,
requestId: RequestId,
slotIndex: UInt256,
slotIndex: uint64,
collateralPerByte: UInt256,
): Future[?!Reservation] {.async.} =
if self.createReservationThrowBytesOutOfBoundsError:

2
tests/codex/helpers/mocksalesagent.nim
View File

@ -12,6 +12,6 @@ method onFailed*(agent: SalesAgent, requestId: RequestId) =
failedCalled = true
method onSlotFilled*(
agent: SalesAgent, requestId: RequestId, slotIndex: UInt256
agent: SalesAgent, requestId: RequestId, slotIndex: uint64
) {.base.} =
slotFilledCalled = true

6
tests/codex/helpers/mockslotqueueitem.nim
View File

@ -4,11 +4,11 @@ import pkg/codex/sales/slotqueue
type MockSlotQueueItem* = object
requestId*: RequestId
slotIndex*: uint16
slotSize*: UInt256
duration*: UInt256
slotSize*: uint64
duration*: uint64
pricePerBytePerSecond*: UInt256
collateralPerByte*: UInt256
expiry*: UInt256
expiry*: uint64
seen*: bool
proc toSlotQueueItem*(item: MockSlotQueueItem): SlotQueueItem =

2
tests/codex/node/helpers.nim
View File

@ -6,6 +6,7 @@ import pkg/chronos
import pkg/codex/codextypes
import pkg/codex/chunker
import pkg/codex/stores
import pkg/taskpools
import ../../asynctest
@ -118,6 +119,7 @@ template setupAndTearDown*() {.dirty.} =
engine = engine,
prover = Prover.none,
discovery = blockDiscovery,
taskpool = Taskpool.new(),
)
teardown:

12
tests/codex/node/testcontracts.nim
View File

@ -75,10 +75,9 @@ asyncchecksuite "Test Node - Host contracts":
let
manifestBlock =
bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet()
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider)
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, Taskpool.new)
manifestCid = manifestBlock.cid
manifestCidStr = $(manifestCid)
(await localStore.putBlock(manifestBlock)).tryGet()
@ -99,7 +98,7 @@ asyncchecksuite "Test Node - Host contracts":
expectedExpiry: SecondsSince1970 = clock.now + DefaultBlockTtl.seconds + 11123
expiryUpdateCallback = !sales.onExpiryUpdate
(await expiryUpdateCallback(manifestCidStr, expectedExpiry)).tryGet()
(await expiryUpdateCallback(manifestCid, expectedExpiry)).tryGet()
for index in 0 ..< manifest.blocksCount:
let
@ -116,8 +115,9 @@ asyncchecksuite "Test Node - Host contracts":
test "onStore callback":
let onStore = !sales.onStore
var request = StorageRequest.example
request.content.cid = $verifiableBlock.cid
request.expiry = (getTime() + DefaultBlockTtl.toTimesDuration + 1.hours).toUnix.u256
request.content.cid = verifiableBlock.cid
request.expiry =
(getTime() + DefaultBlockTtl.toTimesDuration + 1.hours).toUnix.uint64
var fetchedBytes: uint = 0
let onBlocks = proc(blocks: seq[bt.Block]): Future[?!void] {.async.} =
@ -125,7 +125,7 @@ asyncchecksuite "Test Node - Host contracts":
fetchedBytes += blk.data.len.uint
return success()
(await onStore(request, 1.u256, onBlocks)).tryGet()
(await onStore(request, 1.uint64, onBlocks)).tryGet()
check fetchedBytes == 12 * DefaultBlockSize.uint
let indexer = verifiable.protectedStrategy.init(

38
tests/codex/node/testnode.nim
View File

@ -12,6 +12,7 @@ import pkg/questionable/results
import pkg/stint
import pkg/poseidon2
import pkg/poseidon2/io
import pkg/taskpools
import pkg/nitro
import pkg/codexdht/discv5/protocol as discv5
@ -37,6 +38,7 @@ import ../examples
import ../helpers
import ../helpers/mockmarket
import ../helpers/mockclock
import ../slots/helpers
import ./helpers
@ -66,7 +68,7 @@ asyncchecksuite "Test Node - Basic":
# https://github.com/codex-storage/nim-codex/issues/699
let
cstore = CountingStore.new(engine, localStore)
node = CodexNodeRef.new(switch, cstore, engine, blockDiscovery)
node = CodexNodeRef.new(switch, cstore, engine, blockDiscovery, Taskpool.new())
missingCid =
Cid.init("zDvZRwzmCvtiyubW9AecnxgLnXK8GrBvpQJBDzToxmzDN6Nrc2CZ").get()
@ -137,7 +139,8 @@ asyncchecksuite "Test Node - Basic":
test "Setup purchase request":
let
erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider)
erasure =
Erasure.new(store, leoEncoderProvider, leoDecoderProvider, Taskpool.new())
manifest = await storeDataGetManifest(localStore, chunker)
manifestBlock =
bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet()
@ -154,15 +157,40 @@ asyncchecksuite "Test Node - Basic":
cid = manifestBlock.cid,
nodes = 5,
tolerance = 2,
duration = 100.u256,
duration = 100.uint64,
pricePerBytePerSecond = 1.u256,
proofProbability = 3.u256,
expiry = 200.u256,
expiry = 200.uint64,
collateralPerByte = 1.u256,
)
).tryGet
check:
(await verifiableBlock.cid in localStore) == true
request.content.cid == $verifiableBlock.cid
request.content.cid == verifiableBlock.cid
request.content.merkleRoot == builder.verifyRoot.get.toBytes
test "Should delete a single block":
let randomBlock = bt.Block.new("Random block".toBytes).tryGet()
(await localStore.putBlock(randomBlock)).tryGet()
check (await randomBlock.cid in localStore) == true
(await node.delete(randomBlock.cid)).tryGet()
check (await randomBlock.cid in localStore) == false
test "Should delete an entire dataset":
let
blocks = await makeRandomBlocks(datasetSize = 2048, blockSize = 256'nb)
manifest = await storeDataGetManifest(localStore, blocks)
manifestBlock = (await store.storeManifest(manifest)).tryGet()
manifestCid = manifestBlock.cid
check await manifestCid in localStore
for blk in blocks:
check await blk.cid in localStore
(await node.delete(manifestCid)).tryGet()
check not await manifestCid in localStore
for blk in blocks:
check not (await blk.cid in localStore)

4
tests/codex/sales/helpers/periods.nim
View File

@ -3,6 +3,6 @@ import ../../helpers/mockclock
proc advanceToNextPeriod*(clock: MockClock, market: Market) {.async.} =
let periodicity = await market.periodicity()
let period = periodicity.periodOf(clock.now().u256)
let period = periodicity.periodOf(clock.now().Timestamp)
let periodEnd = periodicity.periodEnd(period)
clock.set((periodEnd + 1).truncate(int))
clock.set(periodEnd.toSecondsSince1970 + 1)

2
tests/codex/sales/states/testcancelled.nim
View File

@ -14,7 +14,7 @@ import ../../helpers/mockclock
asyncchecksuite "sales state 'cancelled'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
let clock = MockClock.new()
let currentCollateral = UInt256.example

2
tests/codex/sales/states/testdownloading.nim
View File

@ -10,7 +10,7 @@ import ../../helpers
checksuite "sales state 'downloading'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
var state: SaleDownloading
setup:

2
tests/codex/sales/states/testerrored.nim
View File

@ -14,7 +14,7 @@ import ../../helpers/mockclock
asyncchecksuite "sales state 'errored'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
let market = MockMarket.new()
let clock = MockClock.new()

4
tests/codex/sales/states/testfilled.nim
View File

@ -16,7 +16,7 @@ import ../../helpers
checksuite "sales state 'filled'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
var market: MockMarket
var slot: MockSlot
@ -36,7 +36,7 @@ checksuite "sales state 'filled'":
market.requestEnds[request.id] = 321
onExpiryUpdatePassedExpiry = -1
let onExpiryUpdate = proc(
rootCid: string, expiry: SecondsSince1970
rootCid: Cid, expiry: SecondsSince1970
): Future[?!void] {.async.} =
onExpiryUpdatePassedExpiry = expiry
return success()

2
tests/codex/sales/states/testfilling.nim
View File

@ -9,7 +9,7 @@ import ../../helpers
checksuite "sales state 'filling'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
var state: SaleFilling
setup:

2
tests/codex/sales/states/testfinished.nim
View File

@ -15,7 +15,7 @@ import ../../helpers/mockclock
asyncchecksuite "sales state 'finished'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
let clock = MockClock.new()
let currentCollateral = UInt256.example

2
tests/codex/sales/states/testignored.nim
View File

@ -14,7 +14,7 @@ import ../../helpers/mockclock
asyncchecksuite "sales state 'ignored'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
let market = MockMarket.new()
let clock = MockClock.new()

2
tests/codex/sales/states/testinitialproving.nim
View File

@ -20,7 +20,7 @@ import ../helpers/periods
asyncchecksuite "sales state 'initialproving'":
let proof = Groth16Proof.example
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
let market = MockMarket.new()
let clock = MockClock.new()

2
tests/codex/sales/states/testpayout.nim
View File

@ -15,7 +15,7 @@ import ../../helpers/mockclock
asyncchecksuite "sales state 'payout'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
let clock = MockClock.new()
let currentCollateral = UInt256.example

8
tests/codex/sales/states/testpreparing.nim
View File

@ -22,7 +22,7 @@ import ../../helpers/mockclock
asyncchecksuite "sales state 'preparing'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
let market = MockMarket.new()
let clock = MockClock.new()
var agent: SalesAgent
@ -34,9 +34,9 @@ asyncchecksuite "sales state 'preparing'":
setup:
availability = Availability.init(
totalSize = request.ask.slotSize + 100.u256,
freeSize = request.ask.slotSize + 100.u256,
duration = request.ask.duration + 60.u256,
totalSize = request.ask.slotSize + 100.uint64,
freeSize = request.ask.slotSize + 100.uint64,
duration = request.ask.duration + 60.uint64,
minPricePerBytePerSecond = request.ask.pricePerBytePerSecond,
totalCollateral = request.ask.collateralPerSlot * request.ask.slots.u256,
)

4
tests/codex/sales/states/testproving.nim
View File

@ -40,9 +40,9 @@ asyncchecksuite "sales state 'proving'":
proc advanceToNextPeriod(market: Market) {.async.} =
let periodicity = await market.periodicity()
let current = periodicity.periodOf(clock.now().u256)
let current = periodicity.periodOf(clock.now().Timestamp)
let periodEnd = periodicity.periodEnd(current)
clock.set(periodEnd.truncate(int64) + 1)
clock.set(periodEnd.toSecondsSince1970 + 1)
test "switches to cancelled state when request expires":
let next = state.onCancelled(request)

4
tests/codex/sales/states/testsimulatedproving.nim
View File

@ -56,9 +56,9 @@ asyncchecksuite "sales state 'simulated-proving'":
proc advanceToNextPeriod(market: Market) {.async.} =
let periodicity = await market.periodicity()
let current = periodicity.periodOf(clock.now().u256)
let current = periodicity.periodOf(clock.now().Timestamp)
let periodEnd = periodicity.periodEnd(current)
clock.set(periodEnd.truncate(int64) + 1)
clock.set(periodEnd.toSecondsSince1970 + 1)
proc waitForProvingRounds(market: Market, rounds: int) {.async.} =
var rnds = rounds - 1 # proof round runs prior to advancing

2
tests/codex/sales/states/testslotreserving.nim
View File

@ -19,7 +19,7 @@ import ../../helpers/mockclock
asyncchecksuite "sales state 'SlotReserving'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
var market: MockMarket
var clock: MockClock
var agent: SalesAgent

2
tests/codex/sales/states/testunknown.nim
View File

@ -16,7 +16,7 @@ import ../../helpers
checksuite "sales state 'unknown'":
let request = StorageRequest.example
let slotIndex = (request.ask.slots div 2).u256
let slotIndex = request.ask.slots div 2
let slotId = slotId(request.id, slotIndex)
var market: MockMarket

41
tests/codex/sales/testreservations.nim
View File

@ -41,17 +41,17 @@ asyncchecksuite "Reservations module":
proc createAvailability(): Availability =
let example = Availability.example(collateralPerByte)
let totalSize = rand(100000 .. 200000).u256
let totalCollateral = totalSize * collateralPerByte
let totalSize = rand(100000 .. 200000).uint64
let totalCollateral = totalSize.u256 * collateralPerByte
let availability = waitFor reservations.createAvailability(
totalSize, example.duration, example.minPricePerBytePerSecond, totalCollateral
)
return availability.get
proc createReservation(availability: Availability): Reservation =
let size = rand(1 ..< availability.freeSize.truncate(int))
let size = rand(1 ..< availability.freeSize.int)
let reservation = waitFor reservations.createReservation(
availability.id, size.u256, RequestId.example, UInt256.example, 1.u256
availability.id, size.uint64, RequestId.example, uint64.example, 1.u256
)
return reservation.get
@ -64,8 +64,8 @@ asyncchecksuite "Reservations module":
check (await reservations.all(Availability)).get.len == 0
test "generates unique ids for storage availability":
let availability1 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256, 5.u256)
let availability2 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256, 5.u256)
let availability1 = Availability.init(1.uint64, 2.uint64, 3.uint64, 4.u256, 5.u256)
let availability2 = Availability.init(1.uint64, 2.uint64, 3.uint64, 4.u256, 5.u256)
check availability1.id != availability2.id
test "can reserve available storage":
@ -75,7 +75,7 @@ asyncchecksuite "Reservations module":
test "creating availability reserves bytes in repo":
let orig = repo.available.uint
let availability = createAvailability()
check repo.available.uint == (orig.u256 - availability.freeSize).truncate(uint)
check repo.available.uint == orig - availability.freeSize
test "can get all availabilities":
let availability1 = createAvailability()
@ -129,7 +129,7 @@ asyncchecksuite "Reservations module":
test "cannot create reservation with non-existant availability":
let availability = Availability.example
let created = await reservations.createReservation(
availability.id, UInt256.example, RequestId.example, UInt256.example, 1.u256
availability.id, uint64.example, RequestId.example, uint64.example, 1.u256
)
check created.isErr
check created.error of NotExistsError
@ -140,7 +140,7 @@ asyncchecksuite "Reservations module":
availability.id,
availability.totalSize + 1,
RequestId.example,
UInt256.example,
uint64.example,
UInt256.example,
)
check created.isErr
@ -153,12 +153,12 @@ asyncchecksuite "Reservations module":
availability.id,
availability.totalSize - 1,
RequestId.example,
UInt256.example,
uint64.example,
UInt256.example,
)
let two = reservations.createReservation(
availability.id, availability.totalSize, RequestId.example, UInt256.example,
availability.id, availability.totalSize, RequestId.example, uint64.example,
UInt256.example,
)
@ -228,7 +228,7 @@ asyncchecksuite "Reservations module":
let reservation = createReservation(availability)
let orig = availability.freeSize - reservation.size
let origQuota = repo.quotaReservedBytes
let returnedBytes = reservation.size + 200.u256
let returnedBytes = reservation.size + 200.uint64
check isOk await reservations.returnBytesToAvailability(
reservation.availabilityId, reservation.id, returnedBytes
@ -238,7 +238,7 @@ asyncchecksuite "Reservations module":
let updated = !(await reservations.get(key, Availability))
check updated.freeSize > orig
check (updated.freeSize - orig) == 200.u256
check (updated.freeSize - orig) == 200.uint64
check (repo.quotaReservedBytes - origQuota) == 200.NBytes
test "update releases quota when lowering size":
@ -271,14 +271,14 @@ asyncchecksuite "Reservations module":
let availability = createAvailability()
let reservation = createReservation(availability)
let updated = await reservations.release(
reservation.id, reservation.availabilityId, (reservation.size + 1).truncate(uint)
reservation.id, reservation.availabilityId, reservation.size + 1
)
check updated.isErr
check updated.error of BytesOutOfBoundsError
test "cannot release bytes from non-existant reservation":
let availability = createAvailability()
let reservation = createReservation(availability)
discard createReservation(availability)
let updated = await reservations.release(ReservationId.example, availability.id, 1)
check updated.isErr
check updated.error of NotExistsError
@ -297,7 +297,7 @@ asyncchecksuite "Reservations module":
var added: Availability
reservations.onAvailabilityAdded = proc(a: Availability) {.async.} =
added = a
availability.freeSize += 1.u256
availability.freeSize += 1
discard await reservations.update(availability)
check added == availability
@ -307,7 +307,7 @@ asyncchecksuite "Reservations module":
var called = false
reservations.onAvailabilityAdded = proc(a: Availability) {.async.} =
called = true
availability.freeSize -= 1.u256
availability.freeSize -= 1
discard await reservations.update(availability)
check not called
@ -356,14 +356,11 @@ asyncchecksuite "Reservations module":
check reservations.hasAvailable(DefaultQuotaBytes.uint - 1)
test "reports quota not available to be reserved":
check not reservations.hasAvailable(DefaultQuotaBytes.uint + 1)
check not reservations.hasAvailable(DefaultQuotaBytes.uint64 + 1)
test "fails to create availability with size that is larger than available quota":
let created = await reservations.createAvailability(
(DefaultQuotaBytes.uint + 1).u256,
UInt256.example,
UInt256.example,
UInt256.example,
DefaultQuotaBytes.uint64 + 1, uint64.example, UInt256.example, UInt256.example
)
check created.isErr
check created.error of ReserveFailedError

150
tests/codex/sales/testsales.nim
View File

@ -36,18 +36,21 @@ asyncchecksuite "Sales - start":
var repo: RepoStore
var queue: SlotQueue
var itemsProcessed: seq[SlotQueueItem]
var expiry: SecondsSince1970
setup:
request = StorageRequest(
ask: StorageAsk(
slots: 4,
slotSize: 100.u256,
duration: 60.u256,
slotSize: 100.uint64,
duration: 60.uint64,
pricePerBytePerSecond: 1.u256,
collateralPerByte: 1.u256,
),
content: StorageContent(cid: "some cid"),
expiry: (getTime() + initDuration(hours = 1)).toUnix.u256,
content: StorageContent(
cid: Cid.init("zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob").tryGet
),
expiry: (getTime() + initDuration(hours = 1)).toUnix.uint64,
)
market = MockMarket.new()
@ -59,12 +62,12 @@ asyncchecksuite "Sales - start":
sales = Sales.new(market, clock, repo)
reservations = sales.context.reservations
sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
request: StorageRequest, slot: uint64, onBatch: BatchProc
): Future[?!void] {.async.} =
return success()
sales.onExpiryUpdate = proc(
rootCid: string, expiry: SecondsSince1970
rootCid: Cid, expiry: SecondsSince1970
): Future[?!void] {.async.} =
return success()
@ -74,7 +77,8 @@ asyncchecksuite "Sales - start":
): Future[?!Groth16Proof] {.async.} =
return success(proof)
itemsProcessed = @[]
request.expiry = (clock.now() + 42).u256
expiry = (clock.now() + 42)
request.expiry = expiry.uint64
teardown:
await sales.stop()
@ -82,7 +86,7 @@ asyncchecksuite "Sales - start":
await repoTmp.destroyDb()
await metaTmp.destroyDb()
proc fillSlot(slotIdx: UInt256 = 0.u256) {.async.} =
proc fillSlot(slotIdx: uint64 = 0.uint64) {.async.} =
let address = await market.getSigner()
let slot =
MockSlot(requestId: request.id, slotIndex: slotIdx, proof: proof, host: address)
@ -95,16 +99,15 @@ asyncchecksuite "Sales - start":
request.ask.slots = 2
market.requested = @[request]
market.requestState[request.id] = RequestState.New
market.requestExpiry[request.id] = expiry
let slot0 =
MockSlot(requestId: request.id, slotIndex: 0.u256, proof: proof, host: me)
let slot0 = MockSlot(requestId: request.id, slotIndex: 0, proof: proof, host: me)
await fillSlot(slot0.slotIndex)
let slot1 =
MockSlot(requestId: request.id, slotIndex: 1.u256, proof: proof, host: me)
let slot1 = MockSlot(requestId: request.id, slotIndex: 1, proof: proof, host: me)
await fillSlot(slot1.slotIndex)
market.activeSlots[me] = @[request.slotId(0.u256), request.slotId(1.u256)]
market.activeSlots[me] = @[request.slotId(0), request.slotId(1)]
market.requested = @[request]
market.activeRequests[me] = @[request.id]
@ -112,10 +115,10 @@ asyncchecksuite "Sales - start":
check eventually sales.agents.len == 2
check sales.agents.any(
agent => agent.data.requestId == request.id and agent.data.slotIndex == 0.u256
agent => agent.data.requestId == request.id and agent.data.slotIndex == 0.uint64
)
check sales.agents.any(
agent => agent.data.requestId == request.id and agent.data.slotIndex == 1.u256
agent => agent.data.requestId == request.id and agent.data.slotIndex == 1.uint64
)
asyncchecksuite "Sales":
@ -124,7 +127,7 @@ asyncchecksuite "Sales":
repoTmp = TempLevelDb.new()
metaTmp = TempLevelDb.new()
var totalAvailabilitySize: UInt256
var totalAvailabilitySize: uint64
var minPricePerBytePerSecond: UInt256
var requestedCollateralPerByte: UInt256
var totalCollateral: UInt256
@ -139,27 +142,29 @@ asyncchecksuite "Sales":
var itemsProcessed: seq[SlotQueueItem]
setup:
totalAvailabilitySize = 100.u256
totalAvailabilitySize = 100.uint64
minPricePerBytePerSecond = 1.u256
requestedCollateralPerByte = 1.u256
totalCollateral = requestedCollateralPerByte * totalAvailabilitySize
totalCollateral = requestedCollateralPerByte * totalAvailabilitySize.stuint(256)
availability = Availability.init(
totalSize = totalAvailabilitySize,
freeSize = totalAvailabilitySize,
duration = 60.u256,
duration = 60.uint64,
minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = totalCollateral,
)
request = StorageRequest(
ask: StorageAsk(
slots: 4,
slotSize: 100.u256,
duration: 60.u256,
slotSize: 100.uint64,
duration: 60.uint64,
pricePerBytePerSecond: minPricePerBytePerSecond,
collateralPerByte: 1.u256,
),
content: StorageContent(cid: "some cid"),
expiry: (getTime() + initDuration(hours = 1)).toUnix.u256,
content: StorageContent(
cid: Cid.init("zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob").tryGet
),
expiry: (getTime() + initDuration(hours = 1)).toUnix.uint64,
)
market = MockMarket.new()
@ -176,12 +181,12 @@ asyncchecksuite "Sales":
sales = Sales.new(market, clock, repo)
reservations = sales.context.reservations
sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
request: StorageRequest, slot: uint64, onBatch: BatchProc
): Future[?!void] {.async.} =
return success()
sales.onExpiryUpdate = proc(
rootCid: string, expiry: SecondsSince1970
rootCid: Cid, expiry: SecondsSince1970
): Future[?!void] {.async.} =
return success()
@ -281,13 +286,13 @@ asyncchecksuite "Sales":
test "removes slot index from slot queue once SlotFilled emitted":
let request1 = await addRequestToSaturatedQueue()
market.emitSlotFilled(request1.id, 1.u256)
market.emitSlotFilled(request1.id, 1.uint64)
let expected = SlotQueueItem.init(request1, 1'u16)
check always (not itemsProcessed.contains(expected))
test "removes slot index from slot queue once SlotReservationsFull emitted":
let request1 = await addRequestToSaturatedQueue()
market.emitSlotReservationsFull(request1.id, 1.u256)
market.emitSlotReservationsFull(request1.id, 1.uint64)
let expected = SlotQueueItem.init(request1, 1'u16)
check always (not itemsProcessed.contains(expected))
@ -298,7 +303,7 @@ asyncchecksuite "Sales":
createAvailability()
market.requested.add request # "contract" must be able to return request
market.emitSlotFreed(request.id, 2.u256)
market.emitSlotFreed(request.id, 2.uint64)
let expected = SlotQueueItem.init(request, 2.uint16)
check eventually itemsProcessed.contains(expected)
@ -343,10 +348,10 @@ asyncchecksuite "Sales":
test "availability size is reduced by request slot size when fully downloaded":
sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
request: StorageRequest, slot: uint64, onBatch: BatchProc
): Future[?!void] {.async.} =
let blk = bt.Block.new(@[1.byte]).get
await onBatch(blk.repeat(request.ask.slotSize.truncate(int)))
await onBatch(blk.repeat(request.ask.slotSize.int))
createAvailability()
await market.requestStorage(request)
@ -354,16 +359,16 @@ asyncchecksuite "Sales":
availability.freeSize - request.ask.slotSize
test "non-downloaded bytes are returned to availability once finished":
var slotIndex = 0.u256
var slotIndex = 0.uint64
sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
request: StorageRequest, slot: uint64, onBatch: BatchProc
): Future[?!void] {.async.} =
slotIndex = slot
let blk = bt.Block.new(@[1.byte]).get
await onBatch(@[blk])
let sold = newFuture[void]()
sales.onSale = proc(request: StorageRequest, slotIndex: UInt256) =
sales.onSale = proc(request: StorageRequest, slotIndex: uint64) =
sold.complete()
createAvailability()
@ -374,7 +379,7 @@ asyncchecksuite "Sales":
# complete request
market.slotState[request.slotId(slotIndex)] = SlotState.Finished
clock.advance(request.ask.duration.truncate(int64))
clock.advance(request.ask.duration.int64)
check eventually getAvailability().freeSize == origSize - 1
@ -406,17 +411,17 @@ asyncchecksuite "Sales":
test "ignores request when slot state is not free":
createAvailability()
await market.requestStorage(request)
market.slotState[request.slotId(0.u256)] = SlotState.Filled
market.slotState[request.slotId(1.u256)] = SlotState.Filled
market.slotState[request.slotId(2.u256)] = SlotState.Filled
market.slotState[request.slotId(3.u256)] = SlotState.Filled
market.slotState[request.slotId(0.uint64)] = SlotState.Filled
market.slotState[request.slotId(1.uint64)] = SlotState.Filled
market.slotState[request.slotId(2.uint64)] = SlotState.Filled
market.slotState[request.slotId(3.uint64)] = SlotState.Filled
check wasIgnored()
test "retrieves and stores data locally":
var storingRequest: StorageRequest
var storingSlot: UInt256
var storingSlot: uint64
sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
request: StorageRequest, slot: uint64, onBatch: BatchProc
): Future[?!void] {.async.} =
storingRequest = request
storingSlot = slot
@ -424,29 +429,12 @@ asyncchecksuite "Sales":
createAvailability()
await market.requestStorage(request)
check eventually storingRequest == request
check storingSlot < request.ask.slots.u256
test "handles errors during state run":
var saleFailed = false
sales.onProve = proc(
slot: Slot, challenge: ProofChallenge
): Future[?!Groth16Proof] {.async.} =
# raise exception so machine.onError is called
raise newException(ValueError, "some error")
# onClear is called in SaleErrored.run
sales.onClear = proc(request: StorageRequest, idx: UInt256) =
saleFailed = true
createAvailability()
await market.requestStorage(request)
await allowRequestToStart()
check eventually saleFailed
check storingSlot < request.ask.slots
test "makes storage available again when data retrieval fails":
let error = newException(IOError, "data retrieval failed")
sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
request: StorageRequest, slot: uint64, onBatch: BatchProc
): Future[?!void] {.async.} =
return failure(error)
createAvailability()
@ -455,7 +443,7 @@ asyncchecksuite "Sales":
test "generates proof of storage":
var provingRequest: StorageRequest
var provingSlot: UInt256
var provingSlot: uint64
sales.onProve = proc(
slot: Slot, challenge: ProofChallenge
): Future[?!Groth16Proof] {.async.} =
@ -467,7 +455,7 @@ asyncchecksuite "Sales":
await allowRequestToStart()
check eventually provingRequest == request
check provingSlot < request.ask.slots.u256
check provingSlot < request.ask.slots
test "fills a slot":
createAvailability()
@ -476,14 +464,14 @@ asyncchecksuite "Sales":
check eventually market.filled.len > 0
check market.filled[0].requestId == request.id
check market.filled[0].slotIndex < request.ask.slots.u256
check market.filled[0].slotIndex < request.ask.slots
check market.filled[0].proof == proof
check market.filled[0].host == await market.getSigner()
test "calls onFilled when slot is filled":
var soldRequest = StorageRequest.default
var soldSlotIndex = UInt256.high
sales.onSale = proc(request: StorageRequest, slotIndex: UInt256) =
var soldSlotIndex = uint64.high
sales.onSale = proc(request: StorageRequest, slotIndex: uint64) =
soldRequest = request
soldSlotIndex = slotIndex
createAvailability()
@ -491,7 +479,7 @@ asyncchecksuite "Sales":
await allowRequestToStart()
check eventually soldRequest == request
check soldSlotIndex < request.ask.slots.u256
check soldSlotIndex < request.ask.slots
test "calls onClear when storage becomes available again":
# fail the proof intentionally to trigger `agent.finish(success=false)`,
@ -501,8 +489,8 @@ asyncchecksuite "Sales":
): Future[?!Groth16Proof] {.async.} =
raise newException(IOError, "proof failed")
var clearedRequest: StorageRequest
var clearedSlotIndex: UInt256
sales.onClear = proc(request: StorageRequest, slotIndex: UInt256) =
var clearedSlotIndex: uint64
sales.onClear = proc(request: StorageRequest, slotIndex: uint64) =
clearedRequest = request
clearedSlotIndex = slotIndex
createAvailability()
@ -510,19 +498,19 @@ asyncchecksuite "Sales":
await allowRequestToStart()
check eventually clearedRequest == request
check clearedSlotIndex < request.ask.slots.u256
check clearedSlotIndex < request.ask.slots
test "makes storage available again when other host fills the slot":
let otherHost = Address.example
sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
request: StorageRequest, slot: uint64, onBatch: BatchProc
): Future[?!void] {.async.} =
await sleepAsync(chronos.hours(1))
return success()
createAvailability()
await market.requestStorage(request)
for slotIndex in 0 ..< request.ask.slots:
market.fillSlot(request.id, slotIndex.u256, proof, otherHost)
market.fillSlot(request.id, slotIndex.uint64, proof, otherHost)
check eventually (await reservations.all(Availability)).get == @[availability]
test "makes storage available again when request expires":
@ -531,7 +519,7 @@ asyncchecksuite "Sales":
let origSize = availability.freeSize
sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
request: StorageRequest, slot: uint64, onBatch: BatchProc
): Future[?!void] {.async.} =
await sleepAsync(chronos.hours(1))
return success()
@ -551,12 +539,12 @@ asyncchecksuite "Sales":
# ensure only one slot, otherwise once bytes are returned to the
# availability, the queue will be unpaused and availability will be consumed
# by other slots
request.ask.slots = 1.uint64
request.ask.slots = 1
market.requestExpiry[request.id] = expiry
let origSize = availability.freeSize
sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
request: StorageRequest, slot: uint64, onBatch: BatchProc
): Future[?!void] {.async.} =
await sleepAsync(chronos.hours(1))
return success()
@ -583,21 +571,19 @@ asyncchecksuite "Sales":
market.requestState[request.id] = RequestState.New
market.requestEnds[request.id] = request.expiry.toSecondsSince1970
proc fillSlot(slotIdx: UInt256 = 0.u256) {.async.} =
proc fillSlot(slotIdx: uint64 = 0) {.async.} =
let address = await market.getSigner()
let slot =
MockSlot(requestId: request.id, slotIndex: slotIdx, proof: proof, host: address)
market.filled.add slot
market.slotState[slotId(request.id, slotIdx)] = SlotState.Filled
let slot0 =
MockSlot(requestId: request.id, slotIndex: 0.u256, proof: proof, host: me)
let slot0 = MockSlot(requestId: request.id, slotIndex: 0, proof: proof, host: me)
await fillSlot(slot0.slotIndex)
let slot1 =
MockSlot(requestId: request.id, slotIndex: 1.u256, proof: proof, host: me)
let slot1 = MockSlot(requestId: request.id, slotIndex: 1, proof: proof, host: me)
await fillSlot(slot1.slotIndex)
market.activeSlots[me] = @[request.slotId(0.u256), request.slotId(1.u256)]
market.activeSlots[me] = @[request.slotId(0), request.slotId(1)]
market.requested = @[request]
market.activeRequests[me] = @[request.id]
@ -605,16 +591,16 @@ asyncchecksuite "Sales":
check eventually sales.agents.len == 2
check sales.agents.any(
agent => agent.data.requestId == request.id and agent.data.slotIndex == 0.u256
agent => agent.data.requestId == request.id and agent.data.slotIndex == 0.uint64
)
check sales.agents.any(
agent => agent.data.requestId == request.id and agent.data.slotIndex == 1.u256
agent => agent.data.requestId == request.id and agent.data.slotIndex == 1.uint64
)
test "deletes inactive reservations on load":
createAvailability()
discard await reservations.createReservation(
availability.id, 100.u256, RequestId.example, UInt256.example, UInt256.example
availability.id, 100.uint64, RequestId.example, 0.uint64, UInt256.example
)
check (await reservations.all(Reservation)).get.len == 1
await sales.load()

32
tests/codex/sales/testsalesagent.nim
View File

@ -4,7 +4,6 @@ import pkg/codex/sales
import pkg/codex/sales/salesagent
import pkg/codex/sales/salescontext
import pkg/codex/sales/statemachine
import pkg/codex/sales/states/errorhandling
import ../../asynctest
import ../helpers/mockmarket
@ -15,18 +14,12 @@ import ../examples
var onCancelCalled = false
var onFailedCalled = false
var onSlotFilledCalled = false
var onErrorCalled = false
type
MockState = ref object of SaleState
MockErrorState = ref object of ErrorHandlingState
type MockState = ref object of SaleState
method `$`*(state: MockState): string =
"MockState"
method `$`*(state: MockErrorState): string =
"MockErrorState"
method onCancelled*(state: MockState, request: StorageRequest): ?State =
onCancelCalled = true
@ -34,31 +27,24 @@ method onFailed*(state: MockState, request: StorageRequest): ?State =
onFailedCalled = true
method onSlotFilled*(
state: MockState, requestId: RequestId, slotIndex: UInt256
state: MockState, requestId: RequestId, slotIndex: uint64
): ?State =
onSlotFilledCalled = true
method onError*(state: MockErrorState, err: ref CatchableError): ?State =
onErrorCalled = true
method run*(state: MockErrorState, machine: Machine): Future[?State] {.async.} =
raise newException(ValueError, "failure")
asyncchecksuite "Sales agent":
let request = StorageRequest.example
var agent: SalesAgent
var context: SalesContext
var slotIndex: UInt256
var slotIndex: uint64
var market: MockMarket
var clock: MockClock
setup:
market = MockMarket.new()
market.requestExpiry[request.id] =
getTime().toUnix() + request.expiry.truncate(int64)
market.requestExpiry[request.id] = getTime().toUnix() + request.expiry.int64
clock = MockClock.new()
context = SalesContext(market: market, clock: clock)
slotIndex = 0.u256
slotIndex = 0.uint64
onCancelCalled = false
onFailedCalled = false
onSlotFilledCalled = false
@ -123,7 +109,9 @@ asyncchecksuite "Sales agent":
agent.start(MockState.new())
await agent.subscribe()
agent.onFulfilled(request.id)
check eventually agent.data.cancelled.cancelled()
# Note: futures that are cancelled, and do not re-raise the CancelledError
# will have a state of completed, not cancelled.
check eventually agent.data.cancelled.completed()
test "current state onFailed called when onFailed called":
agent.start(MockState.new())
@ -134,7 +122,3 @@ asyncchecksuite "Sales agent":
agent.start(MockState.new())
agent.onSlotFilled(request.id, slotIndex)
check eventually onSlotFilledCalled
test "ErrorHandlingState.onError can be overridden at the state level":
agent.start(MockErrorState.new())
check eventually onErrorCalled

76
tests/codex/sales/testslotqueue.nim
View File

@ -146,18 +146,18 @@ suite "Slot queue":
test "correctly compares SlotQueueItems":
var requestA = StorageRequest.example
requestA.ask.duration = 1.u256
requestA.ask.duration = 1.uint64
requestA.ask.pricePerBytePerSecond = 1.u256
check requestA.ask.pricePerSlot == 1.u256 * requestA.ask.slotSize
check requestA.ask.pricePerSlot == 1.u256 * requestA.ask.slotSize.u256
requestA.ask.collateralPerByte = 100000.u256
requestA.expiry = 1001.u256
requestA.expiry = 1001.uint64
var requestB = StorageRequest.example
requestB.ask.duration = 100.u256
requestB.ask.duration = 100.uint64
requestB.ask.pricePerBytePerSecond = 1000.u256
check requestB.ask.pricePerSlot == 100000.u256 * requestB.ask.slotSize
check requestB.ask.pricePerSlot == 100000.u256 * requestB.ask.slotSize.u256
requestB.ask.collateralPerByte = 1.u256
requestB.expiry = 1000.u256
requestB.expiry = 1000.uint64
let itemA = SlotQueueItem.init(requestA, 0)
let itemB = SlotQueueItem.init(requestB, 0)
@ -169,21 +169,21 @@ suite "Slot queue":
let itemA = MockSlotQueueItem(
requestId: request.id,
slotIndex: 0,
slotSize: 1.u256,
duration: 1.u256,
slotSize: 1.uint64,
duration: 1.uint64,
pricePerBytePerSecond: 2.u256, # profitability is higher (good)
collateralPerByte: 1.u256,
expiry: 1.u256,
expiry: 1.uint64,
seen: true, # seen (bad), more weight than profitability
)
let itemB = MockSlotQueueItem(
requestId: request.id,
slotIndex: 0,
slotSize: 1.u256,
duration: 1.u256,
slotSize: 1.uint64,
duration: 1.uint64,
pricePerBytePerSecond: 1.u256, # profitability is lower (bad)
collateralPerByte: 1.u256,
expiry: 1.u256,
expiry: 1.uint64,
seen: false, # not seen (good)
)
check itemB.toSlotQueueItem < itemA.toSlotQueueItem # B higher priority than A
@ -194,22 +194,22 @@ suite "Slot queue":
let itemA = MockSlotQueueItem(
requestId: request.id,
slotIndex: 0,
slotSize: 1.u256,
duration: 1.u256,
slotSize: 1.uint64,
duration: 1.uint64,
pricePerBytePerSecond: 1.u256, # reward is lower (bad)
collateralPerByte: 1.u256, # collateral is lower (good)
expiry: 1.u256,
expiry: 1.uint64,
seen: false,
)
let itemB = MockSlotQueueItem(
requestId: request.id,
slotIndex: 0,
slotSize: 1.u256,
duration: 1.u256,
slotSize: 1.uint64,
duration: 1.uint64,
pricePerBytePerSecond: 2.u256,
# reward is higher (good), more weight than collateral
collateralPerByte: 2.u256, # collateral is higher (bad)
expiry: 1.u256,
expiry: 1.uint64,
seen: false,
)
@ -220,21 +220,21 @@ suite "Slot queue":
let itemA = MockSlotQueueItem(
requestId: request.id,
slotIndex: 0,
slotSize: 1.u256,
duration: 1.u256,
slotSize: 1.uint64,
duration: 1.uint64,
pricePerBytePerSecond: 1.u256,
collateralPerByte: 2.u256, # collateral is higher (bad)
expiry: 2.u256, # expiry is longer (good)
expiry: 2.uint64, # expiry is longer (good)
seen: false,
)
let itemB = MockSlotQueueItem(
requestId: request.id,
slotIndex: 0,
slotSize: 1.u256,
duration: 1.u256,
slotSize: 1.uint64,
duration: 1.uint64,
pricePerBytePerSecond: 1.u256,
collateralPerByte: 1.u256, # collateral is lower (good), more weight than expiry
expiry: 1.u256, # expiry is shorter (bad)
expiry: 1.uint64, # expiry is shorter (bad)
seen: false,
)
@ -245,21 +245,21 @@ suite "Slot queue":
let itemA = MockSlotQueueItem(
requestId: request.id,
slotIndex: 0,
slotSize: 1.u256, # slotSize is smaller (good)
duration: 1.u256,
slotSize: 1.uint64, # slotSize is smaller (good)
duration: 1.uint64,
pricePerBytePerSecond: 1.u256,
collateralPerByte: 1.u256,
expiry: 1.u256, # expiry is shorter (bad)
expiry: 1.uint64, # expiry is shorter (bad)
seen: false,
)
let itemB = MockSlotQueueItem(
requestId: request.id,
slotIndex: 0,
slotSize: 2.u256, # slotSize is larger (bad)
duration: 1.u256,
slotSize: 2.uint64, # slotSize is larger (bad)
duration: 1.uint64,
pricePerBytePerSecond: 1.u256,
collateralPerByte: 1.u256,
expiry: 2.u256, # expiry is longer (good), more weight than slotSize
expiry: 2.uint64, # expiry is longer (good), more weight than slotSize
seen: false,
)
@ -270,21 +270,21 @@ suite "Slot queue":
let itemA = MockSlotQueueItem(
requestId: request.id,
slotIndex: 0,
slotSize: 2.u256, # slotSize is larger (bad)
duration: 1.u256,
slotSize: 2.uint64, # slotSize is larger (bad)
duration: 1.uint64,
pricePerBytePerSecond: 1.u256,
collateralPerByte: 1.u256,
expiry: 1.u256, # expiry is shorter (bad)
expiry: 1.uint64, # expiry is shorter (bad)
seen: false,
)
let itemB = MockSlotQueueItem(
requestId: request.id,
slotIndex: 0,
slotSize: 1.u256, # slotSize is smaller (good)
duration: 1.u256,
slotSize: 1.uint64, # slotSize is smaller (good)
duration: 1.uint64,
pricePerBytePerSecond: 1.u256,
collateralPerByte: 1.u256,
expiry: 1.u256,
expiry: 1.uint64,
seen: false,
)
@ -460,14 +460,14 @@ suite "Slot queue":
test "sorts items by expiry descending (longer expiry = higher priority)":
var request = StorageRequest.example
let item0 = SlotQueueItem.init(request, 0)
request.expiry += 1.u256
request.expiry += 1
let item1 = SlotQueueItem.init(request, 1)
check item1 < item0
test "sorts items by slot size descending (bigger dataset = higher profitability = higher priority)":
var request = StorageRequest.example
let item0 = SlotQueueItem.init(request, 0)
request.ask.slotSize += 1.u256
request.ask.slotSize += 1
let item1 = SlotQueueItem.init(request, 1)
check item1 < item0

13
tests/codex/slots/helpers.nim
View File

@ -15,9 +15,7 @@ import pkg/codex/rng
import ../helpers
proc storeManifest*(
store: BlockStore, manifest: Manifest
): Future[?!bt.Block] {.async.} =
proc makeManifestBlock*(manifest: Manifest): ?!bt.Block =
without encodedVerifiable =? manifest.encode(), err:
trace "Unable to encode manifest"
return failure(err)
@ -26,6 +24,15 @@ proc storeManifest*(
trace "Unable to create block from manifest"
return failure(error)
success blk
proc storeManifest*(
store: BlockStore, manifest: Manifest
): Future[?!bt.Block] {.async.} =
without blk =? makeManifestBlock(manifest), err:
trace "Unable to create manifest block", err = err.msg
return failure(err)
if err =? (await store.putBlock(blk)).errorOption:
trace "Unable to store manifest block", cid = blk.cid, err = err.msg
return failure(err)

2
tests/codex/stores/testqueryiterhelper.nim
View File

@ -1,6 +1,6 @@
import std/sugar
import pkg/stew/results
import pkg/results
import pkg/questionable
import pkg/chronos
import pkg/datastore/typedds

Some files were not shown because too many files have changed in this diff Show More