diff --git a/codex.nim b/codex.nim index e2c6033e..7749bdee 100644 --- a/codex.nim +++ b/codex.nim @@ -38,33 +38,35 @@ when isMainModule: when defined(posix): import system/ansi_c - type - CodexStatus {.pure.} = enum - Stopped, - Stopping, - Running + type CodexStatus {.pure.} = enum + Stopped + Stopping + Running let config = CodexConf.load( version = codexFullVersion, envVarsPrefix = "codex", - secondarySources = proc (config: CodexConf, sources: auto) {.gcsafe, raises: [ConfigurationError].} = - if configFile =? config.configFile: - sources.addConfigFile(Toml, configFile) + secondarySources = proc( + config: CodexConf, sources: auto + ) {.gcsafe, raises: [ConfigurationError].} = + if configFile =? config.configFile: + sources.addConfigFile(Toml, configFile) + , ) config.setupLogging() config.setupMetrics() - if not(checkAndCreateDataDir((config.dataDir).string)): + if not (checkAndCreateDataDir((config.dataDir).string)): # We are unable to access/create data folder or data folder's # permissions are insecure. quit QuitFailure - if config.prover() and not(checkAndCreateDataDir((config.circuitDir).string)): + if config.prover() and not (checkAndCreateDataDir((config.circuitDir).string)): quit QuitFailure trace "Data dir initialized", dir = $config.dataDir - if not(checkAndCreateDataDir((config.dataDir / "repo"))): + if not (checkAndCreateDataDir((config.dataDir / "repo"))): # We are unable to access/create data folder or data folder's # permissions are insecure. quit QuitFailure @@ -83,11 +85,12 @@ when isMainModule: config.dataDir / config.netPrivKeyFile privateKey = setupKey(keyPath).expect("Should setup private key!") - server = try: - CodexServer.new(config, privateKey) - except Exception as exc: - error "Failed to start Codex", msg = exc.msg - quit QuitFailure + server = + try: + CodexServer.new(config, privateKey) + except Exception as exc: + error "Failed to start Codex", msg = exc.msg + quit QuitFailure ## Ctrl+C handling proc doShutdown() = @@ -101,7 +104,9 @@ when isMainModule: # workaround for https://github.com/nim-lang/Nim/issues/4057 try: setupForeignThreadGc() - except Exception as exc: raiseAssert exc.msg # shouldn't happen + except Exception as exc: + raiseAssert exc.msg + # shouldn't happen notice "Shutting down after having received SIGINT" doShutdown() diff --git a/codex/contracts/config.nim b/codex/contracts/config.nim index 5493c643..986b1944 100644 --- a/codex/contracts/config.nim +++ b/codex/contracts/config.nim @@ -10,13 +10,16 @@ type MarketplaceConfig* = object collateral*: CollateralConfig proofs*: ProofConfig + reservations*: SlotReservationsConfig + requestDurationLimit*: UInt256 CollateralConfig* = object repairRewardPercentage*: uint8 # percentage of remaining collateral slot has after it has been freed maxNumberOfSlashes*: uint8 # frees slot when the number of slashes reaches this value - slashCriterion*: uint16 # amount of proofs missed that lead to slashing slashPercentage*: uint8 # percentage of the collateral that is slashed + validatorRewardPercentage*: uint8 + # percentage of the slashed amount going to the validators ProofConfig* = object period*: UInt256 # proofs requirements are calculated per period (in seconds) @@ -28,6 +31,9 @@ type # blocks. Should be a prime number to ensure there are no cycles. downtimeProduct*: uint8 + SlotReservationsConfig* = object + maxReservations*: uint8 + func fromTuple(_: type ProofConfig, tupl: tuple): ProofConfig = ProofConfig( period: tupl[0], @@ -37,16 +43,27 @@ func fromTuple(_: type ProofConfig, tupl: tuple): ProofConfig = downtimeProduct: tupl[4], ) +func fromTuple(_: type SlotReservationsConfig, tupl: tuple): SlotReservationsConfig = + SlotReservationsConfig(maxReservations: tupl[0]) + func fromTuple(_: type CollateralConfig, tupl: tuple): CollateralConfig = CollateralConfig( repairRewardPercentage: tupl[0], maxNumberOfSlashes: tupl[1], - slashCriterion: tupl[2], - slashPercentage: tupl[3], + slashPercentage: tupl[2], + validatorRewardPercentage: tupl[3], ) func fromTuple(_: type MarketplaceConfig, tupl: tuple): MarketplaceConfig = - MarketplaceConfig(collateral: tupl[0], proofs: tupl[1]) + MarketplaceConfig( + collateral: tupl[0], + proofs: tupl[1], + reservations: tupl[2], + requestDurationLimit: tupl[3], + ) + +func solidityType*(_: type SlotReservationsConfig): string = + solidityType(SlotReservationsConfig.fieldTypes) func solidityType*(_: type ProofConfig): string = solidityType(ProofConfig.fieldTypes) @@ -55,7 +72,10 @@ func solidityType*(_: type CollateralConfig): string = solidityType(CollateralConfig.fieldTypes) func solidityType*(_: type MarketplaceConfig): string = - solidityType(CollateralConfig.fieldTypes) + solidityType(MarketplaceConfig.fieldTypes) + +func encode*(encoder: var AbiEncoder, slot: SlotReservationsConfig) = + encoder.write(slot.fieldValues) func encode*(encoder: var AbiEncoder, slot: ProofConfig) = encoder.write(slot.fieldValues) @@ -70,6 +90,10 @@ func decode*(decoder: var AbiDecoder, T: type ProofConfig): ?!T = let tupl = ?decoder.read(ProofConfig.fieldTypes) success ProofConfig.fromTuple(tupl) +func decode*(decoder: var AbiDecoder, T: type SlotReservationsConfig): ?!T = + let tupl = ?decoder.read(SlotReservationsConfig.fieldTypes) + success SlotReservationsConfig.fromTuple(tupl) + func decode*(decoder: var AbiDecoder, T: type CollateralConfig): ?!T = let tupl = ?decoder.read(CollateralConfig.fieldTypes) success CollateralConfig.fromTuple(tupl) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 9157b269..208dbe07 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -91,9 +91,14 @@ method proofTimeout*(market: OnChainMarket): Future[UInt256] {.async.} = method repairRewardPercentage*(market: OnChainMarket): Future[uint8] {.async.} = convertEthersError: - let config = await market.contract.configuration() + let config = await market.config() return config.collateral.repairRewardPercentage +method requestDurationLimit*(market: OnChainMarket): Future[UInt256] {.async.} = + convertEthersError: + let config = await market.config() + return config.requestDurationLimit + method proofDowntime*(market: OnChainMarket): Future[uint8] {.async.} = convertEthersError: let config = await market.config() diff --git a/codex/contracts/marketplace.nim b/codex/contracts/marketplace.nim index 87fd1e47..091f45db 100644 --- a/codex/contracts/marketplace.nim +++ b/codex/contracts/marketplace.nim @@ -42,6 +42,7 @@ type Marketplace_InsufficientCollateral* = object of SolidityError Marketplace_InsufficientReward* = object of SolidityError Marketplace_InvalidCid* = object of SolidityError + Marketplace_DurationExceedsLimit* = object of SolidityError Proofs_InsufficientBlockHeight* = object of SolidityError Proofs_InvalidProof* = object of SolidityError Proofs_ProofAlreadySubmitted* = object of SolidityError diff --git a/codex/market.nim b/codex/market.nim index bc325cd9..66f31804 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -78,6 +78,9 @@ method proofTimeout*(market: Market): Future[UInt256] {.base, async.} = method repairRewardPercentage*(market: Market): Future[uint8] {.base, async.} = raiseAssert("not implemented") +method requestDurationLimit*(market: Market): Future[UInt256] {.base, async.} = + raiseAssert("not implemented") + method proofDowntime*(market: Market): Future[uint8] {.base, async.} = raiseAssert("not implemented") diff --git a/codex/purchasing.nim b/codex/purchasing.nim index 4ab84405..25a35137 100644 --- a/codex/purchasing.nim +++ b/codex/purchasing.nim @@ -14,7 +14,7 @@ export purchase type Purchasing* = ref object - market: Market + market*: Market clock: Clock purchases: Table[PurchaseId, Purchase] proofProbability*: UInt256 diff --git a/codex/rest/api.nim b/codex/rest/api.nim index f64a6f20..8ba1abae 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -637,6 +637,14 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = without params =? StorageRequestParams.fromJson(body), error: return RestApiResponse.error(Http400, error.msg, headers = headers) + let requestDurationLimit = await contracts.purchasing.market.requestDurationLimit + if params.duration > requestDurationLimit: + return RestApiResponse.error( + Http400, + "Duration exceeds limit of " & $requestDurationLimit & " seconds", + headers = headers, + ) + let nodes = params.nodes |? 3 let tolerance = params.tolerance |? 1 diff --git a/codex/validation.nim b/codex/validation.nim index 6e3135e4..6659bc5b 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -22,8 +22,6 @@ type Validation* = ref object proofTimeout: UInt256 config: ValidationConfig -const MaxStorageRequestDuration = 30.days - logScope: topics = "codex validator" @@ -122,7 +120,10 @@ proc epochForDurationBackFromNow( proc restoreHistoricalState(validation: Validation) {.async.} = trace "Restoring historical state..." - let startTimeEpoch = validation.epochForDurationBackFromNow(MaxStorageRequestDuration) + let requestDurationLimit = await validation.market.requestDurationLimit + let startTimeEpoch = validation.epochForDurationBackFromNow( + seconds(requestDurationLimit.truncate(int64)) + ) let slotFilledEvents = await validation.market.queryPastSlotFilledEvents(fromTime = startTimeEpoch) for event in slotFilledEvents: diff --git a/config.nims b/config.nims index 6a4767ad..05a31fff 100644 --- a/config.nims +++ b/config.nims @@ -1,21 +1,24 @@ - include "build.nims" import std/os const currentDir = currentSourcePath()[0 .. ^(len("config.nims") + 1)] when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and - # BEWARE - # In Nim 1.6, config files are evaluated with a working directory - # matching where the Nim command was invocated. This means that we - # must do all file existence checks with full absolute paths: - system.fileExists(currentDir & "nimbus-build-system.paths"): + # BEWARE + # In Nim 1.6, config files are evaluated with a working directory + # matching where the Nim command was invocated. This means that we + # must do all file existence checks with full absolute paths: + system.fileExists(currentDir & "nimbus-build-system.paths"): include "nimbus-build-system.paths" when defined(release): - switch("nimcache", joinPath(currentSourcePath.parentDir, "nimcache/release/$projectName")) + switch( + "nimcache", joinPath(currentSourcePath.parentDir, "nimcache/release/$projectName") + ) else: - switch("nimcache", joinPath(currentSourcePath.parentDir, "nimcache/debug/$projectName")) + switch( + "nimcache", joinPath(currentSourcePath.parentDir, "nimcache/debug/$projectName") + ) when defined(limitStackUsage): # This limits stack usage of each individual function to 1MB - the option is @@ -34,7 +37,8 @@ when defined(windows): # increase stack size switch("passL", "-Wl,--stack,8388608") # https://github.com/nim-lang/Nim/issues/4057 - --tlsEmulation:off + --tlsEmulation: + off if defined(i386): # set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM switch("passL", "-Wl,--large-address-aware") @@ -63,30 +67,47 @@ else: # ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes) switch("passC", "-mno-avx512vl") ---tlsEmulation:off ---threads:on ---opt:speed ---excessiveStackTrace:on +--tlsEmulation: + off +--threads: + on +--opt: + speed +--excessiveStackTrace: + on # enable metric collection ---define:metrics +--define: + metrics # for heap-usage-by-instance-type metrics and object base-type strings ---define:nimTypeNames ---styleCheck:usages ---styleCheck:error ---maxLoopIterationsVM:1000000000 ---fieldChecks:on ---warningAsError:"ProveField:on" +--define: + nimTypeNames +--styleCheck: + usages +--styleCheck: + error +--maxLoopIterationsVM: + 1000000000 +--fieldChecks: + on +--warningAsError: + "ProveField:on" when (NimMajor, NimMinor) >= (1, 4): - --warning:"ObservableStores:off" - --warning:"LockLevel:off" - --hint:"XCannotRaiseY:off" + --warning: + "ObservableStores:off" + --warning: + "LockLevel:off" + --hint: + "XCannotRaiseY:off" when (NimMajor, NimMinor) >= (1, 6): - --warning:"DotLikeOps:off" + --warning: + "DotLikeOps:off" when (NimMajor, NimMinor, NimPatch) >= (1, 6, 11): - --warning:"BareExcept:off" + --warning: + "BareExcept:off" when (NimMajor, NimMinor) >= (2, 0): - --mm:refc + --mm: + refc switch("define", "withoutPCRE") @@ -94,10 +115,12 @@ switch("define", "withoutPCRE") # "--debugger:native" build. It can be increased with `ulimit -n 1024`. if not defined(macosx): # add debugging symbols and original files and line numbers - --debugger:native + --debugger: + native if not (defined(windows) and defined(i386)) and not defined(disable_libbacktrace): # light-weight stack traces using libbacktrace and libunwind - --define:nimStackTraceOverride + --define: + nimStackTraceOverride switch("import", "libbacktrace") # `switch("warning[CaseTransition]", "off")` fails with "Error: invalid command line option: '--warning[CaseTransition]'" diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index bb0eaaa2..3638d11e 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -122,12 +122,14 @@ proc new*(_: type MockMarket, clock: ?Clock = Clock.none): MockMarket = collateral: CollateralConfig( repairRewardPercentage: 10, maxNumberOfSlashes: 5, - slashCriterion: 3, slashPercentage: 10, + validatorRewardPercentage: 20, ), proofs: ProofConfig( period: 10.u256, timeout: 5.u256, downtime: 64.uint8, downtimeProduct: 67.uint8 ), + reservations: SlotReservationsConfig(maxReservations: 3), + requestDurationLimit: (60 * 60 * 24 * 30).u256, ) MockMarket( signer: Address.example, config: config, canReserveSlot: true, clock: clock @@ -142,6 +144,9 @@ method periodicity*(mock: MockMarket): Future[Periodicity] {.async.} = method proofTimeout*(market: MockMarket): Future[UInt256] {.async.} = return market.config.proofs.timeout +method requestDurationLimit*(market: MockMarket): Future[UInt256] {.async.} = + return market.config.requestDurationLimit + method proofDowntime*(market: MockMarket): Future[uint8] {.async.} = return market.config.proofs.downtime diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index 13b06500..3918791e 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -103,6 +103,26 @@ twonodessuite "REST API": check responseBefore.status == "400 Bad Request" check responseBefore.body == "Tolerance needs to be bigger then zero" + test "request storage fails if duration exceeds limit", twoNodesConfig: + let data = await RandomChunker.example(blocks = 2) + let cid = client1.upload(data).get + let duration = (31 * 24 * 60 * 60).u256 + # 31 days TODO: this should not be hardcoded, but waits for https://github.com/codex-storage/nim-codex/issues/1056 + let proofProbability = 3.u256 + let expiry = 30.uint + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 2 + let pricePerBytePerSecond = 1.u256 + + var responseBefore = client1.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes.uint, tolerance.uint, + ) + + check responseBefore.status == "400 Bad Request" + check "Duration exceeds limit of" in responseBefore.body + test "request storage fails if nodes and tolerance aren't correct", twoNodesConfig: let data = await RandomChunker.example(blocks = 2) let cid = client1.upload(data).get diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth index 0f2012b1..ff82c26b 160000 --- a/vendor/codex-contracts-eth +++ b/vendor/codex-contracts-eth @@ -1 +1 @@ -Subproject commit 0f2012b1442c404605c8ba9dcae2f4e53058cd2c +Subproject commit ff82c26b3669b52a09280c634141dace7f04659a