2023-04-19 13:06:00 +00:00
|
|
|
import std/sets
|
|
|
|
import std/sequtils
|
|
|
|
import pkg/chronos
|
|
|
|
import ./market
|
|
|
|
import ./clock
|
feat: create logging proxy (#663)
* implement a logging proxy
The logging proxy:
- prevents the need to import chronicles (as well as export except toJson),
- prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization,
- and handles json formatting correctly in chronicles json sinks
* Rename logging -> logutils to avoid ambiguity with common names
* clean up
* add setProperty for JsonRecord, remove nim-json-serialization conflict
* Allow specifying textlines and json format separately
Not specifying a LogFormat will apply the formatting to both textlines and json sinks.
Specifying a LogFormat will apply the formatting to only that sink.
* remove unneeded usages of std/json
We only need to import utils/json instead of std/json
* move serialization from rest/json to utils/json so it can be shared
* fix NoColors ambiguity
Was causing unit tests to fail on Windows.
* Remove nre usage to fix Windows error
Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :)
* Add logutils module doc
* Shorten logutils.formatIt for `NBytes`
Both json and textlines formatIt were not needed, and could be combined into one formatIt
* remove debug integration test config
debug output and logformat of json for integration test logs
* Use ## module doc to support docgen
* bump nim-poseidon2 to export fromBytes
Before the changes in this branch, fromBytes was likely being resolved by nim-stew, or other dependency. With the changes in this branch, that dependency was removed and fromBytes could no longer be resolved. By exporting fromBytes from nim-poseidon, the correct resolution is now happening.
* fixes to get compiling after rebasing master
* Add support for Result types being logged using formatIt
2024-01-23 07:35:03 +00:00
|
|
|
import ./logutils
|
2023-04-19 13:06:00 +00:00
|
|
|
|
|
|
|
export market
|
|
|
|
export sets
|
|
|
|
|
|
|
|
type
|
|
|
|
Validation* = ref object
|
|
|
|
slots: HashSet[SlotId]
|
|
|
|
maxSlots: int
|
|
|
|
clock: Clock
|
|
|
|
market: Market
|
|
|
|
subscriptions: seq[Subscription]
|
|
|
|
running: Future[void]
|
|
|
|
periodicity: Periodicity
|
|
|
|
proofTimeout: UInt256
|
|
|
|
|
|
|
|
logScope:
|
|
|
|
topics = "codex validator"
|
|
|
|
|
2023-06-22 15:11:18 +00:00
|
|
|
proc new*(
|
|
|
|
_: type Validation,
|
|
|
|
clock: Clock,
|
|
|
|
market: Market,
|
|
|
|
maxSlots: int
|
|
|
|
): Validation =
|
|
|
|
## Create a new Validation instance
|
2023-04-19 13:06:00 +00:00
|
|
|
Validation(clock: clock, market: market, maxSlots: maxSlots)
|
|
|
|
|
|
|
|
proc slots*(validation: Validation): seq[SlotId] =
|
|
|
|
validation.slots.toSeq
|
|
|
|
|
2024-05-06 08:12:49 +00:00
|
|
|
proc iterateSlots(validation: Validation, action: proc(s: SlotId): Future[void] {.async.}) {.async.} =
|
|
|
|
# Copy of hashSet, for iteration.
|
|
|
|
let slots = validation.slots
|
|
|
|
for slotId in slots:
|
|
|
|
await action(slotId)
|
|
|
|
|
2024-02-16 22:12:16 +00:00
|
|
|
proc getCurrentPeriod(validation: Validation): UInt256 =
|
|
|
|
return validation.periodicity.periodOf(validation.clock.now().u256)
|
2023-04-19 13:06:00 +00:00
|
|
|
|
|
|
|
proc waitUntilNextPeriod(validation: Validation) {.async.} =
|
2024-02-16 22:12:16 +00:00
|
|
|
let period = validation.getCurrentPeriod()
|
2023-04-19 13:06:00 +00:00
|
|
|
let periodEnd = validation.periodicity.periodEnd(period)
|
|
|
|
trace "Waiting until next period", currentPeriod = period
|
|
|
|
await validation.clock.waitUntil(periodEnd.truncate(int64) + 1)
|
|
|
|
|
|
|
|
proc subscribeSlotFilled(validation: Validation) {.async.} =
|
|
|
|
proc onSlotFilled(requestId: RequestId, slotIndex: UInt256) =
|
|
|
|
let slotId = slotId(requestId, slotIndex)
|
|
|
|
if slotId notin validation.slots:
|
|
|
|
if validation.slots.len < validation.maxSlots:
|
feat: create logging proxy (#663)
* implement a logging proxy
The logging proxy:
- prevents the need to import chronicles (as well as export except toJson),
- prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization,
- and handles json formatting correctly in chronicles json sinks
* Rename logging -> logutils to avoid ambiguity with common names
* clean up
* add setProperty for JsonRecord, remove nim-json-serialization conflict
* Allow specifying textlines and json format separately
Not specifying a LogFormat will apply the formatting to both textlines and json sinks.
Specifying a LogFormat will apply the formatting to only that sink.
* remove unneeded usages of std/json
We only need to import utils/json instead of std/json
* move serialization from rest/json to utils/json so it can be shared
* fix NoColors ambiguity
Was causing unit tests to fail on Windows.
* Remove nre usage to fix Windows error
Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :)
* Add logutils module doc
* Shorten logutils.formatIt for `NBytes`
Both json and textlines formatIt were not needed, and could be combined into one formatIt
* remove debug integration test config
debug output and logformat of json for integration test logs
* Use ## module doc to support docgen
* bump nim-poseidon2 to export fromBytes
Before the changes in this branch, fromBytes was likely being resolved by nim-stew, or other dependency. With the changes in this branch, that dependency was removed and fromBytes could no longer be resolved. By exporting fromBytes from nim-poseidon, the correct resolution is now happening.
* fixes to get compiling after rebasing master
* Add support for Result types being logged using formatIt
2024-01-23 07:35:03 +00:00
|
|
|
trace "Adding slot", slotId
|
2023-04-19 13:06:00 +00:00
|
|
|
validation.slots.incl(slotId)
|
|
|
|
let subscription = await validation.market.subscribeSlotFilled(onSlotFilled)
|
|
|
|
validation.subscriptions.add(subscription)
|
|
|
|
|
|
|
|
proc removeSlotsThatHaveEnded(validation: Validation) {.async.} =
|
|
|
|
var ended: HashSet[SlotId]
|
2024-05-06 08:12:49 +00:00
|
|
|
proc onSlot(slotId: SlotId) {.async.} =
|
2023-04-19 13:06:00 +00:00
|
|
|
let state = await validation.market.slotState(slotId)
|
|
|
|
if state != SlotState.Filled:
|
feat: create logging proxy (#663)
* implement a logging proxy
The logging proxy:
- prevents the need to import chronicles (as well as export except toJson),
- prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization,
- and handles json formatting correctly in chronicles json sinks
* Rename logging -> logutils to avoid ambiguity with common names
* clean up
* add setProperty for JsonRecord, remove nim-json-serialization conflict
* Allow specifying textlines and json format separately
Not specifying a LogFormat will apply the formatting to both textlines and json sinks.
Specifying a LogFormat will apply the formatting to only that sink.
* remove unneeded usages of std/json
We only need to import utils/json instead of std/json
* move serialization from rest/json to utils/json so it can be shared
* fix NoColors ambiguity
Was causing unit tests to fail on Windows.
* Remove nre usage to fix Windows error
Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :)
* Add logutils module doc
* Shorten logutils.formatIt for `NBytes`
Both json and textlines formatIt were not needed, and could be combined into one formatIt
* remove debug integration test config
debug output and logformat of json for integration test logs
* Use ## module doc to support docgen
* bump nim-poseidon2 to export fromBytes
Before the changes in this branch, fromBytes was likely being resolved by nim-stew, or other dependency. With the changes in this branch, that dependency was removed and fromBytes could no longer be resolved. By exporting fromBytes from nim-poseidon, the correct resolution is now happening.
* fixes to get compiling after rebasing master
* Add support for Result types being logged using formatIt
2024-01-23 07:35:03 +00:00
|
|
|
trace "Removing slot", slotId
|
2023-04-19 13:06:00 +00:00
|
|
|
ended.incl(slotId)
|
2024-05-06 08:12:49 +00:00
|
|
|
await validation.iterateSlots(onSlot)
|
2023-04-19 13:06:00 +00:00
|
|
|
validation.slots.excl(ended)
|
|
|
|
|
|
|
|
proc markProofAsMissing(validation: Validation,
|
|
|
|
slotId: SlotId,
|
|
|
|
period: Period) {.async.} =
|
2023-06-22 10:32:18 +00:00
|
|
|
logScope:
|
2024-02-16 22:12:16 +00:00
|
|
|
currentPeriod = validation.getCurrentPeriod()
|
2023-06-22 10:32:18 +00:00
|
|
|
|
2023-04-19 13:06:00 +00:00
|
|
|
try:
|
|
|
|
if await validation.market.canProofBeMarkedAsMissing(slotId, period):
|
feat: create logging proxy (#663)
* implement a logging proxy
The logging proxy:
- prevents the need to import chronicles (as well as export except toJson),
- prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization,
- and handles json formatting correctly in chronicles json sinks
* Rename logging -> logutils to avoid ambiguity with common names
* clean up
* add setProperty for JsonRecord, remove nim-json-serialization conflict
* Allow specifying textlines and json format separately
Not specifying a LogFormat will apply the formatting to both textlines and json sinks.
Specifying a LogFormat will apply the formatting to only that sink.
* remove unneeded usages of std/json
We only need to import utils/json instead of std/json
* move serialization from rest/json to utils/json so it can be shared
* fix NoColors ambiguity
Was causing unit tests to fail on Windows.
* Remove nre usage to fix Windows error
Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :)
* Add logutils module doc
* Shorten logutils.formatIt for `NBytes`
Both json and textlines formatIt were not needed, and could be combined into one formatIt
* remove debug integration test config
debug output and logformat of json for integration test logs
* Use ## module doc to support docgen
* bump nim-poseidon2 to export fromBytes
Before the changes in this branch, fromBytes was likely being resolved by nim-stew, or other dependency. With the changes in this branch, that dependency was removed and fromBytes could no longer be resolved. By exporting fromBytes from nim-poseidon, the correct resolution is now happening.
* fixes to get compiling after rebasing master
* Add support for Result types being logged using formatIt
2024-01-23 07:35:03 +00:00
|
|
|
trace "Marking proof as missing", slotId, periodProofMissed = period
|
2023-04-19 13:06:00 +00:00
|
|
|
await validation.market.markProofAsMissing(slotId, period)
|
2023-12-19 04:29:18 +00:00
|
|
|
else:
|
|
|
|
let inDowntime {.used.} = await validation.market.inDowntime(slotId)
|
|
|
|
trace "Proof not missing", checkedPeriod = period, inDowntime
|
2023-04-19 13:06:00 +00:00
|
|
|
except CancelledError:
|
|
|
|
raise
|
|
|
|
except CatchableError as e:
|
2023-06-22 10:32:18 +00:00
|
|
|
error "Marking proof as missing failed", msg = e.msg
|
2023-04-19 13:06:00 +00:00
|
|
|
|
|
|
|
proc markProofsAsMissing(validation: Validation) {.async.} =
|
2024-05-06 08:12:49 +00:00
|
|
|
proc onSlot(slotId: SlotId) {.async.} =
|
2024-02-16 22:12:16 +00:00
|
|
|
let previousPeriod = validation.getCurrentPeriod() - 1
|
2023-04-19 13:06:00 +00:00
|
|
|
await validation.markProofAsMissing(slotId, previousPeriod)
|
2024-05-06 08:12:49 +00:00
|
|
|
await validation.iterateSlots(onSlot)
|
2023-04-19 13:06:00 +00:00
|
|
|
|
|
|
|
proc run(validation: Validation) {.async.} =
|
|
|
|
trace "Validation started"
|
|
|
|
try:
|
|
|
|
while true:
|
|
|
|
await validation.waitUntilNextPeriod()
|
|
|
|
await validation.removeSlotsThatHaveEnded()
|
|
|
|
await validation.markProofsAsMissing()
|
|
|
|
except CancelledError:
|
|
|
|
trace "Validation stopped"
|
|
|
|
discard
|
|
|
|
except CatchableError as e:
|
|
|
|
error "Validation failed", msg = e.msg
|
|
|
|
|
|
|
|
proc start*(validation: Validation) {.async.} =
|
|
|
|
validation.periodicity = await validation.market.periodicity()
|
|
|
|
validation.proofTimeout = await validation.market.proofTimeout()
|
|
|
|
await validation.subscribeSlotFilled()
|
|
|
|
validation.running = validation.run()
|
|
|
|
|
|
|
|
proc stop*(validation: Validation) {.async.} =
|
|
|
|
await validation.running.cancelAndWait()
|
|
|
|
while validation.subscriptions.len > 0:
|
|
|
|
let subscription = validation.subscriptions.pop()
|
|
|
|
await subscription.unsubscribe()
|