Merge branch 'master' into feat/chronos-v4

* add special repo_version version to the Nim compiler which picks up what's in the repo
This commit is contained in:
gmega 2024-02-06 17:33:00 -03:00
commit f92d069d82
No known key found for this signature in database
GPG Key ID: FFD8DAF00660270F
22 changed files with 193 additions and 124 deletions

79
.github/workflows/ci-reusable.yml vendored Normal file
View File

@ -0,0 +1,79 @@
name: Reusable - CI
on:
workflow_call:
inputs:
matrix:
type: string
cache_nonce:
default: '0'
description: Allows for easily busting actions/cache caches
required: false
type: string
env:
cache_nonce: ${{ inputs.cache_nonce }}
jobs:
build:
strategy:
matrix:
include: ${{ fromJson(inputs.matrix) }}
defaults:
run:
shell: ${{ matrix.shell }} {0}
name: '${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}'
runs-on: ${{ matrix.builder }}
timeout-minutes: 80
steps:
- name: Checkout sources
uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Nimbus Build System
uses: ./.github/actions/nimbus-build-system
with:
os: ${{ matrix.os }}
shell: ${{ matrix.shell }}
nim_version: ${{ matrix.nim_version }}
## Part 1 Tests ##
- name: Unit tests
if: matrix.tests == 'unittest' || matrix.tests == 'all'
run: ./env.sh make -j${ncpu} test
# workaround for https://github.com/NomicFoundation/hardhat/issues/3877
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 18.15
- name: Start Ethereum node with Codex contracts
if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'all'
working-directory: vendor/codex-contracts-eth
env:
MSYS2_PATH_TYPE: inherit
run: |
npm install
npm start &
## Part 2 Tests ##
- name: Contract tests
if: matrix.tests == 'contract' || matrix.tests == 'all'
run: ./env.sh make -j${ncpu} testContracts
## Part 3 Tests ##
- name: Integration tests
if: matrix.tests == 'integration' || matrix.tests == 'all'
run: ./env.sh make -j${ncpu} testIntegration
status:
if: always()
needs: [build]
runs-on: ubuntu-latest
steps:
- if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }}
run: exit 1

View File

@ -1,91 +1,45 @@
name: CI
on:
push:
branches:
- master
pull_request:
workflow_dispatch:
env:
cache_nonce: 0 # Allows for easily busting actions/cache caches
nim_version:
repo_current # This is the version that's currently in the repo
concurrency:
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
cancel-in-progress: true
jobs:
build:
strategy:
matrix:
include:
- os: linux
cpu: amd64
builder: ubuntu-latest
shell: bash --noprofile --norc -e -o pipefail
tests: all
- os: macos
cpu: amd64
builder: macos-latest
shell: bash --noprofile --norc -e -o pipefail
tests: all
- os: windows
cpu: amd64
builder: windows-latest
shell: msys2
tests: unittest
- os: windows
cpu: amd64
builder: windows-latest
shell: msys2
tests: contract
- os: windows
cpu: amd64
builder: windows-latest
shell: msys2
tests: integration
defaults:
run:
shell: ${{ matrix.shell }} {0}
name: '${{ matrix.os }}-${{ matrix.cpu }}-tests-${{ matrix.tests }}'
runs-on: ${{ matrix.builder }}
timeout-minutes: 80
matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.matrix.outputs.matrix }}
cache_nonce: ${{ env.cache_nonce }}
steps:
- name: Checkout sources
uses: actions/checkout@v4
with:
submodules: recursive
- name: Compute matrix
id: matrix
uses: fabiocaccamo/create-matrix-action@v4
with:
matrix: |
os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {all}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
os {macos}, cpu {amd64}, builder {macos-latest}, tests {all}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2}
os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2}
os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}
- name: Setup Nimbus Build System
uses: ./.github/actions/nimbus-build-system
with:
os: ${{ matrix.os }}
shell: ${{ matrix.shell }}
## Part 1 Tests ##
- name: Unit tests
if: matrix.tests == 'unittest' || matrix.tests == 'all'
run: ./env.sh make -j${ncpu} test
# workaround for https://github.com/NomicFoundation/hardhat/issues/3877
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 18.15
- name: Start Ethereum node with Codex contracts
if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'all'
working-directory: vendor/codex-contracts-eth
env:
MSYS2_PATH_TYPE: inherit
run: |
npm install
npm start &
## Part 2 Tests ##
- name: Contract tests
if: matrix.tests == 'contract' || matrix.tests == 'all'
run: ./env.sh make -j${ncpu} testContracts
## Part 3 Tests ##
- name: Integration tests
if: matrix.tests == 'integration' || matrix.tests == 'all'
run: ./env.sh make -j${ncpu} testIntegration
build:
needs: matrix
uses: ./.github/workflows/ci-reusable.yml
with:
matrix: ${{ needs.matrix.outputs.matrix }}
cache_nonce: ${{ needs.matrix.outputs.cache_nonce }}
coverage:
continue-on-error: true

View File

@ -1,4 +1,4 @@
name: Docker - Reusable
name: Reusable - Docker
on:

30
.github/workflows/nim-matrix.yml vendored Normal file
View File

@ -0,0 +1,30 @@
name: Nim matrix
on:
merge_group:
workflow_dispatch:
env:
cache_nonce: 0 # Allows for easily busting actions/cache caches
nim_version: v1.6.14, v1.6.16, v1.6.18, v2.0.0, v2.0.2
jobs:
matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.matrix.outputs.matrix }}
cache_nonce: ${{ env.cache_nonce }}
steps:
- name: Compute matrix
id: matrix
uses: fabiocaccamo/create-matrix-action@v4
with:
matrix: |
os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {all}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
build:
needs: matrix
uses: ./.github/workflows/ci-reusable.yml
with:
matrix: ${{ needs.matrix.outputs.matrix }}
cache_nonce: ${{ needs.matrix.outputs.cache_nonce }}

View File

@ -1,6 +1,5 @@
import pkg/chronos
import pkg/stew/endians2
import pkg/upraises
import pkg/stint
type
@ -8,7 +7,7 @@ type
SecondsSince1970* = int64
Timeout* = object of CatchableError
method now*(clock: Clock): SecondsSince1970 {.base, upraises: [].} =
method now*(clock: Clock): Future[SecondsSince1970] {.base, async.} =
raiseAssert "not implemented"
method waitUntil*(clock: Clock, time: SecondsSince1970) {.base, async.} =

View File

@ -47,12 +47,12 @@ method stop*(clock: OnChainClock) {.async.} =
await clock.subscription.unsubscribe()
clock.started = false
method now*(clock: OnChainClock): SecondsSince1970 {.raises: [].}=
method now*(clock: OnChainClock): Future[SecondsSince1970] {.async.} =
when codex_use_hardhat:
# hardhat's latest block.timestamp is usually 1s behind the block timestamp
# in the newHeads event. When testing, always return the latest block.
try:
if queriedBlock =? (waitFor clock.provider.getBlock(BlockTag.latest)):
if queriedBlock =? (await clock.provider.getBlock(BlockTag.latest)):
trace "using last block timestamp for clock.now",
lastBlockTimestamp = queriedBlock.timestamp.truncate(int64),
cachedBlockTimestamp = clock.lastBlockTime.truncate(int64)
@ -66,6 +66,6 @@ method now*(clock: OnChainClock): SecondsSince1970 {.raises: [].}=
return toUnix(getTime() + clock.offset)
method waitUntil*(clock: OnChainClock, time: SecondsSince1970) {.async.} =
while (let difference = time - clock.now(); difference > 0):
while (let difference = time - (await clock.now()); difference > 0):
clock.newBlock.clear()
discard await clock.newBlock.wait().withTimeout(chronos.seconds(difference))

View File

@ -53,7 +53,7 @@ proc populate*(purchasing: Purchasing,
if result.ask.proofProbability == 0.u256:
result.ask.proofProbability = purchasing.proofProbability
if result.expiry == 0.u256:
result.expiry = (purchasing.clock.now().u256 + purchasing.requestExpiryInterval)
result.expiry = (await purchasing.clock.now()).u256 + purchasing.requestExpiryInterval
if result.nonce == Nonce.default:
var id = result.nonce.toArray
doAssert randomBytes(id) == 32

View File

@ -328,10 +328,12 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
if node.clock.isNil:
return RestApiResponse.error(Http500)
if expiry <= node.clock.now.u256:
let now = (await node.clock.now).u256
if expiry <= now:
return RestApiResponse.error(Http400, "Expiry needs to be in future")
if expiry > node.clock.now.u256 + params.duration:
if expiry > now + params.duration:
return RestApiResponse.error(Http400, "Expiry has to be before the request's end (now + duration)")
without purchaseId =? await node.requestStorage(

View File

@ -73,8 +73,9 @@ proc subscribeCancellation(agent: SalesAgent) {.async.} =
return
while true:
let deadline = max(clock.now, request.expiry.truncate(int64)) + 1
trace "Waiting for request to be cancelled", now=clock.now, expiry=deadline
let now = await clock.now
let deadline = max(now, request.expiry.truncate(int64)) + 1
trace "Waiting for request to be cancelled", now=now, expiry=deadline
await clock.waitUntil(deadline)
without state =? await agent.retrieveRequestState():
@ -85,7 +86,7 @@ proc subscribeCancellation(agent: SalesAgent) {.async.} =
agent.schedule(cancelledEvent(request))
break
debug "The request is not yet canceled, even though it should be. Waiting for some more time.", currentState = state, now=clock.now
debug "The request is not yet canceled, even though it should be. Waiting for some more time.", currentState = state, now=now
data.cancelled = onCancelled()

View File

@ -57,7 +57,7 @@ proc proveLoop(
proc getCurrentPeriod(): Future[Period] {.async.} =
let periodicity = await market.periodicity()
return periodicity.periodOf(clock.now().u256)
return periodicity.periodOf((await clock.now()).u256)
proc waitUntilPeriod(period: Period) {.async.} =
let periodicity = await market.periodicity()

View File

@ -59,7 +59,7 @@ proc deleteExpiredBlock(self: BlockMaintainer, cid: Cid): Future[void] {.async.}
trace "Unable to delete block from repoStore"
proc processBlockExpiration(self: BlockMaintainer, be: BlockExpiration): Future[void] {.async} =
if be.expiration < self.clock.now:
if be.expiration < (await self.clock.now):
await self.deleteExpiredBlock(be.cid)
else:
inc self.offset

View File

@ -229,12 +229,12 @@ proc getBlockExpirationEntry(
proc getBlockExpirationEntry(
self: RepoStore,
cid: Cid,
ttl: ?Duration): ?!BatchEntry =
ttl: ?Duration): Future[?!BatchEntry] {.async.} =
## Get an expiration entry for a batch for duration since "now"
##
let duration = ttl |? self.blockTtl
self.getBlockExpirationEntry(cid, self.clock.now() + duration.seconds)
self.getBlockExpirationEntry(cid, (await self.clock.now()) + duration.seconds)
method ensureExpiry*(
self: RepoStore,
@ -340,7 +340,7 @@ method putBlock*(
trace "Updating quota", used
batch.add((QuotaUsedKey, @(used.uint64.toBytesBE)))
without blockExpEntry =? self.getBlockExpirationEntry(blk.cid, ttl), err:
without blockExpEntry =? (await self.getBlockExpirationEntry(blk.cid, ttl)), err:
trace "Unable to create block expiration metadata key", err = err.msg
return failure(err)
batch.add(blockExpEntry)

View File

@ -1,10 +1,10 @@
import std/times
import pkg/upraises
import pkg/chronos
import ./clock
type
SystemClock* = ref object of Clock
method now*(clock: SystemClock): SecondsSince1970 {.upraises: [].} =
method now*(clock: SystemClock): Future[SecondsSince1970] {.async.} =
let now = times.now().utc
now.toTime().toUnix()

View File

@ -34,11 +34,11 @@ proc new*(
proc slots*(validation: Validation): seq[SlotId] =
validation.slots.toSeq
proc getCurrentPeriod(validation: Validation): UInt256 =
return validation.periodicity.periodOf(validation.clock.now().u256)
proc getCurrentPeriod(validation: Validation): Future[UInt256] {.async.} =
return validation.periodicity.periodOf((await validation.clock.now()).u256)
proc waitUntilNextPeriod(validation: Validation) {.async.} =
let period = validation.getCurrentPeriod()
let period = await validation.getCurrentPeriod()
let periodEnd = validation.periodicity.periodEnd(period)
trace "Waiting until next period", currentPeriod = period
await validation.clock.waitUntil(periodEnd.truncate(int64) + 1)
@ -66,7 +66,7 @@ proc markProofAsMissing(validation: Validation,
slotId: SlotId,
period: Period) {.async.} =
logScope:
currentPeriod = validation.getCurrentPeriod()
currentPeriod = (await validation.getCurrentPeriod())
try:
if await validation.market.canProofBeMarkedAsMissing(slotId, period):
@ -82,7 +82,7 @@ proc markProofAsMissing(validation: Validation,
proc markProofsAsMissing(validation: Validation) {.async.} =
for slotId in validation.slots:
let previousPeriod = validation.getCurrentPeriod() - 1
let previousPeriod = (await validation.getCurrentPeriod()) - 1
await validation.markProofAsMissing(slotId, previousPeriod)
proc run(validation: Validation) {.async.} =

11
env.sh
View File

@ -1,6 +1,6 @@
#!/usr/bin/env bash
# This is the compiler version that will get used everywher.
# This is the compiler version that will get used everywhere by default.
NIM_VERSION="f45bdea94ac4ed9a9bae03426275456aeb0cab2a"
NIM_REPO_URL="https://github.com/gmega/Nim"
@ -11,9 +11,12 @@ ABS_PATH="$(cd ${REL_PATH}; pwd)"
ENV_FILE="${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh"
# Allows the user to override the default Nim compiler version and repo URL.
export NIM_COMMIT="${NIM_COMMIT:-${NIM_VERSION}}"
export NIM_REPO="${NIM_REPO:-${NIM_REPO_URL}}"
# This makes it look nicer in the CI: if the version in the matrix says
# "repo_current", then we'll use the version that's registered here.
if [ "${NIM_COMMIT}" = "repo_current" ] || [ "${NIM_COMMIT}" = "" ]; then
export NIM_COMMIT="${NIM_VERSION}"
export NIM_REPO="${NIM_REPO_URL}"
fi
if ! [ -f "$ENV_FILE" ]; then
# Before the first "make update", the env file doesn't exist.

View File

@ -32,11 +32,11 @@ proc set*(clock: MockClock, time: SecondsSince1970) =
proc advance*(clock: MockClock, seconds: int64) =
clock.set(clock.time + seconds)
method now*(clock: MockClock): SecondsSince1970 {.raises: [].} =
method now*(clock: MockClock): Future[SecondsSince1970] {.async.} =
clock.time
method waitUntil*(clock: MockClock, time: SecondsSince1970) {.async.} =
if time > clock.now():
if time > (await clock.now()):
let future = newFuture[void]()
clock.waiting.add(Waiting(until: time, future: future))
await future

View File

@ -98,7 +98,7 @@ asyncchecksuite "Test Node - Host contracts":
test "onExpiryUpdate callback":
let
# The blocks have set default TTL, so in order to update it we have to have larger TTL
expectedExpiry: SecondsSince1970 = clock.now + DefaultBlockTtl.seconds + 11123
expectedExpiry: SecondsSince1970 = (await clock.now) + DefaultBlockTtl.seconds + 11123
expiryUpdateCallback = !sales.onExpiryUpdate
(await expiryUpdateCallback(manifestCidStr, expectedExpiry)).tryGet()

View File

@ -67,7 +67,7 @@ asyncchecksuite "Sales - start":
sales.onProve = proc(slot: Slot, challenge: ProofChallenge): Future[?!seq[byte]] {.async.} =
return success(proof)
itemsProcessed = @[]
request.expiry = (clock.now() + 42).u256
request.expiry = ((await clock.now()) + 42).u256
teardown:
await sales.stop()

View File

@ -161,7 +161,7 @@ checksuite "Purchasing state machine":
market.requestState[request5.id] = RequestState.Failed
# ensure the started state doesn't error, giving a false positive test result
market.requestEnds[request2.id] = clock.now() - 1
market.requestEnds[request2.id] = (await clock.now()) - 1
await purchasing.load()
check eventually purchasing.getPurchase(PurchaseId(request1.id)).?finished == false.some
@ -182,7 +182,7 @@ checksuite "Purchasing state machine":
test "moves to PurchaseStarted when request state is Started":
let request = StorageRequest.example
let purchase = Purchase.new(request, market, clock)
market.requestEnds[request.id] = clock.now() + request.ask.duration.truncate(int64)
market.requestEnds[request.id] = (await clock.now()) + request.ask.duration.truncate(int64)
market.requested = @[request]
market.requestState[request.id] = RequestState.Started
let next = await PurchaseUnknown().run(purchase)
@ -215,7 +215,7 @@ checksuite "Purchasing state machine":
test "moves to PurchaseFailed state once RequestFailed emitted":
let request = StorageRequest.example
let purchase = Purchase.new(request, market, clock)
market.requestEnds[request.id] = clock.now() + request.ask.duration.truncate(int64)
market.requestEnds[request.id] = (await clock.now()) + request.ask.duration.truncate(int64)
let future = PurchaseStarted().run(purchase)
market.emitRequestFailed(request.id)
@ -226,7 +226,7 @@ checksuite "Purchasing state machine":
test "moves to PurchaseFinished state once request finishes":
let request = StorageRequest.example
let purchase = Purchase.new(request, market, clock)
market.requestEnds[request.id] = clock.now() + request.ask.duration.truncate(int64)
market.requestEnds[request.id] = (await clock.now()) + request.ask.duration.truncate(int64)
let future = PurchaseStarted().run(purchase)
clock.advance(request.ask.duration.truncate(int64))

View File

@ -1,14 +1,15 @@
import std/times
import std/unittest
import codex/systemclock
import ../asynctest
import ./helpers
checksuite "SystemClock":
asyncchecksuite "SystemClock":
test "Should get now":
let clock = SystemClock.new()
let expectedNow = times.now().utc
let now = clock.now()
let now = (await clock.now())
check now == expectedNow.toTime().toUnix()

View File

@ -31,9 +31,9 @@ asyncchecksuite "validation":
teardown:
await validation.stop()
proc advanceToNextPeriod =
proc advanceToNextPeriod {.async.} =
let periodicity = Periodicity(seconds: period.u256)
let period = periodicity.periodOf(clock.now().u256)
let period = periodicity.periodOf((await clock.now()).u256)
let periodEnd = periodicity.periodEnd(period)
clock.set((periodEnd + 1).truncate(int))
@ -48,20 +48,20 @@ asyncchecksuite "validation":
test "when slot state changes, it is removed from the list":
await market.fillSlot(slot.request.id, slot.slotIndex, @[], collateral)
market.slotState[slot.id] = state
advanceToNextPeriod()
await advanceToNextPeriod()
check eventually validation.slots.len == 0
test "when a proof is missed, it is marked as missing":
await market.fillSlot(slot.request.id, slot.slotIndex, @[], collateral)
market.setCanProofBeMarkedAsMissing(slot.id, true)
advanceToNextPeriod()
await advanceToNextPeriod()
await sleepAsync(1.millis)
check market.markedAsMissingProofs.contains(slot.id)
test "when a proof can not be marked as missing, it will not be marked":
await market.fillSlot(slot.request.id, slot.slotIndex, @[], collateral)
market.setCanProofBeMarkedAsMissing(slot.id, false)
advanceToNextPeriod()
await advanceToNextPeriod()
await sleepAsync(1.millis)
check market.markedAsMissingProofs.len == 0

View File

@ -17,23 +17,23 @@ ethersuite "On-Chain Clock":
test "returns the current time of the EVM":
let latestBlock = (!await ethProvider.getBlock(BlockTag.latest))
let timestamp = latestBlock.timestamp.truncate(int64)
check clock.now() == timestamp
check (await clock.now()) == timestamp
test "updates time with timestamp of new blocks":
let future = (getTime() + 42.years).toUnix
discard await ethProvider.send("evm_setNextBlockTimestamp", @[%future])
discard await ethProvider.send("evm_mine")
check clock.now() == future
check (await clock.now()) == future
test "can wait until a certain time is reached by the chain":
let future = clock.now() + 42 # seconds
let future = (await clock.now()) + 42 # seconds
let waiting = clock.waitUntil(future)
discard await ethProvider.send("evm_setNextBlockTimestamp", @[%future])
discard await ethProvider.send("evm_mine")
check await waiting.withTimeout(chronos.milliseconds(100))
test "can wait until a certain time is reached by the wall-clock":
let future = clock.now() + 1 # seconds
let future = (await clock.now()) + 1 # seconds
let waiting = clock.waitUntil(future)
check await waiting.withTimeout(chronos.seconds(2))