From 876a4cd684db54bedb386611da15f42a58dee252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 20 May 2021 17:52:28 +0200 Subject: [PATCH 01/15] CI cron: try a different expression syntax --- .github/workflows/cron.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 429847f37..8aaad6e32 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -1,7 +1,7 @@ name: Daily on: schedule: - - cron: "45 3 * * *" + - cron: "55 15 * * *" jobs: build: @@ -41,7 +41,7 @@ jobs: name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' runs-on: ${{ matrix.builder }} - continue-on-error: ${{ matrix.branch }} == 'devel' + continue-on-error: ${{ matrix.branch == 'devel' }} steps: - name: Checkout nimbus-eth2 uses: actions/checkout@v2 From 57259cf0d917ee0efb7e5c58766dab0ffdd89468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 20 May 2021 18:21:16 +0200 Subject: [PATCH 02/15] CI cron: debug fixture download --- .github/workflows/cron.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 8aaad6e32..778b5c07d 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -1,7 +1,7 @@ name: Daily on: schedule: - - cron: "55 15 * * *" + - cron: "25 16 * * *" jobs: build: @@ -171,7 +171,7 @@ jobs: - name: Get the Ethereum Foundation fixtures shell: bash run: | - scripts/setup_official_tests.sh fixturesCache + V=1 scripts/setup_official_tests.sh fixturesCache - name: Build Nim and Nimbus dependencies shell: bash From dd84ecd520efc284b70f05c1a4631a30a1865472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 20 May 2021 18:21:16 +0200 Subject: [PATCH 03/15] CI cron: debug fixture download --- .github/workflows/cron.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 778b5c07d..6ce13fbea 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -1,7 +1,7 @@ name: Daily on: schedule: - - cron: "25 16 * * *" + - cron: "30 16 * * *" jobs: build: From 00dadaba15c0db7c86ff4ec4b45ed78be88dc9c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 20 May 2021 18:36:33 +0200 Subject: [PATCH 04/15] CI cron: reschedule it --- .github/workflows/cron.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 6ce13fbea..41e8e5b56 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -1,7 +1,7 @@ name: Daily on: schedule: - - cron: "30 16 * * *" + - cron: "40 16 * * *" jobs: build: From 74722f16959a84ad72a664597440f20bb515f2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 20 May 2021 19:02:06 +0200 Subject: [PATCH 05/15] CI cron: reschedule --- .github/workflows/cron.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 41e8e5b56..e764b8bbd 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -1,7 +1,7 @@ name: Daily on: schedule: - - cron: "40 16 * * *" + - cron: "40 17 * * *" jobs: build: From 8b15c5d75f55007635cb1656b5c525196336847c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 20 May 2021 20:00:38 +0200 Subject: [PATCH 06/15] CI cron: change step order --- .github/workflows/cron.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index e764b8bbd..f210d953b 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -1,7 +1,7 @@ name: Daily on: schedule: - - cron: "40 17 * * *" + - cron: "10 20 * * *" jobs: build: @@ -151,6 +151,11 @@ jobs: brew install gnu-getopt brew link --force gnu-getopt + - name: Build Nim and Nimbus dependencies + shell: bash + run: | + make -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update + - name: Get latest fixtures commit hash id: fixtures_version shell: bash @@ -171,12 +176,7 @@ jobs: - name: Get the Ethereum Foundation fixtures shell: bash run: | - V=1 scripts/setup_official_tests.sh fixturesCache - - - name: Build Nim and Nimbus dependencies - shell: bash - run: | - make -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update + scripts/setup_official_tests.sh fixturesCache - name: Smoke test the Beacon Node and Validator Client with all tracing enabled shell: bash From a2cbcff89c4c4e56fc6fc6afff2bb9f02db361fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Wed, 9 Jun 2021 02:20:29 +0200 Subject: [PATCH 07/15] CI: nightly build workflow --- .github/workflows/nightly_build.yml | 308 ++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 .github/workflows/nightly_build.yml diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml new file mode 100644 index 000000000..e8fae6533 --- /dev/null +++ b/.github/workflows/nightly_build.yml @@ -0,0 +1,308 @@ +name: Nightly build +on: + schedule: + - cron: "45 4 * * *" + workflow_dispatch: + #pull_request: + +jobs: + build-amd64: + name: Linux AMD64 release asset + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: unstable + + - name: Build project + id: make_dist + run: | + make dist-amd64 + cd dist + ARCHIVE=$(echo *.tar.gz) + tar -xzf ${ARCHIVE} + NEW_ARCHIVE_DIR="nimbus-eth2_Linux_amd64_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" + mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} + tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz + echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} + + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: Linux_amd64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: Linux_amd64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum + retention-days: 2 + + build-arm64: + name: Linux ARM64 release asset + runs-on: ubuntu-latest + steps: + - name: Install packages + env: + DEBIAN_FRONTEND: "noninteractive" + TZ: "Etc/UTC" + run: | + sudo apt-get -qq update + sudo apt-get -qq -y install binfmt-support qemu-user-static + + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: unstable + + - name: Build project + id: make_dist + run: | + make dist-arm64 + cd dist + ARCHIVE=$(echo *.tar.gz) + tar -xzf ${ARCHIVE} + NEW_ARCHIVE_DIR="nimbus-eth2_Linux_arm64v8_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" + mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} + tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz + echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} + + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: Linux_arm64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: Linux_arm64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum + retention-days: 2 + + build-arm: + name: Linux ARM release asset + runs-on: ubuntu-latest + steps: + - name: Install packages + env: + DEBIAN_FRONTEND: "noninteractive" + TZ: "Etc/UTC" + run: | + sudo apt-get -qq update + sudo apt-get -qq -y install binfmt-support qemu-user-static + + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: unstable + + - name: Build project + id: make_dist + run: | + make dist-arm + cd dist + ARCHIVE=$(echo *.tar.gz) + tar -xzf ${ARCHIVE} + NEW_ARCHIVE_DIR="nimbus-eth2_Linux_arm32v7_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" + mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} + tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz + echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} + + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: Linux_arm_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: Linux_arm_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum + retention-days: 2 + + build-win64: + name: Windows AMD64 release asset + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: unstable + + - name: Build project + id: make_dist + run: | + make dist-win64 + cd dist + ARCHIVE=$(echo *.tar.gz) + tar -xzf ${ARCHIVE} + NEW_ARCHIVE_DIR="nimbus-eth2_Windows_amd64_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" + mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} + tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz + echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} + + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: Windows_amd64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: Windows_amd64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum + retention-days: 2 + + build-macos-amd64: + name: macOS AMD64 release asset + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: unstable + + - name: Build project + id: make_dist + run: | + make dist-macos + cd dist + ARCHIVE=$(echo *.tar.gz) + tar -xzf ${ARCHIVE} + NEW_ARCHIVE_DIR="nimbus-eth2_macOS_amd64_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" + mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} + tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz + echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} + + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_amd64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_amd64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum + retention-days: 2 + + build-macos-arm64: + name: macOS ARM64 release asset + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: unstable + + - name: Build project + id: make_dist + run: | + make dist-macos-arm64 + cd dist + ARCHIVE=$(echo *.tar.gz) + tar -xzf ${ARCHIVE} + NEW_ARCHIVE_DIR="nimbus-eth2_macOS_arm64_$(date +%Y%m%d)_$(git rev-parse --short=8 HEAD)" + mv ${ARCHIVE%.tar.gz} ${NEW_ARCHIVE_DIR} + tar -czf ${NEW_ARCHIVE_DIR}.tar.gz ${NEW_ARCHIVE_DIR} + echo "::set-output name=archive::"${NEW_ARCHIVE_DIR}.tar.gz + echo "::set-output name=archive_dir::"${NEW_ARCHIVE_DIR} + + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_arm64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_arm64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum + retention-days: 2 + + prepare-prerelease: + name: Prepare pre-release + needs: [build-amd64, build-arm64, build-arm, build-win64, build-macos-amd64, build-macos-arm64] + runs-on: ubuntu-latest + steps: + - name: Download artefacts + uses: actions/download-artifact@v2 + + - name: Create release notes + run: | + cat > release_notes.md <> release_notes.md + cat Linux_amd64_checksum/* >> release_notes.md + echo '# Linux ARM64' >> release_notes.md + cat Linux_arm64_checksum/* >> release_notes.md + echo '# Linux ARM' >> release_notes.md + cat Linux_arm_checksum/* >> release_notes.md + echo '# Windows AMD64' >> release_notes.md + cat Windows_amd64_checksum/* >> release_notes.md + echo '# macOS AMD64' >> release_notes.md + cat macOS_amd64_checksum/* >> release_notes.md + echo '# macOS ARM64' >> release_notes.md + cat macOS_arm64_checksum/* >> release_notes.md + echo '```' >> release_notes.md + + - name: Delete tag + uses: dev-drprasad/delete-tag-and-release@v0.2.0 + with: + delete_release: true + tag_name: latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create prerelease + run: | + gh release create latest --prerelease --target unstable \ + --title 'Nightly build ("unstable" branch)' --notes-file release_notes.md \ + Linux_amd64_archive/* \ + Linux_arm64_archive/* \ + Linux_arm_archive/* \ + Windows_amd64_archive/* \ + macOS_amd64_archive/* \ + macOS_arm64_archive/* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Delete artefacts + uses: geekyeggo/delete-artifact@v1 + with: + failOnError: false + name: | + Linux_amd64_archive + Linux_amd64_checksum + Linux_arm64_archive + Linux_arm64_checksum + Linux_arm_archive + Linux_arm_checksum + Windows_amd64_archive + Windows_amd64_checksum + macOS_amd64_archive + macOS_amd64_checksum + macOS_arm64_archive + macOS_arm64_checksum + From 94b1ef55682df399a3808ba54717f4c16f9a422c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Wed, 9 Jun 2021 17:39:42 +0200 Subject: [PATCH 08/15] fix nightly build (#2640) --- .github/workflows/nightly_build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index e8fae6533..45276381b 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -243,6 +243,11 @@ jobs: needs: [build-amd64, build-arm64, build-arm, build-win64, build-macos-amd64, build-macos-arm64] runs-on: ubuntu-latest steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: unstable + - name: Download artefacts uses: actions/download-artifact@v2 @@ -271,7 +276,7 @@ jobs: uses: dev-drprasad/delete-tag-and-release@v0.2.0 with: delete_release: true - tag_name: latest + tag_name: latest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 7ffa8c280f4c48f458418e8a192d5179a4606675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 10 Jun 2021 13:03:26 +0200 Subject: [PATCH 09/15] nightly build: change tag name (#2643) --- .github/workflows/nightly_build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index 45276381b..756c11a5c 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -276,13 +276,13 @@ jobs: uses: dev-drprasad/delete-tag-and-release@v0.2.0 with: delete_release: true - tag_name: latest + tag_name: nightly env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Create prerelease run: | - gh release create latest --prerelease --target unstable \ + gh release create nightly --prerelease --target unstable \ --title 'Nightly build ("unstable" branch)' --notes-file release_notes.md \ Linux_amd64_archive/* \ Linux_arm64_archive/* \ From 9d48097de0ba67141947a3eab569e1c5f3a02e77 Mon Sep 17 00:00:00 2001 From: tersec Date: Thu, 17 Jun 2021 07:22:39 +0000 Subject: [PATCH 10/15] fix assertion in beacon block creation rollback/restore (#2655) --- beacon_chain/validators/validator_duties.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_chain/validators/validator_duties.nim b/beacon_chain/validators/validator_duties.nim index b4caacc88..2995e0bf7 100644 --- a/beacon_chain/validators/validator_duties.nim +++ b/beacon_chain/validators/validator_duties.nim @@ -313,7 +313,7 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode, # TODO address this ugly workaround - there should probably be a # `state_transition` that takes a `StateData` instead and updates # the block as well - doAssert v.addr == addr proposalStateAddr.data + doAssert v.addr == addr proposalStateAddr.data.hbsPhase0 assign(proposalStateAddr[], poolPtr.headState) return makeBeaconBlock( From 9ad8310157c91c08f39134d927e8bcb55d6a7f3d Mon Sep 17 00:00:00 2001 From: tersec Date: Thu, 17 Jun 2021 11:51:04 +0000 Subject: [PATCH 11/15] doppelganger detection walltime refactor (#2656) * strawman doppelganger detection walltime refactor * move DoppelgangerProtection to Eth2Processor * increase comment precision * document difference between broadcastStartEpoch and nodeLaunchSlot, and allow for one-slot overlap to avoid false positives on intra-slot restarts --- .../gossip_processing/eth2_processor.nim | 34 +++++++++++++------ beacon_chain/spec/datatypes/base.nim | 3 -- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index f70d9275c..34e2bbb8d 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -45,6 +45,18 @@ declareHistogram beacon_block_delay, "Time(s) between slot start and beacon block reception", buckets = delayBuckets type + DoppelgangerProtection = object + broadcastStartEpoch*: Epoch ##\ + ## Set anew, each time gossip is re-enabled after syncing completes, so + ## might reset multiple times per instance. This allows some safe level + ## of gossip interleaving between nodes so long as they don't gossip at + ## the same time. + + nodeLaunchSlot: Slot ##\ + ## Set once, at node launch. This functions as a basic protection against + ## false positives from attestations persisting within the gossip network + ## across quick restarts. + Eth2Processor* = object doppelGangerDetectionEnabled*: bool getWallTime*: GetWallTimeFn @@ -88,6 +100,8 @@ proc new*(T: type Eth2Processor, getWallTime: GetWallTimeFn): ref Eth2Processor = (ref Eth2Processor)( doppelGangerDetectionEnabled: doppelGangerDetectionEnabled, + doppelgangerDetection: DoppelgangerProtection( + nodeLaunchSlot: getWallTime().slotOrZero), getWallTime: getWallTime, blockProcessor: blockProcessor, dag: dag, @@ -158,15 +172,15 @@ proc blockValidator*( proc checkForPotentialDoppelganger( self: var Eth2Processor, attestation: Attestation, - attesterIndices: openArray[ValidatorIndex], wallSlot: Slot) = - let epoch = wallSlot.epoch - - # Only check for current epoch, not potential attestations bouncing around - # from up to several minutes prior. - if attestation.data.slot.epoch < epoch: + attesterIndices: openArray[ValidatorIndex]) = + # Only check for attestations after node launch. There might be one slot of + # overlap in quick intra-slot restarts so trade off a few true negatives in + # the service of avoiding more likely false positives. + if attestation.data.slot <= self.doppelgangerDetection.nodeLaunchSlot + 1: return - if epoch < self.doppelgangerDetection.broadcastStartEpoch: + if attestation.data.slot.epoch < + self.doppelgangerDetection.broadcastStartEpoch: let tgtBlck = self.dag.getRef(attestation.data.target.root) doAssert not tgtBlck.isNil # because attestation is valid above @@ -221,8 +235,7 @@ proc attestationValidator*( let (attestation_index, sig) = v.get() - self[].checkForPotentialDoppelganger( - attestation, [attestation_index], wallSlot) + self[].checkForPotentialDoppelganger(attestation, [attestation_index]) trace "Attestation validated" self.attestationPool[].addAttestation( @@ -269,8 +282,7 @@ proc aggregateValidator*( let (attesting_indices, sig) = v.get() self[].checkForPotentialDoppelganger( - signedAggregateAndProof.message.aggregate, attesting_indices, - wallSlot) + signedAggregateAndProof.message.aggregate, attesting_indices) trace "Aggregate validated", aggregator_index = signedAggregateAndProof.message.aggregator_index, diff --git a/beacon_chain/spec/datatypes/base.nim b/beacon_chain/spec/datatypes/base.nim index def96bf40..17ff531a7 100644 --- a/beacon_chain/spec/datatypes/base.nim +++ b/beacon_chain/spec/datatypes/base.nim @@ -472,9 +472,6 @@ type current_justified_checkpoint*: Checkpoint finalized_checkpoint*: Checkpoint - DoppelgangerProtection* = object - broadcastStartEpoch*: Epoch - type # Caches for computing justificiation, rewards and penalties - based on # implementation in Lighthouse: From e14873757ea4ad2d96bf24473715f6fbdffa58a2 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Thu, 17 Jun 2021 16:38:25 +0200 Subject: [PATCH 12/15] Fix #2595 - recompute wallSlot across async calls (#2650) --- .../gossip_processing/eth2_processor.nim | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index 34e2bbb8d..8d68311c7 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -208,9 +208,8 @@ proc attestationValidator*( attestation = shortLog(attestation) subnet_id - let - wallTime = self.getWallTime() - (afterGenesis, wallSlot) = wallTime.toSlot() + let wallTime = self.getWallTime() + var (afterGenesis, wallSlot) = wallTime.toSlot() if not afterGenesis: notice "Attestation before genesis" @@ -227,9 +226,11 @@ proc attestationValidator*( self.batchCrypto, attestation, wallTime, subnet_id, checkSignature) if v.isErr(): debug "Dropping attestation", validationError = v.error - return v.error[0] + # Due to async validation the wallSlot here might have changed + (afterGenesis, wallSlot) = self.getWallTime().toSlot() + beacon_attestations_received.inc() beacon_attestation_delay.observe(delay.toFloatSeconds()) @@ -250,9 +251,8 @@ proc aggregateValidator*( aggregate = shortLog(signedAggregateAndProof.message.aggregate) signature = shortLog(signedAggregateAndProof.signature) - let - wallTime = self.getWallTime() - (afterGenesis, wallSlot) = wallTime.toSlot() + let wallTime = self.getWallTime() + var (afterGenesis, wallSlot) = wallTime.toSlot() if not afterGenesis: notice "Aggregate before genesis" @@ -276,6 +276,9 @@ proc aggregateValidator*( wallSlot return v.error[0] + # Due to async validation the wallSlot here might have changed + (afterGenesis, wallSlot) = self.getWallTime().toSlot() + beacon_aggregates_received.inc() beacon_aggregate_delay.observe(delay.toFloatSeconds()) @@ -286,8 +289,7 @@ proc aggregateValidator*( trace "Aggregate validated", aggregator_index = signedAggregateAndProof.message.aggregator_index, - selection_proof = signedAggregateAndProof.message.selection_proof, - wallSlot + selection_proof = signedAggregateAndProof.message.selection_proof self.attestationPool[].addAttestation( signedAggregateAndProof.message.aggregate, attesting_indices, sig, wallSlot) From a3322acee9c957c46c2debe7b472fb51a979396f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 17 Jun 2021 16:52:11 +0200 Subject: [PATCH 13/15] Revert "fix assertion in beacon block creation rollback/restore (#2655)" This reverts commit 9d48097de0ba67141947a3eab569e1c5f3a02e77. --- beacon_chain/validators/validator_duties.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_chain/validators/validator_duties.nim b/beacon_chain/validators/validator_duties.nim index 2995e0bf7..b4caacc88 100644 --- a/beacon_chain/validators/validator_duties.nim +++ b/beacon_chain/validators/validator_duties.nim @@ -313,7 +313,7 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode, # TODO address this ugly workaround - there should probably be a # `state_transition` that takes a `StateData` instead and updates # the block as well - doAssert v.addr == addr proposalStateAddr.data.hbsPhase0 + doAssert v.addr == addr proposalStateAddr.data assign(proposalStateAddr[], poolPtr.headState) return makeBeaconBlock( From 533cd25e1c51078302c7e316ae3f95cf4fa937ed Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Fri, 18 Jun 2021 15:43:54 +0300 Subject: [PATCH 14/15] Add CONFIG_NAME parameter to REST API /eth/v1/config/spec response. (#2652) * Add CONFIG_NAME parameter to /eth/v1/config/spec response. * Add missed comma. * Use `const_preset` instead of CONFIG_NAME. --- beacon_chain/rpc/config_rest_api.nim | 2 ++ ncli/resttest-rules.json | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/beacon_chain/rpc/config_rest_api.nim b/beacon_chain/rpc/config_rest_api.nim index 359fe619d..d4dc49031 100644 --- a/beacon_chain/rpc/config_rest_api.nim +++ b/beacon_chain/rpc/config_rest_api.nim @@ -35,6 +35,8 @@ proc installConfigApiHandlers*(router: var RestRouter, node: BeaconNode) = "/api/eth/v1/config/spec") do () -> RestApiResponse: return RestApiResponse.jsonResponse( ( + CONFIG_NAME: + const_preset, MAX_COMMITTEES_PER_SLOT: Base10.toString(MAX_COMMITTEES_PER_SLOT), TARGET_COMMITTEE_SIZE: diff --git a/ncli/resttest-rules.json b/ncli/resttest-rules.json index d88bf6a0e..fb7d3506f 100644 --- a/ncli/resttest-rules.json +++ b/ncli/resttest-rules.json @@ -2471,7 +2471,7 @@ "response": { "status": {"operator": "equals", "value": "200"}, "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals"}], - "body": [{"operator": "jstructcmps", "start": ["data"], "value": {"MAX_COMMITTEES_PER_SLOT": "", "TARGET_COMMITTEE_SIZE": "", "MAX_VALIDATORS_PER_COMMITTEE": "", "MIN_PER_EPOCH_CHURN_LIMIT": "", "CHURN_LIMIT_QUOTIENT": "", "SHUFFLE_ROUND_COUNT": "", "MIN_GENESIS_ACTIVE_VALIDATOR_COUNT": "", "MIN_GENESIS_TIME": "", "HYSTERESIS_QUOTIENT": "", "HYSTERESIS_DOWNWARD_MULTIPLIER": "", "HYSTERESIS_UPWARD_MULTIPLIER": "", "SAFE_SLOTS_TO_UPDATE_JUSTIFIED": "", "ETH1_FOLLOW_DISTANCE": "", "TARGET_AGGREGATORS_PER_COMMITTEE": "", "RANDOM_SUBNETS_PER_VALIDATOR": "", "EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION": "", "SECONDS_PER_ETH1_BLOCK": "", "DEPOSIT_CHAIN_ID": "", "DEPOSIT_NETWORK_ID": "", "DEPOSIT_CONTRACT_ADDRESS": "", "MIN_DEPOSIT_AMOUNT": "", "MAX_EFFECTIVE_BALANCE": "", "EJECTION_BALANCE": "", "EFFECTIVE_BALANCE_INCREMENT": "", "GENESIS_FORK_VERSION": "", "BLS_WITHDRAWAL_PREFIX": "", "GENESIS_DELAY": "", "SECONDS_PER_SLOT": "", "MIN_ATTESTATION_INCLUSION_DELAY": "", "SLOTS_PER_EPOCH": "", "MIN_SEED_LOOKAHEAD": "", "MAX_SEED_LOOKAHEAD": "", "EPOCHS_PER_ETH1_VOTING_PERIOD": "", "SLOTS_PER_HISTORICAL_ROOT": "", "MIN_VALIDATOR_WITHDRAWABILITY_DELAY": "", "SHARD_COMMITTEE_PERIOD": "", "MIN_EPOCHS_TO_INACTIVITY_PENALTY": "", "EPOCHS_PER_HISTORICAL_VECTOR": "", "EPOCHS_PER_SLASHINGS_VECTOR": "", "HISTORICAL_ROOTS_LIMIT": "", "VALIDATOR_REGISTRY_LIMIT": "", "BASE_REWARD_FACTOR": "", "WHISTLEBLOWER_REWARD_QUOTIENT": "", "PROPOSER_REWARD_QUOTIENT": "", "INACTIVITY_PENALTY_QUOTIENT": "", "MIN_SLASHING_PENALTY_QUOTIENT" :"", "PROPORTIONAL_SLASHING_MULTIPLIER" :"", "MAX_PROPOSER_SLASHINGS" :"", "MAX_ATTESTER_SLASHINGS" :"", "MAX_ATTESTATIONS" :"", "MAX_DEPOSITS" :"", "MAX_VOLUNTARY_EXITS" :"", "DOMAIN_BEACON_PROPOSER" :"", "DOMAIN_BEACON_ATTESTER" :"", "DOMAIN_RANDAO" :"", "DOMAIN_DEPOSIT" :"", "DOMAIN_VOLUNTARY_EXIT" :"", "DOMAIN_SELECTION_PROOF" :"", "DOMAIN_AGGREGATE_AND_PROOF" :""}}] + "body": [{"operator": "jstructcmps", "start": ["data"], "value": {"CONFIG_NAME": "", "MAX_COMMITTEES_PER_SLOT": "", "TARGET_COMMITTEE_SIZE": "", "MAX_VALIDATORS_PER_COMMITTEE": "", "MIN_PER_EPOCH_CHURN_LIMIT": "", "CHURN_LIMIT_QUOTIENT": "", "SHUFFLE_ROUND_COUNT": "", "MIN_GENESIS_ACTIVE_VALIDATOR_COUNT": "", "MIN_GENESIS_TIME": "", "HYSTERESIS_QUOTIENT": "", "HYSTERESIS_DOWNWARD_MULTIPLIER": "", "HYSTERESIS_UPWARD_MULTIPLIER": "", "SAFE_SLOTS_TO_UPDATE_JUSTIFIED": "", "ETH1_FOLLOW_DISTANCE": "", "TARGET_AGGREGATORS_PER_COMMITTEE": "", "RANDOM_SUBNETS_PER_VALIDATOR": "", "EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION": "", "SECONDS_PER_ETH1_BLOCK": "", "DEPOSIT_CHAIN_ID": "", "DEPOSIT_NETWORK_ID": "", "DEPOSIT_CONTRACT_ADDRESS": "", "MIN_DEPOSIT_AMOUNT": "", "MAX_EFFECTIVE_BALANCE": "", "EJECTION_BALANCE": "", "EFFECTIVE_BALANCE_INCREMENT": "", "GENESIS_FORK_VERSION": "", "BLS_WITHDRAWAL_PREFIX": "", "GENESIS_DELAY": "", "SECONDS_PER_SLOT": "", "MIN_ATTESTATION_INCLUSION_DELAY": "", "SLOTS_PER_EPOCH": "", "MIN_SEED_LOOKAHEAD": "", "MAX_SEED_LOOKAHEAD": "", "EPOCHS_PER_ETH1_VOTING_PERIOD": "", "SLOTS_PER_HISTORICAL_ROOT": "", "MIN_VALIDATOR_WITHDRAWABILITY_DELAY": "", "SHARD_COMMITTEE_PERIOD": "", "MIN_EPOCHS_TO_INACTIVITY_PENALTY": "", "EPOCHS_PER_HISTORICAL_VECTOR": "", "EPOCHS_PER_SLASHINGS_VECTOR": "", "HISTORICAL_ROOTS_LIMIT": "", "VALIDATOR_REGISTRY_LIMIT": "", "BASE_REWARD_FACTOR": "", "WHISTLEBLOWER_REWARD_QUOTIENT": "", "PROPOSER_REWARD_QUOTIENT": "", "INACTIVITY_PENALTY_QUOTIENT": "", "MIN_SLASHING_PENALTY_QUOTIENT" :"", "PROPORTIONAL_SLASHING_MULTIPLIER" :"", "MAX_PROPOSER_SLASHINGS" :"", "MAX_ATTESTER_SLASHINGS" :"", "MAX_ATTESTATIONS" :"", "MAX_DEPOSITS" :"", "MAX_VOLUNTARY_EXITS" :"", "DOMAIN_BEACON_PROPOSER" :"", "DOMAIN_BEACON_ATTESTER" :"", "DOMAIN_RANDAO" :"", "DOMAIN_DEPOSIT" :"", "DOMAIN_VOLUNTARY_EXIT" :"", "DOMAIN_SELECTION_PROOF" :"", "DOMAIN_AGGREGATE_AND_PROOF" :""}}] } }, { From 4ef784fa68211b0c59b3631c4a200fe5e239835b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mamy=20Andr=C3=A9-Ratsimbazafy?= Date: Mon, 21 Jun 2021 14:02:12 +0200 Subject: [PATCH 15/15] v1.4.0 --- CHANGELOG.md | 33 ++++++++++++++++++++++++++++++++- beacon_chain/version.nim | 3 +-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 685eab1ec..66fcdf116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,35 @@ +2021-06-21 v1.4.0 +================= + +This release is marked as low-urgency - please update at your convenience. + +It contains **improvements to attestation effectiveness and CPU usage**. + +It also contains **improvements to the RPC APIs**, as suggested by DappNode and RocketPool, and in preparation for our refactored validator client. +### Outdated Nimbus instances + +A reminder that if you're running Nimbus with a version prior to `1.0.10` (March 2021) you are exposed to a vulnerability in our core cryptography library (this library is used by all eth2 clients). See this [blst security advisory](https://github.com/supranational/blst/security/advisories/GHSA-x279-68rr-jp4p) for more information. If this concerns you, please update as soon as you can. + +## Notable changes +### We've added: + +* Nightly builds for the very adventurous. (https://github.com/status-im/nimbus-eth2/pull/2640) + * We expect users of nightly builds to be comfortable providing debugging logs. + +### We've fixed: + +* RPC API endpoints (https://github.com/status-im/nimbus-eth2/pull/2585, https://github.com/status-im/nimbus-eth2/pull/2586) + * `/eth/v1/beacon/pool/attestations` + * `/api/eth/v1/validator/aggregate_and_proofs` +* Doppelganger detection: fixed false positive on fast restart (https://github.com/status-im/nimbus-eth2/pull/2656/) + + +### We've improved + +* Database read performance improvements during epoch transitions and startup (https://github.com/status-im/nimbus-eth2/pull/2639, https://github.com/status-im/nimbus-eth2/pull/2617) +* Better usage of caches, specially when validating attestations (https://github.com/status-im/nimbus-eth2/pull/2631) + + 2021-05-17 v1.3.0 ================= @@ -705,4 +737,3 @@ the Eth2 mainnet launch. * Support for storing the validator signing keys in a separate process, isolated from the network, with a minimal attack surface. - diff --git a/beacon_chain/version.nim b/beacon_chain/version.nim index 397f10d81..95ea89568 100644 --- a/beacon_chain/version.nim +++ b/beacon_chain/version.nim @@ -15,7 +15,7 @@ when not defined(nimscript): const versionMajor* = 1 - versionMinor* = 3 + versionMinor* = 4 versionBuild* = 0 versionBlob* = "stateofus" # Single word - ends up in the default graffitti @@ -43,4 +43,3 @@ func shortNimBanner*(): string = tmp[0] & " (" & gitHash & ")" else: tmp[0] -