status-im/nimbus-eth2
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth2.git synced 2025-01-10 22:36:01 +00:00
Code Issues Projects Releases Wiki Activity

Slashing db pruning [Merge only after v2 has been default for 1 noticeable release] (#2452)

Browse Source 
* Enable slashing DB pruning

* integrate slashing DB pruning with onSlotEnd

* rebase tests
This commit is contained in:
Mamy Ratsimbazafy 2021-05-10 16:32:28 +02:00 committed by GitHub
parent f9b964ca5d
commit e6b559a35a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 293 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
AllTests-mainnet.md
View File

@ -226,7 +226,11 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
## Slashing Protection DB [Preset: mainnet]
```diff
+ Attestation ordering #1698 OK
+ Don't prune the very last attestation(s) even by mistake OK
+ Don't prune the very last block even by mistake OK
+ Empty database [Preset: mainnet] OK
+ Pruning attestations works OK
+ Pruning blocks works OK
+ SP for block proposal - backtracking append OK
+ SP for block proposal - linear append OK
+ SP for same epoch attestation target - linear append OK
@ -234,7 +238,7 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
+ SP for surrounding attestations OK
+ Test valid attestation #1699 OK
```
OK: 8/8 Fail: 0/8 Skip: 0/8
OK: 12/12 Fail: 0/12 Skip: 0/12
## Spec datatypes
```diff
+ Graffiti bytes OK
@ -319,4 +323,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
OK: 1/1 Fail: 0/1 Skip: 0/1
---TOTAL---
OK: 176/185 Fail: 0/185 Skip: 9/185
OK: 180/189 Fail: 0/189 Skip: 9/189

11
beacon_chain/nimbus_beacon_node.nim
View File

@ -875,8 +875,19 @@ proc onSlotEnd(node: BeaconNode, slot: Slot) {.async.} =
# Things we do when slot processing has ended and we're about to wait for the
# next slot
if node.chainDag.needStateCachesAndForkChoicePruning():
if node.attachedValidators.validators.len > 0:
node.attachedValidators
.slashingProtection
# pruning is only done if the DB is set to pruning mode.
.pruneAfterFinalization(
node.chainDag.finalizedHead.slot.compute_epoch_at_slot()
)
# Delay part of pruning until latency critical duties are done.
# The other part of pruning, `pruneBlocksDAG`, is done eagerly.
# ----
# This is the last pruning to do as it clears the "needPruning" condition.
node.consensusManager[].pruneStateCachesAndForkChoice()
when declared(GC_fullCollect):

20
beacon_chain/validators/slashing_protection.nim
View File

@ -267,15 +267,19 @@ proc pruneAfterFinalization*(
db: SlashingProtectionDB,
finalizedEpoch: Epoch
) =
# TODO
# call sqlPruneAfterFinalizationBlocks
# and sqlPruneAfterFinalizationAttestations
# and test that wherever pruning happens, tests still pass
# and/or devise new tests
## Prune blocks and attestations after a specified `finalizedEpoch`
## The block with the highest slot
## and the attestation(s) with the highest source and target epochs
## are never pruned.
##
## This ensures that even if pruning is called with an incorrect epoch
## slashing protection can fallback to the minimal / high-watermark protection mode.
##
## Pruning is only done if pruning is enabled (DB in kLowWatermarkV2 mode)
## Pruning is only triggered on v2 database.
# {.error: "NotImplementedError".}
fatal "Pruning is not implemented"
quit 1
if kLowWatermarkV2 in db.modes:
db.db_v2.pruneAfterFinalization(finalizedEpoch)
# The high-level import/export functions are
# - importSlashingInterchange

107
beacon_chain/validators/slashing_protection_v2.nim
View File

@ -16,7 +16,7 @@ import
chronicles,
sqlite3_abi,
# Internal
../spec/[datatypes, digest, crypto],
../spec/[datatypes, digest, crypto, helpers],
../ssz,
./slashing_protection_common
@ -206,8 +206,8 @@ type
sqlInsertBlock: SqliteStmt[(ValidatorInternalID, int64, Hash32), void]
sqlPruneValidatorBlocks: SqliteStmt[(ValidatorInternalID, int64), void]
sqlPruneValidatorAttestations: SqliteStmt[(ValidatorInternalID, int64, int64), void]
sqlPruneAfterFinalizationBlocks: SqliteStmt[(ValidatorInternalID, int64), void]
sqlPruneAfterFinalizationAttestations: SqliteStmt[(ValidatorInternalID, int64), void]
sqlPruneAfterFinalizationBlocks: SqliteStmt[int64, void]
sqlPruneAfterFinalizationAttestations: SqliteStmt[(int64, int64), void]
# Cached queries - read
sqlGetValidatorInternalID: SqliteStmt[PubKeyBytes, ValidatorInternalID]
sqlAttForSameTargetEpoch: SqliteStmt[(ValidatorInternalID, int64), Hash32]
@ -485,44 +485,42 @@ proc setupCachedQueries(db: SlashingProtectionDB_v2) =
""", (ValidatorInternalID, int64, int64), void
).get()
# TODO: test and activate pruning after finalization
db.sqlPruneAfterFinalizationBlocks = db.backend.prepareStmt("""
DELETE
FROM
signed_blocks AS sb1
WHERE 1=1
and sb1.slot < ?
-- Keep the most recent slot per validator
and sb1.slot <> (
SELECT MAX(sb2.slot)
FROM signed_blocks AS sb2
WHERE sb2.validator_id = sb1.validator_id
)
""", int64, void
).get()
# db.sqlPruneAfterFinalizationBlocks = db.backend.prepareStmt("""
# DELETE
# FROM
# signed_blocks sb1
# WHERE 1=1
# and sb1.slot < ?
# -- Keep the most recent slot per validator
# and sb1.slot <> (
# SELECT MAX(sb2.slot)
# FROM signed_blocks AS sb2
# WHERE sb2.validator_id = sb1.validator_id
# )
# """, (ValidatorInternalID, int64), void
# ).get()
#
# db.sqlPruneAfterFinalizationAttestations = db.backend.prepareStmt("""
# DELETE
# FROM
# signed_attestations
# WHERE 1=1
# and source_epoch < ?
# and target_epoch < ?
# -- Keep the most recent source_epoch per validator
# and sa1.source_epoch <> (
# SELECT MAX(sas.source_epoch)
# FROM signed_attestations AS sas
# WHERE sa1.validator_id = sas.validator_id
# )
# -- And the most recent target_epoch per validator
# and sa1.target_epoch <> (
# SELECT MAX(sat.target_epoch)
# FROM signed_attestations AS sat
# WHERE sa1.validator_id = sat.validator_id
# )
# """, (ValidatorInternalID, int64, int64), void
# ).get()
db.sqlPruneAfterFinalizationAttestations = db.backend.prepareStmt("""
DELETE
FROM
signed_attestations AS sa1
WHERE 1=1
and source_epoch < ?
and target_epoch < ?
-- Keep the most recent source_epoch per validator
and sa1.source_epoch <> (
SELECT MAX(sas.source_epoch)
FROM signed_attestations AS sas
WHERE sa1.validator_id = sas.validator_id
)
-- And the most recent target_epoch per validator
and sa1.target_epoch <> (
SELECT MAX(sat.target_epoch)
FROM signed_attestations AS sat
WHERE sa1.validator_id = sat.validator_id
)
""", (int64, int64), void
).get()
# DB Multiversioning
# -------------------------------------------------------------
@ -1136,15 +1134,30 @@ proc pruneAfterFinalization*(
db: SlashingProtectionDB_v2,
finalizedEpoch: Epoch
) =
warn "Slashing DB pruning after finalization is not supported on the v2 of our database. Request ignored.",
finalizedEpoch = shortLog(finalizedEpoch)
## Prune blocks and attestations after a specified `finalizedEpoch`
## The block with the highest slot
## and the attestation(s) with the highest source and target epochs
## are never pruned.
##
## This ensures that even if pruning is called with an incorrect epoch
## slashing protection can fallback to the minimal / high-watermark protection mode.
# TODO
# call sqlPruneAfterFinalizationBlocks
# and sqlPruneAfterFinalizationAttestations
# and test that wherever pruning happens, tests still pass
# and/or devise new tests
block: # Prune blocks
let finalizedSlot = compute_start_slot_at_epoch(finalizedEpoch)
let status = db.sqlPruneAfterFinalizationBlocks
.exec(int64 finalizedSlot)
doAssert status.isOk(),
"SQLite error when pruning validator attestations: " & $status.error & "\n" &
"for " &
"finalizedEpoch: " & $finalizedEpoch &
", firstSlotOfFinalizedEpoch: " & $finalizedSlot
block: # Prune attestations
let status = db.sqlPruneAfterFinalizationAttestations
.exec((int64 finalizedEpoch, int64 finalizedEpoch))
doAssert status.isOk(),
"SQLite error when pruning validator attestations: " & $status.error & "\n" &
"for finalized epoch: " & $finalizedEpoch
# Interchange
# --------------------------------------------

205
tests/slashing_protection/test_slashing_protection_db.nim
View File

@ -15,7 +15,7 @@ import
stew/results,
# Internal
../../beacon_chain/validators/slashing_protection,
../../beacon_chain/spec/[datatypes, digest, crypto, presets],
../../beacon_chain/spec/[datatypes, digest, crypto, presets, helpers],
# Test utilies
../testutil
@ -625,3 +625,206 @@ suite "Slashing Protection DB" & preset():
fakeValidator(100),
Epoch 20, Epoch 30
).isOk
test "Pruning blocks works":
block:
sqlite3db_delete(TestDir, TestDbName)
let db = SlashingProtectionDB.init(
default(Eth2Digest),
TestDir,
TestDbName
)
defer:
db.close()
sqlite3db_delete(TestDir, TestDbName)
db.registerBlock(
ValidatorIndex(100),
fakeValidator(100),
Slot 10,
fakeRoot(10)
).expect("registered block")
db.registerBlock(
ValidatorIndex(100),
fakeValidator(100),
Slot 1000,
fakeRoot(20)
).expect("registered block")
# After pruning, duplicate becomes a min slot violation
doAssert db.checkSlashableBlockProposal(
ValidatorIndex(100),
fakeValidator(100),
Slot 10,
).error.kind == DoubleProposal
db.pruneAfterFinalization(
compute_epoch_at_slot(Slot 1000)
)
doAssert db.checkSlashableBlockProposal(
ValidatorIndex(100),
fakeValidator(100),
Slot 10,
).error.kind == MinSlotViolation
test "Don't prune the very last block even by mistake":
block:
sqlite3db_delete(TestDir, TestDbName)
let db = SlashingProtectionDB.init(
default(Eth2Digest),
TestDir,
TestDbName
)
defer:
db.close()
sqlite3db_delete(TestDir, TestDbName)
db.registerBlock(
ValidatorIndex(100),
fakeValidator(100),
Slot 10,
fakeRoot(10)
).expect("registered block")
db.registerBlock(
ValidatorIndex(100),
fakeValidator(100),
Slot 1000,
fakeRoot(20)
).expect("registered block")
doAssert db.checkSlashableBlockProposal(
ValidatorIndex(100),
fakeValidator(100),
Slot 1000,
).error.kind == DoubleProposal
# Pruning far in the future
db.pruneAfterFinalization(
compute_epoch_at_slot(Slot 10000)
)
# Last block is still there
doAssert db.checkSlashableBlockProposal(
ValidatorIndex(100),
fakeValidator(100),
Slot 1000,
).error.kind == DoubleProposal
test "Pruning attestations works":
block:
sqlite3db_delete(TestDir, TestDbName)
let db = SlashingProtectionDB.init(
default(Eth2Digest),
TestDir,
TestDbName
)
defer:
db.close()
sqlite3db_delete(TestDir, TestDbName)
db.registerAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 10, Epoch 20,
fakeRoot(20)
).expect("registered block")
db.registerAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 40, Epoch 50,
fakeRoot(50)
).expect("registered block")
# After pruning, duplicate becomes a min source epoch violation
doAssert db.checkSlashableAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 10, Epoch 20
).error.kind == DoubleVote
# After pruning, surrounding vote becomes a min source epoch violation
doAssert db.checkSlashableAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 9, Epoch 21
).error.kind == SurroundVote
# After pruning, surrounded vote becomes a min source epoch violation
doAssert db.checkSlashableAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 11, Epoch 19
).error.kind == SurroundVote
# --------------------------------
db.pruneAfterFinalization(
Epoch 40
)
# --------------------------------
doAssert db.checkSlashableAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 10, Epoch 20
).error.kind == MinSourceViolation
doAssert db.checkSlashableAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 9, Epoch 21
).error.kind == MinSourceViolation
doAssert db.checkSlashableAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 11, Epoch 19
).error.kind == MinSourceViolation
# TODO is it possible to actually trigger MinTargetViolation
# given all the other constraints?
test "Don't prune the very last attestation(s) even by mistake":
block:
sqlite3db_delete(TestDir, TestDbName)
let db = SlashingProtectionDB.init(
default(Eth2Digest),
TestDir,
TestDbName
)
defer:
db.close()
sqlite3db_delete(TestDir, TestDbName)
db.registerAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 10, Epoch 20,
fakeRoot(20)
).expect("registered block")
db.registerAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 40, Epoch 50,
fakeRoot(50)
).expect("registered block")
# --------------------------------
db.pruneAfterFinalization(
compute_epoch_at_slot(Slot 10000)
)
# --------------------------------
doAssert db.checkSlashableAttestation(
ValidatorIndex(100),
fakeValidator(100),
Epoch 40, Epoch 50
).error.kind == DoubleVote
# TODO is it possible to actually to have
# the MAX(SourceEpoch) and MAX(TargetEpoch)
# on 2 different attestations
# given all the other constraints?