Slashing db pruning [Merge only after v2 has been default for 1 noticeable release] (#2452)
* Enable slashing DB pruning * integrate slashing DB pruning with onSlotEnd * rebase tests
This commit is contained in:
Mamy Ratsimbazafy
GitHub
parent
f9b964ca5d
commit
e6b559a35a
|
|
@ -226,7 +226,11 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
|
|||
## Slashing Protection DB [Preset: mainnet]
|
||||
```diff
|
||||
+ Attestation ordering #1698 OK
|
||||
+ Don't prune the very last attestation(s) even by mistake OK
|
||||
+ Don't prune the very last block even by mistake OK
|
||||
+ Empty database [Preset: mainnet] OK
|
||||
+ Pruning attestations works OK
|
||||
+ Pruning blocks works OK
|
||||
+ SP for block proposal - backtracking append OK
|
||||
+ SP for block proposal - linear append OK
|
||||
+ SP for same epoch attestation target - linear append OK
|
||||
|
|
@ -234,7 +238,7 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
|
|||
+ SP for surrounding attestations OK
|
||||
+ Test valid attestation #1699 OK
|
||||
```
|
||||
OK: 8/8 Fail: 0/8 Skip: 0/8
|
||||
OK: 12/12 Fail: 0/12 Skip: 0/12
|
||||
## Spec datatypes
|
||||
```diff
|
||||
+ Graffiti bytes OK
|
||||
|
|
@ -319,4 +323,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
|
|||
OK: 1/1 Fail: 0/1 Skip: 0/1
|
||||
|
||||
---TOTAL---
|
||||
OK: 176/185 Fail: 0/185 Skip: 9/185
|
||||
OK: 180/189 Fail: 0/189 Skip: 9/189
|
||||
|
|
|
|||
|
|
@ -875,8 +875,19 @@ proc onSlotEnd(node: BeaconNode, slot: Slot) {.async.} =
|
|||
# Things we do when slot processing has ended and we're about to wait for the
|
||||
# next slot
|
||||
|
||||
if node.chainDag.needStateCachesAndForkChoicePruning():
|
||||
if node.attachedValidators.validators.len > 0:
|
||||
node.attachedValidators
|
||||
.slashingProtection
|
||||
# pruning is only done if the DB is set to pruning mode.
|
||||
.pruneAfterFinalization(
|
||||
node.chainDag.finalizedHead.slot.compute_epoch_at_slot()
|
||||
)
|
||||
|
||||
# Delay part of pruning until latency critical duties are done.
|
||||
# The other part of pruning, `pruneBlocksDAG`, is done eagerly.
|
||||
# ----
|
||||
# This is the last pruning to do as it clears the "needPruning" condition.
|
||||
node.consensusManager[].pruneStateCachesAndForkChoice()
|
||||
|
||||
when declared(GC_fullCollect):
|
||||
|
|
|
|||
|
|
@ -267,15 +267,19 @@ proc pruneAfterFinalization*(
|
|||
db: SlashingProtectionDB,
|
||||
finalizedEpoch: Epoch
|
||||
) =
|
||||
# TODO
|
||||
# call sqlPruneAfterFinalizationBlocks
|
||||
# and sqlPruneAfterFinalizationAttestations
|
||||
# and test that wherever pruning happens, tests still pass
|
||||
# and/or devise new tests
|
||||
## Prune blocks and attestations after a specified `finalizedEpoch`
|
||||
## The block with the highest slot
|
||||
## and the attestation(s) with the highest source and target epochs
|
||||
## are never pruned.
|
||||
##
|
||||
## This ensures that even if pruning is called with an incorrect epoch
|
||||
## slashing protection can fallback to the minimal / high-watermark protection mode.
|
||||
##
|
||||
## Pruning is only done if pruning is enabled (DB in kLowWatermarkV2 mode)
|
||||
## Pruning is only triggered on v2 database.
|
||||
|
||||
# {.error: "NotImplementedError".}
|
||||
fatal "Pruning is not implemented"
|
||||
quit 1
|
||||
if kLowWatermarkV2 in db.modes:
|
||||
db.db_v2.pruneAfterFinalization(finalizedEpoch)
|
||||
|
||||
# The high-level import/export functions are
|
||||
# - importSlashingInterchange
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import
|
|||
chronicles,
|
||||
sqlite3_abi,
|
||||
# Internal
|
||||
../spec/[datatypes, digest, crypto],
|
||||
../spec/[datatypes, digest, crypto, helpers],
|
||||
../ssz,
|
||||
./slashing_protection_common
|
||||
|
||||
|
|
@ -206,8 +206,8 @@ type
|
|||
sqlInsertBlock: SqliteStmt[(ValidatorInternalID, int64, Hash32), void]
|
||||
sqlPruneValidatorBlocks: SqliteStmt[(ValidatorInternalID, int64), void]
|
||||
sqlPruneValidatorAttestations: SqliteStmt[(ValidatorInternalID, int64, int64), void]
|
||||
sqlPruneAfterFinalizationBlocks: SqliteStmt[(ValidatorInternalID, int64), void]
|
||||
sqlPruneAfterFinalizationAttestations: SqliteStmt[(ValidatorInternalID, int64), void]
|
||||
sqlPruneAfterFinalizationBlocks: SqliteStmt[int64, void]
|
||||
sqlPruneAfterFinalizationAttestations: SqliteStmt[(int64, int64), void]
|
||||
# Cached queries - read
|
||||
sqlGetValidatorInternalID: SqliteStmt[PubKeyBytes, ValidatorInternalID]
|
||||
sqlAttForSameTargetEpoch: SqliteStmt[(ValidatorInternalID, int64), Hash32]
|
||||
|
|
@ -485,44 +485,42 @@ proc setupCachedQueries(db: SlashingProtectionDB_v2) =
|
|||
""", (ValidatorInternalID, int64, int64), void
|
||||
).get()
|
||||
|
||||
# TODO: test and activate pruning after finalization
|
||||
db.sqlPruneAfterFinalizationBlocks = db.backend.prepareStmt("""
|
||||
DELETE
|
||||
FROM
|
||||
signed_blocks AS sb1
|
||||
WHERE 1=1
|
||||
and sb1.slot < ?
|
||||
-- Keep the most recent slot per validator
|
||||
and sb1.slot <> (
|
||||
SELECT MAX(sb2.slot)
|
||||
FROM signed_blocks AS sb2
|
||||
WHERE sb2.validator_id = sb1.validator_id
|
||||
)
|
||||
""", int64, void
|
||||
).get()
|
||||
|
||||
# db.sqlPruneAfterFinalizationBlocks = db.backend.prepareStmt("""
|
||||
# DELETE
|
||||
# FROM
|
||||
# signed_blocks sb1
|
||||
# WHERE 1=1
|
||||
# and sb1.slot < ?
|
||||
# -- Keep the most recent slot per validator
|
||||
# and sb1.slot <> (
|
||||
# SELECT MAX(sb2.slot)
|
||||
# FROM signed_blocks AS sb2
|
||||
# WHERE sb2.validator_id = sb1.validator_id
|
||||
# )
|
||||
# """, (ValidatorInternalID, int64), void
|
||||
# ).get()
|
||||
#
|
||||
# db.sqlPruneAfterFinalizationAttestations = db.backend.prepareStmt("""
|
||||
# DELETE
|
||||
# FROM
|
||||
# signed_attestations
|
||||
# WHERE 1=1
|
||||
# and source_epoch < ?
|
||||
# and target_epoch < ?
|
||||
# -- Keep the most recent source_epoch per validator
|
||||
# and sa1.source_epoch <> (
|
||||
# SELECT MAX(sas.source_epoch)
|
||||
# FROM signed_attestations AS sas
|
||||
# WHERE sa1.validator_id = sas.validator_id
|
||||
# )
|
||||
# -- And the most recent target_epoch per validator
|
||||
# and sa1.target_epoch <> (
|
||||
# SELECT MAX(sat.target_epoch)
|
||||
# FROM signed_attestations AS sat
|
||||
# WHERE sa1.validator_id = sat.validator_id
|
||||
# )
|
||||
# """, (ValidatorInternalID, int64, int64), void
|
||||
# ).get()
|
||||
db.sqlPruneAfterFinalizationAttestations = db.backend.prepareStmt("""
|
||||
DELETE
|
||||
FROM
|
||||
signed_attestations AS sa1
|
||||
WHERE 1=1
|
||||
and source_epoch < ?
|
||||
and target_epoch < ?
|
||||
-- Keep the most recent source_epoch per validator
|
||||
and sa1.source_epoch <> (
|
||||
SELECT MAX(sas.source_epoch)
|
||||
FROM signed_attestations AS sas
|
||||
WHERE sa1.validator_id = sas.validator_id
|
||||
)
|
||||
-- And the most recent target_epoch per validator
|
||||
and sa1.target_epoch <> (
|
||||
SELECT MAX(sat.target_epoch)
|
||||
FROM signed_attestations AS sat
|
||||
WHERE sa1.validator_id = sat.validator_id
|
||||
)
|
||||
""", (int64, int64), void
|
||||
).get()
|
||||
|
||||
# DB Multiversioning
|
||||
# -------------------------------------------------------------
|
||||
|
|
@ -1136,15 +1134,30 @@ proc pruneAfterFinalization*(
|
|||
db: SlashingProtectionDB_v2,
|
||||
finalizedEpoch: Epoch
|
||||
) =
|
||||
warn "Slashing DB pruning after finalization is not supported on the v2 of our database. Request ignored.",
|
||||
finalizedEpoch = shortLog(finalizedEpoch)
|
||||
## Prune blocks and attestations after a specified `finalizedEpoch`
|
||||
## The block with the highest slot
|
||||
## and the attestation(s) with the highest source and target epochs
|
||||
## are never pruned.
|
||||
##
|
||||
## This ensures that even if pruning is called with an incorrect epoch
|
||||
## slashing protection can fallback to the minimal / high-watermark protection mode.
|
||||
|
||||
# TODO
|
||||
# call sqlPruneAfterFinalizationBlocks
|
||||
# and sqlPruneAfterFinalizationAttestations
|
||||
# and test that wherever pruning happens, tests still pass
|
||||
# and/or devise new tests
|
||||
block: # Prune blocks
|
||||
let finalizedSlot = compute_start_slot_at_epoch(finalizedEpoch)
|
||||
let status = db.sqlPruneAfterFinalizationBlocks
|
||||
.exec(int64 finalizedSlot)
|
||||
doAssert status.isOk(),
|
||||
"SQLite error when pruning validator attestations: " & $status.error & "\n" &
|
||||
"for " &
|
||||
"finalizedEpoch: " & $finalizedEpoch &
|
||||
", firstSlotOfFinalizedEpoch: " & $finalizedSlot
|
||||
|
||||
block: # Prune attestations
|
||||
let status = db.sqlPruneAfterFinalizationAttestations
|
||||
.exec((int64 finalizedEpoch, int64 finalizedEpoch))
|
||||
doAssert status.isOk(),
|
||||
"SQLite error when pruning validator attestations: " & $status.error & "\n" &
|
||||
"for finalized epoch: " & $finalizedEpoch
|
||||
|
||||
# Interchange
|
||||
# --------------------------------------------
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import
|
|||
stew/results,
|
||||
# Internal
|
||||
../../beacon_chain/validators/slashing_protection,
|
||||
../../beacon_chain/spec/[datatypes, digest, crypto, presets],
|
||||
../../beacon_chain/spec/[datatypes, digest, crypto, presets, helpers],
|
||||
# Test utilies
|
||||
../testutil
|
||||
|
||||
|
|
@ -625,3 +625,206 @@ suite "Slashing Protection DB" & preset():
|
|||
fakeValidator(100),
|
||||
Epoch 20, Epoch 30
|
||||
).isOk
|
||||
|
||||
test "Pruning blocks works":
|
||||
block:
|
||||
sqlite3db_delete(TestDir, TestDbName)
|
||||
let db = SlashingProtectionDB.init(
|
||||
default(Eth2Digest),
|
||||
TestDir,
|
||||
TestDbName
|
||||
)
|
||||
defer:
|
||||
db.close()
|
||||
sqlite3db_delete(TestDir, TestDbName)
|
||||
|
||||
db.registerBlock(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Slot 10,
|
||||
fakeRoot(10)
|
||||
).expect("registered block")
|
||||
db.registerBlock(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Slot 1000,
|
||||
fakeRoot(20)
|
||||
).expect("registered block")
|
||||
|
||||
# After pruning, duplicate becomes a min slot violation
|
||||
doAssert db.checkSlashableBlockProposal(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Slot 10,
|
||||
).error.kind == DoubleProposal
|
||||
|
||||
db.pruneAfterFinalization(
|
||||
compute_epoch_at_slot(Slot 1000)
|
||||
)
|
||||
|
||||
doAssert db.checkSlashableBlockProposal(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Slot 10,
|
||||
).error.kind == MinSlotViolation
|
||||
|
||||
test "Don't prune the very last block even by mistake":
|
||||
block:
|
||||
sqlite3db_delete(TestDir, TestDbName)
|
||||
let db = SlashingProtectionDB.init(
|
||||
default(Eth2Digest),
|
||||
TestDir,
|
||||
TestDbName
|
||||
)
|
||||
defer:
|
||||
db.close()
|
||||
sqlite3db_delete(TestDir, TestDbName)
|
||||
|
||||
db.registerBlock(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Slot 10,
|
||||
fakeRoot(10)
|
||||
).expect("registered block")
|
||||
db.registerBlock(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Slot 1000,
|
||||
fakeRoot(20)
|
||||
).expect("registered block")
|
||||
|
||||
doAssert db.checkSlashableBlockProposal(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Slot 1000,
|
||||
).error.kind == DoubleProposal
|
||||
|
||||
# Pruning far in the future
|
||||
db.pruneAfterFinalization(
|
||||
compute_epoch_at_slot(Slot 10000)
|
||||
)
|
||||
|
||||
# Last block is still there
|
||||
doAssert db.checkSlashableBlockProposal(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Slot 1000,
|
||||
).error.kind == DoubleProposal
|
||||
|
||||
test "Pruning attestations works":
|
||||
block:
|
||||
sqlite3db_delete(TestDir, TestDbName)
|
||||
let db = SlashingProtectionDB.init(
|
||||
default(Eth2Digest),
|
||||
TestDir,
|
||||
TestDbName
|
||||
)
|
||||
defer:
|
||||
db.close()
|
||||
sqlite3db_delete(TestDir, TestDbName)
|
||||
|
||||
|
||||
db.registerAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 10, Epoch 20,
|
||||
fakeRoot(20)
|
||||
).expect("registered block")
|
||||
|
||||
db.registerAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 40, Epoch 50,
|
||||
fakeRoot(50)
|
||||
).expect("registered block")
|
||||
|
||||
# After pruning, duplicate becomes a min source epoch violation
|
||||
doAssert db.checkSlashableAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 10, Epoch 20
|
||||
).error.kind == DoubleVote
|
||||
|
||||
# After pruning, surrounding vote becomes a min source epoch violation
|
||||
doAssert db.checkSlashableAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 9, Epoch 21
|
||||
).error.kind == SurroundVote
|
||||
|
||||
# After pruning, surrounded vote becomes a min source epoch violation
|
||||
doAssert db.checkSlashableAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 11, Epoch 19
|
||||
).error.kind == SurroundVote
|
||||
|
||||
# --------------------------------
|
||||
db.pruneAfterFinalization(
|
||||
Epoch 40
|
||||
)
|
||||
# --------------------------------
|
||||
|
||||
doAssert db.checkSlashableAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 10, Epoch 20
|
||||
).error.kind == MinSourceViolation
|
||||
|
||||
doAssert db.checkSlashableAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 9, Epoch 21
|
||||
).error.kind == MinSourceViolation
|
||||
|
||||
doAssert db.checkSlashableAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 11, Epoch 19
|
||||
).error.kind == MinSourceViolation
|
||||
|
||||
# TODO is it possible to actually trigger MinTargetViolation
|
||||
# given all the other constraints?
|
||||
|
||||
test "Don't prune the very last attestation(s) even by mistake":
|
||||
block:
|
||||
sqlite3db_delete(TestDir, TestDbName)
|
||||
let db = SlashingProtectionDB.init(
|
||||
default(Eth2Digest),
|
||||
TestDir,
|
||||
TestDbName
|
||||
)
|
||||
defer:
|
||||
db.close()
|
||||
sqlite3db_delete(TestDir, TestDbName)
|
||||
|
||||
db.registerAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 10, Epoch 20,
|
||||
fakeRoot(20)
|
||||
).expect("registered block")
|
||||
|
||||
db.registerAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 40, Epoch 50,
|
||||
fakeRoot(50)
|
||||
).expect("registered block")
|
||||
|
||||
# --------------------------------
|
||||
db.pruneAfterFinalization(
|
||||
compute_epoch_at_slot(Slot 10000)
|
||||
)
|
||||
# --------------------------------
|
||||
|
||||
doAssert db.checkSlashableAttestation(
|
||||
ValidatorIndex(100),
|
||||
fakeValidator(100),
|
||||
Epoch 40, Epoch 50
|
||||
).error.kind == DoubleVote
|
||||
|
||||
# TODO is it possible to actually to have
|
||||
# the MAX(SourceEpoch) and MAX(TargetEpoch)
|
||||
# on 2 different attestations
|
||||
# given all the other constraints?
|
||||
|
|
|
|||
Loading…
Reference in New Issue