status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Minify slashing protection before SQLite (#3393)

This commit is contained in:
Mamy Ratsimbazafy 2022-03-04 15:43:34 +01:00 committed by GitHub
parent 8967f9cf01
commit ef7e8bdbd2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 133 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
AllTests-mainnet.md
View File

@ -311,7 +311,7 @@ OK: 12/12 Fail: 0/12 Skip: 0/12
+ Slashing test: multiple_interchanges_overlapping_validators_repeat_idem.json OK
+ Slashing test: multiple_interchanges_single_validator_fail_iff_imported.json OK
+ Slashing test: multiple_interchanges_single_validator_first_surrounds_second.json OK
Slashing test: multiple_interchanges_single_validator_multiple_blocks_out_of_order.json Skip
+ Slashing test: multiple_interchanges_single_validator_multiple_blocks_out_of_order.json OK
+ Slashing test: multiple_interchanges_single_validator_second_surrounds_first.json OK
+ Slashing test: multiple_interchanges_single_validator_single_att_out_of_order.json OK
+ Slashing test: multiple_interchanges_single_validator_single_block_out_of_order.json OK
@ -324,7 +324,7 @@ OK: 12/12 Fail: 0/12 Skip: 0/12
+ Slashing test: single_validator_multiple_blocks_and_attestations.json OK
+ Slashing test: single_validator_out_of_order_attestations.json OK
+ Slashing test: single_validator_out_of_order_blocks.json OK
+ Slashing test: single_validator_resign_attestation.json OK
Slashing test: single_validator_resign_attestation.json Skip
+ Slashing test: single_validator_resign_block.json OK
+ Slashing test: single_validator_single_attestation.json OK
+ Slashing test: single_validator_single_block.json OK
@ -337,18 +337,12 @@ OK: 12/12 Fail: 0/12 Skip: 0/12
+ Slashing test: single_validator_slashable_blocks_no_root.json OK
+ Slashing test: single_validator_source_greater_than_target.json OK
+ Slashing test: single_validator_source_greater_than_target_sensible_iff_minified.json OK
+ Slashing test: single_validator_source_greater_than_target_surrounded.json OK
Slashing test: single_validator_source_greater_than_target_surrounded.json Skip
Slashing test: single_validator_source_greater_than_target_surrounding.json Skip
+ Slashing test: single_validator_two_blocks_no_signing_root.json OK
+ Slashing test: wrong_genesis_validators_root.json OK
```
OK: 36/38 Fail: 0/38 Skip: 2/38
## Slashing Protection DB - Interchange [Preset: mainnet]
```diff
+ Smoke test - Complete format - Invalid database is refused [Preset: mainnet] OK
+ Smoke test - Complete format [Preset: mainnet] OK
```
OK: 2/2 Fail: 0/2 Skip: 0/2
OK: 35/38 Fail: 0/38 Skip: 3/38
## Slashing Protection DB [Preset: mainnet]
```diff
+ Attestation ordering #1698 OK
@ -519,4 +513,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
OK: 1/1 Fail: 0/1 Skip: 0/1
---TOTAL---
OK: 285/289 Fail: 0/289 Skip: 4/289
OK: 282/287 Fail: 0/287 Skip: 5/287

100
beacon_chain/validators/slashing_protection_common.nim
View File

@ -294,6 +294,7 @@ proc importInterchangeV5Impl*(
continue
key.get()
# TODO: with minification sorting is unnecessary, cleanup
# Sort by ascending minimum slot so that we don't trigger MinSlotViolation
spdir.data[v].signed_blocks.sort do (a, b: SPDIR_SignedBlock) -> int:
result = cmp(a.slot.int, b.slot.int)
@ -305,6 +306,8 @@ proc importInterchangeV5Impl*(
const ZeroDigest = Eth2Digest()
let (dbSlot, dbSource, dbTarget) = db.retrieveLatestValidatorData(parsedKey)
# Blocks
# ---------------------------------------------------
# After import we need to prune the DB from everything
@ -312,9 +315,12 @@ proc importInterchangeV5Impl*(
# This ensures that even if 2 slashing DB are imported in the wrong order
# (the last before the earliest) the minSlotViolation check stays consistent.
var maxValidSlotSeen = -1
if dbSlot.isSome():
maxValidSlotSeen = int dbSlot.get()
for b in 0 ..< spdir.data[v].signed_blocks.len:
template B: untyped = spdir.data[v].signed_blocks[b]
if spdir.data[v].signed_blocks.len >= 1:
# Minification, to limit Sqlite IO we only import the last block after sorting
template B: untyped = spdir.data[v].signed_blocks[^1]
let status = db.registerBlock(
parsedKey, B.slot.Slot, B.signing_root.Eth2Digest
)
@ -332,20 +338,19 @@ proc importInterchangeV5Impl*(
warn "Block already exists in the DB",
pubkey = spdir.data[v].pubkey.PubKeyBytes.toHex(),
candidateBlock = B
continue
else:
error "Slashable block. Skipping its import.",
pubkey = spdir.data[v].pubkey.PubKeyBytes.toHex(),
candidateBlock = B,
conflict = status.error()
result = siPartial
continue
if B.slot.int > maxValidSlotSeen:
maxValidSlotSeen = B.slot.int
maxValidSlotSeen = int B.slot
# Now prune everything that predates
# this interchange file max slot
# this DB or interchange file max slot
# Even if the block is not imported, pruning will keep the latest one.
db.pruneBlocks(parsedKey, Slot maxValidSlotSeen)
# Attestations
@ -356,76 +361,43 @@ proc importInterchangeV5Impl*(
# (the last before the earliest) the minEpochViolation check stays consistent.
var maxValidSourceEpochSeen = -1
var maxValidTargetEpochSeen = -1
if dbSource.isSome():
maxValidSourceEpochSeen = int dbSource.get()
if dbTarget.isSome():
maxValidTargetEpochSeen = int dbTarget.get()
# We do a first pass over the data to find the max source/target seen
for a in 0 ..< spdir.data[v].signed_attestations.len:
template A: untyped = spdir.data[v].signed_attestations[a]
let status = db.registerAttestation(
parsedKey,
A.source_epoch.Epoch,
A.target_epoch.Epoch,
A.signing_root.Eth2Digest
)
if status.isErr():
# We might be importing a duplicate which EIP-3076 allows
# there is no reason during normal operation to integrate
# a duplicate so checkSlashableAttestation would have rejected it.
# We special-case that for imports.
if status.error.kind == DoubleVote and
A.signing_root.Eth2Digest != ZeroDigest and
status.error.existingAttestation == A.signing_root.Eth2Digest:
warn "Attestation already exists in the DB",
pubkey = spdir.data[v].pubkey.PubKeyBytes.toHex(),
candidateAttestation = A,
conflict = status.error()
continue
elif status.error.kind == SurroundVote:
doAssert A.source_epoch.Epoch == status.error.sourceSlashable
doAssert A.target_epoch.Epoch == status.error.targetSlashable
# Formal proof of correctness: https://github.com/michaelsproul/slashing-proofs
let synth = SPDIR_SignedAttestation(
source_epoch: EpochString max(status.error.sourceSlashable, status.error.sourceExisting),
target_epoch: EpochString max(status.error.targetSlashable, status.error.targetExisting)
)
warn "Slashable surround vote. Constructing a synthetic attestation to reconcile DB and import",
pubkey = spdir.data[v].pubkey.PubKeyBytes.toHex(),
candidateAttestation = A,
conflict = status.error(),
syntheticAttestation = synth
db.registerSyntheticAttestation(
parsedKey,
synth.source_epoch.Epoch,
synth.target_epoch.Epoch
)
if synth.source_epoch.int > maxValidSourceEpochSeen:
maxValidSourceEpochSeen = synth.source_epoch.int
if synth.target_epoch.int > maxValidTargetEpochSeen:
maxValidTargetEpochSeen = synth.target_epoch.int
result = siPartial
continue
else:
error "Slashable vote. Skipping its import.",
pubkey = spdir.data[v].pubkey.PubKeyBytes.toHex(),
candidateAttestation = A,
conflict = status.error()
result = siPartial
continue
if A.source_epoch.int > maxValidSourceEpochSeen:
maxValidSourceEpochSeen = A.source_epoch.int
if A.target_epoch.int > maxValidTargetEpochSeen:
maxValidTargetEpochSeen = A.target_epoch.int
# Now prune everything that predates
# this interchange file max slot
if maxValidSourceEpochSeen < 0 or maxValidTargetEpochSeen < 0:
doAssert maxValidSourceEpochSeen == -1 and maxValidTargetEpochSeen == -1
notice "No attestation found in slashing interchange file for validator",
pubkey = spdir.data[v].pubkey.PubKeyBytes.toHex()
continue
# See formal proof https://github.com/michaelsproul/slashing-proofs
# of synthetic attestation
if not(maxValidSourceEpochSeen < maxValidTargetEpochSeen) and
not(maxValidSourceEpochSeen == 0 and maxValidTargetEpochSeen == 0):
# Special-case genesis (Slashing prot is deactivated anyway)
warn "Invalid attestation(s), source epochs should be less than target epochs, skipping import",
pubkey = spdir.data[v].pubkey.PubKeyBytes.toHex(),
maxValidSourceEpochSeen = maxValidSourceEpochSeen,
maxValidTargetEpochSeen = maxValidTargetEpochSeen
result = siPartial
continue
db.registerSyntheticAttestation(
parsedKey,
Epoch maxValidSourceEpochSeen,
Epoch maxValidTargetEpochSeen
)
db.pruneAttestations(parsedKey, maxValidSourceEpochSeen, maxValidTargetEpochSeen)

65
beacon_chain/validators/slashing_protection_v2.nim
View File

@ -219,6 +219,7 @@ type
sqlAttMinSourceTargetEpochs: SqliteStmt[ValidatorInternalID, (int64, int64)]
sqlBlockForSameSlot: SqliteStmt[(ValidatorInternalID, int64), Hash32]
sqlBlockMinSlot: SqliteStmt[ValidatorInternalID, int64]
sqlMaxBlockAtt: SqliteStmt[ValidatorInternalID, (int64, int64, int64)]
internalIds: Table[ValidatorIndex, ValidatorInternalID]
@ -245,16 +246,8 @@ template dispose(sqlStmt: SqliteStmt) =
proc setupDB(db: SlashingProtectionDB_v2, genesis_validators_root: Eth2Digest) =
## Initial setup of the DB
# Naming:
# - We use the same naming as https://eips.ethereum.org/EIPS/eip-3076
# and Lighthouse to allow loading/exporting without the Intermediate
# interchange format (provided we agree on a metadata format as well)
#
# - https://github.com/sigp/lighthouse/blob/v1.1.0/validator_client/slashing_protection/src/slashing_database.rs#L59-L88
#
# Differences
# - Lighthouse uses public_key instead of pubkey as in spec
# TODO - the Metadata table is a remnant from the v1 of the DB and should be removed
block: # Metadata
db.backend.exec("""
CREATE TABLE metadata(
@ -590,6 +583,21 @@ proc setupCachedQueries(db: SlashingProtectionDB_v2) =
""", ValidatorInternalID, void).get()
db.sqlCommitTransaction = db.backend.prepareStmt("""COMMIT TRANSACTION;""", NoParams, void).get()
db.sqlMaxBlockAtt = db.backend.prepareStmt("""
SELECT
MAX(slot), MAX(source_epoch), MAX(target_epoch)
FROM
validators v
LEFT JOIN
signed_blocks b on v.id = b.validator_id
LEFT JOIN
signed_attestations a on v.id = a.validator_id
WHERE
id = ?
GROUP BY
NULL
""", ValidatorInternalID, (int64, int64, int64)).get()
# DB Multiversioning
# -------------------------------------------------------------
@ -1235,12 +1243,49 @@ proc pruneAfterFinalization*(
# Interchange
# --------------------------------------------
proc retrieveLatestValidatorData*(
db: SlashingProtectionDB_v2,
validator: ValidatorPubkey
): tuple[
maxBlockSlot: Option[Slot],
maxAttSourceEpoch: Option[Epoch],
maxAttTargetEpoch: Option[Epoch]] =
let valID = db.getOrRegisterValidator(none(ValidatorIndex), validator)
var slot, source, target: int64
let status = db.sqlMaxBlockAtt.exec(
valID
) do (res: tuple[slot, source, target: int64]):
slot = res.slot
source = res.source
target = res.target
doAssert status.isOk(),
"SQLite error when querying validator: " & $status.error & "\n" &
"for validatorID " & $valID & " (0x" & $validator & ")"
# TODO: sqlite partial results ugly kludge
# if we find blocks but no attestation
# source and target would be set to 0 (from NULL in sqlite)
# 0 isn't an issue since it refers to Genesis (is it possible to have genesis epoch != 0?)
# but let's deal with those here
if slot != 0:
result.maxBlockSlot = some(Slot slot)
if source != 0:
result.maxAttSourceEpoch = some(Epoch source)
if target != 0:
result.maxAttTargetEpoch = some(Epoch target)
proc registerSyntheticAttestation*(
db: SlashingProtectionDB_v2,
validator: ValidatorPubKey,
source, target: Epoch) =
## Add a synthetic attestation to the slashing protection DB
doAssert source < target
# Spec require source < target (except genesis?), for synthetic attestation for slashing protection we want max(source, target)
doAssert (source < target) or (source == Epoch(0) and target == Epoch(0))
let valID = db.getOrRegisterValidator(none(ValidatorIndex), validator)

1
tests/all_tests.nim
View File

@ -41,7 +41,6 @@ import # Unit test
./fork_choice/tests_fork_choice,
./consensus_spec/all_tests as consensus_all_tests,
./slashing_protection/test_fixtures,
./slashing_protection/test_slashing_interchange,
./slashing_protection/test_slashing_protection_db
import # Refactor state transition unit tests

43
tests/slashing_protection/test_fixtures.nim
View File

@ -173,7 +173,7 @@ proc runTest(identifier: string) =
"Unexpected error:\n" &
" " & $status & "\n"
elif step.contains_slashable_data:
doAssert siPartial == status,
doAssert status in {siPartial, siSuccess},
"Unexpected error:\n" &
" " & $status & "\n"
else:
@ -182,8 +182,9 @@ proc runTest(identifier: string) =
" " & $status & "\n"
for blck in step.blocks:
let pubkey = ValidatorPubKey.fromRaw(blck.pubkey.PubKeyBytes).get()
let status = db.db_v2.checkSlashableBlockProposal(none(ValidatorIndex),
ValidatorPubKey.fromRaw(blck.pubkey.PubKeyBytes).get(),
pubkey,
Slot blck.slot
)
if blck.should_succeed:
@ -191,6 +192,18 @@ proc runTest(identifier: string) =
"Unexpected error:\n" &
" " & $status & "\n" &
" for " & $toHexLogs(blck)
# https://github.com/eth-clients/slashing-protection-interchange-tests/pull/14
# Successful blocks are to be incoporated in the DB
if status.isOk(): # Skip duplicates
let status = db.db_v2.registerBlock(
none(ValidatorIndex),
pubkey, Slot blck.slot,
Eth2Digest blck.signing_root
)
doAssert status.isOk(),
"Failure to register block: " & $status
else:
doAssert status.isErr(),
"Unexpected success:\n" &
@ -198,8 +211,10 @@ proc runTest(identifier: string) =
" for " & $toHexLogs(blck)
for att in step.attestations:
let pubkey = ValidatorPubKey.fromRaw(att.pubkey.PubKeyBytes).get()
let status = db.db_v2.checkSlashableAttestation(none(ValidatorIndex),
ValidatorPubKey.fromRaw(att.pubkey.PubKeyBytes).get(),
pubkey,
Epoch att.source_epoch,
Epoch att.target_epoch
)
@ -208,6 +223,19 @@ proc runTest(identifier: string) =
"Unexpected error:\n" &
" " & $status & "\n" &
" for " & $toHexLogs(att)
# https://github.com/eth-clients/slashing-protection-interchange-tests/pull/14
# Successful attestations are to be incoporated in the DB
if status.isOk(): # Skip duplicates
let status = db.db_v2.registerAttestation(
none(ValidatorIndex),
pubkey,
Epoch att.source_epoch,
Epoch att.target_epoch,
Eth2Digest att.signing_root
)
doAssert status.isOk(),
"Failure to register attestation: " & $status
else:
doAssert status.isErr(),
"Unexpected success:\n" &
@ -223,14 +251,17 @@ suite "Slashing Interchange tests " & preset():
InterchangeTestsDir, relative = true, checkDir = true):
test "Slashing test: " & path:
if path == "multiple_interchanges_single_validator_multiple_blocks_out_of_order.json":
# TODO: test relying on undocumented behavior (if signing a test block is possible import it)
# https://github.com/eth-clients/slashing-protection-interchange-tests/pull/12#issuecomment-1011158701
if path == "single_validator_source_greater_than_target_surrounded.json":
# TODO: test relying on invalid behavior source > target
skip()
elif path == "single_validator_source_greater_than_target_surrounding.json":
# TODO: test relying on unclear minification behavior:
# creating an invalid minified attestation with source > target
# or setting target = max(source, target)
skip()
elif path == "single_validator_resign_attestation.json":
# It's simpler to just disallow register an attestation twice for the same (source, target)
# rather than also checking the actual signing_root
skip()
else:
runTest(path)

125
tests/slashing_protection/test_slashing_interchange.nim
View File

@ -1,125 +0,0 @@
# Nimbus
# Copyright (c) 2018-2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or https://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or https://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
# Standard library
std/[os],
# Status lib
eth/db/[kvstore, kvstore_sqlite3],
stew/results,
nimcrypto/utils,
# Internal
../../beacon_chain/validators/[slashing_protection, slashing_protection_v2],
../../beacon_chain/spec/datatypes/base,
# Test utilies
../testutil
func fakeRoot(index: SomeInteger): Eth2Digest =
## Create fake roots
## Those are just the value serialized in big-endian
## We prevent zero hash special case via a power of 2 prefix
result.data[0 ..< 8] = (1'u64 shl 32 + index.uint64).toBytesBE()
func hexToDigest(hex: string): Eth2Digest =
Eth2Digest.fromHex(hex)
proc sqlite3db_delete(basepath, dbname: string) =
removeFile(basepath / dbname&".sqlite3-shm")
removeFile(basepath / dbname&".sqlite3-wal")
removeFile(basepath / dbname&".sqlite3")
const TestDir = ""
const TestDbName = "test_slashprot"
suite "Slashing Protection DB - Interchange" & preset():
# https://hackmd.io/@sproul/Bk0Y0qdGD#Format-1-Complete
# https://eips.ethereum.org/EIPS/eip-3076
sqlite3db_delete(TestDir, TestDbName)
test "Smoke test - Complete format" & preset():
let genesis_validators_root = hexToDigest"0x04700007fabc8282644aed6d1c7c9e21d38a03a0c4ba193f3afe428824b3a673"
block: # export
let db = SlashingProtectionDB.init(
genesis_validators_root,
TestDir,
TestDbName
)
defer:
db.close()
sqlite3db_delete(TestDir, TestDbName)
let pubkey = ValidatorPubKey
.fromHex"0xb845089a1457f811bfc000588fbb4e713669be8ce060ea6be3c6ece09afc3794106c91ca73acda5e5457122d58723bed"
.get()
check:
db.db_v2.registerBlock(
pubkey,
Slot 81952,
hexToDigest"0x4ff6f743a43f3b4f95350831aeaf0a122a1a392922c45d804280284a69eb850b"
).isOk()
# db.registerBlock(
# pubkey,
# Slot 81951,
# fakeRoot(65535)
# )
db.db_v2.registerAttestation(
pubkey,
source = Epoch 2290,
target = Epoch 3007,
hexToDigest"0x587d6a4f59a58fe24f406e0502413e77fe1babddee641fda30034ed37ecc884d"
).isOk()
db.db_v2.registerAttestation(
pubkey,
source = Epoch 2290,
target = Epoch 3008,
fakeRoot(65535)
).isOk()
db.exportSlashingInterchange(currentSourcePath.parentDir/"test_complete_export_slashing_protection.json")
block: # import - zero root db
let db2 = SlashingProtectionDB.init(
Eth2Digest(),
TestDir,
TestDbName
)
defer:
db2.close()
sqlite3db_delete(TestDir, TestDbName)
doAssert siSuccess == db2.importSlashingInterchange(currentSourcePath.parentDir/"test_complete_export_slashing_protection.json")
db2.exportSlashingInterchange(currentSourcePath.parentDir/"test_complete_export_slashing_protection_roundtrip1.json")
block: # import - same root db
let db3 = SlashingProtectionDB.init(
genesis_validators_root,
TestDir,
TestDbName
)
defer:
db3.close()
sqlite3db_delete(TestDir, TestDbName)
doAssert siSuccess == db3.importSlashingInterchange(currentSourcePath.parentDir/"test_complete_export_slashing_protection.json")
db3.exportSlashingInterchange(currentSourcePath.parentDir/"test_complete_export_slashing_protection_roundtrip2.json")
test "Smoke test - Complete format - Invalid database is refused" & preset():
block: # import - invalid root db
let invalid_genvalroot = hexToDigest"0x1234"
let db4 = SlashingProtectionDB.init(
invalid_genvalroot,
TestDir,
TestDbName
)
defer:
db4.close()
sqlite3db_delete(TestDir, TestDbName)
doAssert siFailure == db4.importSlashingInterchange(currentSourcePath.parentDir/"test_complete_export_slashing_protection.json")