status-im/nimbus-eth2
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth2.git synced 2025-01-22 04:24:05 +00:00
Code Issues Projects Releases Wiki Activity

support readOnly backend in LC client/data dbs (#4518)

Browse Source 
When accessing DB in `readOnly` mode that does not already have latest
schema, initial writes trigger `attempt to write a readonly database`.
Avoid that by only writing schema when DB is not `readOnly`, and provide
data from legacy tables if such are present.
This commit is contained in:
Etan Kissling 2023-01-18 03:06:23 +01:00 committed by GitHub
parent 073c544f0c
commit 9cceb1b4a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 121 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
beacon_chain/beacon_chain_db_light_client.nim
View File

@ -52,6 +52,7 @@ type
keepFromStmt: SqliteStmt[int64, void] keepFromStmt: SqliteStmt[int64, void]
LegacyBestLightClientUpdateStore = object LegacyBestLightClientUpdateStore = object
getStmt: SqliteStmt[int64, (int64, seq[byte])]
putStmt: SqliteStmt[(int64, seq[byte]), void] putStmt: SqliteStmt[(int64, seq[byte]), void]
delStmt: SqliteStmt[int64, void] delStmt: SqliteStmt[int64, void]
delFromStmt: SqliteStmt[int64, void] delFromStmt: SqliteStmt[int64, void]
@ -94,9 +95,17 @@ type
## Tracks the finalized sync committee periods for which complete data ## Tracks the finalized sync committee periods for which complete data
## has been imported (from `dag.tail.slot`). ## has been imported (from `dag.tail.slot`).
template disposeSafe(s: untyped): untyped =
if distinctBase(s) != nil:
s.dispose()
s = nil
proc initCurrentBranchesStore( proc initCurrentBranchesStore(
backend: SqStoreRef, backend: SqStoreRef,
name: string): KvResult[CurrentSyncCommitteeBranchStore] = name: string): KvResult[CurrentSyncCommitteeBranchStore] =
if backend.readOnly and not ? backend.hasTable(name):
return ok CurrentSyncCommitteeBranchStore()
? backend.exec(""" ? backend.exec("""
CREATE TABLE IF NOT EXISTS `""" & name & """` ( CREATE TABLE IF NOT EXISTS `""" & name & """` (
`slot` INTEGER PRIMARY KEY, -- `Slot` (up through 2^63-1) `slot` INTEGER PRIMARY KEY, -- `Slot` (up through 2^63-1)
@ -131,15 +140,16 @@ proc initCurrentBranchesStore(
putStmt: putStmt, putStmt: putStmt,
keepFromStmt: keepFromStmt) keepFromStmt: keepFromStmt)
func close(store: CurrentSyncCommitteeBranchStore) = func close(store: var CurrentSyncCommitteeBranchStore) =
store.containsStmt.dispose() store.containsStmt.disposeSafe()
store.getStmt.dispose() store.getStmt.disposeSafe()
store.putStmt.dispose() store.putStmt.disposeSafe()
store.keepFromStmt.dispose() store.keepFromStmt.disposeSafe()
func hasCurrentSyncCommitteeBranch*( func hasCurrentSyncCommitteeBranch*(
db: LightClientDataDB, slot: Slot): bool = db: LightClientDataDB, slot: Slot): bool =
if not slot.isSupportedBySQLite: if not slot.isSupportedBySQLite or
distinctBase(db.currentBranches.containsStmt) == nil:
return false return false
var exists: int64 var exists: int64
for res in db.currentBranches.containsStmt.exec(slot.int64, exists): for res in db.currentBranches.containsStmt.exec(slot.int64, exists):
@ -150,7 +160,8 @@ func hasCurrentSyncCommitteeBranch*(
proc getCurrentSyncCommitteeBranch*( proc getCurrentSyncCommitteeBranch*(
db: LightClientDataDB, slot: Slot): altair.CurrentSyncCommitteeBranch = db: LightClientDataDB, slot: Slot): altair.CurrentSyncCommitteeBranch =
if not slot.isSupportedBySQLite: if not slot.isSupportedBySQLite or
distinctBase(db.currentBranches.getStmt) == nil:
return default(altair.CurrentSyncCommitteeBranch) return default(altair.CurrentSyncCommitteeBranch)
var branch: seq[byte] var branch: seq[byte]
for res in db.currentBranches.getStmt.exec(slot.int64, branch): for res in db.currentBranches.getStmt.exec(slot.int64, branch):
@ -165,6 +176,7 @@ proc getCurrentSyncCommitteeBranch*(
func putCurrentSyncCommitteeBranch*( func putCurrentSyncCommitteeBranch*(
db: LightClientDataDB, slot: Slot, db: LightClientDataDB, slot: Slot,
branch: altair.CurrentSyncCommitteeBranch) = branch: altair.CurrentSyncCommitteeBranch) =
doAssert not db.backend.readOnly # All `stmt` are non-nil
if not slot.isSupportedBySQLite: if not slot.isSupportedBySQLite:
return return
let res = db.currentBranches.putStmt.exec((slot.int64, SSZ.encode(branch))) let res = db.currentBranches.putStmt.exec((slot.int64, SSZ.encode(branch)))
@ -174,6 +186,9 @@ proc initLegacyBestUpdatesStore(
backend: SqStoreRef, backend: SqStoreRef,
name: string, name: string,
): KvResult[LegacyBestLightClientUpdateStore] = ): KvResult[LegacyBestLightClientUpdateStore] =
if backend.readOnly and not ? backend.hasTable(name):
return ok LegacyBestLightClientUpdateStore()
? backend.exec(""" ? backend.exec("""
CREATE TABLE IF NOT EXISTS `""" & name & """` ( CREATE TABLE IF NOT EXISTS `""" & name & """` (
`period` INTEGER PRIMARY KEY, -- `SyncCommitteePeriod` `period` INTEGER PRIMARY KEY, -- `SyncCommitteePeriod`
@ -181,7 +196,13 @@ proc initLegacyBestUpdatesStore(
); );
""") """)
const legacyKind = Base10.toString(ord(LightClientDataFork.Altair).uint)
let let
getStmt = backend.prepareStmt("""
SELECT """ & legacyKind & """ AS `kind`, `update`
FROM `""" & name & """`
WHERE `period` = ?;
""", int64, (int64, seq[byte]), managed = false).expect("SQL query OK")
putStmt = backend.prepareStmt(""" putStmt = backend.prepareStmt("""
REPLACE INTO `""" & name & """` ( REPLACE INTO `""" & name & """` (
`period`, `update` `period`, `update`
@ -201,21 +222,26 @@ proc initLegacyBestUpdatesStore(
""", int64, void, managed = false).expect("SQL query OK") """, int64, void, managed = false).expect("SQL query OK")
ok LegacyBestLightClientUpdateStore( ok LegacyBestLightClientUpdateStore(
getStmt: getStmt,
putStmt: putStmt, putStmt: putStmt,
delStmt: delStmt, delStmt: delStmt,
delFromStmt: delFromStmt, delFromStmt: delFromStmt,
keepFromStmt: keepFromStmt) keepFromStmt: keepFromStmt)
func close(store: LegacyBestLightClientUpdateStore) = func close(store: var LegacyBestLightClientUpdateStore) =
store.putStmt.dispose() store.getStmt.disposeSafe()
store.delStmt.dispose() store.putStmt.disposeSafe()
store.delFromStmt.dispose() store.delStmt.disposeSafe()
store.keepFromStmt.dispose() store.delFromStmt.disposeSafe()
store.keepFromStmt.disposeSafe()
proc initBestUpdatesStore( proc initBestUpdatesStore(
backend: SqStoreRef, backend: SqStoreRef,
name, legacyAltairName: string, name, legacyAltairName: string,
): KvResult[BestLightClientUpdateStore] = ): KvResult[BestLightClientUpdateStore] =
if backend.readOnly and not ? backend.hasTable(name):
return ok BestLightClientUpdateStore()
? backend.exec(""" ? backend.exec("""
CREATE TABLE IF NOT EXISTS `""" & name & """` ( CREATE TABLE IF NOT EXISTS `""" & name & """` (
`period` INTEGER PRIMARY KEY, -- `SyncCommitteePeriod` `period` INTEGER PRIMARY KEY, -- `SyncCommitteePeriod`
@ -223,7 +249,7 @@ proc initBestUpdatesStore(
`update` BLOB -- `LightClientUpdate` (SSZ) `update` BLOB -- `LightClientUpdate` (SSZ)
); );
""") """)
block: if ? backend.hasTable(legacyAltairName):
# SyncCommitteePeriod -> altair.LightClientUpdate # SyncCommitteePeriod -> altair.LightClientUpdate
const legacyKind = Base10.toString(ord(LightClientDataFork.Altair).uint) const legacyKind = Base10.toString(ord(LightClientDataFork.Altair).uint)
? backend.exec(""" ? backend.exec("""
@ -267,19 +293,20 @@ proc initBestUpdatesStore(
delFromStmt: delFromStmt, delFromStmt: delFromStmt,
keepFromStmt: keepFromStmt) keepFromStmt: keepFromStmt)
func close(store: BestLightClientUpdateStore) = func close(store: var BestLightClientUpdateStore) =
store.getStmt.dispose() store.getStmt.disposeSafe()
store.putStmt.dispose() store.putStmt.disposeSafe()
store.delStmt.dispose() store.delStmt.disposeSafe()
store.delFromStmt.dispose() store.delFromStmt.disposeSafe()
store.keepFromStmt.dispose() store.keepFromStmt.disposeSafe()
proc getBestUpdate*( proc getBestUpdate*(
db: LightClientDataDB, period: SyncCommitteePeriod db: LightClientDataDB, period: SyncCommitteePeriod
): ForkedLightClientUpdate = ): ForkedLightClientUpdate =
doAssert period.isSupportedBySQLite doAssert period.isSupportedBySQLite
var update: (int64, seq[byte]) var update: (int64, seq[byte])
for res in db.bestUpdates.getStmt.exec(period.int64, update): template body: untyped =
res.expect("SQL query OK") res.expect("SQL query OK")
try: try:
withAll(LightClientDataFork): withAll(LightClientDataFork):
@ -297,9 +324,19 @@ proc getBestUpdate*(
period, kind = update[0], exc = exc.msg period, kind = update[0], exc = exc.msg
return default(ForkedLightClientUpdate) return default(ForkedLightClientUpdate)
if distinctBase(db.bestUpdates.getStmt) != nil:
for res in db.bestUpdates.getStmt.exec(period.int64, update):
body
elif distinctBase(db.legacyBestUpdates.getStmt) != nil:
for res in db.legacyBestUpdates.getStmt.exec(period.int64, update):
body
else:
return default(ForkedLightClientUpdate)
func putBestUpdate*( func putBestUpdate*(
db: LightClientDataDB, period: SyncCommitteePeriod, db: LightClientDataDB, period: SyncCommitteePeriod,
update: ForkedLightClientUpdate) = update: ForkedLightClientUpdate) =
doAssert not db.backend.readOnly # All `stmt` are non-nil
doAssert period.isSupportedBySQLite doAssert period.isSupportedBySQLite
withForkyUpdate(update): withForkyUpdate(update):
when lcDataFork > LightClientDataFork.None: when lcDataFork > LightClientDataFork.None:
@ -341,6 +378,9 @@ proc putUpdateIfBetter*(
proc initSealedPeriodsStore( proc initSealedPeriodsStore(
backend: SqStoreRef, backend: SqStoreRef,
name: string): KvResult[SealedSyncCommitteePeriodStore] = name: string): KvResult[SealedSyncCommitteePeriodStore] =
if backend.readOnly and not ? backend.hasTable(name):
return ok SealedSyncCommitteePeriodStore()
? backend.exec(""" ? backend.exec("""
CREATE TABLE IF NOT EXISTS `""" & name & """` ( CREATE TABLE IF NOT EXISTS `""" & name & """` (
`period` INTEGER PRIMARY KEY -- `SyncCommitteePeriod` `period` INTEGER PRIMARY KEY -- `SyncCommitteePeriod`
@ -373,15 +413,17 @@ proc initSealedPeriodsStore(
delFromStmt: delFromStmt, delFromStmt: delFromStmt,
keepFromStmt: keepFromStmt) keepFromStmt: keepFromStmt)
func close(store: SealedSyncCommitteePeriodStore) = func close(store: var SealedSyncCommitteePeriodStore) =
store.containsStmt.dispose() store.containsStmt.disposeSafe()
store.putStmt.dispose() store.putStmt.disposeSafe()
store.delFromStmt.dispose() store.delFromStmt.disposeSafe()
store.keepFromStmt.dispose() store.keepFromStmt.disposeSafe()
func isPeriodSealed*( func isPeriodSealed*(
db: LightClientDataDB, period: SyncCommitteePeriod): bool = db: LightClientDataDB, period: SyncCommitteePeriod): bool =
doAssert period.isSupportedBySQLite doAssert period.isSupportedBySQLite
if distinctBase(db.sealedPeriods.containsStmt) == nil:
return false
var exists: int64 var exists: int64
for res in db.sealedPeriods.containsStmt.exec(period.int64, exists): for res in db.sealedPeriods.containsStmt.exec(period.int64, exists):
res.expect("SQL query OK") res.expect("SQL query OK")
@ -391,12 +433,14 @@ func isPeriodSealed*(
func sealPeriod*( func sealPeriod*(
db: LightClientDataDB, period: SyncCommitteePeriod) = db: LightClientDataDB, period: SyncCommitteePeriod) =
doAssert not db.backend.readOnly # All `stmt` are non-nil
doAssert period.isSupportedBySQLite doAssert period.isSupportedBySQLite
let res = db.sealedPeriods.putStmt.exec(period.int64) let res = db.sealedPeriods.putStmt.exec(period.int64)
res.expect("SQL query OK") res.expect("SQL query OK")
func delNonFinalizedPeriodsFrom*( func delNonFinalizedPeriodsFrom*(
db: LightClientDataDB, minPeriod: SyncCommitteePeriod) = db: LightClientDataDB, minPeriod: SyncCommitteePeriod) =
doAssert not db.backend.readOnly # All `stmt` are non-nil
doAssert minPeriod.isSupportedBySQLite doAssert minPeriod.isSupportedBySQLite
block: block:
let res = db.sealedPeriods.delFromStmt.exec(minPeriod.int64) let res = db.sealedPeriods.delFromStmt.exec(minPeriod.int64)
@ -411,6 +455,7 @@ func delNonFinalizedPeriodsFrom*(
func keepPeriodsFrom*( func keepPeriodsFrom*(
db: LightClientDataDB, minPeriod: SyncCommitteePeriod) = db: LightClientDataDB, minPeriod: SyncCommitteePeriod) =
doAssert not db.backend.readOnly # All `stmt` are non-nil
doAssert minPeriod.isSupportedBySQLite doAssert minPeriod.isSupportedBySQLite
block: block:
let res = db.sealedPeriods.keepFromStmt.exec(minPeriod.int64) let res = db.sealedPeriods.keepFromStmt.exec(minPeriod.int64)

62
beacon_chain/light_client_db.nim
View File

@ -32,6 +32,7 @@ type
Finalized = 1 # Latest finalized header Finalized = 1 # Latest finalized header
LegacyLightClientHeadersStore = object LegacyLightClientHeadersStore = object
getStmt: SqliteStmt[int64, (int64, seq[byte])]
putStmt: SqliteStmt[(int64, seq[byte]), void] putStmt: SqliteStmt[(int64, seq[byte]), void]
LightClientHeadersStore = object LightClientHeadersStore = object
@ -59,9 +60,17 @@ type
## SyncCommitteePeriod -> altair.SyncCommittee ## SyncCommitteePeriod -> altair.SyncCommittee
## Stores finalized `SyncCommittee` by sync committee period. ## Stores finalized `SyncCommittee` by sync committee period.
template disposeSafe(s: untyped): untyped =
if distinctBase(s) != nil:
s.dispose()
s = nil
proc initLegacyLightClientHeadersStore( proc initLegacyLightClientHeadersStore(
backend: SqStoreRef, backend: SqStoreRef,
name: string): KvResult[LegacyLightClientHeadersStore] = name: string): KvResult[LegacyLightClientHeadersStore] =
if backend.readOnly and not ? backend.hasTable(name):
return ok LegacyLightClientHeadersStore()
? backend.exec(""" ? backend.exec("""
CREATE TABLE IF NOT EXISTS `""" & name & """` ( CREATE TABLE IF NOT EXISTS `""" & name & """` (
`kind` INTEGER PRIMARY KEY, -- `LightClientHeaderKey` `kind` INTEGER PRIMARY KEY, -- `LightClientHeaderKey`
@ -69,7 +78,13 @@ proc initLegacyLightClientHeadersStore(
); );
""") """)
const legacyKind = Base10.toString(ord(LightClientDataFork.Altair).uint)
let let
getStmt = backend.prepareStmt("""
SELECT """ & legacyKind & """ AS `kind`, `header`
FROM `""" & name & """`
WHERE `""" & name & """`.`kind` = ?;
""", int64, (int64, seq[byte]), managed = false).expect("SQL query OK")
putStmt = backend.prepareStmt(""" putStmt = backend.prepareStmt("""
REPLACE INTO `""" & name & """` ( REPLACE INTO `""" & name & """` (
`kind`, `header` `kind`, `header`
@ -78,14 +93,19 @@ proc initLegacyLightClientHeadersStore(
.expect("SQL query OK") .expect("SQL query OK")
ok LegacyLightClientHeadersStore( ok LegacyLightClientHeadersStore(
getStmt: getStmt,
putStmt: putStmt) putStmt: putStmt)
func close(store: LegacyLightClientHeadersStore) = func close(store: var LegacyLightClientHeadersStore) =
store.putStmt.dispose() store.getStmt.disposeSafe()
store.putStmt.disposeSafe()
proc initLightClientHeadersStore( proc initLightClientHeadersStore(
backend: SqStoreRef, backend: SqStoreRef,
name, legacyAltairName: string): KvResult[LightClientHeadersStore] = name, legacyAltairName: string): KvResult[LightClientHeadersStore] =
if backend.readOnly and not ? backend.hasTable(name):
return ok LightClientHeadersStore()
? backend.exec(""" ? backend.exec("""
CREATE TABLE IF NOT EXISTS `""" & name & """` ( CREATE TABLE IF NOT EXISTS `""" & name & """` (
`key` INTEGER PRIMARY KEY, -- `LightClientHeaderKey` `key` INTEGER PRIMARY KEY, -- `LightClientHeaderKey`
@ -93,7 +113,7 @@ proc initLightClientHeadersStore(
`header` BLOB -- `LightClientHeader` (SSZ) `header` BLOB -- `LightClientHeader` (SSZ)
); );
""") """)
block: if ? backend.hasTable(legacyAltairName):
# LightClientHeaderKey -> altair.LightClientHeader # LightClientHeaderKey -> altair.LightClientHeader
const legacyKind = Base10.toString(ord(LightClientDataFork.Altair).uint) const legacyKind = Base10.toString(ord(LightClientDataFork.Altair).uint)
? backend.exec(""" ? backend.exec("""
@ -121,15 +141,16 @@ proc initLightClientHeadersStore(
getStmt: getStmt, getStmt: getStmt,
putStmt: putStmt) putStmt: putStmt)
func close(store: LightClientHeadersStore) = func close(store: var LightClientHeadersStore) =
store.getStmt.dispose() store.getStmt.disposeSafe()
store.putStmt.dispose() store.putStmt.disposeSafe()
proc getLatestFinalizedHeader*( proc getLatestFinalizedHeader*(
db: LightClientDB): ForkedLightClientHeader = db: LightClientDB): ForkedLightClientHeader =
const key = LightClientHeaderKey.Finalized const key = LightClientHeaderKey.Finalized
var header: (int64, seq[byte]) var header: (int64, seq[byte])
for res in db.headers.getStmt.exec(key.int64, header): template body: untyped =
res.expect("SQL query OK") res.expect("SQL query OK")
try: try:
withAll(LightClientDataFork): withAll(LightClientDataFork):
@ -147,8 +168,18 @@ proc getLatestFinalizedHeader*(
key, kind = header[0], exc = exc.msg key, kind = header[0], exc = exc.msg
return default(ForkedLightClientHeader) return default(ForkedLightClientHeader)
if distinctBase(db.headers.getStmt) != nil:
for res in db.headers.getStmt.exec(key.int64, header):
body
elif distinctBase(db.legacyHeaders.getStmt) != nil:
for res in db.legacyHeaders.getStmt.exec(key.int64, header):
body
else:
return default(ForkedLightClientHeader)
func putLatestFinalizedHeader*( func putLatestFinalizedHeader*(
db: LightClientDB, header: ForkedLightClientHeader) = db: LightClientDB, header: ForkedLightClientHeader) =
doAssert not db.backend.readOnly # All `stmt` are non-nil
withForkyHeader(header): withForkyHeader(header):
when lcDataFork > LightClientDataFork.None: when lcDataFork > LightClientDataFork.None:
block: block:
@ -161,6 +192,9 @@ func putLatestFinalizedHeader*(
let res = db.legacyHeaders.putStmt.exec( let res = db.legacyHeaders.putStmt.exec(
(key.int64, SSZ.encode(forkyHeader))) (key.int64, SSZ.encode(forkyHeader)))
res.expect("SQL query OK") res.expect("SQL query OK")
else:
# Keep legacy table at best Altair header.
discard
block: block:
let period = forkyHeader.beacon.slot.sync_committee_period let period = forkyHeader.beacon.slot.sync_committee_period
doAssert period.isSupportedBySQLite doAssert period.isSupportedBySQLite
@ -171,6 +205,9 @@ func putLatestFinalizedHeader*(
func initSyncCommitteesStore( func initSyncCommitteesStore(
backend: SqStoreRef, backend: SqStoreRef,
name: string): KvResult[SyncCommitteeStore] = name: string): KvResult[SyncCommitteeStore] =
if backend.readOnly and not ? backend.hasTable(name):
return ok SyncCommitteeStore()
? backend.exec(""" ? backend.exec("""
CREATE TABLE IF NOT EXISTS `""" & name & """` ( CREATE TABLE IF NOT EXISTS `""" & name & """` (
`period` INTEGER PRIMARY KEY, -- `SyncCommitteePeriod` `period` INTEGER PRIMARY KEY, -- `SyncCommitteePeriod`
@ -199,14 +236,16 @@ func initSyncCommitteesStore(
putStmt: putStmt, putStmt: putStmt,
keepFromStmt: keepFromStmt) keepFromStmt: keepFromStmt)
func close(store: SyncCommitteeStore) = func close(store: var SyncCommitteeStore) =
store.getStmt.dispose() store.getStmt.disposeSafe()
store.putStmt.dispose() store.putStmt.disposeSafe()
store.keepFromStmt.dispose() store.keepFromStmt.disposeSafe()
proc getSyncCommittee*( proc getSyncCommittee*(
db: LightClientDB, period: SyncCommitteePeriod): Opt[altair.SyncCommittee] = db: LightClientDB, period: SyncCommitteePeriod): Opt[altair.SyncCommittee] =
doAssert period.isSupportedBySQLite doAssert period.isSupportedBySQLite
if distinctBase(db.syncCommittees.getStmt) == nil:
return Opt.none(altair.SyncCommittee)
var syncCommittee: seq[byte] var syncCommittee: seq[byte]
for res in db.syncCommittees.getStmt.exec(period.int64, syncCommittee): for res in db.syncCommittees.getStmt.exec(period.int64, syncCommittee):
res.expect("SQL query OK") res.expect("SQL query OK")
@ -220,6 +259,7 @@ proc getSyncCommittee*(
func putSyncCommittee*( func putSyncCommittee*(
db: LightClientDB, period: SyncCommitteePeriod, db: LightClientDB, period: SyncCommitteePeriod,
syncCommittee: altair.SyncCommittee) = syncCommittee: altair.SyncCommittee) =
doAssert not db.backend.readOnly # All `stmt` are non-nil
doAssert period.isSupportedBySQLite doAssert period.isSupportedBySQLite
let res = db.syncCommittees.putStmt.exec( let res = db.syncCommittees.putStmt.exec(
(period.int64, SSZ.encode(syncCommittee))) (period.int64, SSZ.encode(syncCommittee)))