Initial commit.
This commit is contained in:
cheatfate
parent
219916162e
commit
735566fad0
|
|
@ -19,6 +19,12 @@ import
|
|||
|
||||
export rest_utils
|
||||
|
||||
const
|
||||
MINIMAL_VALIDATORS_BF_COUNT = 1_000_000
|
||||
## Minimal size of validators bloom filter. Current mainnet number of
|
||||
## validators is near 700k, so you could update this number if it exceeds.
|
||||
## This number only affects first-run nodes.
|
||||
|
||||
logScope: topics = "rest_beaconapi"
|
||||
|
||||
proc validateBeaconApiQueries*(key: string, value: string): int =
|
||||
|
|
@ -123,7 +129,58 @@ proc toString*(kind: ValidatorFilterKind): string =
|
|||
of ValidatorFilterKind.WithdrawalDone:
|
||||
"withdrawal_done"
|
||||
|
||||
proc getFinalizedBlockSlotId(node: BeaconNode): BlockSlotId =
|
||||
node.dag.finalizedHead.toBlockSlotId().expect("not nil")
|
||||
|
||||
proc getValidatorsCount(node: BeaconNode): int =
|
||||
node.withStateForBlockSlotId(node.getFinalizedBlockSlotId()):
|
||||
return len(getStateField(state, validators))
|
||||
0
|
||||
|
||||
proc updateBloomFilter(bf: var BloomFilter, node: BeaconNode) =
|
||||
var count = 0
|
||||
let startTime = Moment.now()
|
||||
node.withStateForBlockSlotId(node.getFinalizedBlockSlotId()):
|
||||
let validatorsCount = lenu64(getStateField(state, validators))
|
||||
if validatorsCount > lenu64(bf):
|
||||
let validatorsInFilter = lenu64(bf)
|
||||
for index in validatorsInFilter ..< validatorsCount:
|
||||
bf.registerKey(
|
||||
distinctBase(getStateField(state, validators).data)[index].pubkey)
|
||||
inc(count)
|
||||
if count > 0:
|
||||
debug "Validators bloom filter updated", count = count,
|
||||
in_filter_count = len(bf), elapsed_time = (Moment.now() - startTime)
|
||||
|
||||
proc getRecentValidators(
|
||||
startIndex: ValidatorIndex,
|
||||
forkedState: ForkedHashedBeaconState
|
||||
): Table[ValidatorPubKey, ValidatorIndex] =
|
||||
var res: Table[ValidatorPubKey, ValidatorIndex]
|
||||
withState(forkedState):
|
||||
let validatorsCount = lenu64(forkyState.data.validators)
|
||||
for index in uint64(startIndex) ..< validatorsCount:
|
||||
let pubkey =
|
||||
distinctBase(forkyState.data.validators).data[index].pubkey
|
||||
res[pubkey] = ValidatorIndex(index)
|
||||
res
|
||||
|
||||
proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
|
||||
let
|
||||
validatorsCount = max(node.getValidatorsCount(),
|
||||
MINIMAL_VALIDATORS_BF_COUNT)
|
||||
# We expect 133% of validators to be registered while node is
|
||||
# running. If number of validators will exceed this value we going
|
||||
# to lose bloom filter's efficiency only.
|
||||
itemsCount = validatorsCount + (validatorsCount div 3)
|
||||
bitsCount = getBitsCount(int(itemsCount))
|
||||
|
||||
var validatorsBloomFilter = BloomFilter.init(bitsCount)
|
||||
debug "Validators bloom filter initialized", bits_count = bitsCount,
|
||||
actual_bits_count = validatorsBloomFilter.bitsCount(),
|
||||
memory_size = formatSize(validatorsBloomFilter.bytesCount())
|
||||
validatorsBloomFilter.updateBloomFilter(node)
|
||||
|
||||
# https://github.com/ethereum/EIPs/blob/master/EIPS/eip-4881.md
|
||||
router.api(MethodGet, "/eth/v1/beacon/deposit_snapshot") do () -> RestApiResponse:
|
||||
let snapshot = node.db.getDepositTreeSnapshot().valueOr:
|
||||
|
|
@ -271,10 +328,16 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
|
|||
$res.error())
|
||||
res.get()
|
||||
|
||||
# Update bloom filter to latest finalized head validators.
|
||||
validatorsBloomFilter.updateBloomFilter(node)
|
||||
|
||||
node.withStateForBlockSlotId(bslot):
|
||||
let
|
||||
current_epoch = getStateField(state, slot).epoch()
|
||||
validatorsCount = lenu64(getStateField(state, validators))
|
||||
var recentValidators =
|
||||
getRecentValidators(ValidatorIndex(len(validatorsBloomFilter)),
|
||||
state)
|
||||
|
||||
let indices =
|
||||
block:
|
||||
|
|
@ -305,6 +368,8 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
|
|||
|
||||
if len(keyset) > 0:
|
||||
let optIndices = keysToIndices(node.restKeysCache, state,
|
||||
validatorsBloomFilter,
|
||||
recentValidators,
|
||||
keyset.toSeq())
|
||||
# Remove all the duplicates.
|
||||
for item in optIndices:
|
||||
|
|
@ -378,16 +443,23 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
|
|||
return RestApiResponse.jsonError(Http404, StateNotFoundError,
|
||||
$error)
|
||||
|
||||
# Update bloom filter to latest finalized head validators.
|
||||
validatorsBloomFilter.updateBloomFilter(node)
|
||||
|
||||
node.withStateForBlockSlotId(bslot):
|
||||
let
|
||||
current_epoch = getStateField(state, slot).epoch()
|
||||
validatorsCount = lenu64(getStateField(state, validators))
|
||||
|
||||
var recentValidators =
|
||||
getRecentValidators(ValidatorIndex(len(validatorsBloomFilter)),
|
||||
state)
|
||||
let vindex =
|
||||
block:
|
||||
case vid.kind
|
||||
of ValidatorQueryKind.Key:
|
||||
let optIndices = keysToIndices(node.restKeysCache, state, [vid.key])
|
||||
let optIndices = keysToIndices(node.restKeysCache, state,
|
||||
validatorsBloomFilter,
|
||||
recentValidators, [vid.key])
|
||||
if optIndices[0].isNone():
|
||||
return RestApiResponse.jsonError(Http404, ValidatorNotFoundError)
|
||||
optIndices[0].get()
|
||||
|
|
@ -450,8 +522,15 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
|
|||
MaximumNumberOfValidatorIdsError)
|
||||
ires
|
||||
|
||||
# Update bloom filter to latest finalized head validators.
|
||||
validatorsBloomFilter.updateBloomFilter(node)
|
||||
|
||||
node.withStateForBlockSlotId(bslot):
|
||||
let validatorsCount = lenu64(getStateField(state, validators))
|
||||
let
|
||||
validatorsCount = lenu64(getStateField(state, validators))
|
||||
var recentValidators =
|
||||
getRecentValidators(ValidatorIndex(len(validatorsBloomFilter)),
|
||||
state)
|
||||
|
||||
let indices =
|
||||
block:
|
||||
|
|
@ -481,6 +560,8 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
|
|||
|
||||
if len(keyset) > 0:
|
||||
let optIndices = keysToIndices(node.restKeysCache, state,
|
||||
validatorsBloomFilter,
|
||||
recentValidators,
|
||||
keyset.toSeq())
|
||||
# Remove all the duplicates.
|
||||
for item in optIndices:
|
||||
|
|
@ -664,7 +745,14 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
|
|||
# the state will be obtained.
|
||||
bslot.slot.epoch()
|
||||
|
||||
# Update bloom filter to latest finalized head validators.
|
||||
validatorsBloomFilter.updateBloomFilter(node)
|
||||
|
||||
node.withStateForBlockSlotId(bslot):
|
||||
var recentValidators =
|
||||
getRecentValidators(ValidatorIndex(len(validatorsBloomFilter)),
|
||||
state)
|
||||
|
||||
let keys =
|
||||
block:
|
||||
let res = syncCommitteeParticipants(state, qepoch)
|
||||
|
|
@ -680,7 +768,9 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
|
|||
let indices =
|
||||
block:
|
||||
var res: seq[ValidatorIndex]
|
||||
let optIndices = keysToIndices(node.restKeysCache, state, keys)
|
||||
let optIndices = keysToIndices(node.restKeysCache, state,
|
||||
validatorsBloomFilter,
|
||||
recentValidators, keys)
|
||||
# Remove all the duplicates.
|
||||
for item in optIndices:
|
||||
if item.isNone():
|
||||
|
|
|
|||
|
|
@ -8,14 +8,16 @@
|
|||
{.push raises: [].}
|
||||
|
||||
import std/[options, macros],
|
||||
stew/byteutils, presto,
|
||||
../spec/[forks],
|
||||
stew/[byteutils, endians2], presto,
|
||||
../spec/[forks, crypto],
|
||||
../spec/eth2_apis/[rest_types, eth2_rest_serialization, rest_common],
|
||||
../validators/validator_duties,
|
||||
../consensus_object_pools/blockchain_dag,
|
||||
../beacon_node,
|
||||
"."/[rest_constants, state_ttl_cache]
|
||||
|
||||
from std/math import isPowerOfTwo, nextPowerOfTwo
|
||||
|
||||
export
|
||||
options, eth2_rest_serialization, blockchain_dag, presto, rest_types,
|
||||
rest_constants, rest_common
|
||||
|
|
@ -24,6 +26,15 @@ type
|
|||
ValidatorIndexError* {.pure.} = enum
|
||||
UnsupportedValue, TooHighValue
|
||||
|
||||
BloomFilter* = object
|
||||
words: seq[uint]
|
||||
length: int
|
||||
mask: uint
|
||||
used: int
|
||||
|
||||
func checkKey*(bf: BloomFilter, pubkey: ValidatorPubKey): bool {.
|
||||
raises: [].}
|
||||
|
||||
func match(data: openArray[char], charset: set[char]): int =
|
||||
for ch in data:
|
||||
if ch notin charset:
|
||||
|
|
@ -255,6 +266,8 @@ func syncCommitteeParticipants*(forkedState: ForkedHashedBeaconState,
|
|||
|
||||
func keysToIndices*(cacheTable: var Table[ValidatorPubKey, ValidatorIndex],
|
||||
forkedState: ForkedHashedBeaconState,
|
||||
bloomFilter: BloomFilter,
|
||||
recentValidators: var Table[ValidatorPubKey, ValidatorIndex],
|
||||
keys: openArray[ValidatorPubKey]
|
||||
): seq[Option[ValidatorIndex]] =
|
||||
var indices = newSeq[Option[ValidatorIndex]](len(keys))
|
||||
|
|
@ -262,13 +275,22 @@ func keysToIndices*(cacheTable: var Table[ValidatorPubKey, ValidatorIndex],
|
|||
var keyset =
|
||||
block:
|
||||
var res: Table[ValidatorPubKey, int]
|
||||
for inputIndex, pubkey in keys:
|
||||
# Try to search in cache first.
|
||||
cacheTable.withValue(pubkey, vindex):
|
||||
if uint64(vindex[]) < totalValidatorsInState:
|
||||
indices[inputIndex] = some(vindex[])
|
||||
do:
|
||||
res[pubkey] = inputIndex
|
||||
for inputIndex, pubkey in keys.pairs():
|
||||
if bloomFilter.checkKey(pubkey):
|
||||
# The validator's public key is possible in our registry, we will
|
||||
# check in cache first.
|
||||
cacheTable.withValue(pubkey, vindex):
|
||||
if uint64(vindex[]) < totalValidatorsInState:
|
||||
indices[inputIndex] = some(vindex[])
|
||||
do:
|
||||
res[pubkey] = inputIndex
|
||||
else:
|
||||
# The validator's public key is not in bloom filter, but it could
|
||||
# be inside `recentValidators`, so we check it.
|
||||
recentValidators.withValue(pubkey, rindex):
|
||||
indices[inputIndex] = some(rindex[])
|
||||
do:
|
||||
res[pubkey] = inputIndex
|
||||
res
|
||||
if len(keyset) > 0:
|
||||
for validatorIndex, validator in getStateField(forkedState, validators):
|
||||
|
|
@ -334,3 +356,124 @@ proc verifyRandao*(
|
|||
|
||||
verify_epoch_signature(
|
||||
fork, genesis_validators_root, slot.epoch, proposer_pubkey, randao)
|
||||
|
||||
template len*(bf: BloomFilter): int =
|
||||
bf.used
|
||||
|
||||
template divShift(): uint =
|
||||
when sizeof(uint) == 8: 6'u else: 5'u
|
||||
|
||||
template modMask(): uint =
|
||||
when sizeof(uint) == 8: 63'u else: 31'u
|
||||
|
||||
func raiseBit*(bf: var BloomFilter, pos: Natural) {.inline.} =
|
||||
doAssert(pos < bf.length)
|
||||
let index = uint(pos) shr divShift()
|
||||
bf.words[index] = bf.words[index] or (1'u shl (uint(pos) and modMask()))
|
||||
|
||||
func getBit*(bf: BloomFilter, pos: Natural): bool {.inline.} =
|
||||
doAssert(pos < bf.length)
|
||||
let index = uint(pos) shr divShift()
|
||||
(bf.words[index] and (1'u shl (uint(pos) and modMask()))) != 0'u
|
||||
|
||||
proc getBitsCount*(itemsCount: int): int =
|
||||
## We are using 8 hashes and we want to get 0.0001 false positive rate, so we
|
||||
## should use `m/n == 21` according to Table 3 of
|
||||
## https://pages.cs.wisc.edu/~cao/papers/summary-cache/node8.html
|
||||
itemsCount * 21
|
||||
|
||||
proc init*(T: typedesc[BloomFilter], bitsCount: int): T =
|
||||
when sizeof(int) == 8:
|
||||
doAssert bitsCount <= 0x4000_0000_0000_0000
|
||||
else:
|
||||
doAssert bitsCount <= 0x4000_0000
|
||||
|
||||
let
|
||||
correctedBits = nextPowerOfTwo(bitsCount + 1)
|
||||
correctedSize = (correctedBits + int(modMask())) shr int(divShift())
|
||||
|
||||
BloomFilter(
|
||||
words: newSeq[uint](correctedSize),
|
||||
mask: uint(correctedBits - 1),
|
||||
length: correctedBits
|
||||
)
|
||||
|
||||
func itemsCount*(bf: BloomFilter): int =
|
||||
len(bf.words)
|
||||
|
||||
func bytesCount*(bf: BloomFilter): int =
|
||||
len(bf.words) * sizeof(uint)
|
||||
|
||||
func bitsCount*(bf: BloomFilter): int =
|
||||
bf.length
|
||||
|
||||
template toHashes(pubkey: ValidatorPubKey): auto =
|
||||
var pos: array[8, uint32]
|
||||
when sizeof(int) == 8:
|
||||
let
|
||||
pos0 = uint64.fromBytesLE(pubkey.blob.toOpenArray(0, 7)) xor
|
||||
uint64.fromBytesLE(pubkey.blob.toOpenArray(8, 15))
|
||||
pos1 = uint64.fromBytesLE(pubkey.blob.toOpenArray(16, 23)) xor
|
||||
uint64.fromBytesLE(pubkey.blob.toOpenArray(24, 31))
|
||||
pos2 = uint64.fromBytesLE(pubkey.blob.toOpenArray(32, 39)) xor
|
||||
uint64.fromBytesLE(pubkey.blob.toOpenArray(40, 47))
|
||||
pos3 = uint64.fromBytesLE(pubkey.blob.toOpenArray(0, 7)) xor
|
||||
uint64.fromBytesLE(pubkey.blob.toOpenArray(40, 47))
|
||||
pos[0] = uint32(pos0 and uint64(bf.mask))
|
||||
pos[1] = uint32((pos0 shr 32) and uint64(bf.mask))
|
||||
pos[2] = uint32(pos1 and uint64(bf.mask))
|
||||
pos[3] = uint32((pos1 shr 32) and uint64(bf.mask))
|
||||
pos[4] = uint32(pos2 and uint64(bf.mask))
|
||||
pos[5] = uint32((pos2 shr 32) and uint64(bf.mask))
|
||||
pos[6] = uint32(pos3 and uint64(bf.mask))
|
||||
pos[7] = uint32((pos3 shr 32) and uint64(bf.mask))
|
||||
else:
|
||||
pos[0] = (uint32.fromBytesLE(pubkey.blob.toOpenArray(0, 3)) xor
|
||||
uint32.fromBytesLE(pubkey.blob.toOpenArray(4, 7))) and
|
||||
uint32(bf.mask)
|
||||
pos[1] = (uint32.fromBytesLE(pubkey.blob.toOpenArray(8, 11)) xor
|
||||
uint32.fromBytesLE(pubkey.blob.toOpenArray(12, 15))) and
|
||||
uint32(bf.mask)
|
||||
pos[2] = (uint32.fromBytesLE(pubkey.blob.toOpenArray(16, 19)) xor
|
||||
uint32.fromBytesLE(pubkey.blob.toOpenArray(20, 23))) and
|
||||
uint32(bf.mask)
|
||||
pos[3] = (uint32.fromBytesLE(pubkey.blob.toOpenArray(24, 27)) xor
|
||||
uint32.fromBytesLE(pubkey.blob.toOpenArray(28, 31))) and
|
||||
uint32(bf.mask)
|
||||
pos[4] = (uint32.fromBytesLE(pubkey.blob.toOpenArray(32, 35)) xor
|
||||
uint32.fromBytesLE(pubkey.blob.toOpenArray(36, 39))) and
|
||||
uint32(bf.mask)
|
||||
pos[5] = (uint32.fromBytesLE(pubkey.blob.toOpenArray(40, 43)) xor
|
||||
uint32.fromBytesLE(pubkey.blob.toOpenArray(44, 47))) and
|
||||
uint32(bf.mask)
|
||||
pos[6] = (uint32.fromBytesLE(pubkey.blob.toOpenArray(0, 3)) xor
|
||||
uint32.fromBytesLE(pubkey.blob.toOpenArray(44, 47))) and
|
||||
uint32(bf.mask)
|
||||
pos[7] = (uint32.fromBytesLE(pubkey.blob.toOpenArray(4, 7)) xor
|
||||
uint32.fromBytesLE(pubkey.blob.toOpenArray(40, 43))) and
|
||||
uint32(bf.mask)
|
||||
pos
|
||||
|
||||
proc registerKey*(bf: var BloomFilter, pubkey: ValidatorPubKey) =
|
||||
let hashes = pubkey.toHashes()
|
||||
bf.raiseBit(hashes[0])
|
||||
bf.raiseBit(hashes[1])
|
||||
bf.raiseBit(hashes[2])
|
||||
bf.raiseBit(hashes[3])
|
||||
bf.raiseBit(hashes[4])
|
||||
bf.raiseBit(hashes[5])
|
||||
bf.raiseBit(hashes[6])
|
||||
bf.raiseBit(hashes[7])
|
||||
inc(bf.used)
|
||||
|
||||
func checkKey*(bf: BloomFilter, pubkey: ValidatorPubKey): bool =
|
||||
let hashes = pubkey.toHashes()
|
||||
if not(bf.getBit(hashes[0])): return false
|
||||
if not(bf.getBit(hashes[1])): return false
|
||||
if not(bf.getBit(hashes[2])): return false
|
||||
if not(bf.getBit(hashes[3])): return false
|
||||
if not(bf.getBit(hashes[4])): return false
|
||||
if not(bf.getBit(hashes[5])): return false
|
||||
if not(bf.getBit(hashes[6])): return false
|
||||
if not(bf.getBit(hashes[7])): return false
|
||||
true
|
||||
|
|
|
|||
Loading…
Reference in New Issue