Rename --validator-source to --web3-signer-url and document it (#5389)

Also allows multiple instances to be configured
This commit is contained in:
zah 2023-09-06 22:04:10 +03:00 committed by GitHub
parent b8db44d761
commit 2b5bd74e15
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 180 additions and 112 deletions

View File

@ -162,14 +162,14 @@ type
desc: "A directory containing validator keystores"
name: "validators-dir" .}: Option[InputDir]
validatorsSource* {.
web3signers* {.
desc: "Remote Web3Signer URL that will be used as a source of validators"
name: "validators-source"}: Option[string]
name: "web3-signer-url" .}: seq[Uri]
validatorsSourceInverval* {.
desc: "Number of minutes between validator list updates"
name: "validators-source-interval"
defaultValue: 60 .}: Natural
web3signerUpdateInterval* {.
desc: "Number of seconds between validator list updates"
name: "web3-signer-update-interval"
defaultValue: 3600 .}: Natural
secretsDirFlag* {.
desc: "A directory containing validator keystore passwords"
@ -885,14 +885,14 @@ type
desc: "A directory containing validator keystores"
name: "validators-dir" .}: Option[InputDir]
validatorsSource* {.
web3signers* {.
desc: "Remote Web3Signer URL that will be used as a source of validators"
name: "validators-source"}: Option[string]
name: "web3-signer-url" .}: seq[Uri]
validatorsSourceInverval* {.
desc: "Number of minutes between validator list updates"
name: "validators-source-interval"
defaultValue: 60 .}: Natural
web3signerUpdateInterval* {.
desc: "Number of seconds between validator list updates"
name: "web3-signer-update-interval"
defaultValue: 3600 .}: Natural
secretsDirFlag* {.
desc: "A directory containing validator keystore passwords"

View File

@ -1617,7 +1617,17 @@ proc run(node: BeaconNode) {.raises: [CatchableError].} =
waitFor node.updateGossipStatus(wallSlot)
asyncSpawn pollForDynamicValidators(node)
for web3signerUrl in node.config.web3signers:
# TODO
# The current strategy polls all remote signers independently
# from each other which may lead to some race conditions of
# validators are migrated from one signer to another
# (because the updates to our validator pool are not atomic).
# Consider using different strategies that would detect such
# race conditions.
asyncSpawn node.pollForDynamicValidators(
web3signerUrl, node.config.web3signerUpdateInterval)
asyncSpawn runSlotLoop(node, wallTime, onSlotStart)
asyncSpawn runOnSecondLoop(node)
asyncSpawn runQueueProcessingLoop(node.blockProcessor)

View File

@ -85,16 +85,28 @@ proc initGenesis(vc: ValidatorClientRef): Future[RestGenesis] {.async.} =
dec(counter)
return melem
proc initValidators(vc: ValidatorClientRef): Future[bool] {.async.} =
info "Loading validators", validatorsDir = vc.config.validatorsDir()
var duplicates: seq[ValidatorPubKey]
for keystore in listLoadableKeystores(vc.config, vc.keystoreCache):
vc.addValidator(keystore)
let res = await queryValidatorsSource(vc.config)
proc addValidatorsFromWeb3Signer(vc: ValidatorClientRef, web3signerUrl: Uri) {.async.} =
let res = await queryValidatorsSource(web3signerUrl)
if res.isOk():
let dynamicKeystores = res.get()
for keystore in dynamicKeystores:
vc.addValidator(keystore)
proc initValidators(vc: ValidatorClientRef): Future[bool] {.async.} =
info "Loading validators", validatorsDir = vc.config.validatorsDir()
for keystore in listLoadableKeystores(vc.config, vc.keystoreCache):
vc.addValidator(keystore)
let web3signerValidatorsFuts = mapIt(
vc.config.web3signers,
vc.addValidatorsFromWeb3Signer(it))
# We use `allFutures` because all failures are already reported as
# user-visible warnings in `queryValidatorsSource`.
# We don't consider them fatal because the Web3Signer may be experiencing
# a temporary hiccup that will be resolved later.
await allFutures(web3signerValidatorsFuts)
true
proc initClock(vc: ValidatorClientRef): Future[BeaconClock] {.async.} =

View File

@ -589,15 +589,17 @@ proc validatorIndexLoop(service: DutiesServiceRef) {.async.} =
await service.pollForValidatorIndices()
await service.waitForNextSlot()
proc dynamicValidatorsLoop*(service: DutiesServiceRef) {.async.} =
proc dynamicValidatorsLoop*(service: DutiesServiceRef,
web3signerUrl: Uri,
intervalInSeconds: int) {.async.} =
let vc = service.client
doAssert(vc.config.validatorsSourceInverval > 0)
doAssert(intervalInSeconds > 0)
proc addValidatorProc(data: KeystoreData) =
vc.addValidator(data)
var
timeout = minutes(vc.config.validatorsSourceInverval)
timeout = seconds(intervalInSeconds)
exitLoop = false
while not(exitLoop):
@ -606,15 +608,16 @@ proc dynamicValidatorsLoop*(service: DutiesServiceRef) {.async.} =
await sleepAsync(timeout)
timeout =
block:
let res = await vc.config.queryValidatorsSource()
let res = await queryValidatorsSource(web3signerUrl)
if res.isOk():
let keystores = res.get()
debug "Validators source has been polled for validators",
debug "Web3Signer has been polled for validators",
keystores_found = len(keystores),
validators_source = vc.config.validatorsSource
vc.attachedValidators.updateDynamicValidators(keystores,
web3signer_url = web3signerUrl
vc.attachedValidators.updateDynamicValidators(web3signerUrl,
keystores,
addValidatorProc)
minutes(vc.config.validatorsSourceInverval)
seconds(intervalInSeconds)
else:
seconds(5)
false
@ -694,12 +697,13 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
service.validatorRegisterLoop()
else:
nil
dynamicFut =
if vc.config.validatorsSourceInverval > 0:
service.dynamicValidatorsLoop()
dynamicFuts =
if vc.config.web3signerUpdateInterval > 0:
mapIt(vc.config.web3signers,
service.dynamicValidatorsLoop(it, vc.config.web3signerUpdateInterval))
else:
debug "Dynamic validators update loop disabled"
nil
@[]
while true:
# This loop could look much more nicer/better, when
@ -713,8 +717,9 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
FutureBase(indicesFut),
FutureBase(syncFut),
FutureBase(prepareFut),
FutureBase(dynamicFut)
]
for fut in dynamicFuts:
futures.add fut
if not(isNil(registerFut)): futures.add(FutureBase(registerFut))
discard await race(futures)
checkAndRestart(AttesterLoop, attestFut, service.attesterDutiesLoop())
@ -727,9 +732,11 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
if not(isNil(registerFut)):
checkAndRestart(ValidatorRegisterLoop, registerFut,
service.validatorRegisterLoop())
if not(isNil(dynamicFut)):
checkAndRestart(DynamicValidatorsLoop, dynamicFut,
service.dynamicValidatorsLoop())
for i in 0 ..< dynamicFuts.len:
checkAndRestart(DynamicValidatorsLoop, dynamicFuts[i],
service.dynamicValidatorsLoop(
vc.config.web3signers[i],
vc.config.web3signerUpdateInterval))
false
except CancelledError:
debug "Service interrupted"
@ -746,8 +753,9 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
pending.add(prepareFut.cancelAndWait())
if not(isNil(registerFut)) and not(registerFut.finished()):
pending.add(registerFut.cancelAndWait())
if not(isNil(dynamicFut)) and not(dynamicFut.finished()):
pending.add(dynamicFut.cancelAndWait())
for dynamicFut in dynamicFuts:
if not dynamicFut.finished():
pending.add(dynamicFut.cancelAndWait())
if not(isNil(service.pollingAttesterDutiesTask)) and
not(service.pollingAttesterDutiesTask.finished()):
pending.add(service.pollingAttesterDutiesTask.cancelAndWait())

View File

@ -114,31 +114,10 @@ proc getValidator*(validators: auto,
Opt.some ValidatorAndIndex(index: ValidatorIndex(idx),
validator: validators[idx])
proc addValidators*(node: BeaconNode) =
info "Loading validators", validatorsDir = node.config.validatorsDir(),
keystore_cache_available = not(isNil(node.keystoreCache))
let epoch = node.currentSlot().epoch
for keystore in listLoadableKeystores(node.config, node.keystoreCache):
let
data = withState(node.dag.headState):
getValidator(forkyState.data.validators.asSeq(), keystore.pubkey)
index =
if data.isSome():
Opt.some(data.get().index)
else:
Opt.none(ValidatorIndex)
feeRecipient = node.consensusManager[].getFeeRecipient(
keystore.pubkey, index, epoch)
gasLimit = node.consensusManager[].getGasLimit(keystore.pubkey)
v = node.attachedValidators[].addValidator(keystore, feeRecipient,
gasLimit)
v.updateValidator(data)
proc addValidatorsFromWeb3Signer(node: BeaconNode, web3signerUrl: Uri, epoch: Epoch) {.async.} =
let dynamicStores =
try:
let res = waitFor(queryValidatorsSource(node.config))
let res = await queryValidatorsSource(web3signerUrl)
if res.isErr():
# Error is already reported via log warning.
default(seq[KeystoreData])
@ -166,8 +145,47 @@ proc addValidators*(node: BeaconNode) =
gasLimit)
v.updateValidator(data)
proc pollForDynamicValidators*(node: BeaconNode) {.async.} =
if node.config.validatorsSourceInverval == 0:
proc addValidators*(node: BeaconNode) =
info "Loading validators", validatorsDir = node.config.validatorsDir(),
keystore_cache_available = not(isNil(node.keystoreCache))
let epoch = node.currentSlot().epoch
for keystore in listLoadableKeystores(node.config, node.keystoreCache):
let
data = withState(node.dag.headState):
getValidator(forkyState.data.validators.asSeq(), keystore.pubkey)
index =
if data.isSome():
Opt.some(data.get().index)
else:
Opt.none(ValidatorIndex)
feeRecipient = node.consensusManager[].getFeeRecipient(
keystore.pubkey, index, epoch)
gasLimit = node.consensusManager[].getGasLimit(keystore.pubkey)
v = node.attachedValidators[].addValidator(keystore, feeRecipient,
gasLimit)
v.updateValidator(data)
try:
# We use `allFutures` because all failures are already reported as
# user-visible warnings in `queryValidatorsSource`.
# We don't consider them fatal because the Web3Signer may be experiencing
# a temporary hiccup that will be resolved later.
waitFor allFutures(mapIt(node.config.web3signers,
node.addValidatorsFromWeb3Signer(it, epoch)))
except CatchableError as err:
# This should never happen because all errors are handled within
# `addValidatorsFromWeb3Signer`. Furthermore, the code above is
# using `allFutures` which is guaranteed to not raise exceptions.
# Nevertheless, we need it to make the compiler's exception tracking happy.
debug "Unexpected error while fetching the list of validators from a remote signer",
err = err.msg
proc pollForDynamicValidators*(node: BeaconNode,
web3signerUrl: Uri,
intervalInSeconds: int) {.async.} =
if intervalInSeconds == 0:
return
proc addValidatorProc(keystore: KeystoreData) =
@ -182,7 +200,7 @@ proc pollForDynamicValidators*(node: BeaconNode) {.async.} =
gasLimit)
var
timeout = minutes(node.config.validatorsSourceInverval)
timeout = seconds(intervalInSeconds)
exitLoop = false
while not(exitLoop):
@ -191,15 +209,16 @@ proc pollForDynamicValidators*(node: BeaconNode) {.async.} =
await sleepAsync(timeout)
timeout =
block:
let res = await node.config.queryValidatorsSource()
let res = await queryValidatorsSource(web3signerUrl)
if res.isOk():
let keystores = res.get()
debug "Validators source has been polled for validators",
keystores_found = len(keystores),
validators_source = node.config.validatorsSource
node.attachedValidators.updateDynamicValidators(keystores,
web3signer_url = web3signerUrl
node.attachedValidators.updateDynamicValidators(web3signerUrl,
keystores,
addValidatorProc)
minutes(node.config.validatorsSourceInverval)
seconds(intervalInSeconds)
else:
# In case of error we going to repeat our call with much smaller
# interval.

View File

@ -629,15 +629,11 @@ proc existsKeystore(keystoreDir: string,
return true
false
proc queryValidatorsSource*(config: AnyConf): Future[QueryResult] {.async.} =
proc queryValidatorsSource*(web3signerUrl: Uri): Future[QueryResult] {.async.} =
var keystores: seq[KeystoreData]
if config.validatorsSource.isNone() or
len(config.validatorsSource.get()) == 0:
return QueryResult.ok(keystores)
let vsource = config.validatorsSource.get()
logScope:
validators_source = vsource
web3signer_url = web3signerUrl
let
httpFlags: HttpClientFlags = {}
@ -645,7 +641,7 @@ proc queryValidatorsSource*(config: AnyConf): Future[QueryResult] {.async.} =
socketFlags = {SocketFlags.TcpNoDelay}
client =
block:
let res = RestClientRef.new(vsource, prestoFlags,
let res = RestClientRef.new($web3signerUrl, prestoFlags,
httpFlags, socketFlags = socketFlags)
if res.isErr():
warn "Unable to resolve validator's source distributed signer " &
@ -686,7 +682,7 @@ proc queryValidatorsSource*(config: AnyConf): Future[QueryResult] {.async.} =
handle: FileLockHandle(opened: false),
pubkey: pubkey,
remotes: @[RemoteSignerInfo(
url: HttpHostUri(parseUri(vsource)),
url: HttpHostUri(web3signerUrl),
pubkey: pubkey)],
flags: {RemoteKeystoreFlag.DynamicKeystore},
remoteType: RemoteSignerType.Web3Signer))

View File

@ -380,6 +380,7 @@ func triggersDoppelganger*(
v.isSome() and v[].triggersDoppelganger(epoch)
proc updateDynamicValidators*(pool: ref ValidatorPool,
web3signerUrl: Uri,
keystores: openArray[KeystoreData],
addProc: AddValidatorProc) =
var
@ -399,7 +400,12 @@ proc updateDynamicValidators*(pool: ref ValidatorPool,
if keystore.isSome():
# Just update validator's `data` field with new data from keystore.
validator.data = keystore.get()
else:
elif validator.data.remotes[0].url == HttpHostUri(web3signerUrl):
# The "dynamic" keystores are guaratneed to not be distributed
# so they have a single remote. This code ensures that we are
# deleting all previous dynamically obtained keystores which
# were associated with a particular Web3Signer when the same
# signer no longer serves them.
deleteValidators.add(validator.pubkey)
for pubkey in deleteValidators:
@ -517,7 +523,7 @@ proc getBlockSignature*(v: AttachedValidator, fork: Fork,
fork, genesis_validators_root, slot, block_root,
v.data.privateKey).toValidatorSig())
of ValidatorKind.Remote:
let web3SignerRequest =
let web3signerRequest =
when blck is ForkedBlindedBeaconBlock:
case blck.kind
of ConsensusFork.Phase0, ConsensusFork.Altair, ConsensusFork.Bellatrix:
@ -617,7 +623,7 @@ proc getBlockSignature*(v: AttachedValidator, fork: Fork,
Web3SignerForkedBeaconBlock(kind: ConsensusFork.Deneb,
data: blck.denebData.toBeaconBlockHeader),
proofs)
await v.signData(web3SignerRequest)
await v.signData(web3signerRequest)
# https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/deneb/validator.md#constructing-the-signedblobsidecars
proc getBlobSignature*(v: AttachedValidator, fork: Fork,

View File

@ -33,8 +33,8 @@ The following options are available:
--network The Eth2 network to join [=mainnet].
-d, --data-dir The directory where nimbus will store all blockchain data.
--validators-dir A directory containing validator keystores.
--validators-source Remote Web3Signer URL that will be used as a source of validators.
--validators-source-interval Number of minutes between validator list updates [=60].
--web3-signer-url Remote Web3Signer URL that will be used as a source of validators.
--web3-signer-update-interval Number of seconds between validator list updates [=3600].
--secrets-dir A directory containing validator keystore passwords.
--wallets-dir A directory containing wallet files.
--web3-url One or more execution layer Engine API URLs.

View File

@ -3,9 +3,19 @@
[Web3Signer](https://docs.web3signer.consensys.net/en/latest/) is a remote signing server developed by Consensys.
It offers a [standardized REST API](https://consensys.github.io/web3signer/web3signer-eth2.html) allowing the Nimbus beacon node or validator client to operate without storing any validator keys locally.
Remote validators can be permanently added to a Nimbus installation (or more precisely to a particular [data directory](./data-dir.md)) either on-the-fly through the [`POST /eth/v1/remotekeys`](https://ethereum.github.io/keymanager-APIs/#/Remote%20Key%20Manager/ImportRemoteKeys) request when the [Keymanager API](./keymanager-api.md) is enabled or by manually creating a remote keystore file within the [validators directory](./data-dir.md#secrets-and-validators) of the client which will be loaded upon the next restart.
You can instruct Nimbus to connect to a Web3Signer instance by supplying the `--web3-signer-url` command-line option. Since Nimbus obtains the list of validator keys automatically through the [`/api/v1/eth2/publicKeys`](https://consensys.github.io/web3signer/web3signer-eth2.html#tag/Public-Key/operation/ETH2_LIST) Web3Signer API endpoint, no further configuration is required.
Here is an example `remote_keystore.json` file:
!!! info
By default, the list of validators will be refreshed once per hour. You can change the number of seconds between two updates with the `--web3signer-update-interval` command-line option.
!!! tip
You can use multiple Web3Signer instances by specifying the `--web3-signer-url` parameter multiple times.
Alternatively, if you prefer not to depend on the automatic validator discovery mechanism or wish to take advantage of the advanced configurations described below, you have the option to permanently add multiple remote validators to a particular Nimbus data directory. This can be accomplished in two ways:
**On-the-fly Addition**: Utilize the [`POST /eth/v1/remotekeys`](https://ethereum.github.io/keymanager-APIs/#/Remote%20Key%20Manager/ImportRemoteKeys) request when the Keymanager API is enabled. This allows you to dynamically add and remove remote validators as needed.
**Manual Configuration**: You can manually create a remote keystore file within the [validators directory](./data-dir.md#secrets-and-validators) of the client. This configuration will be loaded during the next restart of the client. Here is an example `remote_keystore.json` file:
```
{

View File

@ -25,10 +25,14 @@ func createLocal(pubkey: ValidatorPubKey): KeystoreData =
func createRemote(pubkey: ValidatorPubKey): KeystoreData =
KeystoreData(kind: KeystoreKind.Remote, pubkey: pubkey)
func createDynamic(pubkey: ValidatorPubKey): KeystoreData =
func createDynamic(url: Uri, pubkey: ValidatorPubKey): KeystoreData =
KeystoreData(kind: KeystoreKind.Remote, pubkey: pubkey,
remotes: @[RemoteSignerInfo(url: HttpHostUri(url))],
flags: {RemoteKeystoreFlag.DynamicKeystore})
const
remoteSignerUrl = parseUri("http://nimbus.team/signer1")
func makeValidatorAndIndex(
index: ValidatorIndex, activation_epoch: Epoch): Opt[ValidatorAndIndex] =
Opt.some ValidatorAndIndex(
@ -159,7 +163,7 @@ suite "Validator pool":
config =
try:
BeaconNodeConf.load(cmdLine =
mapIt(["--validators-source=http://" & $serverAddress], it))
mapIt(["--web3-signer-url=http://" & $serverAddress], it))
except Exception as exc:
raiseAssert exc.msg
@ -167,27 +171,30 @@ suite "Validator pool":
try:
block:
testStage = 0
let res = await queryValidatorsSource(config)
let res = await queryValidatorsSource(config.web3signers[0])
check:
res.isOk()
checkResponse(
res.get(),
[createDynamic(createPubKey(1)), createDynamic(createPubKey(2))])
[
createDynamic(remoteSignerUrl, createPubKey(1)),
createDynamic(remoteSignerUrl, createPubKey(2))
])
block:
testStage = 1
let res = await queryValidatorsSource(config)
let res = await queryValidatorsSource(config.web3signers[0])
check:
res.isOk()
checkResponse(res.get(), [createDynamic(createPubKey(1))])
checkResponse(res.get(), [createDynamic(remoteSignerUrl, createPubKey(1))])
block:
testStage = 2
let res = await queryValidatorsSource(config)
let res = await queryValidatorsSource(config.web3signers[0])
check:
res.isOk()
len(res.get()) == 0
block:
testStage = 3
let res = await queryValidatorsSource(config)
let res = await queryValidatorsSource(config.web3signers[0])
check:
res.isErr()
finally:
@ -214,7 +221,7 @@ suite "Validator pool":
var pool = (ref ValidatorPool)()
discard pool[].addValidator(createLocal(createPubKey(1)), fee, gas)
discard pool[].addValidator(createRemote(createPubKey(2)), fee, gas)
discard pool[].addValidator(createDynamic(createPubKey(3)), fee, gas)
discard pool[].addValidator(createDynamic(remoteSignerUrl, createPubKey(3)), fee, gas)
proc addValidator(data: KeystoreData) {.gcsafe.} =
discard pool[].addValidator(data, fee, gas)
@ -225,16 +232,16 @@ suite "Validator pool":
expected = [
createLocal(createPubKey(1)),
createRemote(createPubKey(2)),
createDynamic(createPubKey(3)),
createDynamic(createPubKey(4)),
createDynamic(createPubKey(5))
createDynamic(remoteSignerUrl, createPubKey(3)),
createDynamic(remoteSignerUrl, createPubKey(4)),
createDynamic(remoteSignerUrl, createPubKey(5))
]
keystores = [
createDynamic(createPubKey(3)),
createDynamic(createPubKey(4)),
createDynamic(createPubKey(5))
createDynamic(remoteSignerUrl, createPubKey(3)),
createDynamic(remoteSignerUrl, createPubKey(4)),
createDynamic(remoteSignerUrl, createPubKey(5))
]
pool.updateDynamicValidators(keystores, addValidator)
pool.updateDynamicValidators(remoteSignerUrl, keystores, addValidator)
pool[].checkPool(expected)
# Removing dynamic keystores.
@ -243,12 +250,12 @@ suite "Validator pool":
expected = [
createLocal(createPubKey(1)),
createRemote(createPubKey(2)),
createDynamic(createPubKey(3))
createDynamic(remoteSignerUrl, createPubKey(3))
]
keystores = [
createDynamic(createPubKey(3)),
createDynamic(remoteSignerUrl, createPubKey(3)),
]
pool.updateDynamicValidators(keystores, addValidator)
pool.updateDynamicValidators(remoteSignerUrl, keystores, addValidator)
pool[].checkPool(expected)
# Adding and removing keystores at same time.
@ -257,14 +264,14 @@ suite "Validator pool":
expected = [
createLocal(createPubKey(1)),
createRemote(createPubKey(2)),
createDynamic(createPubKey(4)),
createDynamic(createPubKey(5))
createDynamic(remoteSignerUrl, createPubKey(4)),
createDynamic(remoteSignerUrl, createPubKey(5))
]
keystores = [
createDynamic(createPubKey(4)),
createDynamic(createPubKey(5))
createDynamic(remoteSignerUrl, createPubKey(4)),
createDynamic(remoteSignerUrl, createPubKey(5))
]
pool.updateDynamicValidators(keystores, addValidator)
pool.updateDynamicValidators(remoteSignerUrl, keystores, addValidator)
pool[].checkPool(expected)
# Adding dynamic keystores with keys which are static.
@ -273,14 +280,14 @@ suite "Validator pool":
expected = [
createLocal(createPubKey(1)),
createRemote(createPubKey(2)),
createDynamic(createPubKey(3))
createDynamic(remoteSignerUrl, createPubKey(3))
]
keystores = [
createDynamic(createPubKey(1)),
createDynamic(createPubKey(2)),
createDynamic(createPubKey(3)),
createDynamic(remoteSignerUrl, createPubKey(1)),
createDynamic(remoteSignerUrl, createPubKey(2)),
createDynamic(remoteSignerUrl, createPubKey(3)),
]
pool.updateDynamicValidators(keystores, addValidator)
pool.updateDynamicValidators(remoteSignerUrl, keystores, addValidator)
pool[].checkPool(expected)
# Empty response
@ -291,5 +298,5 @@ suite "Validator pool":
createRemote(createPubKey(2))
]
var keystores: seq[KeystoreData]
pool.updateDynamicValidators(keystores, addValidator)
pool.updateDynamicValidators(remoteSignerUrl, keystores, addValidator)
pool[].checkPool(expected)