status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #1259 from status-im/devel 

merge "devel" into "master"
This commit is contained in:
Ștefan Talpalaru 2020-06-30 00:45:04 +02:00 committed by GitHub
parent db92c2f254 b4db2ad693
commit 4a2e180653
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
130 changed files with 3721 additions and 2906 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.editorconfig Normal file
View File

@ -0,0 +1,14 @@
root = true
[*.nim]
indent_style = space
indent_size = 2
[*.sh]
indent_style = space
indent_size = 2
[Makefile]
ident_size = 2
ident_style = tab

25
.gitmodules vendored
View File

@ -153,3 +153,28 @@
url = https://github.com/status-im/nim-rocksdb.git url = https://github.com/status-im/nim-rocksdb.git
ignore = dirty ignore = dirty
branch = master branch = master
[submodule "vendor/asynctools"]
path = vendor/asynctools
url = https://github.com/cheatfate/asynctools.git
ignore = dirty
branch = master
[submodule "vendor/karax"]
path = vendor/karax
url = https://github.com/pragmagic/karax.git
ignore = dirty
branch = master
[submodule "vendor/jswebsockets"]
path = vendor/jswebsockets
url = https://github.com/stisa/jswebsockets.git
ignore = dirty
branch = master
[submodule "vendor/websocket.nim"]
path = vendor/websocket.nim
url = https://github.com/niv/websocket.nim.git
ignore = dirty
branch = master
[submodule "vendor/nim-chronicles-tail"]
path = vendor/nim-chronicles-tail
url = https://github.com/status-im/nim-chronicles-tail.git
ignore = dirty
branch = master

1
.nvmrc Normal file
View File

@ -0,0 +1 @@
v13.12.0

268
AllTests-minimal.md
View File

@ -1,268 +0,0 @@
AllTests-minimal
===
## Attestation pool processing [Preset: minimal]
```diff
+ Attestations may arrive in any order [Preset: minimal] OK
+ Attestations may overlap, bigger first [Preset: minimal] OK
+ Attestations may overlap, smaller first [Preset: minimal] OK
+ Attestations should be combined [Preset: minimal] OK
+ Can add and retrieve simple attestation [Preset: minimal] OK
+ Fork choice returns block with attestation OK
+ Fork choice returns latest block with no attestations OK
```
OK: 7/7 Fail: 0/7 Skip: 0/7
## Beacon chain DB [Preset: minimal]
```diff
+ empty database [Preset: minimal] OK
+ find ancestors [Preset: minimal] OK
+ sanity check blocks [Preset: minimal] OK
+ sanity check genesis roundtrip [Preset: minimal] OK
+ sanity check states [Preset: minimal] OK
```
OK: 5/5 Fail: 0/5 Skip: 0/5
## Beacon node
```diff
+ Compile OK
```
OK: 1/1 Fail: 0/1 Skip: 0/1
## Beacon state [Preset: minimal]
```diff
+ Smoke test initialize_beacon_state_from_eth1 [Preset: minimal] OK
```
OK: 1/1 Fail: 0/1 Skip: 0/1
## Block pool processing [Preset: minimal]
```diff
+ Can add same block twice [Preset: minimal] OK
+ Reverse order block add & get [Preset: minimal] OK
+ Simple block add&get [Preset: minimal] OK
+ getRef returns nil for missing blocks OK
+ loadTailState gets genesis block on first load [Preset: minimal] OK
+ updateHead updates head and headState [Preset: minimal] OK
+ updateStateData sanity [Preset: minimal] OK
```
OK: 7/7 Fail: 0/7 Skip: 0/7
## Block processing [Preset: minimal]
```diff
+ Attestation gets processed at epoch [Preset: minimal] OK
+ Passes from genesis state, empty block [Preset: minimal] OK
+ Passes from genesis state, no block [Preset: minimal] OK
+ Passes through epoch update, empty block [Preset: minimal] OK
+ Passes through epoch update, no block [Preset: minimal] OK
```
OK: 5/5 Fail: 0/5 Skip: 0/5
## BlockPool finalization tests [Preset: minimal]
```diff
+ init with gaps [Preset: minimal] OK
+ prune heads on finalization [Preset: minimal] OK
```
OK: 2/2 Fail: 0/2 Skip: 0/2
## BlockRef and helpers [Preset: minimal]
```diff
+ getAncestorAt sanity [Preset: minimal] OK
+ isAncestorOf sanity [Preset: minimal] OK
```
OK: 2/2 Fail: 0/2 Skip: 0/2
## BlockSlot and helpers [Preset: minimal]
```diff
+ atSlot sanity [Preset: minimal] OK
+ parent sanity [Preset: minimal] OK
```
OK: 2/2 Fail: 0/2 Skip: 0/2
## Fork Choice + Finality [Preset: minimal]
```diff
+ fork_choice - testing finality #01 OK
+ fork_choice - testing finality #02 OK
+ fork_choice - testing no votes OK
+ fork_choice - testing with votes OK
```
OK: 4/4 Fail: 0/4 Skip: 0/4
## Honest validator
```diff
+ General pubsub topics: OK
+ Mainnet attestation topics OK
```
OK: 2/2 Fail: 0/2 Skip: 0/2
## Interop
```diff
+ Interop genesis OK
+ Interop signatures OK
+ Mocked start private key OK
```
OK: 3/3 Fail: 0/3 Skip: 0/3
## Keystore
```diff
+ Pbkdf2 decryption OK
+ Pbkdf2 encryption OK
+ Pbkdf2 errors OK
```
OK: 3/3 Fail: 0/3 Skip: 0/3
## Mocking utilities
```diff
+ merkle_minimal OK
```
OK: 1/1 Fail: 0/1 Skip: 0/1
## Official - constants & config [Preset: minimal]
```diff
+ BASE_REWARD_FACTOR 64 [Preset: minimal] OK
+ BLS_WITHDRAWAL_PREFIX "0x00" [Preset: minimal] OK
+ CHURN_LIMIT_QUOTIENT 65536 [Preset: minimal] OK
+ CUSTODY_PERIOD_TO_RANDAO_PADDING 2048 [Preset: minimal] OK
DEPOSIT_CONTRACT_ADDRESS "0x1234567890123456789012345678901234567 Skip
+ DOMAIN_AGGREGATE_AND_PROOF "0x06000000" [Preset: minimal] OK
+ DOMAIN_BEACON_ATTESTER "0x01000000" [Preset: minimal] OK
+ DOMAIN_BEACON_PROPOSER "0x00000000" [Preset: minimal] OK
+ DOMAIN_CUSTODY_BIT_SLASHING "0x83000000" [Preset: minimal] OK
+ DOMAIN_DEPOSIT "0x03000000" [Preset: minimal] OK
+ DOMAIN_LIGHT_CLIENT "0x82000000" [Preset: minimal] OK
+ DOMAIN_RANDAO "0x02000000" [Preset: minimal] OK
+ DOMAIN_SELECTION_PROOF "0x05000000" [Preset: minimal] OK
+ DOMAIN_SHARD_COMMITTEE "0x81000000" [Preset: minimal] OK
+ DOMAIN_SHARD_PROPOSAL "0x80000000" [Preset: minimal] OK
+ DOMAIN_VOLUNTARY_EXIT "0x04000000" [Preset: minimal] OK
+ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS 4096 [Preset: minimal] OK
+ EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE 2 [Preset: minimal] OK
+ EFFECTIVE_BALANCE_INCREMENT 1000000000 [Preset: minimal] OK
+ EJECTION_BALANCE 16000000000 [Preset: minimal] OK
+ EPOCHS_PER_CUSTODY_PERIOD 2048 [Preset: minimal] OK
+ EPOCHS_PER_ETH1_VOTING_PERIOD 4 [Preset: minimal] OK
+ EPOCHS_PER_HISTORICAL_VECTOR 64 [Preset: minimal] OK
+ EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION 256 [Preset: minimal] OK
+ EPOCHS_PER_SLASHINGS_VECTOR 64 [Preset: minimal] OK
+ ETH1_FOLLOW_DISTANCE 16 [Preset: minimal] OK
+ GASPRICE_ADJUSTMENT_COEFFICIENT 8 [Preset: minimal] OK
+ GENESIS_DELAY 300 [Preset: minimal] OK
GENESIS_FORK_VERSION "0x00000001" [Preset: minimal] Skip
+ HISTORICAL_ROOTS_LIMIT 16777216 [Preset: minimal] OK
+ HYSTERESIS_DOWNWARD_MULTIPLIER 1 [Preset: minimal] OK
+ HYSTERESIS_QUOTIENT 4 [Preset: minimal] OK
+ HYSTERESIS_UPWARD_MULTIPLIER 5 [Preset: minimal] OK
+ INACTIVITY_PENALTY_QUOTIENT 16777216 [Preset: minimal] OK
+ INITIAL_ACTIVE_SHARDS 4 [Preset: minimal] OK
+ LIGHT_CLIENT_COMMITTEE_PERIOD 256 [Preset: minimal] OK
+ LIGHT_CLIENT_COMMITTEE_SIZE 128 [Preset: minimal] OK
+ MAX_ATTESTATIONS 128 [Preset: minimal] OK
+ MAX_ATTESTER_SLASHINGS 2 [Preset: minimal] OK
+ MAX_COMMITTEES_PER_SLOT 4 [Preset: minimal] OK
+ MAX_CUSTODY_KEY_REVEALS 256 [Preset: minimal] OK
+ MAX_CUSTODY_SLASHINGS 1 [Preset: minimal] OK
+ MAX_DEPOSITS 16 [Preset: minimal] OK
+ MAX_EARLY_DERIVED_SECRET_REVEALS 1 [Preset: minimal] OK
+ MAX_EFFECTIVE_BALANCE 32000000000 [Preset: minimal] OK
+ MAX_EPOCHS_PER_CROSSLINK 4 [Preset: minimal] OK
+ MAX_GASPRICE 16384 [Preset: minimal] OK
+ MAX_PROPOSER_SLASHINGS 16 [Preset: minimal] OK
+ MAX_REVEAL_LATENESS_DECREMENT 128 [Preset: minimal] OK
+ MAX_SEED_LOOKAHEAD 4 [Preset: minimal] OK
+ MAX_SHARDS 8 [Preset: minimal] OK
+ MAX_SHARD_BLOCKS_PER_ATTESTATION 12 [Preset: minimal] OK
+ MAX_SHARD_BLOCK_CHUNKS 4 [Preset: minimal] OK
+ MAX_VALIDATORS_PER_COMMITTEE 2048 [Preset: minimal] OK
+ MAX_VOLUNTARY_EXITS 16 [Preset: minimal] OK
+ MINOR_REWARD_QUOTIENT 256 [Preset: minimal] OK
+ MIN_ATTESTATION_INCLUSION_DELAY 1 [Preset: minimal] OK
+ MIN_DEPOSIT_AMOUNT 1000000000 [Preset: minimal] OK
+ MIN_EPOCHS_TO_INACTIVITY_PENALTY 4 [Preset: minimal] OK
+ MIN_GASPRICE 8 [Preset: minimal] OK
+ MIN_GENESIS_ACTIVE_VALIDATOR_COUNT 64 [Preset: minimal] OK
+ MIN_GENESIS_TIME 1578009600 [Preset: minimal] OK
+ MIN_PER_EPOCH_CHURN_LIMIT 4 [Preset: minimal] OK
+ MIN_SEED_LOOKAHEAD 1 [Preset: minimal] OK
+ MIN_SLASHING_PENALTY_QUOTIENT 32 [Preset: minimal] OK
+ MIN_VALIDATOR_WITHDRAWABILITY_DELAY 256 [Preset: minimal] OK
+ ONLINE_PERIOD 8 [Preset: minimal] OK
+ PHASE_1_FORK_VERSION "0x01000001" [Preset: minimal] OK
+ PHASE_1_GENESIS_SLOT 8 [Preset: minimal] OK
+ PROPOSER_REWARD_QUOTIENT 8 [Preset: minimal] OK
+ RANDAO_PENALTY_EPOCHS 2 [Preset: minimal] OK
+ RANDOM_SUBNETS_PER_VALIDATOR 1 [Preset: minimal] OK
+ SAFE_SLOTS_TO_UPDATE_JUSTIFIED 2 [Preset: minimal] OK
+ SECONDS_PER_ETH1_BLOCK 14 [Preset: minimal] OK
+ SECONDS_PER_SLOT 6 [Preset: minimal] OK
+ SHARD_BLOCK_CHUNK_SIZE 262144 [Preset: minimal] OK
SHARD_BLOCK_OFFSETS [1,2,3,5,8,13,21,34,55,89,144,233] [Pres Skip
+ SHARD_COMMITTEE_PERIOD 64 [Preset: minimal] OK
+ SHUFFLE_ROUND_COUNT 10 [Preset: minimal] OK
+ SLOTS_PER_EPOCH 8 [Preset: minimal] OK
+ SLOTS_PER_HISTORICAL_ROOT 64 [Preset: minimal] OK
+ TARGET_AGGREGATORS_PER_COMMITTEE 16 [Preset: minimal] OK
+ TARGET_COMMITTEE_SIZE 4 [Preset: minimal] OK
+ TARGET_SHARD_BLOCK_SIZE 196608 [Preset: minimal] OK
+ VALIDATOR_REGISTRY_LIMIT 1099511627776 [Preset: minimal] OK
+ WHISTLEBLOWER_REWARD_QUOTIENT 512 [Preset: minimal] OK
```
OK: 83/86 Fail: 0/86 Skip: 3/86
## PeerPool testing suite
```diff
+ Access peers by key test OK
+ Acquire from empty pool OK
+ Acquire/Sorting and consistency test OK
+ Iterators test OK
+ Peer lifetime test OK
+ Safe/Clear test OK
+ Score check test OK
+ addPeer() test OK
+ addPeerNoWait() test OK
+ deletePeer() test OK
```
OK: 10/10 Fail: 0/10 Skip: 0/10
## SSZ dynamic navigator
```diff
+ navigating fields OK
```
OK: 1/1 Fail: 0/1 Skip: 0/1
## SSZ navigator
```diff
+ basictype OK
+ lists with max size OK
+ simple object fields OK
```
OK: 3/3 Fail: 0/3 Skip: 0/3
## Spec helpers
```diff
+ integer_squareroot OK
```
OK: 1/1 Fail: 0/1 Skip: 0/1
## Sync protocol
```diff
+ Compile OK
```
OK: 1/1 Fail: 0/1 Skip: 0/1
## Zero signature sanity checks
```diff
+ SSZ serialization roundtrip of SignedBeaconBlockHeader OK
```
OK: 1/1 Fail: 0/1 Skip: 0/1
## [Unit - Spec - Block processing] Attestations [Preset: minimal]
```diff
+ Valid attestation OK
+ Valid attestation from previous epoch OK
```
OK: 2/2 Fail: 0/2 Skip: 0/2
## [Unit - Spec - Block processing] Deposits [Preset: minimal]
```diff
+ Deposit at MAX_EFFECTIVE_BALANCE balance (32 ETH) OK
+ Deposit over MAX_EFFECTIVE_BALANCE balance (32 ETH) OK
+ Deposit under MAX_EFFECTIVE_BALANCE balance (32 ETH) OK
+ Validator top-up OK
```
OK: 4/4 Fail: 0/4 Skip: 0/4
## [Unit - Spec - Epoch processing] Justification and Finalization [Preset: minimal]
```diff
+ Rule I - 234 finalization with enough support OK
+ Rule I - 234 finalization without support OK
+ Rule II - 23 finalization with enough support OK
+ Rule II - 23 finalization without support OK
+ Rule III - 123 finalization with enough support OK
+ Rule III - 123 finalization without support OK
+ Rule IV - 12 finalization with enough support OK
+ Rule IV - 12 finalization without support OK
```
OK: 8/8 Fail: 0/8 Skip: 0/8
## hash
```diff
+ HashArray OK
```
OK: 1/1 Fail: 0/1 Skip: 0/1
---TOTAL---
OK: 160/163 Fail: 0/163 Skip: 3/163

183
FixtureAll-minimal.md
View File

@ -1,183 +0,0 @@
FixtureAll-minimal
===
## Official - Epoch Processing - Final updates [Preset: minimal]
```diff
+ Final updates - effective_balance_hysteresis [Preset: minimal] OK
+ Final updates - eth1_vote_no_reset [Preset: minimal] OK
+ Final updates - eth1_vote_reset [Preset: minimal] OK
+ Final updates - historical_root_accumulator [Preset: minimal] OK
```
OK: 4/4 Fail: 0/4 Skip: 0/4
## Official - Epoch Processing - Justification & Finalization [Preset: minimal]
```diff
+ Justification & Finalization - 123_ok_support [Preset: minimal] OK
+ Justification & Finalization - 123_poor_support [Preset: minimal] OK
+ Justification & Finalization - 12_ok_support [Preset: minimal] OK
+ Justification & Finalization - 12_ok_support_messed_target [Preset: minimal] OK
+ Justification & Finalization - 12_poor_support [Preset: minimal] OK
+ Justification & Finalization - 234_ok_support [Preset: minimal] OK
+ Justification & Finalization - 234_poor_support [Preset: minimal] OK
+ Justification & Finalization - 23_ok_support [Preset: minimal] OK
+ Justification & Finalization - 23_poor_support [Preset: minimal] OK
```
OK: 9/9 Fail: 0/9 Skip: 0/9
## Official - Epoch Processing - Registry updates [Preset: minimal]
```diff
+ Registry updates - activation_queue_activation_and_ejection [Preset: minimal] OK
+ Registry updates - activation_queue_efficiency [Preset: minimal] OK
+ Registry updates - activation_queue_no_activation_no_finality [Preset: minimal] OK
+ Registry updates - activation_queue_sorting [Preset: minimal] OK
+ Registry updates - activation_queue_to_activated_if_finalized [Preset: minimal] OK
+ Registry updates - add_to_activation_queue [Preset: minimal] OK
+ Registry updates - ejection [Preset: minimal] OK
+ Registry updates - ejection_past_churn_limit [Preset: minimal] OK
```
OK: 8/8 Fail: 0/8 Skip: 0/8
## Official - Epoch Processing - Slashings [Preset: minimal]
```diff
+ Slashings - max_penalties [Preset: minimal] OK
+ Slashings - scaled_penalties [Preset: minimal] OK
+ Slashings - small_penalty [Preset: minimal] OK
```
OK: 3/3 Fail: 0/3 Skip: 0/3
## Official - Operations - Attestations [Preset: minimal]
```diff
+ [Invalid] after_epoch_slots OK
+ [Invalid] bad_source_root OK
+ [Invalid] before_inclusion_delay OK
+ [Invalid] empty_participants_seemingly_valid_sig OK
+ [Invalid] empty_participants_zeroes_sig OK
+ [Invalid] future_target_epoch OK
+ [Invalid] invalid_attestation_signature OK
+ [Invalid] invalid_current_source_root OK
+ [Invalid] invalid_index OK
+ [Invalid] mismatched_target_and_slot OK
+ [Invalid] new_source_epoch OK
+ [Invalid] old_source_epoch OK
+ [Invalid] old_target_epoch OK
+ [Invalid] source_root_is_target_root OK
+ [Invalid] too_few_aggregation_bits OK
+ [Invalid] too_many_aggregation_bits OK
+ [Invalid] wrong_index_for_committee_signature OK
+ [Invalid] wrong_index_for_slot OK
+ [Valid] success OK
+ [Valid] success_multi_proposer_index_iterations OK
+ [Valid] success_previous_epoch OK
```
OK: 21/21 Fail: 0/21 Skip: 0/21
## Official - Operations - Attester slashing [Preset: minimal]
```diff
+ [Invalid] att1_bad_extra_index OK
+ [Invalid] att1_bad_replaced_index OK
+ [Invalid] att1_duplicate_index_double_signed OK
+ [Invalid] att1_duplicate_index_normal_signed OK
+ [Invalid] att2_bad_extra_index OK
+ [Invalid] att2_bad_replaced_index OK
+ [Invalid] att2_duplicate_index_double_signed OK
+ [Invalid] att2_duplicate_index_normal_signed OK
+ [Invalid] invalid_sig_1 OK
+ [Invalid] invalid_sig_1_and_2 OK
+ [Invalid] invalid_sig_2 OK
+ [Invalid] no_double_or_surround OK
+ [Invalid] participants_already_slashed OK
+ [Invalid] same_data OK
+ [Invalid] unsorted_att_1 OK
+ [Invalid] unsorted_att_2 OK
+ [Valid] success_already_exited_long_ago OK
+ [Valid] success_already_exited_recent OK
+ [Valid] success_double OK
+ [Valid] success_surround OK
```
OK: 20/20 Fail: 0/20 Skip: 0/20
## Official - Operations - Block header [Preset: minimal]
```diff
+ [Invalid] invalid_multiple_blocks_single_slot OK
+ [Invalid] invalid_parent_root OK
+ [Invalid] invalid_proposer_index OK
+ [Invalid] invalid_slot_block_header OK
+ [Invalid] proposer_slashed OK
+ [Valid] success_block_header OK
```
OK: 6/6 Fail: 0/6 Skip: 0/6
## Official - Operations - Deposits [Preset: minimal]
```diff
+ [Invalid] bad_merkle_proof OK
+ [Invalid] wrong_deposit_for_deposit_count OK
+ [Valid] invalid_sig_new_deposit OK
+ [Valid] invalid_sig_other_version OK
+ [Valid] invalid_sig_top_up OK
+ [Valid] invalid_withdrawal_credentials_top_up OK
+ [Valid] new_deposit_max OK
+ [Valid] new_deposit_over_max OK
+ [Valid] new_deposit_under_max OK
+ [Valid] success_top_up OK
+ [Valid] valid_sig_but_forked_state OK
```
OK: 11/11 Fail: 0/11 Skip: 0/11
## Official - Operations - Proposer slashing [Preset: minimal]
```diff
+ [Invalid] identifier OK
+ [Valid] identifier OK
```
OK: 2/2 Fail: 0/2 Skip: 0/2
## Official - Operations - Voluntary exit [Preset: minimal]
```diff
+ [Invalid] invalid_signature OK
+ [Invalid] validator_already_exited OK
+ [Invalid] validator_exit_in_future OK
+ [Invalid] validator_invalid_validator_index OK
+ [Invalid] validator_not_active OK
+ [Invalid] validator_not_active_long_enough OK
+ [Valid] default_exit_epoch_subsequent_exit OK
+ [Valid] success OK
+ [Valid] success_exit_queue OK
```
OK: 9/9 Fail: 0/9 Skip: 0/9
## Official - Sanity - Blocks [Preset: minimal]
```diff
+ [Invalid] double_same_proposer_slashings_same_block OK
+ [Invalid] double_similar_proposer_slashings_same_block OK
+ [Invalid] double_validator_exit_same_block OK
+ [Invalid] duplicate_attester_slashing OK
+ [Invalid] expected_deposit_in_block OK
+ [Invalid] invalid_block_sig OK
+ [Invalid] invalid_proposer_index_sig_from_expected_proposer OK
+ [Invalid] invalid_proposer_index_sig_from_proposer_index OK
+ [Invalid] invalid_state_root OK
+ [Invalid] parent_from_same_slot OK
+ [Invalid] prev_slot_block_transition OK
+ [Invalid] proposal_for_genesis_slot OK
+ [Invalid] same_slot_block_transition OK
+ [Invalid] zero_block_sig OK
+ [Valid] attestation OK
+ [Valid] attester_slashing OK
+ [Valid] balance_driven_status_transitions OK
+ [Valid] deposit_in_block OK
+ [Valid] deposit_top_up OK
+ [Valid] empty_block_transition OK
+ [Valid] empty_epoch_transition OK
+ [Valid] empty_epoch_transition_not_finalizing OK
+ [Valid] high_proposer_index OK
+ [Valid] historical_batch OK
+ [Valid] multiple_attester_slashings_no_overlap OK
+ [Valid] multiple_attester_slashings_partial_overlap OK
+ [Valid] multiple_different_proposer_slashings_same_block OK
+ [Valid] multiple_different_validator_exits_same_block OK
+ [Valid] proposer_after_inactive_index OK
+ [Valid] proposer_slashing OK
+ [Valid] skipped_slots OK
+ [Valid] voluntary_exit OK
```
OK: 32/32 Fail: 0/32 Skip: 0/32
## Official - Sanity - Slots [Preset: minimal]
```diff
+ Slots - double_empty_epoch OK
+ Slots - empty_epoch OK
+ Slots - over_epoch_boundary OK
+ Slots - slots_1 OK
+ Slots - slots_2 OK
```
OK: 5/5 Fail: 0/5 Skip: 0/5
---TOTAL---
OK: 130/130 Fail: 0/130 Skip: 0/130

36
FixtureSSZConsensus-minimal.md
View File

@ -1,36 +0,0 @@
FixtureSSZConsensus-minimal
===
## Official - SSZ consensus objects [Preset: minimal]
```diff
+ Testing AggregateAndProof OK
+ Testing Attestation OK
+ Testing AttestationData OK
+ Testing AttesterSlashing OK
+ Testing BeaconBlock OK
+ Testing BeaconBlockBody OK
+ Testing BeaconBlockHeader OK
+ Testing BeaconState OK
+ Testing Checkpoint OK
+ Testing Deposit OK
+ Testing DepositData OK
+ Testing DepositMessage OK
+ Testing Eth1Block OK
+ Testing Eth1Data OK
+ Testing Fork OK
+ Testing ForkData OK
+ Testing HistoricalBatch OK
+ Testing IndexedAttestation OK
+ Testing PendingAttestation OK
+ Testing ProposerSlashing OK
+ Testing SignedAggregateAndProof OK
+ Testing SignedBeaconBlock OK
+ Testing SignedBeaconBlockHeader OK
+ Testing SignedVoluntaryExit OK
+ Testing SigningData OK
+ Testing Validator OK
+ Testing VoluntaryExit OK
```
OK: 27/27 Fail: 0/27 Skip: 0/27
---TOTAL---
OK: 27/27 Fail: 0/27 Skip: 0/27

21
FixtureSSZGeneric-minimal.md
View File

@ -1,21 +0,0 @@
FixtureSSZGeneric-minimal
===
## Official - SSZ generic types
```diff
Testing basic_vector inputs - invalid - skipping Vector[uint128, N] and Vector[uint256, N] Skip
+ Testing basic_vector inputs - valid - skipping Vector[uint128, N] and Vector[uint256, N] OK
+ Testing bitlist inputs - invalid OK
+ Testing bitlist inputs - valid OK
Testing bitvector inputs - invalid Skip
+ Testing bitvector inputs - valid OK
+ Testing boolean inputs - invalid OK
+ Testing boolean inputs - valid OK
+ Testing containers inputs - invalid - skipping BitsStruct OK
+ Testing containers inputs - valid - skipping BitsStruct OK
+ Testing uints inputs - invalid - skipping uint128 and uint256 OK
+ Testing uints inputs - valid - skipping uint128 and uint256 OK
```
OK: 10/12 Fail: 0/12 Skip: 2/12
---TOTAL---
OK: 10/12 Fail: 0/12 Skip: 2/12

60
Makefile
View File

@ -15,23 +15,25 @@ BUILD_SYSTEM_DIR := vendor/nimbus-build-system
# unconditionally built by the default Make target # unconditionally built by the default Make target
TOOLS := \ TOOLS := \
validator_client \
beacon_node \ beacon_node \
block_sim \
deposit_contract \
inspector \ inspector \
logtrace \ logtrace \
deposit_contract \ nbench \
nbench_spec_scenarios \
ncli_db \
ncli_hash_tree_root \ ncli_hash_tree_root \
ncli_pretty \ ncli_pretty \
ncli_query \ ncli_query \
ncli_transition \ ncli_transition \
ncli_db \
process_dashboard \ process_dashboard \
stack_sizes \ stack_sizes \
state_sim \ state_sim \
block_sim \ validator_client
nbench \
nbench_spec_scenarios
# bench_bls_sig_agggregation TODO reenable after bls v0.10.1 changes # bench_bls_sig_agggregation TODO reenable after bls v0.10.1 changes
TOOLS_DIRS := \ TOOLS_DIRS := \
beacon_chain \ beacon_chain \
benchmarks \ benchmarks \
@ -47,7 +49,7 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
update \ update \
test \ test \
$(TOOLS) \ $(TOOLS) \
clean_eth2_network_simulation_files \ clean_eth2_network_simulation_all \
eth2_network_simulation \ eth2_network_simulation \
clean-testnet0 \ clean-testnet0 \
testnet0 \ testnet0 \
@ -55,12 +57,6 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
testnet1 \ testnet1 \
clean \ clean \
libbacktrace \ libbacktrace \
clean-schlesi \
schlesi \
schlesi-dev \
clean-witti \
witti \
witti-dev \
book \ book \
publish-book publish-book
@ -99,7 +95,7 @@ else
NIM_PARAMS := $(NIM_PARAMS) -d:release NIM_PARAMS := $(NIM_PARAMS) -d:release
endif endif
deps: | deps-common beacon_chain.nims deps: | deps-common nat-libs beacon_chain.nims
ifneq ($(USE_LIBBACKTRACE), 0) ifneq ($(USE_LIBBACKTRACE), 0)
deps: | libbacktrace deps: | libbacktrace
endif endif
@ -131,11 +127,14 @@ $(TOOLS): | build deps
echo -e $(BUILD_MSG) "build/$@" && \ echo -e $(BUILD_MSG) "build/$@" && \
$(ENV_SCRIPT) nim c -o:build/$@ $(NIM_PARAMS) "$${TOOL_DIR}/$@.nim" $(ENV_SCRIPT) nim c -o:build/$@ $(NIM_PARAMS) "$${TOOL_DIR}/$@.nim"
clean_eth2_network_simulation_files: clean_eth2_network_simulation_data:
rm -rf tests/simulation/data
clean_eth2_network_simulation_all:
rm -rf tests/simulation/{data,validators} rm -rf tests/simulation/{data,validators}
eth2_network_simulation: | build deps clean_eth2_network_simulation_files eth2_network_simulation: | build deps clean_eth2_network_simulation_data
+ GIT_ROOT="$$PWD" NIMFLAGS="$(NIMFLAGS)" LOG_LEVEL="$(LOG_LEVEL)" tests/simulation/start.sh + GIT_ROOT="$$PWD" NIMFLAGS="$(NIMFLAGS)" LOG_LEVEL="$(LOG_LEVEL)" tests/simulation/start-in-tmux.sh
clean-testnet0: clean-testnet0:
rm -rf build/data/testnet0* rm -rf build/data/testnet0*
@ -147,25 +146,24 @@ clean-testnet1:
# - try SCRIPT_PARAMS="--skipGoerliKey" # - try SCRIPT_PARAMS="--skipGoerliKey"
testnet0 testnet1: | build deps testnet0 testnet1: | build deps
source scripts/$@.env; \ source scripts/$@.env; \
NIM_PARAMS="$(subst ",\",$(NIM_PARAMS))" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims $(SCRIPT_PARAMS) --const-preset=$$CONST_PRESET --dev-build $@ NIM_PARAMS="$(NIM_PARAMS)" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims $(SCRIPT_PARAMS) --const-preset=$$CONST_PRESET --dev-build $@
clean-schlesi: clean-altona:
rm -rf build/data/shared_schlesi* rm -rf build/data/shared_altona*
schlesi: | build deps altona: | build deps
NIM_PARAMS="$(subst ",\",$(NIM_PARAMS))" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims $(SCRIPT_PARAMS) shared/schlesi NIM_PARAMS="$(NIM_PARAMS)" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims $(SCRIPT_PARAMS) shared/altona
schlesi-dev: | build deps altona-dev: | build deps
NIM_PARAMS="$(subst ",\",$(NIM_PARAMS))" LOG_LEVEL="DEBUG; TRACE:discv5,networking; REQUIRED:none; DISABLED:none" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims $(SCRIPT_PARAMS) shared/schlesi NIM_PARAMS="$(NIM_PARAMS)" LOG_LEVEL="DEBUG; TRACE:discv5,networking; REQUIRED:none; DISABLED:none" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims $(SCRIPT_PARAMS) shared/altona
clean-witti: ctail: | build deps
rm -rf build/data/shared_witti* mkdir -p vendor/.nimble/bin/
$(ENV_SCRIPT) nim -d:danger -o:vendor/.nimble/bin/ctail c vendor/nim-chronicles-tail/ctail.nim
witti: | build deps ntu: | build deps
NIM_PARAMS="$(subst ",\",$(NIM_PARAMS))" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims $(SCRIPT_PARAMS) shared/witti mkdir -p vendor/.nimble/bin/
$(ENV_SCRIPT) nim -d:danger -o:vendor/.nimble/bin/ntu c vendor/nim-testutils/ntu.nim
witti-dev: | build deps
NIM_PARAMS="$(subst ",\",$(NIM_PARAMS))" LOG_LEVEL="DEBUG; TRACE:discv5,networking; REQUIRED:none; DISABLED:none" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims $(SCRIPT_PARAMS) shared/witti
clean: | clean-common clean: | clean-common
rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*ssz*,beacon_node_*,block_sim,state_sim,transition*} rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*ssz*,beacon_node_*,block_sim,state_sim,transition*}

50
README.md
View File

@ -1,4 +1,5 @@
# Nimbus Eth2 (Beacon Chain) # Nimbus Eth2 (Beacon Chain)
[![Build Status (Travis)](https://img.shields.io/travis/status-im/nim-beacon-chain/master.svg?label=Linux%20/%20macOS "Linux/macOS build status (Travis)")](https://travis-ci.org/status-im/nim-beacon-chain) [![Build Status (Travis)](https://img.shields.io/travis/status-im/nim-beacon-chain/master.svg?label=Linux%20/%20macOS "Linux/macOS build status (Travis)")](https://travis-ci.org/status-im/nim-beacon-chain)
[![Build Status (Azure)](https://dev.azure.com/nimbus-dev/nim-beacon-chain/_apis/build/status/status-im.nim-beacon-chain?branchName=master)](https://dev.azure.com/nimbus-dev/nim-beacon-chain/_build/latest?definitionId=3&branchName=master) [![Build Status (Azure)](https://dev.azure.com/nimbus-dev/nim-beacon-chain/_apis/build/status/status-im.nim-beacon-chain?branchName=master)](https://dev.azure.com/nimbus-dev/nim-beacon-chain/_build/latest?definitionId=3&branchName=master)
[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
@ -6,29 +7,21 @@
![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg) ![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)
[![Discord: Nimbus](https://img.shields.io/badge/discord-nimbus-orange.svg)](https://discord.gg/XRxWahP) [![Discord: Nimbus](https://img.shields.io/badge/discord-nimbus-orange.svg)](https://discord.gg/XRxWahP)
[![Gitter: #status-im/nimbus](https://img.shields.io/badge/gitter-status--im%2Fnimbus-orange.svg)](https://gitter.im/status-im/nimbus)
[![Status: #nimbus-general](https://img.shields.io/badge/status-nimbus--general-orange.svg)](https://join.status.im/nimbus-general) [![Status: #nimbus-general](https://img.shields.io/badge/status-nimbus--general-orange.svg)](https://join.status.im/nimbus-general)
Welcome to Nimbus for Ethereum 2.0.
Nimbus beacon chain is a research implementation of the beacon chain component of the upcoming Ethereum Serenity upgrade, aka Eth2. Nimbus beacon chain is a research implementation of the beacon chain component of the upcoming Ethereum Serenity upgrade, aka Eth2.
## Related <!-- START doctoc generated TOC please keep comment here to allow auto update -->
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
* [status-im/nimbus](https://github.com/status-im/nimbus/): Nimbus for Ethereum 1
* [ethereum/eth2.0-specs](https://github.com/ethereum/eth2.0-specs/tree/v0.12.1#phase-0): Serenity specification that this project implements
You can check where the beacon chain fits in the Ethereum ecosystem our Two-Point-Oh series: https://our.status.im/tag/two-point-oh/ - [Documentation](#documentation)
## Table of Contents
- [Nimbus Eth2 (Beacon Chain)](#nimbus-eth2-beacon-chain)
- [Related](#related) - [Related](#related)
- [Table of Contents](#table-of-contents)
- [Prerequisites for everyone](#prerequisites-for-everyone) - [Prerequisites for everyone](#prerequisites-for-everyone)
- [Linux](#linux) - [Linux](#linux)
- [MacOS](#macos) - [MacOS](#macos)
- [Windows](#windows) - [Windows](#windows)
- [Android](#android)
- [For users](#for-users) - [For users](#for-users)
- [Connecting to testnets](#connecting-to-testnets) - [Connecting to testnets](#connecting-to-testnets)
- [Getting metrics from a local testnet client](#getting-metrics-from-a-local-testnet-client) - [Getting metrics from a local testnet client](#getting-metrics-from-a-local-testnet-client)
@ -46,6 +39,20 @@ You can check where the beacon chain fits in the Ethereum ecosystem our Two-Poin
- [CI setup](#ci-setup) - [CI setup](#ci-setup)
- [License](#license) - [License](#license)
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
## Documentation
You can find complete information about running a beacon node and operating as a validator in [The Book](https://status-im.github.io/nim-beacon-chain/).
## Related projects
* [status-im/nimbus](https://github.com/status-im/nimbus/): Nimbus for Ethereum 1
* [ethereum/eth2.0-specs](https://github.com/ethereum/eth2.0-specs/tree/v0.12.1#phase-0): Serenity specification that this project implements
You can check where the beacon chain fits in the Ethereum ecosystem our Two-Point-Oh series: https://our.status.im/tag/two-point-oh/
## Prerequisites for everyone ## Prerequisites for everyone
At the moment, Nimbus has to be built from source. At the moment, Nimbus has to be built from source.
@ -191,6 +198,21 @@ make VALIDATORS=192 NODES=6 USER_NODES=1 eth2_network_simulation
# looks like from a single nodes' perspective. # looks like from a single nodes' perspective.
``` ```
By default, all validators are loaded within the beacon nodes, but if you want to use
external processes as validator clients you can pass `BN_VC_VALIDATOR_SPLIT=yes` as an
additional argument to the `make eth2_network_simulation` command and that will split
the `VALIDATORS` between beacon nodes and validator clients - for example with `192`
validators and `6` nodes you will end up with 6 beacon node and 6 validator client
processes, where each of them will handle 16 validators.
By default, the simulation will start from a pre-generated genesis state. If you wish to
simulate the bootstrap process with a Ethereum 1.0 validator deposit contract, start the
simulation with `WAIT_GENESIS=yes`
```
make eth2_network_simulation WAIT_GENESIS=yes
```
You can also separate the output from each beacon node in its own panel, using [multitail](http://www.vanheusden.com/multitail/): You can also separate the output from each beacon node in its own panel, using [multitail](http://www.vanheusden.com/multitail/):
```bash ```bash
@ -296,7 +318,9 @@ After cloning the repo:
```bash ```bash
# The first `make` invocation will update all Git submodules. # The first `make` invocation will update all Git submodules.
# You'll run `make update` after each `git pull`, in the future, to keep those submodules up to date. # You'll run `make update` after each `git pull`, in the future, to keep those submodules up to date.
make
# Build beacon_node and all the tools, using 4 parallel Make jobs
make -j4
# Run tests # Run tests
make test make test

2
azure-pipelines.yml
View File

@ -1,7 +1,7 @@
jobs: jobs:
- job: Windows - job: Windows
timeoutInMinutes: 80 timeoutInMinutes: 90
pool: pool:
vmImage: windows-latest vmImage: windows-latest

28
beacon_chain.nimble
View File

@ -49,7 +49,7 @@ proc buildAndRunBinary(name: string, srcDir = "./", params = "", cmdParams = "",
task moduleTests, "Run all module tests": task moduleTests, "Run all module tests":
buildAndRunBinary "beacon_node", "beacon_chain/", buildAndRunBinary "beacon_node", "beacon_chain/",
"-d:chronicles_log_level=TRACE " & "-d:chronicles_log_level=TRACE " &
"-d:const_preset=minimal -d:ETH2_SPEC=\"v0.12.1\" -d:BLS_ETH2_SPEC=\"v0.12.x\" " & "-d:const_preset=minimal " &
"-d:testutils_test_build" "-d:testutils_test_build"
### tasks ### tasks
@ -58,27 +58,23 @@ task test, "Run all tests":
# pieces of code get tested regularly. Increased test output verbosity is the # pieces of code get tested regularly. Increased test output verbosity is the
# price we pay for that. # price we pay for that.
# Just the part of minimal config which explicitly differs from mainnet
buildAndRunBinary "test_fixture_const_sanity_check", "tests/official/", "-d:const_preset=minimal"
# Mainnet config # Mainnet config
buildAndRunBinary "proto_array", "beacon_chain/fork_choice/", "-d:const_preset=mainnet -d:ETH2_SPEC=\"v0.12.1\" -d:BLS_ETH2_SPEC=\"v0.12.x\"" buildAndRunBinary "proto_array", "beacon_chain/fork_choice/", "-d:const_preset=mainnet"
buildAndRunBinary "fork_choice", "beacon_chain/fork_choice/", "-d:const_preset=mainnet -d:ETH2_SPEC=\"v0.12.1\" -d:BLS_ETH2_SPEC=\"v0.12.x\"" buildAndRunBinary "fork_choice", "beacon_chain/fork_choice/", "-d:const_preset=mainnet"
buildAndRunBinary "all_tests", "tests/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:ETH2_SPEC=\"v0.12.1\" -d:BLS_ETH2_SPEC=\"v0.12.x\"" buildAndRunBinary "all_tests", "tests/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet"
# Generic SSZ test, doesn't use consensus objects minimal/mainnet presets # Generic SSZ test, doesn't use consensus objects minimal/mainnet presets
buildAndRunBinary "test_fixture_ssz_generic_types", "tests/official/", "-d:chronicles_log_level=TRACE" buildAndRunBinary "test_fixture_ssz_generic_types", "tests/official/", "-d:chronicles_log_level=TRACE"
# Consensus object SSZ tests # Consensus object SSZ tests
# 0.11.3 buildAndRunBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet"
buildAndRunBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:ETH2_SPEC=\"v0.11.3\""
# 0.12.1 # 0.12.1
buildAndRunBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:ETH2_SPEC=\"v0.12.1\" -d:BLS_ETH2_SPEC=\"v0.12.x\"" buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet"
# 0.11.3
buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:ETH2_SPEC=\"v0.11.3\""
# 0.12.1
buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:ETH2_SPEC=\"v0.12.1\" -d:BLS_ETH2_SPEC=\"v0.12.x\""
# State sim; getting into 4th epoch useful to trigger consensus checks
buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet -d:ETH2_SPEC=\"v0.12.1\" -d:BLS_ETH2_SPEC=\"v0.12.x\"", "--validators=2000 --slots=128"
# State and block sims; getting to 4th epoch triggers consensus checks
buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet", "--validators=2000 --slots=128"
buildAndRunBinary "block_sim", "research/", "-d:const_preset=mainnet", "--validators=2000 --slots=128"

47
beacon_chain/attestation_aggregation.nim
View File

@ -10,8 +10,10 @@
import import
options, chronicles, options, chronicles,
./spec/[ ./spec/[
beaconstate, datatypes, crypto, digest, helpers, validator, signatures], beaconstate, datatypes, crypto, digest, helpers, network, validator,
./block_pool, ./attestation_pool, ./beacon_node_types, ./ssz signatures],
./block_pool, ./block_pools/candidate_chains, ./attestation_pool,
./beacon_node_types, ./ssz
logScope: logScope:
topics = "att_aggr" topics = "att_aggr"
@ -73,22 +75,14 @@ proc aggregate_attestations*(
none(AggregateAndProof) none(AggregateAndProof)
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#attestation-subnets # https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#attestation-subnets
proc isValidAttestation*( proc isValidAttestation*(
pool: AttestationPool, attestation: Attestation, current_slot: Slot, pool: var AttestationPool, attestation: Attestation, current_slot: Slot,
topicCommitteeIndex: uint64): bool = topicCommitteeIndex: uint64): bool =
logScope: logScope:
topics = "att_aggr valid_att" topics = "att_aggr valid_att"
received_attestation = shortLog(attestation) received_attestation = shortLog(attestation)
# The attestation's committee index (attestation.data.index) is for the
# correct subnet.
if attestation.data.index != topicCommitteeIndex:
debug "attestation's committee index not for the correct subnet",
topicCommitteeIndex = topicCommitteeIndex
return false
if not (attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= if not (attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >=
current_slot and current_slot >= attestation.data.slot): current_slot and current_slot >= attestation.data.slot):
debug "attestation.data.slot not within ATTESTATION_PROPAGATION_SLOT_RANGE" debug "attestation.data.slot not within ATTESTATION_PROPAGATION_SLOT_RANGE"
@ -133,18 +127,35 @@ proc isValidAttestation*(
# TODO: consider a "slush pool" of attestations whose blocks have not yet # TODO: consider a "slush pool" of attestations whose blocks have not yet
# propagated - i.e. imagine that attestations are smaller than blocks and # propagated - i.e. imagine that attestations are smaller than blocks and
# therefore propagate faster, thus reordering their arrival in some nodes # therefore propagate faster, thus reordering their arrival in some nodes
if pool.blockPool.get(attestation.data.beacon_block_root).isNone(): let attestationBlck = pool.blockPool.getRef(attestation.data.beacon_block_root)
if attestationBlck.isNil:
debug "block doesn't exist in block pool" debug "block doesn't exist in block pool"
pool.blockPool.addMissing(attestation.data.beacon_block_root)
return false
pool.blockPool.withState(
pool.blockPool.tmpState,
BlockSlot(blck: attestationBlck, slot: attestation.data.slot)):
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/p2p-interface.md#attestation-subnets
# [REJECT] The attestation is for the correct subnet (i.e.
# compute_subnet_for_attestation(state, attestation) == subnet_id).
let
epochInfo = blck.getEpochInfo(state)
requiredSubnetIndex =
compute_subnet_for_attestation(
epochInfo.shuffled_active_validator_indices.len.uint64, attestation)
if requiredSubnetIndex != topicCommitteeIndex:
debug "isValidAttestation: attestation's committee index not for the correct subnet",
topicCommitteeIndex = topicCommitteeIndex,
attestation_data_index = attestation.data.index,
requiredSubnetIndex = requiredSubnetIndex
return false return false
# The signature of attestation is valid. # The signature of attestation is valid.
# TODO need to know above which validator anyway, and this is too general var cache = getEpochCache(blck, state)
# as it supports aggregated attestations (which this can't be)
var cache = get_empty_per_epoch_cache()
if not is_valid_indexed_attestation( if not is_valid_indexed_attestation(
pool.blockPool.headState.data.data, state, get_indexed_attestation(state, attestation, cache), {}):
get_indexed_attestation(
pool.blockPool.headState.data.data, attestation, cache), {}):
debug "signature verification failed" debug "signature verification failed"
return false return false

16
beacon_chain/attestation_pool.nim
View File

@ -61,7 +61,7 @@ proc slotIndex(
doAssert attestationSlot >= pool.startingSlot, doAssert attestationSlot >= pool.startingSlot,
""" """
We should have checked in validate that attestation is newer than We should have checked in addResolved that attestation is newer than
finalized_slot and we never prune things before that, per below condition! finalized_slot and we never prune things before that, per below condition!
""" & """ &
", attestationSlot: " & $shortLog(attestationSlot) & ", attestationSlot: " & $shortLog(attestationSlot) &
@ -145,6 +145,16 @@ proc addResolved(pool: var AttestationPool, blck: BlockRef, attestation: Attesta
attestation = shortLog(attestation), attestation = shortLog(attestation),
blockSlot = shortLog(blck.slot) blockSlot = shortLog(blck.slot)
return return
if attestation.data.slot < pool.startingSlot:
# It can happen that attestations in blocks for example are included even
# though they no longer are relevant for finalization - let's clear
# these out
debug "Old attestation",
attestation = shortLog(attestation),
startingSlot = pool.startingSlot
return
# if not isValidAttestationSlot(attestation.data.slot, blck.slot): # if not isValidAttestationSlot(attestation.data.slot, blck.slot):
# # Logging in isValidAttestationSlot # # Logging in isValidAttestationSlot
# return # return
@ -161,9 +171,7 @@ proc addResolved(pool: var AttestationPool, blck: BlockRef, attestation: Attesta
# on the state and those that don't to cheaply # on the state and those that don't to cheaply
# discard invalid attestations before rewinding state. # discard invalid attestations before rewinding state.
# TODO: stateCache usage if not isValidAttestationTargetEpoch(state, attestation.data):
var stateCache = get_empty_per_epoch_cache()
if not isValidAttestationTargetEpoch(state, attestation):
notice "Invalid attestation", notice "Invalid attestation",
attestation = shortLog(attestation), attestation = shortLog(attestation),
current_epoch = get_current_epoch(state), current_epoch = get_current_epoch(state),

63
beacon_chain/beacon_chain_db.nim
View File

@ -4,8 +4,8 @@ import
typetraits, stew/[results, objects, endians2], typetraits, stew/[results, objects, endians2],
serialization, chronicles, snappy, serialization, chronicles, snappy,
eth/db/kvstore, eth/db/kvstore,
./spec/[datatypes, digest, crypto], ./spec/[datatypes, digest, crypto, state_transition],
./ssz/[ssz_serialization, merkleization], ./state_transition ./ssz/[ssz_serialization, merkleization]
type type
BeaconChainDB* = ref object BeaconChainDB* = ref object
@ -94,24 +94,31 @@ proc get(db: BeaconChainDB, key: openArray[byte], T: type Eth2Digest): Opt[T] =
res res
proc get(db: BeaconChainDB, key: openArray[byte], T: typedesc): Opt[T] = proc get(db: BeaconChainDB, key: openArray[byte], res: var auto): bool =
var res: Opt[T] var found = false
# TODO address is needed because there's no way to express lifetimes in nim
# we'll use unsafeAddr to find the code later
var resPtr = unsafeAddr res # callback is local, ptr wont escape
proc decode(data: openArray[byte]) = proc decode(data: openArray[byte]) =
try: try:
res.ok SSZ.decode(snappy.decode(data), T) resPtr[] = SSZ.decode(snappy.decode(data), type res)
found = true
except SerializationError as e: except SerializationError as e:
# If the data can't be deserialized, it could be because it's from a # If the data can't be deserialized, it could be because it's from a
# version of the software that uses a different SSZ encoding # version of the software that uses a different SSZ encoding
warn "Unable to deserialize data, old database?", warn "Unable to deserialize data, old database?",
err = e.msg, typ = name(T), dataLen = data.len err = e.msg, typ = name(type res), dataLen = data.len
discard discard
discard db.backend.get(key, decode).expect("working database") discard db.backend.get(key, decode).expect("working database")
res found
proc putBlock*(db: BeaconChainDB, key: Eth2Digest, value: SignedBeaconBlock) = proc putBlock*(db: BeaconChainDB, key: Eth2Digest, value: SignedBeaconBlock) =
db.put(subkey(type value, key), value) db.put(subkey(type value, key), value)
proc putBlock*(db: BeaconChainDB, key: Eth2Digest, value: TrustedSignedBeaconBlock) =
db.put(subkey(SignedBeaconBlock, key), value)
proc putState*(db: BeaconChainDB, key: Eth2Digest, value: BeaconState) = proc putState*(db: BeaconChainDB, key: Eth2Digest, value: BeaconState) =
# TODO prune old states - this is less easy than it seems as we never know # TODO prune old states - this is less easy than it seems as we never know
@ -126,7 +133,9 @@ proc putStateRoot*(db: BeaconChainDB, root: Eth2Digest, slot: Slot,
value: Eth2Digest) = value: Eth2Digest) =
db.put(subkey(root, slot), value) db.put(subkey(root, slot), value)
proc putBlock*(db: BeaconChainDB, value: SignedBeaconBlock) = proc putBlock*(db: BeaconChainDB, value: SomeSignedBeaconBlock) =
# TODO this should perhaps be a TrustedSignedBeaconBlock, but there's no
# trivial way to coerce one type into the other, as it stands..
db.putBlock(hash_tree_root(value.message), value) db.putBlock(hash_tree_root(value.message), value)
proc delBlock*(db: BeaconChainDB, key: Eth2Digest) = proc delBlock*(db: BeaconChainDB, key: Eth2Digest) =
@ -145,8 +154,11 @@ proc putHeadBlock*(db: BeaconChainDB, key: Eth2Digest) =
proc putTailBlock*(db: BeaconChainDB, key: Eth2Digest) = proc putTailBlock*(db: BeaconChainDB, key: Eth2Digest) =
db.put(subkey(kTailBlock), key) db.put(subkey(kTailBlock), key)
proc getBlock*(db: BeaconChainDB, key: Eth2Digest): Opt[SignedBeaconBlock] = proc getBlock*(db: BeaconChainDB, key: Eth2Digest): Opt[TrustedSignedBeaconBlock] =
db.get(subkey(SignedBeaconBlock, key), SignedBeaconBlock) # We only store blocks that we trust in the database
result.ok(TrustedSignedBeaconBlock())
if not db.get(subkey(SignedBeaconBlock, key), result.get):
result.err()
proc getState*( proc getState*(
db: BeaconChainDB, key: Eth2Digest, output: var BeaconState, db: BeaconChainDB, key: Eth2Digest, output: var BeaconState,
@ -159,20 +171,11 @@ proc getState*(
# https://github.com/nim-lang/Nim/issues/14126 # https://github.com/nim-lang/Nim/issues/14126
# TODO RVO is inefficient for large objects: # TODO RVO is inefficient for large objects:
# https://github.com/nim-lang/Nim/issues/13879 # https://github.com/nim-lang/Nim/issues/13879
# TODO address is needed because there's no way to express lifetimes in nim if not db.get(subkey(BeaconState, key), output):
# we'll use unsafeAddr to find the code later rollback(output)
let outputAddr = unsafeAddr output # callback is local false
proc decode(data: openArray[byte]) = else:
try: true
# TODO can't write to output directly..
assign(outputAddr[], SSZ.decode(snappy.decode(data), BeaconState))
except SerializationError as e:
# If the data can't be deserialized, it could be because it's from a
# version of the software that uses a different SSZ encoding
warn "Unable to deserialize data, old database?", err = e.msg
rollback(outputAddr[])
db.backend.get(subkey(BeaconState, key), decode).expect("working database")
proc getStateRoot*(db: BeaconChainDB, proc getStateRoot*(db: BeaconChainDB,
root: Eth2Digest, root: Eth2Digest,
@ -192,14 +195,14 @@ proc containsState*(db: BeaconChainDB, key: Eth2Digest): bool =
db.backend.contains(subkey(BeaconState, key)).expect("working database") db.backend.contains(subkey(BeaconState, key)).expect("working database")
iterator getAncestors*(db: BeaconChainDB, root: Eth2Digest): iterator getAncestors*(db: BeaconChainDB, root: Eth2Digest):
tuple[root: Eth2Digest, blck: SignedBeaconBlock] = tuple[root: Eth2Digest, blck: TrustedSignedBeaconBlock] =
## Load a chain of ancestors for blck - returns a list of blocks with the ## Load a chain of ancestors for blck - returns a list of blocks with the
## oldest block last (blck will be at result[0]). ## oldest block last (blck will be at result[0]).
## ##
## The search will go on until the ancestor cannot be found. ## The search will go on until the ancestor cannot be found.
var root = root var res: tuple[root: Eth2Digest, blck: TrustedSignedBeaconBlock]
while (let blck = db.getBlock(root); blck.isOk()): res.root = root
yield (root, blck.get()) while db.get(subkey(SignedBeaconBlock, res.root), res.blck):
yield res
root = blck.get().message.parent_root res.root = res.blck.message.parent_root

314
beacon_chain/beacon_node.nim
View File

@ -7,7 +7,7 @@
import import
# Standard library # Standard library
os, tables, random, strutils, times, math, algorithm, os, tables, strutils, times, math, terminal,
# Nimble packages # Nimble packages
stew/[objects, byteutils], stew/shims/macros, stew/[objects, byteutils], stew/shims/macros,
@ -19,19 +19,17 @@ import
# Local modules # Local modules
spec/[datatypes, digest, crypto, beaconstate, helpers, network], spec/[datatypes, digest, crypto, beaconstate, helpers, network],
spec/presets/custom, spec/state_transition, spec/presets/custom,
conf, time, beacon_chain_db, validator_pool, extras, conf, time, beacon_chain_db, validator_pool, extras,
attestation_pool, block_pool, eth2_network, eth2_discovery, attestation_pool, block_pool, eth2_network, eth2_discovery,
beacon_node_common, beacon_node_types, block_pools/block_pools_types, beacon_node_common, beacon_node_types, block_pools/block_pools_types,
nimbus_binary_common, nimbus_binary_common,
mainchain_monitor, version, ssz/[merkleization], sszdump, mainchain_monitor, version, ssz/[merkleization], sszdump,
sync_protocol, request_manager, keystore_management, interop, statusbar, sync_protocol, request_manager, keystore_management, interop, statusbar,
sync_manager, state_transition, sync_manager, validator_duties, validator_api, attestation_aggregation
validator_duties, validator_api, attestation_aggregation
const const
genesisFile* = "genesis.ssz" genesisFile* = "genesis.ssz"
timeToInitNetworkingBeforeGenesis = chronos.seconds(10)
hasPrompt = not defined(withoutPrompt) hasPrompt = not defined(withoutPrompt)
type type
@ -95,12 +93,14 @@ proc getStateFromSnapshot(conf: BeaconNodeConf): NilableBeaconStateRef =
genesisPath, dataDir = conf.dataDir.string genesisPath, dataDir = conf.dataDir.string
writeGenesisFile = true writeGenesisFile = true
genesisPath = snapshotPath genesisPath = snapshotPath
else: elif fileExists(genesisPath):
try: try: snapshotContents = readFile(genesisPath)
snapshotContents = readFile(genesisPath)
except CatchableError as err: except CatchableError as err:
error "Failed to read genesis file", err = err.msg error "Failed to read genesis file", err = err.msg
quit 1 quit 1
else:
# No snapshot was provided. We should wait for genesis.
return nil
result = try: result = try:
newClone(SSZ.decode(snapshotContents, BeaconState)) newClone(SSZ.decode(snapshotContents, BeaconState))
@ -144,20 +144,34 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
# Try file from command line first # Try file from command line first
if genesisState.isNil: if genesisState.isNil:
# Didn't work, try creating a genesis state using main chain monitor if conf.web3Url.len == 0:
fatal "Web3 URL not specified"
quit 1
if conf.depositContractAddress.len == 0:
fatal "Deposit contract address not specified"
quit 1
if conf.depositContractDeployedAt.isNone:
# When we don't have a known genesis state, the network metadata
# must specify the deployment block of the contract.
fatal "Deposit contract deployment block not specified"
quit 1
# TODO Could move this to a separate "GenesisMonitor" process or task # TODO Could move this to a separate "GenesisMonitor" process or task
# that would do only this - see Paul's proposal for this. # that would do only this - see Paul's proposal for this.
if conf.web3Url.len > 0 and conf.depositContractAddress.len > 0:
mainchainMonitor = MainchainMonitor.init( mainchainMonitor = MainchainMonitor.init(
web3Provider(conf.web3Url), web3Provider(conf.web3Url),
conf.depositContractAddress, conf.depositContractAddress,
Eth2Digest()) Eth1Data(block_hash: conf.depositContractDeployedAt.get, deposit_count: 0))
mainchainMonitor.start() mainchainMonitor.start()
else:
error "No initial state, need genesis state or deposit contract address"
quit 1
genesisState = await mainchainMonitor.getGenesis() genesisState = await mainchainMonitor.waitGenesis()
info "Eth2 genesis state detected",
genesisTime = genesisState.genesisTime,
eth1Block = genesisState.eth1_data.block_hash,
totalDeposits = genesisState.eth1_data.deposit_count
# This is needed to prove the not nil property from here on # This is needed to prove the not nil property from here on
if genesisState == nil: if genesisState == nil:
@ -193,7 +207,7 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
mainchainMonitor = MainchainMonitor.init( mainchainMonitor = MainchainMonitor.init(
web3Provider(conf.web3Url), web3Provider(conf.web3Url),
conf.depositContractAddress, conf.depositContractAddress,
blockPool.headState.data.data.eth1_data.block_hash) blockPool.headState.data.data.eth1_data)
# TODO if we don't have any validators attached, we don't need a mainchain # TODO if we don't have any validators attached, we don't need a mainchain
# monitor # monitor
mainchainMonitor.start() mainchainMonitor.start()
@ -213,7 +227,6 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
nickname: nickname, nickname: nickname,
network: network, network: network,
netKeys: netKeys, netKeys: netKeys,
requestManager: RequestManager.init(network),
db: db, db: db,
config: conf, config: conf,
attachedValidators: ValidatorPool.init(), attachedValidators: ValidatorPool.init(),
@ -227,7 +240,12 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
topicAggregateAndProofs: topicAggregateAndProofs, topicAggregateAndProofs: topicAggregateAndProofs,
) )
traceAsyncErrors res.addLocalValidators() res.requestManager = RequestManager.init(network,
proc(signedBlock: SignedBeaconBlock) =
onBeaconBlock(res, signedBlock)
)
await res.addLocalValidators()
# This merely configures the BeaconSync # This merely configures the BeaconSync
# The traffic will be started when we join the network. # The traffic will be started when we join the network.
@ -322,11 +340,12 @@ proc storeBlock(
# The block we received contains attestations, and we might not yet know about # The block we received contains attestations, and we might not yet know about
# all of them. Let's add them to the attestation pool. # all of them. Let's add them to the attestation pool.
let currentSlot = node.beaconClock.now.toSlot
if currentSlot.afterGenesis and
signedBlock.message.slot.epoch + 1 >= currentSlot.slot.epoch:
for attestation in signedBlock.message.body.attestations: for attestation in signedBlock.message.body.attestations:
node.onAttestation(attestation) debug "Attestation from block",
attestation = shortLog(attestation),
cat = "consensus" # Tag "consensus|attestation"?
node.attestationPool.add(attestation)
ok() ok()
proc onBeaconBlock(node: BeaconNode, signedBlock: SignedBeaconBlock) = proc onBeaconBlock(node: BeaconNode, signedBlock: SignedBeaconBlock) =
@ -356,7 +375,7 @@ func verifyFinalization(node: BeaconNode, slot: Slot) =
proc onSlotStart(node: BeaconNode, lastSlot, scheduledSlot: Slot) {.gcsafe, async.} = proc onSlotStart(node: BeaconNode, lastSlot, scheduledSlot: Slot) {.gcsafe, async.} =
## Called at the beginning of a slot - usually every slot, but sometimes might ## Called at the beginning of a slot - usually every slot, but sometimes might
## skip a few in case we're running late. ## skip a few in case we're running late.
## lastSlot: the last slot that we sucessfully processed, so we know where to ## lastSlot: the last slot that we successfully processed, so we know where to
## start work from ## start work from
## scheduledSlot: the slot that we were aiming for, in terms of timing ## scheduledSlot: the slot that we were aiming for, in terms of timing
@ -501,21 +520,8 @@ proc handleMissingBlocks(node: BeaconNode) =
let missingBlocks = node.blockPool.checkMissing() let missingBlocks = node.blockPool.checkMissing()
if missingBlocks.len > 0: if missingBlocks.len > 0:
var left = missingBlocks.len var left = missingBlocks.len
info "Requesting detected missing blocks", blocks = shortLog(missingBlocks)
info "Requesting detected missing blocks", missingBlocks node.requestManager.fetchAncestorBlocks(missingBlocks)
node.requestManager.fetchAncestorBlocks(missingBlocks) do (b: SignedBeaconBlock):
onBeaconBlock(node, b)
# TODO instead of waiting for a full second to try the next missing block
# fetching, we'll do it here again in case we get all blocks we asked
# for (there might be new parents to fetch). of course, this is not
# good because the onSecond fetching also kicks in regardless but
# whatever - this is just a quick fix for making the testnet easier
# work with while the sync problem is dealt with more systematically
# dec left
# if left == 0:
# discard setTimer(Moment.now()) do (p: pointer):
# handleMissingBlocks(node)
proc onSecond(node: BeaconNode) {.async.} = proc onSecond(node: BeaconNode) {.async.} =
## This procedure will be called once per second. ## This procedure will be called once per second.
@ -569,7 +575,7 @@ proc runForwardSyncLoop(node: BeaconNode) {.async.} =
# We doing round manually because stdlib.round is deprecated # We doing round manually because stdlib.round is deprecated
storeSpeed = round(v * 10000) / 10000 storeSpeed = round(v * 10000) / 10000
info "Forward sync blocks got imported sucessfully", count = len(list), info "Forward sync blocks got imported successfully", count = len(list),
local_head_slot = getLocalHeadSlot(), store_speed = storeSpeed local_head_slot = getLocalHeadSlot(), store_speed = storeSpeed
ok() ok()
@ -705,6 +711,22 @@ proc installDebugApiHandlers(rpcServer: RpcServer, node: BeaconNode) =
return res return res
rpcServer.rpc("peers") do () -> JsonNode:
var res = newJObject()
var peers = newJArray()
for id, peer in node.network.peerPool:
peers.add(
%(
info: shortLog(peer.info),
wasDialed: peer.wasDialed,
connectionState: $peer.connectionState,
score: peer.score,
)
)
res.add("peers", peers)
return res
proc installRpcHandlers(rpcServer: RpcServer, node: BeaconNode) = proc installRpcHandlers(rpcServer: RpcServer, node: BeaconNode) =
rpcServer.installValidatorApiHandlers(node) rpcServer.installValidatorApiHandlers(node)
rpcServer.installBeaconApiHandlers(node) rpcServer.installBeaconApiHandlers(node)
@ -723,7 +745,7 @@ proc installAttestationHandlers(node: BeaconNode) =
proc attestationValidator(attestation: Attestation, proc attestationValidator(attestation: Attestation,
committeeIndex: uint64): bool = committeeIndex: uint64): bool =
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#attestation-subnets # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/p2p-interface.md#attestation-subnets
let (afterGenesis, slot) = node.beaconClock.now().toSlot() let (afterGenesis, slot) = node.beaconClock.now().toSlot()
if not afterGenesis: if not afterGenesis:
return false return false
@ -731,27 +753,17 @@ proc installAttestationHandlers(node: BeaconNode) =
var attestationSubscriptions: seq[Future[void]] = @[] var attestationSubscriptions: seq[Future[void]] = @[]
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#mainnet-3 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/p2p-interface.md#attestations-and-aggregation
for it in 0'u64 ..< ATTESTATION_SUBNET_COUNT.uint64: for it in 0'u64 ..< ATTESTATION_SUBNET_COUNT.uint64:
closureScope: closureScope:
let ci = it let ci = it
attestationSubscriptions.add(node.network.subscribe( attestationSubscriptions.add(node.network.subscribe(
getMainnetAttestationTopic(node.forkDigest, ci), attestationHandler, getAttestationTopic(node.forkDigest, ci), attestationHandler,
# This proc needs to be within closureScope; don't lift out of loop. # This proc needs to be within closureScope; don't lift out of loop.
proc(attestation: Attestation): bool = proc(attestation: Attestation): bool =
attestationValidator(attestation, ci) attestationValidator(attestation, ci)
)) ))
when ETH2_SPEC == "v0.11.3":
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#interop-3
attestationSubscriptions.add(node.network.subscribe(
getInteropAttestationTopic(node.forkDigest), attestationHandler,
proc(attestation: Attestation): bool =
# isValidAttestation checks attestation.data.index == topicCommitteeIndex
# which doesn't make sense here, so rig that check to vacuously pass.
attestationValidator(attestation, attestation.data.index)
))
waitFor allFutures(attestationSubscriptions) waitFor allFutures(attestationSubscriptions)
proc stop*(node: BeaconNode) = proc stop*(node: BeaconNode) =
@ -799,6 +811,8 @@ proc run*(node: BeaconNode) =
node.onSecondLoop = runOnSecondLoop(node) node.onSecondLoop = runOnSecondLoop(node)
node.forwardSyncLoop = runForwardSyncLoop(node) node.forwardSyncLoop = runForwardSyncLoop(node)
node.requestManager.start()
# main event loop # main event loop
while status == BeaconNodeStatus.Running: while status == BeaconNodeStatus.Running:
try: try:
@ -819,21 +833,20 @@ proc createPidFile(filename: string) =
proc initializeNetworking(node: BeaconNode) {.async.} = proc initializeNetworking(node: BeaconNode) {.async.} =
node.network.startListening() node.network.startListening()
let addressFile = node.config.dataDir / "beacon_node.address" let addressFile = node.config.dataDir / "beacon_node.enr"
writeFile(addressFile, node.network.announcedENR.toURI) writeFile(addressFile, node.network.announcedENR.toURI)
await node.network.startLookingForPeers() await node.network.startLookingForPeers()
info "Networking initialized",
enr = node.network.announcedENR.toURI,
libp2p = shortLog(node.network.switch.peerInfo)
proc start(node: BeaconNode) = proc start(node: BeaconNode) =
let let
head = node.blockPool.head head = node.blockPool.head
finalizedHead = node.blockPool.finalizedHead finalizedHead = node.blockPool.finalizedHead
genesisTime = node.beaconClock.fromNow(toBeaconTime(Slot 0))
let genesisTime = node.beaconClock.fromNow(toBeaconTime(Slot 0))
if genesisTime.inFuture and genesisTime.offset > timeToInitNetworkingBeforeGenesis:
info "Waiting for the genesis event", genesisIn = genesisTime.offset
waitFor sleepAsync(genesisTime.offset - timeToInitNetworkingBeforeGenesis)
info "Starting beacon node", info "Starting beacon node",
version = fullVersionStr, version = fullVersionStr,
@ -851,6 +864,9 @@ proc start(node: BeaconNode) =
cat = "init", cat = "init",
pcs = "start_beacon_node" pcs = "start_beacon_node"
if genesisTime.inFuture:
notice "Waiting for genesis", genesisIn = genesisTime.offset
waitFor node.initializeNetworking() waitFor node.initializeNetworking()
node.run() node.run()
@ -871,7 +887,7 @@ func formatGwei(amount: uint64): string =
when hasPrompt: when hasPrompt:
from unicode import Rune from unicode import Rune
import terminal, prompt import prompt
proc providePromptCompletions*(line: seq[Rune], cursorPos: int): seq[string] = proc providePromptCompletions*(line: seq[Rune], cursorPos: int): seq[string] =
# TODO # TODO
@ -998,6 +1014,104 @@ when hasPrompt:
# var t: Thread[ptr Prompt] # var t: Thread[ptr Prompt]
# createThread(t, processPromptCommands, addr p) # createThread(t, processPromptCommands, addr p)
proc createWalletInteractively(conf: BeaconNodeConf): OutFile {.raises: [Defect].} =
if conf.nonInteractive:
fatal "Wallets can be created only in interactive mode"
quit 1
var mnemonic = generateMnemonic()
defer: keystore_management.burnMem(mnemonic)
template readLine: string =
try: stdin.readLine()
except IOError:
fatal "Failed to read data from stdin"
quit 1
echo "The created wallet will be protected with a password " &
"that applies only to the current Nimbus installation. " &
"In case you lose your wallet and you need to restore " &
"it on a different machine, you must use the following " &
"seed recovery phrase: \n"
echo $mnemonic
echo "Please back up the seed phrase now to a safe location as " &
"if you are protecting a sensitive password. The seed phrase " &
"be used to withdrawl funds from your wallet.\n"
echo "Did you back up your seed recovery phrase? (please type 'yes' to continue or press enter to quit)"
while true:
let answer = readLine()
if answer == "":
quit 1
elif answer != "yes":
echo "To continue, please type 'yes' (without the quotes) or press enter to quit"
else:
break
echo "When you perform operations with your wallet such as withdrawals " &
"and additional deposits, you'll be asked to enter a password. " &
"Please note that this password is local to the current Nimbus " &
"installation and can be changed at any time."
while true:
var password, confirmedPassword: TaintedString
try:
let status = try:
readPasswordFromStdin("Please enter a password:", password) and
readPasswordFromStdin("Please repeat the password:", confirmedPassword)
except IOError:
fatal "Failed to read password interactively"
quit 1
if status:
if password != confirmedPassword:
echo "Passwords don't match, please try again"
else:
var name: WalletName
if conf.createdWalletName.isSome:
name = conf.createdWalletName.get
else:
echo "For your convenience, the wallet can be identified with a name " &
"of your choice. Please enter a wallet name below or press ENTER " &
"to continue with a machine-generated name."
while true:
var enteredName = readLine()
if enteredName.len > 0:
name = try: WalletName.parseCmdArg(enteredName)
except CatchableError as err:
echo err.msg & ". Please try again."
continue
break
let (uuid, walletContent) = KdfPbkdf2.createWalletContent(mnemonic, name)
try:
var outWalletFile: OutFile
if conf.createdWalletFile.isSome:
outWalletFile = conf.createdWalletFile.get
createDir splitFile(string outWalletFile).dir
else:
let walletsDir = conf.walletsDir
createDir walletsDir
outWalletFile = OutFile(walletsDir / addFileExt(string uuid, "json"))
writeFile(string outWalletFile, string walletContent)
return outWalletFile
except CatchableError as err:
fatal "Failed to write wallet file", err = err.msg
quit 1
if not status:
fatal "Failed to read a password from stdin"
quit 1
finally:
keystore_management.burnMem(password)
keystore_management.burnMem(confirmedPassword)
programMain: programMain:
let config = makeBannerAndConfig(clientId, BeaconNodeConf) let config = makeBannerAndConfig(clientId, BeaconNodeConf)
@ -1005,8 +1119,10 @@ programMain:
case config.cmd case config.cmd
of createTestnet: of createTestnet:
var deposits: seq[Deposit] var
var i = -1 depositDirs: seq[string]
deposits: seq[Deposit]
i = -1
for kind, dir in walkDir(config.testnetDepositsDir.string): for kind, dir in walkDir(config.testnetDepositsDir.string):
if kind != pcDir: if kind != pcDir:
continue continue
@ -1015,13 +1131,19 @@ programMain:
if i < config.firstValidator.int: if i < config.firstValidator.int:
continue continue
depositDirs.add dir
# Add deposits, in order, to pass Merkle validation
sort(depositDirs, system.cmp)
for dir in depositDirs:
let depositFile = dir / "deposit.json" let depositFile = dir / "deposit.json"
try: try:
deposits.add Json.loadFile(depositFile, Deposit) deposits.add Json.loadFile(depositFile, Deposit)
except SerializationError as err: except SerializationError as err:
stderr.write "Error while loading a deposit file:\n" stderr.write "Error while loading a deposit file:\n"
stderr.write err.formatMsg(depositFile), "\n" stderr.write err.formatMsg(depositFile), "\n"
stderr.write "Please regenerate the deposit files by running makeDeposits again\n" stderr.write "Please regenerate the deposit files by running 'beacon_node deposits create' again\n"
quit 1 quit 1
let let
@ -1031,7 +1153,7 @@ programMain:
else: waitFor getLatestEth1BlockHash(config.web3Url) else: waitFor getLatestEth1BlockHash(config.web3Url)
var var
initialState = initialize_beacon_state_from_eth1( initialState = initialize_beacon_state_from_eth1(
eth1Hash, startTime, deposits, {skipBlsValidation, skipMerkleValidation}) eth1Hash, startTime, deposits, {skipBlsValidation})
# https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state # https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
initialState.genesis_time = startTime initialState.genesis_time = startTime
@ -1064,22 +1186,6 @@ programMain:
writeFile(bootstrapFile, bootstrapEnr.tryGet().toURI) writeFile(bootstrapFile, bootstrapEnr.tryGet().toURI)
echo "Wrote ", bootstrapFile echo "Wrote ", bootstrapFile
of importValidator:
template reportFailureFor(keyExpr) =
error "Failed to import validator key", key = keyExpr
programResult = 1
if config.keyFiles.len == 0:
stderr.write "Please specify at least one keyfile to import."
quit 1
for keyFile in config.keyFiles:
try:
saveValidatorKey(keyFile.string.extractFilename,
readFile(keyFile.string), config)
except:
reportFailureFor keyFile.string
of noCommand: of noCommand:
debug "Launching beacon node", debug "Launching beacon node",
version = fullVersionStr, version = fullVersionStr,
@ -1109,34 +1215,44 @@ programMain:
else: else:
node.start() node.start()
of makeDeposits: of deposits:
case config.depositsCmd
of DepositsCmd.create:
createDir(config.outValidatorsDir) createDir(config.outValidatorsDir)
createDir(config.outSecretsDir) createDir(config.outSecretsDir)
let let deposits = generateDeposits(
deposits = generateDeposits(
config.totalDeposits, config.totalDeposits,
config.outValidatorsDir, config.outValidatorsDir,
config.outSecretsDir).tryGet config.outSecretsDir)
if config.web3Url.len > 0 and config.depositContractAddress.len > 0: if deposits.isErr:
fatal "Failed to generate deposits", err = deposits.error
quit 1
if not config.dontSend:
waitFor sendDeposits(config, deposits.value)
of DepositsCmd.send:
if config.minDelay > config.maxDelay: if config.minDelay > config.maxDelay:
echo "The minimum delay should not be larger than the maximum delay" echo "The minimum delay should not be larger than the maximum delay"
quit 1 quit 1
var delayGenerator: DelayGenerator let deposits = loadDeposits(config.depositsDir)
if config.maxDelay > 0.0: waitFor sendDeposits(config, deposits)
delayGenerator = proc (): chronos.Duration {.gcsafe.} =
chronos.milliseconds (rand(config.minDelay..config.maxDelay)*1000).int
info "Sending deposits", of DepositsCmd.status:
web3 = config.web3Url, # TODO
depositContract = config.depositContractAddress echo "The status command is not implemented yet"
quit 1
waitFor sendDeposits(
deposits,
config.web3Url,
config.depositContractAddress,
config.depositPrivateKey,
delayGenerator)
of wallets:
case config.walletsCmd:
of WalletsCmd.create:
let walletFile = createWalletInteractively(config)
of WalletsCmd.list:
# TODO
discard
of WalletsCmd.restore:
# TODO
discard

21
beacon_chain/block_pool.nim
View File

@ -35,7 +35,7 @@ type
# Quarantine dispatch # Quarantine dispatch
# -------------------------------------------- # --------------------------------------------
func checkMissing*(pool: var BlockPool): seq[FetchRecord] {.noInit.} = func checkMissing*(pool: var BlockPool): seq[FetchRecord] =
checkMissing(pool.quarantine) checkMissing(pool.quarantine)
# CandidateChains # CandidateChains
@ -127,6 +127,9 @@ proc latestJustifiedBlock*(pool: BlockPool): BlockSlot =
## as the latest finalized block ## as the latest finalized block
latestJustifiedBlock(pool.dag) latestJustifiedBlock(pool.dag)
proc addMissing*(pool: var BlockPool, broot: Eth2Digest) {.inline.} =
pool.quarantine.addMissing(broot)
proc isInitialized*(T: type BlockPool, db: BeaconChainDB): bool = proc isInitialized*(T: type BlockPool, db: BeaconChainDB): bool =
isInitialized(CandidateChains, db) isInitialized(CandidateChains, db)
@ -152,7 +155,8 @@ template justifiedState*(pool: BlockPool): StateData =
pool.dag.justifiedState pool.dag.justifiedState
template withState*( template withState*(
pool: BlockPool, cache: var StateData, blockSlot: BlockSlot, body: untyped): untyped = pool: BlockPool, cache: var StateData, blockSlot: BlockSlot, body: untyped):
untyped =
## Helper template that updates state to a particular BlockSlot - usage of ## Helper template that updates state to a particular BlockSlot - usage of
## cache is unsafe outside of block. ## cache is unsafe outside of block.
## TODO async transformations will lead to a race where cache gets updated ## TODO async transformations will lead to a race where cache gets updated
@ -160,6 +164,19 @@ template withState*(
withState(pool.dag, cache, blockSlot, body) withState(pool.dag, cache, blockSlot, body)
template withEpochState*(
pool: BlockPool, cache: var StateData, blockSlot: BlockSlot, body: untyped):
untyped =
## Helper template that updates state to a state with an epoch matching the
## epoch of blockSlot. This aims to be at least as fast as withState, quick
## enough to expose to unautheticated, remote use, but trades off that it's
## possible for it to decide that finding a state from a matching epoch may
## provide too expensive for such use cases.
##
## cache is unsafe outside of block.
withEpochState(pool.dag, cache, blockSlot, body)
proc updateStateData*(pool: BlockPool, state: var StateData, bs: BlockSlot) = proc updateStateData*(pool: BlockPool, state: var StateData, bs: BlockSlot) =
## Rewind or advance state such that it matches the given block and slot - ## Rewind or advance state such that it matches the given block and slot -
## this may include replaying from an earlier snapshot if blck is on a ## this may include replaying from an earlier snapshot if blck is on a

6
beacon_chain/block_pools/block_pools_types.nim
View File

@ -37,7 +37,7 @@ type
## ##
## Invalid blocks are dropped immediately. ## Invalid blocks are dropped immediately.
pending*: Table[Eth2Digest, SignedBeaconBlock] ##\ orphans*: Table[Eth2Digest, SignedBeaconBlock] ##\
## Blocks that have passed validation but that we lack a link back to tail ## Blocks that have passed validation but that we lack a link back to tail
## for - when we receive a "missing link", we can use this data to build ## for - when we receive a "missing link", we can use this data to build
## an entire branch ## an entire branch
@ -49,12 +49,10 @@ type
inAdd*: bool inAdd*: bool
MissingBlock* = object MissingBlock* = object
slots*: uint64 # number of slots that are suspected missing
tries*: int tries*: int
FetchRecord* = object FetchRecord* = object
root*: Eth2Digest root*: Eth2Digest
historySlots*: uint64
CandidateChains* = ref object CandidateChains* = ref object
## Pool of blocks responsible for keeping a DAG of resolved blocks. ## Pool of blocks responsible for keeping a DAG of resolved blocks.
@ -145,7 +143,7 @@ type
BlockData* = object BlockData* = object
## Body and graph in one ## Body and graph in one
data*: SignedBeaconBlock data*: TrustedSignedBeaconBlock # We trust all blocks we have a ref for
refs*: BlockRef refs*: BlockRef
StateData* = object StateData* = object

65
beacon_chain/block_pools/candidate_chains.nim
View File

@ -10,8 +10,8 @@
import import
chronicles, options, sequtils, tables, chronicles, options, sequtils, tables,
metrics, metrics,
../ssz/merkleization, ../beacon_chain_db, ../state_transition, ../extras, ../ssz/merkleization, ../beacon_chain_db, ../extras,
../spec/[crypto, datatypes, digest, helpers, validator], ../spec/[crypto, datatypes, digest, helpers, validator, state_transition],
block_pools_types block_pools_types
declareCounter beacon_reorgs_total, "Total occurrences of reorganizations of the chain" # On fork choice declareCounter beacon_reorgs_total, "Total occurrences of reorganizations of the chain" # On fork choice
@ -20,7 +20,8 @@ declareCounter beacon_state_data_cache_misses, "dag.cachedStates misses"
logScope: topics = "hotdb" logScope: topics = "hotdb"
proc putBlock*(dag: var CandidateChains, blockRoot: Eth2Digest, signedBlock: SignedBeaconBlock) {.inline.} = proc putBlock*(
dag: var CandidateChains, blockRoot: Eth2Digest, signedBlock: SignedBeaconBlock) =
dag.db.putBlock(blockRoot, signedBlock) dag.db.putBlock(blockRoot, signedBlock)
proc updateStateData*( proc updateStateData*(
@ -53,7 +54,7 @@ func parent*(bs: BlockSlot): BlockSlot =
slot: bs.slot - 1 slot: bs.slot - 1
) )
func populateEpochCache*(state: BeaconState, epoch: Epoch): EpochRef = func populateEpochCache(state: BeaconState, epoch: Epoch): EpochRef =
result = (EpochRef)( result = (EpochRef)(
epoch: state.slot.compute_epoch_at_slot, epoch: state.slot.compute_epoch_at_slot,
shuffled_active_validator_indices: shuffled_active_validator_indices:
@ -129,6 +130,22 @@ func get_ancestor*(blck: BlockRef, slot: Slot): BlockRef =
blck = blck.parent blck = blck.parent
iterator get_ancestors_in_epoch(blockSlot: BlockSlot): BlockSlot =
let min_slot =
blockSlot.slot.compute_epoch_at_slot.compute_start_slot_at_epoch
var blockSlot = blockSlot
while true:
for slot in countdown(blockSlot.slot, max(blockSlot.blck.slot, min_slot)):
yield BlockSlot(blck: blockSlot.blck, slot: slot)
if blockSlot.blck.parent.isNil or blockSlot.blck.slot <= min_slot:
break
doAssert blockSlot.blck.slot > blockSlot.blck.parent.slot
blockSlot =
BlockSlot(blck: blockSlot.blck.parent, slot: blockSlot.blck.slot - 1)
func atSlot*(blck: BlockRef, slot: Slot): BlockSlot = func atSlot*(blck: BlockRef, slot: Slot): BlockSlot =
## Return a BlockSlot at a given slot, with the block set to the closest block ## Return a BlockSlot at a given slot, with the block set to the closest block
## available. If slot comes from before the block, a suitable block ancestor ## available. If slot comes from before the block, a suitable block ancestor
@ -148,7 +165,7 @@ func getEpochInfo*(blck: BlockRef, state: BeaconState): EpochRef =
if matching_epochinfo.len == 0: if matching_epochinfo.len == 0:
let cache = populateEpochCache(state, state_epoch) let cache = populateEpochCache(state, state_epoch)
blck.epochsInfo.add(cache) blck.epochsInfo.add(cache)
trace "candidate_chains.skipAndUpdateState(): back-filling parent.epochInfo", trace "candidate_chains.getEpochInfo: back-filling parent.epochInfo",
state_slot = state.slot state_slot = state.slot
cache cache
elif matching_epochinfo.len == 1: elif matching_epochinfo.len == 1:
@ -169,7 +186,7 @@ func init(T: type BlockRef, root: Eth2Digest, slot: Slot): BlockRef =
slot: slot slot: slot
) )
func init*(T: type BlockRef, root: Eth2Digest, blck: BeaconBlock): BlockRef = func init*(T: type BlockRef, root: Eth2Digest, blck: SomeBeaconBlock): BlockRef =
BlockRef.init(root, blck.slot) BlockRef.init(root, blck.slot)
proc init*(T: type CandidateChains, db: BeaconChainDB, proc init*(T: type CandidateChains, db: BeaconChainDB,
@ -492,10 +509,10 @@ proc skipAndUpdateState(
ok ok
proc rewindState(dag: CandidateChains, state: var StateData, bs: BlockSlot): proc rewindState(dag: CandidateChains, state: var StateData, bs: BlockSlot):
seq[BlockData] = seq[BlockRef] =
logScope: pcs = "replay_state" logScope: pcs = "replay_state"
var ancestors = @[dag.get(bs.blck)] var ancestors = @[bs.blck]
# Common case: the last block applied is the parent of the block to apply: # Common case: the last block applied is the parent of the block to apply:
if not bs.blck.parent.isNil and state.blck.root == bs.blck.parent.root and if not bs.blck.parent.isNil and state.blck.root == bs.blck.parent.root and
state.data.data.slot < bs.blck.slot: state.data.data.slot < bs.blck.slot:
@ -522,7 +539,7 @@ proc rewindState(dag: CandidateChains, state: var StateData, bs: BlockSlot):
break # Bug probably! break # Bug probably!
if parBs.blck != curBs.blck: if parBs.blck != curBs.blck:
ancestors.add(dag.get(parBs.blck)) ancestors.add(parBs.blck)
# TODO investigate replacing with getStateCached, by refactoring whole # TODO investigate replacing with getStateCached, by refactoring whole
# function. Empirically, this becomes pretty rare once good caches are # function. Empirically, this becomes pretty rare once good caches are
@ -531,12 +548,12 @@ proc rewindState(dag: CandidateChains, state: var StateData, bs: BlockSlot):
if idx >= 0: if idx >= 0:
assign(state.data, dag.cachedStates[idx].state[]) assign(state.data, dag.cachedStates[idx].state[])
let ancestor = ancestors.pop() let ancestor = ancestors.pop()
state.blck = ancestor.refs state.blck = ancestor
beacon_state_data_cache_hits.inc() beacon_state_data_cache_hits.inc()
trace "Replaying state transitions via in-memory cache", trace "Replaying state transitions via in-memory cache",
stateSlot = shortLog(state.data.data.slot), stateSlot = shortLog(state.data.data.slot),
ancestorStateRoot = shortLog(ancestor.data.message.state_root), ancestorStateRoot = shortLog(state.data.root),
ancestorStateSlot = shortLog(state.data.data.slot), ancestorStateSlot = shortLog(state.data.data.slot),
slot = shortLog(bs.slot), slot = shortLog(bs.slot),
blockRoot = shortLog(bs.blck.root), blockRoot = shortLog(bs.blck.root),
@ -568,7 +585,7 @@ proc rewindState(dag: CandidateChains, state: var StateData, bs: BlockSlot):
let let
ancestor = ancestors.pop() ancestor = ancestors.pop()
root = stateRoot.get() root = stateRoot.get()
found = dag.getState(dag.db, root, ancestor.refs, state) found = dag.getState(dag.db, root, ancestor, state)
if not found: if not found:
# TODO this should only happen if the database is corrupt - we walked the # TODO this should only happen if the database is corrupt - we walked the
@ -584,7 +601,6 @@ proc rewindState(dag: CandidateChains, state: var StateData, bs: BlockSlot):
trace "Replaying state transitions", trace "Replaying state transitions",
stateSlot = shortLog(state.data.data.slot), stateSlot = shortLog(state.data.data.slot),
ancestorStateRoot = shortLog(ancestor.data.message.state_root),
ancestorStateSlot = shortLog(state.data.data.slot), ancestorStateSlot = shortLog(state.data.data.slot),
slot = shortLog(bs.slot), slot = shortLog(bs.slot),
blockRoot = shortLog(bs.blck.root), blockRoot = shortLog(bs.blck.root),
@ -618,6 +634,24 @@ proc getStateDataCached(dag: CandidateChains, state: var StateData, bs: BlockSlo
false false
template withEpochState*(
dag: CandidateChains, cache: var StateData, blockSlot: BlockSlot, body: untyped): untyped =
## Helper template that updates state to a particular BlockSlot - usage of
## cache is unsafe outside of block.
## TODO async transformations will lead to a race where cache gets updated
## while waiting for future to complete - catch this here somehow?
for ancestor in get_ancestors_in_epoch(blockSlot):
if getStateDataCached(dag, cache, ancestor):
break
template hashedState(): HashedBeaconState {.inject, used.} = cache.data
template state(): BeaconState {.inject, used.} = cache.data.data
template blck(): BlockRef {.inject, used.} = cache.blck
template root(): Eth2Digest {.inject, used.} = cache.data.root
body
proc updateStateData*(dag: CandidateChains, state: var StateData, bs: BlockSlot) = proc updateStateData*(dag: CandidateChains, state: var StateData, bs: BlockSlot) =
## Rewind or advance state such that it matches the given block and slot - ## Rewind or advance state such that it matches the given block and slot -
## this may include replaying from an earlier snapshot if blck is on a ## this may include replaying from an earlier snapshot if blck is on a
@ -655,10 +689,7 @@ proc updateStateData*(dag: CandidateChains, state: var StateData, bs: BlockSlot)
# no state root calculation will take place here, because we can load # no state root calculation will take place here, because we can load
# the final state root from the block itself. # the final state root from the block itself.
let ok = let ok =
dag.skipAndUpdateState( dag.skipAndUpdateState(state, dag.get(ancestors[i]), {}, false)
state, ancestors[i],
{skipBlsValidation, skipStateRootValidation},
false)
doAssert ok, "Blocks in database should never fail to apply.." doAssert ok, "Blocks in database should never fail to apply.."
# We save states here - blocks were guaranteed to have passed through the save # We save states here - blocks were guaranteed to have passed through the save

40
beacon_chain/block_pools/clearance.nim
View File

@ -10,9 +10,9 @@
import import
chronicles, sequtils, tables, chronicles, sequtils, tables,
metrics, stew/results, metrics, stew/results,
../ssz/merkleization, ../state_transition, ../extras, ../ssz/merkleization, ../extras,
../spec/[crypto, datatypes, digest, helpers, signatures], ../spec/[crypto, datatypes, digest, helpers, signatures, state_transition],
block_pools_types, candidate_chains block_pools_types, candidate_chains, quarantine
export results export results
@ -32,7 +32,7 @@ func getOrResolve*(dag: CandidateChains, quarantine: var Quarantine, root: Eth2D
result = dag.getRef(root) result = dag.getRef(root)
if result.isNil: if result.isNil:
quarantine.missing[root] = MissingBlock(slots: 1) quarantine.missing[root] = MissingBlock()
proc add*( proc add*(
dag: var CandidateChains, quarantine: var Quarantine, dag: var CandidateChains, quarantine: var Quarantine,
@ -99,12 +99,12 @@ proc addResolvedBlock(
defer: quarantine.inAdd = false defer: quarantine.inAdd = false
var keepGoing = true var keepGoing = true
while keepGoing: while keepGoing:
let retries = quarantine.pending let retries = quarantine.orphans
for k, v in retries: for k, v in retries:
discard add(dag, quarantine, k, v) discard add(dag, quarantine, k, v)
# Keep going for as long as the pending dag is shrinking # Keep going for as long as the pending dag is shrinking
# TODO inefficient! so what? # TODO inefficient! so what?
keepGoing = quarantine.pending.len < retries.len keepGoing = quarantine.orphans.len < retries.len
blockRef blockRef
proc add*( proc add*(
@ -165,9 +165,9 @@ proc add*(
return err Invalid return err Invalid
# The block might have been in either of pending or missing - we don't want # The block might have been in either of `orphans` or `missing` - we don't
# any more work done on its behalf # want any more work done on its behalf
quarantine.pending.del(blockRoot) quarantine.orphans.del(blockRoot)
# The block is resolved, now it's time to validate it to ensure that the # The block is resolved, now it's time to validate it to ensure that the
# blocks we add to the database are clean for the given state # blocks we add to the database are clean for the given state
@ -209,7 +209,7 @@ proc add*(
# the pending dag calls this function back later in a loop, so as long # the pending dag calls this function back later in a loop, so as long
# as dag.add(...) requires a SignedBeaconBlock, easier to keep them in # as dag.add(...) requires a SignedBeaconBlock, easier to keep them in
# pending too. # pending too.
quarantine.pending[blockRoot] = signedBlock quarantine.add(dag, signedBlock, some(blockRoot))
# TODO possibly, it makes sense to check the database - that would allow sync # TODO possibly, it makes sense to check the database - that would allow sync
# to simply fill up the database with random blocks the other clients # to simply fill up the database with random blocks the other clients
@ -217,7 +217,7 @@ proc add*(
# junk that's not part of the block graph # junk that's not part of the block graph
if blck.parent_root in quarantine.missing or if blck.parent_root in quarantine.missing or
blck.parent_root in quarantine.pending: blck.parent_root in quarantine.orphans:
return err MissingParent return err MissingParent
# This is an unresolved block - put its parent on the missing list for now... # This is an unresolved block - put its parent on the missing list for now...
@ -232,24 +232,11 @@ proc add*(
# filter. # filter.
# TODO when we receive the block, we don't know how many others we're missing # TODO when we receive the block, we don't know how many others we're missing
# from that branch, so right now, we'll just do a blind guess # from that branch, so right now, we'll just do a blind guess
let parentSlot = blck.slot - 1
quarantine.missing[blck.parent_root] = MissingBlock(
slots:
# The block is at least two slots ahead - try to grab whole history
if parentSlot > dag.head.blck.slot:
parentSlot - dag.head.blck.slot
else:
# It's a sibling block from a branch that we're missing - fetch one
# epoch at a time
max(1.uint64, SLOTS_PER_EPOCH.uint64 -
(parentSlot.uint64 mod SLOTS_PER_EPOCH.uint64))
)
debug "Unresolved block (parent missing)", debug "Unresolved block (parent missing)",
blck = shortLog(blck), blck = shortLog(blck),
blockRoot = shortLog(blockRoot), blockRoot = shortLog(blockRoot),
pending = quarantine.pending.len, orphans = quarantine.orphans.len,
missing = quarantine.missing.len, missing = quarantine.missing.len,
cat = "filtering" cat = "filtering"
@ -345,8 +332,7 @@ proc isValidBeaconBlock*(
# not specific to this, but by the pending dag keying on the htr of the # not specific to this, but by the pending dag keying on the htr of the
# BeaconBlock, not SignedBeaconBlock, opens up certain spoofing attacks. # BeaconBlock, not SignedBeaconBlock, opens up certain spoofing attacks.
debug "parent unknown, putting block in quarantine" debug "parent unknown, putting block in quarantine"
quarantine.pending[hash_tree_root(signed_beacon_block.message)] = quarantine.add(dag, signed_beacon_block)
signed_beacon_block
return err(MissingParent) return err(MissingParent)
# The proposer signature, signed_beacon_block.signature, is valid with # The proposer signature, signed_beacon_block.signature, is valid with

27
beacon_chain/block_pools/quarantine.nim
View File

@ -6,13 +6,15 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms. # at your option. This file may not be copied, modified, or distributed except according to those terms.
import import
chronicles, tables, chronicles, tables, options,
stew/bitops2, stew/bitops2,
metrics, metrics,
../spec/digest, ../spec/[datatypes, digest],
../ssz/merkleization,
block_pools_types block_pools_types
export options
logScope: topics = "quarant" logScope: topics = "quarant"
{.push raises: [Defect].} {.push raises: [Defect].}
@ -35,4 +37,21 @@ func checkMissing*(quarantine: var Quarantine): seq[FetchRecord] =
# simple (simplistic?) exponential backoff for retries.. # simple (simplistic?) exponential backoff for retries..
for k, v in quarantine.missing.pairs(): for k, v in quarantine.missing.pairs():
if countOnes(v.tries.uint64) == 1: if countOnes(v.tries.uint64) == 1:
result.add(FetchRecord(root: k, historySlots: v.slots)) result.add(FetchRecord(root: k))
func addMissing*(quarantine: var Quarantine, broot: Eth2Digest) {.inline.} =
discard quarantine.missing.hasKeyOrPut(broot, MissingBlock())
func add*(quarantine: var Quarantine, dag: CandidateChains,
sblck: SignedBeaconBlock,
broot: Option[Eth2Digest] = none[Eth2Digest]()) =
## Adds block to quarantine's `orphans` and `missing` lists.
let blockRoot = if broot.isSome():
broot.get()
else:
hash_tree_root(sblck.message)
quarantine.orphans[blockRoot] = sblck
let parentRoot = sblck.message.parent_root
quarantine.addMissing(parentRoot)

210
beacon_chain/conf.nim
View File

@ -5,7 +5,7 @@ import
chronicles, confutils, json_serialization, chronicles, confutils, json_serialization,
confutils/defs, confutils/std/net, confutils/defs, confutils/std/net,
chronicles/options as chroniclesOptions, chronicles/options as chroniclesOptions,
spec/[crypto, keystore] spec/[crypto, keystore, digest]
export export
defs, enabledLogLevel, parseCmdArg, completeCmdArg defs, enabledLogLevel, parseCmdArg, completeCmdArg
@ -15,9 +15,19 @@ type
BNStartUpCmd* = enum BNStartUpCmd* = enum
noCommand noCommand
importValidator
createTestnet createTestnet
makeDeposits deposits
wallets
WalletsCmd* {.pure.} = enum
create = "Creates a new EIP-2386 wallet"
restore = "Restores a wallet from cold storage"
list = "Lists details about all wallets"
DepositsCmd* {.pure.} = enum
create = "Creates validator keystores and deposits"
send = "Sends prepared deposits to the validator deposit contract"
status = "Displays status information about all deposits"
VCStartUpCmd* = enum VCStartUpCmd* = enum
VCNoCommand VCNoCommand
@ -31,32 +41,36 @@ type
BeaconNodeConf* = object BeaconNodeConf* = object
logLevel* {. logLevel* {.
defaultValue: "DEBUG" defaultValue: "DEBUG"
desc: "Sets the log level." desc: "Sets the log level"
name: "log-level" }: string name: "log-level" }: string
eth1Network* {. eth1Network* {.
defaultValue: goerli defaultValue: goerli
desc: "The Eth1 network tracked by the beacon node." desc: "The Eth1 network tracked by the beacon node"
name: "eth1-network" }: Eth1Network name: "eth1-network" }: Eth1Network
dataDir* {. dataDir* {.
defaultValue: config.defaultDataDir() defaultValue: config.defaultDataDir()
desc: "The directory where nimbus will store all blockchain data." desc: "The directory where nimbus will store all blockchain data"
abbr: "d" abbr: "d"
name: "data-dir" }: OutDir name: "data-dir" }: OutDir
web3Url* {. web3Url* {.
defaultValue: "" defaultValue: ""
desc: "URL of the Web3 server to observe Eth1." desc: "URL of the Web3 server to observe Eth1"
name: "web3-url" }: string name: "web3-url" }: string
depositContractAddress* {. depositContractAddress* {.
defaultValue: "" defaultValue: ""
desc: "Address of the deposit contract." desc: "Address of the deposit contract"
name: "deposit-contract" }: string name: "deposit-contract" }: string
depositContractDeployedAt* {.
desc: "The Eth1 block hash where the deposit contract has been deployed"
name: "deposit-contract-block" }: Option[Eth2Digest]
nonInteractive* {. nonInteractive* {.
desc: "Do not display interative prompts. Quit on missing configuration." desc: "Do not display interative prompts. Quit on missing configuration"
name: "non-interactive" }: bool name: "non-interactive" }: bool
case cmd* {. case cmd* {.
@ -65,28 +79,28 @@ type
of noCommand: of noCommand:
bootstrapNodes* {. bootstrapNodes* {.
desc: "Specifies one or more bootstrap nodes to use when connecting to the network." desc: "Specifies one or more bootstrap nodes to use when connecting to the network"
abbr: "b" abbr: "b"
name: "bootstrap-node" }: seq[string] name: "bootstrap-node" }: seq[string]
bootstrapNodesFile* {. bootstrapNodesFile* {.
defaultValue: "" defaultValue: ""
desc: "Specifies a line-delimited file of bootsrap Ethereum network addresses." desc: "Specifies a line-delimited file of bootstrap Ethereum network addresses"
name: "bootstrap-file" }: InputFile name: "bootstrap-file" }: InputFile
libp2pAddress* {. libp2pAddress* {.
defaultValue: defaultListenAddress(config) defaultValue: defaultListenAddress(config)
desc: "Listening address for the Ethereum LibP2P traffic." desc: "Listening address for the Ethereum LibP2P traffic"
name: "listen-address" }: ValidIpAddress name: "listen-address" }: ValidIpAddress
tcpPort* {. tcpPort* {.
defaultValue: defaultEth2TcpPort defaultValue: defaultEth2TcpPort
desc: "Listening TCP port for Ethereum LibP2P traffic." desc: "Listening TCP port for Ethereum LibP2P traffic"
name: "tcp-port" }: Port name: "tcp-port" }: Port
udpPort* {. udpPort* {.
defaultValue: defaultEth2TcpPort defaultValue: defaultEth2TcpPort
desc: "Listening UDP port for node discovery." desc: "Listening UDP port for node discovery"
name: "udp-port" }: Port name: "udp-port" }: Port
maxPeers* {. maxPeers* {.
@ -96,62 +110,66 @@ type
nat* {. nat* {.
desc: "Specify method to use for determining public address. " & desc: "Specify method to use for determining public address. " &
"Must be one of: any, none, upnp, pmp, extip:<IP>." "Must be one of: any, none, upnp, pmp, extip:<IP>"
defaultValue: "any" }: string defaultValue: "any" }: string
validators* {. validators* {.
required required
desc: "Path to a validator private key, as generated by makeDeposits." desc: "Path to a validator keystore"
abbr: "v" abbr: "v"
name: "validator" }: seq[ValidatorKeyPath] name: "validator" }: seq[ValidatorKeyPath]
validatorsDirFlag* {. validatorsDirFlag* {.
desc: "A directory containing validator keystores." desc: "A directory containing validator keystores"
name: "validators-dir" }: Option[InputDir] name: "validators-dir" }: Option[InputDir]
secretsDirFlag* {. secretsDirFlag* {.
desc: "A directory containing validator keystore passwords." desc: "A directory containing validator keystore passwords"
name: "secrets-dir" }: Option[InputDir] name: "secrets-dir" }: Option[InputDir]
walletsDirFlag* {.
desc: "A directory containing wallet files"
name: "wallets-dir" }: Option[InputDir]
stateSnapshot* {. stateSnapshot* {.
desc: "Json file specifying a recent state snapshot." desc: "Json file specifying a recent state snapshot"
abbr: "s" abbr: "s"
name: "state-snapshot" }: Option[InputFile] name: "state-snapshot" }: Option[InputFile]
nodeName* {. nodeName* {.
defaultValue: "" defaultValue: ""
desc: "A name for this node that will appear in the logs. " & desc: "A name for this node that will appear in the logs. " &
"If you set this to 'auto', a persistent automatically generated ID will be seleceted for each --dataDir folder." "If you set this to 'auto', a persistent automatically generated ID will be selected for each --data-dir folder"
name: "node-name" }: string name: "node-name" }: string
verifyFinalization* {. verifyFinalization* {.
defaultValue: false defaultValue: false
desc: "Specify whether to verify finalization occurs on schedule, for testing." desc: "Specify whether to verify finalization occurs on schedule, for testing"
name: "verify-finalization" }: bool name: "verify-finalization" }: bool
stopAtEpoch* {. stopAtEpoch* {.
defaultValue: 0 defaultValue: 0
desc: "A positive epoch selects the epoch at which to stop." desc: "A positive epoch selects the epoch at which to stop"
name: "stop-at-epoch" }: uint64 name: "stop-at-epoch" }: uint64
metricsEnabled* {. metricsEnabled* {.
defaultValue: false defaultValue: false
desc: "Enable the metrics server." desc: "Enable the metrics server"
name: "metrics" }: bool name: "metrics" }: bool
metricsAddress* {. metricsAddress* {.
defaultValue: defaultAdminListenAddress(config) defaultValue: defaultAdminListenAddress(config)
desc: "Listening address of the metrics server." desc: "Listening address of the metrics server"
name: "metrics-address" }: ValidIpAddress name: "metrics-address" }: ValidIpAddress
metricsPort* {. metricsPort* {.
defaultValue: 8008 defaultValue: 8008
desc: "Listening HTTP port of the metrics server." desc: "Listening HTTP port of the metrics server"
name: "metrics-port" }: Port name: "metrics-port" }: Port
statusBarEnabled* {. statusBarEnabled* {.
defaultValue: true defaultValue: true
desc: "Display a status bar at the bottom of the terminal screen." desc: "Display a status bar at the bottom of the terminal screen"
name: "status-bar" }: bool name: "status-bar" }: bool
statusBarContents* {. statusBarContents* {.
@ -160,7 +178,7 @@ type
"head: $head_root:$head_epoch:$head_epoch_slot;" & "head: $head_root:$head_epoch:$head_epoch_slot;" &
"time: $epoch:$epoch_slot ($slot)|" & "time: $epoch:$epoch_slot ($slot)|" &
"ETH: $attached_validators_balance" "ETH: $attached_validators_balance"
desc: "Textual template for the contents of the status bar." desc: "Textual template for the contents of the status bar"
name: "status-bar-contents" }: string name: "status-bar-contents" }: string
rpcEnabled* {. rpcEnabled* {.
@ -170,7 +188,7 @@ type
rpcPort* {. rpcPort* {.
defaultValue: defaultEth2RpcPort defaultValue: defaultEth2RpcPort
desc: "HTTP port for the JSON-RPC service." desc: "HTTP port for the JSON-RPC service"
name: "rpc-port" }: Port name: "rpc-port" }: Port
rpcAddress* {. rpcAddress* {.
@ -185,87 +203,138 @@ type
of createTestnet: of createTestnet:
testnetDepositsDir* {. testnetDepositsDir* {.
desc: "Directory containing validator keystores." desc: "Directory containing validator keystores"
name: "validators-dir" }: InputDir name: "validators-dir" }: InputDir
totalValidators* {. totalValidators* {.
desc: "The number of validator deposits in the newly created chain." desc: "The number of validator deposits in the newly created chain"
name: "total-validators" }: uint64 name: "total-validators" }: uint64
firstValidator* {. firstValidator* {.
defaultValue: 0 defaultValue: 0
desc: "Index of first validator to add to validator list." desc: "Index of first validator to add to validator list"
name: "first-validator" }: uint64 name: "first-validator" }: uint64
lastUserValidator* {. lastUserValidator* {.
defaultValue: config.totalValidators - 1, defaultValue: config.totalValidators - 1,
desc: "The last validator index that will free for taking from a testnet participant." desc: "The last validator index that will free for taking from a testnet participant"
name: "last-user-validator" }: uint64 name: "last-user-validator" }: uint64
bootstrapAddress* {. bootstrapAddress* {.
defaultValue: ValidIpAddress.init("127.0.0.1") defaultValue: ValidIpAddress.init("127.0.0.1")
desc: "The public IP address that will be advertised as a bootstrap node for the testnet." desc: "The public IP address that will be advertised as a bootstrap node for the testnet"
name: "bootstrap-address" }: ValidIpAddress name: "bootstrap-address" }: ValidIpAddress
bootstrapPort* {. bootstrapPort* {.
defaultValue: defaultEth2TcpPort defaultValue: defaultEth2TcpPort
desc: "The TCP/UDP port that will be used by the bootstrap node." desc: "The TCP/UDP port that will be used by the bootstrap node"
name: "bootstrap-port" }: Port name: "bootstrap-port" }: Port
genesisOffset* {. genesisOffset* {.
defaultValue: 5 defaultValue: 5
desc: "Seconds from now to add to genesis time." desc: "Seconds from now to add to genesis time"
name: "genesis-offset" }: int name: "genesis-offset" }: int
outputGenesis* {. outputGenesis* {.
desc: "Output file where to write the initial state snapshot." desc: "Output file where to write the initial state snapshot"
name: "output-genesis" }: OutFile name: "output-genesis" }: OutFile
withGenesisRoot* {. withGenesisRoot* {.
defaultValue: false defaultValue: false
desc: "Include a genesis root in 'network.json'." desc: "Include a genesis root in 'network.json'"
name: "with-genesis-root" }: bool name: "with-genesis-root" }: bool
outputBootstrapFile* {. outputBootstrapFile* {.
desc: "Output file with list of bootstrap nodes for the network." desc: "Output file with list of bootstrap nodes for the network"
name: "output-bootstrap-file" }: OutFile name: "output-bootstrap-file" }: OutFile
of importValidator: of wallets:
keyFiles* {. case walletsCmd* {.command.}: WalletsCmd
desc: "File with validator key to be imported (in hex form)." of WalletsCmd.create:
name: "keyfile" }: seq[ValidatorKeyPath] createdWalletName* {.
desc: "An easy-to-remember name for the wallet of your choice"
name: "name"}: Option[WalletName]
of makeDeposits: nextAccount* {.
desc: "Initial value for the 'nextaccount' property of the wallet"
name: "next-account" }: Option[Natural]
createdWalletFile* {.
desc: "Output wallet file"
name: "out" }: Option[OutFile]
of WalletsCmd.restore:
restoredWalletName* {.
desc: "An easy-to-remember name for the wallet of your choice"
name: "name"}: Option[WalletName]
restoredDepositsCount* {.
desc: "Expected number of deposits to recover. If not specified, " &
"Nimbus will try to guess the number by inspecting the latest " &
"beacon state"
name: "deposits".}: Option[Natural]
restoredWalletFile* {.
desc: "Output wallet file"
name: "out" }: Option[OutFile]
of WalletsCmd.list:
discard
of deposits:
depositPrivateKey* {.
defaultValue: ""
desc: "Private key of the controlling (sending) account",
name: "deposit-private-key" }: string
case depositsCmd* {.command.}: DepositsCmd
of DepositsCmd.create:
totalDeposits* {. totalDeposits* {.
defaultValue: 1 defaultValue: 1
desc: "Number of deposits to generate." desc: "Number of deposits to generate"
name: "count" }: int name: "count" }: int
existingWalletId* {.
desc: "An existing wallet ID. If not specified, a new wallet will be created"
name: "wallet" }: Option[WalletName]
outValidatorsDir* {. outValidatorsDir* {.
defaultValue: "validators" defaultValue: "validators"
desc: "Output folder for validator keystores and deposits." desc: "Output folder for validator keystores and deposits"
name: "out-validators-dir" }: string name: "out-deposits-dir" }: string
outSecretsDir* {. outSecretsDir* {.
defaultValue: "secrets" defaultValue: "secrets"
desc: "Output folder for randomly generated keystore passphrases." desc: "Output folder for randomly generated keystore passphrases"
name: "out-secrets-dir" }: string name: "out-secrets-dir" }: string
depositPrivateKey* {. dontSend* {.
defaultValue: "" defaultValue: false,
desc: "Private key of the controlling (sending) account.", desc: "By default, all created deposits are also immediately sent " &
name: "deposit-private-key" }: string "to the validator deposit contract. You can use this option to " &
"prevent this behavior. Use the `deposits send` command to send " &
"the deposit transactions at your convenience later"
name: "dont-send" .}: bool
of DepositsCmd.send:
depositsDir* {.
defaultValue: "validators"
desc: "A folder with validator metadata created by the `deposits create` command"
name: "deposits-dir" }: string
minDelay* {. minDelay* {.
defaultValue: 0.0 defaultValue: 0.0
desc: "Minimum possible delay between making two deposits (in seconds)." desc: "Minimum possible delay between making two deposits (in seconds)"
name: "min-delay" }: float name: "min-delay" }: float
maxDelay* {. maxDelay* {.
defaultValue: 0.0 defaultValue: 0.0
desc: "Maximum possible delay between making two deposits (in seconds)." desc: "Maximum possible delay between making two deposits (in seconds)"
name: "max-delay" }: float name: "max-delay" }: float
of DepositsCmd.status:
discard
ValidatorClientConf* = object ValidatorClientConf* = object
logLevel* {. logLevel* {.
defaultValue: "DEBUG" defaultValue: "DEBUG"
@ -274,12 +343,12 @@ type
dataDir* {. dataDir* {.
defaultValue: config.defaultDataDir() defaultValue: config.defaultDataDir()
desc: "The directory where nimbus will store all blockchain data." desc: "The directory where nimbus will store all blockchain data"
abbr: "d" abbr: "d"
name: "data-dir" }: OutDir name: "data-dir" }: OutDir
nonInteractive* {. nonInteractive* {.
desc: "Do not display interative prompts. Quit on missing configuration." desc: "Do not display interative prompts. Quit on missing configuration"
name: "non-interactive" }: bool name: "non-interactive" }: bool
case cmd* {. case cmd* {.
@ -289,26 +358,26 @@ type
of VCNoCommand: of VCNoCommand:
rpcPort* {. rpcPort* {.
defaultValue: defaultEth2RpcPort defaultValue: defaultEth2RpcPort
desc: "HTTP port of the server to connect to for RPC." desc: "HTTP port of the server to connect to for RPC"
name: "rpc-port" }: Port name: "rpc-port" }: Port
rpcAddress* {. rpcAddress* {.
defaultValue: defaultAdminListenAddress(config) defaultValue: defaultAdminListenAddress(config)
desc: "Address of the server to connect to for RPC." desc: "Address of the server to connect to for RPC"
name: "rpc-address" }: ValidIpAddress name: "rpc-address" }: ValidIpAddress
validators* {. validators* {.
required required
desc: "Path to a validator key store, as generated by makeDeposits." desc: "Attach a validator by supplying a keystore path"
abbr: "v" abbr: "v"
name: "validator" }: seq[ValidatorKeyPath] name: "validator" }: seq[ValidatorKeyPath]
validatorsDirFlag* {. validatorsDirFlag* {.
desc: "A directory containing validator keystores." desc: "A directory containing validator keystores"
name: "validators-dir" }: Option[InputDir] name: "validators-dir" }: Option[InputDir]
secretsDirFlag* {. secretsDirFlag* {.
desc: "A directory containing validator keystore passwords." desc: "A directory containing validator keystore passwords"
name: "secrets-dir" }: Option[InputDir] name: "secrets-dir" }: Option[InputDir]
proc defaultDataDir*(conf: BeaconNodeConf|ValidatorClientConf): string = proc defaultDataDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
@ -343,12 +412,33 @@ proc createDumpDirs*(conf: BeaconNodeConf) =
# Dumping is mainly a debugging feature, so ignore these.. # Dumping is mainly a debugging feature, so ignore these..
warn "Cannot create dump directories", msg = err.msg warn "Cannot create dump directories", msg = err.msg
func parseCmdArg*(T: type Eth2Digest, input: TaintedString): T
{.raises: [ValueError, Defect].} =
fromHex(T, string input)
func completeCmdArg*(T: type Eth2Digest, input: TaintedString): seq[string] =
return @[]
func parseCmdArg*(T: type WalletName, input: TaintedString): T
{.raises: [ValueError, Defect].} =
if input.len == 0:
raise newException(ValueError, "The wallet name should not be empty")
if input[0] == '_':
raise newException(ValueError, "The wallet name should not start with an underscore")
return T(input)
func completeCmdArg*(T: type WalletName, input: TaintedString): seq[string] =
return @[]
func validatorsDir*(conf: BeaconNodeConf|ValidatorClientConf): string = func validatorsDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
string conf.validatorsDirFlag.get(InputDir(conf.dataDir / "validators")) string conf.validatorsDirFlag.get(InputDir(conf.dataDir / "validators"))
func secretsDir*(conf: BeaconNodeConf|ValidatorClientConf): string = func secretsDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
string conf.secretsDirFlag.get(InputDir(conf.dataDir / "secrets")) string conf.secretsDirFlag.get(InputDir(conf.dataDir / "secrets"))
func walletsDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
string conf.walletsDirFlag.get(InputDir(conf.dataDir / "wallets"))
func databaseDir*(conf: BeaconNodeConf|ValidatorClientConf): string = func databaseDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
conf.dataDir / "db" conf.dataDir / "db"

11
beacon_chain/deposit_contract.nim
View File

@ -40,7 +40,7 @@ type
contract(Deposit): contract(Deposit):
proc drain() proc drain()
proc deployContract*(web3: Web3, code: string): Future[Address] {.async.} = proc deployContract*(web3: Web3, code: string): Future[ReceiptObject] {.async.} =
var code = code var code = code
if code[1] notin {'x', 'X'}: if code[1] notin {'x', 'X'}:
code = "0x" & code code = "0x" & code
@ -51,8 +51,7 @@ proc deployContract*(web3: Web3, code: string): Future[Address] {.async.} =
gasPrice: 1.some) gasPrice: 1.some)
let r = await web3.send(tr) let r = await web3.send(tr)
let receipt = await web3.getMinedTransactionReceipt(r) result = await web3.getMinedTransactionReceipt(r)
result = receipt.contractAddress.get
proc sendEth(web3: Web3, to: string, valueEth: int): Future[TxHash] = proc sendEth(web3: Web3, to: string, valueEth: int): Future[TxHash] =
let tr = EthSend( let tr = EthSend(
@ -67,7 +66,7 @@ proc main() {.async.} =
let cfg = CliConfig.load() let cfg = CliConfig.load()
let web3 = await newWeb3(cfg.web3Url) let web3 = await newWeb3(cfg.web3Url)
if cfg.privateKey.len != 0: if cfg.privateKey.len != 0:
web3.privateKey = PrivateKey.fromHex(cfg.privateKey)[] web3.privateKey = some(PrivateKey.fromHex(cfg.privateKey)[])
else: else:
let accounts = await web3.provider.eth_accounts() let accounts = await web3.provider.eth_accounts()
doAssert(accounts.len > 0) doAssert(accounts.len > 0)
@ -75,8 +74,8 @@ proc main() {.async.} =
case cfg.cmd case cfg.cmd
of StartUpCommand.deploy: of StartUpCommand.deploy:
let contractAddress = await web3.deployContract(contractCode) let receipt = await web3.deployContract(contractCode)
echo "0x", contractAddress echo "0x", receipt.contractAddress.get, ";", receipt.blockHash
of StartUpCommand.drain: of StartUpCommand.drain:
let sender = web3.contractSender(Deposit, Address.fromHex(cfg.contractAddress)) let sender = web3.contractSender(Deposit, Address.fromHex(cfg.contractAddress))
discard await sender.drain().send(gasPrice = 1) discard await sender.drain().send(gasPrice = 1)

9
beacon_chain/eth2_discovery.nim
View File

@ -51,7 +51,7 @@ proc loadBootstrapFile*(bootstrapFile: string,
localPubKey: PublicKey) = localPubKey: PublicKey) =
if bootstrapFile.len == 0: return if bootstrapFile.len == 0: return
let ext = splitFile(bootstrapFile).ext let ext = splitFile(bootstrapFile).ext
if cmpIgnoreCase(ext, ".txt") == 0: if cmpIgnoreCase(ext, ".txt") == 0 or cmpIgnoreCase(ext, ".enr") == 0 :
try: try:
for ln in lines(bootstrapFile): for ln in lines(bootstrapFile):
addBootstrapNode(ln, bootstrapEnrs, localPubKey) addBootstrapNode(ln, bootstrapEnrs, localPubKey)
@ -64,7 +64,7 @@ proc loadBootstrapFile*(bootstrapFile: string,
# removal of YAML metadata. # removal of YAML metadata.
try: try:
for ln in lines(bootstrapFile): for ln in lines(bootstrapFile):
addBootstrapNode(string(ln[3..^2]), bootstrapEnrs, localPubKey) addBootstrapNode(string(ln.strip()[3..^2]), bootstrapEnrs, localPubKey)
except IOError as e: except IOError as e:
error "Could not read bootstrap file", msg = e.msg error "Could not read bootstrap file", msg = e.msg
quit 1 quit 1
@ -75,7 +75,7 @@ proc loadBootstrapFile*(bootstrapFile: string,
proc new*(T: type Eth2DiscoveryProtocol, proc new*(T: type Eth2DiscoveryProtocol,
conf: BeaconNodeConf, conf: BeaconNodeConf,
ip: Option[ValidIpAddress], tcpPort, udpPort: Port, ip: Option[ValidIpAddress], tcpPort, udpPort: Port,
rawPrivKeyBytes: openarray[byte], pk: PrivateKey,
enrFields: openarray[(string, seq[byte])]): enrFields: openarray[(string, seq[byte])]):
T {.raises: [Exception, Defect].} = T {.raises: [Exception, Defect].} =
# TODO # TODO
@ -83,8 +83,7 @@ proc new*(T: type Eth2DiscoveryProtocol,
# * for setting up a specific key # * for setting up a specific key
# * for using a persistent database # * for using a persistent database
let let
pk = PrivateKey.fromRaw(rawPrivKeyBytes).expect("Valid private key") ourPubKey = pk.toPublicKey()
ourPubKey = pk.toPublicKey().expect("Public key from valid private key")
# TODO: `newMemoryDB()` causes raises: [Exception] # TODO: `newMemoryDB()` causes raises: [Exception]
db = DiscoveryDB.init(newMemoryDB()) db = DiscoveryDB.init(newMemoryDB())

150
beacon_chain/eth2_network.nim
View File

@ -4,18 +4,18 @@ import
options as stdOptions, options as stdOptions,
# Status libs # Status libs
stew/[varints, base58, endians2, results, byteutils], stew/[varints, base58, base64, endians2, results, byteutils],
stew/shims/net as stewNet, stew/shims/net as stewNet,
stew/shims/[macros, tables], stew/shims/[macros, tables],
faststreams/[inputs, outputs, buffers], snappy, snappy/framing, faststreams/[inputs, outputs, buffers], snappy, snappy/framing,
json_serialization, json_serialization/std/[net, options], json_serialization, json_serialization/std/[net, options],
chronos, chronicles, metrics, chronos, chronicles, metrics,
# TODO: create simpler to use libp2p modules that use re-exports # TODO: create simpler to use libp2p modules that use re-exports
libp2p/[switch, standard_setup, peerinfo, peer, connection, errors, libp2p/[switch, standard_setup, peerinfo, peer, errors,
multiaddress, multicodec, crypto/crypto, crypto/secp, multiaddress, multicodec, crypto/crypto, crypto/secp,
protocols/identify, protocols/protocol], protocols/identify, protocols/protocol],
libp2p/protocols/secure/[secure, secio], libp2p/protocols/secure/[secure, secio],
libp2p/protocols/pubsub/[pubsub, floodsub, rpc/messages], libp2p/protocols/pubsub/[pubsub, floodsub, rpc/message, rpc/messages],
libp2p/transports/tcptransport, libp2p/transports/tcptransport,
libp2p/stream/lpstream, libp2p/stream/lpstream,
eth/[keys, async_utils], eth/p2p/p2p_protocol_dsl, eth/[keys, async_utils], eth/p2p/p2p_protocol_dsl,
@ -63,6 +63,7 @@ type
connQueue: AsyncQueue[PeerInfo] connQueue: AsyncQueue[PeerInfo]
seenTable: Table[PeerID, SeenItem] seenTable: Table[PeerID, SeenItem]
connWorkers: seq[Future[void]] connWorkers: seq[Future[void]]
connTable: Table[PeerID, PeerInfo]
forkId: ENRForkID forkId: ENRForkID
EthereumNode = Eth2Node # needed for the definitions in p2p_backends_helpers EthereumNode = Eth2Node # needed for the definitions in p2p_backends_helpers
@ -144,9 +145,10 @@ type
MessageContentPrinter* = proc(msg: pointer): string {.gcsafe.} MessageContentPrinter* = proc(msg: pointer): string {.gcsafe.}
DisconnectionReason* = enum DisconnectionReason* = enum
ClientShutDown # might see other values on the wire!
IrrelevantNetwork ClientShutDown = 1
FaultOrError IrrelevantNetwork = 2
FaultOrError = 3
PeerDisconnected* = object of CatchableError PeerDisconnected* = object of CatchableError
reason*: DisconnectionReason reason*: DisconnectionReason
@ -192,8 +194,6 @@ const
TTFB_TIMEOUT* = 5.seconds TTFB_TIMEOUT* = 5.seconds
RESP_TIMEOUT* = 10.seconds RESP_TIMEOUT* = 10.seconds
readTimeoutErrorMsg = "Exceeded read timeout for a request"
NewPeerScore* = 200 NewPeerScore* = 200
## Score which will be assigned to new connected Peer ## Score which will be assigned to new connected Peer
PeerScoreLowLimit* = 0 PeerScoreLowLimit* = 0
@ -274,10 +274,6 @@ proc openStream(node: Eth2Node,
else: else:
raise raise
func peerId(conn: Connection): PeerID =
# TODO: Can this be `nil`?
conn.peerInfo.peerId
proc init*(T: type Peer, network: Eth2Node, info: PeerInfo): Peer {.gcsafe.} proc init*(T: type Peer, network: Eth2Node, info: PeerInfo): Peer {.gcsafe.}
proc getPeer*(node: Eth2Node, peerInfo: PeerInfo): Peer {.gcsafe.} = proc getPeer*(node: Eth2Node, peerInfo: PeerInfo): Peer {.gcsafe.} =
@ -454,17 +450,6 @@ when useNativeSnappy:
else: else:
include libp2p_streams_backend include libp2p_streams_backend
template awaitWithTimeout[T](operation: Future[T],
deadline: Future[void],
onTimeout: untyped): T =
let f = operation
await f or deadline
if not f.finished:
cancel f
onTimeout
else:
f.read
proc makeEth2Request(peer: Peer, protocolId: string, requestBytes: Bytes, proc makeEth2Request(peer: Peer, protocolId: string, requestBytes: Bytes,
ResponseMsg: type, ResponseMsg: type,
timeout: Duration): Future[NetRes[ResponseMsg]] timeout: Duration): Future[NetRes[ResponseMsg]]
@ -495,7 +480,7 @@ proc init*[MsgType](T: type SingleChunkResponse[MsgType],
peer: Peer, conn: Connection, noSnappy: bool): T = peer: Peer, conn: Connection, noSnappy: bool): T =
T(UntypedResponse(peer: peer, stream: conn, noSnappy: noSnappy)) T(UntypedResponse(peer: peer, stream: conn, noSnappy: noSnappy))
template write*[M](r: MultipleChunksResponse[M], val: M): untyped = template write*[M](r: MultipleChunksResponse[M], val: auto): untyped =
sendResponseChunkObj(UntypedResponse(r), val) sendResponseChunkObj(UntypedResponse(r), val)
template send*[M](r: SingleChunkResponse[M], val: auto): untyped = template send*[M](r: SingleChunkResponse[M], val: auto): untyped =
@ -572,6 +557,11 @@ proc handleIncomingStream(network: Eth2Node,
try: try:
let peer = peerFromStream(network, conn) let peer = peerFromStream(network, conn)
# TODO peer connection setup is broken, update info in some better place
# whenever race is fix:
# https://github.com/status-im/nim-beacon-chain/issues/1157
peer.info = conn.peerInfo
template returnInvalidRequest(msg: ErrorMsg) = template returnInvalidRequest(msg: ErrorMsg) =
await sendErrorResponse(peer, conn, noSnappy, InvalidRequest, msg) await sendErrorResponse(peer, conn, noSnappy, InvalidRequest, msg)
return return
@ -735,10 +725,13 @@ proc connectWorker(network: Eth2Node) {.async.} =
let pi = await network.connQueue.popFirst() let pi = await network.connQueue.popFirst()
let r1 = network.peerPool.hasPeer(pi.peerId) let r1 = network.peerPool.hasPeer(pi.peerId)
let r2 = network.isSeen(pi) let r2 = network.isSeen(pi)
let r3 = network.connTable.hasKey(pi.peerId)
if not(r1) and not(r2): if not(r1) and not(r2) and not(r3):
# We trying to connect to peers which are not present in our PeerPool and network.connTable[pi.peerId] = pi
# not present in our SeenTable. try:
# We trying to connect to peers which are not in PeerPool, SeenTable and
# ConnTable.
var fut = network.dialPeer(pi) var fut = network.dialPeer(pi)
# We discarding here just because we going to check future state, to avoid # We discarding here just because we going to check future state, to avoid
# condition where connection happens and timeout reached. # condition where connection happens and timeout reached.
@ -755,10 +748,12 @@ proc connectWorker(network: Eth2Node) {.async.} =
debug "Connection to remote peer timed out", peer = pi.id debug "Connection to remote peer timed out", peer = pi.id
inc nbc_timeout_dials inc nbc_timeout_dials
network.addSeen(pi, SeenTableTimeTimeout) network.addSeen(pi, SeenTableTimeTimeout)
finally:
network.connTable.del(pi.peerId)
else: else:
trace "Peer is already connected or already seen", peer = pi.id, trace "Peer is already connected, connecting or already seen",
peer_pool_has_peer = $r1, seen_table_has_peer = $r2, peer = pi.id, peer_pool_has_peer = $r1, seen_table_has_peer = $r2,
seen_table_size = len(network.seenTable) connecting_peer = $r3, seen_table_size = len(network.seenTable)
proc runDiscoveryLoop*(node: Eth2Node) {.async.} = proc runDiscoveryLoop*(node: Eth2Node) {.async.} =
debug "Starting discovery loop" debug "Starting discovery loop"
@ -809,11 +804,12 @@ proc init*(T: type Eth2Node, conf: BeaconNodeConf, enrForkId: ENRForkID,
result.connectTimeout = 10.seconds result.connectTimeout = 10.seconds
result.seenThreshold = 10.minutes result.seenThreshold = 10.minutes
result.seenTable = initTable[PeerID, SeenItem]() result.seenTable = initTable[PeerID, SeenItem]()
result.connTable = initTable[PeerID, PeerInfo]()
result.connQueue = newAsyncQueue[PeerInfo](ConcurrentConnections) result.connQueue = newAsyncQueue[PeerInfo](ConcurrentConnections)
result.metadata = getPersistentNetMetadata(conf) result.metadata = getPersistentNetMetadata(conf)
result.forkId = enrForkId result.forkId = enrForkId
result.discovery = Eth2DiscoveryProtocol.new( result.discovery = Eth2DiscoveryProtocol.new(
conf, ip, tcpPort, udpPort, privKey.toRaw, conf, ip, tcpPort, udpPort, privKey,
{"eth2": SSZ.encode(result.forkId), "attnets": SSZ.encode(result.metadata.attnets)}) {"eth2": SSZ.encode(result.forkId), "attnets": SSZ.encode(result.metadata.attnets)})
newSeq result.protocolStates, allProtocols.len newSeq result.protocolStates, allProtocols.len
@ -826,7 +822,7 @@ proc init*(T: type Eth2Node, conf: BeaconNodeConf, enrForkId: ENRForkID,
msg.protocolMounter result msg.protocolMounter result
template publicKey*(node: Eth2Node): keys.PublicKey = template publicKey*(node: Eth2Node): keys.PublicKey =
node.discovery.privKey.toPublicKey.tryGet() node.discovery.privKey.toPublicKey
template addKnownPeer*(node: Eth2Node, peer: enr.Record) = template addKnownPeer*(node: Eth2Node, peer: enr.Record) =
node.discovery.addNode peer node.discovery.addNode peer
@ -1083,6 +1079,13 @@ proc getPersistentNetKeys*(conf: BeaconNodeConf): KeyPair =
KeyPair(seckey: privKey, pubkey: privKey.getKey().tryGet()) KeyPair(seckey: privKey, pubkey: privKey.getKey().tryGet())
func gossipId(data: openArray[byte]): string =
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/p2p-interface.md#topics-and-messages
base64.encode(Base64Url, sha256.digest(data).data)
func msgIdProvider(m: messages.Message): string =
gossipId(m.data)
proc createEth2Node*(conf: BeaconNodeConf, enrForkId: ENRForkID): Future[Eth2Node] {.async, gcsafe.} = proc createEth2Node*(conf: BeaconNodeConf, enrForkId: ENRForkID): Future[Eth2Node] {.async, gcsafe.} =
var var
(extIp, extTcpPort, extUdpPort) = setupNat(conf) (extIp, extTcpPort, extUdpPort) = setupNat(conf)
@ -1100,7 +1103,8 @@ proc createEth2Node*(conf: BeaconNodeConf, enrForkId: ENRForkID): Future[Eth2Nod
var switch = newStandardSwitch(some keys.seckey, hostAddress, var switch = newStandardSwitch(some keys.seckey, hostAddress,
triggerSelf = true, gossip = true, triggerSelf = true, gossip = true,
sign = false, verifySignature = false, sign = false, verifySignature = false,
transportFlags = {ServerFlags.ReuseAddr}) transportFlags = {ServerFlags.ReuseAddr},
msgIdProvider = msgIdProvider)
result = Eth2Node.init(conf, enrForkId, switch, result = Eth2Node.init(conf, enrForkId, switch,
extIp, extTcpPort, extUdpPort, extIp, extTcpPort, extUdpPort,
keys.seckey.asEthKey) keys.seckey.asEthKey)
@ -1138,69 +1142,47 @@ proc subscribe*[MsgType](node: Eth2Node,
topic: string, topic: string,
msgHandler: proc(msg: MsgType) {.gcsafe.}, msgHandler: proc(msg: MsgType) {.gcsafe.},
msgValidator: proc(msg: MsgType): bool {.gcsafe.} ) {.async, gcsafe.} = msgValidator: proc(msg: MsgType): bool {.gcsafe.} ) {.async, gcsafe.} =
template execMsgHandler(peerExpr, gossipBytes, gossipTopic, useSnappy) = proc execMsgHandler(topic: string, data: seq[byte]) {.async, gcsafe.} =
inc nbc_gossip_messages_received inc nbc_gossip_messages_received
trace "Incoming pubsub message received", trace "Incoming pubsub message received",
peer = peerExpr, len = gossipBytes.len, topic = gossipTopic, len = data.len, topic, msgId = gossipId(data)
message_id = `$`(sha256.digest(gossipBytes)) try:
when useSnappy: msgHandler SSZ.decode(snappy.decode(data), MsgType)
msgHandler SSZ.decode(snappy.decode(gossipBytes), MsgType) except CatchableError as err:
else: debug "Gossip msg handler error",
msgHandler SSZ.decode(gossipBytes, MsgType) msg = err.msg, len = data.len, topic, msgId = gossipId(data)
# All message types which are subscribed to should be validated; putting
# this in subscribe(...) ensures that the default approach is correct.
template execMsgValidator(gossipBytes, gossipTopic, useSnappy): bool =
trace "Incoming pubsub message received for validation",
len = gossipBytes.len, topic = gossipTopic,
message_id = `$`(sha256.digest(gossipBytes))
when useSnappy:
msgValidator SSZ.decode(snappy.decode(gossipBytes), MsgType)
else:
msgValidator SSZ.decode(gossipBytes, MsgType)
# Validate messages as soon as subscribed # Validate messages as soon as subscribed
let incomingMsgValidator = proc(topic: string, proc execValidator(
message: GossipMsg): Future[bool] topic: string, message: GossipMsg): Future[bool] {.async, gcsafe.} =
{.async, gcsafe.} = trace "Validating incoming gossip message",
return execMsgValidator(message.data, topic, false) len = message.data.len, topic, msgId = gossipId(message.data)
let incomingMsgValidatorSnappy = proc(topic: string, try:
message: GossipMsg): Future[bool] return msgValidator SSZ.decode(snappy.decode(message.data), MsgType)
{.async, gcsafe.} = except CatchableError as err:
return execMsgValidator(message.data, topic, true) debug "Gossip validation error",
msg = err.msg, msgId = gossipId(message.data)
return false
node.switch.addValidator(topic, incomingMsgValidator) node.switch.addValidator(topic & "_snappy", execValidator)
node.switch.addValidator(topic & "_snappy", incomingMsgValidatorSnappy)
let incomingMsgHandler = proc(topic: string, await node.switch.subscribe(topic & "_snappy", execMsgHandler)
data: seq[byte]) {.async, gcsafe.} =
execMsgHandler "unknown", data, topic, false
let incomingMsgHandlerSnappy = proc(topic: string,
data: seq[byte]) {.async, gcsafe.} =
execMsgHandler "unknown", data, topic, true
var switchSubscriptions: seq[Future[void]] = @[] proc traceMessage(fut: FutureBase, msgId: string) =
switchSubscriptions.add(node.switch.subscribe(topic, incomingMsgHandler))
switchSubscriptions.add(node.switch.subscribe(topic & "_snappy", incomingMsgHandlerSnappy))
await allFutures(switchSubscriptions)
proc traceMessage(fut: FutureBase, digest: MDigest[256]) =
fut.addCallback do (arg: pointer): fut.addCallback do (arg: pointer):
if not(fut.failed): if not(fut.failed):
trace "Outgoing pubsub message sent", message_id = `$`(digest) trace "Outgoing pubsub message sent", msgId
elif fut.error != nil:
debug "Gossip message not sent", msgId, err = fut.error.msg
else:
debug "Unexpected future state for gossip", msgId, state = fut.state
proc broadcast*(node: Eth2Node, topic: string, msg: auto) = proc broadcast*(node: Eth2Node, topic: string, msg: auto) =
inc nbc_gossip_messages_sent inc nbc_gossip_messages_sent
let broadcastBytes = SSZ.encode(msg) let
var fut = node.switch.publish(topic, broadcastBytes) data = snappy.encode(SSZ.encode(msg))
traceMessage(fut, sha256.digest(broadcastBytes)) var futSnappy = node.switch.publish(topic & "_snappy", data)
traceAsyncErrors(fut) traceMessage(futSnappy, gossipId(data))
# also publish to the snappy-compressed topics
let snappyEncoded = snappy.encode(broadcastBytes)
var futSnappy = node.switch.publish(topic & "_snappy", snappyEncoded)
traceMessage(futSnappy, sha256.digest(snappyEncoded))
traceAsyncErrors(futSnappy)
# TODO: # TODO:
# At the moment, this is just a compatiblity shim for the existing RLPx functionality. # At the moment, this is just a compatiblity shim for the existing RLPx functionality.

5
beacon_chain/extras.nim
View File

@ -20,17 +20,12 @@
type type
UpdateFlag* = enum UpdateFlag* = enum
skipMerkleValidation ##\
## When processing deposits, skip verifying the Merkle proof trees of each
## deposit.
skipBlsValidation ##\ skipBlsValidation ##\
## Skip verification of BLS signatures in block processing. ## Skip verification of BLS signatures in block processing.
## Predominantly intended for use in testing, e.g. to allow extra coverage. ## Predominantly intended for use in testing, e.g. to allow extra coverage.
## Also useful to avoid unnecessary work when replaying known, good blocks. ## Also useful to avoid unnecessary work when replaying known, good blocks.
skipStateRootValidation ##\ skipStateRootValidation ##\
## Skip verification of block state root. ## Skip verification of block state root.
skipBlockParentRootValidation ##\
## Skip verification that the block's parent root matches the previous block header.
verifyFinalization verifyFinalization
UpdateFlags* = set[UpdateFlag] UpdateFlags* = set[UpdateFlag]

28
beacon_chain/inspector.nim
View File

@ -6,7 +6,7 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms. # at your option. This file may not be copied, modified, or distributed except according to those terms.
import strutils, os, tables, options import strutils, os, tables, options
import confutils, chronicles, chronos import confutils, chronicles, chronos
import libp2p/[switch, standard_setup, connection, multiaddress, multicodec, import libp2p/[switch, standard_setup, multiaddress, multicodec,
peer, peerinfo, peer] peer, peerinfo, peer]
import libp2p/crypto/crypto as lcrypto import libp2p/crypto/crypto as lcrypto
import libp2p/crypto/secp as lsecp import libp2p/crypto/secp as lsecp
@ -52,10 +52,8 @@ type
next_fork_version*: Version next_fork_version*: Version
next_fork_epoch*: Epoch next_fork_epoch*: Epoch
# TODO remove InteropAttestations when Altona launches
TopicFilter* {.pure.} = enum TopicFilter* {.pure.} = enum
Blocks, Attestations, Exits, ProposerSlashing, AttesterSlashings, Blocks, Attestations, Exits, ProposerSlashing, AttesterSlashings
InteropAttestations
BootstrapKind* {.pure.} = enum BootstrapKind* {.pure.} = enum
Enr, MultiAddr Enr, MultiAddr
@ -197,18 +195,12 @@ func getTopics(forkDigest: ForkDigest,
of TopicFilter.AttesterSlashings: of TopicFilter.AttesterSlashings:
let topic = getAttesterSlashingsTopic(forkDigest) let topic = getAttesterSlashingsTopic(forkDigest)
@[topic, topic & "_snappy"] @[topic, topic & "_snappy"]
of TopicFilter.InteropAttestations:
when ETH2_SPEC == "v0.11.3":
let topic = getInteropAttestationTopic(forkDigest)
@[topic, topic & "_snappy"]
else:
@[]
of TopicFilter.Attestations: of TopicFilter.Attestations:
var topics = newSeq[string](ATTESTATION_SUBNET_COUNT * 2) var topics = newSeq[string](ATTESTATION_SUBNET_COUNT * 2)
var offset = 0 var offset = 0
for i in 0'u64 ..< ATTESTATION_SUBNET_COUNT.uint64: for i in 0'u64 ..< ATTESTATION_SUBNET_COUNT.uint64:
topics[offset] = getMainnetAttestationTopic(forkDigest, i) topics[offset] = getAttestationTopic(forkDigest, i)
topics[offset + 1] = getMainnetAttestationTopic(forkDigest, i) & "_snappy" topics[offset + 1] = getAttestationTopic(forkDigest, i) & "_snappy"
offset += 2 offset += 2
topics topics
@ -537,10 +529,6 @@ proc pubsubLogger(conf: InspectorConf, switch: Switch,
elif topic.endsWith(topicAggregateAndProofsSuffix) or elif topic.endsWith(topicAggregateAndProofsSuffix) or
topic.endsWith(topicAggregateAndProofsSuffix & "_snappy"): topic.endsWith(topicAggregateAndProofsSuffix & "_snappy"):
info "AggregateAndProof", msg = SSZ.decode(buffer, AggregateAndProof) info "AggregateAndProof", msg = SSZ.decode(buffer, AggregateAndProof)
when ETH2_SPEC == "v0.11.3":
if topic.endsWith(topicInteropAttestationSuffix) or
topic.endsWith(topicInteropAttestationSuffix & "_snappy"):
info "Attestation", msg = SSZ.decode(buffer, Attestation)
except CatchableError as exc: except CatchableError as exc:
info "Unable to decode message", errMsg = exc.msg info "Unable to decode message", errMsg = exc.msg
@ -708,8 +696,6 @@ proc run(conf: InspectorConf) {.async.} =
topics.incl({TopicFilter.Blocks, TopicFilter.Attestations, topics.incl({TopicFilter.Blocks, TopicFilter.Attestations,
TopicFilter.Exits, TopicFilter.ProposerSlashing, TopicFilter.Exits, TopicFilter.ProposerSlashing,
TopicFilter.AttesterSlashings}) TopicFilter.AttesterSlashings})
when ETH2_SPEC == "v0.11.3":
topics.incl({TopicFilter.AttesterSlashings})
break break
elif lcitem == "a": elif lcitem == "a":
topics.incl(TopicFilter.Attestations) topics.incl(TopicFilter.Attestations)
@ -723,16 +709,10 @@ proc run(conf: InspectorConf) {.async.} =
topics.incl(TopicFilter.AttesterSlashings) topics.incl(TopicFilter.AttesterSlashings)
else: else:
discard discard
when ETH2_SPEC == "v0.11.3":
if lcitem == "ia":
topics.incl(TopicFilter.InteropAttestations)
else: else:
topics.incl({TopicFilter.Blocks, TopicFilter.Attestations, topics.incl({TopicFilter.Blocks, TopicFilter.Attestations,
TopicFilter.Exits, TopicFilter.ProposerSlashing, TopicFilter.Exits, TopicFilter.ProposerSlashing,
TopicFilter.AttesterSlashings}) TopicFilter.AttesterSlashings})
when ETH2_SPEC == "v0.11.3":
topics.incl({TopicFilter.AttesterSlashings})
proc pubsubTrampoline(topic: string, proc pubsubTrampoline(topic: string,
data: seq[byte]): Future[void] {.gcsafe.} = data: seq[byte]): Future[void] {.gcsafe.} =

51
beacon_chain/keystore_management.nim
View File

@ -1,9 +1,12 @@
import import
os, strutils, terminal, os, strutils, terminal, random,
chronicles, chronos, blscurve, nimcrypto, json_serialization, serialization, chronicles, chronos, blscurve, nimcrypto, json_serialization, serialization,
web3, stint, eth/keys, confutils, web3, stint, eth/keys, confutils,
spec/[datatypes, digest, crypto, keystore], conf, ssz/merkleization, merkle_minimal spec/[datatypes, digest, crypto, keystore], conf, ssz/merkleization, merkle_minimal
export
keystore
contract(DepositContract): contract(DepositContract):
proc deposit(pubkey: Bytes48, withdrawalCredentials: Bytes32, signature: Bytes96, deposit_data_root: FixedBytes[32]) proc deposit(pubkey: Bytes48, withdrawalCredentials: Bytes32, signature: Bytes96, deposit_data_root: FixedBytes[32])
@ -109,7 +112,7 @@ proc generateDeposits*(totalValidators: int,
let credentials = generateCredentials(password = password) let credentials = generateCredentials(password = password)
let let
keyName = $(credentials.signingKey.toPubKey) keyName = intToStr(i, 6) & "_" & $(credentials.signingKey.toPubKey)
validatorDir = validatorsDir / keyName validatorDir = validatorsDir / keyName
passphraseFile = secretsDir / keyName passphraseFile = secretsDir / keyName
depositFile = validatorDir / depositFileName depositFile = validatorDir / depositFileName
@ -139,14 +142,34 @@ proc generateDeposits*(totalValidators: int,
ok deposits ok deposits
proc loadDeposits*(depositsDir: string): seq[Deposit] =
try:
for kind, dir in walkDir(depositsDir):
if kind == pcDir:
let depositFile = dir / depositFileName
try:
result.add Json.loadFile(depositFile, Deposit)
except IOError as err:
error "Failed to open deposit file", depositFile, err = err.msg
quit 1
except SerializationError as err:
error "Invalid deposit file", error = formatMsg(err, depositFile)
quit 1
except OSError as err:
error "Deposits directory not accessible",
path = depositsDir, err = err.msg
quit 1
{.pop.} {.pop.}
# TODO: async functions should note take `seq` inputs because
# this leads to full copies.
proc sendDeposits*(deposits: seq[Deposit], proc sendDeposits*(deposits: seq[Deposit],
web3Url, depositContractAddress, privateKey: string, web3Url, depositContractAddress, privateKey: string,
delayGenerator: DelayGenerator = nil) {.async.} = delayGenerator: DelayGenerator = nil) {.async.} =
var web3 = await newWeb3(web3Url) var web3 = await newWeb3(web3Url)
if privateKey.len != 0: if privateKey.len != 0:
web3.privateKey = PrivateKey.fromHex(privateKey).tryGet web3.privateKey = some(PrivateKey.fromHex(privateKey).tryGet)
else: else:
let accounts = await web3.provider.eth_accounts() let accounts = await web3.provider.eth_accounts()
if accounts.len == 0: if accounts.len == 0:
@ -158,12 +181,32 @@ proc sendDeposits*(deposits: seq[Deposit],
let depositContract = web3.contractSender(DepositContract, contractAddress) let depositContract = web3.contractSender(DepositContract, contractAddress)
for i, dp in deposits: for i, dp in deposits:
discard await depositContract.deposit( let status = await depositContract.deposit(
Bytes48(dp.data.pubKey.toRaw()), Bytes48(dp.data.pubKey.toRaw()),
Bytes32(dp.data.withdrawal_credentials.data), Bytes32(dp.data.withdrawal_credentials.data),
Bytes96(dp.data.signature.toRaw()), Bytes96(dp.data.signature.toRaw()),
FixedBytes[32](hash_tree_root(dp.data).data)).send(value = 32.u256.ethToWei, gasPrice = 1) FixedBytes[32](hash_tree_root(dp.data).data)).send(value = 32.u256.ethToWei, gasPrice = 1)
info "Deposit sent", status = $status
if delayGenerator != nil: if delayGenerator != nil:
await sleepAsync(delayGenerator()) await sleepAsync(delayGenerator())
proc sendDeposits*(config: BeaconNodeConf,
deposits: seq[Deposit]) {.async.} =
var delayGenerator: DelayGenerator
if config.maxDelay > 0.0:
delayGenerator = proc (): chronos.Duration {.gcsafe.} =
chronos.milliseconds (rand(config.minDelay..config.maxDelay)*1000).int
info "Sending deposits",
web3 = config.web3Url,
depositContract = config.depositContractAddress
await sendDeposits(
deposits,
config.web3Url,
config.depositContractAddress,
config.depositPrivateKey,
delayGenerator)

743
beacon_chain/mainchain_monitor.nim
View File

@ -1,7 +1,13 @@
import import
deques, tables, hashes, options, deques, tables, hashes, options, strformat,
chronos, web3, json, chronicles, chronos, web3, web3/ethtypes, json, chronicles, eth/async_utils,
spec/[datatypes, digest, crypto, beaconstate, helpers] spec/[datatypes, digest, crypto, beaconstate, helpers],
merkle_minimal
from times import epochTime
export
ethtypes
contract(DepositContract): contract(DepositContract):
proc deposit(pubkey: Bytes48, proc deposit(pubkey: Bytes48,
@ -17,7 +23,6 @@ contract(DepositContract):
amount: Bytes8, amount: Bytes8,
signature: Bytes96, signature: Bytes96,
index: Bytes8) {.event.} index: Bytes8) {.event.}
# TODO # TODO
# The raises list of this module are still not usable due to general # The raises list of this module are still not usable due to general
# Exceptions being reported from Chronos's asyncfutures2. # Exceptions being reported from Chronos's asyncfutures2.
@ -31,30 +36,29 @@ type
timestamp*: Eth1BlockTimestamp timestamp*: Eth1BlockTimestamp
deposits*: seq[Deposit] deposits*: seq[Deposit]
voteData*: Eth1Data voteData*: Eth1Data
knownGoodDepositsCount*: Option[uint64]
Eth1Chain* = object Eth1Chain* = object
knownStart: Eth1Data
knownStartBlockNum: Option[Eth1BlockNumber]
blocks: Deque[Eth1Block] blocks: Deque[Eth1Block]
blocksByHash: Table[BlockHash, Eth1Block] blocksByHash: Table[BlockHash, Eth1Block]
allDeposits*: seq[Deposit]
MainchainMonitor* = ref object MainchainMonitor* = ref object
startBlock: BlockHash
depositContractAddress: Address depositContractAddress: Address
dataProviderFactory*: DataProviderFactory dataProviderFactory*: DataProviderFactory
genesisState: NilableBeaconStateRef genesisState: NilableBeaconStateRef
genesisStateFut: Future[void] genesisStateFut: Future[void]
genesisMonitoringFut: Future[void]
eth1Chain: Eth1Chain eth1Chain: Eth1Chain
depositQueue: AsyncQueue[DepositQueueElem] depositQueue: AsyncQueue[BlockHeader]
runFut: Future[void] runFut: Future[void]
Web3EventType = enum
NewEvent
RemovedEvent
DepositQueueElem = (BlockHash, Web3EventType)
DataProvider* = object of RootObj DataProvider* = object of RootObj
DataProviderRef* = ref DataProvider DataProviderRef* = ref DataProvider
@ -69,7 +73,7 @@ type
url: string url: string
web3: Web3 web3: Web3
ns: Sender[DepositContract] ns: Sender[DepositContract]
subscription: Subscription blockHeadersSubscription: Subscription
Web3DataProviderRef* = ref Web3DataProvider Web3DataProviderRef* = ref Web3DataProvider
@ -86,11 +90,21 @@ type
const const
reorgDepthLimit = 1000 reorgDepthLimit = 1000
web3Timeouts = 5.seconds
followDistanceInSeconds = uint64(SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE)
totalDepositsNeededForGenesis = uint64 max(SLOTS_PER_EPOCH,
MIN_GENESIS_ACTIVE_VALIDATOR_COUNT)
# TODO Nim's analysis on the lock level of the methods in this # TODO Nim's analysis on the lock level of the methods in this
# module seems broken. Investigate and file this as an issue. # module seems broken. Investigate and file this as an issue.
{.push warning[LockLevel]: off.} {.push warning[LockLevel]: off.}
static:
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#genesis
when SPEC_VERSION == "0.12.1":
doAssert SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE < GENESIS_DELAY,
"Invalid configuration: GENESIS_DELAY is set too low"
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#get_eth1_data # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#get_eth1_data
func compute_time_at_slot(state: BeaconState, slot: Slot): uint64 = func compute_time_at_slot(state: BeaconState, slot: Slot): uint64 =
state.genesis_time + slot * SECONDS_PER_SLOT state.genesis_time + slot * SECONDS_PER_SLOT
@ -106,12 +120,15 @@ func is_candidate_block(blk: Eth1Block, period_start: uint64): bool =
(blk.timestamp + SECONDS_PER_ETH1_BLOCK.uint64 * ETH1_FOLLOW_DISTANCE.uint64 <= period_start) and (blk.timestamp + SECONDS_PER_ETH1_BLOCK.uint64 * ETH1_FOLLOW_DISTANCE.uint64 <= period_start) and
(blk.timestamp + SECONDS_PER_ETH1_BLOCK.uint64 * ETH1_FOLLOW_DISTANCE.uint64 * 2 >= period_start) (blk.timestamp + SECONDS_PER_ETH1_BLOCK.uint64 * ETH1_FOLLOW_DISTANCE.uint64 * 2 >= period_start)
func asEth2Digest(x: BlockHash): Eth2Digest = func asEth2Digest*(x: BlockHash): Eth2Digest =
Eth2Digest(data: array[32, byte](x)) Eth2Digest(data: array[32, byte](x))
template asBlockHash(x: Eth2Digest): BlockHash = template asBlockHash(x: Eth2Digest): BlockHash =
BlockHash(x.data) BlockHash(x.data)
func shortLog(b: Eth1Block): string =
&"{b.number}:{shortLog b.voteData.block_hash}"
func getDepositsInRange(eth1Chain: Eth1Chain, func getDepositsInRange(eth1Chain: Eth1Chain,
sinceBlock, latestBlock: Eth1BlockNumber): seq[Deposit] = sinceBlock, latestBlock: Eth1BlockNumber): seq[Deposit] =
## Returns all deposits that happened AFTER the block `sinceBlock` (not inclusive). ## Returns all deposits that happened AFTER the block `sinceBlock` (not inclusive).
@ -147,55 +164,44 @@ proc findParent*(eth1Chain: Eth1Chain, blk: BlockObject): Eth1Block =
parentHash = blk.parentHash.toHex, parentNumber = result.number parentHash = blk.parentHash.toHex, parentNumber = result.number
result = nil result = nil
when false:
func getCacheIdx(eth1Chain: Eth1Chain, blockNumber: Eth1BlockNumber): int =
if eth1Chain.blocks.len == 0:
return -1
let idx = blockNumber - eth1Chain.blocks[0].number
if idx < 0 or idx >= eth1Chain.blocks.len:
return -1
idx
func `{}`*(eth1Chain: Eth1Chain, blockNumber: Eth1BlockNumber): Eth1Block =
## Finds a block in our cache that corresponds to a particular Eth block
## number. May return `nil` if we don't have such a block in the cache.
let idx = eth1Chain.getCacheIdx(blockNumber)
if idx != -1: eth1Chain.blocks[idx] else: nil
func latestCandidateBlock(eth1Chain: Eth1Chain, periodStart: uint64): Eth1Block = func latestCandidateBlock(eth1Chain: Eth1Chain, periodStart: uint64): Eth1Block =
for i in countdown(eth1Chain.blocks.len - 1, 0): for i in countdown(eth1Chain.blocks.len - 1, 0):
let blk = eth1Chain.blocks[i] let blk = eth1Chain.blocks[i]
if is_candidate_block(blk, periodStart): if is_candidate_block(blk, periodStart):
return blk return blk
func trimHeight(eth1Chain: var Eth1Chain, blockNumber: Eth1BlockNumber) = func popBlock(eth1Chain: var Eth1Chain) =
## Removes all blocks above certain `blockNumber`
if eth1Chain.blocks.len == 0:
return
let newLen = max(0, int(blockNumber - eth1Chain.blocks[0].number + 1))
for i in newLen ..< eth1Chain.blocks.len:
let removed = eth1Chain.blocks.popLast let removed = eth1Chain.blocks.popLast
eth1Chain.blocksByHash.del removed.voteData.block_hash.asBlockHash eth1Chain.blocksByHash.del removed.voteData.block_hash.asBlockHash
template purgeChain*(eth1Chain: var Eth1Chain, blk: Eth1Block) = func trimHeight(eth1Chain: var Eth1Chain, blockNumber: Eth1BlockNumber) =
## This is used when we discover that a previously considered block ## Removes all blocks above certain `blockNumber`
## is no longer part of the selected chain (due to a reorg). We can while eth1Chain.blocks.len > 0:
## then remove from our chain together with all blocks that follow it. if eth1Chain.blocks.peekLast.number > blockNumber:
trimHeight(eth1Chain, blk.number - 1) eth1Chain.popBlock()
else:
break
func purgeChain*(eth1Chain: var Eth1Chain, blockHash: BlockHash) =
let blk = eth1Chain.findBlock(blockHash)
if blk != nil: eth1Chain.purgeChain(blk)
template purgeDescendants*(eth1CHain: Eth1Chain, blk: Eth1Block) =
trimHeight(eth1Chain, blk.number)
func addBlock*(eth1Chain: var Eth1Chain, newBlock: Eth1Block) =
if eth1Chain.blocks.len > 0: if eth1Chain.blocks.len > 0:
doAssert eth1Chain.blocks.peekLast.number + 1 == newBlock.number eth1Chain.allDeposits.setLen(eth1Chain.blocks[^1].voteData.deposit_count)
else:
eth1Chain.allDeposits.setLen(0)
func isSuccessorBlock(eth1Chain: Eth1Chain, newBlock: Eth1Block): bool =
let currentDepositCount = if eth1Chain.blocks.len == 0:
eth1Chain.knownStart.deposit_count
else:
let lastBlock = eth1Chain.blocks.peekLast
if lastBlock.number >= newBlock.number: return false
lastBlock.voteData.deposit_count
(currentDepositCount + newBlock.deposits.len.uint64) == newBlock.voteData.deposit_count
func addSuccessorBlock*(eth1Chain: var Eth1Chain, newBlock: Eth1Block): bool =
result = isSuccessorBlock(eth1Chain, newBlock)
if result:
eth1Chain.allDeposits.add newBlock.deposits
reset newBlock.deposits
eth1Chain.blocks.addLast newBlock eth1Chain.blocks.addLast newBlock
eth1Chain.blocksByHash[newBlock.voteData.block_hash.asBlockHash] = newBlock eth1Chain.blocksByHash[newBlock.voteData.block_hash.asBlockHash] = newBlock
@ -207,6 +213,9 @@ func allDeposits*(eth1Chain: Eth1Chain): seq[Deposit] =
for blk in eth1Chain.blocks: for blk in eth1Chain.blocks:
result.add blk.deposits result.add blk.deposits
func clear*(eth1Chain: var Eth1Chain) =
eth1Chain = default(Eth1Chain)
template hash*(x: Eth1Block): Hash = template hash*(x: Eth1Block): Hash =
hash(x.voteData.block_hash.data) hash(x.voteData.block_hash.data)
@ -219,8 +228,15 @@ method getBlockByHash*(p: DataProviderRef, hash: BlockHash): Future[BlockObject]
locks: 0 locks: 0
# raises: [Defect] # raises: [Defect]
.} = .} =
discard notImplemented
# notImplemented
method getBlockByNumber*(p: DataProviderRef, hash: Eth1BlockNumber): Future[BlockObject] {.
base
gcsafe
locks: 0
# raises: [Defect]
.} =
notImplemented
method onDisconnect*(p: DataProviderRef, handler: DisconnectHandler) {. method onDisconnect*(p: DataProviderRef, handler: DisconnectHandler) {.
base base
@ -230,9 +246,9 @@ method onDisconnect*(p: DataProviderRef, handler: DisconnectHandler) {.
.} = .} =
notImplemented notImplemented
method onDepositEvent*(p: DataProviderRef, method onBlockHeaders*(p: DataProviderRef,
startBlock: Eth1BlockNumber, blockHeaderHandler: BlockHeaderHandler,
handler: DepositEventHandler): Future[void] {. errorHandler: SubscriptionErrorHandler): Future[void] {.
base base
gcsafe gcsafe
locks: 0 locks: 0
@ -249,7 +265,7 @@ method close*(p: DataProviderRef): Future[void] {.
notImplemented notImplemented
method fetchDepositData*(p: DataProviderRef, method fetchDepositData*(p: DataProviderRef,
web3Block: BlockObject): Future[Eth1Block] {. fromBlock, toBlock: Eth1BlockNumber): Future[seq[Eth1Block]] {.
base base
gcsafe gcsafe
locks: 0 locks: 0
@ -257,6 +273,153 @@ method fetchDepositData*(p: DataProviderRef,
.} = .} =
notImplemented notImplemented
method fetchBlockDetails(p: DataProviderRef, blk: Eth1Block): Future[void] {.
base
gcsafe
locks: 0
# raises: [Defect, CatchableError]
.} =
notImplemented
proc new*(T: type Web3DataProvider,
web3Url: string,
depositContractAddress: Address): Future[ref Web3DataProvider] {.
async
# raises: [Defect]
.} =
try:
type R = ref T
let
web3 = await newWeb3(web3Url)
ns = web3.contractSender(DepositContract, depositContractAddress)
return R(url: web3Url, web3: web3, ns: ns)
except CatchableError:
return nil
func web3Provider*(web3Url: string): DataProviderFactory =
proc factory(depositContractAddress: Address): Future[DataProviderRef] {.async.} =
result = await Web3DataProvider.new(web3Url, depositContractAddress)
DataProviderFactory(desc: "web3(" & web3Url & ")", new: factory)
method close*(p: Web3DataProviderRef): Future[void] {.async, locks: 0.} =
if p.blockHeadersSubscription != nil:
await p.blockHeadersSubscription.unsubscribe()
await p.web3.close()
method getBlockByHash*(p: Web3DataProviderRef, hash: BlockHash): Future[BlockObject] =
return p.web3.provider.eth_getBlockByHash(hash, false)
method getBlockByNumber*(p: Web3DataProviderRef, number: Eth1BlockNumber): Future[BlockObject] =
return p.web3.provider.eth_getBlockByNumber(&"0x{number:X}", false)
proc getBlockNumber(p: DataProviderRef, hash: BlockHash): Future[Eth1BlockNumber] {.async.} =
try:
let blk = awaitWithTimeout(p.getBlockByHash(hash), web3Timeouts):
return 0
return Eth1BlockNumber(blk.number)
except CatchableError as exc:
notice "Failed to get Eth1 block number from hash",
hash = $hash, err = exc.msg
raise
template readJsonField(j: JsonNode,
fieldName: string,
ValueType: type): untyped =
var res: ValueType
fromJson(j[fieldName], fieldName, res)
res
proc readJsonDeposits(depositsList: JsonNode): seq[Eth1Block] =
if depositsList.kind != JArray:
raise newException(CatchableError,
"Web3 provider didn't return a list of deposit events")
var lastEth1Block: Eth1Block
for logEvent in depositsList:
let
blockNumber = Eth1BlockNumber readJsonField(logEvent, "blockNumber", Quantity)
blockHash = readJsonField(logEvent, "blockHash", BlockHash)
logData = strip0xPrefix(logEvent["data"].getStr)
if lastEth1Block == nil or lastEth1Block.number != blockNumber:
lastEth1Block = Eth1Block(
number: blockNumber,
voteData: Eth1Data(block_hash: blockHash.asEth2Digest))
result.add lastEth1Block
var
pubkey: Bytes48
withdrawalCredentials: Bytes32
amount: Bytes8
signature: Bytes96
index: Bytes8
var offset = 0
offset += decode(logData, offset, pubkey)
offset += decode(logData, offset, withdrawalCredentials)
offset += decode(logData, offset, amount)
offset += decode(logData, offset, signature)
offset += decode(logData, offset, index)
lastEth1Block.deposits.add Deposit(
data: DepositData(
pubkey: ValidatorPubKey.init(array[48, byte](pubkey)),
withdrawal_credentials: Eth2Digest(data: array[32, byte](withdrawalCredentials)),
amount: bytes_to_int(array[8, byte](amount)),
signature: ValidatorSig.init(array[96, byte](signature))))
method fetchDepositData*(p: Web3DataProviderRef,
fromBlock, toBlock: Eth1BlockNumber): Future[seq[Eth1Block]]
{.async, locks: 0.} =
info "Obtaining deposit log events", fromBlock, toBlock
return readJsonDeposits(await p.ns.getJsonLogs(DepositEvent,
fromBlock = some blockId(fromBlock),
toBlock = some blockId(toBlock)))
method fetchBlockDetails(p: Web3DataProviderRef, blk: Eth1Block) {.async.} =
let
web3Block = p.getBlockByNumber(blk.number)
depositRoot = p.ns.get_deposit_root.call(blockNumber = blk.number)
rawCount = p.ns.get_deposit_count.call(blockNumber = blk.number)
discard await web3Block
discard await depositRoot
discard await rawCount
let depositCount = bytes_to_int(array[8, byte](rawCount.read))
blk.timestamp = Eth1BlockTimestamp(web3Block.read.timestamp)
blk.voteData.deposit_count = depositCount
blk.voteData.deposit_root = depositRoot.read.asEth2Digest
method onDisconnect*(p: Web3DataProviderRef, handler: DisconnectHandler) {.
gcsafe
locks: 0
# raises: []
.} =
p.web3.onDisconnect = handler
method onBlockHeaders*(p: Web3DataProviderRef,
blockHeaderHandler: BlockHeaderHandler,
errorHandler: SubscriptionErrorHandler): Future[void] {.
async
gcsafe
locks: 0
# raises: []
.} =
if p.blockHeadersSubscription != nil:
await p.blockHeadersSubscription.unsubscribe()
info "Waiting for new Eth1 block headers"
let options = newJObject()
p.blockHeadersSubscription = await p.web3.subscribeForBlockHeaders(
options, blockHeaderHandler, errorHandler)
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md#get_eth1_data # https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md#get_eth1_data
func getBlockProposalData*(eth1Chain: Eth1Chain, func getBlockProposalData*(eth1Chain: Eth1Chain,
state: BeaconState): (Eth1Data, seq[Deposit]) = state: BeaconState): (Eth1Data, seq[Deposit]) =
@ -296,147 +459,163 @@ template getBlockProposalData*(m: MainchainMonitor, state: BeaconState): untyped
proc init*(T: type MainchainMonitor, proc init*(T: type MainchainMonitor,
dataProviderFactory: DataProviderFactory, dataProviderFactory: DataProviderFactory,
depositContractAddress: string, depositContractAddress: string,
startBlock: Eth2Digest): T = startPosition: Eth1Data): T =
T(depositContractAddress: Address.fromHex(depositContractAddress), T(depositQueue: newAsyncQueue[BlockHeader](),
depositQueue: newAsyncQueue[DepositQueueElem](), dataProviderFactory: dataProviderFactory,
startBlock: BlockHash(startBlock.data), depositContractAddress: Address.fromHex(depositContractAddress),
dataProviderFactory: dataProviderFactory) eth1Chain: Eth1Chain(knownStart: startPosition))
const MIN_GENESIS_TIME = 0 proc isCandidateForGenesis(timeNow: float, blk: Eth1Block): bool =
if float(blk.timestamp + followDistanceInSeconds) > timeNow:
return false
proc readJsonDeposits(json: JsonNode): seq[Deposit] = if genesis_time_from_eth1_timestamp(blk.timestamp) < MIN_GENESIS_TIME:
if json.kind != JArray: return false
raise newException(CatchableError,
"Web3 provider didn't return a list of deposit events")
for logEvent in json: if blk.knownGoodDepositsCount.isSome:
var logData = strip0xPrefix(json["data"].getStr) blk.knownGoodDepositsCount.get >= totalDepositsNeededForGenesis
var else:
pubkey: Bytes48 blk.voteData.deposit_count >= totalDepositsNeededForGenesis
withdrawalCredentials: Bytes32
amount: Bytes8
signature: Bytes96
index: Bytes8
var offset = 0 proc minGenesisCandidateBlockIdx(eth1Chain: Eth1Chain): Option[int]
offset = decode(logData, offset, pubkey) {.raises: [Defect].} =
offset = decode(logData, offset, withdrawalCredentials) if eth1Chain.blocks.len == 0:
offset = decode(logData, offset, amount)
offset = decode(logData, offset, signature)
offset = decode(logData, offset, index)
result.add Deposit(
# proof: TODO
data: DepositData(
pubkey: ValidatorPubKey.init(array[48, byte](pubkey)),
withdrawal_credentials: Eth2Digest(data: array[32, byte](withdrawalCredentials)),
amount: bytes_to_int(array[8, byte](amount)),
signature: ValidatorSig.init(array[96, byte](signature))))
proc checkForGenesisEvent(m: MainchainMonitor) =
if not m.genesisState.isNil:
return return
let lastBlock = m.eth1Chain.blocks.peekLast let now = epochTime()
const totalDepositsNeeded = max(SLOTS_PER_EPOCH, if not isCandidateForGenesis(now, eth1Chain.blocks.peekLast):
MIN_GENESIS_ACTIVE_VALIDATOR_COUNT) return
if lastBlock.timestamp.uint64 >= MIN_GENESIS_TIME.uint64 and var candidatePos = eth1Chain.blocks.len - 1
m.eth1Chain.totalDeposits >= totalDepositsNeeded: while candidatePos > 1:
# This block is a genesis candidate if not isCandidateForGenesis(now, eth1Chain.blocks[candidatePos - 1]):
let startTime = lastBlock.timestamp.uint64 break
var s = initialize_beacon_state_from_eth1(lastBlock.voteData.block_hash, dec candidatePos
startTime, m.eth1Chain.allDeposits, {})
if is_valid_genesis_state(s[]): return some(candidatePos)
# https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
s.genesis_time = startTime proc createBeaconStateAux(eth1Block: Eth1Block,
deposits: var openarray[Deposit]): BeaconStateRef =
attachMerkleProofs deposits
result = initialize_beacon_state_from_eth1(eth1Block.voteData.block_hash,
eth1Block.timestamp.uint64,
deposits, {})
let activeValidators = get_active_validator_indices(result[], GENESIS_EPOCH)
eth1Block.knownGoodDepositsCount = some len(activeValidators).uint64
proc createBeaconState(eth1Chain: var Eth1Chain, eth1Block: Eth1Block): BeaconStateRef =
createBeaconStateAux(
eth1Block,
eth1Chain.allDeposits.toOpenArray(0, int(eth1Block.voteData.deposit_count - 1)))
proc signalGenesis(m: MainchainMonitor, genesisState: BeaconStateRef) =
m.genesisState = genesisState
m.genesisState = s
if not m.genesisStateFut.isNil: if not m.genesisStateFut.isNil:
m.genesisStateFut.complete() m.genesisStateFut.complete()
m.genesisStateFut = nil m.genesisStateFut = nil
proc processDeposits(m: MainchainMonitor, dataProvider: DataProviderRef) {. proc findGenesisBlockInRange(m: MainchainMonitor,
async startBlock, endBlock: Eth1Block): Future[Eth1Block]
# raises: [Defect] {.async.} =
.} = let dataProvider = await m.dataProviderFactory.new(m.depositContractAddress)
# ATTENTION! if dataProvider == nil:
# Please note that this code is using a queue to guarantee the error "Failed to initialize Eth1 data provider",
# strict serial order of processing of deposits. If we had the provider = m.dataProviderFactory.desc
# same code embedded in the deposit contracts events handler, raise newException(CatchableError, "Failed to initialize Eth1 data provider")
# it could easily re-order the steps due to the intruptable
# interleaved execution of async code.
while true:
let (blockHash, eventType) = await m.depositQueue.popFirst()
if eventType == RemovedEvent: var
m.eth1Chain.purgeChain(blockHash) startBlock = startBlock
continue endBlock = endBlock
depositData = startBlock.voteData
let cachedBlock = m.eth1Chain.findBlock(blockHash) while startBlock.number + 1 < endBlock.number:
if cachedBlock == nil:
try:
let let
web3Block = await dataProvider.getBlockByHash(blockHash) startBlockTime = genesis_time_from_eth1_timestamp(startBlock.timestamp)
eth1Block = await dataProvider.fetchDepositData(web3Block) secondsPerBlock = float(endBlock.timestamp - startBlock.timestamp) /
float(endBlock.number - startBlock.number)
blocksToJump = max(float(MIN_GENESIS_TIME - startBlockTime) / secondsPerBlock, 1.0)
candidateNumber = min(endBlock.number - 1, startBlock.number + blocksToJump.uint64)
candidateBlock = await dataProvider.getBlockByNumber(candidateNumber)
if m.eth1Chain.blocks.len > 0: var candidateAsEth1Block = Eth1Block(number: candidateBlock.number.uint64,
var cachedParent = m.eth1Chain.findParent(web3Block) timestamp: candidateBlock.timestamp.uint64,
if cachedParent == nil: voteData: depositData)
# We are missing the parent block. candidateAsEth1Block.voteData.block_hash = candidateBlock.hash.asEth2Digest
# This shouldn't be happening if the deposits events are reported in
# proper order, but nevertheless let's try to repair our chain:
var chainOfParents = newSeq[Eth1Block]()
var parentHash = web3Block.parentHash
var expectedParentBlockNumber = web3Block.number.uint64 - 1
warn "Eth1 parent block missing. Attempting to request from the network",
parentHash = parentHash.toHex
info "Probing possible genesis block",
`block` = candidateBlock.number.uint64,
timestamp = genesis_time_from_eth1_timestamp(candidateBlock.timestamp.uint64)
if genesis_time_from_eth1_timestamp(candidateBlock.timestamp.uint64) < MIN_GENESIS_TIME:
startBlock = candidateAsEth1Block
else:
endBlock = candidateAsEth1Block
return endBlock
proc checkForGenesisLoop(m: MainchainMonitor) {.async.} =
while true: while true:
if chainOfParents.len > reorgDepthLimit: if not m.genesisState.isNil:
error "Detected Eth1 re-org exceeded the maximum depth limit", return
headBlockHash = web3Block.hash.toHex,
ourHeadHash = m.eth1Chain.blocks.peekLast.voteData.block_hash
raise newException(ReorgDepthLimitExceeded, "Reorg depth limit exceeded")
let parentWeb3Block = await dataProvider.getBlockByHash(parentHash) try:
if parentWeb3Block.number.uint64 != expectedParentBlockNumber: let genesisCandidateIdx = m.eth1Chain.minGenesisCandidateBlockIdx
error "Eth1 data provider supplied invalid parent block", if genesisCandidateIdx.isSome:
parentBlockNumber = parentWeb3Block.number.uint64, let
expectedParentBlockNumber, parentHash = parentHash.toHex genesisCandidateIdx = genesisCandidateIdx.get
raise newException(CorruptDataProvider, genesisCandidate = m.eth1Chain.blocks[genesisCandidateIdx]
"Parent block with incorrect number") candidateState = m.eth1Chain.createBeaconState(genesisCandidate)
chainOfParents.add(await dataProvider.fetchDepositData(parentWeb3Block)) if genesisCandidate.knownGoodDepositsCount.get >= totalDepositsNeededForGenesis:
let localParent = m.eth1Chain.findParent(parentWeb3Block) # We have a candidate state on our hands, but our current Eth1Chain
if localParent != nil: # may consist only of blocks that have deposits attached to them
m.eth1Chain.purgeDescendants(localParent) # while the real genesis may have happened in a block without any
for i in countdown(chainOfParents.len - 1, 0): # deposits (triggered by MIN_GENESIS_TIME).
m.eth1Chain.addBlock chainOfParents[i] #
cachedParent = m.eth1Chain.blocks.peekLast # This can happen when the beacon node is launched after the genesis
break # event. We take a short cut when constructing the initial Eth1Chain
# by downloading only deposit log entries. Thus, we'll see all the
# blocks with deposits, but not the regular blocks in between.
#
# We'll handle this special case below by examing whether we are in
# this potential scenario and we'll use a fast guessing algorith to
# discover the ETh1 block with minimal valid genesis time.
if genesisCandidateIdx > 0:
let preceedingEth1Block = m.eth1Chain.blocks[genesisCandidateIdx - 1]
if preceedingEth1Block.voteData.deposit_root == genesisCandidate.voteData.deposit_root:
preceedingEth1Block.knownGoodDepositsCount = genesisCandidate.knownGoodDepositsCount
else:
discard m.eth1Chain.createBeaconState(preceedingEth1Block)
dec expectedParentBlockNumber if preceedingEth1Block.knownGoodDepositsCount.get >= totalDepositsNeededForGenesis and
parentHash = parentWeb3Block.parentHash genesisCandidate.number - preceedingEth1Block.number > 1:
let genesisBlock = await m.findGenesisBlockInRange(preceedingEth1Block, genesisCandidate)
if genesisBlock.number != genesisCandidate.number:
m.signalGenesis m.eth1Chain.createBeaconState(genesisBlock)
return
m.eth1Chain.purgeDescendants(cachedParent) m.signalGenesis candidateState
return
else:
info "Eth2 genesis candidate block rejected",
`block` = shortLog(genesisCandidate),
validDeposits = genesisCandidate.knownGoodDepositsCount.get,
needed = totalDepositsNeededForGenesis
else:
# TODO: check for a stale monitor
discard
except CatchableError as err:
debug "Unexpected error in checkForGenesisLoop", err = err.msg
m.eth1Chain.addBlock eth1Block await sleepAsync(1.seconds)
m.checkForGenesisEvent()
except CatchableError: proc waitGenesis*(m: MainchainMonitor): Future[BeaconStateRef] {.async.} =
# Connection problem? Put the unprocessed deposit back to queue.
# Raising the exception here will lead to a restart of the whole monitor.
m.depositQueue.addFirstNoWait((blockHash, eventType))
raise
proc isRunning*(m: MainchainMonitor): bool =
not m.runFut.isNil
proc getGenesis*(m: MainchainMonitor): Future[BeaconStateRef] {.async.} =
if m.genesisState.isNil: if m.genesisState.isNil:
if m.genesisStateFut.isNil: if m.genesisStateFut.isNil:
m.genesisStateFut = newFuture[void]("getGenesis") m.genesisStateFut = newFuture[void]("waitGenesis")
m.genesisMonitoringFut = m.checkForGenesisLoop()
await m.genesisStateFut await m.genesisStateFut
m.genesisStateFut = nil m.genesisStateFut = nil
@ -446,85 +625,105 @@ proc getGenesis*(m: MainchainMonitor): Future[BeaconStateRef] {.async.} =
result = new BeaconStateRef # make the compiler happy result = new BeaconStateRef # make the compiler happy
raiseAssert "Unreachable code" raiseAssert "Unreachable code"
method getBlockByHash*(p: Web3DataProviderRef, hash: BlockHash): Future[BlockObject] = func totalNonFinalizedBlocks(eth1Chain: Eth1Chain): Natural =
discard # TODO: implement this precisely
# p.web3.provider.eth_getBlockByHash(hash, false) eth1Chain.blocks.len
method close*(p: Web3DataProviderRef): Future[void] {.async, locks: 0.} = func latestEth1Data(eth1Chain: Eth1Chain): Eth1Data =
if p.subscription != nil: if eth1Chain.blocks.len > 0:
await p.subscription.unsubscribe() eth1Chain.blocks[^1].voteData
await p.web3.close() else:
eth1Chain.knownStart
method fetchDepositData*(p: Web3DataProviderRef, func knownInvalidDepositsCount(eth1Chain: Eth1Chain): uint64 =
web3Block: BlockObject): Future[Eth1Block] {.async, locks: 0.} = for i in countdown(eth1Chain.blocks.len - 1, 0):
let let blk = eth1Chain.blocks[i]
blockHash = web3Block.hash if blk.knownGoodDepositsCount.isSome:
depositRoot = await p.ns.get_deposit_root.call(blockNumber = web3Block.number.uint64) return blk.voteData.deposit_count - blk.knownGoodDepositsCount.get
rawCount = await p.ns.get_deposit_count.call(blockNumber = web3Block.number.uint64)
depositCount = bytes_to_int(array[8, byte](rawCount))
depositsJson = await p.ns.getJsonLogs(DepositEvent, blockHash = some(blockHash))
deposits = readJsonDeposits(depositsJson)
return Eth1Block( return 0
number: Eth1BlockNumber(web3Block.number),
timestamp: Eth1BlockTimestamp(web3Block.timestamp),
deposits: deposits,
voteData: Eth1Data(deposit_root: depositRoot.asEth2Digest,
deposit_count: depositCount,
block_hash: blockHash.asEth2Digest))
method onDisconnect*(p: Web3DataProviderRef, handler: DisconnectHandler) {. func maxValidDeposits(eth1Chain: Eth1Chain): uint64 =
gcsafe if eth1Chain.blocks.len > 0:
locks: 0 let lastBlock = eth1Chain.blocks[^1]
# raises: [] lastBlock.knownGoodDepositsCount.get(
.} = lastBlock.voteData.deposit_count - eth1Chain.knownInvalidDepositsCount)
p.web3.onDisconnect = handler else:
0
method onDepositEvent*(p: Web3DataProviderRef, proc processDeposits(m: MainchainMonitor,
startBlock: Eth1BlockNumber, dataProvider: DataProviderRef) {.async.} =
handler: DepositEventHandler): Future[void] {. # ATTENTION!
async # Please note that this code is using a queue to guarantee the
gcsafe # strict serial order of processing of deposits. If we had the
locks: 0 # same code embedded in the deposit contracts events handler,
# raises: [] # it could easily re-order the steps due to the intruptable
.} = # interleaved execution of async code.
if p.subscription != nil: while true:
await p.subscription.unsubscribe() let blk = await m.depositQueue.popFirst()
m.eth1Chain.trimHeight(Eth1BlockNumber(blk.number) - 1)
p.subscription = await p.ns.subscribe( let latestKnownBlock = if m.eth1Chain.blocks.len > 0:
DepositEvent, %*{"fromBlock": startBlock}, handler) m.eth1Chain.blocks[^1].number
elif m.eth1Chain.knownStartBlockNum.isSome:
m.eth1Chain.knownStartBlockNum.get
else:
m.eth1Chain.knownStartBlockNum = some(
await dataProvider.getBlockNumber(m.eth1Chain.knownStart.block_hash.asBlockHash))
m.eth1Chain.knownStartBlockNum.get
proc getBlockNumber(p: DataProviderRef, hash: BlockHash): Future[Quantity] {.async.} = let eth1Blocks = await dataProvider.fetchDepositData(latestKnownBlock + 1,
debug "Querying block number", hash = $hash Eth1BlockNumber blk.number)
if eth1Blocks.len == 0:
if m.eth1Chain.maxValidDeposits > totalDepositsNeededForGenesis and
m.eth1Chain.knownStart.deposit_count == 0:
let latestEth1Data = m.eth1Chain.latestEth1Data
try: for missingBlockNum in latestKnownBlock + 1 ..< Eth1BlockNumber(blk.number):
let blk = await p.getBlockByHash(hash) let missingBlock = await dataProvider.getBlockByNumber(missingBlockNum)
return blk.number doAssert m.eth1Chain.addSuccessorBlock Eth1Block(
except CatchableError as exc: number: Eth1BlockNumber(missingBlock.number),
notice "Failed to get Eth1 block number from hash", timestamp: Eth1BlockTimestamp(missingBlock.timestamp),
hash = $hash, err = exc.msg voteData: latestEth1Data)
raise
proc new*(T: type Web3DataProvider, doAssert m.eth1Chain.addSuccessorBlock Eth1Block(
web3Url: string, number: Eth1BlockNumber(blk.number),
depositContractAddress: Address): Future[ref Web3DataProvider] {. timestamp: Eth1BlockTimestamp(blk.timestamp),
async voteData: latestEth1Data)
# raises: [Defect] else:
.} = template logBlockProcessed(blk) =
try: info "Eth1 block processed",
type R = ref T `block` = shortLog(blk), totalDeposits = blk.voteData.deposit_count
let
web3 = await newWeb3(web3Url)
ns = web3.contractSender(DepositContract, depositContractAddress)
return R(url: web3Url, web3: web3, ns: ns)
except CatchableError:
return nil
func web3Provider*(web3Url: string): DataProviderFactory = await dataProvider.fetchBlockDetails(eth1Blocks[0])
proc factory(depositContractAddress: Address): Future[DataProviderRef] {.async.} = if m.eth1Chain.addSuccessorBlock(eth1Blocks[0]):
result = await Web3DataProvider.new(web3Url, depositContractAddress) logBlockProcessed eth1Blocks[0]
DataProviderFactory(desc: "web3(" & web3Url & ")", new: factory) for i in 1 ..< eth1Blocks.len:
await dataProvider.fetchBlockDetails(eth1Blocks[i])
if m.eth1Chain.addSuccessorBlock(eth1Blocks[i]):
logBlockProcessed eth1Blocks[i]
else:
raise newException(CorruptDataProvider,
"A non-successor Eth1 block reported")
else:
# A non-continuous chain detected.
# This could be the result of a deeper fork that was not reported
# properly by the web3 provider. Since this should be an extremely
# rare event we can afford to handle it in a relatively inefficient
# manner. Let's delete half of our non-finalized chain and try again.
let blocksToPop = max(1, m.eth1Chain.totalNonFinalizedBlocks div 2)
warn "Web3 provider responded with a non-continous chain of deposits.",
backtrackedDeposits = blocksToPop
for i in 0 ..< blocksToPop:
m.eth1Chain.popBlock()
m.depositQueue.addFirstNoWait blk
proc isRunning*(m: MainchainMonitor): bool =
not m.runFut.isNil
func `===`(json: JsonNode, boolean: bool): bool =
json.kind == JBool and json.bval == boolean
proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} = proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
if delayBeforeStart != ZeroDuration: if delayBeforeStart != ZeroDuration:
@ -535,44 +734,37 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
error "Failed to initialize Eth1 data provider", error "Failed to initialize Eth1 data provider",
provider = m.dataProviderFactory.desc provider = m.dataProviderFactory.desc
raise newException(CatchableError, "Failed to initialize Eth1 data provider") raise newException(CatchableError, "Failed to initialize Eth1 data provider")
defer: await close(dataProvider)
let processFut = m.processDeposits(dataProvider) try:
defer: await processFut info "Starting Eth1 deposit contract monitoring",
dataProvider.onDisconnect do:
error "Eth1 data provider disconnected",
provider = m.dataProviderFactory.desc
processFut.cancel()
let startBlkNum = await dataProvider.getBlockNumber(m.startBlock)
notice "Monitoring eth1 deposits",
fromBlock = startBlkNum.uint64,
contract = $m.depositContractAddress, contract = $m.depositContractAddress,
url = m.dataProviderFactory.desc url = m.dataProviderFactory.desc
await dataProvider.onDepositEvent(Eth1BlockNumber(startBlkNum)) do ( await dataProvider.onBlockHeaders do (blk: BlockHeader)
pubkey: Bytes48, {.raises: [Defect], gcsafe}:
withdrawalCredentials: Bytes32,
amount: Bytes8,
signature: Bytes96, merkleTreeIndex: Bytes8, j: JsonNode)
{.raises: [Defect], gcsafe.}:
try: try:
let m.depositQueue.addLastNoWait(blk)
blockHash = BlockHash.fromHex(j["blockHash"].getStr()) except AsyncQueueFullError:
eventType = if j.hasKey("removed"): RemovedEvent raiseAssert "The depositQueue has no size limit"
else: NewEvent except Exception:
# TODO Investigate why this exception is being raised
raiseAssert "queue.addLastNoWait should not raise exceptions"
do (err: CatchableError):
debug "Error while processing Eth1 block headers subscription", err = err.msg
m.depositQueue.addLastNoWait((blockHash, eventType)) await m.processDeposits(dataProvider)
except CatchableError as exc: finally:
warn "Received invalid deposit", err = exc.msg, j await close(dataProvider)
except Exception as err:
# chronos still raises exceptions which inherit directly from Exception proc safeCancel(fut: var Future[void]) =
if err[] of Defect: if not fut.isNil and not fut.finished:
raise (ref Defect)(err) fut.cancel()
else: fut = nil
warn "Received invalid deposit", err = err.msg, j
proc stop*(m: MainchainMonitor) =
safeCancel m.runFut
safeCancel m.genesisMonitoringFut
proc start(m: MainchainMonitor, delayBeforeStart: Duration) = proc start(m: MainchainMonitor, delayBeforeStart: Duration) =
if m.runFut.isNil: if m.runFut.isNil:
@ -583,7 +775,7 @@ proc start(m: MainchainMonitor, delayBeforeStart: Duration) =
if runFut.error[] of CatchableError: if runFut.error[] of CatchableError:
if runFut == m.runFut: if runFut == m.runFut:
error "Mainchain monitor failure, restarting", err = runFut.error.msg error "Mainchain monitor failure, restarting", err = runFut.error.msg
m.runFut = nil m.stop()
m.start(5.seconds) m.start(5.seconds)
else: else:
fatal "Fatal exception reached", err = runFut.error.msg fatal "Fatal exception reached", err = runFut.error.msg
@ -592,11 +784,6 @@ proc start(m: MainchainMonitor, delayBeforeStart: Duration) =
proc start*(m: MainchainMonitor) {.inline.} = proc start*(m: MainchainMonitor) {.inline.} =
m.start(0.seconds) m.start(0.seconds)
proc stop*(m: MainchainMonitor) =
if not m.runFut.isNil:
m.runFut.cancel()
m.runFut = nil
proc getLatestEth1BlockHash*(url: string): Future[Eth2Digest] {.async.} = proc getLatestEth1BlockHash*(url: string): Future[Eth2Digest] {.async.} =
let web3 = await newWeb3(url) let web3 = await newWeb3(url)
try: try:

2
beacon_chain/merkle_minimal.nim
View File

@ -83,7 +83,7 @@ proc getMerkleProof*[Depth: static int](tree: SparseMerkleTree[Depth],
else: else:
result[depth] = zeroHashes[depth] result[depth] = zeroHashes[depth]
proc attachMerkleProofs*(deposits: var seq[Deposit]) = func attachMerkleProofs*(deposits: var openarray[Deposit]) =
let deposit_data_roots = mapIt(deposits, it.data.hash_tree_root) let deposit_data_roots = mapIt(deposits, it.data.hash_tree_root)
var var
deposit_data_sums: seq[Eth2Digest] deposit_data_sums: seq[Eth2Digest]

172
beacon_chain/request_manager.nim
View File

@ -1,71 +1,133 @@
import import options, sequtils, strutils
options, random, import chronos, chronicles
chronos, chronicles, import spec/[datatypes, digest], eth2_network, beacon_node_types, sync_protocol,
spec/datatypes, sync_manager, ssz/merkleization
eth2_network, beacon_node_types, sync_protocol,
eth/async_utils logScope:
topics = "requman"
const
MAX_REQUEST_BLOCKS* = 4 # Specification's value is 1024.
## Maximum number of blocks, which can be requested by beaconBlocksByRoot.
PARALLEL_REQUESTS* = 2
## Number of peers we using to resolve our request.
type type
RequestManager* = object RequestManager* = object
network*: Eth2Node network*: Eth2Node
queue*: AsyncQueue[FetchRecord]
responseHandler*: FetchAncestorsResponseHandler
loopFuture: Future[void]
proc init*(T: type RequestManager, network: Eth2Node): T =
T(network: network)
type
FetchAncestorsResponseHandler = proc (b: SignedBeaconBlock) {.gcsafe.} FetchAncestorsResponseHandler = proc (b: SignedBeaconBlock) {.gcsafe.}
proc fetchAncestorBlocksFromPeer( func shortLog*(x: seq[Eth2Digest]): string =
peer: Peer, "[" & x.mapIt(shortLog(it)).join(", ") & "]"
rec: FetchRecord,
responseHandler: FetchAncestorsResponseHandler) {.async.} =
# TODO: It's not clear if this function follows the intention of the
# FetchRecord data type. Perhaps it is supposed to get a range of blocks
# instead. In order to do this, we'll need the slot number of the known
# block to be stored in the FetchRecord, so we can ask for a range of
# blocks starting N positions before this slot number.
try:
let blocks = await peer.beaconBlocksByRoot(BlockRootsList @[rec.root])
if blocks.isOk:
for b in blocks.get:
responseHandler(b)
except CatchableError as err:
debug "Error while fetching ancestor blocks",
err = err.msg, root = rec.root, peer = peer
proc fetchAncestorBlocksFromNetwork( func shortLog*(x: seq[FetchRecord]): string =
network: Eth2Node, "[" & x.mapIt(shortLog(it.root)).join(", ") & "]"
rec: FetchRecord,
responseHandler: FetchAncestorsResponseHandler) {.async.} = proc init*(T: type RequestManager, network: Eth2Node,
responseCb: FetchAncestorsResponseHandler): T =
T(
network: network, queue: newAsyncQueue[FetchRecord](),
responseHandler: responseCb
)
proc checkResponse(roots: openArray[Eth2Digest],
blocks: openArray[SignedBeaconBlock]): bool =
## This procedure checks peer's response.
var checks = @roots
if len(blocks) > len(roots):
return false
for blk in blocks:
let blockRoot = hash_tree_root(blk.message)
let res = checks.find(blockRoot)
if res == -1:
return false
else:
checks.del(res)
return true
proc fetchAncestorBlocksFromNetwork(rman: RequestManager,
items: seq[Eth2Digest]) {.async.} =
var peer: Peer var peer: Peer
try: try:
peer = await network.peerPool.acquire() peer = await rman.network.peerPool.acquire()
let blocks = await peer.beaconBlocksByRoot(BlockRootsList @[rec.root]) debug "Requesting blocks by root", peer = peer, blocks = shortLog(items),
peer_score = peer.getScore()
let blocks = await peer.beaconBlocksByRoot(BlockRootsList items)
if blocks.isOk: if blocks.isOk:
for b in blocks.get: let ublocks = blocks.get()
responseHandler(b) if checkResponse(items, ublocks):
except CatchableError as err: for b in ublocks:
debug "Error while fetching ancestor blocks", rman.responseHandler(b)
err = err.msg, root = rec.root, peer = peer peer.updateScore(PeerScoreGoodBlocks)
else:
peer.updateScore(PeerScoreBadResponse)
else:
peer.updateScore(PeerScoreNoBlocks)
except CancelledError as exc:
raise exc
except CatchableError as exc:
debug "Error while fetching ancestor blocks", exc = exc.msg,
items = shortLog(items), peer = peer, peer_score = peer.getScore()
raise exc
finally: finally:
if not(isNil(peer)): if not(isNil(peer)):
network.peerPool.release(peer) rman.network.peerPool.release(peer)
proc fetchAncestorBlocks*(requestManager: RequestManager, proc requestManagerLoop(rman: RequestManager) {.async.} =
roots: seq[FetchRecord], var rootList = newSeq[Eth2Digest]()
responseHandler: FetchAncestorsResponseHandler) = var workers = newSeq[Future[void]](PARALLEL_REQUESTS)
# TODO: we could have some fancier logic here: while true:
# try:
# * Keeps track of what was requested rootList.setLen(0)
# (this would give a little bit of time for the asked peer to respond) let req = await rman.queue.popFirst()
# rootList.add(req.root)
# * Keep track of the average latency of each peer
# (we can give priority to peers with better latency)
#
const ParallelRequests = 2
for i in 0 ..< ParallelRequests: var count = min(MAX_REQUEST_BLOCKS - 1, len(rman.queue))
traceAsyncErrors fetchAncestorBlocksFromNetwork(requestManager.network, while count > 0:
roots.sample(), rootList.add(rman.queue.popFirstNoWait().root)
responseHandler) dec(count)
let start = SyncMoment.now(Slot(0))
for i in 0 ..< PARALLEL_REQUESTS:
workers[i] = rman.fetchAncestorBlocksFromNetwork(rootList)
# We do not care about
await allFutures(workers)
let finish = SyncMoment.now(Slot(0) + uint64(len(rootList)))
var succeed = 0
for worker in workers:
if worker.finished() and not(worker.failed()):
inc(succeed)
debug "Request manager tick", blocks_count = len(rootList),
succeed = succeed,
failed = (len(workers) - succeed),
queue_size = len(rman.queue),
sync_speed = speed(start, finish)
except CatchableError as exc:
debug "Got a problem in request manager", exc = exc.msg
proc start*(rman: var RequestManager) =
## Start Request Manager's loop.
rman.loopFuture = requestManagerLoop(rman)
proc stop*(rman: RequestManager) =
## Stop Request Manager's loop.
if not(isNil(rman.loopFuture)):
rman.loopFuture.cancel()
proc fetchAncestorBlocks*(rman: RequestManager, roots: seq[FetchRecord]) =
## Enqueue list missing blocks roots ``roots`` for download by
## Request Manager ``rman``.
for item in roots:
rman.queue.addLastNoWait(item)

121
beacon_chain/spec/beaconstate.nim
View File

@ -14,7 +14,7 @@ import
./crypto, ./datatypes, ./digest, ./helpers, ./signatures, ./validator, ./crypto, ./datatypes, ./digest, ./helpers, ./signatures, ./validator,
../../nbench/bench_lab ../../nbench/bench_lab
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#is_valid_merkle_branch # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#is_valid_merkle_branch
func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openarray[Eth2Digest], depth: uint64, index: uint64, root: Eth2Digest): bool {.nbench.}= func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openarray[Eth2Digest], depth: uint64, index: uint64, root: Eth2Digest): bool {.nbench.}=
## Check if ``leaf`` at ``index`` verifies against the Merkle ``root`` and ## Check if ``leaf`` at ``index`` verifies against the Merkle ``root`` and
## ``branch``. ## ``branch``.
@ -32,37 +32,37 @@ func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openarray[Eth2Digest], de
value = eth2digest(buf) value = eth2digest(buf)
value == root value == root
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#increase_balance # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#increase_balance
func increase_balance*( func increase_balance*(
state: var BeaconState, index: ValidatorIndex, delta: Gwei) = state: var BeaconState, index: ValidatorIndex, delta: Gwei) =
# Increase the validator balance at index ``index`` by ``delta``. # Increase the validator balance at index ``index`` by ``delta``.
state.balances[index] += delta state.balances[index] += delta
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#decrease_balance # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#decrease_balance
func decrease_balance*( func decrease_balance*(
state: var BeaconState, index: ValidatorIndex, delta: Gwei) = state: var BeaconState, index: ValidatorIndex, delta: Gwei) =
## Decrease the validator balance at index ``index`` by ``delta``, with # Decrease the validator balance at index ``index`` by ``delta``, with
## underflow protection. # underflow protection.
state.balances[index] = state.balances[index] =
if delta > state.balances[index]: if delta > state.balances[index]:
0'u64 0'u64
else: else:
state.balances[index] - delta state.balances[index] - delta
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#deposits # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#deposits
proc process_deposit*( proc process_deposit*(
state: var BeaconState, deposit: Deposit, flags: UpdateFlags = {}): bool {.nbench.}= state: var BeaconState, deposit: Deposit, flags: UpdateFlags = {}): bool {.nbench.}=
# Process an Eth1 deposit, registering a validator or increasing its balance. # Process an Eth1 deposit, registering a validator or increasing its balance.
# Verify the Merkle branch # Verify the Merkle branch
if skipMerkleValidation notin flags and not is_valid_merkle_branch( if not is_valid_merkle_branch(
hash_tree_root(deposit.data), hash_tree_root(deposit.data),
deposit.proof, deposit.proof,
DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the `List` length mix-in DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the `List` length mix-in
state.eth1_deposit_index, state.eth1_deposit_index,
state.eth1_data.deposit_root, state.eth1_data.deposit_root,
): ):
notice "Deposit merkle validation failed", notice "Deposit Merkle validation failed",
proof = deposit.proof, deposit_root = state.eth1_data.deposit_root, proof = deposit.proof, deposit_root = state.eth1_data.deposit_root,
deposit_index = state.eth1_deposit_index deposit_index = state.eth1_deposit_index
return false return false
@ -83,7 +83,9 @@ proc process_deposit*(
if not verify_deposit_signature(deposit.data): if not verify_deposit_signature(deposit.data):
# It's ok that deposits fail - they get included in blocks regardless # It's ok that deposits fail - they get included in blocks regardless
# TODO spec test? # TODO spec test?
debug "Skipping deposit with invalid signature", # TODO: This is temporary set to trace level in order to deal with the
# large number of invalid deposits on Altona
trace "Skipping deposit with invalid signature",
deposit = shortLog(deposit.data) deposit = shortLog(deposit.data)
return true return true
@ -111,7 +113,7 @@ func compute_activation_exit_epoch(epoch: Epoch): Epoch =
## ``epoch`` take effect. ## ``epoch`` take effect.
epoch + 1 + MAX_SEED_LOOKAHEAD epoch + 1 + MAX_SEED_LOOKAHEAD
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_validator_churn_limit # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_validator_churn_limit
func get_validator_churn_limit(state: BeaconState, cache: var StateCache): func get_validator_churn_limit(state: BeaconState, cache: var StateCache):
uint64 = uint64 =
# Return the validator churn limit for the current epoch. # Return the validator churn limit for the current epoch.
@ -119,7 +121,7 @@ func get_validator_churn_limit(state: BeaconState, cache: var StateCache):
len(cache.shuffled_active_validator_indices) div len(cache.shuffled_active_validator_indices) div
CHURN_LIMIT_QUOTIENT).uint64 CHURN_LIMIT_QUOTIENT).uint64
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#initiate_validator_exit # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#initiate_validator_exit
func initiate_validator_exit*(state: var BeaconState, func initiate_validator_exit*(state: var BeaconState,
index: ValidatorIndex, cache: var StateCache) = index: ValidatorIndex, cache: var StateCache) =
# Initiate the exit of the validator with index ``index``. # Initiate the exit of the validator with index ``index``.
@ -148,7 +150,7 @@ func initiate_validator_exit*(state: var BeaconState,
validator.withdrawable_epoch = validator.withdrawable_epoch =
validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#slash_validator # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#slash_validator
proc slash_validator*(state: var BeaconState, slashed_index: ValidatorIndex, proc slash_validator*(state: var BeaconState, slashed_index: ValidatorIndex,
cache: var StateCache) = cache: var StateCache) =
# Slash the validator with index ``index``. # Slash the validator with index ``index``.
@ -192,7 +194,15 @@ proc slash_validator*(state: var BeaconState, slashed_index: ValidatorIndex,
increase_balance( increase_balance(
state, whistleblower_index, whistleblowing_reward - proposer_reward) state, whistleblower_index, whistleblowing_reward - proposer_reward)
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#genesis func genesis_time_from_eth1_timestamp*(eth1_timestamp: uint64): uint64 =
# TODO: remove once we switch completely to v0.12.1
when SPEC_VERSION == "0.12.1":
eth1_timestamp + GENESIS_DELAY
else:
const SECONDS_PER_DAY = uint64(60*60*24)
eth1_timestamp + 2'u64 * SECONDS_PER_DAY - (eth1_timestamp mod SECONDS_PER_DAY)
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#genesis
proc initialize_beacon_state_from_eth1*( proc initialize_beacon_state_from_eth1*(
eth1_block_hash: Eth2Digest, eth1_block_hash: Eth2Digest,
eth1_timestamp: uint64, eth1_timestamp: uint64,
@ -214,20 +224,20 @@ proc initialize_beacon_state_from_eth1*(
# at that point :) # at that point :)
doAssert deposits.len >= SLOTS_PER_EPOCH doAssert deposits.len >= SLOTS_PER_EPOCH
const SECONDS_PER_DAY = uint64(60*60*24)
var state = BeaconStateRef( var state = BeaconStateRef(
fork: Fork( fork: Fork(
previous_version: Version(GENESIS_FORK_VERSION), previous_version: Version(GENESIS_FORK_VERSION),
current_version: Version(GENESIS_FORK_VERSION), current_version: Version(GENESIS_FORK_VERSION),
epoch: GENESIS_EPOCH), epoch: GENESIS_EPOCH),
genesis_time: genesis_time: genesis_time_from_eth1_timestamp(eth1_timestamp),
eth1_timestamp + 2'u64 * SECONDS_PER_DAY -
(eth1_timestamp mod SECONDS_PER_DAY),
eth1_data: eth1_data:
Eth1Data(block_hash: eth1_block_hash, deposit_count: uint64(len(deposits))), Eth1Data(block_hash: eth1_block_hash, deposit_count: uint64(len(deposits))),
latest_block_header: latest_block_header:
BeaconBlockHeader( BeaconBlockHeader(
body_root: hash_tree_root(BeaconBlockBody( body_root: hash_tree_root(BeaconBlockBody(
# This differs from the spec intentionally.
# We must specify the default value for `ValidatorSig`
# in order to get a correct `hash_tree_root`.
randao_reveal: ValidatorSig(kind: OpaqueBlob) randao_reveal: ValidatorSig(kind: OpaqueBlob)
)) ))
) )
@ -273,11 +283,11 @@ proc initialize_hashed_beacon_state_from_eth1*(
eth1_block_hash, eth1_timestamp, deposits, flags) eth1_block_hash, eth1_timestamp, deposits, flags)
HashedBeaconState(data: genesisState[], root: hash_tree_root(genesisState[])) HashedBeaconState(data: genesisState[], root: hash_tree_root(genesisState[]))
func is_valid_genesis_state*(state: BeaconState): bool = func is_valid_genesis_state*(state: BeaconState, active_validator_indices: seq[ValidatorIndex]): bool =
if state.genesis_time < MIN_GENESIS_TIME: if state.genesis_time < MIN_GENESIS_TIME:
return false return false
# This is an okay get_active_validator_indices(...) for the time being. # This is an okay get_active_validator_indices(...) for the time being.
if len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: if len(active_validator_indices) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT:
return false return false
return true return true
@ -303,12 +313,12 @@ func get_block_root_at_slot*(state: BeaconState,
doAssert slot < state.slot doAssert slot < state.slot
state.block_roots[slot mod SLOTS_PER_HISTORICAL_ROOT] state.block_roots[slot mod SLOTS_PER_HISTORICAL_ROOT]
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_block_root # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_block_root
func get_block_root*(state: BeaconState, epoch: Epoch): Eth2Digest = func get_block_root*(state: BeaconState, epoch: Epoch): Eth2Digest =
# Return the block root at the start of a recent ``epoch``. # Return the block root at the start of a recent ``epoch``.
get_block_root_at_slot(state, compute_start_slot_at_epoch(epoch)) get_block_root_at_slot(state, compute_start_slot_at_epoch(epoch))
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_total_balance # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_total_balance
func get_total_balance*(state: BeaconState, validators: auto): Gwei = func get_total_balance*(state: BeaconState, validators: auto): Gwei =
## Return the combined effective balance of the ``indices``. ## Return the combined effective balance of the ``indices``.
## ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. ## ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero.
@ -317,15 +327,13 @@ func get_total_balance*(state: BeaconState, validators: auto): Gwei =
foldl(validators, a + state.validators[b].effective_balance, 0'u64) foldl(validators, a + state.validators[b].effective_balance, 0'u64)
) )
# XXX: Move to state_transition_epoch.nim? # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#is_eligible_for_activation_queue
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#is_eligible_for_activation_queue
func is_eligible_for_activation_queue(validator: Validator): bool = func is_eligible_for_activation_queue(validator: Validator): bool =
# Check if ``validator`` is eligible to be placed into the activation queue. # Check if ``validator`` is eligible to be placed into the activation queue.
validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and
validator.effective_balance == MAX_EFFECTIVE_BALANCE validator.effective_balance == MAX_EFFECTIVE_BALANCE
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#is_eligible_for_activation # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#is_eligible_for_activation
func is_eligible_for_activation(state: BeaconState, validator: Validator): func is_eligible_for_activation(state: BeaconState, validator: Validator):
bool = bool =
# Check if ``validator`` is eligible for activation. # Check if ``validator`` is eligible for activation.
@ -394,21 +402,31 @@ proc process_registry_updates*(state: var BeaconState,
validator.activation_epoch = validator.activation_epoch =
compute_activation_exit_epoch(get_current_epoch(state)) compute_activation_exit_epoch(get_current_epoch(state))
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#is_valid_indexed_attestation # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#is_valid_indexed_attestation
proc is_valid_indexed_attestation*( func is_valid_indexed_attestation*(
state: BeaconState, indexed_attestation: IndexedAttestation, state: BeaconState, indexed_attestation: SomeIndexedAttestation,
flags: UpdateFlags): bool = flags: UpdateFlags): bool =
# Check if ``indexed_attestation`` has sorted and unique indices and a valid # Check if ``indexed_attestation`` is not empty, has sorted and unique
# aggregate signature. # indices and has a valid aggregate signature.
# TODO: this is noSideEffect besides logging
# https://github.com/status-im/nim-chronicles/issues/62 template is_sorted_and_unique(s: untyped): bool =
for i in 1 ..< s.len:
if s[i - 1].uint64 >= s[i].uint64:
return false
true
# Not from spec, but this function gets used in front-line roles, not just
# behind firewall.
let num_validators = state.validators.len.uint64
if anyIt(indexed_attestation.attesting_indices, it >= num_validators):
trace "indexed attestation: not all indices valid validators"
return false
# Verify indices are sorted and unique # Verify indices are sorted and unique
# TODO: A simple loop can verify that the indicates are monotonically let indices = indexed_attestation.attesting_indices.asSeq
# increasing and non-repeating here! if len(indices) == 0 or not is_sorted_and_unique(indices):
let indices = indexed_attestation.attesting_indices trace "indexed attestation: indices not sorted and unique"
if indices.asSeq != sorted(toHashSet(indices.asSeq).toSeq, system.cmp):
notice "indexed attestation: indices not sorted"
return false return false
# Verify aggregate signature # Verify aggregate signature
@ -418,7 +436,7 @@ proc is_valid_indexed_attestation*(
if not verify_attestation_signature( if not verify_attestation_signature(
state.fork, state.genesis_validators_root, indexed_attestation.data, state.fork, state.genesis_validators_root, indexed_attestation.data,
pubkeys, indexed_attestation.signature): pubkeys, indexed_attestation.signature):
notice "indexed attestation: signature verification failure" trace "indexed attestation: signature verification failure"
return false return false
true true
@ -449,7 +467,7 @@ func get_attesting_indices*(state: BeaconState,
if bits[i]: if bits[i]:
result.incl index result.incl index
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_indexed_attestation # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_indexed_attestation
func get_indexed_attestation*(state: BeaconState, attestation: Attestation, func get_indexed_attestation*(state: BeaconState, attestation: Attestation,
stateCache: var StateCache): IndexedAttestation = stateCache: var StateCache): IndexedAttestation =
# Return the indexed attestation corresponding to ``attestation``. # Return the indexed attestation corresponding to ``attestation``.
@ -466,6 +484,22 @@ func get_indexed_attestation*(state: BeaconState, attestation: Attestation,
signature: attestation.signature signature: attestation.signature
) )
func get_indexed_attestation*(state: BeaconState, attestation: TrustedAttestation,
stateCache: var StateCache): TrustedIndexedAttestation =
# Return the indexed attestation corresponding to ``attestation``.
let
attesting_indices =
get_attesting_indices(
state, attestation.data, attestation.aggregation_bits, stateCache)
TrustedIndexedAttestation(
attesting_indices:
List[uint64, MAX_VALIDATORS_PER_COMMITTEE].init(
sorted(mapIt(attesting_indices.toSeq, it.uint64), system.cmp)),
data: attestation.data,
signature: attestation.signature
)
# Attestation validation # Attestation validation
# ------------------------------------------------------------------------------------------ # ------------------------------------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#attestations # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#attestations
@ -496,7 +530,7 @@ proc isValidAttestationSlot*(attestationSlot, stateSlot: Slot): bool =
# TODO remove/merge with p2p-interface validation # TODO remove/merge with p2p-interface validation
proc isValidAttestationTargetEpoch*( proc isValidAttestationTargetEpoch*(
state: BeaconState, attestation: Attestation): bool = state: BeaconState, data: AttestationData): bool =
# TODO what constitutes a valid attestation when it's about to be added to # TODO what constitutes a valid attestation when it's about to be added to
# the pool? we're interested in attestations that will become viable # the pool? we're interested in attestations that will become viable
# for inclusion in blocks in the future and on any fork, so we need to # for inclusion in blocks in the future and on any fork, so we need to
@ -509,7 +543,6 @@ proc isValidAttestationTargetEpoch*(
# include an attestation in a block even if the corresponding validator # include an attestation in a block even if the corresponding validator
# was slashed in the same epoch - there's no penalty for doing this and # was slashed in the same epoch - there's no penalty for doing this and
# the vote counting logic will take care of any ill effects (TODO verify) # the vote counting logic will take care of any ill effects (TODO verify)
let data = attestation.data
# TODO re-enable check # TODO re-enable check
#if not (data.crosslink.shard < SHARD_COUNT): #if not (data.crosslink.shard < SHARD_COUNT):
# notice "Attestation shard too high", # notice "Attestation shard too high",
@ -539,7 +572,7 @@ proc isValidAttestationTargetEpoch*(
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.2/specs/phase0/beacon-chain.md#attestations # https://github.com/ethereum/eth2.0-specs/blob/v0.11.2/specs/phase0/beacon-chain.md#attestations
proc check_attestation*( proc check_attestation*(
state: BeaconState, attestation: Attestation, flags: UpdateFlags, state: BeaconState, attestation: SomeAttestation, flags: UpdateFlags,
stateCache: var StateCache): bool = stateCache: var StateCache): bool =
## Check that an attestation follows the rules of being included in the state ## Check that an attestation follows the rules of being included in the state
## at the current slot. When acting as a proposer, the same rules need to ## at the current slot. When acting as a proposer, the same rules need to
@ -564,7 +597,7 @@ proc check_attestation*(
committee_count = get_committee_count_at_slot(state, data.slot)) committee_count = get_committee_count_at_slot(state, data.slot))
return return
if not isValidAttestationTargetEpoch(state, attestation): if not isValidAttestationTargetEpoch(state, data):
# Logging in isValidAttestationTargetEpoch # Logging in isValidAttestationTargetEpoch
return return
@ -602,7 +635,7 @@ proc check_attestation*(
true true
proc process_attestation*( proc process_attestation*(
state: var BeaconState, attestation: Attestation, flags: UpdateFlags, state: var BeaconState, attestation: SomeAttestation, flags: UpdateFlags,
stateCache: var StateCache): bool {.nbench.}= stateCache: var StateCache): bool {.nbench.}=
# In the spec, attestation validation is mixed with state mutation, so here # In the spec, attestation validation is mixed with state mutation, so here
# we've split it into two functions so that the validation logic can be # we've split it into two functions so that the validation logic can be

48
beacon_chain/spec/crypto.nim
View File

@ -71,6 +71,11 @@ type
RandomSourceDepleted* = object of CatchableError RandomSourceDepleted* = object of CatchableError
TrustedSig* = object
data*: array[RawSigSize, byte]
SomeSig* = TrustedSig | ValidatorSig
func `==`*(a, b: BlsValue): bool = func `==`*(a, b: BlsValue): bool =
if a.kind != b.kind: return false if a.kind != b.kind: return false
if a.kind == Real: if a.kind == Real:
@ -86,7 +91,7 @@ template `==`*[N, T](a: T, b: BlsValue[N, T]): bool =
# API # API
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#bls-signatures # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#bls-signatures
func toPubKey*(privkey: ValidatorPrivKey): ValidatorPubKey = func toPubKey*(privkey: ValidatorPrivKey): ValidatorPubKey =
## Create a private key from a public key ## Create a private key from a public key
@ -218,6 +223,9 @@ func toRaw*(x: BlsValue): auto =
else: else:
x.blob x.blob
func toRaw*(x: TrustedSig): auto =
x.data
func toHex*(x: BlsCurveType): string = func toHex*(x: BlsCurveType): string =
toHex(toRaw(x)) toHex(toRaw(x))
@ -260,30 +268,47 @@ template hash*(x: BlsCurveType): Hash =
# Serialization # Serialization
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
{.pragma: serializationRaises, raises: [SerializationError, IOError, Defect].}
proc writeValue*(writer: var JsonWriter, value: ValidatorPubKey) {. proc writeValue*(writer: var JsonWriter, value: ValidatorPubKey) {.
inline, raises: [IOError, Defect].} = inline, raises: [IOError, Defect].} =
writer.writeValue(value.toHex()) writer.writeValue(value.toHex())
proc readValue*(reader: var JsonReader, value: var ValidatorPubKey) {. proc readValue*(reader: var JsonReader, value: var ValidatorPubKey)
inline, raises: [Exception].} = {.serializationRaises.} =
value = ValidatorPubKey.fromHex(reader.readValue(string)).tryGet() let key = ValidatorPubKey.fromHex(reader.readValue(string))
if key.isOk:
value = key.get
else:
# TODO: Can we provide better diagnostic?
raiseUnexpectedValue(reader, "Valid hex-encoded public key expected")
proc writeValue*(writer: var JsonWriter, value: ValidatorSig) {. proc writeValue*(writer: var JsonWriter, value: ValidatorSig) {.
inline, raises: [IOError, Defect].} = inline, raises: [IOError, Defect].} =
# Workaround: https://github.com/status-im/nim-beacon-chain/issues/374 # Workaround: https://github.com/status-im/nim-beacon-chain/issues/374
writer.writeValue(value.toHex()) writer.writeValue(value.toHex())
proc readValue*(reader: var JsonReader, value: var ValidatorSig) {. proc readValue*(reader: var JsonReader, value: var ValidatorSig)
inline, raises: [Exception].} = {.serializationRaises.} =
value = ValidatorSig.fromHex(reader.readValue(string)).tryGet() let sig = ValidatorSig.fromHex(reader.readValue(string))
if sig.isOk:
value = sig.get
else:
# TODO: Can we provide better diagnostic?
raiseUnexpectedValue(reader, "Valid hex-encoded signature expected")
proc writeValue*(writer: var JsonWriter, value: ValidatorPrivKey) {. proc writeValue*(writer: var JsonWriter, value: ValidatorPrivKey) {.
inline, raises: [IOError, Defect].} = inline, raises: [IOError, Defect].} =
writer.writeValue(value.toHex()) writer.writeValue(value.toHex())
proc readValue*(reader: var JsonReader, value: var ValidatorPrivKey) {. proc readValue*(reader: var JsonReader, value: var ValidatorPrivKey)
inline, raises: [Exception].} = {.serializationRaises.} =
value = ValidatorPrivKey.fromHex(reader.readValue(string)).tryGet() let key = ValidatorPrivKey.fromHex(reader.readValue(string))
if key.isOk:
value = key.get
else:
# TODO: Can we provide better diagnostic?
raiseUnexpectedValue(reader, "Valid hex-encoded private key expected")
template fromSszBytes*(T: type BlsValue, bytes: openArray[byte]): auto = template fromSszBytes*(T: type BlsValue, bytes: openArray[byte]): auto =
let v = fromRaw(T, bytes) let v = fromRaw(T, bytes)
@ -308,6 +333,9 @@ func shortLog*(x: ValidatorPrivKey): string =
## Logging for raw unwrapped BLS types ## Logging for raw unwrapped BLS types
x.toRaw()[0..3].toHex() x.toRaw()[0..3].toHex()
func shortLog*(x: TrustedSig): string =
x.data[0..3].toHex()
# Initialization # Initialization
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------

158
beacon_chain/spec/datatypes.nim
View File

@ -46,27 +46,15 @@ export
# Eventually, we could also differentiate between user/tainted data and # Eventually, we could also differentiate between user/tainted data and
# internal state that's gone through sanity checks already. # internal state that's gone through sanity checks already.
const ETH2_SPEC* {.strdefine.} = "v0.11.3"
static: doAssert: ETH2_SPEC == "v0.11.3" or ETH2_SPEC == "v0.12.1"
# Constant presets # Constant presets
const const_preset* {.strdefine.} = "mainnet" const const_preset* {.strdefine.} = "mainnet"
when const_preset == "mainnet": when const_preset == "mainnet":
when ETH2_SPEC == "v0.12.1": import ./presets/v0_12_1/mainnet
import ./presets/mainnet
export mainnet export mainnet
else:
import ./presets/mainnet_v0_11_3
export mainnet_v0_11_3
elif const_preset == "minimal": elif const_preset == "minimal":
when ETH2_SPEC == "v0.12.1": import ./presets/v0_12_1/minimal
import ./presets/minimal
export minimal export minimal
else:
import ./presets/minimal_v0_11_3
export minimal_v0_11_3
else: else:
type type
Slot* = distinct uint64 Slot* = distinct uint64
@ -76,11 +64,7 @@ else:
loadCustomPreset const_preset loadCustomPreset const_preset
const const
SPEC_VERSION* = SPEC_VERSION* = "0.12.1" ## \
when ETH2_SPEC == "v0.12.1":
"0.12.1"
else:
"0.11.3" ## \
## Spec version we're aiming to be compatible with, right now ## Spec version we're aiming to be compatible with, right now
GENESIS_SLOT* = Slot(0) GENESIS_SLOT* = Slot(0)
@ -129,12 +113,12 @@ type
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase1/custody-game.md#signature-domain-types # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase1/custody-game.md#signature-domain-types
DOMAIN_CUSTODY_BIT_SLASHING = 0x83 DOMAIN_CUSTODY_BIT_SLASHING = 0x83
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#custom-types # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#custom-types
Domain* = array[32, byte] Domain* = array[32, byte]
# https://github.com/nim-lang/Nim/issues/574 and be consistent across # https://github.com/nim-lang/Nim/issues/574 and be consistent across
# 32-bit and 64-bit word platforms. # 32-bit and 64-bit word platforms.
# TODO VALIDATOR_REGISTRY_LIMIT is 1 shl 40 in 0.8.3, and # TODO VALIDATOR_REGISTRY_LIMIT is 1 shl 40 in 0.12.1, and
# proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} # proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.}
# in Nim/lib/system/gc.nim quite tightly ties seq addressibility # in Nim/lib/system/gc.nim quite tightly ties seq addressibility
# to the system wordsize. This lifts smaller, and now incorrect, # to the system wordsize. This lifts smaller, and now incorrect,
@ -160,6 +144,12 @@ type
data*: AttestationData data*: AttestationData
signature*: ValidatorSig signature*: ValidatorSig
TrustedIndexedAttestation* = object
# TODO ValidatorIndex, but that doesn't serialize properly
attesting_indices*: List[uint64, MAX_VALIDATORS_PER_COMMITTEE]
data*: AttestationData
signature*: TrustedSig
CommitteeValidatorsBits* = BitList[MAX_VALIDATORS_PER_COMMITTEE] CommitteeValidatorsBits* = BitList[MAX_VALIDATORS_PER_COMMITTEE]
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#attestation # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#attestation
@ -168,10 +158,15 @@ type
data*: AttestationData data*: AttestationData
signature*: ValidatorSig signature*: ValidatorSig
TrustedAttestation* = object
aggregation_bits*: CommitteeValidatorsBits
data*: AttestationData
signature*: TrustedSig
Version* = distinct array[4, byte] Version* = distinct array[4, byte]
ForkDigest* = distinct array[4, byte] ForkDigest* = distinct array[4, byte]
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#forkdata # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#forkdata
ForkData* = object ForkData* = object
current_version*: Version current_version*: Version
genesis_validators_root*: Eth2Digest genesis_validators_root*: Eth2Digest
@ -203,7 +198,7 @@ type
data*: DepositData data*: DepositData
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#depositmessage # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#depositmessage
DepositMessage* = object DepositMessage* = object
pubkey*: ValidatorPubKey pubkey*: ValidatorPubKey
withdrawal_credentials*: Eth2Digest withdrawal_credentials*: Eth2Digest
@ -214,6 +209,8 @@ type
pubkey*: ValidatorPubKey pubkey*: ValidatorPubKey
withdrawal_credentials*: Eth2Digest withdrawal_credentials*: Eth2Digest
amount*: Gwei amount*: Gwei
# Cannot use TrustedSig here as invalid signatures are possible and determine
# if the deposit should be added or not during processing
signature*: ValidatorSig # Signing over DepositMessage signature*: ValidatorSig # Signing over DepositMessage
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#voluntaryexit # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#voluntaryexit
@ -242,6 +239,29 @@ type
body*: BeaconBlockBody body*: BeaconBlockBody
TrustedBeaconBlock* = object
## When we receive blocks from outside sources, they are untrusted and go
## through several layers of validation. Blocks that have gone through
## validations can be trusted to be well-formed, with a correct signature,
## having a parent and applying cleanly to the state that their parent
## left them with.
##
## When loading such blocks from the database, to rewind states for example,
## it is expensive to redo the validations (in particular, the signature
## checks), thus `TrustedBlock` uses a `TrustedSig` type to mark that these
## checks can be skipped.
##
## TODO this could probably be solved with some type trickery, but there
## too many bugs in nim around generics handling, and we've used up
## the trickery budget in the serialization library already. Until
## then, the type must be manually kept compatible with its untrusted
## cousin.
slot*: Slot
proposer_index*: uint64
parent_root*: Eth2Digest ##\
state_root*: Eth2Digest ##\
body*: TrustedBeaconBlockBody
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#beaconblockheader # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#beaconblockheader
BeaconBlockHeader* = object BeaconBlockHeader* = object
slot*: Slot slot*: Slot
@ -250,6 +270,11 @@ type
state_root*: Eth2Digest state_root*: Eth2Digest
body_root*: Eth2Digest body_root*: Eth2Digest
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#signingdata
SigningData* = object
object_root*: Eth2Digest
domain*: Domain
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#beaconblockbody # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#beaconblockbody
BeaconBlockBody* = object BeaconBlockBody* = object
randao_reveal*: ValidatorSig randao_reveal*: ValidatorSig
@ -263,8 +288,26 @@ type
deposits*: List[Deposit, MAX_DEPOSITS] deposits*: List[Deposit, MAX_DEPOSITS]
voluntary_exits*: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] voluntary_exits*: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS]
TrustedBeaconBlockBody* = object
randao_reveal*: TrustedSig
eth1_data*: Eth1Data
graffiti*: Eth2Digest # TODO make that raw bytes
# Operations
proposer_slashings*: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS]
attester_slashings*: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS]
attestations*: List[TrustedAttestation, MAX_ATTESTATIONS]
deposits*: List[Deposit, MAX_DEPOSITS]
voluntary_exits*: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS]
SomeSignedBeaconBlock* = SignedBeaconBlock | TrustedSignedBeaconBlock
SomeBeaconBlock* = BeaconBlock | TrustedBeaconBlock
SomeBeaconBlockBody* = BeaconBlockBody | TrustedBeaconBlockBody
SomeAttestation* = Attestation | TrustedAttestation
SomeIndexedAttestation* = IndexedAttestation | TrustedIndexedAttestation
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#beaconstate # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#beaconstate
BeaconStateObj* = object BeaconState* = object
# Versioning # Versioning
genesis_time*: uint64 genesis_time*: uint64
genesis_validators_root*: Eth2Digest genesis_validators_root*: Eth2Digest
@ -316,11 +359,10 @@ type
current_justified_checkpoint*: Checkpoint current_justified_checkpoint*: Checkpoint
finalized_checkpoint*: Checkpoint finalized_checkpoint*: Checkpoint
BeaconState* = BeaconStateObj BeaconStateRef* = ref BeaconState not nil
BeaconStateRef* = ref BeaconStateObj not nil NilableBeaconStateRef* = ref BeaconState
NilableBeaconStateRef* = ref BeaconStateObj
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#validator # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#validator
Validator* = object Validator* = object
pubkey*: ValidatorPubKey pubkey*: ValidatorPubKey
@ -373,6 +415,13 @@ type
deposit_count*: uint64 deposit_count*: uint64
block_hash*: Eth2Digest block_hash*: Eth2Digest
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#eth1block
Eth1Block* = object
timestamp*: uint64
deposit_root*: Eth2Digest
deposit_count*: uint64
# All other eth1 block fields
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#signedvoluntaryexit # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#signedvoluntaryexit
SignedVoluntaryExit* = object SignedVoluntaryExit* = object
message*: VoluntaryExit message*: VoluntaryExit
@ -383,6 +432,10 @@ type
message*: BeaconBlock message*: BeaconBlock
signature*: ValidatorSig signature*: ValidatorSig
TrustedSignedBeaconBlock* = object
message*: TrustedBeaconBlock
signature*: TrustedSig
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#signedbeaconblockheader # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#signedbeaconblockheader
SignedBeaconBlockHeader* = object SignedBeaconBlockHeader* = object
message*: BeaconBlockHeader message*: BeaconBlockHeader
@ -410,31 +463,6 @@ type
committee_count_cache*: Table[Epoch, uint64] committee_count_cache*: Table[Epoch, uint64]
beacon_proposer_indices*: Table[Slot, Option[ValidatorIndex]] beacon_proposer_indices*: Table[Slot, Option[ValidatorIndex]]
JsonError = jsonTypes.JsonError
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#signingdata
# TODO move back into big `type` block
when ETH2_SPEC == "v0.12.1":
type SigningData* = object
object_root*: Eth2Digest
domain*: Domain
else:
type SigningRoot* = object
object_root*: Eth2Digest
domain*: Domain
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#eth1block
when ETH2_SPEC == "v0.12.1":
type Eth1Block* = object
timestamp*: uint64
deposit_root*: Eth2Digest
deposit_count*: uint64
# All other eth1 block fields
else:
type Eth1Block* = object
timestamp*: uint64
# All other eth1 block fields
func shortValidatorKey*(state: BeaconState, validatorIdx: int): string = func shortValidatorKey*(state: BeaconState, validatorIdx: int): string =
($state.validators[validatorIdx].pubkey)[0..7] ($state.validators[validatorIdx].pubkey)[0..7]
@ -487,7 +515,7 @@ template ethTimeUnit(typ: type) {.dirty.} =
writeValue(writer, uint64 value) writeValue(writer, uint64 value)
proc readValue*(reader: var JsonReader, value: var typ) proc readValue*(reader: var JsonReader, value: var typ)
{.raises: [IOError, JsonError, Defect].} = {.raises: [IOError, SerializationError, Defect].} =
value = typ reader.readValue(uint64) value = typ reader.readValue(uint64)
proc writeValue*(writer: var JsonWriter, value: ValidatorIndex) proc writeValue*(writer: var JsonWriter, value: ValidatorIndex)
@ -495,14 +523,14 @@ proc writeValue*(writer: var JsonWriter, value: ValidatorIndex)
writeValue(writer, uint32 value) writeValue(writer, uint32 value)
proc readValue*(reader: var JsonReader, value: var ValidatorIndex) proc readValue*(reader: var JsonReader, value: var ValidatorIndex)
{.raises: [IOError, JsonError, Defect].} = {.raises: [IOError, SerializationError, Defect].} =
value = ValidatorIndex reader.readValue(uint32) value = ValidatorIndex reader.readValue(uint32)
template writeValue*(writer: var JsonWriter, value: Version | ForkDigest) = template writeValue*(writer: var JsonWriter, value: Version | ForkDigest) =
writeValue(writer, $value) writeValue(writer, $value)
proc readValue*(reader: var JsonReader, value: var Version) proc readValue*(reader: var JsonReader, value: var Version)
{.raises: [IOError, JsonError, Defect].} = {.raises: [IOError, SerializationError, Defect].} =
let hex = reader.readValue(string) let hex = reader.readValue(string)
try: try:
hexToByteArray(hex, array[4, byte](value)) hexToByteArray(hex, array[4, byte](value))
@ -510,7 +538,7 @@ proc readValue*(reader: var JsonReader, value: var Version)
raiseUnexpectedValue(reader, "Hex string of 4 bytes expected") raiseUnexpectedValue(reader, "Hex string of 4 bytes expected")
proc readValue*(reader: var JsonReader, value: var ForkDigest) proc readValue*(reader: var JsonReader, value: var ForkDigest)
{.raises: [IOError, JsonError, Defect].} = {.raises: [IOError, SerializationError, Defect].} =
let hex = reader.readValue(string) let hex = reader.readValue(string)
try: try:
hexToByteArray(hex, array[4, byte](value)) hexToByteArray(hex, array[4, byte](value))
@ -518,23 +546,23 @@ proc readValue*(reader: var JsonReader, value: var ForkDigest)
raiseUnexpectedValue(reader, "Hex string of 4 bytes expected") raiseUnexpectedValue(reader, "Hex string of 4 bytes expected")
# `ValidatorIndex` seq handling. # `ValidatorIndex` seq handling.
proc max*(a: ValidatorIndex, b: int) : auto = func max*(a: ValidatorIndex, b: int) : auto =
max(a.int, b) max(a.int, b)
proc `[]`*[T](a: var seq[T], b: ValidatorIndex): var T = func `[]`*[T](a: var seq[T], b: ValidatorIndex): var T =
a[b.int] a[b.int]
proc `[]`*[T](a: seq[T], b: ValidatorIndex): auto = func `[]`*[T](a: seq[T], b: ValidatorIndex): auto =
a[b.int] a[b.int]
proc `[]=`*[T](a: var seq[T], b: ValidatorIndex, c: T) = func `[]=`*[T](a: var seq[T], b: ValidatorIndex, c: T) =
a[b.int] = c a[b.int] = c
# `ValidatorIndex` Nim integration # `ValidatorIndex` Nim integration
proc `==`*(x, y: ValidatorIndex) : bool {.borrow.} proc `==`*(x, y: ValidatorIndex) : bool {.borrow.}
proc `<`*(x, y: ValidatorIndex) : bool {.borrow.} proc `<`*(x, y: ValidatorIndex) : bool {.borrow.}
proc hash*(x: ValidatorIndex): Hash {.borrow.} proc hash*(x: ValidatorIndex): Hash {.borrow.}
proc `$`*(x: ValidatorIndex): auto = $(x.int64) func `$`*(x: ValidatorIndex): auto = $(x.int64)
ethTimeUnit Slot ethTimeUnit Slot
ethTimeUnit Epoch ethTimeUnit Epoch
@ -603,7 +631,7 @@ func shortLog*(s: Slot): uint64 =
func shortLog*(e: Epoch): uint64 = func shortLog*(e: Epoch): uint64 =
e - GENESIS_EPOCH e - GENESIS_EPOCH
func shortLog*(v: BeaconBlock): auto = func shortLog*(v: SomeBeaconBlock): auto =
( (
slot: shortLog(v.slot), slot: shortLog(v.slot),
proposer_index: v.proposer_index, proposer_index: v.proposer_index,
@ -616,7 +644,7 @@ func shortLog*(v: BeaconBlock): auto =
voluntary_exits_len: v.body.voluntary_exits.len(), voluntary_exits_len: v.body.voluntary_exits.len(),
) )
func shortLog*(v: SignedBeaconBlock): auto = func shortLog*(v: SomeSignedBeaconBlock): auto =
( (
blck: shortLog(v.message), blck: shortLog(v.message),
signature: shortLog(v.signature) signature: shortLog(v.signature)
@ -641,7 +669,7 @@ func shortLog*(v: AttestationData): auto =
target_root: shortLog(v.target.root) target_root: shortLog(v.target.root)
) )
func shortLog*(v: Attestation): auto = func shortLog*(v: SomeAttestation): auto =
( (
aggregation_bits: v.aggregation_bits, aggregation_bits: v.aggregation_bits,
data: shortLog(v.data), data: shortLog(v.data),

16
beacon_chain/spec/eth2_apis/beacon_callsigs.nim
View File

@ -1,6 +1,20 @@
import import
options, options,
../datatypes ../[datatypes, digest, crypto],
json_rpc/jsonmarshal,
callsigs_types
proc get_v1_beacon_genesis(): BeaconGenesisTuple
# TODO stateId is part of the REST path
proc get_v1_beacon_states_root(stateId: string): Eth2Digest
# TODO stateId is part of the REST path
proc get_v1_beacon_states_fork(stateId: string): Fork
# TODO: delete old stuff
# https://github.com/ethereum/eth2.0-APIs/blob/master/apis/beacon/basic.md # https://github.com/ethereum/eth2.0-APIs/blob/master/apis/beacon/basic.md
# #

23
beacon_chain/spec/eth2_apis/callsigs_types.nim Normal file
View File

@ -0,0 +1,23 @@
import
# Standard library
options,
# Local modules
# TODO for some reason "../[datatypes, digest, crypto]" results in "Error: cannot open file"
../datatypes,
../digest,
../crypto
type
AttesterDuties* = tuple
public_key: ValidatorPubKey
committee_index: CommitteeIndex
committee_length: uint64
validator_committee_index: uint64
slot: Slot
ValidatorPubkeySlotPair* = tuple[public_key: ValidatorPubKey, slot: Slot]
BeaconGenesisTuple* = tuple
genesis_time: uint64
genesis_validators_root: Eth2Digest
genesis_fork_version: Version

22
beacon_chain/spec/eth2_apis/validator_callsigs.nim
View File

@ -4,24 +4,17 @@ import
# Local modules # Local modules
../[datatypes, digest, crypto], ../[datatypes, digest, crypto],
json_rpc/jsonmarshal, json_rpc/jsonmarshal,
validator_callsigs_types callsigs_types
# TODO check which arguments are part of the path in the REST API
# calls that return a bool are actually without a return type in the main REST API
# spec but nim-json-rpc requires that all RPC calls have a return type.
# TODO this doesn't have "validator" in it's path but is used by the validators nonetheless
proc get_v1_beacon_states_fork(stateId: string): Fork
# TODO this doesn't have "validator" in it's path but is used by the validators nonetheless
proc get_v1_beacon_genesis(): BeaconGenesisTuple
# TODO returns a bool even though in the API there is no return type - because of nim-json-rpc
proc post_v1_beacon_pool_attestations(attestation: Attestation): bool proc post_v1_beacon_pool_attestations(attestation: Attestation): bool
# TODO slot is part of the REST path
proc get_v1_validator_blocks(slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig): BeaconBlock proc get_v1_validator_blocks(slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig): BeaconBlock
# TODO returns a bool even though in the API there is no return type - because of nim-json-rpc
proc post_v1_beacon_blocks(body: SignedBeaconBlock): bool proc post_v1_beacon_blocks(body: SignedBeaconBlock): bool
proc get_v1_validator_attestation_data(slot: Slot, committee_index: CommitteeIndex): AttestationData proc get_v1_validator_attestation_data(slot: Slot, committee_index: CommitteeIndex): AttestationData
@ -31,16 +24,17 @@ proc get_v1_validator_attestation_data(slot: Slot, committee_index: CommitteeInd
# https://docs.google.com/spreadsheets/d/1kVIx6GvzVLwNYbcd-Fj8YUlPf4qGrWUlS35uaTnIAVg/edit?disco=AAAAGh7r_fQ # https://docs.google.com/spreadsheets/d/1kVIx6GvzVLwNYbcd-Fj8YUlPf4qGrWUlS35uaTnIAVg/edit?disco=AAAAGh7r_fQ
proc get_v1_validator_aggregate_attestation(attestation_data: AttestationData): Attestation proc get_v1_validator_aggregate_attestation(attestation_data: AttestationData): Attestation
# TODO returns a bool even though in the API there is no return type - because of nim-json-rpc
proc post_v1_validator_aggregate_and_proof(payload: SignedAggregateAndProof): bool proc post_v1_validator_aggregate_and_proof(payload: SignedAggregateAndProof): bool
# this is a POST instead of a GET because of this: https://docs.google.com/spreadsheets/d/1kVIx6GvzVLwNYbcd-Fj8YUlPf4qGrWUlS35uaTnIAVg/edit?disco=AAAAJk5rbKA # this is a POST instead of a GET because of this: https://docs.google.com/spreadsheets/d/1kVIx6GvzVLwNYbcd-Fj8YUlPf4qGrWUlS35uaTnIAVg/edit?disco=AAAAJk5rbKA
# TODO epoch is part of the REST path
proc post_v1_validator_duties_attester(epoch: Epoch, public_keys: seq[ValidatorPubKey]): seq[AttesterDuties] proc post_v1_validator_duties_attester(epoch: Epoch, public_keys: seq[ValidatorPubKey]): seq[AttesterDuties]
# TODO epoch is part of the REST path
proc get_v1_validator_duties_proposer(epoch: Epoch): seq[ValidatorPubkeySlotPair] proc get_v1_validator_duties_proposer(epoch: Epoch): seq[ValidatorPubkeySlotPair]
proc post_v1_validator_beacon_committee_subscription(committee_index: CommitteeIndex, proc post_v1_validator_beacon_committee_subscriptions(committee_index: CommitteeIndex,
slot: Slot, slot: Slot,
aggregator: bool, aggregator: bool,
validator_pubkey: ValidatorPubKey, validator_pubkey: ValidatorPubKey,
slot_signature: ValidatorSig) slot_signature: ValidatorSig): bool

27
beacon_chain/spec/eth2_apis/validator_callsigs_types.nim
View File

@ -1,27 +0,0 @@
import
# Standard library
options,
# Local modules
# TODO for some reason "../[datatypes, digest, crypto]" results in "Error: cannot open file"
../datatypes,
../digest,
../crypto
type
AttesterDuties* = object
public_key*: ValidatorPubKey
committee_index*: CommitteeIndex
committee_length*: uint64
validator_committee_index*: uint64
slot*: Slot
# TODO do we even need this? how about a simple tuple (alias)?
ValidatorPubkeySlotPair* = object
public_key*: ValidatorPubKey
slot*: Slot
# TODO do we even need this? how about a simple tuple (alias)?
BeaconGenesisTuple* = object
genesis_time*: uint64
genesis_validators_root*: Eth2Digest
genesis_fork_version*: Version

21
beacon_chain/spec/helpers.nim
View File

@ -64,7 +64,13 @@ func get_active_validator_indices*(state: BeaconState, epoch: Epoch):
if is_active_validator(val, epoch): if is_active_validator(val, epoch):
result.add idx.ValidatorIndex result.add idx.ValidatorIndex
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_committee_count_at_slot # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_committee_count_at_slot
func get_committee_count_at_slot*(num_active_validators: auto):
uint64 =
clamp(
num_active_validators div SLOTS_PER_EPOCH div TARGET_COMMITTEE_SIZE,
1, MAX_COMMITTEES_PER_SLOT).uint64
func get_committee_count_at_slot*(state: BeaconState, slot: Slot): uint64 = func get_committee_count_at_slot*(state: BeaconState, slot: Slot): uint64 =
# Return the number of committees at ``slot``. # Return the number of committees at ``slot``.
@ -74,10 +80,7 @@ func get_committee_count_at_slot*(state: BeaconState, slot: Slot): uint64 =
# CommitteeIndex return type here. # CommitteeIndex return type here.
let epoch = compute_epoch_at_slot(slot) let epoch = compute_epoch_at_slot(slot)
let active_validator_indices = get_active_validator_indices(state, epoch) let active_validator_indices = get_active_validator_indices(state, epoch)
let committees_per_slot = clamp( result = get_committee_count_at_slot(len(active_validator_indices))
len(active_validator_indices) div SLOTS_PER_EPOCH div TARGET_COMMITTEE_SIZE,
1, MAX_COMMITTEES_PER_SLOT).uint64
result = committees_per_slot
# Otherwise, get_beacon_committee(...) cannot access some committees. # Otherwise, get_beacon_committee(...) cannot access some committees.
doAssert (SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT).uint64 >= result doAssert (SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT).uint64 >= result
@ -163,7 +166,7 @@ func compute_domain*(
result[0..3] = int_to_bytes4(domain_type.uint64) result[0..3] = int_to_bytes4(domain_type.uint64)
result[4..31] = fork_data_root.data[0..27] result[4..31] = fork_data_root.data[0..27]
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_domain # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_domain
func get_domain*( func get_domain*(
fork: Fork, domain_type: DomainType, epoch: Epoch, genesis_validators_root: Eth2Digest): Domain = fork: Fork, domain_type: DomainType, epoch: Epoch, genesis_validators_root: Eth2Digest): Domain =
## Return the signature domain (fork version concatenated with domain type) ## Return the signature domain (fork version concatenated with domain type)
@ -185,16 +188,10 @@ func get_domain*(
func compute_signing_root*(ssz_object: auto, domain: Domain): Eth2Digest = func compute_signing_root*(ssz_object: auto, domain: Domain): Eth2Digest =
# Return the signing root of an object by calculating the root of the # Return the signing root of an object by calculating the root of the
# object-domain tree. # object-domain tree.
when ETH2_SPEC == "v0.12.1":
let domain_wrapped_object = SigningData( let domain_wrapped_object = SigningData(
object_root: hash_tree_root(ssz_object), object_root: hash_tree_root(ssz_object),
domain: domain domain: domain
) )
else:
let domain_wrapped_object = SigningRoot(
object_root: hash_tree_root(ssz_object),
domain: domain
)
hash_tree_root(domain_wrapped_object) hash_tree_root(domain_wrapped_object)
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_seed # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_seed

169
beacon_chain/spec/keystore.nim
View File

@ -6,10 +6,10 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms. # at your option. This file may not be copied, modified, or distributed except according to those terms.
import import
json, math, strutils, strformat, json, math, strutils, strformat, typetraits,
stew/[results, byteutils, bitseqs, bitops2], stew/shims/macros, stew/[results, byteutils, bitseqs, bitops2], stew/shims/macros,
eth/keyfile/uuid, blscurve, eth/keyfile/uuid, blscurve, json_serialization,
nimcrypto/[sha2, rijndael, pbkdf2, bcmode, hash, sysrand], nimcrypto/[sha2, rijndael, pbkdf2, bcmode, hash, sysrand, utils],
./datatypes, ./crypto, ./digest, ./signatures ./datatypes, ./crypto, ./digest, ./signatures
export export
@ -44,6 +44,17 @@ type
prf: string prf: string
salt: string salt: string
# https://github.com/ethereum/EIPs/blob/4494da0966afa7318ec0157948821b19c4248805/EIPS/eip-2386.md#specification
Wallet* = object
uuid*: UUID
name*: WalletName
version*: uint
walletType* {.serializedFieldName: "type"}: string
# TODO: The use of `JsonString` can be removed once we
# solve the serialization problem for `Crypto[T]`
crypto*: JsonString
nextAccount* {.serializedFieldName: "nextaccount".}: Natural
KdfParams = KdfPbkdf2 | KdfScrypt KdfParams = KdfPbkdf2 | KdfScrypt
Kdf[T: KdfParams] = object Kdf[T: KdfParams] = object
@ -69,12 +80,18 @@ type
signingKeyKind # Also known as voting key signingKeyKind # Also known as voting key
withdrawalKeyKind withdrawalKeyKind
UUID* = distinct string
WalletName* = distinct string
Mnemonic* = distinct string Mnemonic* = distinct string
KeyPath* = distinct string KeyPath* = distinct string
KeyStorePass* = distinct string KeyStorePass* = distinct string
KeyStoreContent* = distinct JsonString
KeySeed* = distinct seq[byte] KeySeed* = distinct seq[byte]
KeyStoreContent* = distinct JsonString
WalletContent* = distinct JsonString
SensitiveData = Mnemonic|KeyStorePass|KeySeed
Credentials* = object Credentials* = object
mnemonic*: Mnemonic mnemonic*: Mnemonic
keyStore*: KeyStoreContent keyStore*: KeyStoreContent
@ -104,6 +121,17 @@ const
# https://github.com/bitcoin/bips/blob/master/bip-0039/bip-0039-wordlists.md # https://github.com/bitcoin/bips/blob/master/bip-0039/bip-0039-wordlists.md
wordListLen = 2048 wordListLen = 2048
UUID.serializesAsBaseIn Json
WalletName.serializesAsBaseIn Json
template `$`*(m: Mnemonic): string =
string(m)
template burnMem*(m: var (SensitiveData|TaintedString)) =
# TODO: `burnMem` in nimcrypto could use distinctBase
# to make its usage less error-prone.
utils.burnMem(string m)
macro wordListArray(filename: static string): array[wordListLen, cstring] = macro wordListArray(filename: static string): array[wordListLen, cstring] =
result = newTree(nnkBracket) result = newTree(nnkBracket)
var words = slurp(filename).split() var words = slurp(filename).split()
@ -152,7 +180,7 @@ func getSeed*(mnemonic: Mnemonic, password: KeyStorePass): KeySeed =
let salt = "mnemonic-" & password.string let salt = "mnemonic-" & password.string
KeySeed sha512.pbkdf2(mnemonic.string, salt, 2048, 64) KeySeed sha512.pbkdf2(mnemonic.string, salt, 2048, 64)
proc generateMnemonic*(words: openarray[cstring], proc generateMnemonic*(words: openarray[cstring] = englishWords,
entropyParam: openarray[byte] = @[]): Mnemonic = entropyParam: openarray[byte] = @[]): Mnemonic =
# https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki#generating-the-mnemonic # https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki#generating-the-mnemonic
doAssert words.len == wordListLen doAssert words.len == wordListLen
@ -226,9 +254,9 @@ proc shaChecksum(key, cipher: openarray[byte]): array[32, byte] =
result = ctx.finish().data result = ctx.finish().data
ctx.clear() ctx.clear()
template tryJsonToCrypto(ks: JsonNode; crypto: typedesc): untyped = template tryJsonToCrypto(json: JsonNode; crypto: typedesc): untyped =
try: try:
ks{"crypto"}.to(Crypto[crypto]) json.to(Crypto[crypto])
except Exception: except Exception:
return err "ks: failed to parse crypto" return err "ks: failed to parse crypto"
@ -238,12 +266,8 @@ template hexToBytes(data, name: string): untyped =
except ValueError: except ValueError:
return err "ks: failed to parse " & name return err "ks: failed to parse " & name
proc decryptKeystore*(data: KeyStoreContent, proc decryptoCryptoField*(json: JsonNode,
password: KeyStorePass): KsResult[ValidatorPrivKey] = password: KeyStorePass): KsResult[seq[byte]] =
# TODO: `parseJson` can raise a general `Exception`
let ks = try: parseJson(data.string)
except Exception: return err "ks: failed to parse keystore"
var var
decKey: seq[byte] decKey: seq[byte]
salt: seq[byte] salt: seq[byte]
@ -251,15 +275,15 @@ proc decryptKeystore*(data: KeyStoreContent,
cipherMsg: seq[byte] cipherMsg: seq[byte]
checksumMsg: seq[byte] checksumMsg: seq[byte]
let kdf = ks{"crypto", "kdf", "function"}.getStr let kdf = json{"kdf", "function"}.getStr
case kdf case kdf
of "scrypt": of "scrypt":
let crypto = tryJsonToCrypto(ks, KdfScrypt) let crypto = tryJsonToCrypto(json, KdfScrypt)
return err "ks: scrypt not supported" return err "ks: scrypt not supported"
of "pbkdf2": of "pbkdf2":
let let
crypto = tryJsonToCrypto(ks, KdfPbkdf2) crypto = tryJsonToCrypto(json, KdfPbkdf2)
kdfParams = crypto.kdf.params kdfParams = crypto.kdf.params
salt = hexToBytes(kdfParams.salt, "salt") salt = hexToBytes(kdfParams.salt, "salt")
@ -288,34 +312,41 @@ proc decryptKeystore*(data: KeyStoreContent,
aesCipher.decrypt(cipherMsg, secret) aesCipher.decrypt(cipherMsg, secret)
aesCipher.clear() aesCipher.clear()
ValidatorPrivKey.fromRaw(secret) ok secret
proc encryptKeystore*(T: type[KdfParams], proc decryptKeystore*(data: KeyStoreContent,
privKey: ValidatorPrivkey, password: KeyStorePass): KsResult[ValidatorPrivKey] =
# TODO: `parseJson` can raise a general `Exception`
let
ks = try: parseJson(data.string)
except Exception: return err "ks: failed to parse keystore"
secret = decryptoCryptoField(ks{"crypto"}, password)
ValidatorPrivKey.fromRaw(? secret)
proc createCryptoField(T: type[KdfParams],
secret: openarray[byte],
password = KeyStorePass "", password = KeyStorePass "",
path = KeyPath "",
salt: openarray[byte] = @[], salt: openarray[byte] = @[],
iv: openarray[byte] = @[], iv: openarray[byte] = @[]): Crypto[T] =
pretty = true): KeyStoreContent = type AES = aes128
var var
secret = privKey.toRaw[^32..^1]
decKey: seq[byte] decKey: seq[byte]
aesCipher: CTR[aes128] aesCipher: CTR[AES]
aesIv = newSeq[byte](aes128.sizeBlock)
kdfSalt = newSeq[byte](saltSize)
cipherMsg = newSeq[byte](secret.len) cipherMsg = newSeq[byte](secret.len)
if salt.len > 0: let kdfSalt = if salt.len > 0:
doAssert salt.len == saltSize doAssert salt.len == saltSize
kdfSalt = @salt @salt
else: else:
getRandomBytesOrPanic(kdfSalt) getRandomBytesOrPanic(saltSize)
if iv.len > 0: let aesIv = if iv.len > 0:
doAssert iv.len == aes128.sizeBlock doAssert iv.len == AES.sizeBlock
aesIv = @iv @iv
else: else:
getRandomBytesOrPanic(aesIv) getRandomBytesOrPanic(AES.sizeBlock)
when T is KdfPbkdf2: when T is KdfPbkdf2:
decKey = sha256.pbkdf2(password.string, kdfSalt, pbkdf2Params.c, decKey = sha256.pbkdf2(password.string, kdfSalt, pbkdf2Params.c,
@ -324,39 +355,83 @@ proc encryptKeystore*(T: type[KdfParams],
var kdf = Kdf[KdfPbkdf2](function: "pbkdf2", params: pbkdf2Params, message: "") var kdf = Kdf[KdfPbkdf2](function: "pbkdf2", params: pbkdf2Params, message: "")
kdf.params.salt = byteutils.toHex(kdfSalt) kdf.params.salt = byteutils.toHex(kdfSalt)
else: else:
return {.fatal: "Other KDFs are supported yet".}
aesCipher.init(decKey.toOpenArray(0, 15), aesIv) aesCipher.init(decKey.toOpenArray(0, 15), aesIv)
aesCipher.encrypt(secret, cipherMsg) aesCipher.encrypt(secret, cipherMsg)
aesCipher.clear() aesCipher.clear()
let pubkey = privKey.toPubKey() let sum = shaChecksum(decKey.toOpenArray(16, 31), cipherMsg)
let Crypto[T](
sum = shaChecksum(decKey.toOpenArray(16, 31), cipherMsg)
uuid = uuidGenerate().get
keystore = Keystore[T](
crypto: Crypto[T](
kdf: kdf, kdf: kdf,
checksum: Checksum( checksum: Checksum(
function: "sha256", function: "sha256",
message: byteutils.toHex(sum) message: byteutils.toHex(sum)),
),
cipher: Cipher( cipher: Cipher(
function: "aes-128-ctr", function: "aes-128-ctr",
params: CipherParams(iv: byteutils.toHex(aesIv)), params: CipherParams(iv: byteutils.toHex(aesIv)),
message: byteutils.toHex(cipherMsg) message: byteutils.toHex(cipherMsg)))
)
), proc encryptKeystore*(T: type[KdfParams],
privKey: ValidatorPrivkey,
password = KeyStorePass "",
path = KeyPath "",
salt: openarray[byte] = @[],
iv: openarray[byte] = @[],
pretty = true): KeyStoreContent =
let
secret = privKey.toRaw[^32..^1]
cryptoField = createCryptoField(T, secret, password, salt, iv)
pubkey = privKey.toPubKey()
uuid = uuidGenerate().expect("Random bytes should be available")
keystore = Keystore[T](
crypto: cryptoField,
pubkey: toHex(pubkey), pubkey: toHex(pubkey),
path: path.string, path: path.string,
uuid: $uuid, uuid: $uuid,
version: 4) version: 4)
KeyStoreContent if pretty: json.pretty(%keystore, indent=4) KeyStoreContent if pretty: json.pretty(%keystore)
else: $(%keystore) else: $(%keystore)
proc createWallet*(T: type[KdfParams],
mnemonic: Mnemonic,
name = WalletName "",
salt: openarray[byte] = @[],
iv: openarray[byte] = @[],
password = KeyStorePass "",
nextAccount = none(Natural),
pretty = true): Wallet =
let
uuid = UUID $(uuidGenerate().expect("Random bytes should be available"))
# Please note that we are passing an empty password here because
# we want the wallet restoration procedure to depend only on the
# mnemonic (the user is asked to treat the mnemonic as a password).
seed = getSeed(mnemonic, KeyStorePass"")
cryptoField = %createCryptoField(T, distinctBase seed, password, salt, iv)
Wallet(
uuid: uuid,
name: if name.string.len > 0: name
else: WalletName(uuid),
version: 1,
walletType: "hierarchical deterministic",
crypto: JsonString(if pretty: json.pretty(cryptoField)
else: $cryptoField),
nextAccount: nextAccount.get(0))
proc createWalletContent*(T: type[KdfParams],
mnemonic: Mnemonic,
name = WalletName "",
salt: openarray[byte] = @[],
iv: openarray[byte] = @[],
password = KeyStorePass "",
nextAccount = none(Natural),
pretty = true): (UUID, WalletContent) =
let wallet = createWallet(T, mnemonic, name, salt, iv, password, nextAccount, pretty)
(wallet.uuid, WalletContent Json.encode(wallet, pretty = pretty))
proc restoreCredentials*(mnemonic: Mnemonic, proc restoreCredentials*(mnemonic: Mnemonic,
password = KeyStorePass ""): Credentials = password = KeyStorePass ""): Credentials =
let let

53
beacon_chain/spec/network.nim
View File

@ -9,17 +9,20 @@
import import
strformat, strformat,
datatypes datatypes, helpers
const const
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/p2p-interface.md#topics-and-messages
topicBeaconBlocksSuffix* = "beacon_block/ssz" topicBeaconBlocksSuffix* = "beacon_block/ssz"
topicMainnetAttestationsSuffix* = "_beacon_attestation/ssz"
topicVoluntaryExitsSuffix* = "voluntary_exit/ssz" topicVoluntaryExitsSuffix* = "voluntary_exit/ssz"
topicProposerSlashingsSuffix* = "proposer_slashing/ssz" topicProposerSlashingsSuffix* = "proposer_slashing/ssz"
topicAttesterSlashingsSuffix* = "attester_slashing/ssz" topicAttesterSlashingsSuffix* = "attester_slashing/ssz"
topicAggregateAndProofsSuffix* = "beacon_aggregate_and_proof/ssz" topicAggregateAndProofsSuffix* = "beacon_aggregate_and_proof/ssz"
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#configuration # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/p2p-interface.md#topics-and-messages
topicMainnetAttestationsSuffix* = "_beacon_attestation/ssz"
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#misc
ATTESTATION_SUBNET_COUNT* = 64 ATTESTATION_SUBNET_COUNT* = 64
defaultEth2TcpPort* = 9000 defaultEth2TcpPort* = 9000
@ -27,55 +30,63 @@ const
# This is not part of the spec yet! # This is not part of the spec yet!
defaultEth2RpcPort* = 9090 defaultEth2RpcPort* = 9090
when ETH2_SPEC == "v0.11.3":
const topicInteropAttestationSuffix* = "beacon_attestation/ssz"
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#topics-and-messages
func getBeaconBlocksTopic*(forkDigest: ForkDigest): string = func getBeaconBlocksTopic*(forkDigest: ForkDigest): string =
try: try:
&"/eth2/{$forkDigest}/{topicBeaconBlocksSuffix}" &"/eth2/{$forkDigest}/{topicBeaconBlocksSuffix}"
except ValueError as e: except ValueError as e:
raiseAssert e.msg raiseAssert e.msg
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#topics-and-messages
func getVoluntaryExitsTopic*(forkDigest: ForkDigest): string = func getVoluntaryExitsTopic*(forkDigest: ForkDigest): string =
try: try:
&"/eth2/{$forkDigest}/{topicVoluntaryExitsSuffix}" &"/eth2/{$forkDigest}/{topicVoluntaryExitsSuffix}"
except ValueError as e: except ValueError as e:
raiseAssert e.msg raiseAssert e.msg
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#topics-and-messages
func getProposerSlashingsTopic*(forkDigest: ForkDigest): string = func getProposerSlashingsTopic*(forkDigest: ForkDigest): string =
try: try:
&"/eth2/{$forkDigest}/{topicProposerSlashingsSuffix}" &"/eth2/{$forkDigest}/{topicProposerSlashingsSuffix}"
except ValueError as e: except ValueError as e:
raiseAssert e.msg raiseAssert e.msg
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#topics-and-messages
func getAttesterSlashingsTopic*(forkDigest: ForkDigest): string = func getAttesterSlashingsTopic*(forkDigest: ForkDigest): string =
try: try:
&"/eth2/{$forkDigest}/{topicAttesterSlashingsSuffix}" &"/eth2/{$forkDigest}/{topicAttesterSlashingsSuffix}"
except ValueError as e: except ValueError as e:
raiseAssert e.msg raiseAssert e.msg
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#topics-and-messages
func getAggregateAndProofsTopic*(forkDigest: ForkDigest): string = func getAggregateAndProofsTopic*(forkDigest: ForkDigest): string =
try: try:
&"/eth2/{$forkDigest}/{topicAggregateAndProofsSuffix}" &"/eth2/{$forkDigest}/{topicAggregateAndProofsSuffix}"
except ValueError as e: except ValueError as e:
raiseAssert e.msg raiseAssert e.msg
when ETH2_SPEC == "v0.11.3": # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#broadcast-attestation
func getInteropAttestationTopic*(forkDigest: ForkDigest): string = func compute_subnet_for_attestation*(
num_active_validators: uint64, attestation: Attestation): uint64 =
# Compute the correct subnet for an attestation for Phase 0.
# Note, this mimics expected Phase 1 behavior where attestations will be
# mapped to their shard subnet.
#
# The spec version has params (state: BeaconState, attestation: Attestation),
# but it's only to call get_committee_count_at_slot(), which needs only epoch
# and the number of active validators.
let
slots_since_epoch_start = attestation.data.slot mod SLOTS_PER_EPOCH
committees_since_epoch_start =
get_committee_count_at_slot(num_active_validators) * slots_since_epoch_start
(committees_since_epoch_start + attestation.data.index) mod ATTESTATION_SUBNET_COUNT
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#broadcast-attestation
func getAttestationTopic*(forkDigest: ForkDigest, subnetIndex: uint64):
string =
# This is for subscribing or broadcasting manually to a known index.
try: try:
&"/eth2/{$forkDigest}/{topicInteropAttestationSuffix}" &"/eth2/{$forkDigest}/beacon_attestation_{subnetIndex}/ssz"
except ValueError as e: except ValueError as e:
raiseAssert e.msg raiseAssert e.msg
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#mainnet-3 func getAttestationTopic*(forkDigest: ForkDigest, attestation: Attestation, num_active_validators: uint64): string =
func getMainnetAttestationTopic*(forkDigest: ForkDigest, committeeIndex: uint64): string = getAttestationTopic(
try: forkDigest,
let topicIndex = committeeIndex mod ATTESTATION_SUBNET_COUNT compute_subnet_for_attestation(num_active_validators, attestation))
&"/eth2/{$forkDigest}/committee_index{topicIndex}{topicMainnetAttestationsSuffix}"
except ValueError as e:
raiseAssert e.msg

3
beacon_chain/spec/presets/custom.nim
View File

@ -73,14 +73,12 @@ type
MIN_EPOCHS_TO_INACTIVITY_PENALTY MIN_EPOCHS_TO_INACTIVITY_PENALTY
MIN_GASPRICE MIN_GASPRICE
MIN_GENESIS_ACTIVE_VALIDATOR_COUNT MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
MIN_GENESIS_DELAY # TODO BLS_SPEC == "v0.11.3" remove
MIN_GENESIS_TIME MIN_GENESIS_TIME
MIN_PER_EPOCH_CHURN_LIMIT MIN_PER_EPOCH_CHURN_LIMIT
MIN_SEED_LOOKAHEAD MIN_SEED_LOOKAHEAD
MIN_SLASHING_PENALTY_QUOTIENT MIN_SLASHING_PENALTY_QUOTIENT
MIN_VALIDATOR_WITHDRAWABILITY_DELAY MIN_VALIDATOR_WITHDRAWABILITY_DELAY
ONLINE_PERIOD ONLINE_PERIOD
PERSISTENT_COMMITTEE_PERIOD # TODO BLS_SPEC == "v0.11.3" remove
PHASE_1_FORK_VERSION PHASE_1_FORK_VERSION
PHASE_1_GENESIS_SLOT PHASE_1_GENESIS_SLOT
PROPOSER_REWARD_QUOTIENT PROPOSER_REWARD_QUOTIENT
@ -142,7 +140,6 @@ const
MIN_DEPOSIT_AMOUNT: "'u64", MIN_DEPOSIT_AMOUNT: "'u64",
MIN_EPOCHS_TO_INACTIVITY_PENALTY: "'u64", MIN_EPOCHS_TO_INACTIVITY_PENALTY: "'u64",
MIN_VALIDATOR_WITHDRAWABILITY_DELAY: "'u64", MIN_VALIDATOR_WITHDRAWABILITY_DELAY: "'u64",
PERSISTENT_COMMITTEE_PERIOD: "'u64",
PHASE_1_FORK_VERSION: forkVersionConversionFn, PHASE_1_FORK_VERSION: forkVersionConversionFn,
PROPOSER_REWARD_QUOTIENT: "'u64", PROPOSER_REWARD_QUOTIENT: "'u64",
SECONDS_PER_SLOT: "'u64", SECONDS_PER_SLOT: "'u64",

211
beacon_chain/spec/presets/mainnet_v0_11_3.nim
View File

@ -1,211 +0,0 @@
# beacon_chain
# Copyright (c) 2018-2020 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# This file contains constants that are part of the spec and thus subject to
# serialization and spec updates.
import
math
type
Slot* = distinct uint64
Epoch* = distinct uint64
{.experimental: "codeReordering".} # SLOTS_PER_EPOCH is use before being defined in spec
const
# Misc
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L6
MAX_COMMITTEES_PER_SLOT* {.intdefine.} = 64
TARGET_COMMITTEE_SIZE* = 2^7 ##\
## Number of validators in the committee attesting to one shard
## Per spec:
## For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds
## [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf);
## with sufficient active validators (at least
## `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures
## committee sizes at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness
## with a Verifiable Delay Function (VDF) will improve committee robustness
## and lower the safe minimum committee size.)
MAX_VALIDATORS_PER_COMMITTEE* = 2048 ##\
## votes
MIN_PER_EPOCH_CHURN_LIMIT* = 4
CHURN_LIMIT_QUOTIENT* = 2^16
SHUFFLE_ROUND_COUNT* = 90
MIN_GENESIS_ACTIVE_VALIDATOR_COUNT* {.intdefine.} = 16384
MIN_GENESIS_TIME* {.intdefine.} = 1578009600
HYSTERESIS_QUOTIENT* = 4
HYSTERESIS_DOWNWARD_MULTIPLIER* = 1
HYSTERESIS_UPWARD_MULTIPLIER* = 5
# Gwei values
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L58
MIN_DEPOSIT_AMOUNT* = 2'u64^0 * 10'u64^9 ##\
## Minimum amounth of ETH that can be deposited in one call - deposits can
## be used either to top up an existing validator or commit to a new one
MAX_EFFECTIVE_BALANCE* = 2'u64^5 * 10'u64^9 ##\
## Maximum amounth of ETH that can be deposited in one call
EJECTION_BALANCE* = 2'u64^4 * 10'u64^9 ##\
## Once the balance of a validator drops below this, it will be ejected from
## the validator pool
EFFECTIVE_BALANCE_INCREMENT* = 2'u64^0 * 10'u64^9
# Initial values
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L70
GENESIS_FORK_VERSION* = [0'u8, 0'u8, 0'u8, 0'u8]
BLS_WITHDRAWAL_PREFIX* = 0'u8
# Time parameters
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L77
MIN_GENESIS_DELAY* = 86400 # 86400 seconds (1 day)
SECONDS_PER_SLOT*{.intdefine.} = 12'u64 # Compile with -d:SECONDS_PER_SLOT=1 for 12x faster slots
## TODO consistent time unit across projects, similar to C++ chrono?
MIN_ATTESTATION_INCLUSION_DELAY* = 1 ##\
## (12 seconds)
## Number of slots that attestations stay in the attestation
## pool before being added to a block.
## The attestation delay exists so that there is time for attestations to
## propagate before the block is created.
## When creating an attestation, the validator will look at the best
## information known to at that time, and may not revise it during the same
## slot (see `is_double_vote`) - the delay gives the validator a chance to
## wait towards the end of the slot and still have time to publish the
## attestation.
SLOTS_PER_EPOCH* {.intdefine.} = 32 ##\
## (~6.4 minutes)
## slots that make up an epoch, at the end of which more heavy
## processing is done
## Compile with -d:SLOTS_PER_EPOCH=4 for shorter epochs
MIN_SEED_LOOKAHEAD* = 1 ##\
## epochs (~6.4 minutes)
MAX_SEED_LOOKAHEAD* = 4 ##\
## epochs (~25.6 minutes)
EPOCHS_PER_ETH1_VOTING_PERIOD* = 32 ##\
## epochs (~3.4 hours)
SLOTS_PER_HISTORICAL_ROOT* = 8192 ##\
## slots (13 hours)
MIN_VALIDATOR_WITHDRAWABILITY_DELAY* = 2'u64^8 ##\
## epochs (~27 hours)
PERSISTENT_COMMITTEE_PERIOD* = 2'u64^11 ##\
## epochs (9 days)
MAX_EPOCHS_PER_CROSSLINK* = 2'u64^6 ##\
## epochs (~7 hours)
MIN_EPOCHS_TO_INACTIVITY_PENALTY* = 2'u64^2 ##\
## epochs (25.6 minutes)
# State vector lengths
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L105
EPOCHS_PER_HISTORICAL_VECTOR* = 65536 ##\
## epochs (~0.8 years)
EPOCHS_PER_SLASHINGS_VECTOR* = 8192 ##\
## epochs (~36 days)
HISTORICAL_ROOTS_LIMIT* = 16777216 ##\
## epochs (~26,131 years)
VALIDATOR_REGISTRY_LIMIT* = 1099511627776
# Reward and penalty quotients
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L117
BASE_REWARD_FACTOR* = 2'u64^6
WHISTLEBLOWER_REWARD_QUOTIENT* = 2'u64^9
PROPOSER_REWARD_QUOTIENT* = 2'u64^3
INACTIVITY_PENALTY_QUOTIENT* = 2'u64^25
MIN_SLASHING_PENALTY_QUOTIENT* = 32 # 2^5
# Max operations per block
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L131
MAX_PROPOSER_SLASHINGS* = 2^4
MAX_ATTESTER_SLASHINGS* = 2^0
MAX_ATTESTATIONS* = 2^7
MAX_DEPOSITS* = 2^4
MAX_VOLUNTARY_EXITS* = 2^4
# Fork choice
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L32
SAFE_SLOTS_TO_UPDATE_JUSTIFIED* = 8 # 96 seconds
# Validators
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L38
ETH1_FOLLOW_DISTANCE* = 1024 # blocks ~ 4 hours
TARGET_AGGREGATORS_PER_COMMITTEE* = 16 # validators
RANDOM_SUBNETS_PER_VALIDATOR* = 1 # subnet
EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION* = 256 # epochs ~ 27 hours
SECONDS_PER_ETH1_BLOCK* = 14 # (estimate from Eth1 mainnet)
# Phase 1: Upgrade from Phase 0
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L161
PHASE_1_FORK_VERSION* = 1
INITIAL_ACTIVE_SHARDS* = 64
# Phase 1: General
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L166
MAX_SHARDS* = 1024
ONLINE_PERIOD* = 8 # epochs (~51 min)
LIGHT_CLIENT_COMMITTEE_SIZE* = 128
LIGHT_CLIENT_COMMITTEE_PERIOD* = 256 # epochs (~27 hours)
SHARD_COMMITTEE_PERIOD* = 256 # epochs (~27 hours)
SHARD_BLOCK_CHUNK_SIZE* = 262144
MAX_SHARD_BLOCK_CHUNKS* = 4
TARGET_SHARD_BLOCK_SIZE* = 196608
MAX_SHARD_BLOCKS_PER_ATTESTATION* = 12
MAX_GASPRICE* = 16384 # Gwei
MIN_GASPRICE* = 8 # Gwei
GASPRICE_ADJUSTMENT_COEFFICIENT* = 8
# Phase 1: Custody game
# ---------------------------------------------------------------
# Time parameters
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L199
RANDAO_PENALTY_EPOCHS* = 2 # epochs (12.8 minutes)
EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS* = 16384 # epochs (~73 days)
EPOCHS_PER_CUSTODY_PERIOD* = 2048 # epochs (~9 days)
CUSTODY_PERIOD_TO_RANDAO_PADDING* = 2048 # epochs (~9 days)
MAX_REVEAL_LATENESS_DECREMENT* = 128 # epochs (~14 hours)
# Max operations
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L211
MAX_CUSTODY_KEY_REVEALS* = 256
MAX_EARLY_DERIVED_SECRET_REVEALS* = 1
MAX_CUSTODY_SLASHINGS* = 1
# Reward and penalty quotients
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L217
EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE* = 2
MINOR_REWARD_QUOTIENT* = 256

190
beacon_chain/spec/presets/minimal_v0_11_3.nim
View File

@ -1,190 +0,0 @@
# beacon_chain
# Copyright (c) 2018-2020 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# This file contains constants that are part of the spec and thus subject to
# serialization and spec updates.
import
math
type
Slot* = distinct uint64
Epoch* = distinct uint64
{.experimental: "codeReordering".} # SLOTS_PER_EPOCH is use before being defined in spec
const
# Misc
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L4
# Changed
MAX_COMMITTEES_PER_SLOT* = 4
TARGET_COMMITTEE_SIZE* = 4
# Unchanged
MAX_VALIDATORS_PER_COMMITTEE* = 2048
MIN_PER_EPOCH_CHURN_LIMIT* = 4
CHURN_LIMIT_QUOTIENT* = 2^16
# Changed
SHUFFLE_ROUND_COUNT* = 10
MIN_GENESIS_ACTIVE_VALIDATOR_COUNT* {.intdefine.} = 64
MIN_GENESIS_TIME* {.intdefine.} = 1578009600 # 3 Jan, 2020
# Unchanged
HYSTERESIS_QUOTIENT* = 4
HYSTERESIS_DOWNWARD_MULTIPLIER* = 1
HYSTERESIS_UPWARD_MULTIPLIER* = 5
# Gwei values
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L58
# Unchanged
MIN_DEPOSIT_AMOUNT* = 2'u64^0 * 10'u64^9
MAX_EFFECTIVE_BALANCE* = 2'u64^5 * 10'u64^9
EJECTION_BALANCE* = 2'u64^4 * 10'u64^9
EFFECTIVE_BALANCE_INCREMENT* = 2'u64^0 * 10'u64^9
# Initial values
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L70
GENESIS_FORK_VERSION* = [0'u8, 0'u8, 0'u8, 1'u8]
BLS_WITHDRAWAL_PREFIX* = 0'u8
# Time parameters
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L77
# Changed: Faster to spin up testnets, but does not give validator
# reasonable warning time for genesis
MIN_GENESIS_DELAY* = 300
# Unchanged
SECONDS_PER_SLOT*{.intdefine.} = 6'u64
# Unchanged
MIN_ATTESTATION_INCLUSION_DELAY* = 1
# Changed
SLOTS_PER_EPOCH* {.intdefine.} = 8
# Unchanged
MIN_SEED_LOOKAHEAD* = 1
MAX_SEED_LOOKAHEAD* = 4
# Changed
EPOCHS_PER_ETH1_VOTING_PERIOD* = 2
SLOTS_PER_HISTORICAL_ROOT* = 64
# Unchanged
MIN_VALIDATOR_WITHDRAWABILITY_DELAY* = 2'u64^8
# Changed
PERSISTENT_COMMITTEE_PERIOD* = 128
# Unchanged
MAX_EPOCHS_PER_CROSSLINK* = 4
# Changed
MIN_EPOCHS_TO_INACTIVITY_PENALTY* = 2'u64^2
# State vector lengths
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L105
# Changed
EPOCHS_PER_HISTORICAL_VECTOR* = 64
EPOCHS_PER_SLASHINGS_VECTOR* = 64
# Unchanged
HISTORICAL_ROOTS_LIMIT* = 16777216
VALIDATOR_REGISTRY_LIMIT* = 1099511627776
# Reward and penalty quotients
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L117
BASE_REWARD_FACTOR* = 2'u64^6
WHISTLEBLOWER_REWARD_QUOTIENT* = 2'u64^9
PROPOSER_REWARD_QUOTIENT* = 2'u64^3
INACTIVITY_PENALTY_QUOTIENT* = 2'u64^25
MIN_SLASHING_PENALTY_QUOTIENT* = 32 # 2^5
# Max operations per block
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L131
MAX_PROPOSER_SLASHINGS* = 2^4
MAX_ATTESTER_SLASHINGS* = 2^0
MAX_ATTESTATIONS* = 2^7
MAX_DEPOSITS* = 2^4
MAX_VOLUNTARY_EXITS* = 2^4
# Fork choice
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L32
# Changed
SAFE_SLOTS_TO_UPDATE_JUSTIFIED* = 2
# Validators
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L38
# Changed
ETH1_FOLLOW_DISTANCE* = 16 # blocks
# Unchanged
TARGET_AGGREGATORS_PER_COMMITTEE* = 16 # validators
RANDOM_SUBNETS_PER_VALIDATOR* = 1 # subnet
EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION* = 256 # epochs ~ 27 hours
SECONDS_PER_ETH1_BLOCK* = 14 # estimate from Eth1 mainnet)
# Phase 1: Upgrade from Phase 0
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L161
PHASE_1_FORK_VERSION* = 16777217
INITIAL_ACTIVE_SHARDS* = 4
# Phase 1: General
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L169
MAX_SHARDS* = 8
ONLINE_PERIOD* = 8 # epochs ~ 51 minutes
LIGHT_CLIENT_COMMITTEE_SIZE* = 128
LIGHT_CLIENT_COMMITTEE_PERIOD* = 256 # epochs
SHARD_COMMITTEE_PERIOD* = 256 # epochs
SHARD_BLOCK_CHUNK_SIZE* = 262144
MAX_SHARD_BLOCK_CHUNKS* = 4
TARGET_SHARD_BLOCK_SIZE* = 196608
MAX_SHARD_BLOCKS_PER_ATTESTATION* = 12
MAX_GASPRICE* = 16384 # Gwei
MIN_GASPRICE* = 8 # Gwei
GASPRICE_ADJUSTMENT_COEFFICIENT* = 8
# Phase 1 - Custody game
# ---------------------------------------------------------------
# Time parameters
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L202
RANDAO_PENALTY_EPOCHS* = 2
EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS* = 4096 # epochs
EPOCHS_PER_CUSTODY_PERIOD* = 2048
CUSTODY_PERIOD_TO_RANDAO_PADDING* = 2048
MAX_REVEAL_LATENESS_DECREMENT* = 128
# Max operations
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L214
MAX_CUSTODY_KEY_REVEALS* = 256
MAX_EARLY_DERIVED_SECRET_REVEALS* = 1
MAX_CUSTODY_SLASHINGS* = 1
# Reward and penalty quotients
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L220
EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE* = 2
MINOR_REWARD_QUOTIENT* = 256

8
beacon_chain/spec/presets/mainnet.nim → beacon_chain/spec/presets/v0_12_1/mainnet.nim
View File

@ -24,7 +24,7 @@ const
MAX_COMMITTEES_PER_SLOT* {.intdefine.} = 64 MAX_COMMITTEES_PER_SLOT* {.intdefine.} = 64
TARGET_COMMITTEE_SIZE* = 2^7 ##\ TARGET_COMMITTEE_SIZE* = 128 ##\
## Number of validators in the committee attesting to one shard ## Number of validators in the committee attesting to one shard
## Per spec: ## Per spec:
## For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds ## For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds
@ -74,7 +74,7 @@ const
# Time parameters # Time parameters
# --------------------------------------------------------------- # ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L77 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L77
GENESIS_DELAY* = 172800 # 172800 seconds (2 days) GENESIS_DELAY* {.intdefine.} = 172800 # 172800 seconds (2 days)
SECONDS_PER_SLOT* {.intdefine.} = 12'u64 # Compile with -d:SECONDS_PER_SLOT=1 for 12x faster slots SECONDS_PER_SLOT* {.intdefine.} = 12'u64 # Compile with -d:SECONDS_PER_SLOT=1 for 12x faster slots
## TODO consistent time unit across projects, similar to C++ chrono? ## TODO consistent time unit across projects, similar to C++ chrono?
@ -161,11 +161,11 @@ const
# Validators # Validators
# --------------------------------------------------------------- # ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L38 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L38
ETH1_FOLLOW_DISTANCE* = 1024 # blocks ~ 4 hours ETH1_FOLLOW_DISTANCE* {.intdefine.} = 1024 # blocks ~ 4 hours
TARGET_AGGREGATORS_PER_COMMITTEE* = 16 # validators TARGET_AGGREGATORS_PER_COMMITTEE* = 16 # validators
RANDOM_SUBNETS_PER_VALIDATOR* = 1 # subnet RANDOM_SUBNETS_PER_VALIDATOR* = 1 # subnet
EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION* = 256 # epochs ~ 27 hours EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION* = 256 # epochs ~ 27 hours
SECONDS_PER_ETH1_BLOCK* = 14 # (estimate from Eth1 mainnet) SECONDS_PER_ETH1_BLOCK* {.intdefine.} = 14 # (estimate from Eth1 mainnet)
# Phase 1: Upgrade from Phase 0 # Phase 1: Upgrade from Phase 0
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L161 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L161

12
beacon_chain/spec/presets/minimal.nim → beacon_chain/spec/presets/v0_12_1/minimal.nim
View File

@ -20,7 +20,7 @@ type
const const
# Misc # Misc
# --------------------------------------------------------------- # ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L4 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/minimal.yaml#L4
# Changed # Changed
MAX_COMMITTEES_PER_SLOT* = 4 MAX_COMMITTEES_PER_SLOT* = 4
@ -63,7 +63,7 @@ const
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/minimal.yaml#L77 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/minimal.yaml#L77
# Changed: Faster to spin up testnets, but does not give validator # Changed: Faster to spin up testnets, but does not give validator
# reasonable warning time for genesis # reasonable warning time for genesis
GENESIS_DELAY* = 300 GENESIS_DELAY* {.intdefine.} = 300
# Unchanged # Unchanged
SECONDS_PER_SLOT*{.intdefine.} = 6'u64 SECONDS_PER_SLOT*{.intdefine.} = 6'u64
@ -95,7 +95,7 @@ const
# State vector lengths # State vector lengths
# --------------------------------------------------------------- # ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L105 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/minimal.yaml#L105
# Changed # Changed
EPOCHS_PER_HISTORICAL_VECTOR* = 64 EPOCHS_PER_HISTORICAL_VECTOR* = 64
@ -127,7 +127,7 @@ const
# Fork choice # Fork choice
# --------------------------------------------------------------- # ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/minimal.yaml#L32 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/minimal.yaml#L32
# Changed # Changed
SAFE_SLOTS_TO_UPDATE_JUSTIFIED* = 2 SAFE_SLOTS_TO_UPDATE_JUSTIFIED* = 2
@ -137,13 +137,13 @@ const
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/minimal.yaml#L38 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/minimal.yaml#L38
# Changed # Changed
ETH1_FOLLOW_DISTANCE* = 16 # blocks ETH1_FOLLOW_DISTANCE* {.intdefine.} = 16 # blocks
# Unchanged # Unchanged
TARGET_AGGREGATORS_PER_COMMITTEE* = 16 # validators TARGET_AGGREGATORS_PER_COMMITTEE* = 16 # validators
RANDOM_SUBNETS_PER_VALIDATOR* = 1 # subnet RANDOM_SUBNETS_PER_VALIDATOR* = 1 # subnet
EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION* = 256 # epochs ~ 27 hours EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION* = 256 # epochs ~ 27 hours
SECONDS_PER_ETH1_BLOCK* = 14 # estimate from Eth1 mainnet) SECONDS_PER_ETH1_BLOCK* {.intdefine.} = 14 # estimate from Eth1 mainnet)
# Phase 1: Upgrade from Phase 0 # Phase 1: Upgrade from Phase 0
# --------------------------------------------------------------- # ---------------------------------------------------------------

21
beacon_chain/spec/signatures.nim
View File

@ -10,6 +10,12 @@
import import
./crypto, ./digest, ./datatypes, ./helpers, ../ssz/merkleization ./crypto, ./digest, ./datatypes, ./helpers, ../ssz/merkleization
template withTrust(sig: SomeSig, body: untyped): bool =
when sig is TrustedSig:
true
else:
body
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#aggregation-selection # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#aggregation-selection
func get_slot_signature*( func get_slot_signature*(
fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot,
@ -34,7 +40,8 @@ func get_epoch_signature*(
func verify_epoch_signature*( func verify_epoch_signature*(
fork: Fork, genesis_validators_root: Eth2Digest, epoch: Epoch, fork: Fork, genesis_validators_root: Eth2Digest, epoch: Epoch,
pubkey: ValidatorPubKey, signature: ValidatorSig): bool = pubkey: ValidatorPubKey, signature: SomeSig): bool =
withTrust(signature):
let let
domain = get_domain(fork, DOMAIN_RANDAO, epoch, genesis_validators_root) domain = get_domain(fork, DOMAIN_RANDAO, epoch, genesis_validators_root)
signing_root = compute_signing_root(epoch, domain) signing_root = compute_signing_root(epoch, domain)
@ -55,8 +62,10 @@ func get_block_signature*(
func verify_block_signature*( func verify_block_signature*(
fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot,
blck: Eth2Digest | BeaconBlock | BeaconBlockHeader, pubkey: ValidatorPubKey, blck: Eth2Digest | SomeBeaconBlock | BeaconBlockHeader,
signature: ValidatorSig): bool = pubkey: ValidatorPubKey,
signature: SomeSig): bool =
withTrust(signature):
let let
epoch = compute_epoch_at_slot(slot) epoch = compute_epoch_at_slot(slot)
domain = get_domain( domain = get_domain(
@ -94,7 +103,8 @@ func verify_attestation_signature*(
fork: Fork, genesis_validators_root: Eth2Digest, fork: Fork, genesis_validators_root: Eth2Digest,
attestation_data: AttestationData, attestation_data: AttestationData,
pubkeys: openArray[ValidatorPubKey], pubkeys: openArray[ValidatorPubKey],
signature: ValidatorSig): bool = signature: SomeSig): bool =
withTrust(signature):
let let
epoch = attestation_data.target.epoch epoch = attestation_data.target.epoch
domain = get_domain( domain = get_domain(
@ -128,7 +138,8 @@ func verify_deposit_signature*(deposit: DepositData): bool =
func verify_voluntary_exit_signature*( func verify_voluntary_exit_signature*(
fork: Fork, genesis_validators_root: Eth2Digest, fork: Fork, genesis_validators_root: Eth2Digest,
voluntary_exit: VoluntaryExit, voluntary_exit: VoluntaryExit,
pubkey: ValidatorPubKey, signature: ValidatorSig): bool = pubkey: ValidatorPubKey, signature: SomeSig): bool =
withTrust(signature):
let let
domain = get_domain( domain = get_domain(
fork, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch, genesis_validators_root) fork, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch, genesis_validators_root)

22
beacon_chain/state_transition.nim → beacon_chain/spec/state_transition.nim
View File

@ -31,10 +31,10 @@ import
tables, tables,
chronicles, chronicles,
stew/results, stew/results,
./extras, ./ssz/merkleization, metrics, ../extras, ../ssz/merkleization, metrics,
./spec/[datatypes, crypto, digest, helpers, signatures, validator], datatypes, crypto, digest, helpers, signatures, validator,
./spec/[state_transition_block, state_transition_epoch], state_transition_block, state_transition_epoch,
../nbench/bench_lab ../../nbench/bench_lab
# https://github.com/ethereum/eth2.0-metrics/blob/master/metrics.md#additional-metrics # https://github.com/ethereum/eth2.0-metrics/blob/master/metrics.md#additional-metrics
declareGauge beacon_current_validators, """Number of status="pending|active|exited|withdrawable" validators in current epoch""" # On epoch transition declareGauge beacon_current_validators, """Number of status="pending|active|exited|withdrawable" validators in current epoch""" # On epoch transition
@ -64,7 +64,7 @@ func get_epoch_validator_count(state: BeaconState): int64 {.nbench.} =
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function
proc verify_block_signature*( proc verify_block_signature*(
state: BeaconState, signed_block: SignedBeaconBlock): bool {.nbench.} = state: BeaconState, signed_block: SomeSignedBeaconBlock): bool {.nbench.} =
let let
proposer_index = signed_block.message.proposer_index proposer_index = signed_block.message.proposer_index
if proposer_index >= state.validators.len.uint64: if proposer_index >= state.validators.len.uint64:
@ -88,11 +88,15 @@ proc verifyStateRoot(state: BeaconState, blck: BeaconBlock): bool =
let state_root = hash_tree_root(state) let state_root = hash_tree_root(state)
if state_root != blck.state_root: if state_root != blck.state_root:
notice "Block: root verification failed", notice "Block: root verification failed",
block_state_root = blck.state_root, state_root block_state_root = shortLog(blck.state_root), state_root = shortLog(state_root)
false false
else: else:
true true
proc verifyStateRoot(state: BeaconState, blck: TrustedBeaconBlock): bool =
# This is inlined in state_transition(...) in spec.
true
type type
RollbackProc* = proc(v: var BeaconState) {.gcsafe, raises: [Defect].} RollbackProc* = proc(v: var BeaconState) {.gcsafe, raises: [Defect].}
@ -170,7 +174,7 @@ proc noRollback*(state: var HashedBeaconState) =
trace "Skipping rollback of broken state" trace "Skipping rollback of broken state"
proc state_transition*( proc state_transition*(
state: var HashedBeaconState, signedBlock: SignedBeaconBlock, state: var HashedBeaconState, signedBlock: SomeSignedBeaconBlock,
# TODO this is ... okay, but not perfect; align with EpochRef # TODO this is ... okay, but not perfect; align with EpochRef
stateCache: var StateCache, stateCache: var StateCache,
flags: UpdateFlags, rollback: RollbackHashedProc): bool {.nbench.} = flags: UpdateFlags, rollback: RollbackHashedProc): bool {.nbench.} =
@ -225,7 +229,7 @@ proc state_transition*(
trace "in state_transition: processing block, signature passed", trace "in state_transition: processing block, signature passed",
signature = signedBlock.signature, signature = signedBlock.signature,
blockRoot = hash_tree_root(signedBlock.message) blockRoot = hash_tree_root(signedBlock.message)
if processBlock(state.data, signedBlock.message, flags, stateCache): if process_block(state.data, signedBlock.message, flags, stateCache):
if skipStateRootValidation in flags or verifyStateRoot(state.data, signedBlock.message): if skipStateRootValidation in flags or verifyStateRoot(state.data, signedBlock.message):
# State root is what it should be - we're done! # State root is what it should be - we're done!
@ -245,7 +249,7 @@ proc state_transition*(
false false
proc state_transition*( proc state_transition*(
state: var HashedBeaconState, signedBlock: SignedBeaconBlock, state: var HashedBeaconState, signedBlock: SomeSignedBeaconBlock,
flags: UpdateFlags, rollback: RollbackHashedProc): bool {.nbench.} = flags: UpdateFlags, rollback: RollbackHashedProc): bool {.nbench.} =
# TODO consider moving this to testutils or similar, since non-testing # TODO consider moving this to testutils or similar, since non-testing
# and fuzzing code should always be coming from blockpool which should # and fuzzing code should always be coming from blockpool which should

32
beacon_chain/spec/state_transition_block.nim
View File

@ -44,12 +44,13 @@ declareGauge beacon_processed_deposits_total, "Number of total deposits included
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#block-header # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#block-header
proc process_block_header*( proc process_block_header*(
state: var BeaconState, blck: BeaconBlock, flags: UpdateFlags, state: var BeaconState, blck: SomeBeaconBlock, flags: UpdateFlags,
stateCache: var StateCache): bool {.nbench.} = stateCache: var StateCache): bool {.nbench.} =
logScope:
blck = shortLog(blck)
# Verify that the slots match # Verify that the slots match
if not (blck.slot == state.slot): if not (blck.slot == state.slot):
notice "Block header: slot mismatch", notice "Block header: slot mismatch",
block_slot = shortLog(blck.slot),
state_slot = shortLog(state.slot) state_slot = shortLog(state.slot)
return false return false
@ -66,16 +67,13 @@ proc process_block_header*(
if not (blck.proposer_index.ValidatorIndex == proposer_index.get): if not (blck.proposer_index.ValidatorIndex == proposer_index.get):
notice "Block header: proposer index incorrect", notice "Block header: proposer index incorrect",
block_proposer_index = blck.proposer_index.ValidatorIndex,
proposer_index = proposer_index.get proposer_index = proposer_index.get
return false return false
# Verify that the parent matches # Verify that the parent matches
if skipBlockParentRootValidation notin flags and not (blck.parent_root == if not (blck.parent_root == hash_tree_root(state.latest_block_header)):
hash_tree_root(state.latest_block_header)):
notice "Block header: previous block root mismatch", notice "Block header: previous block root mismatch",
latest_block_header = state.latest_block_header, latest_block_header = state.latest_block_header,
blck = shortLog(blck),
latest_block_header_root = shortLog(hash_tree_root(state.latest_block_header)) latest_block_header_root = shortLog(hash_tree_root(state.latest_block_header))
return false return false
@ -100,9 +98,9 @@ proc `xor`[T: array](a, b: T): T =
for i in 0..<result.len: for i in 0..<result.len:
result[i] = a[i] xor b[i] result[i] = a[i] xor b[i]
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#randao # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#randao
proc process_randao( proc process_randao(
state: var BeaconState, body: BeaconBlockBody, flags: UpdateFlags, state: var BeaconState, body: SomeBeaconBlockBody, flags: UpdateFlags,
stateCache: var StateCache): bool {.nbench.} = stateCache: var StateCache): bool {.nbench.} =
let let
proposer_index = get_beacon_proposer_index(state, stateCache) proposer_index = get_beacon_proposer_index(state, stateCache)
@ -138,7 +136,7 @@ proc process_randao(
true true
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#eth1-data # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#eth1-data
func process_eth1_data(state: var BeaconState, body: BeaconBlockBody) {.nbench.}= func process_eth1_data(state: var BeaconState, body: SomeBeaconBlockBody) {.nbench.}=
state.eth1_data_votes.add body.eth1_data state.eth1_data_votes.add body.eth1_data
if state.eth1_data_votes.asSeq.count(body.eth1_data) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD.int: if state.eth1_data_votes.asSeq.count(body.eth1_data) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD.int:
@ -201,7 +199,7 @@ proc process_proposer_slashing*(
true true
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#is_slashable_attestation_data # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#is_slashable_attestation_data
func is_slashable_attestation_data( func is_slashable_attestation_data(
data_1: AttestationData, data_2: AttestationData): bool = data_1: AttestationData, data_2: AttestationData): bool =
## Check if ``data_1`` and ``data_2`` are slashable according to Casper FFG ## Check if ``data_1`` and ``data_2`` are slashable according to Casper FFG
@ -285,16 +283,10 @@ proc process_voluntary_exit*(
return false return false
# Verify the validator has been active long enough # Verify the validator has been active long enough
when ETH2_SPEC == "v0.12.1":
if not (get_current_epoch(state) >= validator.activation_epoch + if not (get_current_epoch(state) >= validator.activation_epoch +
SHARD_COMMITTEE_PERIOD): SHARD_COMMITTEE_PERIOD):
notice "Exit: not in validator set long enough" notice "Exit: not in validator set long enough"
return false return false
else:
if not (get_current_epoch(state) >= validator.activation_epoch +
PERSISTENT_COMMITTEE_PERIOD):
notice "Exit: not in validator set long enough"
return false
# Verify signature # Verify signature
if skipBlsValidation notin flags: if skipBlsValidation notin flags:
@ -321,7 +313,7 @@ proc process_voluntary_exit*(
true true
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#operations # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#operations
proc process_operations(state: var BeaconState, body: BeaconBlockBody, proc process_operations(state: var BeaconState, body: SomeBeaconBlockBody,
flags: UpdateFlags, stateCache: var StateCache): bool {.nbench.} = flags: UpdateFlags, stateCache: var StateCache): bool {.nbench.} =
# Verify that outstanding deposits are processed up to the maximum number of # Verify that outstanding deposits are processed up to the maximum number of
# deposits # deposits
@ -357,7 +349,7 @@ proc process_operations(state: var BeaconState, body: BeaconBlockBody,
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#block-processing # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#block-processing
proc process_block*( proc process_block*(
state: var BeaconState, blck: BeaconBlock, flags: UpdateFlags, state: var BeaconState, blck: SomeBeaconBlock, flags: UpdateFlags,
stateCache: var StateCache): bool {.nbench.}= stateCache: var StateCache): bool {.nbench.}=
## When there's a new block, we need to verify that the block is sane and ## When there's a new block, we need to verify that the block is sane and
## update the state accordingly ## update the state accordingly
@ -383,8 +375,8 @@ proc process_block*(
notice "Block header not valid", slot = shortLog(state.slot) notice "Block header not valid", slot = shortLog(state.slot)
return false return false
if not processRandao(state, blck.body, flags, stateCache): if not process_randao(state, blck.body, flags, stateCache):
debug "[Block processing] Randao failure", slot = shortLog(state.slot) debug "Randao failure", slot = shortLog(state.slot)
return false return false
process_eth1_data(state, blck.body) process_eth1_data(state, blck.body)

111
beacon_chain/spec/state_transition_epoch.nim
View File

@ -173,7 +173,7 @@ proc process_justification_and_finalization*(state: var BeaconState,
root: get_block_root(state, previous_epoch)) root: get_block_root(state, previous_epoch))
state.justification_bits.setBit 1 state.justification_bits.setBit 1
info "Justified with previous epoch", debug "Justified with previous epoch",
current_epoch = current_epoch, current_epoch = current_epoch,
checkpoint = shortLog(state.current_justified_checkpoint), checkpoint = shortLog(state.current_justified_checkpoint),
cat = "justification" cat = "justification"
@ -187,7 +187,7 @@ proc process_justification_and_finalization*(state: var BeaconState,
root: get_block_root(state, current_epoch)) root: get_block_root(state, current_epoch))
state.justification_bits.setBit 0 state.justification_bits.setBit 0
info "Justified with current epoch", debug "Justified with current epoch",
current_epoch = current_epoch, current_epoch = current_epoch,
checkpoint = shortLog(state.current_justified_checkpoint), checkpoint = shortLog(state.current_justified_checkpoint),
cat = "justification" cat = "justification"
@ -201,7 +201,7 @@ proc process_justification_and_finalization*(state: var BeaconState,
old_previous_justified_checkpoint.epoch + 3 == current_epoch: old_previous_justified_checkpoint.epoch + 3 == current_epoch:
state.finalized_checkpoint = old_previous_justified_checkpoint state.finalized_checkpoint = old_previous_justified_checkpoint
info "Finalized with rule 234", debug "Finalized with rule 234",
current_epoch = current_epoch, current_epoch = current_epoch,
checkpoint = shortLog(state.finalized_checkpoint), checkpoint = shortLog(state.finalized_checkpoint),
cat = "finalization" cat = "finalization"
@ -212,7 +212,7 @@ proc process_justification_and_finalization*(state: var BeaconState,
old_previous_justified_checkpoint.epoch + 2 == current_epoch: old_previous_justified_checkpoint.epoch + 2 == current_epoch:
state.finalized_checkpoint = old_previous_justified_checkpoint state.finalized_checkpoint = old_previous_justified_checkpoint
info "Finalized with rule 23", debug "Finalized with rule 23",
current_epoch = current_epoch, current_epoch = current_epoch,
checkpoint = shortLog(state.finalized_checkpoint), checkpoint = shortLog(state.finalized_checkpoint),
cat = "finalization" cat = "finalization"
@ -223,7 +223,7 @@ proc process_justification_and_finalization*(state: var BeaconState,
old_current_justified_checkpoint.epoch + 2 == current_epoch: old_current_justified_checkpoint.epoch + 2 == current_epoch:
state.finalized_checkpoint = old_current_justified_checkpoint state.finalized_checkpoint = old_current_justified_checkpoint
info "Finalized with rule 123", debug "Finalized with rule 123",
current_epoch = current_epoch, current_epoch = current_epoch,
checkpoint = shortLog(state.finalized_checkpoint), checkpoint = shortLog(state.finalized_checkpoint),
cat = "finalization" cat = "finalization"
@ -234,7 +234,7 @@ proc process_justification_and_finalization*(state: var BeaconState,
old_current_justified_checkpoint.epoch + 1 == current_epoch: old_current_justified_checkpoint.epoch + 1 == current_epoch:
state.finalized_checkpoint = old_current_justified_checkpoint state.finalized_checkpoint = old_current_justified_checkpoint
info "Finalized with rule 12", debug "Finalized with rule 12",
current_epoch = current_epoch, current_epoch = current_epoch,
checkpoint = shortLog(state.finalized_checkpoint), checkpoint = shortLog(state.finalized_checkpoint),
cat = "finalization" cat = "finalization"
@ -393,7 +393,6 @@ func get_inactivity_penalty_deltas(state: BeaconState, cache: var StateCache):
# Spec constructs rewards anyway; this doesn't # Spec constructs rewards anyway; this doesn't
penalties penalties
when ETH2_SPEC == "v0.12.1":
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_attestation_deltas # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_attestation_deltas
func get_attestation_deltas(state: BeaconState, cache: var StateCache): tuple[a: seq[Gwei], b: seq[Gwei]] = func get_attestation_deltas(state: BeaconState, cache: var StateCache): tuple[a: seq[Gwei], b: seq[Gwei]] =
# Return attestation reward/penalty deltas for each validator. # Return attestation reward/penalty deltas for each validator.
@ -413,104 +412,8 @@ when ETH2_SPEC == "v0.12.1":
inactivity_penalties[it]) inactivity_penalties[it])
(rewards, penalties) (rewards, penalties)
else:
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#rewards-and-penalties-1
func get_attestation_deltas(state: BeaconState, stateCache: var StateCache):
tuple[a: seq[Gwei], b: seq[Gwei]] {.nbench.}=
let
previous_epoch = get_previous_epoch(state)
total_balance = get_total_active_balance(state, stateCache)
var
rewards = repeat(0'u64, len(state.validators))
penalties = repeat(0'u64, len(state.validators))
eligible_validator_indices : seq[ValidatorIndex] = @[]
for index, v in state.validators: # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#process_rewards_and_penalties
if is_active_validator(v, previous_epoch) or
(v.slashed and previous_epoch + 1 < v.withdrawable_epoch):
eligible_validator_indices.add index.ValidatorIndex
# Micro-incentives for matching FFG source, FFG target, and head
let
matching_source_attestations =
get_matching_source_attestations(state, previous_epoch)
matching_target_attestations =
get_matching_target_attestations(state, previous_epoch)
matching_head_attestations =
get_matching_head_attestations(state, previous_epoch)
for attestations in
[matching_source_attestations, matching_target_attestations,
matching_head_attestations]:
let
unslashed_attesting_indices =
get_unslashed_attesting_indices(state, attestations, stateCache)
attesting_balance = get_total_balance(state, unslashed_attesting_indices)
for index in eligible_validator_indices:
if index in unslashed_attesting_indices:
# Factored out from balance totals to avoid uint64 overflow
const increment = EFFECTIVE_BALANCE_INCREMENT
let reward_numerator = get_base_reward(state, index, total_balance) *
(attesting_balance div increment)
rewards[index] += reward_numerator div (total_balance div increment)
else:
penalties[index] += get_base_reward(state, index, total_balance)
# Proposer and inclusion delay micro-rewards
## This depends on matching_source_attestations being an indexable seq, not a
## set, hash table, etc.
let source_attestation_attesting_indices =
mapIt(
matching_source_attestations,
get_attesting_indices(state, it.data, it.aggregation_bits, stateCache))
for index in get_unslashed_attesting_indices(
state, matching_source_attestations, stateCache):
# Translation of attestation = min([...])
doAssert matching_source_attestations.len > 0
# Start by filtering the right attestations
var filtered_matching_source_attestations: seq[PendingAttestation]
for source_attestation_index, a in matching_source_attestations:
if index notin
source_attestation_attesting_indices[source_attestation_index]:
continue
filtered_matching_source_attestations.add a
# The first filtered attestation serves as min until we find something
# better
var attestation = filtered_matching_source_attestations[0]
for source_attestation_index, a in filtered_matching_source_attestations:
if a.inclusion_delay < attestation.inclusion_delay:
attestation = a
let
base_reward = get_base_reward(state, index, total_balance)
proposer_reward = (base_reward div PROPOSER_REWARD_QUOTIENT).Gwei
rewards[attestation.proposer_index.int] += proposer_reward
let max_attester_reward = base_reward - proposer_reward
rewards[index] += max_attester_reward div attestation.inclusion_delay
# Inactivity penalty
let finality_delay = previous_epoch - state.finalized_checkpoint.epoch
if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY:
let matching_target_attesting_indices =
get_unslashed_attesting_indices(
state, matching_target_attestations, stateCache)
for index in eligible_validator_indices:
penalties[index] +=
BASE_REWARDS_PER_EPOCH.uint64 * get_base_reward(state, index, total_balance)
if index notin matching_target_attesting_indices:
penalties[index] +=
state.validators[index].effective_balance *
finality_delay div INACTIVITY_PENALTY_QUOTIENT
(rewards, penalties)
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#rewards-and-penalties-1
func process_rewards_and_penalties( func process_rewards_and_penalties(
state: var BeaconState, cache: var StateCache) {.nbench.}= state: var BeaconState, cache: var StateCache) {.nbench.}=
if get_current_epoch(state) == GENESIS_EPOCH: if get_current_epoch(state) == GENESIS_EPOCH:

16
beacon_chain/spec/validator.nim
View File

@ -12,8 +12,8 @@ import
algorithm, options, sequtils, math, tables, algorithm, options, sequtils, math, tables,
./datatypes, ./digest, ./helpers ./datatypes, ./digest, ./helpers
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#compute_shuffled_index # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#compute_shuffled_index
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#compute_committee # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#compute_committee
func get_shuffled_seq*(seed: Eth2Digest, func get_shuffled_seq*(seed: Eth2Digest,
list_size: uint64, list_size: uint64,
): seq[ValidatorIndex] = ): seq[ValidatorIndex] =
@ -99,7 +99,7 @@ func get_previous_epoch*(state: BeaconState): Epoch =
else: else:
current_epoch - 1 current_epoch - 1
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#compute_committee # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#compute_committee
func compute_committee(indices: seq[ValidatorIndex], seed: Eth2Digest, func compute_committee(indices: seq[ValidatorIndex], seed: Eth2Digest,
index: uint64, count: uint64): seq[ValidatorIndex] = index: uint64, count: uint64): seq[ValidatorIndex] =
## Return the committee corresponding to ``indices``, ``seed``, ``index``, ## Return the committee corresponding to ``indices``, ``seed``, ``index``,
@ -123,7 +123,7 @@ func compute_committee(indices: seq[ValidatorIndex], seed: Eth2Digest,
except KeyError: except KeyError:
raiseAssert("Cached entries are added before use") raiseAssert("Cached entries are added before use")
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_committee # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_beacon_committee
func get_beacon_committee*( func get_beacon_committee*(
state: BeaconState, slot: Slot, index: CommitteeIndex, state: BeaconState, slot: Slot, index: CommitteeIndex,
cache: var StateCache): seq[ValidatorIndex] = cache: var StateCache): seq[ValidatorIndex] =
@ -162,7 +162,7 @@ func get_empty_per_epoch_cache*(): StateCache =
initTable[Epoch, seq[ValidatorIndex]]() initTable[Epoch, seq[ValidatorIndex]]()
result.committee_count_cache = initTable[Epoch, uint64]() result.committee_count_cache = initTable[Epoch, uint64]()
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#compute_proposer_index # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#compute_proposer_index
func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex], func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex],
seed: Eth2Digest): Option[ValidatorIndex] = seed: Eth2Digest): Option[ValidatorIndex] =
# Return from ``indices`` a random index sampled by effective balance. # Return from ``indices`` a random index sampled by effective balance.
@ -193,7 +193,7 @@ func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex],
return some(candidate_index) return some(candidate_index)
i += 1 i += 1
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_proposer_index # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_beacon_proposer_index
func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot: Slot): func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot: Slot):
Option[ValidatorIndex] = Option[ValidatorIndex] =
try: try:
@ -227,7 +227,7 @@ func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot:
except KeyError: except KeyError:
raiseAssert("Cached entries are added before use") raiseAssert("Cached entries are added before use")
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_proposer_index # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_beacon_proposer_index
func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache): func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache):
Option[ValidatorIndex] = Option[ValidatorIndex] =
get_beacon_proposer_index(state, cache, state.slot) get_beacon_proposer_index(state, cache, state.slot)
@ -257,7 +257,7 @@ func get_committee_assignment*(
return some((committee, idx, slot)) return some((committee, idx, slot))
none(tuple[a: seq[ValidatorIndex], b: CommitteeIndex, c: Slot]) none(tuple[a: seq[ValidatorIndex], b: CommitteeIndex, c: Slot])
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/validator.md#validator-assignments # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#validator-assignments
func is_proposer( func is_proposer(
state: BeaconState, validator_index: ValidatorIndex): bool {.used.} = state: BeaconState, validator_index: ValidatorIndex): bool {.used.} =
var cache = get_empty_per_epoch_cache() var cache = get_empty_per_epoch_cache()

3
beacon_chain/ssz/bytes_reader.nim
View File

@ -57,9 +57,6 @@ func fromSszBytes*(T: type Version, bytes: openarray[byte]): T {.raisesssz.} =
raiseIncorrectSize T raiseIncorrectSize T
copyMem(result.addr, unsafeAddr bytes[0], sizeof(result)) copyMem(result.addr, unsafeAddr bytes[0], sizeof(result))
template fromSszBytes*(T: type enum, bytes: openarray[byte]): auto =
T fromSszBytes(uint64, bytes)
template fromSszBytes*(T: type BitSeq, bytes: openarray[byte]): auto = template fromSszBytes*(T: type BitSeq, bytes: openarray[byte]): auto =
BitSeq @bytes BitSeq @bytes

4
beacon_chain/ssz/spec_types.nim
View File

@ -10,7 +10,9 @@ template toSszType*(x: auto): auto =
# Please note that BitArray doesn't need any special treatment here # Please note that BitArray doesn't need any special treatment here
# because it can be considered a regular fixed-size object type. # because it can be considered a regular fixed-size object type.
when x is Slot|Epoch|ValidatorIndex|enum: uint64(x) # enum should not be added here as nim will raise Defect when value is out
# of range
when x is Slot|Epoch|ValidatorIndex: uint64(x)
elif x is Eth2Digest: x.data elif x is Eth2Digest: x.data
elif x is BlsCurveType: toRaw(x) elif x is BlsCurveType: toRaw(x)
elif x is ForkDigest|Version: distinctBase(x) elif x is ForkDigest|Version: distinctBase(x)

6
beacon_chain/sszdump.nim
View File

@ -22,7 +22,11 @@ proc dump*(dir: string, v: SignedBeaconBlock, root: Eth2Digest) =
logErrors: logErrors:
SSZ.saveFile(dir / &"block-{v.message.slot}-{shortLog(root)}.ssz", v) SSZ.saveFile(dir / &"block-{v.message.slot}-{shortLog(root)}.ssz", v)
proc dump*(dir: string, v: SignedBeaconBlock, blck: BlockRef) = proc dump*(dir: string, v: TrustedSignedBeaconBlock, root: Eth2Digest) =
logErrors:
SSZ.saveFile(dir / &"block-{v.message.slot}-{shortLog(root)}.ssz", v)
proc dump*(dir: string, v: SomeSignedBeaconBlock, blck: BlockRef) =
dump(dir, v, blck.root) dump(dir, v, blck.root)
proc dump*(dir: string, v: HashedBeaconState, blck: BlockRef) = proc dump*(dir: string, v: HashedBeaconState, blck: BlockRef) =

17
beacon_chain/sync_protocol.nim
View File

@ -59,6 +59,13 @@ proc shortLog*(s: StatusMsg): auto =
) )
chronicles.formatIt(StatusMsg): shortLog(it) chronicles.formatIt(StatusMsg): shortLog(it)
func disconnectReasonName(reason: uint64): string =
# haha, nim doesn't support uint64 in `case`!
if reason == uint64(ClientShutDown): "Client shutdown"
elif reason == uint64(IrrelevantNetwork): "Irrelevant network"
elif reason == uint64(FaultOrError): "Fault or error"
else: "Disconnected (" & $reason & ")"
proc importBlocks(state: BeaconSyncNetworkState, proc importBlocks(state: BeaconSyncNetworkState,
blocks: openarray[SignedBeaconBlock]) {.gcsafe.} = blocks: openarray[SignedBeaconBlock]) {.gcsafe.} =
for blk in blocks: for blk in blocks:
@ -121,7 +128,8 @@ p2pProtocol BeaconSync(version = 1,
{.libp2pProtocol("metadata", 1).} = {.libp2pProtocol("metadata", 1).} =
return peer.network.metadata return peer.network.metadata
proc beaconBlocksByRange(peer: Peer, proc beaconBlocksByRange(
peer: Peer,
startSlot: Slot, startSlot: Slot,
count: uint64, count: uint64,
step: uint64, step: uint64,
@ -149,7 +157,8 @@ p2pProtocol BeaconSync(version = 1,
debug "Block range request done", debug "Block range request done",
peer, startSlot, count, step, found = count - startIndex peer, startSlot, count, step, found = count - startIndex
proc beaconBlocksByRoot(peer: Peer, proc beaconBlocksByRoot(
peer: Peer,
blockRoots: BlockRootsList, blockRoots: BlockRootsList,
response: MultipleChunksResponse[SignedBeaconBlock]) response: MultipleChunksResponse[SignedBeaconBlock])
{.async, libp2pProtocol("beacon_blocks_by_root", 1).} = {.async, libp2pProtocol("beacon_blocks_by_root", 1).} =
@ -169,9 +178,9 @@ p2pProtocol BeaconSync(version = 1,
peer, roots = blockRoots.len, count, found peer, roots = blockRoots.len, count, found
proc goodbye(peer: Peer, proc goodbye(peer: Peer,
reason: DisconnectionReason) reason: uint64)
{.async, libp2pProtocol("goodbye", 1).} = {.async, libp2pProtocol("goodbye", 1).} =
debug "Received Goodbye message", reason, peer debug "Received Goodbye message", reason = disconnectReasonName(reason), peer
proc setStatusMsg(peer: Peer, statusMsg: StatusMsg) = proc setStatusMsg(peer: Peer, statusMsg: StatusMsg) =
debug "Peer status", peer, statusMsg debug "Peer status", peer, statusMsg

16
beacon_chain/sync_protocol.nim.generated.nim
View File

@ -1,5 +1,5 @@
## Generated at line 87 ## Generated at line 94
type type
BeaconSync* = object BeaconSync* = object
template State*(PROTO: type BeaconSync): type = template State*(PROTO: type BeaconSync): type =
@ -69,15 +69,15 @@ template RecType*(MSG: type beaconBlocksByRootObj): untyped =
BlockRootsList BlockRootsList
type type
goodbyeObj* = distinct DisconnectionReason goodbyeObj* = distinct uint64
template goodbye*(PROTO: type BeaconSync): type = template goodbye*(PROTO: type BeaconSync): type =
DisconnectionReason uint64
template msgProtocol*(MSG: type goodbyeObj): type = template msgProtocol*(MSG: type goodbyeObj): type =
BeaconSync BeaconSync
template RecType*(MSG: type goodbyeObj): untyped = template RecType*(MSG: type goodbyeObj): untyped =
DisconnectionReason uint64
var BeaconSyncProtocolObj = initProtocol("BeaconSync", createPeerState[Peer, var BeaconSyncProtocolObj = initProtocol("BeaconSync", createPeerState[Peer,
ref[BeaconSyncPeerState:ObjectType]], createNetworkState[Eth2Node, ref[BeaconSyncPeerState:ObjectType]], createNetworkState[Eth2Node,
@ -136,7 +136,7 @@ proc beaconBlocksByRoot*(peer: Peer; blockRoots: BlockRootsList;
makeEth2Request(peer, "/eth2/beacon_chain/req/beacon_blocks_by_root/1/", makeEth2Request(peer, "/eth2/beacon_chain/req/beacon_blocks_by_root/1/",
msgBytes, seq[SignedBeaconBlock], timeout) msgBytes, seq[SignedBeaconBlock], timeout)
proc goodbye*(peer: Peer; reason: DisconnectionReason): Future[void] {.gcsafe, proc goodbye*(peer: Peer; reason: uint64): Future[void] {.gcsafe,
libp2pProtocol("goodbye", 1).} = libp2pProtocol("goodbye", 1).} =
var outputStream = memoryOutput() var outputStream = memoryOutput()
var writer = init(WriterType(SSZ), outputStream) var writer = init(WriterType(SSZ), outputStream)
@ -238,7 +238,7 @@ proc beaconBlocksByRootUserHandler(peer: Peer; blockRoots: BlockRootsList; respo
inc found inc found
debug "Block root request done", peer, roots = blockRoots.len, count, found debug "Block root request done", peer, roots = blockRoots.len, count, found
proc goodbyeUserHandler(peer: Peer; reason: DisconnectionReason) {.async, proc goodbyeUserHandler(peer: Peer; reason: uint64) {.async,
libp2pProtocol("goodbye", 1), gcsafe.} = libp2pProtocol("goodbye", 1), gcsafe.} =
type type
CurrentProtocol = BeaconSync CurrentProtocol = BeaconSync
@ -249,7 +249,7 @@ proc goodbyeUserHandler(peer: Peer; reason: DisconnectionReason) {.async,
cast[ref[BeaconSyncNetworkState:ObjectType]](getNetworkState(peer.network, cast[ref[BeaconSyncNetworkState:ObjectType]](getNetworkState(peer.network,
BeaconSyncProtocol)) BeaconSyncProtocol))
debug "Received Goodbye message", reason, peer debug "Received Goodbye message", reason = disconnectReasonName(reason), peer
template callUserHandler(MSG: type statusObj; peer: Peer; stream: Connection; template callUserHandler(MSG: type statusObj; peer: Peer; stream: Connection;
noSnappy: bool; msg: StatusMsg): untyped = noSnappy: bool; msg: StatusMsg): untyped =
@ -338,7 +338,7 @@ proc beaconBlocksByRootMounter(network: Eth2Node) =
"ssz_snappy", handler: snappyThunk) "ssz_snappy", handler: snappyThunk)
template callUserHandler(MSG: type goodbyeObj; peer: Peer; stream: Connection; template callUserHandler(MSG: type goodbyeObj; peer: Peer; stream: Connection;
noSnappy: bool; msg: DisconnectionReason): untyped = noSnappy: bool; msg: uint64): untyped =
goodbyeUserHandler(peer, msg) goodbyeUserHandler(peer, msg)
proc goodbyeMounter(network: Eth2Node) = proc goodbyeMounter(network: Eth2Node) =

159
beacon_chain/validator_api.nim
View File

@ -7,7 +7,7 @@
import import
# Standard library # Standard library
tables, strutils, tables, strutils, parseutils,
# Nimble packages # Nimble packages
stew/[objects], stew/[objects],
@ -19,7 +19,7 @@ import
block_pool, ssz/merkleization, block_pool, ssz/merkleization,
beacon_node_common, beacon_node_types, beacon_node_common, beacon_node_types,
validator_duties, eth2_network, validator_duties, eth2_network,
spec/eth2_apis/validator_callsigs_types, spec/eth2_apis/callsigs_types,
eth2_json_rpc_serialization eth2_json_rpc_serialization
type type
@ -27,63 +27,101 @@ type
logScope: topics = "valapi" logScope: topics = "valapi"
# TODO Probably the `beacon` ones should be defined elsewhere...?
proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) = proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
# TODO Probably the `beacon` ones (and not `validator`) should be defined elsewhere... template withStateForSlot(stateId: string, body: untyped): untyped =
rpcServer.rpc("get_v1_beacon_states_fork") do (stateId: string) -> Fork: var res: BiggestInt
notice "== get_v1_beacon_states_fork", stateId = stateId if parseBiggestInt(stateId, res) == stateId.len:
raise newException(CatchableError, "Not a valid slot number")
let head = node.updateHead()
let blockSlot = head.atSlot(res.Slot)
node.blockPool.withState(node.blockPool.tmpState, blockSlot):
body
rpcServer.rpc("get_v1_beacon_genesis") do () -> BeaconGenesisTuple:
debug "get_v1_beacon_genesis"
return (genesis_time: node.blockPool.headState.data.data.genesis_time,
genesis_validators_root:
node.blockPool.headState.data.data.genesis_validators_root,
genesis_fork_version: Version(GENESIS_FORK_VERSION))
rpcServer.rpc("get_v1_beacon_states_root") do (stateId: string) -> Eth2Digest:
debug "get_v1_beacon_states_root", stateId = stateId
# TODO do we need to call node.updateHead() before using headState?
result = case stateId:
of "head":
node.blockPool.headState.blck.root
of "genesis":
node.blockPool.headState.data.data.genesis_validators_root
of "finalized":
node.blockPool.headState.data.data.finalized_checkpoint.root
of "justified":
node.blockPool.headState.data.data.current_justified_checkpoint.root
else:
if stateId.startsWith("0x"):
# TODO not sure if `fromHex` is the right thing here...
# https://github.com/ethereum/eth2.0-APIs/issues/37#issuecomment-638566144
# we return whatever was passed to us (this is a nonsense request)
fromHex(Eth2Digest, stateId[2..<stateId.len]) # skip first 2 chars
else:
withStateForSlot(stateId):
hashedState.root
rpcServer.rpc("get_v1_beacon_states_fork") do (stateId: string) -> Fork:
debug "get_v1_beacon_states_fork", stateId = stateId
result = case stateId: result = case stateId:
of "head": of "head":
discard node.updateHead() # TODO do we need this?
node.blockPool.headState.data.data.fork node.blockPool.headState.data.data.fork
of "genesis": of "genesis":
Fork(previous_version: Version(GENESIS_FORK_VERSION), Fork(previous_version: Version(GENESIS_FORK_VERSION),
current_version: Version(GENESIS_FORK_VERSION), current_version: Version(GENESIS_FORK_VERSION),
epoch: GENESIS_EPOCH) epoch: GENESIS_EPOCH)
of "finalized": of "finalized":
# TODO node.blockPool.withState(node.blockPool.tmpState, node.blockPool.finalizedHead):
Fork() state.fork
of "justified": of "justified":
# TODO node.blockPool.justifiedState.data.data.fork
Fork()
else: else:
# TODO parse `stateId` as either a number (slot) or a hash (stateRoot) if stateId.startsWith("0x"):
Fork() # TODO not sure if `fromHex` is the right thing here...
# https://github.com/ethereum/eth2.0-APIs/issues/37#issuecomment-638566144
let blckRoot = fromHex(Eth2Digest, stateId[2..<stateId.len]) # skip first 2 chars
let blckRef = node.blockPool.getRef(blckRoot)
if blckRef.isNil:
raise newException(CatchableError, "Block not found")
let blckSlot = blckRef.atSlot(blckRef.slot)
node.blockPool.withState(node.blockPool.tmpState, blckSlot):
state.fork
else:
withStateForSlot(stateId):
state.fork
# TODO Probably the `beacon` ones (and not `validator`) should be defined elsewhere... rpcServer.rpc("post_v1_beacon_pool_attestations") do (
rpcServer.rpc("get_v1_beacon_genesis") do () -> BeaconGenesisTuple: attestation: Attestation) -> bool:
notice "== get_v1_beacon_genesis"
return BeaconGenesisTuple(genesis_time: node.blockPool.headState.data.data.genesis_time,
genesis_validators_root: node.blockPool.headState.data.data.genesis_validators_root,
genesis_fork_version: Version(GENESIS_FORK_VERSION))
rpcServer.rpc("post_v1_beacon_pool_attestations") do (attestation: Attestation) -> bool:
#notice "== post_v1_beacon_pool_attestations"
node.sendAttestation(attestation) node.sendAttestation(attestation)
return true return true
rpcServer.rpc("get_v1_validator_blocks") do ( rpcServer.rpc("get_v1_validator_blocks") do (
slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig) -> BeaconBlock: slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig) -> BeaconBlock:
notice "== get_v1_validator_blocks", slot = slot debug "get_v1_validator_blocks", slot = slot
let head = node.updateHead() let head = node.updateHead()
let proposer = node.blockPool.getProposer(head, slot) let proposer = node.blockPool.getProposer(head, slot)
# TODO how do we handle the case when we cannot return a meaningful block? 404... if proposer.isNone():
doAssert(proposer.isSome()) raise newException(CatchableError, "could not retrieve block for slot: " & $slot)
let valInfo = ValidatorInfoForMakeBeaconBlock(kind: viRandao_reveal, let valInfo = ValidatorInfoForMakeBeaconBlock(kind: viRandao_reveal,
randao_reveal: randao_reveal) randao_reveal: randao_reveal)
let res = makeBeaconBlockForHeadAndSlot( let res = makeBeaconBlockForHeadAndSlot(
node, valInfo, proposer.get()[0], graffiti, head, slot) node, valInfo, proposer.get()[0], graffiti, head, slot)
if res.message.isNone():
# TODO how do we handle the case when we cannot return a meaningful block? 404... raise newException(CatchableError, "could not retrieve block for slot: " & $slot)
doAssert(res.message.isSome()) return res.message.get()
return res.message.get(BeaconBlock()) # returning a default if empty
rpcServer.rpc("post_v1_beacon_blocks") do (body: SignedBeaconBlock) -> bool: rpcServer.rpc("post_v1_beacon_blocks") do (body: SignedBeaconBlock) -> bool:
notice "== post_v1_beacon_blocks" debug "post_v1_beacon_blocks",
slot = body.message.slot,
logScope: pcs = "block_proposal" prop_idx = body.message.proposer_index
let head = node.updateHead() let head = node.updateHead()
if head.slot >= body.message.slot: if head.slot >= body.message.slot:
@ -92,14 +130,15 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
headBlockRoot = shortLog(head.root), headBlockRoot = shortLog(head.root),
slot = shortLog(body.message.slot), slot = shortLog(body.message.slot),
cat = "fastforward" cat = "fastforward"
return false raise newException(CatchableError,
return head != await proposeSignedBlock(node, head, AttachedValidator(), "Proposal is for a past slot: " & $body.message.slot)
body, hash_tree_root(body.message)) if head == await proposeSignedBlock(node, head, AttachedValidator(),
body, hash_tree_root(body.message)):
raise newException(CatchableError, "Could not propose block")
return true
rpcServer.rpc("get_v1_validator_attestation_data") do ( rpcServer.rpc("get_v1_validator_attestation_data") do (
slot: Slot, committee_index: CommitteeIndex) -> AttestationData: slot: Slot, committee_index: CommitteeIndex) -> AttestationData:
#notice "== get_v1_validator_attestation_data"
# Obtain the data to form an attestation
let head = node.updateHead() let head = node.updateHead()
let attestationHead = head.atSlot(slot) let attestationHead = head.atSlot(slot)
node.blockPool.withState(node.blockPool.tmpState, attestationHead): node.blockPool.withState(node.blockPool.tmpState, attestationHead):
@ -107,45 +146,43 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
rpcServer.rpc("get_v1_validator_aggregate_attestation") do ( rpcServer.rpc("get_v1_validator_aggregate_attestation") do (
attestation_data: AttestationData)-> Attestation: attestation_data: AttestationData)-> Attestation:
notice "== get_v1_validator_aggregate_attestation" debug "get_v1_validator_aggregate_attestation"
rpcServer.rpc("post_v1_validator_aggregate_and_proof") do ( rpcServer.rpc("post_v1_validator_aggregate_and_proof") do (
payload: SignedAggregateAndProof) -> bool: payload: SignedAggregateAndProof) -> bool:
notice "== post_v1_validator_aggregate_and_proof"
# TODO is this enough?
node.network.broadcast(node.topicAggregateAndProofs, payload) node.network.broadcast(node.topicAggregateAndProofs, payload)
return true return true
rpcServer.rpc("post_v1_validator_duties_attester") do ( rpcServer.rpc("post_v1_validator_duties_attester") do (
epoch: Epoch, public_keys: seq[ValidatorPubKey]) -> seq[AttesterDuties]: epoch: Epoch, public_keys: seq[ValidatorPubKey]) -> seq[AttesterDuties]:
notice "== post_v1_validator_duties_attester", epoch = epoch debug "post_v1_validator_duties_attester", epoch = epoch
discard node.updateHead() # TODO do we need this? let head = node.updateHead()
let attestationHead = head.atSlot(compute_start_slot_at_epoch(epoch))
node.blockPool.withState(node.blockPool.tmpState, attestationHead):
for pubkey in public_keys: for pubkey in public_keys:
let idx = node.blockPool.headState.data.data.validators.asSeq.findIt(it.pubKey == pubkey) let idx = state.validators.asSeq.findIt(it.pubKey == pubkey)
if idx != -1: if idx == -1:
# TODO this might crash if the requested epoch is further than the BN epoch continue
# because of this: `doAssert epoch <= next_epoch` let ca = state.get_committee_assignment(epoch, idx.ValidatorIndex)
let res = node.blockPool.headState.data.data.get_committee_assignment( if ca.isSome:
epoch, idx.ValidatorIndex) result.add((public_key: pubkey,
if res.isSome: committee_index: ca.get.b,
result.add(AttesterDuties(public_key: pubkey, committee_length: ca.get.a.len.uint64,
committee_index: res.get.b, validator_committee_index: ca.get.a.find(idx.ValidatorIndex).uint64,
committee_length: res.get.a.len.uint64, slot: ca.get.c))
validator_committee_index: res.get.a.find(idx.ValidatorIndex).uint64,
slot: res.get.c))
rpcServer.rpc("get_v1_validator_duties_proposer") do ( rpcServer.rpc("get_v1_validator_duties_proposer") do (
epoch: Epoch) -> seq[ValidatorPubkeySlotPair]: epoch: Epoch) -> seq[ValidatorPubkeySlotPair]:
notice "== get_v1_validator_duties_proposer", epoch = epoch debug "get_v1_validator_duties_proposer", epoch = epoch
let head = node.updateHead() let head = node.updateHead()
for i in 0 ..< SLOTS_PER_EPOCH: for i in 0 ..< SLOTS_PER_EPOCH:
let currSlot = (compute_start_slot_at_epoch(epoch).int + i).Slot let currSlot = (compute_start_slot_at_epoch(epoch).int + i).Slot
let proposer = node.blockPool.getProposer(head, currSlot) let proposer = node.blockPool.getProposer(head, currSlot)
if proposer.isSome(): if proposer.isSome():
result.add(ValidatorPubkeySlotPair(public_key: proposer.get()[1], slot: currSlot)) result.add((public_key: proposer.get()[1], slot: currSlot))
rpcServer.rpc("post_v1_validator_beacon_committee_subscription") do ( rpcServer.rpc("post_v1_validator_beacon_committee_subscriptions") do (
committee_index: CommitteeIndex, slot: Slot, aggregator: bool, committee_index: CommitteeIndex, slot: Slot, aggregator: bool,
validator_pubkey: ValidatorPubKey, slot_signature: ValidatorSig): validator_pubkey: ValidatorPubKey, slot_signature: ValidatorSig) -> bool:
notice "== post_v1_validator_beacon_committee_subscription" debug "post_v1_validator_beacon_committee_subscriptions"
# TODO raise newException(CatchableError, "Not implemented")

83
beacon_chain/validator_client.nim
View File

@ -22,7 +22,7 @@ import
nimbus_binary_common, nimbus_binary_common,
version, ssz/merkleization, version, ssz/merkleization,
sync_manager, keystore_management, sync_manager, keystore_management,
spec/eth2_apis/validator_callsigs_types, spec/eth2_apis/callsigs_types,
eth2_json_rpc_serialization eth2_json_rpc_serialization
logScope: topics = "vc" logScope: topics = "vc"
@ -31,6 +31,7 @@ template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
## Generate client convenience marshalling wrappers from forward declarations ## Generate client convenience marshalling wrappers from forward declarations
createRpcSigs(RpcClient, sourceDir / "spec" / "eth2_apis" / "validator_callsigs.nim") createRpcSigs(RpcClient, sourceDir / "spec" / "eth2_apis" / "validator_callsigs.nim")
createRpcSigs(RpcClient, sourceDir / "spec" / "eth2_apis" / "beacon_callsigs.nim")
type type
ValidatorClient = ref object ValidatorClient = ref object
@ -39,31 +40,66 @@ type
beaconClock: BeaconClock beaconClock: BeaconClock
attachedValidators: ValidatorPool attachedValidators: ValidatorPool
fork: Fork fork: Fork
proposalsForEpoch: Table[Slot, ValidatorPubKey] proposalsForCurrentEpoch: Table[Slot, ValidatorPubKey]
attestationsForEpoch: Table[Slot, seq[AttesterDuties]] attestationsForEpoch: Table[Epoch, Table[Slot, seq[AttesterDuties]]]
beaconGenesis: BeaconGenesisTuple beaconGenesis: BeaconGenesisTuple
proc connectToBN(vc: ValidatorClient) {.gcsafe, async.} =
while true:
try:
await vc.client.connect($vc.config.rpcAddress, Port(vc.config.rpcPort))
info "Connected to BN",
port = vc.config.rpcPort,
address = vc.config.rpcAddress
return
except CatchableError as err:
warn "Could not connect to the BN - retrying!", err = err.msg
await sleepAsync(chronos.seconds(1)) # 1 second before retrying
template attemptUntilSuccess(vc: ValidatorClient, body: untyped) =
while true:
try:
body
break
except CatchableError as err:
warn "Caught an unexpected error", err = err.msg
waitFor vc.connectToBN()
proc getValidatorDutiesForEpoch(vc: ValidatorClient, epoch: Epoch) {.gcsafe, async.} = proc getValidatorDutiesForEpoch(vc: ValidatorClient, epoch: Epoch) {.gcsafe, async.} =
let proposals = await vc.client.get_v1_validator_duties_proposer(epoch) let proposals = await vc.client.get_v1_validator_duties_proposer(epoch)
# update the block proposal duties this VC should do during this epoch # update the block proposal duties this VC should do during this epoch
vc.proposalsForEpoch.clear() vc.proposalsForCurrentEpoch.clear()
for curr in proposals: for curr in proposals:
if vc.attachedValidators.validators.contains curr.public_key: if vc.attachedValidators.validators.contains curr.public_key:
vc.proposalsForEpoch.add(curr.slot, curr.public_key) vc.proposalsForCurrentEpoch.add(curr.slot, curr.public_key)
# couldn't use mapIt in ANY shape or form so reverting to raw loops - sorry Sean Parent :| # couldn't use mapIt in ANY shape or form so reverting to raw loops - sorry Sean Parent :|
var validatorPubkeys: seq[ValidatorPubKey] var validatorPubkeys: seq[ValidatorPubKey]
for key in vc.attachedValidators.validators.keys: for key in vc.attachedValidators.validators.keys:
validatorPubkeys.add key validatorPubkeys.add key
# update the attestation duties this VC should do during this epoch
proc getAttesterDutiesForEpoch(epoch: Epoch) {.gcsafe, async.} =
let attestations = await vc.client.post_v1_validator_duties_attester( let attestations = await vc.client.post_v1_validator_duties_attester(
epoch, validatorPubkeys) epoch, validatorPubkeys)
vc.attestationsForEpoch.clear() # make sure there's an entry
if not vc.attestationsForEpoch.contains epoch:
vc.attestationsForEpoch.add(epoch, Table[Slot, seq[AttesterDuties]]())
for a in attestations: for a in attestations:
if vc.attestationsForEpoch.hasKeyOrPut(a.slot, @[a]): if vc.attestationsForEpoch[epoch].hasKeyOrPut(a.slot, @[a]):
vc.attestationsForEpoch[a.slot].add(a) vc.attestationsForEpoch[epoch][a.slot].add(a)
# obtain the attestation duties this VC should do during the next epoch
await getAttesterDutiesForEpoch(epoch + 1)
# also get the attestation duties for the current epoch if missing
if not vc.attestationsForEpoch.contains epoch:
await getAttesterDutiesForEpoch(epoch)
# cleanup old epoch attestation duties
vc.attestationsForEpoch.del(epoch - 1)
# TODO handle subscriptions to beacon committees for both the next epoch and
# for the current if missing (beacon_committee_subscriptions from the REST api)
# for now we will get the fork each time we update the validator duties for each epoch # for now we will get the fork each time we update the validator duties for each epoch
# TODO should poll occasionally `/v1/config/fork_schedule`
vc.fork = await vc.client.get_v1_beacon_states_fork("head") vc.fork = await vc.client.get_v1_beacon_states_fork("head")
proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, async.} = proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, async.} =
@ -76,6 +112,7 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
let let
slot = wallSlot.slot # afterGenesis == true! slot = wallSlot.slot # afterGenesis == true!
nextSlot = slot + 1 nextSlot = slot + 1
epoch = slot.compute_epoch_at_slot
info "Slot start", info "Slot start",
lastSlot = shortLog(lastSlot), lastSlot = shortLog(lastSlot),
@ -91,11 +128,11 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
# could take up time for attesting... Perhaps this should be called more # could take up time for attesting... Perhaps this should be called more
# than once per epoch because of forks & other events... # than once per epoch because of forks & other events...
if slot.isEpoch: if slot.isEpoch:
await getValidatorDutiesForEpoch(vc, slot.compute_epoch_at_slot) await getValidatorDutiesForEpoch(vc, epoch)
# check if we have a validator which needs to propose on this slot # check if we have a validator which needs to propose on this slot
if vc.proposalsForEpoch.contains slot: if vc.proposalsForCurrentEpoch.contains slot:
let public_key = vc.proposalsForEpoch[slot] let public_key = vc.proposalsForCurrentEpoch[slot]
let validator = vc.attachedValidators.validators[public_key] let validator = vc.attachedValidators.validators[public_key]
let randao_reveal = validator.genRandaoReveal( let randao_reveal = validator.genRandaoReveal(
@ -121,8 +158,8 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
seconds(int64(SECONDS_PER_SLOT)) div 3, slot, "Waiting to send attestations") seconds(int64(SECONDS_PER_SLOT)) div 3, slot, "Waiting to send attestations")
# check if we have validators which need to attest on this slot # check if we have validators which need to attest on this slot
if vc.attestationsForEpoch.contains slot: if vc.attestationsForEpoch[epoch].contains slot:
for a in vc.attestationsForEpoch[slot]: for a in vc.attestationsForEpoch[epoch][slot]:
let validator = vc.attachedValidators.validators[a.public_key] let validator = vc.attachedValidators.validators[a.public_key]
let ad = await vc.client.get_v1_validator_attestation_data(slot, a.committee_index) let ad = await vc.client.get_v1_validator_attestation_data(slot, a.committee_index)
@ -135,7 +172,8 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
discard await vc.client.post_v1_beacon_pool_attestations(attestation) discard await vc.client.post_v1_beacon_pool_attestations(attestation)
except CatchableError as err: except CatchableError as err:
error "Caught an unexpected error", err = err.msg warn "Caught an unexpected error", err = err.msg, slot = shortLog(slot)
await vc.connectToBN()
let let
nextSlotStart = saturate(vc.beaconClock.fromNow(nextSlot)) nextSlotStart = saturate(vc.beaconClock.fromNow(nextSlot))
@ -177,21 +215,16 @@ programMain:
var vc = ValidatorClient( var vc = ValidatorClient(
config: config, config: config,
client: newRpcHttpClient(), client: newRpcHttpClient()
attachedValidators: ValidatorPool.init()
) )
vc.proposalsForEpoch.init()
vc.attestationsForEpoch.init()
# load all the validators from the data dir into memory # load all the validators from the data dir into memory
for curr in vc.config.validatorKeys: for curr in vc.config.validatorKeys:
vc.attachedValidators.addLocalValidator(curr.toPubKey, curr) vc.attachedValidators.addLocalValidator(curr.toPubKey, curr)
# TODO perhaps we should handle the case if the BN is down and try to connect to it waitFor vc.connectToBN()
# untill success, and also later on disconnets we should continue trying to reconnect
waitFor vc.client.connect("localhost", Port(config.rpcPort)) # TODO: use config.rpcAddress
info "Connected to beacon node", port = config.rpcPort
vc.attemptUntilSuccess:
# init the beacon clock # init the beacon clock
vc.beaconGenesis = waitFor vc.client.get_v1_beacon_genesis() vc.beaconGenesis = waitFor vc.client.get_v1_beacon_genesis()
vc.beaconClock = BeaconClock.init(vc.beaconGenesis.genesis_time) vc.beaconClock = BeaconClock.init(vc.beaconGenesis.genesis_time)
@ -201,9 +234,7 @@ programMain:
nextSlot = curSlot + 1 # No earlier than GENESIS_SLOT + 1 nextSlot = curSlot + 1 # No earlier than GENESIS_SLOT + 1
fromNow = saturate(vc.beaconClock.fromNow(nextSlot)) fromNow = saturate(vc.beaconClock.fromNow(nextSlot))
# onSlotStart() requests the validator duties only on the start of each epoch vc.attemptUntilSuccess:
# so we should request the duties here when the VC binary boots up in order
# to handle the case when in the middle of an epoch. Also for the genesis slot.
waitFor vc.getValidatorDutiesForEpoch(curSlot.compute_epoch_at_slot) waitFor vc.getValidatorDutiesForEpoch(curSlot.compute_epoch_at_slot)
info "Scheduling first slot action", info "Scheduling first slot action",

49
beacon_chain/validator_duties.nim
View File

@ -19,10 +19,11 @@ import
# Local modules # Local modules
spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network], spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network],
conf, time, validator_pool, state_transition, spec/state_transition,
attestation_pool, block_pool, eth2_network, keystore_management, conf, time, validator_pool,
beacon_node_common, beacon_node_types, nimbus_binary_common, attestation_pool, block_pool, block_pools/candidate_chains, eth2_network,
mainchain_monitor, version, ssz/merkleization, interop, keystore_management, beacon_node_common, beacon_node_types,
nimbus_binary_common, mainchain_monitor, version, ssz/merkleization, interop,
attestation_aggregation, sync_manager, sszdump attestation_aggregation, sync_manager, sszdump
# Metrics for tracking attestation and beacon block loss # Metrics for tracking attestation and beacon block loss
@ -98,28 +99,45 @@ proc isSynced(node: BeaconNode, head: BlockRef): bool =
else: else:
true true
proc sendAttestation*(node: BeaconNode, attestation: Attestation) = proc sendAttestation*(
node: BeaconNode, attestation: Attestation, num_active_validators: uint64) =
logScope: pcs = "send_attestation" logScope: pcs = "send_attestation"
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md#broadcast-attestation # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/p2p-interface.md#attestations-and-aggregation
node.network.broadcast( node.network.broadcast(
getMainnetAttestationTopic(node.forkDigest, attestation.data.index), getAttestationTopic(node.forkDigest, attestation, num_active_validators),
attestation) attestation)
beacon_attestations_sent.inc() beacon_attestations_sent.inc()
proc sendAttestation*(node: BeaconNode, attestation: Attestation) =
# For the validator API, which doesn't supply num_active_validators.
let attestationBlck =
node.blockPool.getRef(attestation.data.beacon_block_root)
if attestationBlck.isNil:
debug "Attempt to send attestation without corresponding block"
return
node.blockPool.withEpochState(
node.blockPool.tmpState,
BlockSlot(blck: attestationBlck, slot: attestation.data.slot)):
node.sendAttestation(
attestation,
blck.getEpochInfo(state).shuffled_active_validator_indices.len.uint64)
proc createAndSendAttestation(node: BeaconNode, proc createAndSendAttestation(node: BeaconNode,
fork: Fork, fork: Fork,
genesis_validators_root: Eth2Digest, genesis_validators_root: Eth2Digest,
validator: AttachedValidator, validator: AttachedValidator,
attestationData: AttestationData, attestationData: AttestationData,
committeeLen: int, committeeLen: int,
indexInCommittee: int) {.async.} = indexInCommittee: int,
num_active_validators: uint64) {.async.} =
logScope: pcs = "send_attestation" logScope: pcs = "send_attestation"
var attestation = await validator.produceAndSignAttestation(attestationData, committeeLen, indexInCommittee, fork, genesis_validators_root) var attestation = await validator.produceAndSignAttestation(attestationData, committeeLen, indexInCommittee, fork, genesis_validators_root)
node.sendAttestation(attestation) node.sendAttestation(attestation, num_active_validators)
if node.config.dumpEnabled: if node.config.dumpEnabled:
dump(node.config.dumpDirOutgoing, attestation.data, validator.pubKey) dump(node.config.dumpDirOutgoing, attestation.data, validator.pubKey)
@ -311,8 +329,15 @@ proc handleAttestations(node: BeaconNode, head: BlockRef, slot: Slot) =
# the complexity of handling forks correctly - instead, we use an adapted # the complexity of handling forks correctly - instead, we use an adapted
# version here that calculates the committee for a single slot only # version here that calculates the committee for a single slot only
node.blockPool.withState(node.blockPool.tmpState, attestationHead): node.blockPool.withState(node.blockPool.tmpState, attestationHead):
var cache = get_empty_per_epoch_cache() var cache = getEpochCache(attestationHead.blck, state)
let committees_per_slot = get_committee_count_at_slot(state, slot) let
committees_per_slot = get_committee_count_at_slot(state, slot)
num_active_validators =
try:
cache.shuffled_active_validator_indices[
slot.compute_epoch_at_slot].len.uint64
except KeyError:
raiseAssert "getEpochCache(...) didn't fill cache"
for committee_index in 0'u64..<committees_per_slot: for committee_index in 0'u64..<committees_per_slot:
let committee = get_beacon_committee( let committee = get_beacon_committee(
@ -327,7 +352,7 @@ proc handleAttestations(node: BeaconNode, head: BlockRef, slot: Slot) =
for a in attestations: for a in attestations:
traceAsyncErrors createAndSendAttestation( traceAsyncErrors createAndSendAttestation(
node, state.fork, state.genesis_validators_root, a.validator, a.data, node, state.fork, state.genesis_validators_root, a.validator, a.data,
a.committeeLen, a.indexInCommittee) a.committeeLen, a.indexInCommittee, num_active_validators)
proc handleProposal(node: BeaconNode, head: BlockRef, slot: Slot): proc handleProposal(node: BeaconNode, head: BlockRef, slot: Slot):
Future[BlockRef] {.async.} = Future[BlockRef] {.async.} =

5
config.nims
View File

@ -41,6 +41,11 @@ else:
# for heap-usage-by-instance-type metrics and object base-type strings # for heap-usage-by-instance-type metrics and object base-type strings
--define:nimTypeNames --define:nimTypeNames
# switch("define", "snappy_implementation=libp2p")
const currentDir = currentSourcePath()[0 .. ^(len("config.nims") + 1)]
switch("define", "nim_compiler_path=" & currentDir & "env.sh nim")
switch("import", "testutils/moduletests") switch("import", "testutils/moduletests")
const useLibStackTrace = not defined(macosx) and const useLibStackTrace = not defined(macosx) and

7
docker/Dockerfile
View File

@ -3,7 +3,7 @@ FROM debian:bullseye-slim AS build
SHELL ["/bin/bash", "-c"] SHELL ["/bin/bash", "-c"]
RUN apt-get -qq update \ RUN apt-get -qq update \
&& apt-get -qq -y install build-essential make wget libpcre3-dev golang-go git &>/dev/null \ && apt-get -qq -y install build-essential libpcre3-dev git &>/dev/null \
&& apt-get -qq clean \ && apt-get -qq clean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
@ -11,8 +11,7 @@ RUN apt-get -qq update \
RUN cd /root \ RUN cd /root \
&& git clone https://github.com/status-im/nim-beacon-chain.git \ && git clone https://github.com/status-im/nim-beacon-chain.git \
&& cd nim-beacon-chain \ && cd nim-beacon-chain \
&& make -j$(nproc) update \ && make -j$(nproc) update
&& make deps
# Please note that the commands above have the goal of caching the # Please note that the commands above have the goal of caching the
# compilation of Nim, but don't depend on the current git revision. # compilation of Nim, but don't depend on the current git revision.
@ -47,5 +46,7 @@ COPY --from=build /root/nim-beacon-chain/build/beacon_node /usr/bin/
MAINTAINER Zahary Karadjov <zahary@status.im> MAINTAINER Zahary Karadjov <zahary@status.im>
LABEL description="Nimbus installation that can act as an ETH2 network bootstrap node." LABEL description="Nimbus installation that can act as an ETH2 network bootstrap node."
STOPSIGNAL SIGINT
ENTRYPOINT ["/usr/bin/beacon_node"] ENTRYPOINT ["/usr/bin/beacon_node"]

26
docker/manage_testnet_hosts.nims
View File

@ -1,5 +1,5 @@
import import
strformat, os, confutils, algorithm strformat, os, confutils, algorithm, sequtils
type type
Command = enum Command = enum
@ -9,15 +9,15 @@ type
CliConfig = object CliConfig = object
network: string network: string
depositsDir {.
defaultValue: "deposits"
name: "deposits-dir" }: string
case cmd {.command.}: Command case cmd {.command.}: Command
of restart_nodes: of restart_nodes:
discard discard
of reset_network: of reset_network:
depositsDir {.
defaultValue: "deposits"
name: "deposits-dir" }: string
secretsDir {. secretsDir {.
defaultValue: "secrets" defaultValue: "secrets"
name: "secrets-dir" }: string name: "secrets-dir" }: string
@ -38,9 +38,9 @@ type
var conf = load CliConfig var conf = load CliConfig
var var
serverCount = 10 serverCount = 6
instancesCount = 2 instancesCount = 1
validators = listDirs(conf.depositsDir) validators = listDirs(conf.depositsDir).mapIt(splitPath(it)[1])
sort(validators) sort(validators)
@ -115,6 +115,7 @@ of reset_network:
for n, firstValidator, lastValidator in validatorAssignments(): for n, firstValidator, lastValidator in validatorAssignments():
var var
validatorDirs = "" validatorDirs = ""
secretFiles = ""
networkDataFiles = conf.networkDataDir & "/{genesis.ssz,bootstrap_nodes.txt}" networkDataFiles = conf.networkDataDir & "/{genesis.ssz,bootstrap_nodes.txt}"
for i in firstValidator ..< lastValidator: for i in firstValidator ..< lastValidator:
@ -125,15 +126,14 @@ of reset_network:
let dockerPath = &"/docker/{n.container}/data/BeaconNode" let dockerPath = &"/docker/{n.container}/data/BeaconNode"
echo &"echo Syncing {lastValidator - firstValidator} keys starting from {firstValidator} to container {n.container}@{n.server} ... && \\" echo &"echo Syncing {lastValidator - firstValidator} keys starting from {firstValidator} to container {n.container}@{n.server} ... && \\"
echo &" ssh {n.server} 'sudo rm -rf /tmp/nimbus && mkdir -p /tmp/nimbus/{{validators,secrets}}' && \\" echo &" ssh {n.server} 'sudo rm -rf /tmp/nimbus && mkdir -p /tmp/nimbus/{{net-data,validators,secrets}}' && \\"
echo &" rsync -a -zz {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\" echo &" rsync -a -zz {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\"
if validator.len > 0: if validators.len > 0:
echo &" rsync -a -zz {validatorDirs} {n.server}:/tmp/nimbus/validators/ && \\" echo &" rsync -a -zz {validatorDirs} {n.server}:/tmp/nimbus/validators/ && \\"
echo &" rsync -a -zz {secretFiles} {n.server}:/tmp/nimbus/secrets/ && \\" echo &" rsync -a -zz {secretFiles} {n.server}:/tmp/nimbus/secrets/ && \\"
echo &" ssh {n.server} 'sudo docker container stop {n.container}; " & echo &" ssh {n.server} 'sudo docker container stop {n.container}; " &
&"sudo rm -rf {dockerPath}/{{db,validators,secrets}}* && " & &"sudo rm -rf {dockerPath}/{{db,validators,secrets,net-data}}* && " &
(if validators.len > 0: &"sudo mv /tmp/nimbus/* {dockerPath}/ && " else: "") & &"sudo mv /tmp/nimbus/* {dockerPath}/ && " &
&"sudo mv /tmp/nimbus/net-data/* {dockerPath}/ && " &
&"sudo chown dockremap:docker -R {dockerPath}'" &"sudo chown dockremap:docker -R {dockerPath}'"

17
docker/shared_testnet/Dockerfile Normal file
View File

@ -0,0 +1,17 @@
FROM debian:bullseye-slim
SHELL ["/bin/bash", "-c"]
RUN apt-get -qq update \
&& apt-get -qq -y install build-essential libpcre3-dev git &>/dev/null \
&& apt-get -qq clean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
ARG NETWORK
ENV NETWORK=${NETWORK}
STOPSIGNAL SIGINT
COPY "entry_point.sh" "/root/"
ENTRYPOINT ["/root/entry_point.sh"]

28
docker/shared_testnet/Makefile Normal file
View File

@ -0,0 +1,28 @@
SHELL := bash # the shell used internally by "make"
# These default settings can be overriden by exporting env variables
NETWORK ?= witti
IMAGE_TAG ?= testnet2
IMAGE_NAME ?= statusteam/nimbus_beacon_node:$(IMAGE_TAG)
CURRENT_BRANCH = $(shell git rev-parse --abbrev-ref HEAD)
COMPUTER_SAYS_NO = { echo "I'm sorry, Dave. I'm afraid I can't do that."; exit 1; }
.PHONY: build push push-last
build:
@ DOCKER_BUILDKIT=1 \
docker build \
--build-arg="NETWORK=$(NETWORK)" \
-t $(IMAGE_NAME) \
--progress=plain \
.
push: build
+@ $(MAKE) push-last
push-last:
@ [[ "$(CURRENT_BRANCH)" != "devel" ]] && $(COMPUTER_SAYS_NO) || true
docker push $(IMAGE_NAME)

57
docker/shared_testnet/README.md Normal file
View File

@ -0,0 +1,57 @@
## local testing
From the "nim-beacon-chain" repo (top-level dir):
```text
make -C docker/shared_testnet NETWORK=witti build
mkdir tmp
docker run --rm --mount type=bind,source="$(pwd)"/tmp,target=/root/.cache/nimbus --name testnet2 statusteam/nimbus_beacon_node:testnet2 --build
ls -l tmp/nim-beacon-chain/build
docker run --rm --mount type=bind,source="$(pwd)"/tmp,target=/root/.cache/nimbus --name testnet2 -p 127.0.0.1:8008:8008 -p 9000:9000 statusteam/nimbus_beacon_node:testnet2 --run -- --metrics-address=0.0.0.0
# from another terminal
docker ps
docker stop testnet2
# when you're happy with the Docker image:
make -C docker/shared_testnet NETWORK=witti push
```
## setting up remote servers
From the "infra-nimbus" repo:
```text
git pull
ansible-galaxy install -g -f -r ansible/requirements.yml
ansible-playbook ansible/nimbus.yml -i ansible/inventory/test -t beacon-node -u YOUR_USER -K -l nimbus-slaves[5:8]
# faster way to pull the Docker image and recreate the containers (this also stops any running container)
ansible nimbus-slaves[5:8] -i ansible/inventory/test -u YOUR_USER -o -m shell -a "echo; cd /docker/beacon-node-testnet2-1; docker-compose --compatibility pull; docker-compose --compatibility up --no-start; echo '---'" | sed 's/\\n/\n/g'
# build beacon_node in an external volume
ansible nimbus-slaves[5:8] -i ansible/inventory/test -u YOUR_USER -o -m shell -a "echo; cd /docker/beacon-node-testnet2-1; docker-compose --compatibility run --rm beacon_node --build; echo '---'" | sed 's/\\n/\n/g'
```
### create and copy validator keys
Back up "build/data/shared\_witti\_0", if you need to. It will be deleted.
From the nim-beacon-chain repo:
```bash
# If you have "ignorespace" or "ignoreboth" in HISTCONTROL in your ".bashrc", you can prevent
# the key from being stored in your command history by prefixing it with a space.
# See https://www.linuxjournal.com/content/using-bash-history-more-efficiently-histcontrol
./docker/shared_testnet/validator_keys.sh 0xYOUR_ETH1_PRIVATE_GOERLI_KEY
```
### start the containers
From the "infra-nimbus" repo:
```bash
ansible nimbus-slaves[5:8] -i ansible/inventory/test -u YOUR_USER -o -m shell -a "echo; cd /docker/beacon-node-testnet2-1; docker-compose --compatibility up -d; echo '---'" | sed 's/\\n/\n/g'
```

32
docker/shared_testnet/altona_validators.sh Executable file
View File

@ -0,0 +1,32 @@
#!/bin/bash
set -e
cd "$(dirname "${BASH_SOURCE[0]}")/../.."
GROUP=0
TOTAL=$(ls -d ../nimbus-private/altona_deposits/validators/* | wc -l)
#echo "TOTAL=${TOTAL}"
PER_GROUP=$((TOTAL / 4))
#echo "PER_GROUP=${PER_GROUP}"
for N in $(seq 6 9); do
ssh node-0${N}.aws-eu-central-1a.nimbus.test.statusim.net "sudo rm -rf /docker/beacon-node-testnet2-1/data/nim-beacon-chain/build/data/shared_altona_0/secrets"
ssh node-0${N}.aws-eu-central-1a.nimbus.test.statusim.net "sudo rm -rf /docker/beacon-node-testnet2-1/data/nim-beacon-chain/build/data/shared_altona_0/validators"
#echo GROUP="${GROUP}"
for TARGET in "validators" "secrets"; do
DIR_NO=0
ls -d ../nimbus-private/altona_deposits/${TARGET}/* | while read DIR; do
if [[ $DIR_NO -ge $((GROUP * PER_GROUP)) && $DIR_NO -lt $(( (GROUP + 1) * PER_GROUP )) ]]; then
#echo "DIR_NO=${DIR_NO}"
#echo "$DIR"
rsync -a -zz --rsync-path="sudo rsync" "$DIR" node-0${N}.aws-eu-central-1a.nimbus.test.statusim.net:/docker/beacon-node-testnet2-1/data/nim-beacon-chain/build/data/shared_altona_0/${TARGET}/
fi
DIR_NO=$((DIR_NO + 1))
done
done
GROUP=$((GROUP + 1))
ssh node-0${N}.aws-eu-central-1a.nimbus.test.statusim.net "sudo chown -R dockremap:dockremap /docker/beacon-node-testnet2-1/data/nim-beacon-chain/build/data/shared_altona_0/secrets"
ssh node-0${N}.aws-eu-central-1a.nimbus.test.statusim.net "sudo chown -R dockremap:dockremap /docker/beacon-node-testnet2-1/data/nim-beacon-chain/build/data/shared_altona_0/validators"
done

106
docker/shared_testnet/entry_point.sh Executable file
View File

@ -0,0 +1,106 @@
#!/bin/bash
set -e
cd "$(dirname "${BASH_SOURCE[0]}")"
####################
# argument parsing #
####################
! getopt --test > /dev/null
if [ ${PIPESTATUS[0]} != 4 ]; then
echo '`getopt --test` failed in this environment.'
exit 1
fi
OPTS="h"
LONGOPTS="help,network:,build,run"
# default values
NETWORK="altona"
BUILD=0
RUN=0
print_help() {
cat <<EOF
Usage: $(basename $0) <options> -- <beacon_node options>
-h, --help this help message
--network default: ${NETWORK}
--build build the beacon_node
--run run the beacon_node
EOF
}
! PARSED=$(getopt --options=${OPTS} --longoptions=${LONGOPTS} --name "$0" -- "$@")
if [ ${PIPESTATUS[0]} != 0 ]; then
# getopt has complained about wrong arguments to stdout
exit 1
fi
# read getopt's output this way to handle the quoting right
eval set -- "$PARSED"
while true; do
case "$1" in
-h|--help)
print_help
exit
;;
--network)
NETWORK="$2"
shift 2
;;
--build)
BUILD=1
shift
;;
--run)
RUN=1
shift
;;
--)
shift
break
;;
*)
echo "argument parsing error"
print_help
exit 1
esac
done
# docker-compose.yml inserts newlines in our options
if [[ "$(echo $1 | tr -d '[:space:]')" == "--" ]]; then
shift
fi
EXTRA_ARGS="$@"
#########
# build #
#########
if [[ "$BUILD" == "1" ]]; then
# "/root/.cache/nimbus" is the external bind-mounted dir, preserved between runs
cd /root/.cache/nimbus
[[ -d nim-beacon-chain ]] || git clone https://github.com/status-im/nim-beacon-chain.git
cd nim-beacon-chain
git config pull.rebase false
git checkout devel
git pull
# don't use too much RAM
make update
make LOG_LEVEL="TRACE" NIMFLAGS="-d:insecure -d:testnet_servers_image --parallelBuild:1" SCRIPT_PARAMS="--skipGoerliKey --writeLogFile=false --buildOnly" ${NETWORK}
fi
#######
# run #
#######
if [[ "$RUN" == "1" ]]; then
cd /root/.cache/nimbus/nim-beacon-chain
echo $(make SCRIPT_PARAMS="--skipGoerliKey --writeLogFile=false --runOnly --printCmdOnly" ${NETWORK} | tail -n 1) ${EXTRA_ARGS}
# make sure Docker's SIGINT reaches the beacon_node binary
eval $(make SCRIPT_PARAMS="--skipGoerliKey --writeLogFile=false --runOnly --printCmdOnly" ${NETWORK} | tail -n 1) ${EXTRA_ARGS}
fi

31
docker/shared_testnet/validator_keys.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/bash
# This script creates validator keys and uploads them to remote servers,
# assuming your local username is the same as the remote one.
set -e
cd "$(dirname "${BASH_SOURCE[0]}")/../.."
[[ -z "$1" ]] && { echo "Usage: $(basename $0) YOUR_ETH1_PRIVATE_GOERLI_KEY"; exit 1; }
# TODO: make "witti" a parameter
echo -ne "About to delete \"build/data/shared_witti_0\".\nMake a backup, if you need to, then press Enter. >"
read TMP
make clean-witti
for N in $(seq 6 9); do
make SCRIPT_PARAMS="--becomeValidatorOnly --privateGoerliKey=$1" witti && \
ssh node-0${N}.aws-eu-central-1a.nimbus.test.statusim.net "sudo rm -rf /docker/beacon-node-testnet2-1/data/nim-beacon-chain/build/data/shared_witti_0/secrets" && \
ssh node-0${N}.aws-eu-central-1a.nimbus.test.statusim.net "sudo rm -rf /docker/beacon-node-testnet2-1/data/nim-beacon-chain/build/data/shared_witti_0/validators" && \
rsync -a -zz --rsync-path="sudo rsync" build/data/shared_witti_0/{secrets,validators} node-0${N}.aws-eu-central-1a.nimbus.test.statusim.net:/docker/beacon-node-testnet2-1/data/nim-beacon-chain/build/data/shared_witti_0/ && \
ssh node-0${N}.aws-eu-central-1a.nimbus.test.statusim.net "sudo chown -R dockremap:dockremap /docker/beacon-node-testnet2-1/data/nim-beacon-chain/build/data/shared_witti_0/secrets" && \
ssh node-0${N}.aws-eu-central-1a.nimbus.test.statusim.net "sudo chown -R dockremap:dockremap /docker/beacon-node-testnet2-1/data/nim-beacon-chain/build/data/shared_witti_0/validators"
rm -rf build/data/shared_witti_0/{secrets,validators}
# if we're doing it too fast, we get {"code":-32000,"message":"replacement transaction underpriced"}
# or {"code":-32000,"message":"nonce too low"}
echo "Sleeping..."
sleep 120
done

5
docs/src/SUMMARY.md
View File

@ -1,10 +1,9 @@
# Summary # Summary
- [Introduction](./intro.md) - [Introduction](./intro.md)
- [What is Beacon Chain?](./beacon-chain.md)
- [What is Nimbus?](./nimbus.md)
- [Become a Validator](./validator.md)
- [Installation](./install.md) - [Installation](./install.md)
- [Become a Validator](./validator.md)
- [Command-line Options](./cli.md)
- [API](./api.md) - [API](./api.md)
- [Advanced Usage for Developers](./advanced.md) - [Advanced Usage for Developers](./advanced.md)
- [FAQs](./faq.md) - [FAQs](./faq.md)

18
docs/src/advanced.md
View File

@ -8,7 +8,7 @@ The following sections explain how to setup your build environment on your platf
Install Mingw-w64 for your architecture using the "[MinGW-W64 Online Install Mingw-w64 for your architecture using the "[MinGW-W64 Online
Installer](https://sourceforge.net/projects/mingw-w64/files/)" (first link Installer](https://sourceforge.net/projects/mingw-w64/files/)" (first link
under the directory listing). Run it and select your architecture in the setup under the directory listing). Run it and select your architecture in the setup
menu ("i686" on 32-bit, "x86\_64" on 64-bit), set the threads to "win32" and menu (`i686` on 32-bit, `x86_64` on 64-bit), set the threads to `win32` and
the exceptions to "dwarf" on 32-bit and "seh" on 64-bit. Change the the exceptions to "dwarf" on 32-bit and "seh" on 64-bit. Change the
installation directory to "C:\mingw-w64" and add it to your system PATH in "My installation directory to "C:\mingw-w64" and add it to your system PATH in "My
Computer"/"This PC" -> Properties -> Advanced system settings -> Environment Computer"/"This PC" -> Properties -> Advanced system settings -> Environment
@ -17,6 +17,7 @@ Variables -> Path -> Edit -> New -> C:\mingw-w64\mingw64\bin (it's "C:\mingw-w64
Install [Git for Windows](https://gitforwindows.org/) and use a "Git Bash" shell to clone and build nim-beacon-chain. Install [Git for Windows](https://gitforwindows.org/) and use a "Git Bash" shell to clone and build nim-beacon-chain.
If you don't want to compile PCRE separately, you can fetch pre-compiled DLLs with: If you don't want to compile PCRE separately, you can fetch pre-compiled DLLs with:
```bash ```bash
mingw32-make # this first invocation will update the Git submodules mingw32-make # this first invocation will update the Git submodules
mingw32-make fetch-dlls # this will place the right DLLs for your architecture in the "build/" directory mingw32-make fetch-dlls # this will place the right DLLs for your architecture in the "build/" directory
@ -30,14 +31,13 @@ You can now follow those instructions in the previous section by replacing `make
mingw32-make test # run the test suite mingw32-make test # run the test suite
``` ```
### Linux, MacOS ### Linux, macOS
After cloning the repo: After cloning the repo:
```bash ```bash
make # The first `make` invocation will update all Git submodules and prompt you to run `make` again. # Build beacon_node and all the tools, using 4 parallel Make jobs
# It's only required once per Git clone. You'll run `make update` after each `git pull`, in the future, make -j4
# to keep those submodules up to date.
# Run tests # Run tests
make test make test
@ -48,6 +48,7 @@ make update
``` ```
To run a command that might use binaries from the Status Nim fork: To run a command that might use binaries from the Status Nim fork:
```bash ```bash
./env.sh bash # start a new interactive shell with the right env vars set ./env.sh bash # start a new interactive shell with the right env vars set
which nim which nim
@ -62,9 +63,9 @@ nim --version # Nimbus is tested and supported on 1.0.2 at the moment
We recommend you remove any cover or use a fan; the Raspberry Pi will get hot (85°C) and throttle. We recommend you remove any cover or use a fan; the Raspberry Pi will get hot (85°C) and throttle.
* Raspberry PI 3b+ or Raspberry Pi 4b. - Raspberry PI 3b+ or Raspberry Pi 4b.
* 64gb SD Card (less might work too, but the default recommended 4-8GB will probably be too small) - 64gb SD Card (less might work too, but the default recommended 4-8GB will probably be too small)
* [Rasbian Buster Lite](https://www.raspberrypi.org/downloads/raspbian/) - Lite version is enough to get going and will save some disk space! - [Raspbian Buster Lite](https://www.raspberrypi.org/downloads/raspbian/) - Lite version is enough to get going and will save some disk space!
Assuming you're working with a freshly written image: Assuming you're working with a freshly written image:
@ -129,4 +130,3 @@ make -j$(nproc) NIMFLAGS="-d:release" USE_MULTITAIL=yes eth2_network_simulation
```bash ```bash
make USE_LIBBACKTRACE=0 # expect the resulting binaries to be 2-3 times slower make USE_LIBBACKTRACE=0 # expect the resulting binaries to be 2-3 times slower
``` ```

83
docs/src/api.md
View File

@ -1 +1,82 @@
# API
NBC exposes API:s for querying the state of the application at runtime.
:note: Where applicable, this API mimics https://github.com/ethereum/eth2.0-APIs with the exception that JSON-RPC is used instead of http rest - method names, parameters and results are equivalent except for the encoding / access method.
## Introduction
The NBC API is implemented using JSON-RPC 2.0. To query it, you can use a JSON-RPC library in the language of your choice, or a tool like `curl` to access it from the command line. A tool like [jq](https://stedolan.github.io/jq/) is helpful to pretty-print the responses.
```
curl -d '{"jsonrpc":"2.0","id":"id","method":"peers","params":[] }' -H 'Content-Type: application/json' localhost:9190 -s | jq
```
Before you can access the API, make sure it's enabled using the RPC flag (`beacon_node --rpc`):
```
--rpc Enable the JSON-RPC server.
--rpc-port HTTP port for the JSON-RPC service.
--rpc-address Listening address of the RPC server.
```
## Beacon Node API
### getBeaconHead
The latest head slot, as chosen by the latest fork choice.
```
curl -d '{"jsonrpc":"2.0","id":"id","method":"getBeaconHead","params":[] }' -H 'Content-Type: application/json' localhost:9190 -s | jq
```
### getChainHead
Show chain head information, including head, justified and finalized checkpoints.
```
curl -d '{"jsonrpc":"2.0","id":"id","method":"getChainHead","params":[] }' -H 'Content-Type: application/json' localhost:9190 -s | jq
```
### getSyncing
### getBeaconBlock
### getBeaconState
### getNetworkPeerId
### getNetworkPeers
### getNetworkEnr
```
curl -d '{"jsonrpc":"2.0","id":"id","method":"getNetworkEnr","params":[] }' -H 'Content-Type: application/json' localhost:9190 -s | jq
```
## Valdiator API
## Administrative / Debug API
### getNodeVersion
Show version of the software
```
curl -d '{"jsonrpc":"2.0","id":"id","method":"getNodeVersion","params":[] }' -H 'Content-Type: application/json' localhost:9190 -s | jq
```
### getSpecPreset
Show spec constants in use.
```
curl -d '{"jsonrpc":"2.0","id":"id","method":"getSpecPreset","params":[] }' -H 'Content-Type: application/json' localhost:9190 -s | jq
```
### peers
Show a list of peers that the beacon node is connected to.
```
curl -d '{"jsonrpc":"2.0","id":"id","method":"peers","params":[] }' -H 'Content-Type: application/json' localhost:9190 -s | jq
```

18
docs/src/beacon-chain.md
View File

@ -1,18 +0,0 @@
# What is Beacon Chain?
A complete introduction about the beacon chain can be found in the [Ethereum 2.0 blog series](https://our.status.im/two-point-oh-the-beacon-chain/).
In short, the beacon chain is a **new type of blockchain** to help the Ethereum blockchain to smoothly transfer its consensus algorithm from PoW (Proof of Work) to PoS (Proof of Stake), aka Ethereum 2.0. You can also see it as a hybrid PoS + PoW solution.
## Differences Compared to Ethereum 1.0
In traditional PoW, those that propose new blocks are called **_miners_**, whereas in PoS, they are called **_validators_**. In essence, _miners_ rely on actual hardware (such as some specifically manufactured mining machines), while _validators_ rely on just software.
## What it is Like to Be a Validator?
It is obvious that you must have enough computing power or dedicated hardware in order to be a miner, but how about being a validator? Here is a brief overview:
1. A special smart contract named **_deposit contract_** is deployed on the original Ethereum blockchain. Note that in this case, the new beacon chain and the original blockchain co-exists.
2. To "register" as a validator, you have to first deposit **_32 Ether_** from your account to this smart contract.
3. Run the beacon node and wait for the network to sync before your validator is activated.
4. That's all! Remember to stay connected to the network, or you may lose your deposit. :P

109
docs/src/cli.md Normal file
View File

@ -0,0 +1,109 @@
# Command-line Options
You can run your customized beacon node using the `beacon_node` executable. The available options are shown below - you can also run `beacon_node --help` for a reminder.
## Prerequisites
Specifying a genesis file is mandatory to run this executable. You can either get it from the official eth2 repository [here](https://github.com/eth2-clients/eth2-testnets/blob/master/shared/witti/genesis.ssz) or generate your own like [this](https://github.com/status-im/nim-beacon-chain/blob/db92c2f2549a339be60896c3907cefdb394b5e11/scripts/launch_local_testnet.sh#L154) when starting a local testnet. You can also specify the path of your genesis file like [this](https://github.com/status-im/nim-beacon-chain/blob/db92c2f2549a339be60896c3907cefdb394b5e11/scripts/launch_local_testnet.sh#L229).
For example, download a genesis file and then run the following command to start the node:
<img src="./img/beacon_node_example.PNG" alt="" style="margin: 0 40 0 40"/>
## Usage
```
$ ./build/beacon_node --help
Nimbus beacon node v0.3.0 (e537ed9
)
Copyright (c) 2019-2020 Status Research & Development GmbH
Nim Compiler Version 1.3.1 [Windows: amd64]
Compiled at 2020-04-16
Copyright (c) 2006-2020 by Andreas Rumpf
git hash: b4e9f8e814373fc38741736197d88475663ce758
active boot switches: -d:release
Usage:
beacon_node [OPTIONS]... command
The following options are available:
--log-level Sets the log level.
--eth1-network The Eth1 network tracked by the beacon node.
--quick-start Run in quickstart mode
-d, --data-dir The directory where nimbus will store all blockchain data.
--web3-url URL of the Web3 server to observe Eth1.
--deposit-contract Address of the deposit contract.
-b, --bootstrap-node Specifies one or more bootstrap nodes to use when connecting to the
network.
--bootstrap-file Specifies a line-delimited file of bootstrap Ethereum network addresses.
--listen-address Listening address for the Ethereum LibP2P traffic.
--tcp-port Listening TCP port for Ethereum LibP2P traffic.
--udp-port Listening UDP port for node discovery.
--max-peers The maximum number of peers to connect to
--nat Specify method to use for determining public address. Must be one of: any,
none, upnp, pmp, extip:<IP>.
-v, --validator Path to a validator private key, as generated by makeDeposits.
-s, --state-snapshot Json file specifying a recent state snapshot.
--node-name A name for this node that will appear in the logs. If you set this to
'auto', a persistent automatically generated ID will be selected for each
--dataDir folder.
--verify-finalization Specify whether to verify finalization occurs on schedule, for testing.
--stop-at-epoch A positive epoch selects the epoch at which to stop.
--metrics Enable the metrics server.
--metrics-address Listening address of the metrics server.
--metrics-port Listening HTTP port of the metrics server.
--status-bar Display a status bar at the bottom of the terminal screen.
--status-bar-contents Textual template for the contents of the status bar.
--rpc Enable the JSON-RPC server
--rpc-port HTTP port for the JSON-RPC service.
--rpc-address Listening address of the RPC server
--dump Write SSZ dumps of blocks, attestations and states to data dir
Available sub-commands:
beacon_node importValidator [OPTIONS]...
The following options are available:
--keyfile File with validator key to be imported (in hex form).
beacon_node createTestnet [OPTIONS]...
The following options are available:
-d, --validators-dir Directory containing validator descriptors named 'vXXXXXXX.deposit.json'.
--total-validators The number of validators in the newly created chain.
--first-validator Index of first validator to add to validator list.
--last-user-validator The last validator index that will free for taking from a testnet
participant.
--bootstrap-address The public IP address that will be advertised as a bootstrap node for the
testnet.
--bootstrap-port The TCP/UDP port that will be used by the bootstrap node.
-g, --genesis-offset Seconds from now to add to genesis time.
--output-genesis Output file where to write the initial state snapshot.
--with-genesis-root Include a genesis root in 'network.json'.
--output-bootstrap-file Output file with list of bootstrap nodes for the network.
beacon_node makeDeposits [OPTIONS]...
The following options are available:
--quickstart-deposits Number of quick-start deposits to generate.
--random-deposits Number of secure random deposits to generate.
--deposits-dir Folder to write deposits to.
--deposit-private-key Private key of the controlling (sending) account
--min-delay Minimum possible delay between making two deposits (in seconds)
--max-delay Maximum possible delay between making two deposits (in seconds)
beacon_node query command
Available sub-commands:
beacon_node query get <getQueryPath>
<getQueryPath> REST API path to evaluate
```

25
docs/src/contribute.md
View File

@ -2,7 +2,6 @@
Follow these steps to contribute to this book! Follow these steps to contribute to this book!
We use an utility tool called mdBook to create online books from Markdown files. We use an utility tool called mdBook to create online books from Markdown files.
## Before You Start ## Before You Start
@ -18,7 +17,27 @@ We use an utility tool called mdBook to create online books from Markdown files.
## Build and Deploy ## Build and Deploy
1. `mdbook build` The first step is to submit a pull request to the [devel branch](https://github.com/status-im/nim-beacon-chain/tree/devel).
2. `make publish-book` Then, after it is merged, do the following under our main repository:
1. `cd nim-beacon-chain`
2. `git checkout devel`
3. `git pull`
4. `make update` (This is to update the submodules to the latest version)
5. `make publish-book`
## Trouble Shooting
If you see file conflicts in the pull request, this may due to that you have created your new branch from an old version of the `devel` branch. Update your new branch using the following commands:
```
git checkout devel
git pull
make update
git checkout readme
git merge devel
# use something like `git mergetool` to resolve conflicts, then read the instructions for completing the merge (usually just a `git commit`)
# check the output of `git diff devel`
```
Thank you so much for your help to the decentralized and open source community. :) Thank you so much for your help to the decentralized and open source community. :)

34
docs/src/faq.md
View File

@ -1 +1,35 @@
# Frequently Asked Questions # Frequently Asked Questions
## 1. What is Beacon Chain?
A complete introduction about the beacon chain can be found in the [Ethereum 2.0 blog series](https://our.status.im/two-point-oh-the-beacon-chain/).
In short, the beacon chain is a **new type of blockchain** to help the Ethereum blockchain to smoothly transfer its consensus algorithm from PoW (Proof of Work) to PoS (Proof of Stake), aka Ethereum 2.0.
## 2. Differences Between Beacon Chain and Ethereum 1.0
In traditional PoW, those that propose new blocks are called **_miners_**, whereas in PoS, they are called **_validators_**. In essence, _miners_ rely on actual hardware (such as some specifically manufactured mining machines), while _validators_ rely on just software and a good network connection.
## 3. What it is Like to Be a Validator?
It is obvious that you must have enough computing power or dedicated hardware in order to be a miner, but how about being a validator? Here is a brief overview:
1. A special smart contract named **_deposit contract_** is deployed on the original Ethereum blockchain. Note that in this case, the new beacon chain and the original blockchain co-exists.
2. To "register" as a validator, you have to first deposit **_32 Ether_** from your account to this smart contract.
3. Run the beacon node and wait for the network to sync before your validator is activated.
4. That's all! Remember to stay connected to the network, or you may lose some of your deposit, as punishment, depending on how long you're offline. :P
## 4. What is Nimbus?
In a sentence, Nimbus is an Ethereum 1.0 & 2.0 Client for Resource-Restricted Devices.
It is open sourced at [github.com/status-im/nimbus](github.com/status-im/nimbus). Development progress and updates can be viewed at the [Nimbus blog](https://our.status.im/tag/nimbus/).
## Why are metrics not working?
Metrics are currently implemented using a HTTP server that hasn't been hardened sufficiently that it can be exposed as a public endpoint - it must thus be enabled specifically during build:
```
make NIMFLAGS="-d:insecure"
beacon_node --metrics ...
```

BIN
docs/src/img/beacon_node_example.PNG Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 98 KiB

17
docs/src/install.md
View File

@ -1,15 +1,18 @@
# Installation # Installation
Nimbus beacon chain can run on Linux, MacOS, Windows, and Andriod. At the moment, Nimbus has to be built from source. Beacon chain can run on Linux, macOS, Windows, and Android. At the moment, Nimbus has to be built from source.
## External Dependencies ## External Dependencies
- Developer tools (C compiler, Make, Bash, Git) - Developer tools (C compiler, Make, Bash, Git)
- PCRE - PCRE
Nim is not an external dependency, Nimbus will build its own local copy. Nim is not an external dependency, Nimbus will build its own local copy.
## Linux ## Linux
On common Linux distributions the dependencies can be installed with: On common Linux distributions the dependencies can be installed with:
```sh ```sh
# Debian and Ubuntu # Debian and Ubuntu
sudo apt-get install build-essential git libpcre3-dev sudo apt-get install build-essential git libpcre3-dev
@ -21,7 +24,7 @@ dnf install @development-tools pcre
yourAURmanager -S base-devel pcre-static yourAURmanager -S base-devel pcre-static
``` ```
### MacOS ### macOS
Assuming you use [Homebrew](https://brew.sh/) to manage packages Assuming you use [Homebrew](https://brew.sh/) to manage packages
@ -36,12 +39,16 @@ It also provides a downloading script for prebuilt PCRE.
### Android ### Android
* Install the [Termux](https://termux.com) app from FDroid or the Google Play store - Install the [Termux](https://termux.com) app from FDroid or the Google Play store
* Install a [PRoot](https://wiki.termux.com/wiki/PRoot) of your choice following the instructions for your preferred distribution. - Install a [PRoot](https://wiki.termux.com/wiki/PRoot) of your choice following the instructions for your preferred distribution.
Note, the Ubuntu PRoot is known to contain all Nimbus prerequisites compiled on Arm64 architecture (common architecture for Android devices). Note, the Ubuntu PRoot is known to contain all Nimbus prerequisites compiled on Arm64 architecture (common architecture for Android devices).
*Assuming Ubuntu PRoot is used* _Assuming Ubuntu PRoot is used_
```sh ```sh
apt install build-essential git libpcre3-dev apt install build-essential git libpcre3-dev
``` ```
## Next steps
Once you've installed build tools, you're ready to move on to launching the beacon node and becoming a [validator](./validator.md)

21
docs/src/intro.md
View File

@ -1,23 +1,24 @@
# Nimbus Beacon Chain Book # The nim-beacon-chain Book
_Documentation for Nimbus Beacon Chain users and developers._ _Documentation for Nimbus Beacon Chain users and developers._
Nimbus beacon chain is a research implementation of the beacon chain component of the upcoming Ethereum Serenity upgrade, aka Eth2. Nimbus beacon chain is a research implementation of the beacon chain component of the upcoming Ethereum Serenity upgrade, aka Eth2.
- Open sourced at [github.com/status-im/nim-beacon-chain/docs](github.com/status-im/nim-beacon-chain/docs). - Open sourced at [github.com/status-im/nim-beacon-chain](https://github.com/status-im/nim-beacon-chain/tree/master).
- Specification of our implementation can be found at [ethereum/eth2.0-specs](https://github.com/ethereum/eth2.0-specs/tree/v0.11.1#phase-0). - Specification of our implementation can be found at [ethereum/eth2.0-specs](https://github.com/ethereum/eth2.0-specs/tree/v0.12.1#phase-0).
## Overview ## Overview
In this book, we will cover: In this book, we will cover:
1. [What is beacon chain](./beacon-chain.md) and [what is Nimbus](./nimbus.md) to equip you with some basic knowledge. 1. [What is beacon chain](./faq.md#1-what-is-beacon-chain) and [what is Nimbus](./faq.md#4-what-is-nimbus) to equip you with some basic knowledge.
2. How to [become a validator](./validator.md) in Ethereum as a user. 2. [Installation steps](./install.md) outline the prerequisites to get started.
3. [Installation steps](./install.md) for nimbus beacon chain. 3. How to [become a validator](./validator.md) in Ethereum 2.0 as a user, for example on the Altona testnet.
4. The [api documentation](./api.md) for interested developers. 4. [CLI](./cli.md) for running your customized nimbus beacon node.
5. [Advanced usage](./advanced.md) for developers. 5. [API](./api.md) for monitoring your node through `http`.
6. Common [questions and answers](./faq.md) to satisfy your curiosity. 6. [Advanced usage](./advanced.md) for developers.
7. How to [contribute](./contribute.md) to this book. 7. Common [questions and answers](./faq.md) to satisfy your curiosity.
8. How to [contribute](./contribute.md) to this book.
Feel free to give us feedback on how to improve as well as contribute to our book on github. :) Feel free to give us feedback on how to improve as well as contribute to our book on github. :)

6
docs/src/nimbus.md
View File

@ -1,6 +0,0 @@
# What is Nimbus?
In a sentence, Nimbus is an Ethereum 1.0 & 2.0 Client for Resource-Restricted Devices.
It is open sourced at [github.com/status-im/nimbus](github.com/status-im/nimbus). Development progress and updates can be viewed at the [Nimbus blog](https://our.status.im/tag/nimbus/).
## Why should you choose Nimbus?

66
docs/src/validator.md
View File

@ -1,26 +1,29 @@
# Become a Validator # Become a Validator
To become a validator, you have to first connect to a testnet, deposit your Ethers, and sync with the network. To become a validator, you need to install the beacon chain software, acquire 32 ETH, set up your validator account and register with the deposit contract on Ethereum.
There is currently no Eth2 mainnet - all networks are testnets.
## Recommended Testnets ## Recommended Testnets
Though Nimbus can connect to any of the testnets published in the [eth2-clients/eth2-testnets repo](https://github.com/eth2-clients/eth2-testnets/tree/master/nimbus), below are the recommended ones: Though Nimbus can connect to any of the testnets published in the [eth2-clients/eth2-testnets repo](https://github.com/eth2-clients/eth2-testnets/tree/master/nimbus), below are the recommended ones:
- Public Testnet: [witti](https://github.com/goerli/witti) ([explorer](https://witti.beaconcha.in)) - Multi-client Testnet: [altona](https://github.com/goerli/altona) ([explorer](https://altona.beaconcha.in))
- Local Testnet: testnet0 - Nimbus Testnet: testnet0 (experimental, not always active)
## Connecting to Testnets ## Altona
Before we start, we have to obtain 32 Ethers on the Goerli testnet. Then, we can deposit 32 Ethers to the registration smart contract to become a validator. ### Initial setup
1. Open your [MetaMask](https://metamask.io/) wallet, switch to the `Goerli Test Network` option from the top right cornor. Before we start, we have to obtain 32 ETH on the Goerli testnet. Then, we can deposit 32 Ethers to the registration smart contract to become a validator.
1. Open your [MetaMask](https://metamask.io/) wallet, switch to the `Goerli Test Network` option from the top right corner.
2. Copy your account address by clicking on one of your accounts. 2. Copy your account address by clicking on one of your accounts.
3. Post your account address on a social media platform (Twitter or Facebook). Copy the url to the post. 3. Post your account address on a social media platform (Twitter or Facebook). Copy the url to the post.
4. Paste your post url on the [Goerli faucet](https://faucet.goerli.mudit.blog/) and select `Give me Ether > 37.5 Ethers` from the top right cornor of the page. 4. Paste your post url on the [Goerli faucet](https://faucet.goerli.mudit.blog/) and select `Give me Ether > 37.5 Ethers` from the top right cornor of the page.
5. Wait for a few seconds and return to your MetaMask wallet to check if you have successfully received. 5. Wait for a few seconds and return to your MetaMask wallet to check if you have successfully received.
6. Once the [prerequisites](./install.md) are installed, you can connect to testnet0 with the following commands: <br> 6. Once the [prerequisites](./install.md) are installed, you can connect to the altona testnet with the following commands: <br>
- Change `testnet0` to `witti` to connect to the witti testnet.
- **_Remember to replace `make` with `mingw32-make` if using Windows._** - **_Remember to replace `make` with `mingw32-make` if using Windows._**
```bash ```bash
@ -29,13 +32,13 @@ cd nim-beacon-chain
git checkout devel git checkout devel
git pull git pull
make update make update
make testnet0 # This will build Nimbus and all other dependencies make altona # This will build Nimbus and all other dependencies
# and connect you to testnet0 # and connect you to altona
``` ```
<img src="./img/connect_testnet.PNG" alt="" style="margin: 0 40 0 40"/> <img src="./img/connect_testnet.PNG" alt="" style="margin: 0 40 0 40"/>
7. The testnet should now be up and running. Then, you will be prompted to enter your private key of the account you want to deposit the 32 Ether from. Find your private key from MetaMask as below: 7. You will be prompted to enter your private key of the account you want to deposit the 32 Ether from. Find your private key from MetaMask as below:
<img src="./img/export_pkey.PNG" alt="" width="200" style="margin: 0 40 0 40"/> <img src="./img/export_pkey.PNG" alt="" width="200" style="margin: 0 40 0 40"/>
@ -45,16 +48,47 @@ make testnet0 # This will build Nimbus and all other dependencies
<img src="./img/deposit_sent.PNG" alt="" style="margin: 0 40 0 40"/> <img src="./img/deposit_sent.PNG" alt="" style="margin: 0 40 0 40"/>
9. Now you should be syncing with the network. It may take a while (may be quite a few hours). You can know that you are synced if you see the following output. 9. The beacon chain client will start syncing the network while your deposit is being processed. As soon as the deposit has been added, the client will start performing validation duties.
<img src="./img/success.PNG" alt="" style="margin: 0 40 0 40"/> <img src="./img/success.PNG" alt="" style="margin: 0 40 0 40"/>
You can also get a brief estimate of the time remaining until your network gets synced by comparing the output `epoch` value and the one in the blockchain explorer (the [witti explorer](https://witti.beaconcha.in) for example). You can also get a brief estimate of the time remaining until your network gets synced by comparing the output `epoch` value and the one in the blockchain explorer (the [altona explorer](https://altona.beaconcha.in) for example).
## Trouble Shooting ### Upgrading
1. The directory that stores the blockchain data of the testnet is `build/data/testnet0` (replace `testnet0` with other testnet names). Delete this folder if you want to start over. For example, you can start over with a fresh storage if you entered a wrong private key. When restarting the beacon node, the software will resume from where it left off, using your previous deposits.
```
cd nim-beacon-chain
git pull
make update # Update dependencies
make altona # Restart using same keys as last run
```
## Key management
Keys are stored in the `build/data/testnet_name/` folder, under `secrets` and `validators` - make sure to keep these folders backed up.
## Metrics
Metrics are not included in the binary by default - to enable them, use the following options when starting the client:
```
make NIMFLAGS="-d:insecure" altona
```
You can now browse the metrics using a browser and connecting to:
http://localhost:8008/metrics
Make sure to protect this port as the http server used is not considered secure and should not be used by untrusted peers.
## Troubleshooting
1. The directory that stores the blockchain data of the testnet is `build/data/shared_altona_0` (replace `altona` with other testnet names). Delete this folder if you want to start over. For example, you can start over with a fresh storage if you entered a wrong private key.
2. Currently, you have to switch to the devel branch in order to run the validator node successfully. 2. Currently, you have to switch to the devel branch in order to run the validator node successfully.
3. Everytime you want to update your node to the latest version, run `git pull`, `make update`, and then `make testnet0`. 3. Everytime you want to update your node to the latest version, run `git pull`, `make update`, and then `make altona`.
4. If `make update` has been running for too long, you can use `make update V=1` or `make update V=2` for verbose output.

17
env.sh
View File

@ -4,5 +4,22 @@
# and we fall back to a Zsh-specific special var to also support Zsh. # and we fall back to a Zsh-specific special var to also support Zsh.
REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})" REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
ABS_PATH="$(cd ${REL_PATH}; pwd)" ABS_PATH="$(cd ${REL_PATH}; pwd)"
# Activate nvm only when this file is sourced without arguments:
if [ -z "$*" ]; then
if command -v nvm > /dev/null; then
nvm use
command -v ganache-cli > /dev/null || { npm install -g ganache-cli; }
else
echo <<EOF
In order to use Ganache (a development ETH1 chain), please install NVM with:
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
For more info:
https://github.com/nvm-sh/nvm
EOF
fi
fi
source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh

18
grafana/beacon_nodes_Grafana_dashboard.json
View File

@ -12,7 +12,7 @@
}, },
{ {
"datasource": "Prometheus", "datasource": "Prometheus",
"enable": true, "enable": false,
"expr": "changes(beacon_current_epoch{node=\"0\"}[2s])", "expr": "changes(beacon_current_epoch{node=\"0\"}[2s])",
"hide": false, "hide": false,
"iconColor": "#FFA6B0", "iconColor": "#FFA6B0",
@ -44,7 +44,8 @@
"editable": true, "editable": true,
"gnetId": null, "gnetId": null,
"graphTooltip": 0, "graphTooltip": 0,
"id": 13, "id": 23,
"iteration": 1593300421721,
"links": [], "links": [],
"panels": [ "panels": [
{ {
@ -210,14 +211,10 @@
"steppedLine": false, "steppedLine": false,
"targets": [ "targets": [
{ {
"expr": "libp2p_open_bufferstream{node=\"${node}\"}", "expr": "libp2p_open_streams{node=\"${node}\"}",
"legendFormat": "BufferStream", "interval": "",
"legendFormat": "{{type}}",
"refId": "A" "refId": "A"
},
{
"expr": "libp2p_open_connection{node=\"${node}\"}",
"legendFormat": "Connection",
"refId": "B"
} }
], ],
"thresholds": [], "thresholds": [],
@ -1742,7 +1739,6 @@
{ {
"allValue": null, "allValue": null,
"current": { "current": {
"tags": [],
"text": "0", "text": "0",
"value": "0" "value": "0"
}, },
@ -1792,5 +1788,5 @@
"variables": { "variables": {
"list": [] "list": []
}, },
"version": 38 "version": 3
} }

6
nbench/scenarios.nim
View File

@ -9,12 +9,12 @@ import
# Standard library # Standard library
os, tables, os, tables,
# Status libraries # Status libraries
confutils/defs, serialization, confutils/defs, serialization, chronicles,
# Beacon-chain # Beacon-chain
../beacon_chain/spec/[ ../beacon_chain/spec/[
datatypes, crypto, helpers, beaconstate, validator, datatypes, crypto, helpers, beaconstate, validator,
state_transition_block, state_transition_epoch], state_transition_block, state_transition_epoch, state_transition],
../beacon_chain/[state_transition, extras], ../beacon_chain/extras,
../beacon_chain/ssz/[merkleization, ssz_serialization] ../beacon_chain/ssz/[merkleization, ssz_serialization]
# Nimbus Bench - Scenario configuration # Nimbus Bench - Scenario configuration

48
ncli/ncli_db.nim
View File

@ -1,9 +1,9 @@
import import
confutils, stats, chronicles, strformat, tables, confutils, stats, chronicles, strformat, tables,
stew/byteutils, stew/byteutils,
../beacon_chain/[beacon_chain_db, block_pool, extras, state_transition], ../beacon_chain/[beacon_chain_db, block_pool, extras],
../beacon_chain/spec/[crypto, datatypes, digest, helpers], ../beacon_chain/spec/[crypto, datatypes, digest, helpers, state_transition, validator],
../beacon_chain/sszdump, ../beacon_chain/sszdump, ../beacon_chain/ssz/merkleization,
../research/simutils, ../research/simutils,
eth/db/[kvstore, kvstore_sqlite3] eth/db/[kvstore, kvstore_sqlite3]
@ -18,6 +18,7 @@ type
DbCmd* = enum DbCmd* = enum
bench bench
dumpState dumpState
dumpBlock
rewindState rewindState
DbConf = object DbConf = object
@ -32,15 +33,20 @@ type
.}: DbCmd .}: DbCmd
of bench: of bench:
validate* {. slots* {.
defaultValue: true defaultValue: 50000
desc: "Enable BLS validation" }: bool desc: "Number of slots to run benchmark for".}: uint64
of dumpState: of dumpState:
stateRoot* {. stateRoot* {.
argument argument
desc: "State roots to save".}: seq[string] desc: "State roots to save".}: seq[string]
of dumpBlock:
blockRootx* {.
argument
desc: "Block roots to save".}: seq[string]
of rewindState: of rewindState:
blockRoot* {. blockRoot* {.
argument argument
@ -70,7 +76,7 @@ proc cmdBench(conf: DbConf) =
var var
blockRefs: seq[BlockRef] blockRefs: seq[BlockRef]
blocks: seq[SignedBeaconBlock] blocks: seq[TrustedSignedBeaconBlock]
cur = pool.head.blck cur = pool.head.blck
while cur != nil: while cur != nil:
@ -78,6 +84,9 @@ proc cmdBench(conf: DbConf) =
cur = cur.parent cur = cur.parent
for b in 1..<blockRefs.len: # Skip genesis block for b in 1..<blockRefs.len: # Skip genesis block
if blockRefs[blockRefs.len - b - 1].slot > conf.slots:
break
withTimer(timers[tLoadBlock]): withTimer(timers[tLoadBlock]):
blocks.add db.getBlock(blockRefs[blockRefs.len - b - 1].root).get() blocks.add db.getBlock(blockRefs[blockRefs.len - b - 1].root).get()
@ -88,15 +97,17 @@ proc cmdBench(conf: DbConf) =
withTimer(timers[tLoadState]): withTimer(timers[tLoadState]):
discard db.getState(state[].root, state[].data, noRollback) discard db.getState(state[].root, state[].data, noRollback)
let flags = if conf.validate: {} else: {skipBlsValidation}
for b in blocks: for b in blocks:
let let
isEpoch = state[].data.slot.compute_epoch_at_slot != isEpoch = state[].data.slot.compute_epoch_at_slot !=
b.message.slot.compute_epoch_at_slot b.message.slot.compute_epoch_at_slot
withTimer(timers[if isEpoch: tApplyEpochBlock else: tApplyBlock]): withTimer(timers[if isEpoch: tApplyEpochBlock else: tApplyBlock]):
discard state_transition(state[], b, flags, noRollback) if not state_transition(state[], b, {}, noRollback):
dump("./", b, hash_tree_root(b.message))
echo "State transition failed (!)"
quit 1
printTimers(conf.validate, timers) printTimers(false, timers)
proc cmdDumpState(conf: DbConf) = proc cmdDumpState(conf: DbConf) =
let let
@ -114,6 +125,21 @@ proc cmdDumpState(conf: DbConf) =
except CatchableError as e: except CatchableError as e:
echo "Couldn't load ", stateRoot, ": ", e.msg echo "Couldn't load ", stateRoot, ": ", e.msg
proc cmdDumpBlock(conf: DbConf) =
let
db = BeaconChainDB.init(
kvStore SqStoreRef.init(conf.databaseDir.string, "nbc").tryGet())
for blockRoot in conf.blockRootx:
try:
let root = Eth2Digest(data: hexToByteArray[32](blockRoot))
if (let blck = db.getBlock(root); blck.isSome):
dump("./", blck.get(), root)
else:
echo "Couldn't load ", root
except CatchableError as e:
echo "Couldn't load ", blockRoot, ": ", e.msg
proc cmdRewindState(conf: DbConf) = proc cmdRewindState(conf: DbConf) =
echo "Opening database..." echo "Opening database..."
let let
@ -145,5 +171,7 @@ when isMainModule:
cmdBench(conf) cmdBench(conf)
of dumpState: of dumpState:
cmdDumpState(conf) cmdDumpState(conf)
of dumpBlock:
cmdDumpBlock(conf)
of rewindState: of rewindState:
cmdRewindState(conf) cmdRewindState(conf)

8
ncli/ncli_transition.nim
View File

@ -1,16 +1,16 @@
import import
confutils, chronicles, confutils, chronicles,
../beacon_chain/spec/[crypto, datatypes], ../beacon_chain/spec/[crypto, datatypes, state_transition],
../beacon_chain/[extras, state_transition], ../beacon_chain/extras,
../beacon_chain/ssz/[merkleization, ssz_serialization] ../beacon_chain/ssz/[merkleization, ssz_serialization]
cli do(pre: string, blck: string, post: string, verifyStateRoot = false): cli do(pre: string, blck: string, post: string, verifyStateRoot = true):
let let
stateY = (ref HashedBeaconState)( stateY = (ref HashedBeaconState)(
data: SSZ.loadFile(pre, BeaconState), data: SSZ.loadFile(pre, BeaconState),
) )
blckX = SSZ.loadFile(blck, SignedBeaconBlock) blckX = SSZ.loadFile(blck, SignedBeaconBlock)
flags = if verifyStateRoot: {skipStateRootValidation} else: {} flags = if not verifyStateRoot: {skipStateRootValidation} else: {}
stateY.root = hash_tree_root(stateY.data) stateY.root = hash_tree_root(stateY.data)

62
nfuzz/libnfuzz.nim
View File

@ -2,10 +2,10 @@
# https://github.com/nim-lang/Nim/issues/11225 # https://github.com/nim-lang/Nim/issues/11225
import import
stew/ptrops, stew/ranges/ptr_arith, stew/ptrops, stew/ranges/ptr_arith, chronicles,
../beacon_chain/[extras, state_transition], ../beacon_chain/extras,
../beacon_chain/spec/[crypto, datatypes, digest, validator, beaconstate, ../beacon_chain/spec/[crypto, datatypes, digest, validator, beaconstate,
state_transition_block], state_transition_block, state_transition],
../beacon_chain/ssz/[merkleization, ssz_serialization] ../beacon_chain/ssz/[merkleization, ssz_serialization]
type type
@ -33,8 +33,8 @@ type
FuzzCrashError = object of CatchableError FuzzCrashError = object of CatchableError
# TODO: change ptr uint to ptr csize_t when available in newer Nim version. # TODO: change ptr uint to ptr csize_t when available in newer Nim version.
proc copyState(state: BeaconState, output: ptr byte, proc copyState(state: BeaconState, xoutput: ptr byte,
output_size: ptr uint): bool {.raises: [FuzzCrashError, Defect].} = xoutput_size: ptr uint): bool {.raises: [FuzzCrashError, Defect].} =
var resultState = var resultState =
try: try:
SSZ.encode(state) SSZ.encode(state)
@ -42,18 +42,18 @@ proc copyState(state: BeaconState, output: ptr byte,
# Shouldn't occur as the writer isn't a file # Shouldn't occur as the writer isn't a file
raise newException(FuzzCrashError, "Unexpected failure to serialize.", e) raise newException(FuzzCrashError, "Unexpected failure to serialize.", e)
if unlikely(resultState.len.uint > output_size[]): if unlikely(resultState.len.uint > xoutput_size[]):
let msg = ( let msg = (
"Not enough output buffer provided to nimbus harness. Provided: " & "Not enough xoutput buffer provided to nimbus harness. Provided: " &
$(output_size[]) & $(xoutput_size[]) &
"Required: " & "Required: " &
$resultState.len.uint $resultState.len.uint
) )
raise newException(FuzzCrashError, msg) raise newException(FuzzCrashError, msg)
output_size[] = resultState.len.uint xoutput_size[] = resultState.len.uint
# TODO: improvement might be to write directly to buffer with OutputStream # TODO: improvement might be to write directly to buffer with xoutputStream
# and SszWriter (but then need to ensure length doesn't overflow) # and SszWriter (but then need to ensure length doesn't overflow)
copyMem(output, unsafeAddr resultState[0], output_size[]) copyMem(xoutput, unsafeAddr resultState[0], xoutput_size[])
result = true result = true
template decodeAndProcess(typ, process: untyped): bool = template decodeAndProcess(typ, process: untyped): bool =
@ -90,22 +90,22 @@ template decodeAndProcess(typ, process: untyped): bool =
raise newException(FuzzCrashError, "Unexpected Exception in state transition", e) raise newException(FuzzCrashError, "Unexpected Exception in state transition", e)
if processOk: if processOk:
copyState(data.state, output, output_size) copyState(data.state, xoutput, xoutput_size)
else: else:
false false
proc nfuzz_attestation(input: openArray[byte], output: ptr byte, proc nfuzz_attestation(input: openArray[byte], xoutput: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(AttestationInput): decodeAndProcess(AttestationInput):
process_attestation(data.state, data.attestation, flags, cache) process_attestation(data.state, data.attestation, flags, cache)
proc nfuzz_attester_slashing(input: openArray[byte], output: ptr byte, proc nfuzz_attester_slashing(input: openArray[byte], xoutput: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(AttesterSlashingInput): decodeAndProcess(AttesterSlashingInput):
process_attester_slashing(data.state, data.attesterSlashing, flags, cache) process_attester_slashing(data.state, data.attesterSlashing, flags, cache)
proc nfuzz_block(input: openArray[byte], output: ptr byte, proc nfuzz_block(input: openArray[byte], xoutput: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
# There's not a perfect approach here, but it's not worth switching the rest # There's not a perfect approach here, but it's not worth switching the rest
# and requiring HashedBeaconState (yet). So to keep consistent, puts wrapper # and requiring HashedBeaconState (yet). So to keep consistent, puts wrapper
# only in one function. # only in one function.
@ -120,35 +120,35 @@ proc nfuzz_block(input: openArray[byte], output: ptr byte,
decodeAndProcess(BlockInput): decodeAndProcess(BlockInput):
state_transition(data, data.beaconBlock, flags, noRollback) state_transition(data, data.beaconBlock, flags, noRollback)
proc nfuzz_block_header(input: openArray[byte], output: ptr byte, proc nfuzz_block_header(input: openArray[byte], xoutput: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(BlockHeaderInput): decodeAndProcess(BlockHeaderInput):
process_block_header(data.state, data.beaconBlock.message, flags, cache) process_block_header(data.state, data.beaconBlock.message, flags, cache)
proc nfuzz_deposit(input: openArray[byte], output: ptr byte, proc nfuzz_deposit(input: openArray[byte], xoutput: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(DepositInput): decodeAndProcess(DepositInput):
process_deposit(data.state, data.deposit, flags) process_deposit(data.state, data.deposit, flags)
proc nfuzz_proposer_slashing(input: openArray[byte], output: ptr byte, proc nfuzz_proposer_slashing(input: openArray[byte], xoutput: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(ProposerSlashingInput): decodeAndProcess(ProposerSlashingInput):
process_proposer_slashing(data.state, data.proposerSlashing, flags, cache) process_proposer_slashing(data.state, data.proposerSlashing, flags, cache)
proc nfuzz_voluntary_exit(input: openArray[byte], output: ptr byte, proc nfuzz_voluntary_exit(input: openArray[byte], xoutput: ptr byte,
output_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(VoluntaryExitInput): decodeAndProcess(VoluntaryExitInput):
process_voluntary_exit(data.state, data.exit, flags) process_voluntary_exit(data.state, data.exit, flags)
# Note: Could also accept raw input pointer and access list_size + seed here. # Note: Could also accept raw input pointer and access list_size + seed here.
# However, list_size needs to be known also outside this proc to allocate output. # However, list_size needs to be known also outside this proc to allocate xoutput.
# TODO: rework to copy immediatly in an uint8 openArray, considering we have to # TODO: rework to copy immediatly in an uint8 openArray, considering we have to
# go over the list anyhow? # go over the list anyhow?
proc nfuzz_shuffle(input_seed: ptr byte, output: var openArray[uint64]): bool proc nfuzz_shuffle(input_seed: ptr byte, xoutput: var openArray[uint64]): bool
{.exportc, raises: [Defect].} = {.exportc, raises: [Defect].} =
var seed: Eth2Digest var seed: Eth2Digest
# Should be OK as max 2 bytes are passed by the framework. # Should be OK as max 2 bytes are passed by the framework.
let list_size = output.len.uint64 let list_size = xoutput.len.uint64
copyMem(addr(seed.data), input_seed, sizeof(seed.data)) copyMem(addr(seed.data), input_seed, sizeof(seed.data))
@ -162,8 +162,8 @@ proc nfuzz_shuffle(input_seed: ptr byte, output: var openArray[uint64]): bool
for i in 0..<list_size: for i in 0..<list_size:
# ValidatorIndex is currently wrongly uint32 so we copy this 1 by 1, # ValidatorIndex is currently wrongly uint32 so we copy this 1 by 1,
# assumes passed output is zeroed. # assumes passed xoutput is zeroed.
copyMem(offset(addr output, i.int), shuffled_seq[i.int].unsafeAddr, copyMem(offset(addr xoutput, i.int), shuffled_seq[i.int].unsafeAddr,
sizeof(ValidatorIndex)) sizeof(ValidatorIndex))
result = true result = true

5
research/block_sim.nim
View File

@ -20,10 +20,11 @@ import
options, random, tables, options, random, tables,
../tests/[testblockutil], ../tests/[testblockutil],
../beacon_chain/spec/[ ../beacon_chain/spec/[
beaconstate, crypto, datatypes, digest, helpers, validator, signatures], beaconstate, crypto, datatypes, digest, helpers, validator, signatures,
state_transition],
../beacon_chain/[ ../beacon_chain/[
attestation_pool, block_pool, beacon_node_types, beacon_chain_db, attestation_pool, block_pool, beacon_node_types, beacon_chain_db,
interop, state_transition, validator_pool], interop, validator_pool],
eth/db/[kvstore, kvstore_sqlite3], eth/db/[kvstore, kvstore_sqlite3],
../beacon_chain/ssz/[merkleization, ssz_serialization], ../beacon_chain/ssz/[merkleization, ssz_serialization],
./simutils ./simutils

242
scripts/connect_to_testnet.nims
View File

@ -6,10 +6,11 @@ const
bootstrapTxtFileName = "bootstrap_nodes.txt" bootstrapTxtFileName = "bootstrap_nodes.txt"
bootstrapYamlFileName = "boot_enr.yaml" bootstrapYamlFileName = "boot_enr.yaml"
depositContractFileName = "deposit_contract.txt" depositContractFileName = "deposit_contract.txt"
depositContractBlockFileName = "deposit_contract_block.txt"
genesisFile = "genesis.ssz" genesisFile = "genesis.ssz"
configFile = "config.yaml" configFile = "config.yaml"
testnetsRepo = "eth2-testnets" testnetsRepo = "eth2-testnets"
web3Url = "wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a" web3Url = "wss://goerli.infura.io/ws/v3/6224f3c792cc443fafb64e70a98f871e"
let let
testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-clients") testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-clients")
@ -23,9 +24,108 @@ proc validateTestnetName(parts: openarray[string]): auto =
quit 1 quit 1
(parts[0], parts[1]) (parts[0], parts[1])
# reduces the error output when interrupting an external command with Ctrl+C
proc execIgnoringExitCode(s: string) =
try:
exec s
except OsError:
discard
proc updateTestnetsRepo(allTestnetsDir, buildDir: string) =
rmDir(allTestnetsDir)
let cwd = system.getCurrentDir()
cd buildDir
exec &"git clone --quiet --depth=1 {testnetsGitUrl}"
cd cwd
proc makePrometheusConfig(nodeID, baseMetricsPort: int, dataDir: string) =
# macOS may not have gnu-getopts installed and in the PATH
execIgnoringExitCode &"""./scripts/make_prometheus_config.sh --nodes """ & $(1 + nodeID) & &""" --base-metrics-port {baseMetricsPort} --config-file "{dataDir}/prometheus.yml""""
proc buildNode(nimFlags, preset, beaconNodeBinary: string) =
exec &"""nim c {nimFlags} -d:"const_preset={preset}" -o:"{beaconNodeBinary}" beacon_chain/beacon_node.nim"""
proc becomeValidator(validatorsDir, beaconNodeBinary, secretsDir, depositContractOpt, privateGoerliKey: string,
becomeValidatorOnly: bool) =
mode = Silent
var privKey = privateGoerliKey
if privKey.len == 0:
echo "\nPlease enter your Goerli Eth1 private key in hex form (e.g. 0x1a2...f3c) in order to become a validator (you'll need access to 32 GoETH)."
echo "Hit Enter to skip this."
# is there no other way to print without a trailing newline?
exec "printf '> '"
privKey = readLineFromStdin()
if privKey.len > 0:
mkDir validatorsDir
mode = Verbose
exec replace(&"""{beaconNodeBinary} deposits create
--count=1
--out-deposits-dir="{validatorsDir}"
--out-secrets-dir="{secretsDir}"
--deposit-private-key={privKey}
--web3-url={web3Url}
{depositContractOpt}
""", "\n", " ")
mode = Silent
if becomeValidatorOnly:
echo "\nDeposit sent."
else:
echo "\nDeposit sent, wait for confirmation then press enter to continue"
discard readLineFromStdin()
proc runNode(dataDir, beaconNodeBinary, bootstrapFileOpt, depositContractOpt,
genesisFileOpt, natConfig: string,
basePort, nodeID, baseMetricsPort, baseRpcPort: int,
printCmdOnly: bool) =
let logLevel = getEnv("LOG_LEVEL")
var logLevelOpt = ""
if logLevel.len > 0:
logLevelOpt = &"""--log-level="{logLevel}" """
mode = Verbose
var cmd: string
if printCmdOnly:
# When you reinvent getopt() and you forget to support repeating the same
# option to overwrite the old value...
cmd = replace(&"""{beaconNodeBinary}
--data-dir="{dataDir}"
--web3-url={web3Url}
{bootstrapFileOpt}
{depositContractOpt}
{genesisFileOpt} """, "\n", " ")
echo &"cd {dataDir}; exec {cmd}"
else:
cd dataDir
cmd = replace(&"""{beaconNodeBinary}
--data-dir="{dataDir}"
--dump
--web3-url={web3Url}
--nat={natConfig}
--tcp-port=""" & $(basePort + nodeID) & &"""
--udp-port=""" & $(basePort + nodeID) & &"""
--metrics
--metrics-port=""" & $(baseMetricsPort + nodeID) & &"""
--rpc
--rpc-port=""" & $(baseRpcPort + nodeID) & &"""
{bootstrapFileOpt}
{logLevelOpt}
{depositContractOpt}
{genesisFileOpt} """, "\n", " ")
execIgnoringExitCode cmd
cli do (skipGoerliKey {. cli do (skipGoerliKey {.
desc: "Don't prompt for an Eth1 Goerli key to become a validator" .}: bool, desc: "Don't prompt for an Eth1 Goerli key to become a validator" .}: bool,
privateGoerliKey {.
desc: "Use this private Eth1 Goerli key to become a validator (careful with this option, the private key will end up in your shell's command history)" .} = "",
specVersion {.
desc: "Spec version"
name: "spec" .} = "v0.11.3",
constPreset {. constPreset {.
desc: "The Ethereum 2.0 const preset of the network (optional)" desc: "The Ethereum 2.0 const preset of the network (optional)"
name: "const-preset" .} = "", name: "const-preset" .} = "",
@ -42,6 +142,26 @@ cli do (skipGoerliKey {.
baseRpcPort {. baseRpcPort {.
desc: "Base rpc port (nodeID will be added to it)" .} = 9190.int, desc: "Base rpc port (nodeID will be added to it)" .} = 9190.int,
natConfig {.
desc: "Specify method to use for determining public address. " &
"Must be one of: any, none, upnp, pmp, extip:<IP>",
name: "nat" .} = "any",
writeLogFile {.
desc: "Write a log file in dataDir" .} = true,
buildOnly {.
desc: "Just the build, please." .} = false,
becomeValidatorOnly {.
desc: "Just become a validator." .} = false,
runOnly {.
desc: "Just run it." .} = false,
printCmdOnly {.
desc: "Just print the commands (suitable for passing to 'eval'; might replace current shell)." .} = false,
testnetName {.argument .}: string): testnetName {.argument .}: string):
let let
nameParts = testnetName.split "/" nameParts = testnetName.split "/"
@ -52,27 +172,38 @@ cli do (skipGoerliKey {.
buildDir = rootDir / "build" buildDir = rootDir / "build"
allTestnetsDir = buildDir / testnetsRepo allTestnetsDir = buildDir / testnetsRepo
rmDir(allTestnetsDir) if not (runOnly or becomeValidatorOnly):
cd buildDir updateTestnetsRepo(allTestnetsDir, buildDir)
exec &"git clone --quiet --depth=1 {testnetsGitUrl}"
var var
depositContractOpt = "" depositContractOpt = ""
bootstrapFileOpt = "" bootstrapFileOpt = ""
genesisFileOpt = ""
doBuild, doBecomeValidator, doRun = true
# step-skipping logic
if skipGoerliKey:
doBecomeValidator = false
if buildOnly:
doBecomeValidator = false
doRun = false
if becomeValidatorOnly:
doBuild = false
doRun = false
if runOnly:
doBuild = false
doBecomeValidator = false
let
testnetDir = allTestnetsDir / team / testnet
genesisFilePath = testnetDir / genesisFile
let testnetDir = allTestnetsDir / team / testnet
if not system.dirExists(testnetDir): if not system.dirExists(testnetDir):
echo &"No metadata files exists for the '{testnetName}' testnet" echo &"No metadata files exists for the '{testnetName}' testnet"
quit 1 quit 1
proc checkRequiredFile(fileName: string) = if system.fileExists(genesisFilePath):
let filePath = testnetDir / fileName genesisFileOpt = &"--state-snapshot=\"{genesisFilePath}\""
if not system.fileExists(filePath):
echo &"The required file {fileName} is not present in '{testnetDir}'."
quit 1
checkRequiredFile genesisFile
let bootstrapTxtFile = testnetDir / bootstrapTxtFileName let bootstrapTxtFile = testnetDir / bootstrapTxtFileName
if system.fileExists(bootstrapTxtFile): if system.fileExists(bootstrapTxtFile):
@ -80,7 +211,7 @@ cli do (skipGoerliKey {.
else: else:
let bootstrapYamlFile = testnetDir / bootstrapYamlFileName let bootstrapYamlFile = testnetDir / bootstrapYamlFileName
if system.fileExists(bootstrapYamlFile): if system.fileExists(bootstrapYamlFile):
bootstrapFileOpt = &"--enr-bootstrap-file=\"{bootstrapYamlFile}\"" bootstrapFileOpt = &"--bootstrap-file=\"{bootstrapYamlFile}\""
else: else:
echo "Warning: the network metadata doesn't include a bootstrap file" echo "Warning: the network metadata doesn't include a bootstrap file"
@ -89,6 +220,8 @@ cli do (skipGoerliKey {.
preset = constPreset preset = constPreset
if preset.len == 0: preset = "minimal" if preset.len == 0: preset = "minimal"
doAssert specVersion in ["v0.11.3", "v0.12.1"]
let let
dataDirName = testnetName.replace("/", "_") dataDirName = testnetName.replace("/", "_")
.replace("(", "_") .replace("(", "_")
@ -98,8 +231,9 @@ cli do (skipGoerliKey {.
secretsDir = dataDir / "secrets" secretsDir = dataDir / "secrets"
beaconNodeBinary = buildDir / "beacon_node_" & dataDirName beaconNodeBinary = buildDir / "beacon_node_" & dataDirName
var var
nimFlags = "-d:chronicles_log_level=TRACE " & getEnv("NIM_PARAMS") nimFlags = &"-d:chronicles_log_level=TRACE " & getEnv("NIM_PARAMS")
if writeLogFile:
# write the logs to a file # write the logs to a file
nimFlags.add """ -d:"chronicles_sinks=textlines,json[file(nbc""" & staticExec("date +\"%Y%m%d%H%M%S\"") & """.log)]" """ nimFlags.add """ -d:"chronicles_sinks=textlines,json[file(nbc""" & staticExec("date +\"%Y%m%d%H%M%S\"") & """.log)]" """
@ -107,75 +241,21 @@ cli do (skipGoerliKey {.
if system.fileExists(depositContractFile): if system.fileExists(depositContractFile):
depositContractOpt = "--deposit-contract=" & readFile(depositContractFile).strip depositContractOpt = "--deposit-contract=" & readFile(depositContractFile).strip
if system.dirExists(dataDir): let depositContractBlockFile = testnetDir / depositContractBlockFileName
block resetDataDir: if system.fileExists(depositContractBlockFile):
# We reset the testnet data dir if the existing data dir is depositContractOpt.add " --deposit-contract-block=" & readFile(depositContractBlockFile).strip
# incomplete (it misses a genesis file) or if it has a genesis
# file from an older testnet:
if system.fileExists(dataDir/genesisFile):
let localGenesisContent = readFile(dataDir/genesisFile)
let testnetGenesisContent = readFile(testnetDir/genesisFile)
if localGenesisContent == testnetGenesisContent:
break
echo "Detected testnet restart. Deleting previous database..."
rmDir dataDir
proc execIgnoringExitCode(s: string) =
# reduces the error output when interrupting an external command with Ctrl+C
try:
exec s
except OsError:
discard
cd rootDir cd rootDir
mkDir dataDir mkDir dataDir
# macOS may not have gnu-getopts installed and in the PATH if doBuild:
execIgnoringExitCode &"""./scripts/make_prometheus_config.sh --nodes """ & $(1 + nodeID) & &""" --base-metrics-port {baseMetricsPort} --config-file "{dataDir}/prometheus.yml"""" makePrometheusConfig(nodeID, baseMetricsPort, dataDir)
buildNode(nimFlags, preset, beaconNodeBinary)
exec &"""nim c {nimFlags} -d:"const_preset={preset}" -o:"{beaconNodeBinary}" beacon_chain/beacon_node.nim""" if doBecomeValidator and depositContractOpt.len > 0 and not system.dirExists(validatorsDir):
becomeValidator(validatorsDir, beaconNodeBinary, secretsDir, depositContractOpt, privateGoerliKey, becomeValidatorOnly)
if not skipGoerliKey and depositContractOpt.len > 0 and not system.dirExists(validatorsDir):
mode = Silent
echo "\nPlease enter your Goerli Eth1 private key in hex form (e.g. 0x1a2...f3c) in order to become a validator (you'll need access to 32 GoETH)."
echo "Hit Enter to skip this."
# is there no other way to print without a trailing newline?
exec "printf '> '"
let privKey = readLineFromStdin()
if privKey.len > 0:
mkDir validatorsDir
mode = Verbose
exec replace(&"""{beaconNodeBinary} makeDeposits
--count=1
--out-validators-dir="{validatorsDir}"
--out-secrets-dir="{secretsDir}"
--deposit-private-key={privKey}
--web3-url={web3Url}
{depositContractOpt}
""", "\n", " ")
mode = Silent
echo "\nDeposit sent, wait for confirmation then press enter to continue"
discard readLineFromStdin()
let logLevel = getEnv("LOG_LEVEL")
var logLevelOpt = ""
if logLevel.len > 0:
logLevelOpt = &"""--log-level="{logLevel}" """
mode = Verbose
cd dataDir
execIgnoringExitCode replace(&"""{beaconNodeBinary}
--data-dir="{dataDir}"
--dump
--web3-url={web3Url}
--tcp-port=""" & $(basePort + nodeID) & &"""
--udp-port=""" & $(basePort + nodeID) & &"""
--metrics
--metrics-port=""" & $(baseMetricsPort + nodeID) & &"""
--rpc
--rpc-port=""" & $(baseRpcPort + nodeID) & &"""
{bootstrapFileOpt}
{logLevelOpt}
{depositContractOpt}
--state-snapshot="{testnetDir/genesisFile}" """, "\n", " ")
if doRun:
runNode(dataDir, beaconNodeBinary, bootstrapFileOpt, depositContractOpt,
genesisFileOpt, natConfig, basePort, nodeID, baseMetricsPort,
baseRpcPort, printCmdOnly)

73
scripts/launch_local_testnet.sh
View File

@ -23,14 +23,15 @@ if [ ${PIPESTATUS[0]} != 4 ]; then
exit 1 exit 1
fi fi
OPTS="ht:n:d:" OPTS="hgt:n:d:"
LONGOPTS="help,testnet:,nodes:,data-dir:,disable-htop,log-level:,base-port:,base-metrics-port:" LONGOPTS="help,testnet:,nodes:,data-dir:,disable-htop,log-level:,base-port:,base-metrics-port:,with-ganache"
# default values # default values
TESTNET="1" TESTNET="1"
NUM_NODES="10" NUM_NODES="10"
DATA_DIR="local_testnet_data" DATA_DIR="local_testnet_data"
USE_HTOP="1" USE_HTOP="1"
USE_GANACHE="0"
LOG_LEVEL="DEBUG" LOG_LEVEL="DEBUG"
BASE_PORT="9000" BASE_PORT="9000"
BASE_METRICS_PORT="8008" BASE_METRICS_PORT="8008"
@ -44,6 +45,7 @@ CI run: $(basename $0) --disable-htop -- --verify-finalization --stop-at-epoch=5
-h, --help this help message -h, --help this help message
-t, --testnet testnet number (default: ${TESTNET}) -t, --testnet testnet number (default: ${TESTNET})
-n, --nodes number of nodes to launch (default: ${NUM_NODES}) -n, --nodes number of nodes to launch (default: ${NUM_NODES})
-g, --with-ganache simulate a genesis event based on a deposit contract
-d, --data-dir directory where all the node data and logs will end up -d, --data-dir directory where all the node data and logs will end up
(default: "${DATA_DIR}") (default: "${DATA_DIR}")
--base-port bootstrap node's Eth2 traffic port (default: ${BASE_PORT}) --base-port bootstrap node's Eth2 traffic port (default: ${BASE_PORT})
@ -83,6 +85,10 @@ while true; do
USE_HTOP="0" USE_HTOP="0"
shift shift
;; ;;
-g|--with-ganache)
USE_GANACHE="1"
shift
;;
--log-level) --log-level)
LOG_LEVEL="$2" LOG_LEVEL="$2"
shift 2 shift 2
@ -138,14 +144,22 @@ fi
NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh ${NETWORK}) NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh ${NETWORK})
$MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node $MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
./build/beacon_node makeDeposits \ PIDS=""
WEB3_ARG=""
DEPOSIT_CONTRACT_ARG=""
STATE_SNAPSHOT_ARG=""
BOOTSTRAP_TIMEOUT=10 # in seconds
./build/beacon_node deposits create \
--count=${TOTAL_VALIDATORS} \ --count=${TOTAL_VALIDATORS} \
--out-validators-dir="${DEPOSITS_DIR}" \ --out-deposits-dir="${DEPOSITS_DIR}" \
--out-secrets-dir="${SECRETS_DIR}" --out-secrets-dir="${SECRETS_DIR}" \
--dont-send
if [[ $USE_GANACHE == "0" ]]; then
GENESIS_OFFSET=30 GENESIS_OFFSET=30
BOOTSTRAP_IP="127.0.0.1" BOOTSTRAP_IP="127.0.0.1"
./build/beacon_node createTestnet \ ./build/beacon_node createTestnet \
--data-dir="${DATA_DIR}/node0" \ --data-dir="${DATA_DIR}/node0" \
--validators-dir="${DEPOSITS_DIR}" \ --validators-dir="${DEPOSITS_DIR}" \
@ -157,10 +171,39 @@ BOOTSTRAP_IP="127.0.0.1"
--bootstrap-port=${BASE_PORT} \ --bootstrap-port=${BASE_PORT} \
--genesis-offset=${GENESIS_OFFSET} # Delay in seconds --genesis-offset=${GENESIS_OFFSET} # Delay in seconds
STATE_SNAPSHOT_ARG="--state-snapshot=${NETWORK_DIR}/genesis.ssz"
else
make deposit_contract
echo "Launching ganache"
ganache-cli --blockTime 17 --gasLimit 100000000 -e 100000 --verbose > "${DATA_DIR}/log_ganache.txt" 2>&1 &
PIDS="${PIDS},$!"
echo "Deploying deposit contract"
WEB3_ARG="--web3-url=ws://localhost:8545"
DEPOSIT_CONTRACT_ADDRESS=$(./build/deposit_contract deploy $WEB3_ARG)
DEPOSIT_CONTRACT_ARG="--deposit-contract=$DEPOSIT_CONTRACT_ADDRESS"
MIN_DELAY=1
MAX_DELAY=5
BOOTSTRAP_TIMEOUT=$(( MAX_DELAY * TOTAL_VALIDATORS ))
./build/beacon_node deposits send \
--non-interactive \
--deposits-dir="${DEPOSITS_DIR}" \
--min-delay=$MIN_DELAY --max-delay=$MAX_DELAY \
$WEB3_ARG \
$DEPOSIT_CONTRACT_ARG > "${DATA_DIR}/log_deposit_maker.txt" 2>&1 &
PIDS="${PIDS},$!"
fi
./scripts/make_prometheus_config.sh \ ./scripts/make_prometheus_config.sh \
--nodes ${NUM_NODES} \ --nodes ${NUM_NODES} \
--base-metrics-port ${BASE_METRICS_PORT} \ --base-metrics-port ${BASE_METRICS_PORT} \
--config-file "${DATA_DIR}/prometheus.yml" --config-file "${DATA_DIR}/prometheus.yml" || true # TODO: this currently fails on macOS,
# but it can be considered non-critical
# Kill child processes on Ctrl-C/SIGTERM/exit, passing the PID of this shell # Kill child processes on Ctrl-C/SIGTERM/exit, passing the PID of this shell
# instance as the parent and the target process name as a pattern to the # instance as the parent and the target process name as a pattern to the
@ -181,20 +224,20 @@ dump_logs() {
done done
} }
PIDS=""
NODES_WITH_VALIDATORS=${NODES_WITH_VALIDATORS:-4} NODES_WITH_VALIDATORS=${NODES_WITH_VALIDATORS:-4}
BOOTSTRAP_NODE=$(( NUM_NODES - 1 ))
SYSTEM_VALIDATORS=$(( TOTAL_VALIDATORS - USER_VALIDATORS )) SYSTEM_VALIDATORS=$(( TOTAL_VALIDATORS - USER_VALIDATORS ))
VALIDATORS_PER_NODE=$(( SYSTEM_VALIDATORS / NODES_WITH_VALIDATORS )) VALIDATORS_PER_NODE=$(( SYSTEM_VALIDATORS / NODES_WITH_VALIDATORS ))
BOOTSTRAP_TIMEOUT=10 # in seconds
for NUM_NODE in $(seq 0 $((NUM_NODES - 1))); do for NUM_NODE in $(seq $BOOTSTRAP_NODE -1 0); do
if [[ ${NUM_NODE} == 0 ]]; then if [[ ${NUM_NODE} == ${BOOTSTRAP_NODE} ]]; then
BOOTSTRAP_ARG="" BOOTSTRAP_ARG=""
else else
BOOTSTRAP_ARG="--bootstrap-file=${NETWORK_DIR}/bootstrap_nodes.txt" BOOTSTRAP_ENR="${DATA_DIR}/node${BOOTSTRAP_NODE}/beacon_node.enr"
BOOTSTRAP_ARG="--bootstrap-file=${BOOTSTRAP_ENR}"
# Wait for the master node to write out its address file # Wait for the master node to write out its address file
START_TIMESTAMP=$(date +%s) START_TIMESTAMP=$(date +%s)
while [ ! -f "${DATA_DIR}/node0/beacon_node.address" ]; do while [ ! -f "${BOOTSTRAP_ENR}" ]; do
sleep 0.1 sleep 0.1
NOW_TIMESTAMP=$(date +%s) NOW_TIMESTAMP=$(date +%s)
if [[ "$(( NOW_TIMESTAMP - START_TIMESTAMP - GENESIS_OFFSET ))" -ge "$BOOTSTRAP_TIMEOUT" ]]; then if [[ "$(( NOW_TIMESTAMP - START_TIMESTAMP - GENESIS_OFFSET ))" -ge "$BOOTSTRAP_TIMEOUT" ]]; then
@ -226,7 +269,9 @@ for NUM_NODE in $(seq 0 $((NUM_NODES - 1))); do
--udp-port=$(( BASE_PORT + NUM_NODE )) \ --udp-port=$(( BASE_PORT + NUM_NODE )) \
--data-dir="${NODE_DATA_DIR}" \ --data-dir="${NODE_DATA_DIR}" \
${BOOTSTRAP_ARG} \ ${BOOTSTRAP_ARG} \
--state-snapshot="${NETWORK_DIR}/genesis.ssz" \ ${STATE_SNAPSHOT_ARG} \
${WEB3_ARG} \
${DEPOSIT_CONTRACT_ARG} \
--metrics \ --metrics \
--metrics-address="127.0.0.1" \ --metrics-address="127.0.0.1" \
--metrics-port="$(( BASE_METRICS_PORT + NUM_NODE ))" \ --metrics-port="$(( BASE_METRICS_PORT + NUM_NODE ))" \

9
scripts/reset_testnet.sh
View File

@ -65,7 +65,7 @@ if [ "$ETH1_PRIVATE_KEY" != "" ]; then
echo "Done: $DEPOSIT_CONTRACT_ADDRESS" echo "Done: $DEPOSIT_CONTRACT_ADDRESS"
fi fi
echo "Building a local beacon_node instance for 'makeDeposits' and 'createTestnet'" echo "Building a local beacon_node instance for 'deposits create' and 'createTestnet'"
make -j2 NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node process_dashboard make -j2 NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node process_dashboard
echo "Generating Grafana dashboards for remote testnet servers" echo "Generating Grafana dashboards for remote testnet servers"
@ -83,10 +83,11 @@ echo "Building Docker image..."
# in docker/Makefile, and are enabled by default. # in docker/Makefile, and are enabled by default.
make build make build
../build/beacon_node makeDeposits \ ../build/beacon_node deposits create \
--count=$TOTAL_VALIDATORS \ --count=$TOTAL_VALIDATORS \
--out-validators-dir="$DEPOSITS_DIR_ABS" \ --out-deposits-dir="$DEPOSITS_DIR_ABS" \
--out-secrets-dir="$SECRETS_DIR_ABS" --out-secrets-dir="$SECRETS_DIR_ABS" \
--dont-send
../build/beacon_node createTestnet \ ../build/beacon_node createTestnet \
--data-dir="$DATA_DIR_ABS" \ --data-dir="$DATA_DIR_ABS" \

10
scripts/run_ssz_fuzzing_test.nims
View File

@ -1,5 +1,6 @@
import os except dirExists import os except dirExists
import strformat, confutils import strformat, confutils
import testutils/fuzzing_engines
const const
gitRoot = thisDir() / ".." gitRoot = thisDir() / ".."
@ -10,13 +11,8 @@ const
fuzzNims = gitRoot / "vendor" / "nim-testutils" / "testutils" / "fuzzing" / "fuzz.nims" fuzzNims = gitRoot / "vendor" / "nim-testutils" / "testutils" / "fuzzing" / "fuzz.nims"
type
FuzzerKind = enum
libFuzzer
afl
cli do (testname {.argument.}: string, cli do (testname {.argument.}: string,
fuzzer = libFuzzer): fuzzer = defaultFuzzingEngine):
if not dirExists(fixturesDir): if not dirExists(fixturesDir):
echo "Please run `make test` first in order to download the official ETH2 test vectors" echo "Please run `make test` first in order to download the official ETH2 test vectors"
@ -43,5 +39,5 @@ cli do (testname {.argument.}: string,
let testProgram = fuzzingTestsDir / &"ssz_decode_{testname}.nim" let testProgram = fuzzingTestsDir / &"ssz_decode_{testname}.nim"
exec &"""nim "{fuzzNims}" "{fuzzer}" "{testProgram}" "{corpusDir}" """ exec &"""ntu fuzz --fuzzer={fuzzer} --corpus="{corpusDir}" "{testProgram}" """

1
tests/fuzzing/nim.cfg
View File

@ -1,2 +1,3 @@
-d:ssz_testing -d:ssz_testing
-d:"const_preset=mainnet"

4
tests/mocking/mock_attestations.nim
View File

@ -13,9 +13,9 @@ import
sets, sets,
# Specs # Specs
../../beacon_chain/spec/[datatypes, beaconstate, helpers, validator, crypto, ../../beacon_chain/spec/[datatypes, beaconstate, helpers, validator, crypto,
signatures], signatures, state_transition],
# Internals # Internals
../../beacon_chain/[ssz, extras, state_transition], ../../beacon_chain/[ssz, extras],
# Mocking procs # Mocking procs
./mock_blocks, ./mock_blocks,
./mock_validator_keys ./mock_validator_keys

4
tests/mocking/mock_state.nim
View File

@ -10,9 +10,7 @@
import import
# Specs # Specs
../../beacon_chain/spec/[datatypes], ../../beacon_chain/spec/[datatypes, state_transition]
# Internals
../../beacon_chain/state_transition
proc nextEpoch*(state: var HashedBeaconState) = proc nextEpoch*(state: var HashedBeaconState) =
## Transition to the start of the next epoch ## Transition to the start of the next epoch

8
tests/official/test_fixture_const_sanity_check.nim
View File

@ -80,12 +80,14 @@ macro parseNumConsts(file: static string): untyped =
result = quote do: `constsToCheck` result = quote do: `constsToCheck`
const datatypesConsts = @(parseNumConsts(SpecDir/"datatypes.nim")) const
datatypesConsts = @(parseNumConsts(SpecDir/"datatypes.nim"))
specVersionPresetDir = SpecDir/"presets"/("v"&replace(SPEC_VERSION, ".", "_"))
when const_preset == "minimal": when const_preset == "minimal":
const minimalConsts = @(parseNumConsts(SpecDir/"presets"/"minimal.nim")) const minimalConsts = @(parseNumConsts(specVersionPresetDir/"minimal.nim"))
else: else:
const mainnetConsts = @(parseNumConsts(SpecDir/"presets"/"mainnet.nim")) const mainnetConsts = @(parseNumConsts(specVersionPresetDir/"mainnet.nim"))
const IgnoreKeys = [ const IgnoreKeys = [
# Ignore all non-numeric types # Ignore all non-numeric types

4
tests/official/test_fixture_sanity_blocks.nim
View File

@ -11,8 +11,8 @@ import
# Standard library # Standard library
os, sequtils, unittest, os, sequtils, unittest,
# Beacon chain internals # Beacon chain internals
../../beacon_chain/spec/[crypto, datatypes], ../../beacon_chain/spec/[crypto, datatypes, state_transition],
../../beacon_chain/[ssz, state_transition], ../../beacon_chain/ssz,
# Test utilities # Test utilities
../testutil, ../testutil,
./fixtures_utils ./fixtures_utils

3
tests/official/test_fixture_sanity_slots.nim
View File

@ -11,8 +11,7 @@ import
# Standard library # Standard library
os, strutils, unittest, os, strutils, unittest,
# Beacon chain internals # Beacon chain internals
../../beacon_chain/spec/datatypes, ../../beacon_chain/spec/[datatypes, state_transition],
../../beacon_chain/state_transition,
# Test utilities # Test utilities
../testutil, ../testutil,
./fixtures_utils, ./fixtures_utils,

4
tests/official/test_fixture_ssz_consensus_objects.nim
View File

@ -117,11 +117,7 @@ proc runSSZtests() =
checkSSZ(SignedBeaconBlockHeader, path, hash) checkSSZ(SignedBeaconBlockHeader, path, hash)
of "SignedVoluntaryExit": checkSSZ(SignedVoluntaryExit, path, hash) of "SignedVoluntaryExit": checkSSZ(SignedVoluntaryExit, path, hash)
of "SigningData": of "SigningData":
when ETH2_SPEC == "v0.12.1":
checkSSZ(SigningData, path, hash) checkSSZ(SigningData, path, hash)
of "SigningRoot":
when ETH2_SPEC == "v0.11.3":
checkSSZ(SigningRoot, path, hash)
of "Validator": checkSSZ(Validator, path, hash) of "Validator": checkSSZ(Validator, path, hash)
of "VoluntaryExit": checkSSZ(VoluntaryExit, path, hash) of "VoluntaryExit": checkSSZ(VoluntaryExit, path, hash)
else: else:

189
tests/simulation/dashboard/chronicles_tail_p2p_plugin.nim Normal file
View File

@ -0,0 +1,189 @@
import
strformat, jsconsole, jsffi,
karax/[karax, kdom, karaxdsl, vdom],
chronicles_tail/jsplugins
# Make sure that the Karax instance in the plugin is the same one
# as the Karax instance in the enclosing chronicle-tail page.
kxi = getKarax()
type EventsTable = ref object of VComponent
proc renderNetworkEvents(page: VComponent): VNode =
result = buildHtml:
table:
tr:
th: text "Time"
th: text "Nodes"
const
columnWidth = 320
timestampsHeight = 50
eventsMargin = 10
var
eventsTable = newComponent(EventsTable, renderNetworkEvents)
protocolMessages = newJsAssoc[cstring, JsAssoc[cstring, cstring]]()
pendingEvents = newSeq[TailEvent]()
freedColumns = newSeq[int]()
columnBottoms = newSeq[int]()
peerToColumnTable = newJsAssoc[cstring, int]()
lastTimestampBottom = timestampsHeight
proc startsWith*(a, b: cstring): bool {.importcpp: "startsWith", nodecl.}
proc getMsgName(protocol: cstring, msgId: int): cstring =
protocolMessages[protocol][cast[cstring](msgId)]
proc renderEvent(ev: TailEvent): cstring =
var res = newStringOfCap(1024)
let eventType = ev.msg
res.add &"""<div class="event {eventType}">"""
template addField(class, value) =
res.add "<div class=\"" & class & "\">"
res.addEscaped $value
res.add "</div>"
if eventType.startsWith(cstring("peer_")):
addField "peer", ev.peer
addField "port", ev.port
else:
addField "msgName", getMsgName(ev.protocol, ev.msgId)
res.addAsHtml ev.data
res.add """</div>"""
return cstring(res)
proc selectColumn(ev: TailEvent): int =
let key = cast[cstring](ev.port)# & ev.peer
kout ev.msg, key
if ev.msg in [cstring"peer_accepted", "peer_connected"]:
if freedColumns.len > 0:
result = freedColumns.pop()
else:
result = columnBottoms.len
columnBottoms.add(timestampsHeight)
peerToColumnTable[key] = result
elif ev.msg == cstring("peer_disconnected"):
result = peerToColumnTable[key]
discard jsDelete peerToColumnTable[key]
freedColumns.add result
else:
result = peerToColumnTable[key]
template pixels(n: int): cstring =
cast[cstring](n) & "px"
proc addEvent(ev: TailEvent) =
var
row = document.createElement("tr")
timeElem = document.createElement("td")
eventElem = document.createElement("td")
eventsTable = eventsTable.dom
eventsCount = eventsTable.children.len
lastEventRow = eventsTable.children[eventsCount - 1]
row.class = if eventsCount mod 2 == 0: "even" else: "odd"
# Hide the element initially, so we can safely measure its size.
# It has to be added to the DOM before it can be measured.
row.style.visibility = "hidden"
row.appendChild(timeElem)
row.appendChild(eventElem)
timeElem.innerHtml = ev.ts
timeElem.class = "time"
eventElem.innerHTML = renderEvent(ev)
eventsTable.appendChild(row)
let rowHeight = row.offsetHeight
let eventColumn = selectColumn(ev)
let timestampOffset = max(lastTimestampBottom, columnBottoms[eventColumn])
let prevTimestampOffset = lastTimestampBottom - timestampsHeight
lastTimestampBottom = timestampOffset + timestampsHeight
columnBottoms[eventColumn] += rowHeight + eventsMargin
# Make sure the event data is in the right column and that it
# can overflow past the row height:
eventElem.style.paddingLeft = pixels(eventColumn * columnWidth)
# Position the row in its right place and show it:
lastEventRow.style.height = pixels(timestampOffset - prevTimestampOffset)
row.style.top = pixels(timestampOffset)
row.style.visibility = ""
proc networkSectionContent: VNode =
result = buildHtml(tdiv(id = "network")):
text "Network section"
eventsTable
proc tailEventFilter(ev: TailEvent): bool =
if ev.topics != "p2pdump":
return false
if ev.msg == "p2p_protocols":
protocolMessages = cast[type(protocolMessages)](ev.data)
else:
if eventsTable.dom == nil:
pendingEvents.add ev
else:
addEvent ev
return true
proc addPending =
if eventsTable.dom != nil and pendingEvents.len > 0:
defer: pendingEvents.setLen(0)
for ev in pendingEvents:
addEvent ev
let interval = window.setInterval(addPending, 1000)
proc addStyles(styles: cstring) =
var s = document.createElement("style")
s.appendChild document.createTextNode(styles)
document.head.appendChild(s)
once:
addStyles cstring"""
#network > table {
position: relative;
}
#network .event {
border: 1px solid blue;
}
#network .event table {
width: 100%;
}
#network > table > tr {
position: absolute;
display: flex;
flex-direction: row;
border-left: 1px solid red;
}
#network .time {
width: 160px;
}
#network .event {
width: 320px;
}
"""
addSection("Network", networkSectionContent)
addEventFilter(tailEventFilter)
kxi.redraw()

31
tests/simulation/run_node.sh
View File

@ -16,13 +16,19 @@ else
ADDITIONAL_BEACON_NODE_ARGS="" ADDITIONAL_BEACON_NODE_ARGS=""
fi fi
BOOTSTRAP_ARG=""
if [[ ! -z "$1" ]]; then if [[ ! -z "$1" ]]; then
BOOTSTRAP_NODE_ID=$1 BOOTSTRAP_NODE_ID=$1
BOOTSTRAP_ADDRESS_FILE="${SIMULATION_DIR}/node-${BOOTSTRAP_NODE_ID}/beacon_node.address"
shift shift
else else
BOOTSTRAP_NODE_ID=$MASTER_NODE BOOTSTRAP_NODE_ID=$BOOTSTRAP_NODE
BOOTSTRAP_ADDRESS_FILE=$NETWORK_BOOTSTRAP_FILE fi
BOOTSTRAP_ADDRESS_FILE="${SIMULATION_DIR}/node-${BOOTSTRAP_NODE_ID}/beacon_node.enr"
if [[ "$NODE_ID" != "$BOOTSTRAP_NODE" ]]; then
BOOTSTRAP_ARG="--bootstrap-file=$BOOTSTRAP_ADDRESS_FILE"
fi fi
# set up the environment # set up the environment
@ -48,12 +54,12 @@ mkdir -p "$NODE_VALIDATORS_DIR"
rm -rf "$NODE_SECRETS_DIR" rm -rf "$NODE_SECRETS_DIR"
mkdir -p "$NODE_SECRETS_DIR" mkdir -p "$NODE_SECRETS_DIR"
VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES)) VALIDATORS_PER_NODE=$(( NUM_VALIDATORS / (TOTAL_NODES - 1) ))
if [[ $NODE_ID -lt $TOTAL_NODES ]]; then if [[ $NODE_ID -lt $BOOTSTRAP_NODE ]]; then
# if using validator client binaries in addition to beacon nodes # if using validator client binaries in addition to beacon nodes
# we will split the keys for this instance in half between the BN and the VC # we will split the keys for this instance in half between the BN and the VC
if [ "${SPLIT_VALIDATORS_BETWEEN_BN_AND_VC:-}" == "yes" ]; then if [ "${BN_VC_VALIDATOR_SPLIT:-}" == "yes" ]; then
ATTACHED_VALIDATORS=$((VALIDATORS_PER_NODE / 2)) ATTACHED_VALIDATORS=$((VALIDATORS_PER_NODE / 2))
else else
ATTACHED_VALIDATORS=$VALIDATORS_PER_NODE ATTACHED_VALIDATORS=$VALIDATORS_PER_NODE
@ -75,12 +81,18 @@ if [ -f "${SNAPSHOT_FILE}" ]; then
SNAPSHOT_ARG="--state-snapshot=${SNAPSHOT_FILE}" SNAPSHOT_ARG="--state-snapshot=${SNAPSHOT_FILE}"
fi fi
DEPOSIT_CONTRACT_ARGS=""
if [ -f "${DEPOSIT_CONTRACT_FILE}" ]; then
DEPOSIT_CONTRACT_ARGS="$WEB3_ARG \
--deposit-contract=$(cat $DEPOSIT_CONTRACT_FILE) \
--deposit-contract-block=$(cat $DEPOSIT_CONTRACT_BLOCK_FILE)"
fi
cd "$NODE_DATA_DIR" cd "$NODE_DATA_DIR"
# if you want tracing messages, add "--log-level=TRACE" below
$BEACON_NODE_BIN \ $BEACON_NODE_BIN \
--log-level=${LOG_LEVEL:-DEBUG} \ --log-level=${LOG_LEVEL:-DEBUG} \
--bootstrap-file=$BOOTSTRAP_ADDRESS_FILE \ $BOOTSTRAP_ARG \
--data-dir=$NODE_DATA_DIR \ --data-dir=$NODE_DATA_DIR \
--secrets-dir=$NODE_SECRETS_DIR \ --secrets-dir=$NODE_SECRETS_DIR \
--node-name=$NODE_ID \ --node-name=$NODE_ID \
@ -88,8 +100,7 @@ $BEACON_NODE_BIN \
--udp-port=$PORT \ --udp-port=$PORT \
$SNAPSHOT_ARG \ $SNAPSHOT_ARG \
$NAT_ARG \ $NAT_ARG \
$WEB3_ARG \ $DEPOSIT_CONTRACT_ARGS \
--deposit-contract=$DEPOSIT_CONTRACT_ADDRESS \
--rpc \ --rpc \
--rpc-address="127.0.0.1" \ --rpc-address="127.0.0.1" \
--rpc-port="$(( $BASE_RPC_PORT + $NODE_ID ))" \ --rpc-port="$(( $BASE_RPC_PORT + $NODE_ID ))" \

28
tests/simulation/run_validator.sh
View File

@ -15,26 +15,34 @@ source "${SIM_ROOT}/../../env.sh"
cd "$GIT_ROOT" cd "$GIT_ROOT"
VC_DATA_DIR="${SIMULATION_DIR}/validator-$NODE_ID" NODE_DATA_DIR="${SIMULATION_DIR}/validator-$NODE_ID"
NODE_VALIDATORS_DIR=$NODE_DATA_DIR/validators/
NODE_SECRETS_DIR=$NODE_DATA_DIR/secrets/
mkdir -p "$VC_DATA_DIR/validators" rm -rf "$NODE_VALIDATORS_DIR"
rm -f $VC_DATA_DIR/validators/* mkdir -p "$NODE_VALIDATORS_DIR"
rm -rf "$NODE_SECRETS_DIR"
mkdir -p "$NODE_SECRETS_DIR"
VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES))
if [[ $NODE_ID -lt $TOTAL_NODES ]]; then if [[ $NODE_ID -lt $TOTAL_NODES ]]; then
# we will split the keys for this instance in half between the BN and the VC # we will split the keys for this instance in half between the BN and the VC
VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES)) ATTACHED_VALIDATORS=$((VALIDATORS_PER_NODE / 2))
VALIDATORS_PER_NODE_HALF=$((VALIDATORS_PER_NODE / 2))
FIRST_VALIDATOR_IDX=$(( VALIDATORS_PER_NODE * NODE_ID + VALIDATORS_PER_NODE_HALF))
LAST_VALIDATOR_IDX=$(( FIRST_VALIDATOR_IDX + VALIDATORS_PER_NODE_HALF - 1 ))
pushd "$VALIDATORS_DIR" >/dev/null pushd "$VALIDATORS_DIR" >/dev/null
cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) "$VC_DATA_DIR/validators" for VALIDATOR in $(ls | tail -n +$(( ($VALIDATORS_PER_NODE * $NODE_ID) + 1 + $ATTACHED_VALIDATORS )) | head -n $ATTACHED_VALIDATORS); do
cp -ar "$VALIDATOR" "$NODE_VALIDATORS_DIR"
cp -a "$SECRETS_DIR/$VALIDATOR" "$NODE_SECRETS_DIR"
done
popd >/dev/null popd >/dev/null
fi fi
cd "$VC_DATA_DIR" cd "$NODE_DATA_DIR"
$VALIDATOR_CLIENT_BIN \ $VALIDATOR_CLIENT_BIN \
--log-level=${LOG_LEVEL:-DEBUG} \ --log-level=${LOG_LEVEL:-DEBUG} \
--data-dir=$VC_DATA_DIR \ --data-dir=$NODE_DATA_DIR \
--secrets-dir=$NODE_SECRETS_DIR \
--rpc-port="$(( $BASE_RPC_PORT + $NODE_ID ))" --rpc-port="$(( $BASE_RPC_PORT + $NODE_ID ))"

Some files were not shown because too many files have changed in this diff Show More