From ee5d1c1467aab9772a4afab43b1042f1a70ae1e8 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 25 Oct 2019 12:59:56 +0200 Subject: [PATCH] minimize imports, specially for spec - cuts 2/3 of ncli build time (#500) * minimize imports, specially for spec - cuts 2/3 of ncli build time * ptr_arith->ptrops --- beacon_chain/beacon_node.nim | 5 ++--- beacon_chain/beacon_node_types.nim | 8 +------- beacon_chain/block_pool.nim | 1 - beacon_chain/conf.nim | 2 +- beacon_chain/fork_choice.nim | 6 +++--- beacon_chain/interop.nim | 2 +- beacon_chain/spec/crypto.nim | 8 +++++++- beacon_chain/spec/datatypes.nim | 4 ++-- beacon_chain/spec/presets/mainnet.nim | 7 +------ beacon_chain/spec/presets/minimal.nim | 7 +------ beacon_chain/spec/state_transition_block.nim | 4 ++-- beacon_chain/spec/state_transition_epoch.nim | 4 ++-- beacon_chain/spec/validator.nim | 5 ++--- beacon_chain/ssz.nim | 4 ++-- beacon_chain/state_transition.nim | 6 +++--- beacon_chain/sync_protocol.nim | 4 ++-- beacon_chain/trusted_state_snapshots.nim | 4 ++-- ncli/ncli_hash_tree_root.nim | 4 ++-- ncli/ncli_pretty.nim | 4 ++-- ncli/ncli_signing_root.nim | 4 ++-- ncli/ncli_transition.nim | 2 +- ncli/nim.cfg | 1 + research/state_sim.nim | 2 +- 23 files changed, 43 insertions(+), 55 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 1f44eacb9..82935cd25 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -9,11 +9,10 @@ import eth/trie/db, eth/trie/backends/rocksdb_backend, eth/async_utils, # Local modules - spec/[datatypes, digest, crypto, beaconstate, helpers, validator, - state_transition_block, network], + spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network], conf, time, state_transition, fork_choice, ssz, beacon_chain_db, validator_pool, extras, attestation_pool, block_pool, eth2_network, - beacon_node_types, mainchain_monitor, trusted_state_snapshots, version, + beacon_node_types, mainchain_monitor, version, sync_protocol, request_manager, validator_keygen, interop, statusbar const diff --git a/beacon_chain/beacon_node_types.nim b/beacon_chain/beacon_node_types.nim index fee65777f..03bd90b57 100644 --- a/beacon_chain/beacon_node_types.nim +++ b/beacon_chain/beacon_node_types.nim @@ -1,6 +1,6 @@ import sets, deques, tables, options, - eth/keys, stew/[bitseqs, endians2], + stew/[endians2], spec/[datatypes, crypto, digest], beacon_chain_db, conf, mainchain_monitor, eth2_network, time @@ -263,9 +263,3 @@ proc userValidatorsRange*(d: NetworkMetadata): HSlice[int, int] = proc shortLog*(v: AttachedValidator): string = shortLog(v.pubKey) -proc toGaugeValue*(hash: Eth2Digest): int64 = - # Only the last 8 bytes are taken into consideration in accordance - # to the ETH2 metrics spec: - # https://github.com/ethereum/eth2.0-metrics/blob/6a79914cb31f7d54858c7dd57eee75b6162ec737/metrics.md#interop-metrics - cast[int64](uint64.fromBytes(hash.data[24..31], littleEndian)) - diff --git a/beacon_chain/block_pool.nim b/beacon_chain/block_pool.nim index 97a4a433d..185fcf86c 100644 --- a/beacon_chain/block_pool.nim +++ b/beacon_chain/block_pool.nim @@ -61,7 +61,6 @@ proc init*(T: type BlockPool, db: BeaconChainDB): BlockPool = var blocks = {tailRef.root: tailRef}.toTable() latestStateRoot = Option[Eth2Digest]() - headStateBlock = tailRef headRef: BlockRef if headRoot != tailRoot: diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 9e273b62a..996ea18cf 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -1,7 +1,7 @@ import os, options, strformat, confutils/defs, chronicles/options as chroniclesOptions, - spec/[crypto, datatypes], time, version + spec/[crypto] export defs, enabledLogLevel diff --git a/beacon_chain/fork_choice.nim b/beacon_chain/fork_choice.nim index 89bfa9e86..3558cb699 100644 --- a/beacon_chain/fork_choice.nim +++ b/beacon_chain/fork_choice.nim @@ -1,8 +1,8 @@ import - deques, math, options, sequtils, tables, + deques, options, sequtils, tables, chronicles, - ./spec/[beaconstate, datatypes, crypto, digest, helpers, validator], extras, - ./attestation_pool, ./beacon_node_types, ./beacon_chain_db, ./ssz + ./spec/[datatypes, crypto, digest, helpers], + ./attestation_pool, ./beacon_node_types, ./ssz proc get_ancestor(blck: BlockRef, slot: Slot): BlockRef = if blck.slot == slot: diff --git a/beacon_chain/interop.nim b/beacon_chain/interop.nim index ceb31deb2..efdd5cf95 100644 --- a/beacon_chain/interop.nim +++ b/beacon_chain/interop.nim @@ -1,7 +1,7 @@ import os, ospaths, stew/endians2, stint, - ./conf, ./extras, ./ssz, + ./extras, ./ssz, spec/[crypto, datatypes, digest, helpers] func get_eth1data_stub*(deposit_count: uint64, current_epoch: Epoch): Eth1Data = diff --git a/beacon_chain/spec/crypto.nim b/beacon_chain/spec/crypto.nim index 2ee234514..98aa37267 100644 --- a/beacon_chain/spec/crypto.nim +++ b/beacon_chain/spec/crypto.nim @@ -46,7 +46,7 @@ import sequtils, - stew/[objects, byteutils], hashes, nimcrypto/utils, + stew/[endians2, objects, byteutils], hashes, nimcrypto/utils, blscurve, json_serialization, ../version, digest, chronicles @@ -323,3 +323,9 @@ proc writeValue*(writer: var JsonWriter, value: Signature) {.inline.} = proc readValue*(reader: var JsonReader, value: var Signature) {.inline.} = value = Signature.init(reader.readValue(string)) + +proc toGaugeValue*(hash: Eth2Digest): int64 = + # Only the last 8 bytes are taken into consideration in accordance + # to the ETH2 metrics spec: + # https://github.com/ethereum/eth2.0-metrics/blob/6a79914cb31f7d54858c7dd57eee75b6162ec737/metrics.md#interop-metrics + cast[int64](uint64.fromBytesLE(hash.data[24..31])) diff --git a/beacon_chain/spec/datatypes.nim b/beacon_chain/spec/datatypes.nim index 877827714..1ff4f5b5e 100644 --- a/beacon_chain/spec/datatypes.nim +++ b/beacon_chain/spec/datatypes.nim @@ -18,7 +18,7 @@ # types / composition import - macros, hashes, math, json, strutils, tables, + macros, hashes, json, strutils, tables, stew/[byteutils, bitseqs], chronicles, eth/common, ../version, ../ssz/types, ./crypto, ./digest @@ -644,6 +644,6 @@ static: # requires that SHARD_COUNT >= get_committee_count(...) doAssert SHARD_COUNT >= SLOTS_PER_EPOCH, "Shard count should match or exceed number of slots per epoch" -import nimcrypto, json_serialization +import json_serialization export json_serialization export writeValue, readValue diff --git a/beacon_chain/spec/presets/mainnet.nim b/beacon_chain/spec/presets/mainnet.nim index 83db343fc..f467c1994 100644 --- a/beacon_chain/spec/presets/mainnet.nim +++ b/beacon_chain/spec/presets/mainnet.nim @@ -9,12 +9,7 @@ # serialization and spec updates. import - # Standard library - math, - # Third-party - eth/common, - # Internals - ../crypto, ../digest + math type Slot* = distinct uint64 diff --git a/beacon_chain/spec/presets/minimal.nim b/beacon_chain/spec/presets/minimal.nim index a64d7d110..77f78eca0 100644 --- a/beacon_chain/spec/presets/minimal.nim +++ b/beacon_chain/spec/presets/minimal.nim @@ -9,12 +9,7 @@ # serialization and spec updates. import - # Standard library - math, - # Third-party - eth/common, - # Internals - ../crypto, ../digest + math type Slot* = distinct uint64 diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index 26d9db8d8..65fb0705b 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -33,8 +33,8 @@ # now. import # TODO - cleanup imports - algorithm, collections/sets, chronicles, math, options, sequtils, sets, tables, - ../extras, ../ssz, ../beacon_node_types, metrics, + algorithm, collections/sets, chronicles, options, sequtils, sets, tables, + ../extras, ../ssz, metrics, beaconstate, crypto, datatypes, digest, helpers, validator, state_transition_helpers diff --git a/beacon_chain/spec/state_transition_epoch.nim b/beacon_chain/spec/state_transition_epoch.nim index ab5929b84..4f59671c7 100644 --- a/beacon_chain/spec/state_transition_epoch.nim +++ b/beacon_chain/spec/state_transition_epoch.nim @@ -35,7 +35,7 @@ import # TODO - cleanup imports algorithm, math, options, sequtils, tables, stew/[bitseqs, bitops2], chronicles, json_serialization/std/sets, - metrics, ../extras, ../ssz, ../beacon_node_types, + metrics, ../ssz, beaconstate, crypto, datatypes, digest, helpers, validator, state_transition_helpers @@ -459,7 +459,7 @@ func get_crosslink_deltas*(state: BeaconState, cache: var StateCache): shard = (get_start_shard(state, epoch) + offset) mod SHARD_COUNT crosslink_committee = toSet(get_crosslink_committee(state, epoch, shard, cache)) - (winning_crosslink, attesting_indices) = + (_, attesting_indices) = get_winning_crosslink_and_attesting_indices( state, epoch, shard, cache) attesting_balance = get_total_balance(state, attesting_indices) diff --git a/beacon_chain/spec/validator.nim b/beacon_chain/spec/validator.nim index 0fa476146..a06f5eb63 100644 --- a/beacon_chain/spec/validator.nim +++ b/beacon_chain/spec/validator.nim @@ -7,9 +7,8 @@ # Helpers and functions pertaining to managing the validator set import - options, nimcrypto, sequtils, math, tables, chronicles, - ../ssz, - ./crypto, ./datatypes, ./digest, ./helpers + options, nimcrypto, sequtils, math, tables, + ./datatypes, ./digest, ./helpers # TODO: Proceed to renaming and signature changes # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#compute_shuffled_index diff --git a/beacon_chain/ssz.nim b/beacon_chain/ssz.nim index 28d5bd6f9..cdef63af4 100644 --- a/beacon_chain/ssz.nim +++ b/beacon_chain/ssz.nim @@ -10,7 +10,7 @@ import endians, stew/shims/macros, options, algorithm, math, options, - stew/[bitops2, bitseqs, objects, varints], stew/ranges/ptr_arith, stint, + stew/[bitops2, bitseqs, objects, varints, ptrops], stint, faststreams/input_stream, serialization, serialization/testing/tracing, nimcrypto/sha2, blscurve, eth/common, ./spec/[crypto, datatypes, digest], @@ -343,7 +343,7 @@ func addChunk*(merkelizer: SszChunksMerkelizer, data: openarray[byte]) = if not getBitLE(merkelizer.totalChunks, 0): let chunkStartAddr = addr merkelizer.combinedChunks[0].data[0] copyMem(chunkStartAddr, unsafeAddr data[0], data.len) - zeroMem(chunkStartAddr.shift(data.len), bytesPerChunk - data.len) + zeroMem(chunkStartAddr.offset(data.len), bytesPerChunk - data.len) trs "WROTE BASE CHUNK ", merkelizer.combinedChunks[0] else: var hash = mergeBranches(merkelizer.combinedChunks[0], data) diff --git a/beacon_chain/state_transition.nim b/beacon_chain/state_transition.nim index 6b0159f32..1eac7b988 100644 --- a/beacon_chain/state_transition.nim +++ b/beacon_chain/state_transition.nim @@ -31,9 +31,9 @@ # now. import - algorithm, collections/sets, chronicles, math, options, sequtils, sets, tables, - ./extras, ./ssz, ./beacon_node_types, metrics, - ./spec/[beaconstate, crypto, datatypes, digest, helpers, validator], + collections/sets, chronicles, math, options, sets, tables, + ./extras, ./ssz, metrics, + ./spec/[datatypes, digest, helpers, validator], ./spec/[state_transition_block, state_transition_epoch] # https://github.com/ethereum/eth2.0-metrics/blob/master/metrics.md#additional-metrics diff --git a/beacon_chain/sync_protocol.nim b/beacon_chain/sync_protocol.nim index 550f1544d..06d626ff6 100644 --- a/beacon_chain/sync_protocol.nim +++ b/beacon_chain/sync_protocol.nim @@ -1,8 +1,8 @@ import - options, tables, sequtils, algorithm, sets, macros, + options, tables, sets, macros, chronicles, chronos, metrics, stew/ranges/bitranges, spec/[datatypes, crypto, digest, helpers], eth/rlp, - beacon_node_types, eth2_network, beacon_chain_db, block_pool, time, ssz + beacon_node_types, eth2_network, beacon_chain_db, block_pool, ssz when networkBackend == rlpxBackend: import eth/rlp/options as rlpOptions diff --git a/beacon_chain/trusted_state_snapshots.nim b/beacon_chain/trusted_state_snapshots.nim index c6f6398a7..34935ad44 100644 --- a/beacon_chain/trusted_state_snapshots.nim +++ b/beacon_chain/trusted_state_snapshots.nim @@ -1,6 +1,6 @@ import - ospaths, chronos, json_serialization, strformat, - spec/[datatypes, crypto, digest, beaconstate], beacon_chain_db, conf + ospaths, chronos, json_serialization, + spec/[datatypes], beacon_chain_db const WEAK_SUBJECTVITY_PERIOD* = diff --git a/ncli/ncli_hash_tree_root.nim b/ncli/ncli_hash_tree_root.nim index f8329baff..5390e8243 100644 --- a/ncli/ncli_hash_tree_root.nim +++ b/ncli/ncli_hash_tree_root.nim @@ -1,8 +1,8 @@ import confutils, ospaths, strutils, chronicles, json_serialization, nimcrypto/utils, - ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator], - ../beacon_chain/[extras, state_transition, ssz] + ../beacon_chain/spec/[crypto, datatypes, digest], + ../beacon_chain/[ssz] # TODO turn into arguments cli do(kind: string, file: string): diff --git a/ncli/ncli_pretty.nim b/ncli/ncli_pretty.nim index 5f4c950fc..352be6a95 100644 --- a/ncli/ncli_pretty.nim +++ b/ncli/ncli_pretty.nim @@ -1,7 +1,7 @@ import confutils, ospaths, strutils, chronicles, json_serialization, - ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator], - ../beacon_chain/[extras, state_transition, ssz] + ../beacon_chain/spec/[crypto, datatypes, digest], + ../beacon_chain/[ssz] # TODO turn into arguments cli do(kind: string, file: string): diff --git a/ncli/ncli_signing_root.nim b/ncli/ncli_signing_root.nim index 07306ce54..ab937dbfa 100644 --- a/ncli/ncli_signing_root.nim +++ b/ncli/ncli_signing_root.nim @@ -1,8 +1,8 @@ import confutils, ospaths, strutils, chronicles, json_serialization, nimcrypto/utils, - ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator], - ../beacon_chain/[extras, state_transition, ssz] + ../beacon_chain/spec/[crypto, datatypes, digest], + ../beacon_chain/[ssz] # TODO turn into arguments cli do(kind: string, file: string): diff --git a/ncli/ncli_transition.nim b/ncli/ncli_transition.nim index b5796af1d..02a735b1e 100644 --- a/ncli/ncli_transition.nim +++ b/ncli/ncli_transition.nim @@ -1,6 +1,6 @@ import confutils, chronicles, - ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator], + ../beacon_chain/spec/[crypto, datatypes], ../beacon_chain/[extras, state_transition, ssz] cli do(pre: string, blck: string, post: string, verifyStateRoot = false): diff --git a/ncli/nim.cfg b/ncli/nim.cfg index 3f78c9e4f..980403c64 100644 --- a/ncli/nim.cfg +++ b/ncli/nim.cfg @@ -1 +1,2 @@ hints:off +-u:metrics diff --git a/research/state_sim.nim b/research/state_sim.nim index d956d0d61..97a49286c 100644 --- a/research/state_sim.nim +++ b/research/state_sim.nim @@ -4,7 +4,7 @@ import options, sequtils, random, tables, ../tests/[testutil], ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator], - ../beacon_chain/[attestation_pool, extras, ssz, state_transition, fork_choice] + ../beacon_chain/[attestation_pool, extras, ssz] type Timers = enum tBlock = "Process non-epoch slot with block"