From 9ff6118c28a5a96771169f6a11923e448bbb4232 Mon Sep 17 00:00:00 2001 From: tersec Date: Wed, 17 Mar 2021 17:29:34 +0000 Subject: [PATCH 01/12] clear nimcache on make update (#2426) * test clearing nimcache * don't be verbose --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index d078c356e..120deb7b0 100644 --- a/Makefile +++ b/Makefile @@ -137,6 +137,7 @@ endif #- deletes binaries that might need to be rebuilt after a Git pull update: | update-common rm -f build/generate_makefile + rm -fr nimcache/ # nim-libbacktrace libbacktrace: From 51079770b9b39257089c1cd1964747a612318e0a Mon Sep 17 00:00:00 2001 From: Dustin Brody Date: Wed, 17 Mar 2021 19:41:02 +0100 Subject: [PATCH 02/12] disable Nim compiler cache --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6df4c0d5e..e9172b99e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,7 +19,7 @@ def runStages() { } cache(maxCacheSize: 250, caches: [ - [$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/vendor/nimbus-build-system/vendor/Nim/bin"], + #[$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/vendor/nimbus-build-system/vendor/Nim/bin"], [$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/jsonTestsCache"] ]) { stage("Build") { From 732a437fa8eedd88c3aeab7571ed7348463f9c8c Mon Sep 17 00:00:00 2001 From: Dustin Brody Date: Wed, 17 Mar 2021 19:45:27 +0100 Subject: [PATCH 03/12] maybe fix Jenkins syntax --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e9172b99e..fa3202dca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,7 +19,6 @@ def runStages() { } cache(maxCacheSize: 250, caches: [ - #[$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/vendor/nimbus-build-system/vendor/Nim/bin"], [$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/jsonTestsCache"] ]) { stage("Build") { From a3a6aa68e877e6a68bd920af59b898c864b11a7d Mon Sep 17 00:00:00 2001 From: Dustin Brody Date: Wed, 17 Mar 2021 19:58:33 +0100 Subject: [PATCH 04/12] check if Nim-caching version also works --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index fa3202dca..6df4c0d5e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,6 +19,7 @@ def runStages() { } cache(maxCacheSize: 250, caches: [ + [$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/vendor/nimbus-build-system/vendor/Nim/bin"], [$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/jsonTestsCache"] ]) { stage("Build") { From 21b2367bf168b05f9a26f03255975a513d6ee8f2 Mon Sep 17 00:00:00 2001 From: tersec Date: Thu, 18 Mar 2021 21:50:59 +0000 Subject: [PATCH 05/12] Remove Travis and Azure build statuses from README.md (#2433) --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index efe802610..fe678a807 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # Nimbus Eth2 (Beacon Chain) -[![Build Status (Travis)](https://img.shields.io/travis/status-im/nimbus-eth2/master.svg?label=Linux%20/%20macOS "Linux/macOS build status (Travis)")](https://travis-ci.org/status-im/nimbus-eth2) -[![Build Status (Azure)](https://dev.azure.com/nimbus-dev/nimbus-eth2/_apis/build/status/status-im.nimbus-eth2?branchName=master)](https://dev.azure.com/nimbus-dev/nimbus-eth2/_build/latest?definitionId=3&branchName=master) [![Github Actions CI](https://github.com/status-im/nimbus-eth2/workflows/Nimbus%20nimbus-eth2%20CI/badge.svg)](https://github.com/status-im/nim-blscurve/actions?query=workflow%3A%22BLSCurve+CI%22) [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) From 97850741a01b856be97472a605c9def140ac966b Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 19 Mar 2021 08:26:02 +0000 Subject: [PATCH 06/12] remove unused imports in slashing prtection (#2436) --- beacon_chain/validators/slashing_protection_v1.nim | 2 +- tests/slashing_protection/test_migration.nim | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/beacon_chain/validators/slashing_protection_v1.nim b/beacon_chain/validators/slashing_protection_v1.nim index d35987d79..19b7c8eab 100644 --- a/beacon_chain/validators/slashing_protection_v1.nim +++ b/beacon_chain/validators/slashing_protection_v1.nim @@ -11,7 +11,7 @@ import # Status eth/db/[kvstore, kvstore_sqlite3], chronicles, - nimcrypto/[hash, utils], + nimcrypto/hash, serialization, json_serialization, # Internal diff --git a/tests/slashing_protection/test_migration.nim b/tests/slashing_protection/test_migration.nim index d3c251a7e..ce9e414d8 100644 --- a/tests/slashing_protection/test_migration.nim +++ b/tests/slashing_protection/test_migration.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018 Status Research & Development GmbH +# Copyright (c) 2018-2021 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or https://www.apache.org/licenses/LICENSE-2.0) # * MIT license ([LICENSE-MIT](LICENSE-MIT) or https://opensource.org/licenses/MIT) @@ -19,8 +19,7 @@ import # Internal ../../beacon_chain/validators/[ slashing_protection, - slashing_protection_v1, - slashing_protection_v2 + slashing_protection_v1 ], ../../beacon_chain/spec/[datatypes, digest, crypto, presets], # Test utilies From 01fe564e464d3800cee2625ff99e051b84547926 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 19 Mar 2021 10:36:38 +0100 Subject: [PATCH 07/12] chronos-based metrics (#2432) This opens up the road for removing `-d:insecure` for metrics. --- beacon_chain/nimbus_beacon_node.nim | 5 +++-- vendor/nim-chronos | 2 +- vendor/nim-metrics | 2 +- vendor/nim-stew | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index c00ec9321..2fd057316 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -13,7 +13,8 @@ import # Nimble packages stew/[objects, byteutils, endians2, io2], stew/shims/macros, - chronos, confutils, metrics, json_rpc/[rpcclient, rpcserver, jsonmarshal], + chronos, confutils, metrics, metrics/chronos_httpserver, + json_rpc/[rpcclient, rpcserver, jsonmarshal], chronicles, bearssl, blscurve, json_serialization/std/[options, sets, net], serialization/errors, @@ -1637,7 +1638,7 @@ proc doRunBeaconNode(config: var BeaconNodeConf, rng: ref BrHmacDrbgContext) = let metricsAddress = config.metricsAddress notice "Starting metrics HTTP server", url = "http://" & $metricsAddress & ":" & $config.metricsPort & "/metrics" - metrics.startHttpServer($metricsAddress, config.metricsPort) + startMetricsHttpServer($metricsAddress, config.metricsPort) else: warn "Metrics support disabled, see https://status-im.github.io/nimbus-eth2/metrics-pretty-pictures.html#simple-metrics" diff --git a/vendor/nim-chronos b/vendor/nim-chronos index 03707426e..c8eefb938 160000 --- a/vendor/nim-chronos +++ b/vendor/nim-chronos @@ -1 +1 @@ -Subproject commit 03707426e43d03cccc1de2e7284de168b79f7bf6 +Subproject commit c8eefb9382a786993fc703386b0bd446ecf9c037 diff --git a/vendor/nim-metrics b/vendor/nim-metrics index 22a386734..105af2bfb 160000 --- a/vendor/nim-metrics +++ b/vendor/nim-metrics @@ -1 +1 @@ -Subproject commit 22a3867341f7b0a9d55661b41d7ee5febe35c86b +Subproject commit 105af2bfbd4896e8b4086d3dff1e6c187e9d0a41 diff --git a/vendor/nim-stew b/vendor/nim-stew index 42475fd2f..6bcb21184 160000 --- a/vendor/nim-stew +++ b/vendor/nim-stew @@ -1 +1 @@ -Subproject commit 42475fd2f1919acd11be2fdc75fd6e2cefc99e90 +Subproject commit 6bcb21184aeb96ce6c62e187a64d678b74609f1e From baa43ae7f7f58807f2bf5308f8ceb420aae9d3fe Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 19 Mar 2021 12:30:20 +0000 Subject: [PATCH 08/12] add stricter ABI-compatibility checks for punned types (#2434) * add stricter ABI-compatibility checks for punned types * enforce type isomorphism for each pair of cast types to eliminate potential safety hole --- beacon_chain/beacon_chain_db.nim | 4 ++-- beacon_chain/beacon_chain_db_immutable.nim | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim index f86be8f22..05334f9bb 100644 --- a/beacon_chain/beacon_chain_db.nim +++ b/beacon_chain/beacon_chain_db.nim @@ -375,7 +375,7 @@ proc putState*(db: BeaconChainDB, key: Eth2Digest, value: var BeaconState) = updateImmutableValidators(db, db.immutableValidatorsMem, value.validators) db.put( subkey(BeaconStateNoImmutableValidators, key), - cast[ref BeaconStateNoImmutableValidators](addr value)[]) + isomorphicCast[BeaconStateNoImmutableValidators](value)) proc putState*(db: BeaconChainDB, value: var BeaconState) = db.putState(hash_tree_root(value), value) @@ -461,7 +461,7 @@ proc getStateOnlyMutableValidators( case db.get( subkey( BeaconStateNoImmutableValidators, key), - cast[ref BeaconStateNoImmutableValidators](addr output)[]) + isomorphicCast[BeaconStateNoImmutableValidators](output)) of GetResult.found: let numValidators = output.validators.len doAssert db.immutableValidatorsMem.len >= numValidators diff --git a/beacon_chain/beacon_chain_db_immutable.nim b/beacon_chain/beacon_chain_db_immutable.nim index 7df78c54e..ad610d32c 100644 --- a/beacon_chain/beacon_chain_db_immutable.nim +++ b/beacon_chain/beacon_chain_db_immutable.nim @@ -71,13 +71,21 @@ type current_justified_checkpoint*: Checkpoint finalized_checkpoint*: Checkpoint -static: +func getSizeofSig(x: auto, n: int = 0): seq[(string, int, int)] = + for name, value in x.fieldPairs: + when value is tuple|object: + result.add getSizeofSig(value, n + 1) + result.add((name, sizeof(value), n)) + +template isomorphicCast*[T, U](x: var U): T = # Each of these pairs of types has ABI-compatible memory representations, so # that the SSZ serialization can read and write directly from an object with # only mutable portions of BeaconState into a full BeaconState without using - # any extra copies. - doAssert sizeof(Validator) == sizeof(ValidatorStatus) - doAssert sizeof(BeaconState) == sizeof(BeaconStateNoImmutableValidators) + # extra copies. + static: + doAssert sizeof(T) == sizeof(U) + doAssert getSizeofSig(T()) == getSizeofSig(U()) + cast[ref T](addr x)[] proc loadImmutableValidators*(dbSeq: var auto): seq[ImmutableValidatorData] = for i in 0 ..< dbSeq.len: From 3743df64341db5691af6763d820c5311f8fd1cdd Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 22 Mar 2021 08:21:22 +0100 Subject: [PATCH 09/12] introduce unittest2 (#2437) This commit introduces `unittest2`, driven by the need to fix several long-standing issues with `unittest`, in particular exception handling, test isolation, timing and others. Upgrading to `unittest2` generally requires updating the unittest import and potentially fixing the code to support the new `one-proc-per-test` setup. * nim-unittest2 has been moved from Stefans repository to status-im - https://github.com/status-im/nim-unittest2/pull/2 fixes some exception handling issues and brings it in line with latest nim devel - a notable difference to unittest is that each test is run inside a proc which gives it a fresh stack and removes a lot of globals, meaning tests should run under conditions more similar to "normal" code and interfere with each other less - this is a crucial first step to improving the testing experience in general, that stefan implemented for the multithreading support * the multithreading in unittest2 is optional, and disabled by default * nim-testutils contains utilities for fuzzing, as well as other "large:ish" features that have dependencies - https://github.com/status-im/nim-testutils/pull/27 pulls in unittest2 so unittest2 can be kept "low-dependency" * nim-chronos needs the above unittest changes to test its own new, stricter exception handling - https://github.com/status-im/nim-chronos/pull/166 - the strictess introduced is expected to increase with time --- .gitmodules | 5 +++++ vendor/nim-testutils | 2 +- vendor/nim-unittest2 | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) create mode 160000 vendor/nim-unittest2 diff --git a/.gitmodules b/.gitmodules index 116acab0d..e4c89a194 100644 --- a/.gitmodules +++ b/.gitmodules @@ -203,3 +203,8 @@ url = https://github.com/status-im/nimbus-benchmarking.git ignore = untracked branch = master +[submodule "vendor/nim-unittest2"] + path = vendor/nim-unittest2 + url = https://github.com/status-im/nim-unittest2.git + ignore = untracked + branch = master diff --git a/vendor/nim-testutils b/vendor/nim-testutils index cc5d6e461..0f890d4a6 160000 --- a/vendor/nim-testutils +++ b/vendor/nim-testutils @@ -1 +1 @@ -Subproject commit cc5d6e46123e0cf5dfd14f5fc32f0d6f58a20645 +Subproject commit 0f890d4a667fcb2dcafd7243a079e5af2874db1d diff --git a/vendor/nim-unittest2 b/vendor/nim-unittest2 new file mode 160000 index 000000000..93674cbdb --- /dev/null +++ b/vendor/nim-unittest2 @@ -0,0 +1 @@ +Subproject commit 93674cbdbd3ce59e2d4d0cbdfac9ab62d9a6d28f From 895fe4baf12aadd27b400c5e0f9a672db1ca51aa Mon Sep 17 00:00:00 2001 From: Mentor Palokaj Date: Mon, 22 Mar 2021 08:22:56 +0100 Subject: [PATCH 10/12] Make it clear that the default port for ETH2 is 9000 udp/tcp (#2421) * Add default port info * Add default port info * Edit CLI and docs to reflect 9000 port * Apply suggestions from code review Co-authored-by: Jacek Sieka --- beacon_chain/conf.nim | 4 ++-- docs/the_nimbus_book/src/options.md | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 11c6cf093..14b396b15 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -154,12 +154,12 @@ type tcpPort* {. defaultValue: defaultEth2TcpPort - desc: "Listening TCP port for Ethereum LibP2P traffic" + desc: "Listening TCP port for Ethereum LibP2P traffic, the default is 9000" name: "tcp-port" }: Port udpPort* {. defaultValue: defaultEth2TcpPort - desc: "Listening UDP port for node discovery" + desc: "Listening UDP port for node discovery, default is 9000" name: "udp-port" }: Port maxPeers* {. diff --git a/docs/the_nimbus_book/src/options.md b/docs/the_nimbus_book/src/options.md index b93af9840..95ce552e5 100644 --- a/docs/the_nimbus_book/src/options.md +++ b/docs/the_nimbus_book/src/options.md @@ -1,6 +1,6 @@ # Command line options -You can pass any `nimbus_beacon_node` options to the `pyrmont` and `mainnet` scripts. For example, if you wanted to launch Nimbus on mainnet with a different base port, say `9100`, you would run: +You can pass any `nimbus_beacon_node` options to the `pyrmont` and `mainnet` scripts. For example, if you wanted to launch Nimbus on mainnet with different base ports than the default `9000/udp` and `9000/tcp`, say `9100/udp` and `9100/tcp`, you would run: ``` ./run-mainnet-beacon-node.sh --tcp-port=9100 --udp-port=9100 @@ -44,8 +44,8 @@ The following options are available: addresses. --listen-address Listening address for the Ethereum LibP2P and Discovery v5 traffic. - --tcp-port Listening TCP port for Ethereum LibP2P traffic. - --udp-port Listening UDP port for node discovery. + --tcp-port Listening TCP port for Ethereum LibP2P traffic, the default is 9000 + --udp-port Listening UDP port for node discovery, default is 9000 --max-peers The maximum number of peers to connect to. --nat Specify method to use for determining public address. Must be one of: any, none, upnp, pmp, extip:. @@ -87,4 +87,3 @@ Available sub-commands: ... ``` - From dfd99ec94336b784d71845421ac2c15288f8cef1 Mon Sep 17 00:00:00 2001 From: tersec Date: Mon, 22 Mar 2021 14:44:45 +0000 Subject: [PATCH 11/12] Altair (HF1/v1.1.0) minimal and mainnet presets/constants (#2444) * Altair mainnet & minimal presets * std/math not used --- beacon_chain/spec/datatypes/base.nim | 7 ++- beacon_chain/spec/preset_values.nim | 2 + beacon_chain/spec/presets.nim | 8 +++- beacon_chain/spec/presets/altair/mainnet.nim | 50 ++++++++++++++++++++ beacon_chain/spec/presets/altair/minimal.nim | 50 ++++++++++++++++++++ 5 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 beacon_chain/spec/presets/altair/mainnet.nim create mode 100644 beacon_chain/spec/presets/altair/minimal.nim diff --git a/beacon_chain/spec/datatypes/base.nim b/beacon_chain/spec/datatypes/base.nim index aa24becb7..63a5ca39c 100644 --- a/beacon_chain/spec/datatypes/base.nim +++ b/beacon_chain/spec/datatypes/base.nim @@ -114,8 +114,8 @@ template maxSize*(n: int) {.pragma.} type # Domains # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#domain-types DomainType* = enum + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#domain-types DOMAIN_BEACON_PROPOSER = 0 DOMAIN_BEACON_ATTESTER = 1 DOMAIN_RANDAO = 2 @@ -124,6 +124,11 @@ type DOMAIN_SELECTION_PROOF = 5 DOMAIN_AGGREGATE_AND_PROOF = 6 + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L31 + # Needs to be in same enum definition and is safe regardless of whether one + # only accesses phase 0 definitions + DOMAIN_SYNC_COMMITTEE = 7 + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#custom-types Domain* = array[32, byte] diff --git a/beacon_chain/spec/preset_values.nim b/beacon_chain/spec/preset_values.nim index e04d1dc66..d004d05b9 100644 --- a/beacon_chain/spec/preset_values.nim +++ b/beacon_chain/spec/preset_values.nim @@ -1,5 +1,6 @@ type PresetValue* {.pure.} = enum + ALTAIR_FORK_VERSION BASE_REWARD_FACTOR BLS_WITHDRAWAL_PREFIX CHURN_LIMIT_QUOTIENT @@ -12,6 +13,7 @@ type DOMAIN_DEPOSIT DOMAIN_RANDAO DOMAIN_SELECTION_PROOF + DOMAIN_SYNC_COMMITTEE DOMAIN_VOLUNTARY_EXIT EFFECTIVE_BALANCE_INCREMENT EJECTION_BALANCE diff --git a/beacon_chain/spec/presets.nim b/beacon_chain/spec/presets.nim index 1642dc73c..711920fcd 100644 --- a/beacon_chain/spec/presets.nim +++ b/beacon_chain/spec/presets.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2018-2020 Status Research & Development GmbH +# Copyright (c) 2018-2021 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -22,6 +22,7 @@ type RuntimePreset* = object GENESIS_FORK_VERSION*: Version + ALTAIR_FORK_VERSION*: Version GENESIS_DELAY*: uint64 MIN_GENESIS_ACTIVE_VALIDATOR_COUNT*: uint64 MIN_GENESIS_TIME*: uint64 @@ -40,6 +41,7 @@ const MIN_GENESIS_ACTIVE_VALIDATOR_COUNT, MIN_GENESIS_TIME, GENESIS_FORK_VERSION, + ALTAIR_FORK_VERSION, GENESIS_DELAY, ETH1_FOLLOW_DISTANCE, } @@ -60,10 +62,12 @@ const DOMAIN_VOLUNTARY_EXIT, DOMAIN_SELECTION_PROOF, DOMAIN_AGGREGATE_AND_PROOF, + DOMAIN_SYNC_COMMITTEE, CONFIG_NAME } presetValueTypes* = { + ALTAIR_FORK_VERSION: "Version", BLS_WITHDRAWAL_PREFIX: "byte", GENESIS_FORK_VERSION: "Version", }.toTable @@ -133,6 +137,7 @@ const mainnetRuntimePreset* = RuntimePreset( MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384, MIN_GENESIS_TIME: 1606824000, # Dec 1, 2020, 12pm UTC GENESIS_FORK_VERSION: Version [byte 0, 0, 0, 0], + ALTAIR_FORK_VERSION: Version [byte 1, 0, 0, 0], GENESIS_DELAY: 604800, ETH1_FOLLOW_DISTANCE: 2048) @@ -141,6 +146,7 @@ const MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64, MIN_GENESIS_TIME: 1606824000, # Dec 1, 2020, 12pm UTC GENESIS_FORK_VERSION: Version [byte 0, 0, 0, 1], + ALTAIR_FORK_VERSION: Version [byte 1, 0, 0, 0], GENESIS_DELAY: 300, ETH1_FOLLOW_DISTANCE: 16) diff --git a/beacon_chain/spec/presets/altair/mainnet.nim b/beacon_chain/spec/presets/altair/mainnet.nim new file mode 100644 index 000000000..7744f1725 --- /dev/null +++ b/beacon_chain/spec/presets/altair/mainnet.nim @@ -0,0 +1,50 @@ +# beacon_chain +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +# This file contains constants that are part of the spec and thus subject to +# serialization and spec updates. + +const + # Updated penalty values + # --------------------------------------------------------------- + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L5 + CONFIG_NAME* = "mainnet" + + INACTIVITY_PENALTY_QUOTIENT_ALTAIR* = 50331648 ##\ + ## 3 * 2**24 (= 50,331,648) + + MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR* = 64 + PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR* = 2 + + # Misc + # --------------------------------------------------------------- + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L15 + SYNC_COMMITTEE_SIZE* = 1024 + SYNC_SUBCOMMITTEE_SIZE* = 64 + INACTIVITY_SCORE_BIAS* = 4 + + # Time parameters + # --------------------------------------------------------------- + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L25 + EPOCHS_PER_SYNC_COMMITTEE_PERIOD* = 256 + + # Signature domains (DOMAIN_SYNC_COMMITTEE) in spec/datatypes/base + + # Fork + # --------------------------------------------------------------- + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L36 + + # ALTAIR_FORK_VERSION is a runtime preset + + ALTAIR_FORK_SLOT* = 0 # TBD + + # Sync protocol + # --------------------------------------------------------------- + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L43 + MIN_SYNC_COMMITTEE_PARTICIPANTS* = 1 + MAX_VALID_LIGHT_CLIENT_UPDATES* = 8192 + LIGHT_CLIENT_UPDATE_TIMEOUT* = 8192 diff --git a/beacon_chain/spec/presets/altair/minimal.nim b/beacon_chain/spec/presets/altair/minimal.nim new file mode 100644 index 000000000..61b94e3dd --- /dev/null +++ b/beacon_chain/spec/presets/altair/minimal.nim @@ -0,0 +1,50 @@ +# beacon_chain +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +# This file contains constants that are part of the spec and thus subject to +# serialization and spec updates. + +const + # Updated penalty values + # --------------------------------------------------------------- + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/minimal/altair.yaml#L5 + CONFIG_NAME* = "minimal" + + INACTIVITY_PENALTY_QUOTIENT_ALTAIR* = 50331648 ##\ + ## 3 * 2**24 (= 50,331,648) + + MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR* = 64 + PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR* = 2 + + # Misc + # --------------------------------------------------------------- + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/minimal/altair.yaml#L15 + SYNC_COMMITTEE_SIZE* = 32 + SYNC_SUBCOMMITTEE_SIZE* = 16 + INACTIVITY_SCORE_BIAS* = 4 + + # Time parameters + # --------------------------------------------------------------- + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/minimal/altair.yaml#L25 + EPOCHS_PER_SYNC_COMMITTEE_PERIOD* = 8 + + # Signature domains (DOMAIN_SYNC_COMMITTEE) in spec/datatypes/base + + # Fork + # --------------------------------------------------------------- + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/minimal/altair.yaml#L36 + + # ALTAIR_FORK_VERSION is a runtime preset + + ALTAIR_FORK_SLOT* = 0 # TBD + + # Sync protocol + # --------------------------------------------------------------- + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/minimal/altair.yaml#L43 + MIN_SYNC_COMMITTEE_PARTICIPANTS* = 1 + MAX_VALID_LIGHT_CLIENT_UPDATES* = 32 + LIGHT_CLIENT_UPDATE_TIMEOUT* = 32 From b2c4441834aa81f9a5917882b0be64fad18f3bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Mon, 22 Mar 2021 16:40:26 +0100 Subject: [PATCH 12/12] book: document "logrotate" (#2390) * book: document "logrotate" * Update docs/the_nimbus_book/src/log-rotate.md Co-authored-by: Sacha Saint-Leger * Update docs/the_nimbus_book/src/log-rotate.md Co-authored-by: Sacha Saint-Leger * Update docs/the_nimbus_book/src/log-rotate.md Co-authored-by: Sacha Saint-Leger * Update docs/the_nimbus_book/src/log-rotate.md Co-authored-by: Sacha Saint-Leger * Update docs/the_nimbus_book/src/log-rotate.md Co-authored-by: Sacha Saint-Leger * rewording * Update docs/the_nimbus_book/src/log-rotate.md Co-authored-by: Sacha Saint-Leger Co-authored-by: Sacha Saint-Leger --- Makefile | 1 + docs/the_nimbus_book/src/log-rotate.md | 52 ++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 120deb7b0..7d146b798 100644 --- a/Makefile +++ b/Makefile @@ -514,6 +514,7 @@ libnfuzz.a: | build deps book: which mdbook &>/dev/null || { echo "'mdbook' not found in PATH. See 'docs/README.md'. Aborting."; exit 1; } which mdbook-toc &>/dev/null || { echo "'mdbook-toc' not found in PATH. See 'docs/README.md'. Aborting."; exit 1; } + which mdbook-open-on-gh &>/dev/null || { echo "'mdbook-open-on-gh' not found in PATH. See 'docs/README.md'. Aborting."; exit 1; } cd docs/the_nimbus_book && \ mdbook build diff --git a/docs/the_nimbus_book/src/log-rotate.md b/docs/the_nimbus_book/src/log-rotate.md index 426b06167..d8ee4adf7 100644 --- a/docs/the_nimbus_book/src/log-rotate.md +++ b/docs/the_nimbus_book/src/log-rotate.md @@ -1,14 +1,41 @@ # Log rotation -Nimbus logs are written to the console, and optionally to a file. Writing to a file for a long-running process may lead to difficulties when the file grows large. This is typically solved with a *log rotator*. A log rotator is responsible for switching the written to file, as well as compressing and removing old logs. +Nimbus logs are written to stdout, and optionally to a file. Writing to a file for a long-running process may lead to difficulties when the file grows large. This is typically solved with a *log rotator*. A log rotator is responsible for switching the written-to file, as well as compressing and removing old logs. -To set up file-based log rotation, an application such as [rotatelogs](https://httpd.apache.org/docs/2.4/programs/rotatelogs.html) is used - `rotatelogs` is available on most servers and can be used with `docker`, `systemd` and manual setups to write rotated logs files. +## Using "logrotate" -In particular, when using `systemd` and its accompanying `journald` log daemon, this setup avoids clogging the the system log by keeping the Nimbus logs in a separate location. +[logrotate](https://github.com/logrotate/logrotate) provides log rotation and compression. The corresponding package will install its Cron hooks (or Systemd timer) -- all you have to do is add a configuration file for Nimbus-eth2 in "/etc/logrotate.d/nimbus-eth2": -## Compression +```text +/var/log/nimbus-eth2/*.log { + compress + missingok + copytruncate +} +``` -`rotatelogs` works by reading stdin and redirecting it to a file based on a name pattern. Whenever the log is about to be rotated, the application invokes a shell script with the old and new log files. Our aim is to compress the log file to save space. [repo](https://github.com/status-im/nimbus-eth2/tree/unstable/scripts/rotatelogs-compress.sh) provides a helper script to do so: +The above assumes you've configured Nimbus-eth2 to write its logs to "/var/log/nimbus-eth2/" (usually by redirecting stout and stderr from your init script). + +"copytruncate" is required because, when it comes to moving the log file, `logrotate`'s default behaviour requires application support for re-opening that log file at runtime (something which is currently lacking). So, instead of a move, we tell `logrotate` to do a copy and a truncation of the existing file. A few log lines may be lost in the process. + +You can control rotation frequency and the maximum number of log files kept by using the global configuration file - "/etc/logrotate.conf": + +```text +# rotate daily +daily +# only keep logs from the last 7 days +rotate 7 +``` + +## Using "rotatelogs" + +[rotatelogs](https://httpd.apache.org/docs/2.4/programs/rotatelogs.html) is available on most servers and can be used with `Docker`, `Systemd` and manual setups to write rotated logs files. + +In particular, when `Systemd` and its accompanying `Journald` log daemon are used, this setup avoids clogging the system log by keeping the Nimbus logs in a separate location. + +### Compression + +`rotatelogs` works by reading stdin and redirecting it to a file based on a name pattern. Whenever the log is about to be rotated, the application invokes a shell script with the old and new log files. Our aim is to compress the log file to save space. The [Nimbus-eth2 repo](https://github.com/status-im/nimbus-eth2/tree/unstable/scripts/rotatelogs-compress.sh) provides a helper script that does this: ```bash # Create a rotation script for rotatelogs @@ -26,7 +53,7 @@ EOF chmod +x rotatelogs-compress.sh ``` -## Build +### Build Logs in files generally don't benefit from colors. To avoid colors being written to the file, additional flags can be added to the Nimbus [build process](./build.md) -- these flags are best saved in a build script to which one can add more options. Future versions of Nimbus will support disabling colors at runtime. @@ -39,7 +66,7 @@ make NIMFLAGS="-d:chronicles_colors=off -d:chronicles_sinks=textlines" nimbus_be EOF ``` -## Run +### Run The final step is to redirect logs to `rotatelogs` using a pipe when starting Nimbus: @@ -47,7 +74,7 @@ The final step is to redirect logs to `rotatelogs` using a pipe when starting Ni build/nimbus_beacon_node \ --network:pyrmont \ --web3-url="$WEB3URL" \ - --data-dir:$DATADIR | rotatelogs -L "$DATADIR/nbc_bn.log" -p "/path/to/rotatelogs-compress.sh" -D -f -c "$DATADIR/log/nbc_bn_%Y%m%d%H%M%S.log" 3600 + --data-dir:$DATADIR 2>&1 | rotatelogs -L "$DATADIR/nbc_bn.log" -p "/path/to/rotatelogs-compress.sh" -D -f -c "$DATADIR/log/nbc_bn_%Y%m%d%H%M%S.log" 3600 ``` The options used in this example do the following: @@ -58,3 +85,12 @@ The options used in this example do the following: * `-f` - opens the log immediately when starting `rotatelogs` * `-c "$DATADIR/log/nbc_bn_%Y%m%d%H%M%S.log"` - includes timestamp in log filename * `3600` - rotates logs every hour (3600 seconds) + +### Deleting old logs + +`rotatelogs` will not do this for you, so you'll need a Cron script (or Systemd timer): + +```bash +# delete log files older than 7 days +find "$DATADIR/log" -name 'nbc_bn_*.log' -mtime +7 -exec rm '{}' \+ +```