Merge branch 'unstable' of github.com:status-im/nim-beacon-chain into unstable

This commit is contained in:
yslcrypto 2021-03-22 17:16:12 +01:00
commit c7847b39f5
21 changed files with 196 additions and 34 deletions

5
.gitmodules vendored
View File

@ -203,3 +203,8 @@
url = https://github.com/status-im/nimbus-benchmarking.git
ignore = untracked
branch = master
[submodule "vendor/nim-unittest2"]
path = vendor/nim-unittest2
url = https://github.com/status-im/nim-unittest2.git
ignore = untracked
branch = master

View File

@ -137,6 +137,7 @@ endif
#- deletes binaries that might need to be rebuilt after a Git pull
update: | update-common
rm -f build/generate_makefile
rm -fr nimcache/
# nim-libbacktrace
libbacktrace:
@ -513,6 +514,7 @@ libnfuzz.a: | build deps
book:
which mdbook &>/dev/null || { echo "'mdbook' not found in PATH. See 'docs/README.md'. Aborting."; exit 1; }
which mdbook-toc &>/dev/null || { echo "'mdbook-toc' not found in PATH. See 'docs/README.md'. Aborting."; exit 1; }
which mdbook-open-on-gh &>/dev/null || { echo "'mdbook-open-on-gh' not found in PATH. See 'docs/README.md'. Aborting."; exit 1; }
cd docs/the_nimbus_book && \
mdbook build

View File

@ -1,7 +1,5 @@
# Nimbus Eth2 (Beacon Chain)
[![Build Status (Travis)](https://img.shields.io/travis/status-im/nimbus-eth2/master.svg?label=Linux%20/%20macOS "Linux/macOS build status (Travis)")](https://travis-ci.org/status-im/nimbus-eth2)
[![Build Status (Azure)](https://dev.azure.com/nimbus-dev/nimbus-eth2/_apis/build/status/status-im.nimbus-eth2?branchName=master)](https://dev.azure.com/nimbus-dev/nimbus-eth2/_build/latest?definitionId=3&branchName=master)
[![Github Actions CI](https://github.com/status-im/nimbus-eth2/workflows/Nimbus%20nimbus-eth2%20CI/badge.svg)](https://github.com/status-im/nim-blscurve/actions?query=workflow%3A%22BLSCurve+CI%22)
[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)

View File

@ -375,7 +375,7 @@ proc putState*(db: BeaconChainDB, key: Eth2Digest, value: var BeaconState) =
updateImmutableValidators(db, db.immutableValidatorsMem, value.validators)
db.put(
subkey(BeaconStateNoImmutableValidators, key),
cast[ref BeaconStateNoImmutableValidators](addr value)[])
isomorphicCast[BeaconStateNoImmutableValidators](value))
proc putState*(db: BeaconChainDB, value: var BeaconState) =
db.putState(hash_tree_root(value), value)
@ -461,7 +461,7 @@ proc getStateOnlyMutableValidators(
case db.get(
subkey(
BeaconStateNoImmutableValidators, key),
cast[ref BeaconStateNoImmutableValidators](addr output)[])
isomorphicCast[BeaconStateNoImmutableValidators](output))
of GetResult.found:
let numValidators = output.validators.len
doAssert db.immutableValidatorsMem.len >= numValidators

View File

@ -71,13 +71,21 @@ type
current_justified_checkpoint*: Checkpoint
finalized_checkpoint*: Checkpoint
static:
func getSizeofSig(x: auto, n: int = 0): seq[(string, int, int)] =
for name, value in x.fieldPairs:
when value is tuple|object:
result.add getSizeofSig(value, n + 1)
result.add((name, sizeof(value), n))
template isomorphicCast*[T, U](x: var U): T =
# Each of these pairs of types has ABI-compatible memory representations, so
# that the SSZ serialization can read and write directly from an object with
# only mutable portions of BeaconState into a full BeaconState without using
# any extra copies.
doAssert sizeof(Validator) == sizeof(ValidatorStatus)
doAssert sizeof(BeaconState) == sizeof(BeaconStateNoImmutableValidators)
# extra copies.
static:
doAssert sizeof(T) == sizeof(U)
doAssert getSizeofSig(T()) == getSizeofSig(U())
cast[ref T](addr x)[]
proc loadImmutableValidators*(dbSeq: var auto): seq[ImmutableValidatorData] =
for i in 0 ..< dbSeq.len:

View File

@ -154,12 +154,12 @@ type
tcpPort* {.
defaultValue: defaultEth2TcpPort
desc: "Listening TCP port for Ethereum LibP2P traffic"
desc: "Listening TCP port for Ethereum LibP2P traffic, the default is 9000"
name: "tcp-port" }: Port
udpPort* {.
defaultValue: defaultEth2TcpPort
desc: "Listening UDP port for node discovery"
desc: "Listening UDP port for node discovery, default is 9000"
name: "udp-port" }: Port
maxPeers* {.

View File

@ -13,7 +13,8 @@ import
# Nimble packages
stew/[objects, byteutils, endians2, io2], stew/shims/macros,
chronos, confutils, metrics, json_rpc/[rpcclient, rpcserver, jsonmarshal],
chronos, confutils, metrics, metrics/chronos_httpserver,
json_rpc/[rpcclient, rpcserver, jsonmarshal],
chronicles, bearssl, blscurve,
json_serialization/std/[options, sets, net], serialization/errors,
@ -1637,7 +1638,7 @@ proc doRunBeaconNode(config: var BeaconNodeConf, rng: ref BrHmacDrbgContext) =
let metricsAddress = config.metricsAddress
notice "Starting metrics HTTP server",
url = "http://" & $metricsAddress & ":" & $config.metricsPort & "/metrics"
metrics.startHttpServer($metricsAddress, config.metricsPort)
startMetricsHttpServer($metricsAddress, config.metricsPort)
else:
warn "Metrics support disabled, see https://status-im.github.io/nimbus-eth2/metrics-pretty-pictures.html#simple-metrics"

View File

@ -114,8 +114,8 @@ template maxSize*(n: int) {.pragma.}
type
# Domains
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#domain-types
DomainType* = enum
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#domain-types
DOMAIN_BEACON_PROPOSER = 0
DOMAIN_BEACON_ATTESTER = 1
DOMAIN_RANDAO = 2
@ -124,6 +124,11 @@ type
DOMAIN_SELECTION_PROOF = 5
DOMAIN_AGGREGATE_AND_PROOF = 6
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L31
# Needs to be in same enum definition and is safe regardless of whether one
# only accesses phase 0 definitions
DOMAIN_SYNC_COMMITTEE = 7
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#custom-types
Domain* = array[32, byte]

View File

@ -1,5 +1,6 @@
type
PresetValue* {.pure.} = enum
ALTAIR_FORK_VERSION
BASE_REWARD_FACTOR
BLS_WITHDRAWAL_PREFIX
CHURN_LIMIT_QUOTIENT
@ -12,6 +13,7 @@ type
DOMAIN_DEPOSIT
DOMAIN_RANDAO
DOMAIN_SELECTION_PROOF
DOMAIN_SYNC_COMMITTEE
DOMAIN_VOLUNTARY_EXIT
EFFECTIVE_BALANCE_INCREMENT
EJECTION_BALANCE

View File

@ -1,5 +1,5 @@
# beacon_chain
# Copyright (c) 2018-2020 Status Research & Development GmbH
# Copyright (c) 2018-2021 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@ -22,6 +22,7 @@ type
RuntimePreset* = object
GENESIS_FORK_VERSION*: Version
ALTAIR_FORK_VERSION*: Version
GENESIS_DELAY*: uint64
MIN_GENESIS_ACTIVE_VALIDATOR_COUNT*: uint64
MIN_GENESIS_TIME*: uint64
@ -40,6 +41,7 @@ const
MIN_GENESIS_ACTIVE_VALIDATOR_COUNT,
MIN_GENESIS_TIME,
GENESIS_FORK_VERSION,
ALTAIR_FORK_VERSION,
GENESIS_DELAY,
ETH1_FOLLOW_DISTANCE,
}
@ -60,10 +62,12 @@ const
DOMAIN_VOLUNTARY_EXIT,
DOMAIN_SELECTION_PROOF,
DOMAIN_AGGREGATE_AND_PROOF,
DOMAIN_SYNC_COMMITTEE,
CONFIG_NAME
}
presetValueTypes* = {
ALTAIR_FORK_VERSION: "Version",
BLS_WITHDRAWAL_PREFIX: "byte",
GENESIS_FORK_VERSION: "Version",
}.toTable
@ -133,6 +137,7 @@ const mainnetRuntimePreset* = RuntimePreset(
MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384,
MIN_GENESIS_TIME: 1606824000, # Dec 1, 2020, 12pm UTC
GENESIS_FORK_VERSION: Version [byte 0, 0, 0, 0],
ALTAIR_FORK_VERSION: Version [byte 1, 0, 0, 0],
GENESIS_DELAY: 604800,
ETH1_FOLLOW_DISTANCE: 2048)
@ -141,6 +146,7 @@ const
MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64,
MIN_GENESIS_TIME: 1606824000, # Dec 1, 2020, 12pm UTC
GENESIS_FORK_VERSION: Version [byte 0, 0, 0, 1],
ALTAIR_FORK_VERSION: Version [byte 1, 0, 0, 0],
GENESIS_DELAY: 300,
ETH1_FOLLOW_DISTANCE: 16)

View File

@ -0,0 +1,50 @@
# beacon_chain
# Copyright (c) 2021 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# This file contains constants that are part of the spec and thus subject to
# serialization and spec updates.
const
# Updated penalty values
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L5
CONFIG_NAME* = "mainnet"
INACTIVITY_PENALTY_QUOTIENT_ALTAIR* = 50331648 ##\
## 3 * 2**24 (= 50,331,648)
MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR* = 64
PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR* = 2
# Misc
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L15
SYNC_COMMITTEE_SIZE* = 1024
SYNC_SUBCOMMITTEE_SIZE* = 64
INACTIVITY_SCORE_BIAS* = 4
# Time parameters
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L25
EPOCHS_PER_SYNC_COMMITTEE_PERIOD* = 256
# Signature domains (DOMAIN_SYNC_COMMITTEE) in spec/datatypes/base
# Fork
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L36
# ALTAIR_FORK_VERSION is a runtime preset
ALTAIR_FORK_SLOT* = 0 # TBD
# Sync protocol
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/mainnet/altair.yaml#L43
MIN_SYNC_COMMITTEE_PARTICIPANTS* = 1
MAX_VALID_LIGHT_CLIENT_UPDATES* = 8192
LIGHT_CLIENT_UPDATE_TIMEOUT* = 8192

View File

@ -0,0 +1,50 @@
# beacon_chain
# Copyright (c) 2021 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# This file contains constants that are part of the spec and thus subject to
# serialization and spec updates.
const
# Updated penalty values
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/minimal/altair.yaml#L5
CONFIG_NAME* = "minimal"
INACTIVITY_PENALTY_QUOTIENT_ALTAIR* = 50331648 ##\
## 3 * 2**24 (= 50,331,648)
MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR* = 64
PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR* = 2
# Misc
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/minimal/altair.yaml#L15
SYNC_COMMITTEE_SIZE* = 32
SYNC_SUBCOMMITTEE_SIZE* = 16
INACTIVITY_SCORE_BIAS* = 4
# Time parameters
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/minimal/altair.yaml#L25
EPOCHS_PER_SYNC_COMMITTEE_PERIOD* = 8
# Signature domains (DOMAIN_SYNC_COMMITTEE) in spec/datatypes/base
# Fork
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/minimal/altair.yaml#L36
# ALTAIR_FORK_VERSION is a runtime preset
ALTAIR_FORK_SLOT* = 0 # TBD
# Sync protocol
# ---------------------------------------------------------------
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.1/configs/minimal/altair.yaml#L43
MIN_SYNC_COMMITTEE_PARTICIPANTS* = 1
MAX_VALID_LIGHT_CLIENT_UPDATES* = 32
LIGHT_CLIENT_UPDATE_TIMEOUT* = 32

View File

@ -11,7 +11,7 @@ import
# Status
eth/db/[kvstore, kvstore_sqlite3],
chronicles,
nimcrypto/[hash, utils],
nimcrypto/hash,
serialization,
json_serialization,
# Internal

View File

@ -1,14 +1,41 @@
# Log rotation
Nimbus logs are written to the console, and optionally to a file. Writing to a file for a long-running process may lead to difficulties when the file grows large. This is typically solved with a *log rotator*. A log rotator is responsible for switching the written to file, as well as compressing and removing old logs.
Nimbus logs are written to stdout, and optionally to a file. Writing to a file for a long-running process may lead to difficulties when the file grows large. This is typically solved with a *log rotator*. A log rotator is responsible for switching the written-to file, as well as compressing and removing old logs.
To set up file-based log rotation, an application such as [rotatelogs](https://httpd.apache.org/docs/2.4/programs/rotatelogs.html) is used - `rotatelogs` is available on most servers and can be used with `docker`, `systemd` and manual setups to write rotated logs files.
## Using "logrotate"
In particular, when using `systemd` and its accompanying `journald` log daemon, this setup avoids clogging the the system log by keeping the Nimbus logs in a separate location.
[logrotate](https://github.com/logrotate/logrotate) provides log rotation and compression. The corresponding package will install its Cron hooks (or Systemd timer) -- all you have to do is add a configuration file for Nimbus-eth2 in "/etc/logrotate.d/nimbus-eth2":
## Compression
```text
/var/log/nimbus-eth2/*.log {
compress
missingok
copytruncate
}
```
`rotatelogs` works by reading stdin and redirecting it to a file based on a name pattern. Whenever the log is about to be rotated, the application invokes a shell script with the old and new log files. Our aim is to compress the log file to save space. [repo](https://github.com/status-im/nimbus-eth2/tree/unstable/scripts/rotatelogs-compress.sh) provides a helper script to do so:
The above assumes you've configured Nimbus-eth2 to write its logs to "/var/log/nimbus-eth2/" (usually by redirecting stout and stderr from your init script).
"copytruncate" is required because, when it comes to moving the log file, `logrotate`'s default behaviour requires application support for re-opening that log file at runtime (something which is currently lacking). So, instead of a move, we tell `logrotate` to do a copy and a truncation of the existing file. A few log lines may be lost in the process.
You can control rotation frequency and the maximum number of log files kept by using the global configuration file - "/etc/logrotate.conf":
```text
# rotate daily
daily
# only keep logs from the last 7 days
rotate 7
```
## Using "rotatelogs"
[rotatelogs](https://httpd.apache.org/docs/2.4/programs/rotatelogs.html) is available on most servers and can be used with `Docker`, `Systemd` and manual setups to write rotated logs files.
In particular, when `Systemd` and its accompanying `Journald` log daemon are used, this setup avoids clogging the system log by keeping the Nimbus logs in a separate location.
### Compression
`rotatelogs` works by reading stdin and redirecting it to a file based on a name pattern. Whenever the log is about to be rotated, the application invokes a shell script with the old and new log files. Our aim is to compress the log file to save space. The [Nimbus-eth2 repo](https://github.com/status-im/nimbus-eth2/tree/unstable/scripts/rotatelogs-compress.sh) provides a helper script that does this:
```bash
# Create a rotation script for rotatelogs
@ -26,7 +53,7 @@ EOF
chmod +x rotatelogs-compress.sh
```
## Build
### Build
Logs in files generally don't benefit from colors. To avoid colors being written to the file, additional flags can be added to the Nimbus [build process](./build.md) -- these flags are best saved in a build script to which one can add more options. Future versions of Nimbus will support disabling colors at runtime.
@ -39,7 +66,7 @@ make NIMFLAGS="-d:chronicles_colors=off -d:chronicles_sinks=textlines" nimbus_be
EOF
```
## Run
### Run
The final step is to redirect logs to `rotatelogs` using a pipe when starting Nimbus:
@ -47,7 +74,7 @@ The final step is to redirect logs to `rotatelogs` using a pipe when starting Ni
build/nimbus_beacon_node \
--network:pyrmont \
--web3-url="$WEB3URL" \
--data-dir:$DATADIR | rotatelogs -L "$DATADIR/nbc_bn.log" -p "/path/to/rotatelogs-compress.sh" -D -f -c "$DATADIR/log/nbc_bn_%Y%m%d%H%M%S.log" 3600
--data-dir:$DATADIR 2>&1 | rotatelogs -L "$DATADIR/nbc_bn.log" -p "/path/to/rotatelogs-compress.sh" -D -f -c "$DATADIR/log/nbc_bn_%Y%m%d%H%M%S.log" 3600
```
The options used in this example do the following:
@ -58,3 +85,12 @@ The options used in this example do the following:
* `-f` - opens the log immediately when starting `rotatelogs`
* `-c "$DATADIR/log/nbc_bn_%Y%m%d%H%M%S.log"` - includes timestamp in log filename
* `3600` - rotates logs every hour (3600 seconds)
### Deleting old logs
`rotatelogs` will not do this for you, so you'll need a Cron script (or Systemd timer):
```bash
# delete log files older than 7 days
find "$DATADIR/log" -name 'nbc_bn_*.log' -mtime +7 -exec rm '{}' \+
```

View File

@ -1,6 +1,6 @@
# Command line options
You can pass any `nimbus_beacon_node` options to the `pyrmont` and `mainnet` scripts. For example, if you wanted to launch Nimbus on mainnet with a different base port, say `9100`, you would run:
You can pass any `nimbus_beacon_node` options to the `pyrmont` and `mainnet` scripts. For example, if you wanted to launch Nimbus on mainnet with different base ports than the default `9000/udp` and `9000/tcp`, say `9100/udp` and `9100/tcp`, you would run:
```
./run-mainnet-beacon-node.sh --tcp-port=9100 --udp-port=9100
@ -44,8 +44,8 @@ The following options are available:
addresses.
--listen-address Listening address for the Ethereum LibP2P and Discovery v5
traffic.
--tcp-port Listening TCP port for Ethereum LibP2P traffic.
--udp-port Listening UDP port for node discovery.
--tcp-port Listening TCP port for Ethereum LibP2P traffic, the default is 9000
--udp-port Listening UDP port for node discovery, default is 9000
--max-peers The maximum number of peers to connect to.
--nat Specify method to use for determining public address. Must be
one of: any, none, upnp, pmp, extip:<IP>.
@ -87,4 +87,3 @@ Available sub-commands:
...
```

View File

@ -1,5 +1,5 @@
# Nimbus
# Copyright (c) 2018 Status Research & Development GmbH
# Copyright (c) 2018-2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or https://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or https://opensource.org/licenses/MIT)
@ -19,8 +19,7 @@ import
# Internal
../../beacon_chain/validators/[
slashing_protection,
slashing_protection_v1,
slashing_protection_v2
slashing_protection_v1
],
../../beacon_chain/spec/[datatypes, digest, crypto, presets],
# Test utilies

2
vendor/nim-chronos vendored

@ -1 +1 @@
Subproject commit 03707426e43d03cccc1de2e7284de168b79f7bf6
Subproject commit c8eefb9382a786993fc703386b0bd446ecf9c037

2
vendor/nim-metrics vendored

@ -1 +1 @@
Subproject commit 22a3867341f7b0a9d55661b41d7ee5febe35c86b
Subproject commit 105af2bfbd4896e8b4086d3dff1e6c187e9d0a41

2
vendor/nim-stew vendored

@ -1 +1 @@
Subproject commit 42475fd2f1919acd11be2fdc75fd6e2cefc99e90
Subproject commit 6bcb21184aeb96ce6c62e187a64d678b74609f1e

@ -1 +1 @@
Subproject commit cc5d6e46123e0cf5dfd14f5fc32f0d6f58a20645
Subproject commit 0f890d4a667fcb2dcafd7243a079e5af2874db1d

1
vendor/nim-unittest2 vendored Submodule

@ -0,0 +1 @@
Subproject commit 93674cbdbd3ce59e2d4d0cbdfac9ab62d9a6d28f