Logging and startup improvements (#3038)

* Logging and startup improvements

Color support for released binaries!

* startup scripts no longer log to file by default - this only affects
source builds - released binaries don't support file logging
* add --log-stdout option to control logging to stdout (colors, json)
* detect tty:s vs redirected logs and log accordingly
* add option to disable log colors at runtime
* simplify several "common" logs, showing the most important information
earlier and more clearly
* remove line numbers / file information / tid - these take up space and
are of little use to end users
  * still enabled in debug builds and tools
* remove `testnet_servers_image` compile-time option
* server images, released binaries and compile-from-source now offer
the same behaviour and features
* fixes https://github.com/status-im/nimbus-eth2/issues/2326
* fixes https://github.com/status-im/nimbus-eth2/issues/1794
* remove instanteneous block speed from sync message, keeping only
average

before:

```
INF 2021-10-28 16:45:59.000+02:00 Slot start                                 topics="beacnde" tid=386429 file=nimbus_beacon_node.nim:884 lastSlot=2384027 wallSlot=2384028 delay=461us84ns peers=0 head=75a10ee5:3348 headEpoch=104 finalized=cd6804ba:3264 finalizedEpoch=102 sync="wwwwwwwwww:0:0.0000:0.0000:00h00m (3348)"
INF 2021-10-28 16:45:59.046+02:00 Slot end                                   topics="beacnde" tid=386429 file=nimbus_beacon_node.nim:821 slot=2384028 nextSlot=2384029 head=75a10ee5:3348 headEpoch=104 finalizedHead=cd6804ba:3264 finalizedEpoch=102 nextAttestationSlot=-1 nextProposalSlot=-1 nextActionWait=n/a
```

after:

```
INF 2021-10-28 22:43:23.033+02:00 Slot start                                 topics="beacnde" slot=2385815 epoch=74556 sync="DDPDDPUDDD:10:5.2258:01h19m (2361088)" peers=37 head=eacd2dae:2361096 finalized=73782:a4751487 delay=33ms687us715ns
INF 2021-10-28 22:43:23.291+02:00 Slot end                                   topics="beacnde" slot=2385815 nextActionWait=n/a nextAttestationSlot=-1 nextProposalSlot=-1 head=eacd2dae:2361096
```

* fix comment

* documentation updates

* mention `--log-file` may be deprecated in the future
* update various docs

.appveyor.yml

@@ -33,7 +33,6 @@ build_script:
 
 test_script:
   - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% LOG_LEVEL=TRACE
-  - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% LOG_LEVEL=TRACE NIMFLAGS="-d:testnet_servers_image" nimbus_beacon_node
   - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_TEST_FIXTURES_SCRIPT=1 test
 
 deploy: off

.github/workflows/ci.yml

@@ -274,7 +274,7 @@ jobs:
           if [[ "${{ runner.os }}" == "macOS" ]]; then
             ulimit -n 1024
           fi
-          make -j$ncpu ARCH_OVERRIDE=$PLATFORM LOG_LEVEL=TRACE NIMFLAGS="-d:testnet_servers_image" nimbus_beacon_node nimbus_validator_client
+          make -j$ncpu ARCH_OVERRIDE=$PLATFORM LOG_LEVEL=TRACE nimbus_beacon_node nimbus_validator_client
 
       - name: Run nimbus-eth2 tests
         if: matrix.target.TEST_KIND == 'unit-tests'

.travis.yml

@@ -56,5 +56,4 @@ script:
   # Building Nim-1.0.4 takes up to 10 minutes on Travis - the time limit after which jobs are cancelled for having no output
   - make -j${NPROC} NIMFLAGS="--parallelBuild:2" V=1 update # to allow a newer Nim version to be detected
   - make -j${NPROC} NIMFLAGS="--parallelBuild:2" LOG_LEVEL=TRACE
-  - make -j${NPROC} NIMFLAGS="--parallelBuild:2 -d:testnet_servers_image" LOG_LEVEL=TRACE nimbus_beacon_node
   - make -j${NPROC} NIMFLAGS="--parallelBuild:2" DISABLE_TEST_FIXTURES_SCRIPT=1 test

Jenkinsfile

@@ -36,7 +36,7 @@ def runStages() {
 			sh """#!/bin/bash
 			set -e
 			make -j${env.NPROC}
-			make -j${env.NPROC} LOG_LEVEL=TRACE NIMFLAGS='-d:testnet_servers_image' nimbus_beacon_node
+			make -j${env.NPROC} LOG_LEVEL=TRACE
 			"""
 		}
 

azure-pipelines.yml

@@ -64,10 +64,6 @@ jobs:
           git config --global core.autocrlf false
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} LOG_LEVEL=TRACE
-          if [[ $PLATFORM == "x64" ]]; then
-            # everything builds more slowly on 32-bit, since there's no libbacktrace support
-            mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} LOG_LEVEL=TRACE NIMFLAGS="-d:testnet_servers_image" nimbus_beacon_node
-          fi
           file build/nimbus_beacon_node
           # fail fast
           export NIMTEST_ABORT_ON_ERROR=1

beacon_chain/conf.nim

@@ -70,6 +70,13 @@ type
     v2
     both
 
+  StdoutLogKind* {.pure.} = enum
+    Auto = "auto"
+    Colors = "colors"
+    NoColors = "nocolors"
+    Json = "json"
+    None = "none"
+
   SlashProtCmd* = enum
     `import` = "Import a EIP-3076 slashing protection interchange file"
     `export` = "Export a EIP-3076 slashing protection interchange file"
@@ -82,6 +89,12 @@ type
       defaultValue: "INFO"
       name: "log-level" }: string
 
+    logStdout* {.
+      desc: "Specifies what kind of logs should be written to stdout (auto, colors, nocolors, json)"
+      defaultValueDesc: "auto"
+      defaultValue: StdoutLogKind.Auto
+      name: "log-stdout" }: StdoutLogKind
+
     logFile* {.
       desc: "Specifies a path for the written Json log file"
       name: "log-file" }: Option[OutFile]
@@ -518,6 +531,12 @@ type
       defaultValue: "INFO"
       name: "log-level" }: string
 
+    logStdout* {.
+      desc: "Specifies what kind of logs should be written to stdout (auto, colors, nocolors, json)"
+      defaultValueDesc: "auto"
+      defaultValue: StdoutLogKind.Auto
+      name: "log-stdout" }: StdoutLogKind
+
     logFile* {.
       desc: "Specifies a path for the written Json log file"
       name: "log-file" }: Option[OutFile]

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -1220,9 +1220,8 @@ proc updateHead*(
     beacon_reorgs_total.inc()
   else:
     debug "Updated head block",
+      head = shortLog(dag.headState.blck),
       stateRoot = shortLog(getStateRoot(dag.headState.data)),
-      headBlock = shortLog(dag.headState.blck),
-      stateSlot = shortLog(getStateField(dag.headState.data, slot)),
       justified = shortLog(getStateField(
         dag.headState.data, current_justified_checkpoint)),
       finalized = shortLog(getStateField(
@@ -1285,9 +1284,13 @@ proc updateHead*(
     beacon_current_active_validators.set(number_of_active_validators)
 
   if finalizedHead != dag.finalizedHead:
-    notice "Reached new finalization checkpoint",
+    info "Reached new finalization checkpoint",
-      newFinalizedHead = shortLog(finalizedHead),
+      head = shortLog(dag.headState.blck),
-      oldFinalizedHead = shortLog(dag.finalizedHead)
+      stateRoot = shortLog(getStateRoot(dag.headState.data)),
+      justified = shortLog(getStateField(
+        dag.headState.data, current_justified_checkpoint)),
+      finalized = shortLog(getStateField(
+        dag.headState.data, finalized_checkpoint))
 
     dag.finalizedHead = finalizedHead
 

beacon_chain/nimbus_beacon_node.nim

@@ -836,19 +836,14 @@ proc onSlotEnd(node: BeaconNode, slot: Slot) {.async.} =
 
   info "Slot end",
     slot = shortLog(slot),
-    nextSlot = shortLog(slot + 1),
-    head = shortLog(node.dag.head),
-    headEpoch = shortLog(node.dag.head.slot.compute_epoch_at_slot()),
-    finalizedHead = shortLog(node.dag.finalizedHead.blck),
-    finalizedEpoch =
-      shortLog(node.dag.finalizedHead.blck.slot.compute_epoch_at_slot()),
-    nextAttestationSlot = displayInt64(nextAttestationSlot),
-    nextProposalSlot = displayInt64(nextProposalSlot),
     nextActionWait =
       if nextAttestationSlot == FAR_FUTURE_SLOT:
         "n/a"
       else:
-        shortLog(nextActionWaitTime)
+        shortLog(nextActionWaitTime),
+    nextAttestationSlot = displayInt64(nextAttestationSlot),
+    nextProposalSlot = displayInt64(nextProposalSlot),
+    head = shortLog(node.dag.head)
 
   if nextAttestationSlot != FAR_FUTURE_SLOT:
     next_action_wait.set(nextActionWaitTime.toFloatSeconds)
@@ -898,17 +893,16 @@ proc onSlotStart(
     delay = wallTime - expectedSlot.toBeaconTime()
 
   info "Slot start",
-    lastSlot = shortLog(lastSlot),
+    slot = shortLog(wallSlot),
-    wallSlot = shortLog(wallSlot),
+    epoch = shortLog(wallSlot.epoch),
-    delay = shortLog(delay),
-    peers = len(node.network.peerPool),
-    head = shortLog(node.dag.head),
-    headEpoch = shortLog(node.dag.head.slot.compute_epoch_at_slot()),
-    finalized = shortLog(node.dag.finalizedHead.blck),
-    finalizedEpoch = shortLog(finalizedEpoch),
     sync =
       if node.syncManager.inProgress: node.syncManager.syncStatus
-      else: "synced"
+      else: "synced",
+    peers = len(node.network.peerPool),
+    head = shortLog(node.dag.head),
+    finalized = shortLog(getStateField(
+      node.dag.headState.data, finalized_checkpoint)),
+    delay = shortLog(delay)
 
   # Check before any re-scheduling of onSlotStart()
   checkIfShouldStopAtEpoch(wallSlot, node.config.stopAtEpoch)
@@ -1166,6 +1160,10 @@ proc start(node: BeaconNode) {.raises: [Defect, CatchableError].} =
     timeSinceFinalization =
       node.beaconClock.now() - finalizedHead.slot.toBeaconTime(),
     head = shortLog(head),
+    justified = shortLog(getStateField(
+      node.dag.headState.data, current_justified_checkpoint)),
+    finalized = shortLog(getStateField(
+      node.dag.headState.data, finalized_checkpoint)),
     finalizedHead = shortLog(finalizedHead),
     SLOTS_PER_EPOCH,
     SECONDS_PER_SLOT,
@@ -1292,14 +1290,16 @@ proc initStatusBar(node: BeaconNode) {.raises: [Defect, ValueError].} =
     node.config.statusBarContents,
     dataResolver)
 
-  when compiles(defaultChroniclesStream.output.writer):
+  when compiles(defaultChroniclesStream.outputs[0].writer):
-    defaultChroniclesStream.output.writer =
+    let tmp = defaultChroniclesStream.outputs[0].writer
+
+    defaultChroniclesStream.outputs[0].writer =
       proc (logLevel: LogLevel, msg: LogOutputStr) {.raises: [Defect].} =
         try:
           # p.hidePrompt
           erase statusBar
           # p.writeLine msg
-          stdout.write msg
+          tmp(logLevel, msg)
           render statusBar
           # p.showPrompt
         except Exception as e: # render raises Exception
@@ -1827,14 +1827,12 @@ programMain:
   var
     config = makeBannerAndConfig(clientId, BeaconNodeConf)
 
-  setupStdoutLogging(config.logLevel)
-
   if not(checkAndCreateDataDir(string(config.dataDir))):
     # We are unable to access/create data folder or data folder's
     # permissions are insecure.
     quit QuitFailure
 
-  setupLogging(config.logLevel, config.logFile)
+  setupLogging(config.logLevel, config.logStdout, config.logFile)
 
   ## This Ctrl+C handler exits the program in non-graceful way.
   ## It's responsible for handling Ctrl+C in sub-commands such

beacon_chain/nimbus_beacon_node.nim.cfg

@@ -1,10 +1,10 @@
--d:"chronicles_sinks=textlines,json[file]"
+-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
 -d:"chronicles_runtime_filtering=on"
--d:"chronicles_default_output_device=dynamic"
+-d:"chronicles_disable_thread_id"
+
+@if release:
+  -d:"chronicles_line_numbers:0"
+@end
 
 # Use only `secp256k1` public key cryptography as an identity in LibP2P.
 -d:"libp2p_pki_schemes=secp256k1"
-
-@if testnet_servers_image:
-  -d:"chronicles_sinks=json"
-@end

beacon_chain/nimbus_binary_common.nim

@@ -11,7 +11,7 @@
 
 import
   # Standard library
-  std/[os, tables, strutils, typetraits],
+  std/[os, tables, strutils, terminal, typetraits],
 
   # Nimble packages
   chronos, confutils/defs,
@@ -21,7 +21,7 @@ import
   # Local modules
   ./spec/[helpers],
   ./spec/datatypes/base,
-  "."/[beacon_clock, filepath, beacon_node_status]
+  "."/[beacon_clock, conf, filepath, beacon_node_status]
 
 when defined(posix):
   import termios
@@ -33,15 +33,41 @@ type
                            lastSlot: Slot): Future[void] {.gcsafe,
   raises: [Defect].}
 
-proc setupStdoutLogging*(logLevel: string) =
+# silly chronicles, colors is a compile-time property
-  when compiles(defaultChroniclesStream.output.writer):
+proc stripAnsi(v: string): string =
-    defaultChroniclesStream.outputs[0].writer =
+  var
-      proc (logLevel: LogLevel, msg: LogOutputStr) {.gcsafe, raises: [Defect].} =
+    res = newStringOfCap(v.len)
-        try:
+    i: int
-          stdout.write(msg)
+
-          stdout.flushFile()
+  while i < v.len:
-        except IOError as err:
+    let c = v[i]
-          logLoggingFailure(cstring(msg), err)
+    if c == '\x1b':
+      var
+        x = i + 1
+        found = false
+
+      while x < v.len: # look for [..m
+        let c2 = v[x]
+        if x == i + 1:
+          if c2 != '[':
+            break
+        else:
+          if c2 in {'0'..'9'} + {';'}:
+            discard # keep looking
+          elif c2 == 'm':
+            i = x + 1
+            found = true
+            break
+          else:
+            break
+        inc x
+
+      if found: # skip adding c
+        continue
+    res.add c
+    inc i
+
+  res
 
 proc updateLogLevel*(logLevel: string) {.raises: [Defect, ValueError].} =
   # Updates log levels (without clearing old ones)
@@ -56,27 +82,90 @@ proc updateLogLevel*(logLevel: string) {.raises: [Defect, ValueError].} =
       if not setTopicState(topicName, settings.state, settings.logLevel):
         warn "Unrecognized logging topic", topic = topicName
 
-proc setupLogging*(logLevel: string, logFile: Option[OutFile]) =
+proc setupLogging*(
-  if logFile.isSome:
+    logLevel: string, stdoutKind: StdoutLogKind, logFile: Option[OutFile]) =
-    when defaultChroniclesStream.outputs.type.arity > 1:
+  # In the cfg file for nimbus, we create two formats: textlines and json.
-      block openLogFile:
+  # Here, we either write those logs to an output, or not, depending on the
+  # given configuration.
+  # Arguably, if we don't use a format, chronicles should not create it.
+
+  when defaultChroniclesStream.outputs.type.arity != 2:
+    warn "Logging configuration options not enabled in the current build"
+  else:
+    # Naive approach where chronicles will form a string and we will discard
+    # it, even if it could have skipped the formatting phase
+
+    proc noOutput(logLevel: LogLevel, msg: LogOutputStr) = discard
+    proc writeAndFlush(f: File, msg: LogOutputStr) =
+      try:
+        f.write(msg)
+        f.flushFile()
+      except IOError as err:
+        logLoggingFailure(cstring(msg), err)
+
+    proc stdoutFlush(logLevel: LogLevel, msg: LogOutputStr) =
+      writeAndFlush(stdout, msg)
+
+    proc noColorsFlush(logLevel: LogLevel, msg: LogOutputStr) =
+      writeAndFlush(stdout, stripAnsi(msg))
+
+    let fileWriter =
+      if logFile.isSome():
         let
           logFile = logFile.get.string
           logFileDir = splitFile(logFile).dir
-        let lres = secureCreatePath(logFileDir)
+          lres = secureCreatePath(logFileDir)
-        if lres.isErr():
+        if lres.isOk():
+          try:
+            let
+              f = open(logFile, fmAppend)
+              x = proc(logLevel: LogLevel, msg: LogOutputStr) =
+                writeAndFlush(f, msg) # will close when program terminates
+            x
+          except CatchableError as exc:
+            error "Failed to create log file", logFile, msg = exc.msg
+            noOutput
+        else:
           error "Failed to create directory for log file",
                 path = logFileDir, err = ioErrorMsg(lres.error)
-          break openLogFile
+          noOutput
-
-        try:
-          if not defaultChroniclesStream.outputs[1].open(logFile):
-            error "Failed to create log file", logFile
-        except CatchableError as exc:
-          # TODO why is there both exception and bool?
-          error "Failed to create log file", logFile, msg = exc.msg
     else:
-      warn "The --log-file option is not active in the current build"
+      noOutput
+
+    defaultChroniclesStream.outputs[1].writer = fileWriter
+
+    let tmp =
+      if stdoutKind == StdoutLogKind.Auto:
+        if logFile.isSome():
+          # No stdout logging by default, when file logging is enabled
+          StdoutLogKind.None
+        elif isatty(stdout):
+          # On a TTY, let's be fancy
+          StdoutLogKind.Colors
+        else:
+          # When there's no TTY, we output no colors because this matches what
+          # released binaries were doing before auto-detection was around and
+          # looks decent in systemd-captured journals.
+          StdoutLogKind.NoColors
+      else:
+        stdoutKind
+
+    case tmp
+    of StdoutLogKind.Auto: raiseAssert "checked above"
+    of StdoutLogKind.Colors:
+      defaultChroniclesStream.outputs[0].writer = stdoutFlush
+    of StdoutLogKind.NoColors:
+      defaultChroniclesStream.outputs[0].writer = noColorsFlush
+    of StdoutLogKind.Json:
+      defaultChroniclesStream.outputs[0].writer = noOutput
+
+      let prevWriter = defaultChroniclesStream.outputs[1].writer
+      defaultChroniclesStream.outputs[1].writer =
+        proc(logLevel: LogLevel, msg: LogOutputStr) =
+          stdoutFlush(logLevel, msg)
+          prevWriter(logLevel, msg)
+    of StdoutLogKind.None:
+     defaultChroniclesStream.outputs[0].writer = noOutput
 
   try:
     updateLogLevel(logLevel)

beacon_chain/nimbus_validator_client.nim

@@ -146,11 +146,10 @@ proc onSlotStart(vc: ValidatorClientRef, wallTime: BeaconTime,
   checkIfShouldStopAtEpoch(wallSlot.slot, vc.config.stopAtEpoch)
 
   info "Slot start",
-    lastSlot = shortLog(lastSlot),
+    slot = shortLog(wallSlot.slot),
-    wallSlot = shortLog(wallSlot.slot),
-    delay = shortLog(delay),
     attestationIn = vc.getDurationToNextAttestation(wallSlot.slot),
-    blockIn = vc.getDurationToNextBlock(wallSlot.slot)
+    blockIn = vc.getDurationToNextBlock(wallSlot.slot),
+    delay = shortLog(delay)
 
 proc asyncRun*(vc: ValidatorClientRef) {.async.} =
   vc.fallbackService.start()
@@ -164,8 +163,7 @@ programMain:
   let config = makeBannerAndConfig("Nimbus validator client " & fullVersionStr,
                                    ValidatorClientConf)
 
-  setupStdoutLogging(config.logLevel)
+  setupLogging(config.logLevel, config.logStdout, config.logFile)
-  setupLogging(config.logLevel, config.logFile)
 
   case config.cmd
     of VCNoCommand:

beacon_chain/nimbus_validator_client.nim.cfg

@@ -1,11 +1,10 @@
--d:"chronicles_sinks=textlines,json[file]"
+-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
 -d:"chronicles_runtime_filtering=on"
--d:"chronicles_default_output_device=dynamic"
+-d:"chronicles_disable_thread_id"
+
+@if release:
+  -d:"chronicles_line_numbers:0"
+@end
 
 # Use only `secp256k1` public key cryptography as an identity in LibP2P.
 -d:"libp2p_pki_schemes=secp256k1"
-
-@if testnet_servers_image:
-  -d:"chronicles_sinks=json"
-@end
-

beacon_chain/sync/sync_manager.nim

@@ -1151,7 +1151,6 @@ proc syncLoop[A, B](man: SyncManager[A, B]) {.async.} =
 
     # Update status string
     man.syncStatus = map & ":" & $pending & ":" &
-                     man.insSyncSpeed.formatBiggestFloat(ffDecimal, 4) & ":" &
                      man.avgSyncSpeed.formatBiggestFloat(ffDecimal, 4) & ":" &
                      man.timeLeft.toTimeLeftString() &
                      " (" & $man.queue.outSlot & ")"

config.nims

@@ -71,7 +71,7 @@ switch("passL", "-fno-omit-frame-pointer")
 --excessiveStackTrace:on
 # enable metric collection
 --define:metrics
---define:chronicles_line_numbers
+--define:chronicles_line_numbers # These are disabled for release binaries
 # for heap-usage-by-instance-type metrics and object base-type strings
 --define:nimTypeNames
 

docker/dist/entry_point.sh

@@ -20,9 +20,8 @@ PLATFORM="${1}"
 BINARIES="nimbus_beacon_node nimbus_signing_process"
 
 #- we need to build everything against libraries available inside this container, including the Nim compiler
-#- we disable the log file and log colours; the user only has to worry about logging stdout now
 make clean
-NIMFLAGS_COMMON="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none --gcc.options.debug:'-g1' --clang.options.debug:'-gline-tables-only'"
+NIMFLAGS_COMMON="-d:disableMarchNative --gcc.options.debug:'-g1' --clang.options.debug:'-gline-tables-only'"
 if [[ "${PLATFORM}" == "Windows_amd64" ]]; then
   # Cross-compilation using the MXE distribution of Mingw-w64
   export PATH="/usr/lib/mxe/usr/bin:${PATH}"

docs/the_nimbus_book/src/SUMMARY.md

@@ -33,7 +33,7 @@
 - [Set up log rotation](./log-rotate.md)
 - [Verify the integrity of Nimbus](./checksums.md)
 - [Back up your database](./database-backup.md)
-- [Change log level](./log-levels.md)
+- [Set up logging](./logging.md)
 - [Set up email notifications](./email-notifications.md)
 - [Graffiti the blockchain](./graffiti.md)
 - [Optimise for profitability](./profits.md)
@@ -64,3 +64,4 @@
 - [Prater: What you should know](./prater.md)
 - [Security Audit](./audit.md)
 - [FAQ](./faq.md)
+

docs/the_nimbus_book/src/binaries.md

@@ -13,9 +13,3 @@ Click on the `tar.gz` file that corresponds to your OS and architecture, unpack
 We've designed the build process to be reproducible. In practice, this means that anyone can verify that these exact binaries were produced from the corresponding source code commits. For more about the philosophy and importance of this feature see [reproducible-builds.org](https://reproducible-builds.org/).
 
 For instructions on how to reproduce those binaries, see "README.md" inside the archive.
-
-> **N.B.** These binaries are meant to run as a service. As such, they do not
-support the `log-file` [option](./options.md) - duplicating stdout to a
-log file would complicate things. We recommend capturing
-their output (stdout and stderr) with a [log rotation
-helper](./log-rotate.md) instead.

docs/the_nimbus_book/src/docker.md

@@ -8,9 +8,6 @@ These images are simply the contents of [release tarballs](./binaries.md) inside
 
 The unpacked archive is in `/home/user/nimbus-eth2` which is also the default *WORKDIR*. The default *ENTRYPOINT* is the binary itself: `/home/user/nimbus-eth2/build/nimbus\_beacon\_node`
 
-> **N.B.** Docker images do not support the `log-file` [option](./options.md) - The way we recommend setting up docker is to capture its console output with a [log rotation helper](./log-rotate.md).
-
-
 ## Usage
 
 You need to create an external data directory and mount it as a volume inside the container, with  mounting point: `/home/user/nimbus-eth2/build/data`

docs/the_nimbus_book/src/log-levels.md

@@ -1,26 +1 @@
-# Change log level
+This information has moved to [logging](./logging.md).
-
-
-You can customise Nimbus' verbosity with the `--log-level` option.
-
-For example:
-
-```
-./run-mainnet-beacon-node.sh --log-level=WARN
-```
-
-The default value is `INFO`.
-
-Possible values (in order of decreasing verbosity) are:
-
-```
-TRACE
-DEBUG
-INFO
-NOTICE
-WARN
-ERROR
-FATAL
-NONE
-```
-

docs/the_nimbus_book/src/log-rotate.md

@@ -1,10 +1,10 @@
 # Log rotation
 
-Nimbus logs are written to stdout, and optionally to a file. Writing to a file for a long-running process may lead to difficulties when the file grows large. This is typically solved with a *log rotator*. A log rotator is responsible for switching the written-to file, as well as compressing and removing old logs.
+Nimbus logs are written to `stdout`, and can be redirected to a file. Writing to a file for a long-running process may lead to difficulties when the file grows large. This is typically solved with a *log rotator*. A log rotator is responsible for switching the written-to file, as well as compressing and removing old logs.
 
 ## Using `logrotate`
 
-[logrotate](https://github.com/logrotate/logrotate) provides log rotation and compression. The corresponding package will install its Cron hooks (or Systemd timer) -- all you have to do is add a configuration file for Nimbus-eth2 in "/etc/logrotate.d/nimbus-eth2":
+[logrotate](https://github.com/logrotate/logrotate) provides log rotation and compression. The corresponding package will install its Cron hooks (or Systemd timer) -- all you have to do is add a configuration file for Nimbus in "/etc/logrotate.d/nimbus-eth2":
 
 ```text
 /var/log/nimbus-eth2/*.log {
@@ -14,7 +14,7 @@ Nimbus logs are written to stdout, and optionally to a file. Writing to a file f
 }
 ```
 
-The above assumes you've configured Nimbus-eth2 to write its logs to "/var/log/nimbus-eth2/" (usually by redirecting stout and stderr from your init script).
+The above assumes you've configured Nimbus to write its logs to "/var/log/nimbus-eth2/" (usually by redirecting `stdout` and `stderr` from your init script).
 
 "copytruncate" is required because, when it comes to moving the log file, `logrotate`'s default behaviour requires application support for re-opening that log file at runtime (something which is currently lacking). So, instead of a move, we tell `logrotate` to do a copy and a truncation of the existing file. A few log lines may be lost in the process.
 
@@ -29,13 +29,15 @@ rotate 7
 
 ## Using `rotatelogs`
 
-[rotatelogs](https://httpd.apache.org/docs/2.4/programs/rotatelogs.html) is available on most servers and can be used with `Docker`, `Systemd` and manual setups to write rotated logs files.
+[rotatelogs](https://httpd.apache.org/docs/2.4/programs/rotatelogs.html) captures `stdout` logging and redirects it to a file, rotating and compressing on the fly.
 
-In particular, when `Systemd` and its accompanying `Journald` log daemon are used, this setup avoids clogging the system log by keeping the Nimbus logs in a separate location.
+It is available on most servers and can be used with `Docker`, `Systemd` and manual setups to write rotated logs files.
+
+In particular, when `systemd` and its accompanying `journald` log daemon are used, this setup avoids clogging the system log by keeping the Nimbus logs in a separate location.
 
 ### Compression
 
-`rotatelogs` works by reading stdin and redirecting it to a file based on a name pattern. Whenever the log is about to be rotated, the application invokes a shell script with the old and new log files. Our aim is to compress the log file to save space. The [Nimbus-eth2 repo](https://github.com/status-im/nimbus-eth2/tree/unstable/scripts/rotatelogs-compress.sh) provides a helper script that does this:
+`rotatelogs` works by reading `stdin` and redirecting it to a file based on a name pattern. Whenever the log is about to be rotated, the application invokes a shell script with the old and new log files. Our aim is to compress the log file to save space. The [Nimbus-eth2 repo](https://github.com/status-im/nimbus-eth2/tree/unstable/scripts/rotatelogs-compress.sh) provides a helper script that does this:
 
 ```bash
 # Create a rotation script for rotatelogs
@@ -53,19 +55,6 @@ EOF
 chmod +x rotatelogs-compress.sh
 ```
 
-### Build
-
-Logs in files generally don't benefit from colors. To avoid colors being written to the file, additional flags can be added to the Nimbus [build process](./build.md) -- these flags are best saved in a build script to which one can add more options. Future versions of Nimbus will support disabling colors at runtime.
-
-```bash
-
-# Build nimbus with colors disabled
-cat << EOF > build-nbc-text.sh
-#!/bin/bash
-make NIMFLAGS="-d:chronicles_colors=off -d:chronicles_sinks=textlines" nimbus_beacon_node
-EOF
-```
-
 ### Run
 
 The final step is to redirect logs to `rotatelogs` using a pipe when starting Nimbus:

docs/the_nimbus_book/src/logging.md

@@ -0,0 +1,58 @@
+# Logging
+
+Nimbus offers several options for logging - by default, logs are written to stdout using the [chronicles](https://github.com/status-im/nim-chronicles#introduction) `textlines` format which is convenient to read and can be used with tooling for [heroku/logfmt](https://brandur.org/logfmt).
+
+## Change log level
+
+You can customise Nimbus' verbosity with the `--log-level` option.
+
+For example:
+
+```
+./run-mainnet-beacon-node.sh --log-level=WARN
+```
+
+The default value is `INFO`.
+
+Possible values (in order of decreasing verbosity) are:
+
+```
+TRACE
+DEBUG
+INFO
+NOTICE
+WARN
+ERROR
+FATAL
+NONE
+```
+
+## Change logging style
+
+Nimbus supports 3 styles of logging: `colors`, `nocolors` and `json`. In `auto` mode, logs will be printed using either `colors` or `nocolors`.
+
+You can choose a log style with the `--log-stdout` option, which also understands `auto` and `none`:
+
+```
+./run-mainnet-beacon-node.sh --log-stdout=none # disable logging to std out
+./run-mainnet-beacon-node.sh --log-stdout=json # print json logs, one line per item
+```
+
+## Logging to a file
+
+To sending logs to a file, it is recommended to use a redirect of the stdout logs:
+
+```
+# log json to filename.jsonl
+./run-mainnet-beacon-node.sh --log-stdout=json > filename.jsonl
+```
+
+ Keep an eye on the growth of this file with a [log rotator](./log-rotate.md). Logs will be written in the "JSON Lines" format - one `json` entry per line.
+
+Nimbus also supports writing logs to a log file using the `--log-file` option - this is provided for historical reasons and may be removed in future releases.
+
+```
+./run-mainnet-beacon-node.sh --log-file=filename.jsonl # write json logs to the given filename
+```
+
+When the `--log-file` option is enabled, stdout logs will by default be disabled - enable them with the `--log-stdout` option choosing a format other than `auto`.

docs/the_nimbus_book/src/options.md

@@ -23,21 +23,28 @@ The following options are available:
 
      --log-level               Sets the log level for process and topics (e.g. "DEBUG;
                                TRACE:discv5,libp2p; REQUIRED:none; DISABLED:none") [=INFO].
+     --log-stdout              Specifies what kind of logs should be written to stdout
+                               (auto, colors, nocolors, json) [=auto].
      --log-file                Specifies a path for the written Json log file.
      --network                 The Eth2 network to join [=mainnet].
  -d, --data-dir                The directory where nimbus will store all blockchain data.
      --validators-dir          A directory containing validator keystores.
      --secrets-dir             A directory containing validator keystore passwords.
      --wallets-dir             A directory containing wallet files.
-     --web3-url                URL of the Web3 server to observe Eth1.
+     --web3-url                One or more Web3 provider URLs used for obtaining deposit
+                               contract data.
      --non-interactive         Do not display interative prompts. Quit on missing
                                configuration.
      --netkey-file             Source of network (secp256k1) private key file
                                (random|<path>) [=random].
      --insecure-netkey-password  Use pre-generated INSECURE password for network private key
                                file [=false].
-     --agent-string            Node agent string which is used as identifier in network.
+     --agent-string            Node agent string which is used as identifier in network
-     --subscribe-all-subnets   Subscribe to all attestation subnet topics when gossiping.
+                               [=nimbus].
+     --subscribe-all-subnets   Subscribe to all attestation subnet topics when gossiping
+                               [=false].
+     --num-threads             Number of worker threads (set this to 0 to use as many
+                               threads as there are CPU cores available) [=1].
  -b, --bootstrap-node          Specifies one or more bootstrap nodes to use when connecting
                                to the network.
      --bootstrap-file          Specifies a line-delimited file of bootstrap Ethereum network
@@ -48,11 +55,11 @@ The following options are available:
      --udp-port                Listening UDP port for node discovery [=9000].
      --max-peers               The maximum number of peers to connect to [=160].
      --nat                     Specify method to use for determining public address. Must be
-                               one of: any, none, upnp, pmp, extip:<IP>.
+                               one of: any, none, upnp, pmp, extip:<IP> [=any].
      --enr-auto-update         Discovery can automatically update its ENR with the IP
                                address and UDP port as seen by other nodes it communicates
-                               with. This option allows to enable/disable this
+                               with. This option allows to enable/disable this functionality
-                               functionality.
+                               [=false].
      --weak-subjectivity-checkpoint  Weak subjectivity checkpoint in the format
                                block_root:epoch_number.
      --finalized-checkpoint-state  SSZ file specifying a recent finalized state.
@@ -64,19 +71,26 @@ The following options are available:
                                can use a 0x-prefixed hex encoded string to specify raw
                                bytes.
      --verify-finalization     Specify whether to verify finalization occurs on schedule,
-                               for testing.
+                               for testing [=false].
-     --stop-at-epoch           A positive epoch selects the epoch at which to stop.
+     --stop-at-epoch           A positive epoch selects the epoch at which to stop [=0].
      --metrics                 Enable the metrics server [=false].
      --metrics-address         Listening address of the metrics server [=127.0.0.1].
      --metrics-port            Listening HTTP port of the metrics server [=8008].
-     --status-bar              Display a status bar at the bottom of the terminal screen.
+     --status-bar              Display a status bar at the bottom of the terminal screen
+                               [=true].
      --status-bar-contents     Textual template for the contents of the status bar.
      --rpc                     Enable the JSON-RPC server [=false].
      --rpc-port                HTTP port for the JSON-RPC service [=9190].
      --rpc-address             Listening address of the RPC server [=127.0.0.1].
+     --rest                    Enable the REST server [=false].
+     --rest-port               Port for the REST server [=5052].
+     --rest-address            Listening address of the REST server [=127.0.0.1].
+     --validator-api           Enable the REST (BETA version) validator keystore management
+                               API [=false].
      --in-process-validators   Disable the push model (the beacon node tells a signing
                                process with the private keys of the validators what to sign
-                               and when) and load the validators in the beacon node itself.
+                               and when) and load the validators in the beacon node itself
+                               [=true].
      --discv5                  Enable Discovery v5 [=true].
      --dump                    Write SSZ dumps of blocks, attestations and states to data
                                dir [=false].
@@ -86,8 +100,12 @@ The following options are available:
                                /ip4/<address>/tcp/<port>/p2p/<peerId-public-key>. Peering
                                agreements are established out of band and must be
                                reciprocal..
-     --doppelganger-detection  Whether to detect whether another validator is be running the
+     --doppelganger-detection  If enabled, the beacon node prudently listens for 2 epochs
-                               same validator keys [=true].
+                               for attestations from a validator with the same index (a
+                               doppelganger), before sending an attestation itself. This
+                               protects against slashing (due to double-voting) but means
+                               you will miss two attestations when restarting. [=true].
+
 
 ...
 ```

scripts/launch_local_testnet.sh

@@ -228,7 +228,7 @@ if [[ "$ENABLE_LOGTRACE" == "1" ]]; then
   BINARIES="${BINARIES} logtrace"
 fi
 
-$MAKE -j ${NPROC} LOG_LEVEL=TRACE NIMFLAGS="${NIMFLAGS} -d:testnet_servers_image -d:local_testnet -d:const_preset=${CONST_PRESET}" ${BINARIES}
+$MAKE -j ${NPROC} LOG_LEVEL=TRACE NIMFLAGS="${NIMFLAGS} -d:local_testnet -d:const_preset=${CONST_PRESET}" ${BINARIES}
 
 # Kill child processes on Ctrl-C/SIGTERM/exit, passing the PID of this shell
 # instance as the parent and the target process name as a pattern to the

scripts/run-beacon-node.sh

@@ -93,7 +93,6 @@ fi
 exec ${WINPTY} build/${NBC_BINARY} \
   --network=${NETWORK} \
   --data-dir="${DATA_DIR}" \
-  --log-file="${DATA_DIR}/nbc_bn_$(date +"%Y%m%d%H%M%S").log" \
   --tcp-port=$(( ${BASE_P2P_PORT} + ${NODE_ID} )) \
   --udp-port=$(( ${BASE_P2P_PORT} + ${NODE_ID} )) \
   --rest \

tests/simulation/start.sh

@@ -49,7 +49,7 @@ mkdir -p "$SECRETS_DIR"
 
 cd "$GIT_ROOT"
 
-CUSTOM_NIMFLAGS="${NIMFLAGS} -d:useSysAsserts -d:chronicles_sinks:textlines,json[file] -d:const_preset=mainnet -d:local_testnet"
+CUSTOM_NIMFLAGS="${NIMFLAGS} -d:useSysAsserts -d:const_preset=mainnet -d:local_testnet"
 GANACHE_BLOCK_TIME=5
 
 # Run with "SLOTS_PER_EPOCH=8 ./start.sh" to change these

vendor/nim-chronicles

@@ -1 +1 @@
-Subproject commit ba2817f1a9f5ddb59274295f3fe9a54fbf1a5a96
+Subproject commit c277f795ed998bc67312991e944254abed99eafc