Introduce a RuntimePreset object for the overridable testnet preset values

beacon_chain/beacon_node.nim

@@ -1128,6 +1128,8 @@ programMain:
     let
       networkName = config.eth2Network.get
       metadata = case toLowerAscii(networkName)
+        of "mainnet":
+          mainnetMetadata
         of "altona":
           altonaMetadata
         else:

beacon_chain/network_metadata.nim

@@ -1,7 +1,8 @@
 import
   os, strutils,
-  stew/byteutils, nimcrypto/hash,
+  stew/byteutils, stew/shims/macros, nimcrypto/hash,
-  eth/common/[eth_types, eth_types_json_serialization]
+  eth/common/[eth_types, eth_types_json_serialization],
+  spec/presets/custom
 
 # ATTENTION! This file will produce a large C file, because we are inlining
 # genesis states as C literals in the generated code (and blobs in the final
@@ -29,8 +30,11 @@ type
     customEth2Network
     altona
 
+  PresetIncompatible* = object of CatchableError
+
   Eth2NetworkMetadata* = object
     eth1Network*: Eth1Network
+    runtimePreset*: RuntimePreset
 
     # Parsing `enr.Records` is still not possible at compile-time
     bootstrapNodes*: seq[string]
@@ -52,19 +56,60 @@ type
     # unknown genesis state.
     genesisData*: string
 
+const presetValueLoaders = genCode(nnkBracket):
+  for constName in PresetValue:
+    let
+      constNameIdent = ident $constName
+      constType = ident getType(constName)
+
+    yield quote do:
+      proc (preset: RuntimePreset, presetValue: string): bool
+           {.gcsafe, noSideEffect, raises: [Defect].} =
+        try:
+          when `constNameIdent` in runtimeValues:
+            preset.`constNameIdent` = parse(`constType`, presetValue)
+            true
+          else:
+            `constNameIdent` == parse(`constType`, presetValue)
+        except CatchableError:
+          false
+
 proc loadEth2NetworkMetadata*(path: string): Eth2NetworkMetadata
                              {.raises: [CatchableError, Defect].} =
   let
     genesisPath = path / "genesis.ssz"
+    config = readPresetFile(path / "config.yaml")
+
+  var runtimePreset = RuntimePreset()
+
+  for name, value in config.values:
+    if name notin runtimeValues:
+      if not presetValueLoaders[name](runtimePreset, value):
+        raise newException(PresetIncompatible,
+          "The preset '{path}' is not compatible with the current build due to an incompatible value {name} = {value}")
 
   Eth2NetworkMetadata(
     eth1Network: goerli,
+    runtimePreset: runtimePreset,
     bootstrapNodes: readFile(path / "bootstrap_nodes.txt").split("\n"),
     depositContractAddress: Eth1Address.fromHex readFile(path / "deposit_contract.txt").strip,
     depositContractDeployedAt: Eth1BlockHash.fromHex readFile(path / "deposit_contract_block.txt").strip,
     genesisData: if fileExists(genesisPath): readFile(genesisPath) else: "")
 
 const
+  mainnetMetadata* = Eth2NetworkMetadata(
+    eth1Network: mainnet,
+    runtimePreset: RuntimePreset(
+      MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384,
+      MIN_GENESIS_TIME: 1578009600,
+      GENESIS_FORK_VERSION: [byte 0, 0, 0, 0],
+      GENESIS_DELAY: 172800),
+    # TODO The values below are just placeholders for now
+    bootstrapNodes: @[],
+    depositContractAddress: "0x1234567890123456789012345678901234567890",
+    depositContractDeployedAt: "",
+    genesisData: "")
+
   altonaMetadata* = loadEth2NetworkMetadata(
     currentSourcePath.parentDir / ".." / "vendor" / "eth2-testnets" / "shared" / "altona")
 

beacon_chain/spec/beaconstate.nim

@@ -226,8 +226,8 @@ proc initialize_beacon_state_from_eth1*(
 
   var state = BeaconStateRef(
     fork: Fork(
-      previous_version: Version(GENESIS_FORK_VERSION),
+      previous_version: GENESIS_FORK_VERSION,
-      current_version: Version(GENESIS_FORK_VERSION),
+      current_version: GENESIS_FORK_VERSION,
       epoch: GENESIS_EPOCH),
     genesis_time: genesis_time_from_eth1_timestamp(eth1_timestamp),
     eth1_data:

beacon_chain/spec/datatypes.nim

@@ -61,7 +61,7 @@ else:
     Epoch* = distinct uint64
 
   import ./presets/custom
-  loadCustomPreset const_preset
+  createConstantsFromPreset const_preset
 
 const
   SPEC_VERSION* = "0.12.1" ## \
@@ -163,7 +163,6 @@ type
     data*: AttestationData
     signature*: TrustedSig
 
-  Version* = distinct array[4, byte]
   ForkDigest* = distinct array[4, byte]
 
   # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#forkdata

beacon_chain/spec/helpers.nim

@@ -158,7 +158,7 @@ func compute_fork_digest*(current_version: Version,
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#compute_domain
 func compute_domain*(
     domain_type: DomainType,
-    fork_version: Version = Version(GENESIS_FORK_VERSION),
+    fork_version = GENESIS_FORK_VERSION,
     genesis_validators_root: Eth2Digest = ZERO_HASH): Domain =
   # Return the domain for the ``domain_type`` and ``fork_version``.
   let fork_data_root =

beacon_chain/spec/presets/custom.nim

@@ -1,112 +1,87 @@
 import
-  macros, strutils, tables,
+  macros, strutils, parseutils, tables,
   stew/endians2
 
 export
   toBytesBE
 
 type
-  BeaconChainConstants* {.pure.} = enum
+  PresetValue* {.pure.} = enum
-    BASE_REWARDS_PER_EPOCH
     BASE_REWARD_FACTOR
     BLS_WITHDRAWAL_PREFIX
     CHURN_LIMIT_QUOTIENT
-    CUSTODY_PERIOD_TO_RANDAO_PADDING
     DEPOSIT_CONTRACT_ADDRESS
-    DEPOSIT_CONTRACT_TREE_DEPTH
     DOMAIN_AGGREGATE_AND_PROOF
     DOMAIN_BEACON_ATTESTER
     DOMAIN_BEACON_PROPOSER
-    DOMAIN_CUSTODY_BIT_SLASHING
     DOMAIN_DEPOSIT
-    DOMAIN_LIGHT_CLIENT
     DOMAIN_RANDAO
     DOMAIN_SELECTION_PROOF
-    DOMAIN_SHARD_COMMITTEE
-    DOMAIN_SHARD_PROPOSAL
     DOMAIN_VOLUNTARY_EXIT
-    EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS
     EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE
     EFFECTIVE_BALANCE_INCREMENT
     EJECTION_BALANCE
-    EPOCHS_PER_CUSTODY_PERIOD
     EPOCHS_PER_ETH1_VOTING_PERIOD
     EPOCHS_PER_HISTORICAL_VECTOR
     EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION
     EPOCHS_PER_SLASHINGS_VECTOR
     ETH1_FOLLOW_DISTANCE
-    GASPRICE_ADJUSTMENT_COEFFICIENT
-    GENESIS_EPOCH
     GENESIS_FORK_VERSION
     GENESIS_DELAY
-    GENESIS_SLOT
     HISTORICAL_ROOTS_LIMIT
     HYSTERESIS_DOWNWARD_MULTIPLIER
     HYSTERESIS_QUOTIENT
     HYSTERESIS_UPWARD_MULTIPLIER
     INACTIVITY_PENALTY_QUOTIENT
-    INITIAL_ACTIVE_SHARDS
-    JUSTIFICATION_BITS_LENGTH
-    LIGHT_CLIENT_COMMITTEE_PERIOD
-    LIGHT_CLIENT_COMMITTEE_SIZE
     MAX_ATTESTATIONS
     MAX_ATTESTER_SLASHINGS
     MAX_COMMITTEES_PER_SLOT
-    MAX_CUSTODY_KEY_REVEALS
-    MAX_CUSTODY_SLASHINGS
     MAX_DEPOSITS
-    MAX_EARLY_DERIVED_SECRET_REVEALS
     MAX_EFFECTIVE_BALANCE
     MAX_EPOCHS_PER_CROSSLINK
-    MAX_GASPRICE
     MAX_PROPOSER_SLASHINGS
-    MAX_REVEAL_LATENESS_DECREMENT
     MAX_SEED_LOOKAHEAD
-    MAX_SHARDS
-    MAX_SHARD_BLOCKS_PER_ATTESTATION
-    MAX_SHARD_BLOCK_CHUNKS
     MAX_VALIDATORS_PER_COMMITTEE
     MAX_VOLUNTARY_EXITS
-    MINOR_REWARD_QUOTIENT
     MIN_ATTESTATION_INCLUSION_DELAY
     MIN_DEPOSIT_AMOUNT
     MIN_EPOCHS_TO_INACTIVITY_PENALTY
-    MIN_GASPRICE
     MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
     MIN_GENESIS_TIME
     MIN_PER_EPOCH_CHURN_LIMIT
     MIN_SEED_LOOKAHEAD
     MIN_SLASHING_PENALTY_QUOTIENT
     MIN_VALIDATOR_WITHDRAWABILITY_DELAY
-    ONLINE_PERIOD
-    PHASE_1_FORK_VERSION
-    PHASE_1_GENESIS_SLOT
     PROPOSER_REWARD_QUOTIENT
-    RANDAO_PENALTY_EPOCHS
     RANDOM_SUBNETS_PER_VALIDATOR
     SAFE_SLOTS_TO_UPDATE_JUSTIFIED
-    SECONDS_PER_DAY
     SECONDS_PER_ETH1_BLOCK
     SECONDS_PER_SLOT
-    SHARD_BLOCK_CHUNK_SIZE
-    SHARD_BLOCK_OFFSETS
     SHARD_COMMITTEE_PERIOD
     SHUFFLE_ROUND_COUNT
     SLOTS_PER_EPOCH
     SLOTS_PER_HISTORICAL_ROOT
     TARGET_AGGREGATORS_PER_COMMITTEE
     TARGET_COMMITTEE_SIZE
-    TARGET_SHARD_BLOCK_SIZE
     VALIDATOR_REGISTRY_LIMIT
     WHISTLEBLOWER_REWARD_QUOTIENT
 
 const
+  runtimeValues* = {
+    MIN_GENESIS_ACTIVE_VALIDATOR_COUNT,
+    MIN_GENESIS_TIME,
+    DEPOSIT_CONTRACT_ADDRESS,
+    GENESIS_FORK_VERSION,
+    GENESIS_DELAY,
+  }
+
   # These constants cannot really be overriden in a preset.
   # If we encounter them, we'll just ignore the preset value.
-  dubiousConstants = {
+  ignoredValues = {
-    # They are derived from other constants:
+    # The deposit contract address is loaded through a dedicated
-    GENESIS_EPOCH,
+    # metadata file. It would break the property we are exploiting
-    SECONDS_PER_DAY,
+    # right now that all preset values can be parsed as uint64
+    DEPOSIT_CONTRACT_ADDRESS
 
     # These are defined as an enum in datatypes.nim:
     DOMAIN_BEACON_PROPOSER,
@@ -116,74 +91,116 @@ const
     DOMAIN_VOLUNTARY_EXIT,
     DOMAIN_SELECTION_PROOF,
     DOMAIN_AGGREGATE_AND_PROOF,
-    DOMAIN_SHARD_PROPOSAL,
-    DOMAIN_SHARD_COMMITTEE,
-    DOMAIN_LIGHT_CLIENT,
     DOMAIN_CUSTODY_BIT_SLASHING,
   }
 
-const
+  presetValueTypes* = {
-  forkVersionConversionFn = "'u32.toBytesBE"
+    BASE_REWARD_FACTOR: "uint64",
-    # The fork version appears as a number in the preset,
+    BLS_WITHDRAWAL_PREFIX: "byte",
-    # but our codebase works with a byte array.
+    EFFECTIVE_BALANCE_INCREMENT: "uint64",
-
+    EJECTION_BALANCE: "uint64",
-  customTypes = {
+    EPOCHS_PER_SLASHINGS_VECTOR: "uint64",
-    BASE_REWARD_FACTOR: "'u64",
+    GENESIS_FORK_VERSION: "Version",
-    BLS_WITHDRAWAL_PREFIX: ".byte",
+    GENESIS_SLOT: "Slot",
-    EFFECTIVE_BALANCE_INCREMENT: "'u64",
+    INACTIVITY_PENALTY_QUOTIENT: "uint64",
-    EJECTION_BALANCE: "'u64",
+    MAX_EFFECTIVE_BALANCE: "uint64",
-    EPOCHS_PER_SLASHINGS_VECTOR: "'u64",
+    MIN_DEPOSIT_AMOUNT: "uint64",
-    GENESIS_FORK_VERSION: forkVersionConversionFn,
+    MIN_EPOCHS_TO_INACTIVITY_PENALTY: "uint64",
-    GENESIS_SLOT: ".Slot",
+    MIN_VALIDATOR_WITHDRAWABILITY_DELAY: "uint64",
-    INACTIVITY_PENALTY_QUOTIENT: "'u64",
+    PROPOSER_REWARD_QUOTIENT: "uint64",
-    MAX_EFFECTIVE_BALANCE: "'u64",
+    SECONDS_PER_SLOT: "uint64",
-    MIN_DEPOSIT_AMOUNT: "'u64",
+    WHISTLEBLOWER_REWARD_QUOTIENT: "uint64",
-    MIN_EPOCHS_TO_INACTIVITY_PENALTY: "'u64",
-    MIN_VALIDATOR_WITHDRAWABILITY_DELAY: "'u64",
-    PHASE_1_FORK_VERSION: forkVersionConversionFn,
-    PROPOSER_REWARD_QUOTIENT: "'u64",
-    SECONDS_PER_SLOT: "'u64",
-    WHISTLEBLOWER_REWARD_QUOTIENT: "'u64",
   }.toTable
 
+func parse*(T: type uint64, input: string): T
+           {.raises: [ValueError, Defect].} =
+  if input.len > 2 and input[0] == '0' and input[1] == 'x':
+    parseHex(input, result)
+  else:
+    parseInt(input, result)
+
+template parse*(T: type byte, input: string): T =
+  byte parse(uint64, input)
+
+proc parse*(T: type Version, input: string): T =
+  toBytesBE(uint32 parse(uint64, input))
+
+template parse*(T: type Slot, input: string): T =
+  Slot parse(uint64, input)
+
+template getType*(presetValue: PresetValue): string =
+  presetValueTypes.getOrDefault(presetValue, "int")
+
 template entireSet(T: type enum): untyped =
   {low(T) .. high(T)}
 
-macro loadCustomPreset*(path: static string): untyped =
+macro genRuntimePresetType: untyped =
-  result = newStmtList()
+  var fields = newTree(nnkRecList)
 
+  for field in runtimeValues:
+    fields.add newTree(nnkIdentDefs,
+                       ident $field,
+                       ident getType(field),
+                       newEmptyNode()) # default value
+
+  result = newTree(nnkObjectTy,
+                   newEmptyNode(), # pragmas
+                   newEmptyNode(), # base type
+                   fields)
+
+type
+  RuntimePresetObj* = genRuntimePresetType()
+  RuntimePreset* = ref RuntimePresetObj
+
+  PresetFile = object
+    values*: Table[PresetValue, TaintedString]
+    missingValues*: set[PresetValue]
+
+  PresetFileError = object of CatchableError
+
+proc readPresetFile(path: string): PresetFile
+                   {.raises: [IOError, PresetFileError, Defect].} =
   var
-    presetContents = staticRead(path)
-    presetConstants = dubiousConstants
     lineNum = 0
+    presetValues = ignoredValues
 
-  for line in splitLines(presetContents):
+  template lineinfo: string =
+    "$1($2) " % [path, $lineNum]
+
+  template fail(msg) =
+    raise newException(PresetFileError, lineinfo & msg)
+
+  for line in splitLines(readFile(path)):
     inc lineNum
     if line.len == 0 or line[0] == '#': continue
 
-    template lineinfo: string =
+    var lineParts = line.split(":")
-      "$1($2) " % [path, $lineNum]
+    if lineParts.len != 2:
+      fail "Invalid syntax: A preset file should include only assignments in the form 'ConstName: Value'"
 
-    var constParts = line.split(":")
+    let value = try: parseEnum[PresetValue](lineParts[0])
-    if constParts.len != 2:
+                except ValueError: fail "Unrecognized constant in a preset: " & lineParts[0]
-      error lineinfo & "Invalid syntax: A preset file should include only assignments in the form 'ConstName: Value'"
 
-    try:
+    if value in ignoredValues: continue
-      let constant = parseEnum[BeaconChainConstants](constParts[0])
+    presetValues.incl value
-      if constant in dubiousConstants: continue
+    result.add value, lineParts[1]
-      if customTypes.hasKey(constant):
-        constParts.add customTypes[constant]
-      presetConstants.incl constant
-    except ValueError:
-      warning lineinfo & "Unrecognized constant in a preset: " & constParts[0]
-      continue
 
-    if constParts.len == 3:
+  result.missingValues = PresetValue.entireSet - presetValues
-      result.add parseStmt("const $1* {.intdefine.} = $2$3" % constParts)
-    else:
-      result.add parseStmt("const $1* {.intdefine.} = $2" % constParts)
 
-  let missingConstants = BeaconChainConstants.entireSet - presetConstants
+macro createConstantsFromPreset*(path: static string): untyped =
-  if missingConstants.card > 0:
+  result = newStmtList()
-    warning "Missing constants in preset: " & $missingConstants
+
+  let preset = try: loadPreset(path)
+               except PresetError as err: error err.msg
+
+  for name, value in preset:
+    var value = value
+    if presetValueTypes.hasKey(name):
+      let typ = presetValueTypes[name]
+      value = typ & "(" & value & ")"
+
+    result.add parseStmt("const $1* {.intdefine.} = $2" % [name, value])
+
+  if preset.missingValues.card > 0:
+    warning "Missing constants in preset: " & $preset.missingValues
 

beacon_chain/spec/presets/v0_12_1/mainnet.nim

@@ -14,6 +14,7 @@ import
 type
   Slot* = distinct uint64
   Epoch* = distinct uint64
+  Version* = distinct array[4, byte]
 
 {.experimental: "codeReordering".} # SLOTS_PER_EPOCH is use before being defined in spec
 
@@ -41,8 +42,6 @@ const
   MIN_PER_EPOCH_CHURN_LIMIT* = 4
   CHURN_LIMIT_QUOTIENT* = 2^16
   SHUFFLE_ROUND_COUNT* = 90
-  MIN_GENESIS_ACTIVE_VALIDATOR_COUNT* {.intdefine.} = 16384
-  MIN_GENESIS_TIME* {.intdefine.} = 1578009600
 
   HYSTERESIS_QUOTIENT* = 4
   HYSTERESIS_DOWNWARD_MULTIPLIER* = 1
@@ -68,13 +67,11 @@ const
   # Initial values
   # ---------------------------------------------------------------
   # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L70
-  GENESIS_FORK_VERSION* = [0'u8, 0'u8, 0'u8, 0'u8]
   BLS_WITHDRAWAL_PREFIX* = 0'u8
 
   # Time parameters
   # ---------------------------------------------------------------
   # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L77
-  GENESIS_DELAY* {.intdefine.} = 172800 # 172800 seconds (2 days)
 
   SECONDS_PER_SLOT* {.intdefine.} = 12'u64 # Compile with -d:SECONDS_PER_SLOT=1 for 12x faster slots
   ## TODO consistent time unit across projects, similar to C++ chrono?
@@ -193,9 +190,7 @@ const
   # Time parameters
   # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L199
   RANDAO_PENALTY_EPOCHS* = 2 # epochs (12.8 minutes)
-  EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS* = 16384 # epochs (~73 days)
   EPOCHS_PER_CUSTODY_PERIOD* = 2048 # epochs (~9 days)
-  CUSTODY_PERIOD_TO_RANDAO_PADDING* = 2048 # epochs (~9 days)
   MAX_REVEAL_LATENESS_DECREMENT* = 128 # epochs (~14 hours)
 
   # Max operations

beacon_chain/spec/presets/v0_12_1/minimal.nim

@@ -14,6 +14,7 @@ import
 type
   Slot* = distinct uint64
   Epoch* = distinct uint64
+  Version* = distinct array[4, byte]
 
 {.experimental: "codeReordering".} # SLOTS_PER_EPOCH is use before being defined in spec
 
@@ -33,8 +34,6 @@ const
 
   # Changed
   SHUFFLE_ROUND_COUNT* = 10
-  MIN_GENESIS_ACTIVE_VALIDATOR_COUNT* {.intdefine.} = 64
-  MIN_GENESIS_TIME* {.intdefine.} = 1578009600 # 3 Jan, 2020
 
   # Unchanged
   HYSTERESIS_QUOTIENT* = 4
@@ -55,7 +54,6 @@ const
   # ---------------------------------------------------------------
   # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/minimal.yaml#L70
 
-  GENESIS_FORK_VERSION* = [0'u8, 0'u8, 0'u8, 1'u8]
   BLS_WITHDRAWAL_PREFIX* = 0'u8
 
   # Time parameters
@@ -63,7 +61,6 @@ const
   # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/minimal.yaml#L77
   # Changed: Faster to spin up testnets, but does not give validator
   # reasonable warning time for genesis
-  GENESIS_DELAY* {.intdefine.} = 300
 
   # Unchanged
   SECONDS_PER_SLOT*{.intdefine.} = 6'u64
@@ -172,9 +169,7 @@ const
   # Time parameters
   # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/minimal.yaml#L202
   RANDAO_PENALTY_EPOCHS* = 2
-  EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS* = 4096 # epochs
   EPOCHS_PER_CUSTODY_PERIOD* = 2048
-  CUSTODY_PERIOD_TO_RANDAO_PADDING* = 2048
   MAX_REVEAL_LATENESS_DECREMENT* = 128
 
   # Max operations

beacon_chain/validator_api.nim

@@ -45,7 +45,7 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
     return (genesis_time: node.blockPool.headState.data.data.genesis_time,
              genesis_validators_root:
               node.blockPool.headState.data.data.genesis_validators_root,
-             genesis_fork_version: Version(GENESIS_FORK_VERSION))
+             genesis_fork_version: GENESIS_FORK_VERSION)
 
   rpcServer.rpc("get_v1_beacon_states_root") do (stateId: string) -> Eth2Digest:
     debug "get_v1_beacon_states_root", stateId = stateId
@@ -75,8 +75,8 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
       of "head":
         node.blockPool.headState.data.data.fork
       of "genesis":
-        Fork(previous_version: Version(GENESIS_FORK_VERSION),
+        Fork(previous_version: GENESIS_FORK_VERSION,
-             current_version: Version(GENESIS_FORK_VERSION),
+             current_version: GENESIS_FORK_VERSION,
              epoch: GENESIS_EPOCH)
       of "finalized":
         node.blockPool.withState(node.blockPool.tmpState, node.blockPool.finalizedHead):