nimbus-eth2/ncli/ncli_testnet.nim

# beacon_chain
# Copyright (c) 2018-2023 Status Research & Development GmbH
# Licensed and distributed under either of
#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.

{.push raises: [].}

import
  std/[os, sequtils, strutils, options, json, terminal, times],
  chronos, bearssl/rand, chronicles, confutils, stint, json_serialization,
  web3, web3/confutils_defs, eth/keys, eth/p2p/discoveryv5/random2,
  stew/[io2, byteutils], json_rpc/jsonmarshal,
  ../beacon_chain/[conf, filepath],
  ../beacon_chain/el/el_manager,
  ../beacon_chain/networking/eth2_network,
  ../beacon_chain/spec/[beaconstate, eth2_merkleization],
  ../beacon_chain/spec/datatypes/base,
  ../beacon_chain/spec/eth2_apis/eth2_rest_serialization,
  ../beacon_chain/validators/keystore_management,
  ../tests/mocking/mock_genesis,
  ./logtrace

# Compiled version of /scripts/depositContract.v.py in this repo
# The contract was compiled in Remix (https://remix.ethereum.org/) with vyper (remote) compiler.
const depositContractCode = staticRead "../beacon_chain/el/deposit_contract_code.txt"

type
  Eth1Address = web3.Address

  StartUpCommand {.pure.} = enum
    generateDeposits
    createTestnet
    run
    sendDeposits
    analyzeLogs
    deployDepositContract
    sendEth

  CliConfig* = object
    web3Url* {.
      defaultValue: "",
      desc: "URL of the Web3 server to observe Eth1"
      name: "web3-url" }: string

    privateKey* {.
      defaultValue: ""
      desc: "Private key of the controlling account"
      name: "private-key" }: string

    askForKey* {.
      defaultValue: false
      desc: "Ask for an Eth1 private key interactively"
      name: "ask-for-key" }: bool

    eth2Network* {.
      desc: "The Eth2 network preset to use"
      name: "network" }: Option[string]

    case cmd* {.command.}: StartUpCommand
    of StartUpCommand.deployDepositContract:
      discard

    of StartUpCommand.sendEth:
      toAddress* {.name: "to".}: Eth1Address
      valueEth* {.name: "eth".}: string

    of StartUpCommand.generateDeposits:
      simulationDepositsCount* {.
        desc: "The number of validator keystores to generate"
        name: "count" }: Natural

      outValidatorsDir* {.
        desc: "A directory to store the generated validator keystores"
        name: "out-validators-dir" }: OutDir

      outSecretsDir* {.
        desc: "A directory to store the generated keystore password files"
        name: "out-secrets-dir" }: OutDir

      outDepositsFile* {.
        desc: "A LaunchPad deposits file to write"
        name: "out-deposits-file" }: OutFile

      threshold* {.
        defaultValue: 1
        desc: "Used to generate distributed keys"
        name: "threshold" }: uint32

      remoteValidatorsCount* {.
        defaultValue: 0
        desc: "The number of distributed validators validator"
        name: "remote-validators-count" }: uint32

      remoteSignersUrls* {.
        desc: "URLs of the remote signers"
        name: "remote-signer" }: seq[string]

    of StartUpCommand.createTestnet:
      testnetDepositsFile* {.
        desc: "A LaunchPad deposits file for the genesis state validators"
        name: "deposits-file" .}: InputFile

      totalValidators* {.
        desc: "The number of validator deposits in the newly created chain"
        name: "total-validators" .}: uint64

      bootstrapAddress* {.
        desc: "The public IP address that will be advertised as a bootstrap node for the testnet"
        defaultValue: init(ValidIpAddress, defaultAdminListenAddress)
        defaultValueDesc: $defaultAdminListenAddressDesc
        name: "bootstrap-address" .}: ValidIpAddress

      bootstrapPort* {.
        desc: "The TCP/UDP port that will be used by the bootstrap node"
        defaultValue: defaultEth2TcpPort
        defaultValueDesc: $defaultEth2TcpPortDesc
        name: "bootstrap-port" .}: Port

      dataDir* {.
        desc: "Nimbus data directory where the keys of the bootstrap node will be placed"
        name: "data-dir" .}: OutDir

      netKeyFile* {.
        desc: "Source of network (secp256k1) private key file"
        name: "netkey-file" .}: OutFile

      netKeyInsecurePassword* {.
        desc: "Use pre-generated INSECURE password for network private key file"
        defaultValue: false,
        name: "insecure-netkey-password" .}: bool

      genesisTime* {.
        desc: "Unix epoch time of the network genesis"
        name: "genesis-time" .}: Option[uint64]

      genesisOffset* {.
        desc: "Seconds from now to add to genesis time"
        name: "genesis-offset" .}: Option[int]

      executionGenesisBlock* {.
        desc: "The execution genesis block in a merged testnet"
        name: "execution-genesis-block" .}: Option[InputFile]

      capellaForkEpoch* {.
        defaultValue: FAR_FUTURE_EPOCH
        desc: "The epoch of the Capella hard-fork"
        name: "capella-fork-epoch" .}: Epoch

      denebForkEpoch* {.
        defaultValue: FAR_FUTURE_EPOCH
        desc: "The epoch of the Deneb hard-fork"
        name: "deneb-fork-epoch" .}: Epoch

      outputGenesis* {.
        desc: "Output file where to write the initial state snapshot"
        name: "output-genesis" .}: OutFile

      outputDepositTreeSnapshot* {.
        desc: "Output file where to write the initial deposit tree snapshot"
        name: "output-deposit-tree-snapshot" .}: OutFile

      outputBootstrapFile* {.
        desc: "Output file with list of bootstrap nodes for the network"
        name: "output-bootstrap-file" .}: OutFile

    of StartUpCommand.sendDeposits:
      depositsFile* {.
        desc: "A LaunchPad deposits file"
        name: "deposits-file" }: InputFile

      depositContractAddress* {.
        desc: "Address of the deposit contract"
        name: "deposit-contract" }: Eth1Address

      minDelay* {.
        defaultValue: 0.0
        desc: "Minimum possible delay between making two deposits (in seconds)"
        name: "min-delay" }: float

      maxDelay* {.
        defaultValue: 0.0
        desc: "Maximum possible delay between making two deposits (in seconds)"
        name: "max-delay" }: float

    of StartUpCommand.run:
      discard

    of StartUpCommand.analyzeLogs:
      logFiles* {.
        desc: "Specifies one or more log files",
        abbr: "f",
        name: "log-file" .}: seq[string]

      simDir* {.
        desc: "Specifies path to eth2_network_simulation directory",
        defaultValue: "",
        name: "sim-dir" .}: string

      netDir* {.
        desc: "Specifies path to network build directory",
        defaultValue: "",
        name: "net-dir" .}: string

      logDir* {.
        desc: "Specifies path with bunch of logs",
        defaultValue: "",
        name: "log-dir" .}: string

      ignoreSerializationErrors* {.
        desc: "Ignore serialization errors while parsing log files",
        defaultValue: true,
        name: "ignore-errors" .}: bool

      dumpSerializationErrors* {.
        desc: "Dump full serialization errors while parsing log files",
        defaultValue: false ,
        name: "dump-errors" .}: bool

      nodes* {.
        desc: "Specifies node names which logs will be used",
        name: "nodes" .}: seq[string]

      allowedLag* {.
        desc: "Allowed latency lag multiplier",
        defaultValue: 2.0,
        name: "lag" .}: float

      constPreset* {.
        desc: "The const preset being used"
        defaultValue: "mainnet"
        name: "const-preset" .}: string

type
  PubKeyBytes = DynamicBytes[48, 48]
  WithdrawalCredentialsBytes = DynamicBytes[32, 32]
  SignatureBytes = DynamicBytes[96, 96]

contract(DepositContract):
  proc deposit(pubkey: PubKeyBytes,
               withdrawalCredentials: WithdrawalCredentialsBytes,
               signature: SignatureBytes,
               deposit_data_root: FixedBytes[32])

template `as`(address: ethtypes.Address, T: type bellatrix.ExecutionAddress): T =
  T(data: distinctBase(address))

template `as`(address: BlockHash, T: type Eth2Digest): T =
  asEth2Digest(address)

func getOrDefault[T](x: Option[T]): T =
  if x.isSome:
    x.get
  else:
    default T

func `as`(blk: BlockObject, T: type bellatrix.ExecutionPayloadHeader): T =
  T(parent_hash: blk.parentHash as Eth2Digest,
    fee_recipient: blk.miner as ExecutionAddress,
    state_root: blk.stateRoot as Eth2Digest,
    receipts_root: blk.receiptsRoot as Eth2Digest,
    logs_bloom: BloomLogs(data: distinctBase(blk.logsBloom)),
    prev_randao: Eth2Digest(data: blk.difficulty.toByteArrayBE), # Is BE correct here?
    block_number: uint64 blk.number,
    gas_limit: uint64 blk.gasLimit,
    gas_used: uint64 blk.gasUsed,
    timestamp: uint64 blk.timestamp,
    extra_data: List[byte, MAX_EXTRA_DATA_BYTES].init(blk.extraData.bytes),
    base_fee_per_gas: blk.baseFeePerGas.getOrDefault(),
    block_hash: blk.hash as Eth2Digest,
    transactions_root: blk.transactionsRoot as Eth2Digest)

func `as`(blk: BlockObject, T: type capella.ExecutionPayloadHeader): T =
  T(parent_hash: blk.parentHash as Eth2Digest,
    fee_recipient: blk.miner as ExecutionAddress,
    state_root: blk.stateRoot as Eth2Digest,
    receipts_root: blk.receiptsRoot as Eth2Digest,
    logs_bloom: BloomLogs(data: distinctBase(blk.logsBloom)),
    prev_randao: Eth2Digest(data: blk.difficulty.toByteArrayBE),
    block_number: uint64 blk.number,
    gas_limit: uint64 blk.gasLimit,
    gas_used: uint64 blk.gasUsed,
    timestamp: uint64 blk.timestamp,
    extra_data: List[byte, MAX_EXTRA_DATA_BYTES].init(blk.extraData.bytes),
    base_fee_per_gas: blk.baseFeePerGas.getOrDefault(),
    block_hash: blk.hash as Eth2Digest,
    transactions_root: blk.transactionsRoot as Eth2Digest,
    withdrawals_root: blk.withdrawalsRoot.getOrDefault() as Eth2Digest)

func `as`(blk: BlockObject, T: type deneb.ExecutionPayloadHeader): T =
  T(parent_hash: blk.parentHash as Eth2Digest,
    fee_recipient: blk.miner as ExecutionAddress,
    state_root: blk.stateRoot as Eth2Digest,
    receipts_root: blk.receiptsRoot as Eth2Digest,
    logs_bloom: BloomLogs(data: distinctBase(blk.logsBloom)),
    prev_randao: Eth2Digest(data: blk.difficulty.toByteArrayBE),
    block_number: uint64 blk.number,
    gas_limit: uint64 blk.gasLimit,
    gas_used: uint64 blk.gasUsed,
    timestamp: uint64 blk.timestamp,
    extra_data: List[byte, MAX_EXTRA_DATA_BYTES].init(blk.extraData.bytes),
    base_fee_per_gas: blk.baseFeePerGas.getOrDefault(),
    block_hash: blk.hash as Eth2Digest,
    transactions_root: blk.transactionsRoot as Eth2Digest,
    withdrawals_root: blk.withdrawalsRoot.getOrDefault() as Eth2Digest,
    blob_gas_used: uint64 blk.blobGasUsed.getOrDefault(),
    excess_blob_gas: uint64 blk.excessBlobGas.getOrDefault())

proc createDepositTreeSnapshot(deposits: seq[DepositData],
                               blockHash: Eth2Digest,
                               blockHeight: uint64): DepositTreeSnapshot =
  var merkleizer = DepositsMerkleizer.init()
  for i, deposit in deposits:
    let htr = hash_tree_root(deposit)
    merkleizer.addChunk(htr.data)

  DepositTreeSnapshot(
    eth1Block: blockHash,
    depositContractState: merkleizer.toDepositContractState,
    blockHeight: blockHeight)

proc doCreateTestnet*(config: CliConfig,
                      rng: var HmacDrbgContext)
                     {.raises: [CatchableError].} =
  let launchPadDeposits = try:
    Json.loadFile(config.testnetDepositsFile.string, seq[LaunchPadDeposit])
  except SerializationError as err:
    error "Invalid LaunchPad deposits file",
          err = formatMsg(err, config.testnetDepositsFile.string)
    quit 1

  var deposits: seq[DepositData]
  for i in 0 ..< launchPadDeposits.len:
    deposits.add(launchPadDeposits[i] as DepositData)

  let
    startTime = if config.genesisTime.isSome:
      config.genesisTime.get
    else:
      uint64(times.toUnix(times.getTime()) + config.genesisOffset.get(0))
    outGenesis = config.outputGenesis.string
    eth1Hash = mockEth1BlockHash # TODO: Can we set a more appropriate value?
    cfg = getRuntimeConfig(config.eth2Network)

  # This is intentionally left default initialized, when the user doesn't
  # provide an execution genesis block. The generated genesis state will
  # then be considered non-finalized merged state according to the spec.
  var genesisBlock = BlockObject()

  if config.executionGenesisBlock.isSome:
    logScope:
      path = config.executionGenesisBlock.get.string

    if not fileExists(config.executionGenesisBlock.get.string):
      error "The specified execution genesis block file doesn't exist"
      quit 1

    let genesisBlockContents = readAllChars(config.executionGenesisBlock.get.string)
    if genesisBlockContents.isErr:
      error "Failed to read the specified execution genesis block file",
            err = genesisBlockContents.error
      quit 1

    try:
      let blockAsJson = try:
        parseJson genesisBlockContents.get
      except CatchableError as err:
        error "Failed to parse the genesis block json", err = err.msg
        quit 1
      except:
        # TODO The Nim json library should not raise bare exceptions
        raiseAssert "The Nim json library raise a bare exception"
      fromJson(blockAsJson, "", genesisBlock)
    except CatchableError as err:
      error "Failed to load the genesis block from json",
            err = err.msg
      quit 1

  template createAndSaveState(genesisExecutionPayloadHeader: auto): Eth2Digest =
    var initialState = newClone(initialize_beacon_state_from_eth1(
        cfg, eth1Hash, startTime, deposits, genesisExecutionPayloadHeader,
        {skipBlsValidation}))
    # https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
    initialState.genesis_time = startTime

    doAssert initialState.validators.len > 0

    # let outGenesisExt = splitFile(outGenesis).ext
    #if cmpIgnoreCase(outGenesisExt, ".json") == 0:

    # let outGenesisJson = outGenesis & ".json"
    # RestJson.saveFile(outGenesisJson, initialState, pretty = true)
    # info "JSON genesis file written", path = outGenesisJson

    let outSszGenesis = outGenesis.changeFileExt "ssz"
    SSZ.saveFile(outSszGenesis, initialState[])
    info "SSZ genesis file written",
          path = outSszGenesis, fork = toFork(typeof initialState[])

    SSZ.saveFile(
      config.outputDepositTreeSnapshot.string,
      createDepositTreeSnapshot(
        deposits,
        genesisExecutionPayloadHeader.block_hash,
        genesisExecutionPayloadHeader.block_number))

    initialState[].genesis_validators_root

  let genesisValidatorsRoot =
    if config.denebForkEpoch == 0:
      createAndSaveState(genesisBlock as deneb.ExecutionPayloadHeader)
    elif config.capellaForkEpoch == 0:
      createAndSaveState(genesisBlock as capella.ExecutionPayloadHeader)
    else:
      createAndSaveState(genesisBlock as bellatrix.ExecutionPayloadHeader)

  let bootstrapFile = string config.outputBootstrapFile
  if bootstrapFile.len > 0:
    type MetaData = altair.MetaData
    let
      networkKeys = rng.getPersistentNetKeys(
        string config.dataDir, string config.netKeyFile,
        config.netKeyInsecurePassword, allowLoadExisting = false)

      netMetadata = MetaData()
      forkId = getENRForkID(
        cfg,
        Epoch(0),
        genesisValidatorsRoot)
      bootstrapEnr = enr.Record.init(
        1, # sequence number
        networkKeys.seckey.asEthKey,
        some(config.bootstrapAddress),
        some(config.bootstrapPort),
        some(config.bootstrapPort),
        [
          toFieldPair(enrForkIdField, SSZ.encode(forkId)),
          toFieldPair(enrAttestationSubnetsField, SSZ.encode(netMetadata.attnets))
        ])

    writeFile(bootstrapFile, bootstrapEnr.tryGet().toURI)
    echo "Wrote ", bootstrapFile

proc deployContract*(web3: Web3, code: string): Future[ReceiptObject] {.async.} =
  var code = code
  if code[1] notin {'x', 'X'}:
    code = "0x" & code
  let tr = EthSend(
    source: web3.defaultAccount,
    data: code,
    gas: Quantity(3000000).some,
    gasPrice: 1.some)

  let r = await web3.send(tr)
  result = await web3.getMinedTransactionReceipt(r)

proc sendEth(web3: Web3, to: Eth1Address, valueEth: int): Future[TxHash] =
  let tr = EthSend(
    source: web3.defaultAccount,
    gas: Quantity(3000000).some,
    gasPrice: 1.some,
    value: some(valueEth.u256 * 1000000000000000000.u256),
    to: some(to))
  web3.send(tr)

type
  DelayGenerator* = proc(): chronos.Duration {.gcsafe, raises: [].}

proc ethToWei(eth: UInt256): UInt256 =
  eth * 1000000000000000000.u256

proc initWeb3(web3Url, privateKey: string): Future[Web3] {.async.} =
  result = await newWeb3(web3Url)
  if privateKey.len != 0:
    result.privateKey = some(keys.PrivateKey.fromHex(privateKey)[])
  else:
    let accounts = await result.provider.eth_accounts()
    doAssert(accounts.len > 0)
    result.defaultAccount = accounts[0]

# TODO: async functions should note take `seq` inputs because
#       this leads to full copies.
proc sendDeposits*(deposits: seq[LaunchPadDeposit],
                   web3Url, privateKey: string,
                   depositContractAddress: Eth1Address,
                   delayGenerator: DelayGenerator = nil) {.async.} =
  notice "Sending deposits",
    web3 = web3Url,
    depositContract = depositContractAddress

  var web3 = await initWeb3(web3Url, privateKey)
  let gasPrice = int(await web3.provider.eth_gasPrice()) * 2
  let depositContract = web3.contractSender(DepositContract,
                                            Eth1Address depositContractAddress)
  for i in 4200 ..< deposits.len:
    let dp = deposits[i] as DepositData

    while true:
      try:
        let tx = depositContract.deposit(
          PubKeyBytes(@(dp.pubkey.toRaw())),
          WithdrawalCredentialsBytes(@(dp.withdrawal_credentials.data)),
          SignatureBytes(@(dp.signature.toRaw())),
          FixedBytes[32](hash_tree_root(dp).data))

        let status = await tx.send(value = 32.u256.ethToWei, gasPrice = gasPrice)

        info "Deposit sent", tx = $status

        if delayGenerator != nil:
          await sleepAsync(delayGenerator())

        break
      except CatchableError:
        await sleepAsync(chronos.seconds 60)
        web3 = await initWeb3(web3Url, privateKey)

{.pop.} # TODO confutils.nim(775, 17) Error: can raise an unlisted exception: ref IOError
proc main() {.async.} =
  var conf = try: CliConfig.load()
  except CatchableError as exc:
    raise exc
  except Exception as exc: # TODO fix confutils
    raiseAssert exc.msg

  let rng = HmacDrbgContext.new()

  if conf.cmd == StartUpCommand.generateDeposits:
    let
      mnemonic = generateMnemonic(rng[])
      seed = getSeed(mnemonic, KeystorePass.init "")
      cfg = getRuntimeConfig(conf.eth2Network)

    if (let res = secureCreatePath(string conf.outValidatorsDir); res.isErr):
      warn "Could not create validators folder",
        path = string conf.outValidatorsDir, err = ioErrorMsg(res.error)

    if (let res = secureCreatePath(string conf.outSecretsDir); res.isErr):
      warn "Could not create secrets folder",
        path = string conf.outSecretsDir, err = ioErrorMsg(res.error)

    let deposits = generateDeposits(
      cfg,
      rng[],
      seed,
      0, conf.simulationDepositsCount,
      string conf.outValidatorsDir,
      string conf.outSecretsDir,
      conf.remoteSignersUrls,
      conf.threshold,
      conf.remoteValidatorsCount,
      KeystoreMode.Fast)

    if deposits.isErr:
      fatal "Failed to generate deposits", err = deposits.error
      quit 1

    let launchPadDeposits =
      mapIt(deposits.value, LaunchPadDeposit.init(cfg, it))

    Json.saveFile(string conf.outDepositsFile, launchPadDeposits)
    notice "Deposit data written", filename = conf.outDepositsFile
    quit 0

  var deposits: seq[LaunchPadDeposit]
  if conf.cmd == StartUpCommand.sendDeposits:
    deposits = Json.loadFile(string conf.depositsFile, seq[LaunchPadDeposit])

  if conf.askForKey:
    var
      privateKey: string  # TODO consider using a SecretString type
      reasonForKey = ""

    if conf.cmd == StartUpCommand.sendDeposits:
      let
        depositsWord = if deposits.len > 1: "deposits" else: "deposit"
        totalEthNeeded = 32 * deposits.len
      reasonForKey = " in order to make your $1 (you'll need access to $2 ETH)" %
                     [depositsWord, $totalEthNeeded]

    echo "Please enter your Goerli Eth1 private key in hex form (e.g. 0x1a2...f3c)" &
          reasonForKey

    if not readPasswordFromStdin("> ", privateKey):
      error "Failed to read an Eth1 private key from standard input"

    if privateKey.len > 0:
      conf.privateKey = privateKey.string

  case conf.cmd
  of StartUpCommand.createTestnet:
    let rng = HmacDrbgContext.new()
    doCreateTestnet(conf, rng[])

  of StartUpCommand.deployDepositContract:
    let web3 = await initWeb3(conf.web3Url, conf.privateKey)
    let receipt = await web3.deployContract(depositContractCode)
    echo receipt.contractAddress.get, ";", receipt.blockHash

  of StartUpCommand.sendEth:
    let web3 = await initWeb3(conf.web3Url, conf.privateKey)
    echo await sendEth(web3, conf.toAddress, conf.valueEth.parseInt)

  of StartUpCommand.sendDeposits:
    var delayGenerator: DelayGenerator
    if not (conf.maxDelay > 0.0):
      conf.maxDelay = conf.minDelay
    elif conf.minDelay > conf.maxDelay:
      echo "The minimum delay should not be larger than the maximum delay"
      quit 1

    if conf.maxDelay > 0.0:
      delayGenerator = proc (): chronos.Duration =
        let
          minDelay = (conf.minDelay*1000).int64
          maxDelay = (conf.maxDelay*1000).int64
        chronos.milliseconds (rng[].rand(maxDelay - minDelay) + minDelay)

    await sendDeposits(deposits, conf.web3Url, conf.privateKey,
                       conf.depositContractAddress, delayGenerator)

  of StartUpCommand.run:
    discard

  of StartUpCommand.analyzeLogs:
    try:
      logtrace.run(LogTraceConf(
        cmd: logtrace.StartUpCommand.localSimChecks,
        logFiles: conf.logFiles,
        simDir: conf.simDir,
        netDir: conf.netDir,
        logDir: conf.logDir,
        ignoreSerializationErrors: conf.ignoreSerializationErrors,
        dumpSerializationErrors: conf.dumpSerializationErrors,
        nodes: conf.nodes,
        allowedLag: conf.allowedLag,
        constPreset: conf.constPreset
      ))
    except CatchableError as err:
      fatal "Unexpected error in logtrace", err = err.msg
    except Exception as exc:
      # TODO: Investigate where is this coming from?
      fatal "Unexpected exception in logtrace", err = exc.msg

  of StartUpCommand.generateDeposits:
    # This is handled above before the case statement
    discard

when isMainModule:
  waitFor main()