nimbus-eth1/fluffy/tools/eth_data_exporter/exporter_conf.nim

# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed and distributed under either of
#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.

{.push raises: [].}

import std/[os, uri], confutils, chronicles, beacon_chain/spec/digest

proc defaultDataDir*(): string =
  let dataDir =
    when defined(windows):
      "AppData" / "Roaming" / "EthData"
    elif defined(macosx):
      "Library" / "Application Support" / "EthData"
    else:
      ".cache" / "eth-data"

  getHomeDir() / dataDir

type
  Web3UrlKind* = enum
    HttpUrl
    WsUrl

  Web3Url* = object
    kind*: Web3UrlKind
    url*: string

  StorageMode* = enum
    JsonStorage
    DbStorage

const
  defaultDataDirDesc* = defaultDataDir()
  defaultBlockFileName* = "eth-block-data"
  defaultAccumulatorFileName* = "mainnet-master-accumulator.ssz"
  defaultWeb3Url* = Web3Url(kind: HttpUrl, url: "http://127.0.0.1:8545")

type
  ExporterCmd* = enum
    history
    beacon

  HistoryCmd* = enum
    # TODO: Multiline strings doesn't work here anymore with 1.6, and concat of
    # several lines gives the error: Error: Invalid node kind nnkInfix for macros.`$`
    exportBlockData =
      "Export block data (headers, bodies and receipts) to a json format or a database. Some of this functionality is likely to get deprecated"
    exportEpochHeaders =
      "Export block headers from an Ethereum JSON RPC Execution endpoint to *.e2s files arranged per epoch (8192 blocks)"
    verifyEpochHeaders =
      "Verify *.e2s files containing block headers. Verify currently only means being able to RLP decode the block headers"
    exportAccumulatorData =
      "Build and export the master accumulator and historical epoch accumulators. Requires *.e2s block header files generated with the exportHeaders command up until the merge block"
    printAccumulatorData =
      "Print the root hash of the master accumulator and of all historical epoch accumulators. Requires data generated by exportAccumulatorData command"
    exportHeaderRange =
      "Export block headers from an Ethereum JSON RPC Execution endpoint to *.e2s files (unlimited amount)"
    exportHeadersWithProof =
      "Export block headers with proof from *.e2s headers file and epochAccumulator files"
    exportEra1 = "Export historical data to era1 store"
    verifyEra1 = "Read and verify historical data from era1 store"

  BeaconCmd* = enum
    exportLCBootstrap = "Export Light Client Bootstrap"
    exportLCUpdates = "Export Light Client Updates"
    exportLCFinalityUpdate = "Export Light Client Finality Update"
    exportLCOptimisticUpdate = "Export Light Client Optimistic Update"
    exportHistoricalRoots = "Export historical roots from the beacon state (SSZ format)"
    exportBeaconBlockProof =
      "Export EL beacon block proof from era files (Bellatrix and later)"

  ExporterConf* = object
    logLevel* {.
      defaultValue: LogLevel.INFO,
      defaultValueDesc: $LogLevel.INFO,
      desc: "Sets the log level",
      name: "log-level"
    .}: LogLevel
    dataDir* {.
      desc: "The directory where generated data files will be exported to",
      defaultValue: defaultDataDir(),
      defaultValueDesc: $defaultDataDirDesc,
      name: "data-dir"
    .}: OutDir
    case cmd* {.command.}: ExporterCmd
    of ExporterCmd.history:
      web3Url* {.
        desc: "Execution layer JSON-RPC API URL",
        defaultValue: defaultWeb3Url,
        name: "web3-url"
      .}: Web3Url
      case historyCmd* {.command.}: HistoryCmd
      of exportBlockData:
        startBlock* {.
          desc: "Number of the first block to be exported",
          defaultValue: 0,
          name: "start-block"
        .}: uint64
        endBlock* {.
          desc: "Number of the last block to be exported",
          defaultValue: 0,
          name: "end-block"
        .}: uint64
        fileName* {.
          desc: "File name (minus extension) where block data will be exported to",
          defaultValue: defaultBlockFileName,
          defaultValueDesc: $defaultBlockFileName,
          name: "file-name"
        .}: string
        storageMode* {.
          desc: "Storage mode of block data export",
          defaultValue: JsonStorage,
          name: "storage-mode"
        .}: StorageMode
        headersOnly* {.
          desc: "Only export the headers instead of full blocks and receipts",
          defaultValue: false,
          name: "headers-only"
        .}: bool
      of exportEpochHeaders:
        startEpoch* {.
          desc: "Number of the first epoch which should be downloaded",
          defaultValue: 0,
          name: "start-epoch"
        .}: uint64
        endEpoch* {.
          desc: "Number of the last epoch which should be downloaded",
          defaultValue: 1896,
          name: "end-epoch"
        .}: uint64
      # TODO:
      # Although options are the same as for exportHeaders, we can't drop them
      # under the same case of as confutils does not agree with that.
      of verifyEpochHeaders:
        startEpochVerify* {.
          desc: "Number of the first epoch which should be downloaded",
          defaultValue: 0,
          name: "start-epoch"
        .}: uint64
        endEpochVerify* {.
          desc: "Number of the last epoch which should be downloaded",
          defaultValue: 1896,
          name: "end-epoch"
        .}: uint64
      of exportAccumulatorData:
        accumulatorFileName* {.
          desc: "File to which the serialized accumulator is written",
          defaultValue: defaultAccumulatorFileName,
          defaultValueDesc: $defaultAccumulatorFileName,
          name: "accumulator-file-name"
        .}: string
        writeEpochAccumulators* {.
          desc: "Write also the SSZ encoded epoch accumulators to specific files",
          defaultValue: false,
          name: "write-epoch-accumulators"
        .}: bool
      of printAccumulatorData:
        accumulatorFileNamePrint* {.
          desc: "File from which the serialized accumulator is read",
          defaultValue: defaultAccumulatorFileName,
          defaultValueDesc: $defaultAccumulatorFileName,
          name: "accumulator-file-name"
        .}: string
      of exportHeaderRange:
        startBlockNumber* {.
          desc: "Number of the first block header to be exported", name: "start-block"
        .}: uint64
        endBlockNumber* {.
          desc: "Number of the last block header to be exported", name: "end-block"
        .}: uint64
      of exportHeadersWithProof:
        startBlockNumber2* {.
          desc: "Number of the first block header to be exported", name: "start-block"
        .}: uint64
        endBlockNumber2* {.
          desc: "Number of the last block header to be exported", name: "end-block"
        .}: uint64
      of exportEra1:
        era* {.defaultValue: 0, desc: "The era number to write".}: uint64
        eraCount* {.
          defaultValue: 0, name: "count", desc: "Number of eras to write (0=all)"
        .}: uint64
      of verifyEra1:
        era1FileName* {.desc: "Era1 file to read and verify", name: "era1-file-name".}:
          string
    of ExporterCmd.beacon:
      restUrl* {.
        desc: "URL of the beacon node REST service",
        defaultValue: "http://127.0.0.1:5052",
        name: "rest-url"
      .}: string
      case beaconCmd* {.command.}: BeaconCmd
      of exportLCBootstrap:
        trustedBlockRoot* {.
          desc: "Trusted finalized block root of the requested bootstrap",
          name: "trusted-block-root"
        .}: Eth2Digest
      of exportLCUpdates:
        startPeriod* {.
          desc: "Period of the first LC update", defaultValue: 0, name: "start-period"
        .}: uint64
        count* {.
          desc: "Amount of LC updates to request", defaultValue: 1, name: "count"
        .}: uint64
      of exportLCFinalityUpdate:
        discard
      of exportLCOptimisticUpdate:
        discard
      of exportHistoricalRoots:
        discard
      of exportBeaconBlockProof:
        slotNumber* {.
          desc: "The slot for which to export the beacon block proof", name: "slot"
        .}: uint64
        eraDir* {.desc: "Directory containing era files", name: "era-dir".}: InputDir

proc parseCmdArg*(T: type Web3Url, p: string): T {.raises: [ValueError].} =
  let
    url = parseUri(p)
    normalizedScheme = url.scheme.toLowerAscii()

  if (normalizedScheme == "http" or normalizedScheme == "https"):
    Web3Url(kind: HttpUrl, url: p)
  elif (normalizedScheme == "ws" or normalizedScheme == "wss"):
    Web3Url(kind: WsUrl, url: p)
  else:
    raise newException(
      ValueError,
      "The Web3 URL must specify one of following protocols: http/https/ws/wss",
    )

proc completeCmdArg*(T: type Web3Url, val: string): seq[string] =
  return @[]

proc parseCmdArg*(T: type StorageMode, p: string): T {.raises: [ValueError].} =
  if p == "db":
    return DbStorage
  elif p == "json":
    return JsonStorage
  else:
    let msg = "Provided mode: " & p & " is not a valid. Should be `json` or `db`"
    raise newException(ValueError, msg)

proc completeCmdArg*(T: type StorageMode, val: string): seq[string] =
  return @[]

func parseCmdArg*(
    T: type Eth2Digest, input: string
): T {.raises: [ValueError, Defect].} =
  Eth2Digest.fromHex(input)

func completeCmdArg*(T: type Eth2Digest, input: string): seq[string] =
  return @[]