nimbus-eth2/ncli/validator_db_reports.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "071e1cb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autotime\n",
    "%matplotlib inline\n",
    "import string\n",
    "import sqlite3\n",
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import snappy\n",
    "from scipy.interpolate import make_interp_spline\n",
    "from pathlib import Path\n",
    "from io import StringIO"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f00bca92",
   "metadata": {},
   "source": [
    "**Database connection:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f69cd8b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "database_dir = \"../build/data/mainnetValidatorDb/validatorDb.sqlite3\"\n",
    "connection = sqlite3.connect(database_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ccd8945",
   "metadata": {},
   "source": [
    "**Rewards and penalties components:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "229adad0",
   "metadata": {},
   "outputs": [],
   "source": [
    "SOURCE = \"source\"\n",
    "TARGET = \"target\"\n",
    "HEAD = \"head\"\n",
    "INCLUSION_DELAY = \"inclusion_delay\"\n",
    "SYNC_COMMITTEE = \"sync_committee\"\n",
    "\n",
    "CSV_DATA_COLUMNS_NAMES = [\n",
    "    \"source_outcome\",\n",
    "    \"max_source_reward\",\n",
    "    \"target_outcome\",\n",
    "    \"max_target_reward\",\n",
    "    \"head_outcome\",\n",
    "    \"max_head_reward\",\n",
    "    \"inclusion_delay_outcome\",\n",
    "    \"max_inclusion_delay_reward\",\n",
    "    \"sync_committee_outcome\",\n",
    "    \"max_sync_committee_reward\",\n",
    "    \"proposer_outcome\",\n",
    "    \"inactivity_penalty\",\n",
    "    \"slashing_outcome\",\n",
    "    \"deposits\",\n",
    "    \"inclusion_delay\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a747287",
   "metadata": {},
   "source": [
    "**Helper functions:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63fb9f21",
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid_public_key(public_key):\n",
    "    \"\"\"Checks whether a string is a valid hex representation of a public key of an Eth2 validator.\"\"\"\n",
    "    if len(public_key) != 96:\n",
    "        return False\n",
    "    return all(c in string.hexdigits for c in public_key)\n",
    "\n",
    "def idx(public_key):\n",
    "    \"\"\"Returns validator's index by its public key.\"\"\"\n",
    "    \n",
    "    if public_key.startswith(\"0x\"):\n",
    "        public_key = public_key[2:]\n",
    "    \n",
    "    if not valid_public_key(public_key):\n",
    "        raise ValueError(f\"The string '{public_key}' is not a valid public key of a validator.\")\n",
    "        \n",
    "    QUERY_FIELD = \"validator_index\"\n",
    "    query = f\"SELECT {QUERY_FIELD} FROM validators_raw WHERE pubkey=x'{public_key}';\"\n",
    "    query_result = pd.read_sql_query(query, connection)\n",
    "    \n",
    "    if len(query_result[QUERY_FIELD]) == 0:\n",
    "        raise ValueError(f\"Not found a validator with a public key '{public_key}'.\")\n",
    "    \n",
    "    if len(query_result[QUERY_FIELD]) > 1:\n",
    "        raise ValueError(f\"Found multiple validators with a public key '{public_key}'.\")\n",
    "    \n",
    "    return query_result[QUERY_FIELD][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "946762c1",
   "metadata": {},
   "source": [
    "**Input parameters:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9aca3ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 1\n",
    "end_epoch = 94275\n",
    "files_dir = \"../build/data/mainnetCompactedValidatorDb/\"\n",
    "rewards = [SOURCE, TARGET, HEAD, INCLUSION_DELAY, SYNC_COMMITTEE]\n",
    "validators_sets = {\n",
    "    \"set1\": list(range(10)),\n",
    "    \"set2\": list(map(idx, [\n",
    "        \"0x8efba2238a00d678306c6258105b058e3c8b0c1f36e821de42da7319c4221b77aa74135dab1860235e19d6515575c381\",\n",
    "        \"0xa2dce641f347a9e46f58458390e168fa4b3a0166d74fc495457cb00c8e4054b5d284c62aa0d9578af1996c2e08e36fb6\",\n",
    "        \"0x98b7d0eac7ab95d34dbf2b7baa39a8ec451671328c063ab1207c2055d9d5d6f1115403dc5ea19a1111a404823bd9a6e9\",\n",
    "        \"0xb0fd08e2e06d1f4d90d0d6843feb543ebeca684cde397fe230e6cdf6f255d234f2c26f4b36c07170dfdfcbbe355d0848\",\n",
    "        \"0xab7a5aa955382906be3d76e322343bd439e8690f286ecf2f2a7646363b249f5c133d0501d766ccf1aa1640f0283047b3\",\n",
    "        \"0x980c0c001645a00b71c720935ce193e1ed0e917782c4cb07dd476a4fdb7decb8d91daf2770eb413055f0c1d14b5ed6df\",\n",
    "        \"0xac7cbdc535ce8254eb9cdedf10d5b1e75de4cd5e91756c3467d0492b01b70b5c6a81530e9849c6b696c8bc157861d0c3\",\n",
    "        \"0x98ea289db7ea9714699ec93701a3b6db43900e04ae5497be01fa8cc5a56754c23589eaf1f674de718e291376f452d68c\",\n",
    "        \"0x92451d4c099e51f54ab20f5c1a4edf405595c60122ccfb0f39250b7e80986fe0fe457bacd8a887e9087cd6fc323f492c\",\n",
    "        \"0xa06f6c678f0129aec056df309a4fe18760116ecaea2292947c5a9cc997632ff437195309783c269ffca7bb2704e675a0\"\n",
    "        ])),\n",
    "    \"set3\": list(range(20, 30))\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e0fb2da",
   "metadata": {},
   "source": [
    "**Loading the data and losses calculation:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "485a2d7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "COMPACTED_EPOCH_INFO_FILE_PATTERN = re.compile(r\"(\\d{8})\\_(\\d{8})\\.epoch\")\n",
    "\n",
    "def get_first_and_last_epoch(file_name):\n",
    "    m = re.match(COMPACTED_EPOCH_INFO_FILE_PATTERN, file_name)\n",
    "    if m == None:\n",
    "        return None\n",
    "    return int(m.group(1)), int(m.group(2))\n",
    "\n",
    "def isEpochInfoFile(file_name):\n",
    "    r = get_first_and_last_epoch(file_name)\n",
    "    if r == None:\n",
    "        return False\n",
    "    file_start_epoch, file_end_epoch = r\n",
    "    if file_start_epoch > file_end_epoch:\n",
    "        return False\n",
    "    if file_end_epoch < start_epoch:\n",
    "        return False\n",
    "    if file_start_epoch > end_epoch:\n",
    "        return False\n",
    "    return True\n",
    "\n",
    "def adjust_constraints(sorted_file_names):\n",
    "    first_start_epoch, first_end_epoch = get_first_and_last_epoch(sorted_file_names[0])\n",
    "    _, last_end_epoch = get_first_and_last_epoch(sorted_file_names[-1])\n",
    "    global start_epoch, end_epoch, resolution\n",
    "    start_epoch = first_start_epoch\n",
    "    end_epoch = last_end_epoch\n",
    "    resolution = first_end_epoch - first_start_epoch + 1\n",
    "\n",
    "def read_csv(file_path):\n",
    "     return pd.read_csv(\n",
    "        StringIO(snappy.decompress(file_path.read_bytes()).decode(\"utf-8\")),\n",
    "        names = CSV_DATA_COLUMNS_NAMES, usecols = set(range(0, 10)))\n",
    "\n",
    "def get_outcome_var(component):\n",
    "    return component + \"_outcome\"\n",
    "\n",
    "def get_max_reward_var(component):\n",
    "    return \"max_\" + component + \"_reward\"\n",
    "\n",
    "max_reward_vars = [get_max_reward_var(reward_type) for reward_type in rewards]\n",
    "outcome_vars = [get_outcome_var(reward_type) for reward_type in rewards]\n",
    "\n",
    "def sum_max_values(t):\n",
    "    return sum(getattr(t, max_reward) for max_reward in max_reward_vars)\n",
    "\n",
    "def sum_actual_values(t):\n",
    "    return sum(getattr(t, outcome) for outcome in outcome_vars)\n",
    "\n",
    "def compute_losses_median(data):\n",
    "    max_values = data[max_reward_vars].sum(axis = 1)\n",
    "    actual_values = data[outcome_vars].sum(axis = 1)\n",
    "    losses = max_values - actual_values\n",
    "    return losses.median(axis = 0)\n",
    "\n",
    "total_losses_per_epoch_point = {}\n",
    "average_losses_per_epoch_point = {}\n",
    "validators_sets_queries = {}\n",
    "medians = {}\n",
    "\n",
    "for set_name, set_values in validators_sets.items():\n",
    "    total_losses_per_epoch_point[set_name] = {}\n",
    "    average_losses_per_epoch_point[set_name] = {}\n",
    "    validators_sets_queries[set_name] = []\n",
    "\n",
    "file_names = [file_name for file_name in os.listdir(files_dir)\n",
    "              if isEpochInfoFile(file_name)]\n",
    "file_names.sort()\n",
    "adjust_constraints(file_names)\n",
    "\n",
    "previous_validators_count = 0\n",
    "for file_name in file_names:\n",
    "    data = read_csv(Path(files_dir + file_name))\n",
    "    file_first_epoch, file_last_epoch = get_first_and_last_epoch(file_name)\n",
    "    file_epoch_range = file_last_epoch - file_first_epoch + 1\n",
    "    epoch_point = file_first_epoch // resolution\n",
    "    validators_count = len(data.index)\n",
    "    for set_name, validators in validators_sets.items():\n",
    "        for i in range(previous_validators_count, validators_count):\n",
    "            if i in validators:\n",
    "                validators_sets_queries[set_name].append(i)\n",
    "        sums = data.iloc[validators_sets_queries[set_name]].sum(axis = 0)\n",
    "        difference = sum_max_values(sums) - sum_actual_values(sums)\n",
    "        set_validators_count = len(validators_sets_queries[set_name])\n",
    "        average_losses_per_epoch_point[set_name][epoch_point] = \\\n",
    "            difference / set_validators_count if set_validators_count > 0 else 0\n",
    "        total_losses_per_epoch_point[set_name][epoch_point] = difference * file_epoch_range\n",
    "    medians[epoch_point] = compute_losses_median(data)\n",
    "    previous_validators_count = validators_count\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "800ee35b",
   "metadata": {},
   "source": [
    "**Average losses graph:** "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62d1e96d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.subplots(figsize = (20, 5))\n",
    "plt.title(\"Average losses per epoch\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Gwei\")\n",
    "\n",
    "num_samples = (end_epoch - start_epoch + 1) // resolution * 100\n",
    "\n",
    "def plot(set_name, set_values):\n",
    "    epochs = np.array([ep * resolution + resolution // 2 for ep in set_values.keys()])\n",
    "    values = np.array(list(set_values.values()))\n",
    "    spline = make_interp_spline(epochs, values)\n",
    "    x = np.linspace(epochs.min(), epochs.max(), num_samples)\n",
    "    y = spline(x)\n",
    "    plt.plot(x, y, label=set_name)\n",
    "\n",
    "for name, value in average_losses_per_epoch_point.items():\n",
    "    plot(name, value)\n",
    "\n",
    "plot(\"median\", medians)\n",
    "\n",
    "plt.legend(loc=\"best\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fff538c",
   "metadata": {},
   "source": [
    "**Total losses:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ab52601",
   "metadata": {},
   "outputs": [],
   "source": [
    "sets_total_losses = {}\n",
    "for set_name, epoch_points in total_losses_per_epoch_point.items():\n",
    "    sets_total_losses[set_name] = 0\n",
    "    for _, losses in epoch_points.items():\n",
    "        sets_total_losses[set_name] += losses\n",
    "\n",
    "plt.title(\"Total losses\")\n",
    "plt.xlabel(\"Set\")\n",
    "plt.ylabel(\"Ethers\")\n",
    "plt.bar(list(sets_total_losses.keys()), [loss * 1e-9 for loss in sets_total_losses.values()])\n",
    "print(sets_total_losses)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}