logos-storage/logos-storage-research
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-research.git synced 2026-01-04 06:23:09 +00:00
Code Issues Packages Projects Releases Wiki Activity
logos-storage-research/analysis/PoR_Analysis.ipynb

370 lines
76 KiB
Plaintext
Raw Normal View History

Adding PoR analysis model, it includes horizontal and vertical verification frequency calculation and the Storj scenario
2022-08-10 10:16:12 +02:00
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "PoR Analysis",
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# Verification frequency calculation\n",
"\n",
"This code shows how we can compute the minimum horizontal and vertical verification frequency in order to achieve a certain level of guarantee computed in number of nines (e.g., 99.9999999%)\n"
],
"metadata": {
"id": "IAw0wWuN-1Yh"
}
},
{
"cell_type": "code",
"source": [
"import numpy as np\n",
"import random as random\n",
"import matplotlib.pyplot as plt\n",
"\n",
"#@markdown Durability guarantee (three 9s (99.9%) to nine 9s (99.9999999%))\n",
"Durability = 9 #@param {type:\"slider\", min:3, max:9, step:3}\n",
"#@markdown Erasure Coding parameter K\n",
"ECK = 60 #@param {type:\"slider\", min:1, max:100, step:1}\n",
"#@markdown Percentage of data a malicious node discards or refuses/avoids to store\n",
"CheatRatio = 5 #@param {type:\"slider\", min:5, max:95, step:5}\n",
"#@markdown Number of PoR blocks inside an EC block (See model below)\n",
"NbPoRinEC = 500 #@param {type:\"slider\", min:100, max:10000, step:100}\n",
"\n",
"GuaranteeDurability = {3:99.90, 6: 99.99990, 9: 99.99999990}\n",
"ExpectedGuarantee = GuaranteeDurability[Durability]\n",
"\n",
"hx = []\n",
"HorizontalProb = []\n",
"for NbOfHVerif in range(ECK):\n",
" prob = (1-(((ECK-1)/100)**NbOfHVerif))*100\n",
" hx.append(NbOfHVerif)\n",
" HorizontalProb.append(prob)\n",
" if prob > ExpectedGuarantee:\n",
" print(\"Probability of detecting catastrophic failures with %d horizontal verifications is %12.12f%% (Over nine 9s)\" % (NbOfHVerif, prob))\n",
" break\n",
"\n",
"vx = [0]\n",
"VerticalProb = [0]\n",
"for NbOfVVerif in range(1, NbPoRinEC):\n",
" prob = (1-((1-(CheatRatio/100))**NbOfVVerif))*100\n",
" vx.append(NbOfVVerif)\n",
" VerticalProb.append(prob)\n",
" if prob > ExpectedGuarantee:\n",
" print(\"Probability of detecting catastrophic failures with %d vertical verifications %12.12f%% (Over nine 9s)\" % (NbOfVVerif, prob))\n",
" print(\"That is %2.1f%% of the PoR blocks need to be verified\" % (NbOfVVerif*100/NbPoRinEC))\n",
" break\n",
"\n",
"\n",
"fig, ax = plt.subplots()\n",
"plt.grid(axis='x', color='0.9')\n",
"plt.grid(axis='y', color='0.9')\n",
"plt.title(\"Number of horizontal verifications to reach %9.9f guarantee on durability\" % ExpectedGuarantee)\n",
"plt.xlabel(\"Nodes verified from K=%d in the EC\" % ECK)\n",
"plt.ylabel(\"Guarantee of not having catastrophic failures (%)\")\n",
"ax.plot(hx, HorizontalProb)\n",
"#plt.yscale('log')\n",
"plt.show()\n",
"plt.clf()\n",
"\n",
"\n",
"fig, ax = plt.subplots()\n",
"plt.grid(axis='x', color='0.9')\n",
"plt.grid(axis='y', color='0.9')\n",
"plt.title(\"Number of vertical verifications to reach %9.9f guarantee on durability\" % ExpectedGuarantee)\n",
"plt.xlabel(\"Number of PoR blocks verified out of %d\" % NbPoRinEC)\n",
"plt.ylabel(\"Guarantee of not having catastrophic failures (%)\")\n",
"ax.plot(vx, VerticalProb)\n",
"#plt.yscale('log')\n",
"plt.show()\n",
"\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"id": "Y_CIsSJz_Nek",
"outputId": "1b4b7efb-2990-44b2-dd22-67c257d4ff70"
},
"execution_count": 48,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Probability of detecting catastrophic failures with 40 horizontal verifications is 99.999999931753% (Over nine 9s)\n",
"Probability of detecting catastrophic failures with 405 vertical verifications 99.999999904926% (Over nine 9s)\n",
"That is 81.0% of the PoR blocks need to be verified\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"<Figure size 1080x576 with 1 Axes>"
],
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA3sAAAHwCAYAAAAfJXbRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZxcZZno8d/TnT1kJRBIwr7KIi5sInoRXFAQdFxGQQa54DKOyx2vjtuguDveGb24DCPqMIg4qLhCvDrIgM6ooCCQhH0RJN0JIUt39vT23D/O6VAJvVS6u7p6+X0/n/5U1TlV73nqrVOnz1Pve943MhNJkiRJ0tjSUO8AJEmSJElDz2RPkiRJksYgkz1JkiRJGoNM9iRJkiRpDDLZkyRJkqQxyGRPkiRJksYgkz1pDIqIf4uIT9Vp2xERV0TEuoj4fQ/r3xwR/z2E2/twRHxjqMobChFxc0RcNEzb2hgRB5b3p0bEdRHRGhHfj4hzI+I/arDNF0TE/UNd7lgwnJ+9NNLUcv+PiFMiYnkf6/8lIi7u6bkRcXdEnFKLuKSRzmRPGgYR8WhErIqI6RXLLoqIm+sYVq2cDLwEWJSZx9d6Y5n5mcys6cl1RGREHFzLbQxUZu6WmY+UD18LzAd2z8zXZebVmfnSwW5j5/efmf+VmYcNttxdjGFMJlHlceChMmn/eUQsqFg3OyKuLI8dqyLiklqVFREnRcTvI2JDRCyJiJMr1kVEfCQi/hwR6yPimoiYWbF+YUT8JCLWRsTyiHj7TmW/MiKWlXH9NiKOqFg3OSK+GBHN5Q9E/xwREyvWz42IH0XEpoh4LCLO2ansc8rlmyLixxExt5p6H8kiYv/yOzeh3rGMJpn59sz8ZC/rjszMmwEi4pKI+PawBifVkcmeNHwagffUO4hdFRGNu/iS/YBHM3NTLeKpNJ5Phnp57/sBD2Rmx3DHM5qMlP2mbGn4DHA2MBf4E/DvFU/5IjAN2B84HjgvIi4Y6rLKBOk64P8As4HPA9dFxJzytX8FnAc8H1gATAW+XFH2t8vtzQfOAD4TES8qyz4EuBp4e1n2dcBPKz6DDwLHAkcBhwLPAf6+ouyvAm1l2ecCl0XEkWXZRwJfK2ObD2wG/rmn+hlOI2X/Gk3KHxQ8J5VqITP988+/Gv8Bj1Kc1KwFZpfLLgJuLu/vDyQwoeI1NwMXlfffDPyG4oStBXgEOKlc/jiwCji/4rX/BvwLcAOwAfgVsF/F+sPLdWuB+4HX7/Tay4CfAZuAF/fwfhYAPy1f/xDwlnL5hcBWoBPYCHy8h9e+Gfhv4B+BdRQniS/vr+xy3SXAtRQnl+vLOrwE+Ha5/ivldrv/OoBLynXPKOu0BbgbOGun9/xVYHFZX7cCB5Xrfl1+NpvKMv8SmANcDzxZvofrKVoyn/bZ9VBvW4C5FcueDawGJpaP/ydwb1nuL3b63BL4G+BB4E8Vyw4GPk5xUtxexnlhd11XvP7Iis/9CeDD5fLjgd+VdbOirMdJfbz/U4DlFeUOtG6DYp9eVX6eS4Gjeqi3T1PsU1vLGL5SLj8J+APQWt6e1M938APAEmAbMAE4EfhtGfddwCkVz7+g/Bw2UHzf3rZTeWcDd5ZxPwycXvHZf5Li+7oB+A9gXi8x/SPw1Z32j6yon9XAcRXrPwz811CXBZwJ3L1TeQ8AF5b3rwXeX7HupPKzmAbsVm5nj4r1lwNXlfffCSyuWNdA8R04rXx8G/C6ivXnAI+X96dT7NOHVqy/Cvhcef8zwHcq1h1UPn9GL3X0UorjXStFUvgrnjrGXkJ5HOnpmNzX/kD5fSj3r5VljNUcI3rcT4A/l9vuPo49r4pjQ6/H9GqP3xX18D3gW2VcdwPH9lHWS4D7yjr9yi7W6c0U3+3flPvEwVXW84cp9udHgXN3OtZ8qvK5O33/Xwyczo7HybuA1wG37/S+3gv8pLf37Z9/o+nPX1Gk4XMbxT+39w3w9SdQnKjuDnwHuAY4juIf5JuAr0TEbhXPP5fiZGIexUnp1QBlV9IbyjL2BN4A/HNl1yqKE65PAzMoErOdXUPxT3cBRdfBz0TEqZn5TYpf8H+XRffCj/XxXu4vY/s88M2IiL7Krnjt2RQnn7O731O3zHxnud3dKLqTrgN+UnYLu47ihGpP4F3A1RFR2RXxDRQJ0xyKE6BPl2W+sFx/TFn2dylOWK+gaEnbl+JE5Su9vNfK+JopkqrXVCw+B7g2M9sj4myKE5m/APYA/osdW2cAXkVRf5WfF2Vdfwb4bhnnNyvXR8QM4JfAzynq9mDgxnJ1J/C3FJ/H84DTgHf08f4ryx1w3VKcfL+QokVnFvB6YE0P9faRsi66P993lq1Ri4EvUXwnvgAsjojdd359hTdStDzNpmgJWgx8iqIl7H3ADyJij/K5qyiSoJkUJ6BfjIjnlO/5eIqT4feXZb2Q4mSy2znla/YEJtH3dz56uH9UH+sr1w1lWZXr+lsfwGTgkIrl1ZYdVaxfFBGzKPaLjsx8oGL9XRQ/WlDe3tW9IjMfpkwOd3ovRMQ8iuPGhyj2l/spktZq9bo/lPai2I/2A95KdceI3vaT7u/c7HJ//11fx4Yqj+mV+jvGnlU+ZzZFUtjjsa2s0x9StMTOo/jR4/m9bLM351HU1wzgMaqr53nAQuB84PKdjjV9ysyfs+Nx8hiK93hARDxjp7i+tYvvRRqRTPak4fVR4F0VJ5S74k+ZeUVmdgLfBfYBPpGZ2zLzPyhOciqvK1ucmb/OzG3AR4DnRcQ+FP9IHy3L6sjMO4AfUPy62e0nmfmbzOzKzK2VQZRlPB/4QGZuzcw7gW9QdPWq1mOZ+fXyvVwJ7A3Mr7Ls32Xmj8vYtvRUeFm/PwbeVb6/EylaID6XmW2Z+Z8Uv7S/seJlP8rM32fRBfJq4Fm9BZ+ZazLzB5m5OTM3UCQv/6PK9/6d7u2WCe4bymVQJMqfzcx7yzg+AzwrIvareP1nM3Ntb++9D2cCKzPzn8q63ZCZt5bv5/bMvKXcHx6l6BpX7fsZTN22U5zkHQ5E+b5XVLndM4AHM/OqMu5/p2hheGUfr/lSZj5e1t2bgJ9l5s/KfekGih9kXgGQmYsz8+Es/IoimX1BWc6FwL9m5g3la5sy876K7VyRmQ+U2/keve9LPwdeHxHPjIipFMeHpGgx617/wYiYEcU1k/+zYt1QlvU7YEFEvDEiJkbE+RStZJWvvai8lmwWRQsWwLRy//8NcHFETClPzF9T8dpfAv8jigEzJlEkLJN2Kvs9EbFHROwFvLu7bIr9av1O77OVYp+hXN/ax/pKr6BovfxhuR9+iaIVrir97A8AXcDHyuPxliqPEdXuJ9D3saGaYzpQ9fH7v8vvRSdFK+UxvcTUXafXZmY78H/ZhTot/Vtm3l3G3V5FPQNcXNbzryh+sHn9Lm5zB+X/yO9SHBO6uwfvT3Eck0Y9kz1pGGXmMop/IB8cwMufqLi/pSxv52WVLXuPV2x3I0WXnQUUvzSfEBEt3X8UrYB79fTaHiwA1pYnMN0eo/iltVrbTwgyc3N5d7cqy+4rtu6WpmspunddUxHz45nZ1Ue5lScpm9mxLnfexrSI+Fo5MMR6iq6Os6u8vvEHFIn33hS/4HdR/EoPxWdzacXnspaipaPq99+HfSh+eX+aiDg0Iq6PiJXl+/kMxa/n1Rhw3ZaJ4VcounmuiojLo2Lgjyq2+9hOy/rbDyvrbj/gdTt9D06m+OGBiHh5RNwSxaAjLRQntt110mtdlqralzLzl8DHKPaJR8u/DRStLlAkPlsouu3+hKIlp8fRCAdTVmauoWgxfy/FceZ0iiSt+7X/Wj7/ZopufTeVy7vXnwscQFG/l1F0s+4u+z6KFpivUHQRngfcU/HaTwN3UPQ++C3FjzTtZRwbKVp4Ks0s3xdVrK+0gB2PiUkvddmTfvYHgCcrfxir8hhR9TGHvo8N1RzTu1VzjN05rim9XIfYU53u6vFph+dXUc/rcsfrwR8r4xisK4Fzyh/gzgO+VyaB0qhnsicNv48Bb2HHf67d/7wqf7Xv6
},
"metadata": {
"needs_background": "light"
}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"<Figure size 1080x576 with 0 Axes>"
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"<Figure size 1080x576 with 1 Axes>"
],
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA3sAAAHwCAYAAAAfJXbRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd5xddZn48c8zM+mFJCRAICEhFCmKlEgRYe0NRXftYPlhQde1rLu6lrXrqus224qiriuWtSsqrKuiIBZAitICBAiE9Ex6z5Tn98c5IzfDlJtk7pwpn/frdV/3nv6c75x75jz3+z3fE5mJJEmSJGlkaao6AEmSJEnSwDPZkyRJkqQRyGRPkiRJkkYgkz1JkiRJGoFM9iRJkiRpBDLZkyRJkqQRyGRPGoUi4r8j4sMVbTsi4ssRsSEirq9g+4dHxNaIaN7P9VwVEa8eqLj62dbWiFhQfp4QET+OiE0R8Z2IuCAiftaAbZ4dEXcN9HpHgsH820tDTSOP/4h4fEQs62P65yLiPT3NGxG3R8TjGxGXNJyZ7ElDQETcHxFrImJSzbhXR8RVFYbVKI8DngLMyczTGr2xsmyf3DWcmUszc3JmdjR62wOljPe+cvD5wMHAgZn5gsz8emY+dX+3EREZEUfVbPOazHzE/q53L2MYkUlU+V2+p0zafxoRh9ZMmxYRXym//2si4v2NWldEPDYiro+ILRFxS0Q8rmZaRMQ/RsTSiNgcEd+MiKk10w+LiMsiYn1ELIuI13Vb97Mj4rYyrt9FxPE108ZFxH9ExIryR57PRsSYmukzIuIHEbEtIh6IiPO7rfv8cvy2iPhhRMyop9yHsoiYX37nWqqOZTjJzNdl5od6mXZCZl4FEBHvj4ivDWpw0hBlsicNHc3Am6sOYm/tQw3ZPOD+zNzWiHi6jISLqF72YR5wd2a2D3Y8w8lQ+fuXNQ0fAZ4DzACWAP9TM8t/ABOB+cBpwMsi4sKBXleZIP0Y+BdgGvBx4McRMb1c9uXAy4CzgEOBCcCna9b9tXJ7BwPnAh+JiCeU6z4a+DrwunLdPwZ+VPM3eAewEHgkcAxwCvDumnX/J7C7XPcFwMURcUK57hOAz5exHQxsBz7bU/kMpqFyfA0n5Q8KXndKgy0zffnyVfELuJ/igmg9MK0c92rgqvLzfCCBlpplrgJeXX7+f8BvKS72NgL3AY8txz8IrAFeUbPsfwOfA34ObAGuBubVTD+2nLYeuAt4YbdlLwauALYBT+5hfw4FflQufw/wmnL8q4CdQAewFfhAt+XGlfE/smbcLGAHcFA5/Czgj+V8vwNO7FaObwduAXZRXAh3lstvBf6he1lSXDR/GVgBbAB+WI6fDvwEWFuO/wlFbeTDyr+Hfd8BzKgZdzLQCowph18JLCrX+3/dyj6BvwEWA0tqxh0FfIDiorit3J9XlX/j39Qsf0LN32418K5y/GnA78tyWwl8BhhbTvt1uY1t5XpfBDweWFaz3uPKfd4I3A6c1+2Y+E/gcorj6TrgyHJaUByXa4DNwK21f9+adfxTeVzsLGP4TDn+scAfgE3l+2P7+R7V/v1bgDPK42Qj8Cfg8TXzX1j+HbZQfGde2219z6E41jYD9wJPr/nbf4jiO7cF+Bkws5eY/hX4z27HR9aUTyvwmJrp7wKuGeh1UXxvbu+2vruBV5Wfvwu8rWbaY8u/xURgcrmdWTXTLwG+Wn5+A3B5zbQmiu/Ak8rhG4AX1Ew/H3iw/DyJ4pg+pmb6V4GPlZ8/AnyjZtqR5fxTeimjp1KcszZRJIVX89B58v3A12rmnc+e54JejwfK70N5fK0qY6znHNHjcQIsLbe9tXydWce5odfzcr3n4Jpy+DZwaRnX7cDCPtb1FODOskw/s5dlehXFd/u35TFxVJ3l/C6K4/l+4IJu55oP187b7fv/ZODp7Hme/BPwAuDGbvv1d8Blve23L18j5eUvLNLQcQPFP8a37uPyp1Nc5B4IfAP4JvAYin+uLwU+ExGTa+a/gOJCZCbFBe3XAcqmpD8v13EQ8GLgs7XNsigu1v4JmAL8podYvknxD/tQimaHH4mIJ2bmlyh+/f99Fk0T31e7UGbuAr4PvKRm9AuBqzNzTUScDPwX8NpyPz9PUYMwrmb+l1DUPEzLzJdQXFQ9u9zex3uI9asUF7QnlPv7H+X4JookcB5wOMWFymd6WH4PmbmCIql6Xs3o84HvZmZbRDyH4kLmrygS2WvYs3YG4LkUf8/aMqcsr48A3yr350u10yNiCvAL4KcUZX8UcGU5uQN4C8Xf+0zgScDry/WeU87z6HK93+q23jEUtTU/oyijNwJfj4jaZp4vpkhGp1NcXP5TOf6pwDkUNToHUPw91/VQbv9YlsUbyhjeUNZGXQ58iuLv/e/A5RFxYPfla/z5709RE3Q58GGKpP6twPciYlY57xqKJGgqxQXof0TEKeU+n0ZxMfy2cl3nUFxMdjm/XOYgYCx9f2+jh8+P7GN67bSBXFfttP6mB8WPL0fXjK933VHH9DkRcQDFcdGemXfXTP8TxfeR8v1PXRMy817K5LDbvhARMymS1ndSHC93USSt9er1eCgdQnEczQMuor5zRG/HSdd3blp5vP++r3NDneflWj2eg2umn1fOM40iKezx3FaW6fcpamJnUvzocVYv2+zNyyjKawrwAPWV80zgMOAVwCXdzjV9ysyfsud58tEU+3hERBzXLa5L93JfpGHHZE8aWt4LvLHmYnRvLMnML2dxL9q3gLnABzNzV2b+jOIC6aia+S/PzF+XCdY/AmdGxFyKf8L3l+tqz8ybge9R/DLa5bLM/G1mdmbmztogynWcBbw9M3dm5h+BL1I0E6vHNyguZLqcX46D4oLh85l5XWZ2ZOZXKGpwzqiZ/1OZ+WBm7uhvQxExG3gG8LrM3JCZbZl5NUBmrsvM72Xm9szcQpG8/MVe7MNLym1EuT9d+/A64KOZuSiLppgfAU6KiHk1y380M9fXsw/dPAtYlZn/Vpb9lsy8rtyfGzPz2vJvej9Folzv/pxBUbvzsczcnZm/pKjFqE3Kf5CZ15f79HXgpHJ8G8VF3rFAlPu9ss7tngsszsyvlnH/D0UNw7P7WKb27/9S4IrMvKI8Vn9O8aPKMwEy8/LMvDcLV1Mks2eX63kV8F+Z+fNy2eWZeWfNdr6cmXeX2/l2zf5291PghRFxYkRMoPiOJ8UPDF3T3xERU6K4Z/KVNdMGcl2/Bw6NiJdExJiIeAVFLVntsq8u7yU7gKIGC2Biefz/FnhPRIwvL8yfV7PsL4C/iKLDjLEUCcvYbut+c0TMiohDgDd1rZviuNrcbT83URwzlNM39TG91jMpai+/Xx6Hn6KohatLP8cDFK0E3leeU3fUeY6o9ziBvs8N9ZyXgbrPwb8pvxcdFD94PbqXmLrK9LuZ2QZ8gr0o09J/Z+btZdxtdZQzwHvKcr6a4gebF+7lNvdQ/p/7FsU5oat58HyK85g0opnsSUNIZt5G8c/nHfuw+OqazzvK9XUfV1uz92DNdrdSNPc5lOJX6tMjYmPXi6IW8JCelu3BocD68uKnywMUv9LW41fAxIg4PSLmU1wc/aCcNg/4+26xzS23WU9s3c0tY93QfUJETIyIz5cdQ2ymaOo4rc57FL9HkTzPpvgFv5PiV/quffhkTfzrKWo6astnb/ah+/7c29OEiDgmIn4SEavK/fkIxa/n9TiUotldZ8247n/T2gvA7ZTHWpkYfoaimeeaiLgkajr+qGO7D3Qb19+xVFt284AXdDteHgfMBoiIZ0TEtVF0OrKR4sK2q0x6LctSj/vbXWb+AngfxTFxf/naQlHrAkXis4Oi2e5lFDU5PfZGuD/rysx1FM1S/47iXPF0iiSta9n/Kue/iqJZ36/K8V3TLwCOoCjfiynu4eta950UNTCfoWgiPBO4o2bZfwJupmhB8DvghxQ/AqymaGbX/XiYWu4XdUyvdSh7nteSXsqyJ/0cDwBra3/cqvMcUddxUurr3FDPeblLPefg7nGN7+U+xJ7KdG/PT3vMX0c5b
},
"metadata": {
"needs_background": "light"
}
}
]
},
{
"cell_type": "markdown",
"source": [
"# PoR Analysis\n",
"\n",
"## Friendly toy scenario\n",
"\n",
"Feel free to play with these variables as much as you want in order to simulate any extreme scenario you find interesting to explore."
],
"metadata": {
"id": "Bw-GGBdX6tMm"
}
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "n-GB9Vsmp4M1",
"outputId": "a4ab7563-bc61-4169-89e8-7f7432f542a2"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
" *** Network Parameters *** \n",
"\n",
"The network has 15400 nodes\n",
"The average node capacity is 1200 GBs\n",
"The network has 1206000 files\n",
"The average file size in the network is 9765 MBs\n",
"The estimated mean time between failures of the network is 12 hours\n",
"Malicious nodes cheat the system by NOT storing 5% of the data they should store\n",
"\n",
" *** Codex Parameters *** \n",
"\n",
"The erasure coding parameter K is 60\n",
"The erasure coding parameter M is 40\n",
"The lazy repair parameter L is 20\n",
"The Proof of Retrievability number of sectors per block is 10\n",
"The PoR agregation cost is 1 ms per PoR block\n",
"Durability in number of nines: 9\n",
"\n",
" *** Dataset and block structures *** \n",
"\n",
"There are 1206000 files of size 10000000 KB (9765.6 MB)\n",
"The total storage used in the network is 11231.8 TB and there is 6815.1 TB of available storage for a total of 18046.9 TB\n",
"There will be 100 EC blocks of size 162.8 MBs per file\n",
"There is a total of 120600000 EC blocks in the network\n",
"There is an average of 7831.2 EC blocks per node (assuming an homogeneous distribution)\n",
"Each EC block has 550537.6 PoR blocks of size 0.3 KB\n",
"\n",
" *** Durability guarantees and verification frequency *** \n",
"\n",
"Guarantee of data durability is 99.999999900\n",
"Probability of detecting catastrophic failures with 40 horizontal verifications is 99.999999931753% (Over 99.999999900)\n",
"Probability of detecting catastrophic failures with 405 vertical verifications 99.999999904926% (Over 99.999999900)\n",
"That is 0.1% of the PoR blocks need to be verified\n",
"The PoR horizontal verification factor is 40.0\n",
"The PoR vertical verification factor is 7.4%\n",
"\n",
" *** PoR Storage and Time Requirements *** \n",
"\n",
"The total storage required for PoR proofs is 14261.6 MB to check a total of 11231.8TB of data each round\n",
"To verify 40 EC blocks and 7.4% PoR blocks per file in the system:\n",
"Each node will spend 2114.4 minutes (35.2 hours) agregating PoR proofs\n",
"It will take the system 480 hours (20.0 days) to accidentally lose M nodes\n",
"Lazy repair should be triggered after 240 hours (10.0 days)\n",
"Assuming PoR verifications are done once per lazy repair round, PoR verification takes 14.7% of the time\n",
"There are 36.5 PoR verification rounds in a year accumulating 508.3 GB of storage\n"
]
}
],
"source": [
"import numpy as np\n",
"import random as random\n",
"import matplotlib.pyplot as plt\n",
"\n",
"plt.rcParams['figure.figsize'] = [15, 8]\n",
"\n",
"\n",
"#@markdown #Network parameters. \n",
"#@markdown These parameters give the characteristic of the network and we (Codex) have no control over them.\n",
"\n",
"#@markdown Number of nodes in the network\n",
"NetworkSize = 15400 #@param {type:\"slider\", min:100, max:20000, step:100}\n",
"#@markdown Average storage size per node (in GB)\n",
"NodeCapacity = 1200 #@param {type:\"slider\", min:100, max:10000, step:100}\n",
"#@markdown Number of files in the system\n",
"NumberOfFiles = 1206000 #@param {type:\"slider\", min:1000, max:10000000, step:1000}\n",
"#@markdown File size (in KB)\n",
"FileSize = 10000000 #@param {type:\"slider\", min:100, max:10000000, step:100}\n",
"#@markdown Overall network MTBF (in hours)\n",
"MTBF = 12 #@param {type:\"slider\", min:1, max:100, step:1}\n",
"#@markdown Percentage of data a malicious node discards or refuses/avoids to store\n",
"CheatRatio = 5 #@param {type:\"slider\", min:5, max:95, step:5}\n",
"#@markdown Predefined Scenario (Set to 1 to overwrite the above parameters with the values from the Storj network)\n",
"Scenario = 0 #@param {type:\"slider\", min:0, max:1, step:1}\n",
"\n",
"#@markdown ----------------------------------------------------\n",
"#@markdown ----------------------------------------------------\n",
"#@markdown ----------------------------------------------------\n",
"\n",
"\n",
"#@markdown #Codex parameters\n",
"#@markdown The Codex team can tune the following parametersto potimize the system\n",
"\n",
"#@markdown Erasure Coding parameter K\n",
"ECK = 60 #@param {type:\"slider\", min:1, max:100, step:1}\n",
"#@markdown Erasure Coding parameter M\n",
"ECM = 40 #@param {type:\"slider\", min:1, max:100, step:1}\n",
"#@markdown Lazy repair parameter L (start repair after L erasures)\n",
"ECL = 20 #@param {type:\"slider\", min:1, max:100, step:1}\n",
"#@markdown Number of PoR sectors (PoR Parameter S)\n",
"PoRS = 10 #@param {type:\"slider\", min:10, max:1000, step:10}\n",
"#@markdown PoR block agregation cost (in miliseconds)\n",
"AgregationCost = 1 #@param {type:\"slider\", min:1, max:100, step:1}\n",
"#@markdown Durability guarantee (three 9s (99.9%) to nine 9s (99.9999999%))\n",
"Durability = 9 #@param {type:\"slider\", min:3, max:9, step:3}\n",
"\n",
"\n",
"# #@markdown Number of EC blocks to verify among K+M nodes\n",
"#HorizontalVerification = 40 #@param {type:\"slider\", min:1, max:100, step:1}\n",
"##@markdown Percentage of PoR blocks to verify inside an EC block\n",
"#VerticalVerification = 0.2 #@param {type:\"slider\", min:0.1, max:1, step:0.1}\n",
"\n",
"\n",
"if Scenario == 1:\n",
" print(\"\\n***This is the Storj scenario***\\n\")\n",
" NetworkSize = 15400\n",
" NodeCapacity = 1200\n",
" NumberOfFiles = 1200000\n",
" FileSize = 10000000\n",
"\n",
"print(\"\\n *** Network Parameters *** \\n\")\n",
"print(\"The network has %d nodes\" % NetworkSize)\n",
"print(\"The average node capacity is %d GBs\" % NodeCapacity)\n",
"print(\"The network has %d files\" % NumberOfFiles)\n",
"print(\"The average file size in the network is %d MBs\" % (FileSize/1024))\n",
"print(\"The estimated mean time between failures of the network is %d hours\" % MTBF)\n",
"print(\"Malicious nodes cheat the system by NOT storing %d%% of the data they should store\" % CheatRatio)\n",
"\n",
"\n",
"print(\"\\n *** Codex Parameters *** \\n\")\n",
"print(\"The erasure coding parameter K is %d\" % ECK)\n",
"print(\"The erasure coding parameter M is %d\" % ECM)\n",
"print(\"The lazy repair parameter L is %d\" % ECL)\n",
"print(\"The Proof of Retrievability number of sectors per block is %d\" % PoRS)\n",
"print(\"The PoR agregation cost is %d ms per PoR block\" % AgregationCost)\n",
"print(\"Durability in number of nines: %d\" % Durability)\n",
"\n",
"ECblockSize = FileSize/ECK\n",
"PoRblockSize = 31*PoRS/1024\n",
"TotalECBlocks = NumberOfFiles*(ECK+ECM)\n",
"ECBlocksPerNode = TotalECBlocks/NetworkSize\n",
"NbPoRinEC = ECblockSize/PoRblockSize\n",
"\n",
"print(\"\\n *** Dataset and block structures *** \\n\")\n",
"print(\"There are %d files of size %d KB (%2.1f MB)\" % (NumberOfFiles, FileSize, FileSize/1024))\n",
"print(\"The total storage used in the network is %2.1f TB and there is %2.1f TB of available storage for a total of %2.1f TB\" % (TotalUsedCapacity, TotalNetworkCapacity-TotalUsedCapacity, TotalNetworkCapacity))\n",
"print(\"There will be %d EC blocks of size %2.1f MBs per file\" % (ECK+ECM, ECblockSize/1024))\n",
"print(\"There is a total of %d EC blocks in the network\" % TotalECBlocks)\n",
"print(\"There is an average of %2.1f EC blocks per node (assuming an homogeneous distribution)\" % ECBlocksPerNode)\n",
"print(\"Each EC block has %2.1f PoR blocks of size %2.1f KB\" % (NbPoRinEC, PoRblockSize))\n",
"\n",
"\n",
"print(\"\\n *** Durability guarantees and verification frequency *** \\n\")\n",
"GuaranteeDurability = {3:99.90, 6: 99.99990, 9: 99.99999990}\n",
"ExpectedGuarantee = GuaranteeDurability[Durability]\n",
"print(\"Guarantee of data durability is %9.9f\" % ExpectedGuarantee)\n",
"\n",
"for NbOfHVerif in range(ECK):\n",
" prob = (1-(((ECK-1)/100)**NbOfHVerif))*100\n",
" if prob > ExpectedGuarantee:\n",
" print(\"Probability of detecting catastrophic failures with %d horizontal verifications is %12.12f%% (Over %9.9f)\" % (NbOfHVerif, prob, ExpectedGuarantee))\n",
" HorizontalVerification = NbOfHVerif \n",
" break\n",
"\n",
"for NbOfVVerif in range(1, int(NbPoRinEC)):\n",
" prob = (1-((1-(CheatRatio/100))**NbOfVVerif))*100\n",
" if prob > ExpectedGuarantee:\n",
" print(\"Probability of detecting catastrophic failures with %d vertical verifications %12.12f%% (Over %9.9f)\" % (NbOfVVerif, prob, ExpectedGuarantee))\n",
" print(\"That is %2.1f%% of the PoR blocks need to be verified\" % (NbOfVVerif*100/NbPoRinEC))\n",
" VerticalVerification = NbOfVVerif*100/NbPoRinEC\n",
" break\n",
"\n",
"print(\"The PoR horizontal verification factor is %2.1f\" % HorizontalVerification)\n",
"print(\"The PoR vertical verification factor is %2.1f%%\" % (VerticalVerification*100))\n",
"\n",
"\n",
"TotalPoRStorage = NumberOfFiles*HorizontalVerification*PoRblockSize/1024\n",
"NodeAgregationTime = NbPoRinEC*VerticalVerification*ECBlocksPerNode*(HorizontalVerification/(ECK+ECM))*AgregationCost/(1000*60)\n",
"PoRPerYear = 365*24/(ECL*MTBF)\n",
"TotalUsedCapacity = int(NumberOfFiles*FileSize)/int(1024*1024*1024)\n",
"TotalNetworkCapacity = NetworkSize*NodeCapacity/1024\n",
"\n",
"\n",
"print(\"\\n *** PoR Storage and Time Requirements *** \\n\")\n",
"print(\"The total storage required for PoR proofs is %2.1f MB to check a total of %2.1fTB of data each round\" % (TotalPoRStorage, TotalUsedCapacity))\n",
"print(\"To verify %d EC blocks and %2.1f%% PoR blocks per file in the system:\" % (HorizontalVerification, VerticalVerification*100))\n",
"print(\"Each node will spend %2.1f minutes (%2.1f hours) agregating PoR proofs\" % (NodeAgregationTime, NodeAgregationTime/60))\n",
"print(\"It will take the system %d hours (%2.1f days) to accidentally lose M nodes\" % (MTBF*ECM, MTBF*ECM/24))\n",
"print(\"Lazy repair should be triggered after %d hours (%2.1f days)\" % (MTBF*ECL, MTBF*ECL/24))\n",
"print(\"Assuming PoR verifications are done once per lazy repair round, PoR verification takes %2.1f%% of the time\" % ((NodeAgregationTime*100)/(MTBF*ECL*60)))\n",
"print(\"There are %2.1f PoR verification rounds in a year accumulating %2.1f GB of storage\" % (PoRPerYear, PoRPerYear*TotalPoRStorage/1024))\n"
]
},
{
"cell_type": "code",
"source": [
""
],
"metadata": {
"id": "nDubyEGMnOPE"
},
"execution_count": null,
"outputs": []
}
]
}
Reference in New Issue Copy Permalink