From 4c23ff01009b05fbc2bea4a79671381a16d12eea Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 17 Aug 2022 23:57:39 -0600 Subject: [PATCH] first pass at EC structure and placement --- ec-placement.ipynb | 231 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 ec-placement.ipynb diff --git a/ec-placement.ipynb b/ec-placement.ipynb new file mode 100644 index 0000000..e3fcf7a --- /dev/null +++ b/ec-placement.ipynb @@ -0,0 +1,231 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b63223694adb45e2925572977bf8381c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(IntSlider(value=2, description='K', max=256, min=2), IntSlider(value=1, description='M', max=25…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "92669eb1c4484a36a93107690a4c5dc9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from __future__ import print_function\n", + "from ipywidgets import interact, interactive, fixed, interact_manual\n", + "import ipywidgets as widgets\n", + "import itertools as it\n", + "import tabulate as tb\n", + "from functools import reduce\n", + "from operator import concat\n", + "from copy import deepcopy\n", + "\n", + "blocks = []\n", + "\n", + "\n", + "def main(K, M, BC, Dir, Man, Nodes):\n", + " ## K - The number of source blocks to encode\n", + " ## M - The number of parity blocks produced\n", + " ## BC - Initial block count\n", + " ## Dir - Coding direction\n", + " ## Man - Manifest block layout\n", + " ## Nodes - Nodes count\n", + " ##\n", + "\n", + " (rb, eg, n) = calcGeometry(K, M, BC) # Calculate geometry\n", + " blocks = list(range(1, rb + 1)) # Generate source blocks\n", + " ecGroups = makeEcGroups(blocks, K, M, Dir) # Calculate EC groups\n", + " # manifestBlocks = makeManifestBlocks(blocks, K, M, Man)\n", + " manifestBlocks = []\n", + "\n", + " if Man == \"Append\":\n", + " if Dir == \"Horizontal\":\n", + " manifestBlocks = reduce(\n", + " concat,\n", + " [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n", + " else:\n", + " manifestBlocks = reduce(concat, zip(*ecGroups))\n", + " else:\n", + " manifestBlocks = reduce(concat, ecGroups)\n", + "\n", + " print(\"\\n\")\n", + " print(K, M, BC, Dir, Man, Nodes)\n", + " print(\"Rounded Blocks:\", rb)\n", + " print(\"Encoding Groups:\", eg)\n", + " print(\"K + M = \", n)\n", + " print(\"Source Blocks:\", blocks)\n", + " print(\"Read Order:\", manifestBlocks)\n", + " print(\"\\n\",\n", + " tb.tabulate(\n", + " [[\", \".join(map(str, g[:K])), \" ,\".join(map(str, g[-M:]))]\n", + " for g in ecGroups],\n", + " [\"Source\", \"Parity\"]))\n", + "\n", + " print(\"\\n\",\n", + " tb.tabulate(list(map(list, mapNodes(manifestBlocks, Nodes).items())),\n", + " [\"Nodes\", \"Blocks\"]))\n", + "\n", + "\n", + "def calcGeometry(K, M, OB):\n", + " RB = OB + (K - (OB % K))\n", + " EG = (RB // K)\n", + " N = K + M\n", + "\n", + " return (RB, EG, N)\n", + "\n", + "\n", + "def makeManifestBlocks(\n", + " blocks,\n", + " k, m,\n", + " layout):\n", + "\n", + " offset = len(blocks)\n", + " res = []\n", + "\n", + " steps = len(blocks) // (len(blocks) // k)\n", + " if layout == \"Inline\":\n", + " for b in range(0, len(blocks), steps):\n", + " res += blocks[b:b + steps] + list(range(offset + 1, (offset + 1 + m)))\n", + " offset += m\n", + " else:\n", + " res = deepcopy(blocks)\n", + " for b in range(0, len(blocks), steps):\n", + " res += list(range(offset + 1, (offset + 1 + m)))\n", + " offset += m\n", + "\n", + " return res\n", + "\n", + "\n", + "def makeEcGroups(\n", + " blocks,\n", + " k, m,\n", + " direction):\n", + "\n", + " offset = len(blocks)\n", + " res = []\n", + "\n", + " groups = (len(blocks) // k)\n", + " steps = len(blocks) // groups\n", + " if direction == \"Horizontal\":\n", + " for b in range(0, len(blocks), steps):\n", + " res.append(\n", + " blocks[b:b + steps] +\n", + " list(range(offset + 1, (offset + 1 + m))))\n", + " offset += m\n", + " else:\n", + " for s in range(0, groups):\n", + " res.append(\n", + " blocks[s:s+groups+offset:groups] +\n", + " list(range(offset + 1, (offset + 1 + m))))\n", + " offset += m\n", + "\n", + " return res\n", + "\n", + "\n", + "def mapNodes(blocks, hosts, offset=0):\n", + " res = {\"node\" + str((h % hosts)+1): [] for h in range(1, hosts+1)}\n", + " for i, blk in enumerate(blocks):\n", + " res[\"node\" + str((i % hosts)+1)].append(blocks[i])\n", + "\n", + " return res\n", + "\n", + "\n", + "MaxBlocks = 256\n", + "K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n", + "M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n", + "BC = widgets.IntSlider(description=\"Origina blocks count\",\n", + " min=1, max=10000, value=10)\n", + "D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n", + " description=\"EC Coding Direction\")\n", + "Manifest = widgets.Dropdown(\n", + " options=['Inline', 'Append'], description=\"Blocks Manifest Layout\")\n", + "Nodes = widgets.IntSlider(\n", + " description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n", + "\n", + "ui = widgets.VBox([K, M, BC, D, Manifest, Nodes])\n", + "\n", + "\n", + "def updateKRange(*args):\n", + " K.max = MaxBlocks - M.value\n", + "\n", + "\n", + "def updateMRange(*args):\n", + " M.max = MaxBlocks - K.value\n", + "\n", + "\n", + "M.observe(updateKRange, 'value')\n", + "K.observe(updateMRange, 'value')\n", + "\n", + "\n", + "def updateNodesRange(*args):\n", + " Nodes.min = K.value + M.value\n", + "\n", + "\n", + "Nodes.observe(updateNodesRange, 'value')\n", + "\n", + "out = widgets.interactive_output(\n", + " main,\n", + " {\n", + " 'K': K,\n", + " 'M': M,\n", + " 'BC': BC,\n", + " 'Dir': D,\n", + " 'Man': Manifest,\n", + " 'Nodes': Nodes})\n", + "\n", + "display(ui, out)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9.5 64-bit ('3.9.5')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}