From ac3cf527aa594e097519e156b260bf9b141b780e Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 18 Aug 2022 12:11:41 -0600 Subject: [PATCH] move to correct dir and cleanup --- analysis/ec-placement.ipynb | 230 +++++++++++++++++++++++++++++++++++ ec-placement.ipynb | 231 ------------------------------------ 2 files changed, 230 insertions(+), 231 deletions(-) create mode 100644 analysis/ec-placement.ipynb delete mode 100644 ec-placement.ipynb diff --git a/analysis/ec-placement.ipynb b/analysis/ec-placement.ipynb new file mode 100644 index 0000000..0630cf9 --- /dev/null +++ b/analysis/ec-placement.ipynb @@ -0,0 +1,230 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a58d0687e0b9452e93be2ef56ebc68ed", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(IntSlider(value=2, description='K', max=256, min=2), IntSlider(value=1, description='M', max=25…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b3d1ec6e56d4467bb2ec8f202567ee58", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from __future__ import print_function\n", + "from ipywidgets import interact, interactive, fixed, interact_manual\n", + "import ipywidgets as widgets\n", + "import itertools as it\n", + "import tabulate as tb\n", + "from functools import reduce\n", + "from operator import concat\n", + "from copy import deepcopy\n", + "import math\n", + "from collections import OrderedDict\n", + "\n", + "\n", + "def roundBlocks(factor, ob):\n", + " return ob + (factor - (ob % factor))\n", + "\n", + "\n", + "def makeEcGroups(\n", + " blocks,\n", + " k, m,\n", + " direction):\n", + "\n", + " offset = len(blocks)\n", + " res = []\n", + "\n", + " groups = (len(blocks) // k)\n", + " steps = len(blocks) // groups\n", + " if direction == \"Horizontal\":\n", + " for b in range(0, len(blocks), steps):\n", + " res.append(\n", + " blocks[b:b + steps] +\n", + " list(range(offset + 1, (offset + 1 + m))))\n", + " offset += m\n", + " else:\n", + " for s in range(0, groups):\n", + " res.append(\n", + " blocks[s:s+groups+offset:groups] +\n", + " list(range(offset + 1, (offset + 1 + m))))\n", + " offset += m\n", + "\n", + " return res\n", + "\n", + "\n", + "def mapNodes(blocks, hosts, offset=0):\n", + " width = getZeros(len(blocks) or BC.value)\n", + " formatBlock = getBlockFormat(width)\n", + " res = OrderedDict()\n", + "\n", + " for i, _ in enumerate(blocks):\n", + " key = \"node\" + str((i % hosts)+1)\n", + " res[key] = (key in res and res[key]) or []\n", + " res[key].append(formatBlock(blocks[i]))\n", + "\n", + " return res\n", + "\n", + "\n", + "def getBlockFormat(width):\n", + " return lambda b: \"{b:0{fmt}}\".format(b=b, fmt=width)\n", + "\n", + "\n", + "def getZeros(width):\n", + " return int(math.log(width, 10) + 1)\n", + "\n", + "\n", + "def makeReadSeq(K, M, ecGroups, direction, layout):\n", + " readSequence = []\n", + " if layout == \"Append\":\n", + " if direction == \"Horizontal\":\n", + " readSequence = reduce(\n", + " concat,\n", + " [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n", + " else:\n", + " readSequence = reduce(concat, zip(*ecGroups))\n", + " else:\n", + " readSequence = reduce(concat, ecGroups)\n", + "\n", + " return readSequence\n", + "\n", + "\n", + "def buildModel(K, M, blocks, Dir, Layout, Nodes):\n", + " return\n", + "\n", + "\n", + "def main(K, M, BC, Dir, Layout, Nodes):\n", + " # K - The number of source blocks to encode\n", + " # M - The number of parity blocks produced\n", + " # BC - Initial block count\n", + " # Dir - Coding direction\n", + " # Layout - Block layout\n", + " # Nodes - Nodes count\n", + " ##\n", + "\n", + " rb = roundBlocks(Nodes, BC) # Calculate geometry\n", + " blocks = list(range(1, rb + 1)) # Generate source blocks\n", + " ecGroups = makeEcGroups(blocks, K, M, Dir) # Calculate EC groups\n", + " readSequence = makeReadSeq(K, M, ecGroups, Dir, Layout)\n", + "\n", + " width = getZeros(len(readSequence) or BC)\n", + " formatBlock = getBlockFormat(width)\n", + "\n", + " print(\"\\n\")\n", + " print(K, M, BC, Dir, Layout, Nodes)\n", + "\n", + " print(\"Rounded Blocks:\", rb)\n", + " print(\"Encoding Groups:\", len(ecGroups))\n", + " print(\"K + M = \", K + M)\n", + " print(\"Source Blocks:\", \", \".join(map(formatBlock, blocks)))\n", + " print(\"Read Layout:\", \", \".join(map(formatBlock, readSequence)))\n", + "\n", + " groupsTable = tb.tabulate(\n", + " [[\", \".join(map(formatBlock, g[:K])), \", \".join(map(formatBlock, g[-M:]))]\n", + " for g in ecGroups], [\"Source\", \"Parity\"])\n", + " print(\"\\n\", groupsTable)\n", + "\n", + " nodesTable = tb.tabulate(\n", + " list(map(list, mapNodes(readSequence, Nodes).items())), [\"Nodes\", \"Blocks\"])\n", + " print(\"\\n\", nodesTable)\n", + "\n", + "\n", + "MaxBlocks = 256\n", + "K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n", + "M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n", + "BC = widgets.IntSlider(description=\"Origina blocks count\",\n", + " min=1, max=10000, value=10)\n", + "D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n", + " description=\"EC Coding Direction\")\n", + "Layout = widgets.Dropdown(\n", + " options=['Inline', 'Append'], description=\"Blocks Manifest Layout\")\n", + "Nodes = widgets.IntSlider(\n", + " description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n", + "\n", + "ui = widgets.VBox([K, M, BC, D, Layout, Nodes])\n", + "\n", + "\n", + "def updateKRange(*args):\n", + " K.max = MaxBlocks - M.value\n", + "\n", + "\n", + "def updateMRange(*args):\n", + " M.max = MaxBlocks - K.value\n", + "\n", + "\n", + "M.observe(updateKRange, 'value')\n", + "K.observe(updateMRange, 'value')\n", + "\n", + "\n", + "def updateNodesRange(*args):\n", + " Nodes.min = K.value + M.value\n", + "\n", + "\n", + "Nodes.observe(updateNodesRange, 'value')\n", + "\n", + "out = widgets.interactive_output(\n", + " main,\n", + " {\n", + " 'K': K,\n", + " 'M': M,\n", + " 'BC': BC,\n", + " 'Dir': D,\n", + " 'Layout': Layout,\n", + " 'Nodes': Nodes})\n", + "\n", + "display(ui, out)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9.5 64-bit ('3.9.5')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/ec-placement.ipynb b/ec-placement.ipynb deleted file mode 100644 index e3fcf7a..0000000 --- a/ec-placement.ipynb +++ /dev/null @@ -1,231 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b63223694adb45e2925572977bf8381c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "VBox(children=(IntSlider(value=2, description='K', max=256, min=2), IntSlider(value=1, description='M', max=25…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "92669eb1c4484a36a93107690a4c5dc9", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from __future__ import print_function\n", - "from ipywidgets import interact, interactive, fixed, interact_manual\n", - "import ipywidgets as widgets\n", - "import itertools as it\n", - "import tabulate as tb\n", - "from functools import reduce\n", - "from operator import concat\n", - "from copy import deepcopy\n", - "\n", - "blocks = []\n", - "\n", - "\n", - "def main(K, M, BC, Dir, Man, Nodes):\n", - " ## K - The number of source blocks to encode\n", - " ## M - The number of parity blocks produced\n", - " ## BC - Initial block count\n", - " ## Dir - Coding direction\n", - " ## Man - Manifest block layout\n", - " ## Nodes - Nodes count\n", - " ##\n", - "\n", - " (rb, eg, n) = calcGeometry(K, M, BC) # Calculate geometry\n", - " blocks = list(range(1, rb + 1)) # Generate source blocks\n", - " ecGroups = makeEcGroups(blocks, K, M, Dir) # Calculate EC groups\n", - " # manifestBlocks = makeManifestBlocks(blocks, K, M, Man)\n", - " manifestBlocks = []\n", - "\n", - " if Man == \"Append\":\n", - " if Dir == \"Horizontal\":\n", - " manifestBlocks = reduce(\n", - " concat,\n", - " [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n", - " else:\n", - " manifestBlocks = reduce(concat, zip(*ecGroups))\n", - " else:\n", - " manifestBlocks = reduce(concat, ecGroups)\n", - "\n", - " print(\"\\n\")\n", - " print(K, M, BC, Dir, Man, Nodes)\n", - " print(\"Rounded Blocks:\", rb)\n", - " print(\"Encoding Groups:\", eg)\n", - " print(\"K + M = \", n)\n", - " print(\"Source Blocks:\", blocks)\n", - " print(\"Read Order:\", manifestBlocks)\n", - " print(\"\\n\",\n", - " tb.tabulate(\n", - " [[\", \".join(map(str, g[:K])), \" ,\".join(map(str, g[-M:]))]\n", - " for g in ecGroups],\n", - " [\"Source\", \"Parity\"]))\n", - "\n", - " print(\"\\n\",\n", - " tb.tabulate(list(map(list, mapNodes(manifestBlocks, Nodes).items())),\n", - " [\"Nodes\", \"Blocks\"]))\n", - "\n", - "\n", - "def calcGeometry(K, M, OB):\n", - " RB = OB + (K - (OB % K))\n", - " EG = (RB // K)\n", - " N = K + M\n", - "\n", - " return (RB, EG, N)\n", - "\n", - "\n", - "def makeManifestBlocks(\n", - " blocks,\n", - " k, m,\n", - " layout):\n", - "\n", - " offset = len(blocks)\n", - " res = []\n", - "\n", - " steps = len(blocks) // (len(blocks) // k)\n", - " if layout == \"Inline\":\n", - " for b in range(0, len(blocks), steps):\n", - " res += blocks[b:b + steps] + list(range(offset + 1, (offset + 1 + m)))\n", - " offset += m\n", - " else:\n", - " res = deepcopy(blocks)\n", - " for b in range(0, len(blocks), steps):\n", - " res += list(range(offset + 1, (offset + 1 + m)))\n", - " offset += m\n", - "\n", - " return res\n", - "\n", - "\n", - "def makeEcGroups(\n", - " blocks,\n", - " k, m,\n", - " direction):\n", - "\n", - " offset = len(blocks)\n", - " res = []\n", - "\n", - " groups = (len(blocks) // k)\n", - " steps = len(blocks) // groups\n", - " if direction == \"Horizontal\":\n", - " for b in range(0, len(blocks), steps):\n", - " res.append(\n", - " blocks[b:b + steps] +\n", - " list(range(offset + 1, (offset + 1 + m))))\n", - " offset += m\n", - " else:\n", - " for s in range(0, groups):\n", - " res.append(\n", - " blocks[s:s+groups+offset:groups] +\n", - " list(range(offset + 1, (offset + 1 + m))))\n", - " offset += m\n", - "\n", - " return res\n", - "\n", - "\n", - "def mapNodes(blocks, hosts, offset=0):\n", - " res = {\"node\" + str((h % hosts)+1): [] for h in range(1, hosts+1)}\n", - " for i, blk in enumerate(blocks):\n", - " res[\"node\" + str((i % hosts)+1)].append(blocks[i])\n", - "\n", - " return res\n", - "\n", - "\n", - "MaxBlocks = 256\n", - "K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n", - "M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n", - "BC = widgets.IntSlider(description=\"Origina blocks count\",\n", - " min=1, max=10000, value=10)\n", - "D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n", - " description=\"EC Coding Direction\")\n", - "Manifest = widgets.Dropdown(\n", - " options=['Inline', 'Append'], description=\"Blocks Manifest Layout\")\n", - "Nodes = widgets.IntSlider(\n", - " description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n", - "\n", - "ui = widgets.VBox([K, M, BC, D, Manifest, Nodes])\n", - "\n", - "\n", - "def updateKRange(*args):\n", - " K.max = MaxBlocks - M.value\n", - "\n", - "\n", - "def updateMRange(*args):\n", - " M.max = MaxBlocks - K.value\n", - "\n", - "\n", - "M.observe(updateKRange, 'value')\n", - "K.observe(updateMRange, 'value')\n", - "\n", - "\n", - "def updateNodesRange(*args):\n", - " Nodes.min = K.value + M.value\n", - "\n", - "\n", - "Nodes.observe(updateNodesRange, 'value')\n", - "\n", - "out = widgets.interactive_output(\n", - " main,\n", - " {\n", - " 'K': K,\n", - " 'M': M,\n", - " 'BC': BC,\n", - " 'Dir': D,\n", - " 'Man': Manifest,\n", - " 'Nodes': Nodes})\n", - "\n", - "display(ui, out)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.9.5 64-bit ('3.9.5')", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}