From 4c23ff01009b05fbc2bea4a79671381a16d12eea Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 17 Aug 2022 23:57:39 -0600 Subject: [PATCH 1/4] first pass at EC structure and placement --- ec-placement.ipynb | 231 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 ec-placement.ipynb diff --git a/ec-placement.ipynb b/ec-placement.ipynb new file mode 100644 index 0000000..e3fcf7a --- /dev/null +++ b/ec-placement.ipynb @@ -0,0 +1,231 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b63223694adb45e2925572977bf8381c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(IntSlider(value=2, description='K', max=256, min=2), IntSlider(value=1, description='M', max=25…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "92669eb1c4484a36a93107690a4c5dc9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from __future__ import print_function\n", + "from ipywidgets import interact, interactive, fixed, interact_manual\n", + "import ipywidgets as widgets\n", + "import itertools as it\n", + "import tabulate as tb\n", + "from functools import reduce\n", + "from operator import concat\n", + "from copy import deepcopy\n", + "\n", + "blocks = []\n", + "\n", + "\n", + "def main(K, M, BC, Dir, Man, Nodes):\n", + " ## K - The number of source blocks to encode\n", + " ## M - The number of parity blocks produced\n", + " ## BC - Initial block count\n", + " ## Dir - Coding direction\n", + " ## Man - Manifest block layout\n", + " ## Nodes - Nodes count\n", + " ##\n", + "\n", + " (rb, eg, n) = calcGeometry(K, M, BC) # Calculate geometry\n", + " blocks = list(range(1, rb + 1)) # Generate source blocks\n", + " ecGroups = makeEcGroups(blocks, K, M, Dir) # Calculate EC groups\n", + " # manifestBlocks = makeManifestBlocks(blocks, K, M, Man)\n", + " manifestBlocks = []\n", + "\n", + " if Man == \"Append\":\n", + " if Dir == \"Horizontal\":\n", + " manifestBlocks = reduce(\n", + " concat,\n", + " [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n", + " else:\n", + " manifestBlocks = reduce(concat, zip(*ecGroups))\n", + " else:\n", + " manifestBlocks = reduce(concat, ecGroups)\n", + "\n", + " print(\"\\n\")\n", + " print(K, M, BC, Dir, Man, Nodes)\n", + " print(\"Rounded Blocks:\", rb)\n", + " print(\"Encoding Groups:\", eg)\n", + " print(\"K + M = \", n)\n", + " print(\"Source Blocks:\", blocks)\n", + " print(\"Read Order:\", manifestBlocks)\n", + " print(\"\\n\",\n", + " tb.tabulate(\n", + " [[\", \".join(map(str, g[:K])), \" ,\".join(map(str, g[-M:]))]\n", + " for g in ecGroups],\n", + " [\"Source\", \"Parity\"]))\n", + "\n", + " print(\"\\n\",\n", + " tb.tabulate(list(map(list, mapNodes(manifestBlocks, Nodes).items())),\n", + " [\"Nodes\", \"Blocks\"]))\n", + "\n", + "\n", + "def calcGeometry(K, M, OB):\n", + " RB = OB + (K - (OB % K))\n", + " EG = (RB // K)\n", + " N = K + M\n", + "\n", + " return (RB, EG, N)\n", + "\n", + "\n", + "def makeManifestBlocks(\n", + " blocks,\n", + " k, m,\n", + " layout):\n", + "\n", + " offset = len(blocks)\n", + " res = []\n", + "\n", + " steps = len(blocks) // (len(blocks) // k)\n", + " if layout == \"Inline\":\n", + " for b in range(0, len(blocks), steps):\n", + " res += blocks[b:b + steps] + list(range(offset + 1, (offset + 1 + m)))\n", + " offset += m\n", + " else:\n", + " res = deepcopy(blocks)\n", + " for b in range(0, len(blocks), steps):\n", + " res += list(range(offset + 1, (offset + 1 + m)))\n", + " offset += m\n", + "\n", + " return res\n", + "\n", + "\n", + "def makeEcGroups(\n", + " blocks,\n", + " k, m,\n", + " direction):\n", + "\n", + " offset = len(blocks)\n", + " res = []\n", + "\n", + " groups = (len(blocks) // k)\n", + " steps = len(blocks) // groups\n", + " if direction == \"Horizontal\":\n", + " for b in range(0, len(blocks), steps):\n", + " res.append(\n", + " blocks[b:b + steps] +\n", + " list(range(offset + 1, (offset + 1 + m))))\n", + " offset += m\n", + " else:\n", + " for s in range(0, groups):\n", + " res.append(\n", + " blocks[s:s+groups+offset:groups] +\n", + " list(range(offset + 1, (offset + 1 + m))))\n", + " offset += m\n", + "\n", + " return res\n", + "\n", + "\n", + "def mapNodes(blocks, hosts, offset=0):\n", + " res = {\"node\" + str((h % hosts)+1): [] for h in range(1, hosts+1)}\n", + " for i, blk in enumerate(blocks):\n", + " res[\"node\" + str((i % hosts)+1)].append(blocks[i])\n", + "\n", + " return res\n", + "\n", + "\n", + "MaxBlocks = 256\n", + "K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n", + "M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n", + "BC = widgets.IntSlider(description=\"Origina blocks count\",\n", + " min=1, max=10000, value=10)\n", + "D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n", + " description=\"EC Coding Direction\")\n", + "Manifest = widgets.Dropdown(\n", + " options=['Inline', 'Append'], description=\"Blocks Manifest Layout\")\n", + "Nodes = widgets.IntSlider(\n", + " description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n", + "\n", + "ui = widgets.VBox([K, M, BC, D, Manifest, Nodes])\n", + "\n", + "\n", + "def updateKRange(*args):\n", + " K.max = MaxBlocks - M.value\n", + "\n", + "\n", + "def updateMRange(*args):\n", + " M.max = MaxBlocks - K.value\n", + "\n", + "\n", + "M.observe(updateKRange, 'value')\n", + "K.observe(updateMRange, 'value')\n", + "\n", + "\n", + "def updateNodesRange(*args):\n", + " Nodes.min = K.value + M.value\n", + "\n", + "\n", + "Nodes.observe(updateNodesRange, 'value')\n", + "\n", + "out = widgets.interactive_output(\n", + " main,\n", + " {\n", + " 'K': K,\n", + " 'M': M,\n", + " 'BC': BC,\n", + " 'Dir': D,\n", + " 'Man': Manifest,\n", + " 'Nodes': Nodes})\n", + "\n", + "display(ui, out)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9.5 64-bit ('3.9.5')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From ac3cf527aa594e097519e156b260bf9b141b780e Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 18 Aug 2022 12:11:41 -0600 Subject: [PATCH 2/4] move to correct dir and cleanup --- analysis/ec-placement.ipynb | 230 +++++++++++++++++++++++++++++++++++ ec-placement.ipynb | 231 ------------------------------------ 2 files changed, 230 insertions(+), 231 deletions(-) create mode 100644 analysis/ec-placement.ipynb delete mode 100644 ec-placement.ipynb diff --git a/analysis/ec-placement.ipynb b/analysis/ec-placement.ipynb new file mode 100644 index 0000000..0630cf9 --- /dev/null +++ b/analysis/ec-placement.ipynb @@ -0,0 +1,230 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a58d0687e0b9452e93be2ef56ebc68ed", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(IntSlider(value=2, description='K', max=256, min=2), IntSlider(value=1, description='M', max=25…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b3d1ec6e56d4467bb2ec8f202567ee58", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from __future__ import print_function\n", + "from ipywidgets import interact, interactive, fixed, interact_manual\n", + "import ipywidgets as widgets\n", + "import itertools as it\n", + "import tabulate as tb\n", + "from functools import reduce\n", + "from operator import concat\n", + "from copy import deepcopy\n", + "import math\n", + "from collections import OrderedDict\n", + "\n", + "\n", + "def roundBlocks(factor, ob):\n", + " return ob + (factor - (ob % factor))\n", + "\n", + "\n", + "def makeEcGroups(\n", + " blocks,\n", + " k, m,\n", + " direction):\n", + "\n", + " offset = len(blocks)\n", + " res = []\n", + "\n", + " groups = (len(blocks) // k)\n", + " steps = len(blocks) // groups\n", + " if direction == \"Horizontal\":\n", + " for b in range(0, len(blocks), steps):\n", + " res.append(\n", + " blocks[b:b + steps] +\n", + " list(range(offset + 1, (offset + 1 + m))))\n", + " offset += m\n", + " else:\n", + " for s in range(0, groups):\n", + " res.append(\n", + " blocks[s:s+groups+offset:groups] +\n", + " list(range(offset + 1, (offset + 1 + m))))\n", + " offset += m\n", + "\n", + " return res\n", + "\n", + "\n", + "def mapNodes(blocks, hosts, offset=0):\n", + " width = getZeros(len(blocks) or BC.value)\n", + " formatBlock = getBlockFormat(width)\n", + " res = OrderedDict()\n", + "\n", + " for i, _ in enumerate(blocks):\n", + " key = \"node\" + str((i % hosts)+1)\n", + " res[key] = (key in res and res[key]) or []\n", + " res[key].append(formatBlock(blocks[i]))\n", + "\n", + " return res\n", + "\n", + "\n", + "def getBlockFormat(width):\n", + " return lambda b: \"{b:0{fmt}}\".format(b=b, fmt=width)\n", + "\n", + "\n", + "def getZeros(width):\n", + " return int(math.log(width, 10) + 1)\n", + "\n", + "\n", + "def makeReadSeq(K, M, ecGroups, direction, layout):\n", + " readSequence = []\n", + " if layout == \"Append\":\n", + " if direction == \"Horizontal\":\n", + " readSequence = reduce(\n", + " concat,\n", + " [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n", + " else:\n", + " readSequence = reduce(concat, zip(*ecGroups))\n", + " else:\n", + " readSequence = reduce(concat, ecGroups)\n", + "\n", + " return readSequence\n", + "\n", + "\n", + "def buildModel(K, M, blocks, Dir, Layout, Nodes):\n", + " return\n", + "\n", + "\n", + "def main(K, M, BC, Dir, Layout, Nodes):\n", + " # K - The number of source blocks to encode\n", + " # M - The number of parity blocks produced\n", + " # BC - Initial block count\n", + " # Dir - Coding direction\n", + " # Layout - Block layout\n", + " # Nodes - Nodes count\n", + " ##\n", + "\n", + " rb = roundBlocks(Nodes, BC) # Calculate geometry\n", + " blocks = list(range(1, rb + 1)) # Generate source blocks\n", + " ecGroups = makeEcGroups(blocks, K, M, Dir) # Calculate EC groups\n", + " readSequence = makeReadSeq(K, M, ecGroups, Dir, Layout)\n", + "\n", + " width = getZeros(len(readSequence) or BC)\n", + " formatBlock = getBlockFormat(width)\n", + "\n", + " print(\"\\n\")\n", + " print(K, M, BC, Dir, Layout, Nodes)\n", + "\n", + " print(\"Rounded Blocks:\", rb)\n", + " print(\"Encoding Groups:\", len(ecGroups))\n", + " print(\"K + M = \", K + M)\n", + " print(\"Source Blocks:\", \", \".join(map(formatBlock, blocks)))\n", + " print(\"Read Layout:\", \", \".join(map(formatBlock, readSequence)))\n", + "\n", + " groupsTable = tb.tabulate(\n", + " [[\", \".join(map(formatBlock, g[:K])), \", \".join(map(formatBlock, g[-M:]))]\n", + " for g in ecGroups], [\"Source\", \"Parity\"])\n", + " print(\"\\n\", groupsTable)\n", + "\n", + " nodesTable = tb.tabulate(\n", + " list(map(list, mapNodes(readSequence, Nodes).items())), [\"Nodes\", \"Blocks\"])\n", + " print(\"\\n\", nodesTable)\n", + "\n", + "\n", + "MaxBlocks = 256\n", + "K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n", + "M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n", + "BC = widgets.IntSlider(description=\"Origina blocks count\",\n", + " min=1, max=10000, value=10)\n", + "D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n", + " description=\"EC Coding Direction\")\n", + "Layout = widgets.Dropdown(\n", + " options=['Inline', 'Append'], description=\"Blocks Manifest Layout\")\n", + "Nodes = widgets.IntSlider(\n", + " description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n", + "\n", + "ui = widgets.VBox([K, M, BC, D, Layout, Nodes])\n", + "\n", + "\n", + "def updateKRange(*args):\n", + " K.max = MaxBlocks - M.value\n", + "\n", + "\n", + "def updateMRange(*args):\n", + " M.max = MaxBlocks - K.value\n", + "\n", + "\n", + "M.observe(updateKRange, 'value')\n", + "K.observe(updateMRange, 'value')\n", + "\n", + "\n", + "def updateNodesRange(*args):\n", + " Nodes.min = K.value + M.value\n", + "\n", + "\n", + "Nodes.observe(updateNodesRange, 'value')\n", + "\n", + "out = widgets.interactive_output(\n", + " main,\n", + " {\n", + " 'K': K,\n", + " 'M': M,\n", + " 'BC': BC,\n", + " 'Dir': D,\n", + " 'Layout': Layout,\n", + " 'Nodes': Nodes})\n", + "\n", + "display(ui, out)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9.5 64-bit ('3.9.5')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/ec-placement.ipynb b/ec-placement.ipynb deleted file mode 100644 index e3fcf7a..0000000 --- a/ec-placement.ipynb +++ /dev/null @@ -1,231 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b63223694adb45e2925572977bf8381c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "VBox(children=(IntSlider(value=2, description='K', max=256, min=2), IntSlider(value=1, description='M', max=25…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "92669eb1c4484a36a93107690a4c5dc9", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from __future__ import print_function\n", - "from ipywidgets import interact, interactive, fixed, interact_manual\n", - "import ipywidgets as widgets\n", - "import itertools as it\n", - "import tabulate as tb\n", - "from functools import reduce\n", - "from operator import concat\n", - "from copy import deepcopy\n", - "\n", - "blocks = []\n", - "\n", - "\n", - "def main(K, M, BC, Dir, Man, Nodes):\n", - " ## K - The number of source blocks to encode\n", - " ## M - The number of parity blocks produced\n", - " ## BC - Initial block count\n", - " ## Dir - Coding direction\n", - " ## Man - Manifest block layout\n", - " ## Nodes - Nodes count\n", - " ##\n", - "\n", - " (rb, eg, n) = calcGeometry(K, M, BC) # Calculate geometry\n", - " blocks = list(range(1, rb + 1)) # Generate source blocks\n", - " ecGroups = makeEcGroups(blocks, K, M, Dir) # Calculate EC groups\n", - " # manifestBlocks = makeManifestBlocks(blocks, K, M, Man)\n", - " manifestBlocks = []\n", - "\n", - " if Man == \"Append\":\n", - " if Dir == \"Horizontal\":\n", - " manifestBlocks = reduce(\n", - " concat,\n", - " [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n", - " else:\n", - " manifestBlocks = reduce(concat, zip(*ecGroups))\n", - " else:\n", - " manifestBlocks = reduce(concat, ecGroups)\n", - "\n", - " print(\"\\n\")\n", - " print(K, M, BC, Dir, Man, Nodes)\n", - " print(\"Rounded Blocks:\", rb)\n", - " print(\"Encoding Groups:\", eg)\n", - " print(\"K + M = \", n)\n", - " print(\"Source Blocks:\", blocks)\n", - " print(\"Read Order:\", manifestBlocks)\n", - " print(\"\\n\",\n", - " tb.tabulate(\n", - " [[\", \".join(map(str, g[:K])), \" ,\".join(map(str, g[-M:]))]\n", - " for g in ecGroups],\n", - " [\"Source\", \"Parity\"]))\n", - "\n", - " print(\"\\n\",\n", - " tb.tabulate(list(map(list, mapNodes(manifestBlocks, Nodes).items())),\n", - " [\"Nodes\", \"Blocks\"]))\n", - "\n", - "\n", - "def calcGeometry(K, M, OB):\n", - " RB = OB + (K - (OB % K))\n", - " EG = (RB // K)\n", - " N = K + M\n", - "\n", - " return (RB, EG, N)\n", - "\n", - "\n", - "def makeManifestBlocks(\n", - " blocks,\n", - " k, m,\n", - " layout):\n", - "\n", - " offset = len(blocks)\n", - " res = []\n", - "\n", - " steps = len(blocks) // (len(blocks) // k)\n", - " if layout == \"Inline\":\n", - " for b in range(0, len(blocks), steps):\n", - " res += blocks[b:b + steps] + list(range(offset + 1, (offset + 1 + m)))\n", - " offset += m\n", - " else:\n", - " res = deepcopy(blocks)\n", - " for b in range(0, len(blocks), steps):\n", - " res += list(range(offset + 1, (offset + 1 + m)))\n", - " offset += m\n", - "\n", - " return res\n", - "\n", - "\n", - "def makeEcGroups(\n", - " blocks,\n", - " k, m,\n", - " direction):\n", - "\n", - " offset = len(blocks)\n", - " res = []\n", - "\n", - " groups = (len(blocks) // k)\n", - " steps = len(blocks) // groups\n", - " if direction == \"Horizontal\":\n", - " for b in range(0, len(blocks), steps):\n", - " res.append(\n", - " blocks[b:b + steps] +\n", - " list(range(offset + 1, (offset + 1 + m))))\n", - " offset += m\n", - " else:\n", - " for s in range(0, groups):\n", - " res.append(\n", - " blocks[s:s+groups+offset:groups] +\n", - " list(range(offset + 1, (offset + 1 + m))))\n", - " offset += m\n", - "\n", - " return res\n", - "\n", - "\n", - "def mapNodes(blocks, hosts, offset=0):\n", - " res = {\"node\" + str((h % hosts)+1): [] for h in range(1, hosts+1)}\n", - " for i, blk in enumerate(blocks):\n", - " res[\"node\" + str((i % hosts)+1)].append(blocks[i])\n", - "\n", - " return res\n", - "\n", - "\n", - "MaxBlocks = 256\n", - "K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n", - "M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n", - "BC = widgets.IntSlider(description=\"Origina blocks count\",\n", - " min=1, max=10000, value=10)\n", - "D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n", - " description=\"EC Coding Direction\")\n", - "Manifest = widgets.Dropdown(\n", - " options=['Inline', 'Append'], description=\"Blocks Manifest Layout\")\n", - "Nodes = widgets.IntSlider(\n", - " description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n", - "\n", - "ui = widgets.VBox([K, M, BC, D, Manifest, Nodes])\n", - "\n", - "\n", - "def updateKRange(*args):\n", - " K.max = MaxBlocks - M.value\n", - "\n", - "\n", - "def updateMRange(*args):\n", - " M.max = MaxBlocks - K.value\n", - "\n", - "\n", - "M.observe(updateKRange, 'value')\n", - "K.observe(updateMRange, 'value')\n", - "\n", - "\n", - "def updateNodesRange(*args):\n", - " Nodes.min = K.value + M.value\n", - "\n", - "\n", - "Nodes.observe(updateNodesRange, 'value')\n", - "\n", - "out = widgets.interactive_output(\n", - " main,\n", - " {\n", - " 'K': K,\n", - " 'M': M,\n", - " 'BC': BC,\n", - " 'Dir': D,\n", - " 'Man': Manifest,\n", - " 'Nodes': Nodes})\n", - "\n", - "display(ui, out)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.9.5 64-bit ('3.9.5')", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 8cbeb52d9eb838c7f518da21fb36adbad9a2962f Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Thu, 18 Aug 2022 18:02:00 -0600 Subject: [PATCH 3/4] reworked to support state --- analysis/ec-placement.ipynb | 171 ++++++++++++++++++++++++++---------- 1 file changed, 125 insertions(+), 46 deletions(-) diff --git a/analysis/ec-placement.ipynb b/analysis/ec-placement.ipynb index 0630cf9..6a26c3d 100644 --- a/analysis/ec-placement.ipynb +++ b/analysis/ec-placement.ipynb @@ -2,13 +2,13 @@ "cells": [ { "cell_type": "code", - "execution_count": 97, + "execution_count": 77, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a58d0687e0b9452e93be2ef56ebc68ed", + "model_id": "dd5a5f6038bc44d1ac4b044894b50f30", "version_major": 2, "version_minor": 0 }, @@ -22,7 +22,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b3d1ec6e56d4467bb2ec8f202567ee58", + "model_id": "1dbdb23c07e4466e82d422c423af3bad", "version_major": 2, "version_minor": 0 }, @@ -42,13 +42,19 @@ "import tabulate as tb\n", "from functools import reduce\n", "from operator import concat\n", - "from copy import deepcopy\n", - "import math\n", "from collections import OrderedDict\n", + "import math\n", + "\n", + "DEFAULT_BLOCKS = 12\n", + "DEFAULT_NODES = 3\n", + "DEFAULT_K = 2\n", + "DEFAULT_M = 1\n", + "DEFAULT_LAYOUT = \"Inline\"\n", + "DEFAULT_DIRECTION = \"Horizontal\"\n", "\n", "\n", "def roundBlocks(factor, ob):\n", - " return ob + (factor - (ob % factor))\n", + " return ob + (factor - (ob % factor)) if (ob % factor) > 0 else 0\n", "\n", "\n", "def makeEcGroups(\n", @@ -113,59 +119,132 @@ " return readSequence\n", "\n", "\n", - "def buildModel(K, M, blocks, Dir, Layout, Nodes):\n", - " return\n", + "class ECPlacement:\n", + " blocks = []\n", + " nodesMap = OrderedDict()\n", + " blockSeq = []\n", + " ecGroups = []\n", + " rb = 0\n", + "\n", + " def __init__(\n", + " self,\n", + " k=DEFAULT_K, m=DEFAULT_M,\n", + " layout=DEFAULT_LAYOUT,\n", + " direction=DEFAULT_LAYOUT,\n", + " nodes=DEFAULT_NODES,\n", + " bc=DEFAULT_BLOCKS):\n", + " self.blocks = list(range(1, bc + 1))\n", + " self.k = k\n", + " self.m = m\n", + " self.layout = layout\n", + " self.direction = direction\n", + " self.nodes = nodes\n", + " self.bc = bc\n", + "\n", + " def reset(self):\n", + " self.k = DEFAULT_K\n", + " self.m = DEFAULT_M\n", + " self.layout = DEFAULT_LAYOUT\n", + " self.direction = DEFAULT_DIRECTION\n", + " self.nodes = DEFAULT_NODES\n", + " self.bc = DEFAULT_BLOCKS\n", + " self.rb = 0\n", + "\n", + " self.blocks = list(range(1, self.bc + 1))\n", + " self.nodesMap = OrderedDict()\n", + " self.blockSeq = []\n", + " self.ecGroups = []\n", + "\n", + " def update(self, k, m, bc, dir, layout, nodes):\n", + " if self.k != k or self.m != m or self.bc != bc:\n", + " # Calculate geometry\n", + " self.rb = roundBlocks(k, bc)\n", + " if (self.rb + bc) > len(self.blocks):\n", + " # Generate source blocks\n", + " self.blocks += list(\n", + " range(len(self.blocks) + 1, self.rb + 1))\n", + "\n", + " self.bc = bc\n", + " self.k = k\n", + " self.m = m\n", + " self.direction = dir\n", + " self.layout = layout\n", + " self.nodes = nodes\n", + "\n", + " if len(self.blocks) > 0:\n", + " self.ecGroups = makeEcGroups(\n", + " self.blocks, self.k, self.m, self.direction) # Calculate EC groups\n", + "\n", + " self.blockSeq = makeReadSeq(\n", + " self.k, self.m, self.ecGroups, self.direction, self.layout)\n", + "\n", + " self.nodesMap = mapNodes(\n", + " self.blockSeq, self.nodes).items()\n", + "\n", + " return self.blockSeq\n", + "\n", + " def render(self):\n", + " width = getZeros(len(self.blockSeq) or BC)\n", + " formatBlock = getBlockFormat(width)\n", + "\n", + " print(\"\\n\")\n", + " print(\"Rounded Blocks:\", self.rb)\n", + " print(\"Encoding Groups:\", len(self.ecGroups))\n", + " print(\"K + M = \", self.k + self.m)\n", + " print(\"Source Blocks:\", \", \".join(map(formatBlock, self.blocks)))\n", + " print(\"Read Layout:\", \", \".join(map(formatBlock, self.blockSeq)))\n", + "\n", + " groupsTable = tb.tabulate(\n", + " [[\", \".join(map(formatBlock, g[:self.k])), \", \".join(map(formatBlock, g[-self.m:]))]\n", + " for g in self.ecGroups], [\"Source\", \"Parity\"])\n", + " print(\"\\n\", groupsTable)\n", + "\n", + " nodesTable = tb.tabulate(\n", + " list(map(list, self.nodesMap)), [\"Nodes\", \"Blocks\"])\n", + " print(\"\\n\", nodesTable)\n", "\n", "\n", - "def main(K, M, BC, Dir, Layout, Nodes):\n", - " # K - The number of source blocks to encode\n", - " # M - The number of parity blocks produced\n", - " # BC - Initial block count\n", + "ecPlacement = ECPlacement()\n", + "\n", + "def main(k, m, bc, dir, layout, nodes):\n", + " # K - The number of source blocks to encode # BC - Initial block count\n", " # Dir - Coding direction\n", " # Layout - Block layout\n", " # Nodes - Nodes count\n", " ##\n", "\n", - " rb = roundBlocks(Nodes, BC) # Calculate geometry\n", - " blocks = list(range(1, rb + 1)) # Generate source blocks\n", - " ecGroups = makeEcGroups(blocks, K, M, Dir) # Calculate EC groups\n", - " readSequence = makeReadSeq(K, M, ecGroups, Dir, Layout)\n", - "\n", - " width = getZeros(len(readSequence) or BC)\n", - " formatBlock = getBlockFormat(width)\n", - "\n", - " print(\"\\n\")\n", - " print(K, M, BC, Dir, Layout, Nodes)\n", - "\n", - " print(\"Rounded Blocks:\", rb)\n", - " print(\"Encoding Groups:\", len(ecGroups))\n", - " print(\"K + M = \", K + M)\n", - " print(\"Source Blocks:\", \", \".join(map(formatBlock, blocks)))\n", - " print(\"Read Layout:\", \", \".join(map(formatBlock, readSequence)))\n", - "\n", - " groupsTable = tb.tabulate(\n", - " [[\", \".join(map(formatBlock, g[:K])), \", \".join(map(formatBlock, g[-M:]))]\n", - " for g in ecGroups], [\"Source\", \"Parity\"])\n", - " print(\"\\n\", groupsTable)\n", - "\n", - " nodesTable = tb.tabulate(\n", - " list(map(list, mapNodes(readSequence, Nodes).items())), [\"Nodes\", \"Blocks\"])\n", - " print(\"\\n\", nodesTable)\n", + " ecPlacement.update(k, m, bc, dir, layout, nodes)\n", + " ecPlacement.render()\n", "\n", "\n", "MaxBlocks = 256\n", "K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n", "M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n", "BC = widgets.IntSlider(description=\"Origina blocks count\",\n", - " min=1, max=10000, value=10)\n", + " min=1, max=10000, value=12)\n", "D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n", " description=\"EC Coding Direction\")\n", "Layout = widgets.Dropdown(\n", - " options=['Inline', 'Append'], description=\"Blocks Manifest Layout\")\n", + " options=['Inline', 'Append'],\n", + " description=\"Blocks Manifest Layout\")\n", + "\n", + "Clear = widgets.Button(description='Clear')\n", "Nodes = widgets.IntSlider(\n", " description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n", "\n", - "ui = widgets.VBox([K, M, BC, D, Layout, Nodes])\n", + "ui = widgets.VBox([K, M, BC, D, Layout, Nodes, Clear])\n", + "\n", + "\n", + "def clearModelClick(button):\n", + " ecPlacement.reset()\n", + " K.value = K.min\n", + " M.value = M.min\n", + " BC.value = 12\n", + " D.value = \"Horizontal\"\n", + " Layout.value = \"Inline\"\n", + " Nodes.value = 3\n", + "\n", + "Clear.on_click(clearModelClick)\n", "\n", "\n", "def updateKRange(*args):\n", @@ -189,12 +268,12 @@ "out = widgets.interactive_output(\n", " main,\n", " {\n", - " 'K': K,\n", - " 'M': M,\n", - " 'BC': BC,\n", - " 'Dir': D,\n", - " 'Layout': Layout,\n", - " 'Nodes': Nodes})\n", + " 'k': K,\n", + " 'm': M,\n", + " 'bc': BC,\n", + " 'dir': D,\n", + " 'layout': Layout,\n", + " 'nodes': Nodes})\n", "\n", "display(ui, out)\n" ] From faacc0bc0a9f57642a779768474d322e9fedbc0f Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 19 Aug 2022 21:08:51 -0600 Subject: [PATCH 4/4] allow appending blocks and nodes --- analysis/ec-placement.ipynb | 146 ++++++++++++++++++++++++------------ 1 file changed, 98 insertions(+), 48 deletions(-) diff --git a/analysis/ec-placement.ipynb b/analysis/ec-placement.ipynb index 6a26c3d..55f7301 100644 --- a/analysis/ec-placement.ipynb +++ b/analysis/ec-placement.ipynb @@ -2,18 +2,18 @@ "cells": [ { "cell_type": "code", - "execution_count": 77, + "execution_count": 56, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "dd5a5f6038bc44d1ac4b044894b50f30", + "model_id": "0d654a31aa8842bea08f69d17e1e9283", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "VBox(children=(IntSlider(value=2, description='K', max=256, min=2), IntSlider(value=1, description='M', max=25…" + "VBox(children=(Label(value='EC Layout and Placement'), VBox(children=(IntSlider(value=2, description='K', max=…" ] }, "metadata": {}, @@ -22,7 +22,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1dbdb23c07e4466e82d422c423af3bad", + "model_id": "577931878dbf419d85441e582a1760f8", "version_major": 2, "version_minor": 0 }, @@ -54,15 +54,16 @@ "\n", "\n", "def roundBlocks(factor, ob):\n", - " return ob + (factor - (ob % factor)) if (ob % factor) > 0 else 0\n", + " return (factor - (ob % factor)) if (ob % factor) > 0 else 0\n", "\n", "\n", "def makeEcGroups(\n", " blocks,\n", " k, m,\n", - " direction):\n", + " direction,\n", + " offset=0):\n", "\n", - " offset = len(blocks)\n", + " offset = len(blocks) if offset == 0 else offset\n", " res = []\n", "\n", " groups = (len(blocks) // k)\n", @@ -140,6 +141,11 @@ " self.direction = direction\n", " self.nodes = nodes\n", " self.bc = bc\n", + " self.blocks = []\n", + " self.nodesMap = OrderedDict()\n", + " self.blockSeq = []\n", + " self.ecGroups = []\n", + " self.rb = 0\n", "\n", " def reset(self):\n", " self.k = DEFAULT_K\n", @@ -155,14 +161,47 @@ " self.blockSeq = []\n", " self.ecGroups = []\n", "\n", - " def update(self, k, m, bc, dir, layout, nodes):\n", - " if self.k != k or self.m != m or self.bc != bc:\n", + " def update(self, k, m, bc, direction, layout, nodes, append):\n", + " print(k, m, bc, direction, layout, nodes, append)\n", + "\n", + " if append:\n", + " if bc > self.bc:\n", + " appendBlocks = bc - self.bc\n", + " appendBlocks = appendBlocks + roundBlocks(k, appendBlocks)\n", + " self.blocks += list(\n", + " range(self.bc + 1, bc + appendBlocks + 1))\n", + "\n", + " if len(self.blocks) > 0:\n", + " blockSeqLen = len(self.blockSeq)\n", + " print(blockSeqLen)\n", + "\n", + " newBlocks = list(\n", + " range(blockSeqLen + 1, blockSeqLen + appendBlocks + 1))\n", + "\n", + " self.ecGroups += makeEcGroups(\n", + " newBlocks, k, m, direction, blockSeqLen + appendBlocks) # Calculate EC groups\n", + "\n", + " self.blockSeq = makeReadSeq(\n", + " k, m, self.ecGroups, direction, layout)\n", + "\n", + " self.nodesMap = mapNodes(\n", + " self.blockSeq, nodes).items()\n", + " else:\n", " # Calculate geometry\n", " self.rb = roundBlocks(k, bc)\n", - " if (self.rb + bc) > len(self.blocks):\n", - " # Generate source blocks\n", - " self.blocks += list(\n", - " range(len(self.blocks) + 1, self.rb + 1))\n", + " # Generate source blocks\n", + " self.blocks = list(\n", + " range(1, bc + 1 + self.rb))\n", + "\n", + " if len(self.blocks) > 0:\n", + " self.ecGroups = makeEcGroups(\n", + " self.blocks, k, m, direction) # Calculate EC groups\n", + "\n", + " self.blockSeq = makeReadSeq(\n", + " k, m, self.ecGroups, direction, layout)\n", + "\n", + " self.nodesMap = mapNodes(\n", + " self.blockSeq, nodes).items()\n", "\n", " self.bc = bc\n", " self.k = k\n", @@ -171,20 +210,10 @@ " self.layout = layout\n", " self.nodes = nodes\n", "\n", - " if len(self.blocks) > 0:\n", - " self.ecGroups = makeEcGroups(\n", - " self.blocks, self.k, self.m, self.direction) # Calculate EC groups\n", - "\n", - " self.blockSeq = makeReadSeq(\n", - " self.k, self.m, self.ecGroups, self.direction, self.layout)\n", - "\n", - " self.nodesMap = mapNodes(\n", - " self.blockSeq, self.nodes).items()\n", - "\n", - " return self.blockSeq\n", + " self.render()\n", "\n", " def render(self):\n", - " width = getZeros(len(self.blockSeq) or BC)\n", + " width = getZeros(len(self.blockSeq) or self.bc)\n", " formatBlock = getBlockFormat(width)\n", "\n", " print(\"\\n\")\n", @@ -206,56 +235,76 @@ "\n", "ecPlacement = ECPlacement()\n", "\n", - "def main(k, m, bc, dir, layout, nodes):\n", - " # K - The number of source blocks to encode # BC - Initial block count\n", + "\n", + "def init(k, m, bc, dir, layout, nodes, append):\n", + " # K - The number of source blocks to encode\n", + " # BC - Initial block count\n", " # Dir - Coding direction\n", " # Layout - Block layout\n", " # Nodes - Nodes count\n", " ##\n", "\n", - " ecPlacement.update(k, m, bc, dir, layout, nodes)\n", - " ecPlacement.render()\n", + " ecPlacement.update(k, m, bc, dir, layout, nodes, append)\n", "\n", "\n", - "MaxBlocks = 256\n", - "K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n", - "M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n", + "MAX_BLOCKS = 256\n", + "K = widgets.IntSlider(description=\"K\", min=2, max=MAX_BLOCKS)\n", + "M = widgets.IntSlider(description=\"M\", min=1, max=MAX_BLOCKS)\n", "BC = widgets.IntSlider(description=\"Origina blocks count\",\n", - " min=1, max=10000, value=12)\n", + " min=1, max=10000, value=DEFAULT_BLOCKS)\n", + "\n", "D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n", " description=\"EC Coding Direction\")\n", - "Layout = widgets.Dropdown(\n", + "\n", + "BlocksLayout = widgets.Dropdown(\n", " options=['Inline', 'Append'],\n", " description=\"Blocks Manifest Layout\")\n", "\n", "Clear = widgets.Button(description='Clear')\n", "Nodes = widgets.IntSlider(\n", - " description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n", + " description=\"Nodes\", min=K.value + M.value, max=MAX_BLOCKS)\n", "\n", - "ui = widgets.VBox([K, M, BC, D, Layout, Nodes, Clear])\n", + "Append = widgets.Checkbox(\n", + " value=False,\n", + " description='Append',\n", + " disabled=False,\n", + " indent=False)\n", + "\n", + "ui = widgets.VBox([\n", + " widgets.Label(value=\"EC Layout and Placement\"),\n", + " widgets.VBox([K, M, BC, D, BlocksLayout, Nodes, Append]),\n", + " Clear])\n", "\n", "\n", "def clearModelClick(button):\n", " ecPlacement.reset()\n", " K.value = K.min\n", " M.value = M.min\n", + " Nodes.value = Nodes.min = K.value + M.value\n", " BC.value = 12\n", " D.value = \"Horizontal\"\n", - " Layout.value = \"Inline\"\n", - " Nodes.value = 3\n", + " BlocksLayout.value = \"Inline\"\n", + " appendBlocks.value = 0\n", + " appendNodes = 0\n", + " Append.value = False\n", + "\n", "\n", "Clear.on_click(clearModelClick)\n", "\n", "\n", "def updateKRange(*args):\n", - " K.max = MaxBlocks - M.value\n", - "\n", - "\n", - "def updateMRange(*args):\n", - " M.max = MaxBlocks - K.value\n", + " K.max = MAX_BLOCKS - M.value\n", + " Nodes.value = K.value + M.value\n", "\n", "\n", "M.observe(updateKRange, 'value')\n", + "\n", + "\n", + "def updateMRange(*args):\n", + " M.max = MAX_BLOCKS - K.value\n", + " Nodes.value = K.value + M.value\n", + "\n", + "\n", "K.observe(updateMRange, 'value')\n", "\n", "\n", @@ -265,17 +314,18 @@ "\n", "Nodes.observe(updateNodesRange, 'value')\n", "\n", - "out = widgets.interactive_output(\n", - " main,\n", + "init = widgets.interactive_output(\n", + " init,\n", " {\n", " 'k': K,\n", " 'm': M,\n", " 'bc': BC,\n", " 'dir': D,\n", - " 'layout': Layout,\n", - " 'nodes': Nodes})\n", + " 'layout': BlocksLayout,\n", + " 'nodes': Nodes,\n", + " 'append': Append})\n", "\n", - "display(ui, out)\n" + "display(ui, init)\n" ] } ],