mirror of
https://github.com/logos-storage/logos-storage-research.git
synced 2026-01-05 15:03:08 +00:00
Merge faacc0bc0a9f57642a779768474d322e9fedbc0f into 32cf143b5605363208b1cdbf8f2eb137eb519ad1
This commit is contained in:
Dmitriy Ryajov
GitHub
commit
3870a13d76
359
analysis/ec-placement.ipynb
Normal file
359
analysis/ec-placement.ipynb
Normal file
@ -0,0 +1,359 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 56,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "0d654a31aa8842bea08f69d17e1e9283",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"VBox(children=(Label(value='EC Layout and Placement'), VBox(children=(IntSlider(value=2, description='K', max=…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "577931878dbf419d85441e582a1760f8",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Output()"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from __future__ import print_function\n",
|
||||
"from ipywidgets import interact, interactive, fixed, interact_manual\n",
|
||||
"import ipywidgets as widgets\n",
|
||||
"import itertools as it\n",
|
||||
"import tabulate as tb\n",
|
||||
"from functools import reduce\n",
|
||||
"from operator import concat\n",
|
||||
"from collections import OrderedDict\n",
|
||||
"import math\n",
|
||||
"\n",
|
||||
"DEFAULT_BLOCKS = 12\n",
|
||||
"DEFAULT_NODES = 3\n",
|
||||
"DEFAULT_K = 2\n",
|
||||
"DEFAULT_M = 1\n",
|
||||
"DEFAULT_LAYOUT = \"Inline\"\n",
|
||||
"DEFAULT_DIRECTION = \"Horizontal\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def roundBlocks(factor, ob):\n",
|
||||
" return (factor - (ob % factor)) if (ob % factor) > 0 else 0\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def makeEcGroups(\n",
|
||||
" blocks,\n",
|
||||
" k, m,\n",
|
||||
" direction,\n",
|
||||
" offset=0):\n",
|
||||
"\n",
|
||||
" offset = len(blocks) if offset == 0 else offset\n",
|
||||
" res = []\n",
|
||||
"\n",
|
||||
" groups = (len(blocks) // k)\n",
|
||||
" steps = len(blocks) // groups\n",
|
||||
" if direction == \"Horizontal\":\n",
|
||||
" for b in range(0, len(blocks), steps):\n",
|
||||
" res.append(\n",
|
||||
" blocks[b:b + steps] +\n",
|
||||
" list(range(offset + 1, (offset + 1 + m))))\n",
|
||||
" offset += m\n",
|
||||
" else:\n",
|
||||
" for s in range(0, groups):\n",
|
||||
" res.append(\n",
|
||||
" blocks[s:s+groups+offset:groups] +\n",
|
||||
" list(range(offset + 1, (offset + 1 + m))))\n",
|
||||
" offset += m\n",
|
||||
"\n",
|
||||
" return res\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def mapNodes(blocks, hosts, offset=0):\n",
|
||||
" width = getZeros(len(blocks) or BC.value)\n",
|
||||
" formatBlock = getBlockFormat(width)\n",
|
||||
" res = OrderedDict()\n",
|
||||
"\n",
|
||||
" for i, _ in enumerate(blocks):\n",
|
||||
" key = \"node\" + str((i % hosts)+1)\n",
|
||||
" res[key] = (key in res and res[key]) or []\n",
|
||||
" res[key].append(formatBlock(blocks[i]))\n",
|
||||
"\n",
|
||||
" return res\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def getBlockFormat(width):\n",
|
||||
" return lambda b: \"{b:0{fmt}}\".format(b=b, fmt=width)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def getZeros(width):\n",
|
||||
" return int(math.log(width, 10) + 1)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def makeReadSeq(K, M, ecGroups, direction, layout):\n",
|
||||
" readSequence = []\n",
|
||||
" if layout == \"Append\":\n",
|
||||
" if direction == \"Horizontal\":\n",
|
||||
" readSequence = reduce(\n",
|
||||
" concat,\n",
|
||||
" [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n",
|
||||
" else:\n",
|
||||
" readSequence = reduce(concat, zip(*ecGroups))\n",
|
||||
" else:\n",
|
||||
" readSequence = reduce(concat, ecGroups)\n",
|
||||
"\n",
|
||||
" return readSequence\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class ECPlacement:\n",
|
||||
" blocks = []\n",
|
||||
" nodesMap = OrderedDict()\n",
|
||||
" blockSeq = []\n",
|
||||
" ecGroups = []\n",
|
||||
" rb = 0\n",
|
||||
"\n",
|
||||
" def __init__(\n",
|
||||
" self,\n",
|
||||
" k=DEFAULT_K, m=DEFAULT_M,\n",
|
||||
" layout=DEFAULT_LAYOUT,\n",
|
||||
" direction=DEFAULT_LAYOUT,\n",
|
||||
" nodes=DEFAULT_NODES,\n",
|
||||
" bc=DEFAULT_BLOCKS):\n",
|
||||
" self.blocks = list(range(1, bc + 1))\n",
|
||||
" self.k = k\n",
|
||||
" self.m = m\n",
|
||||
" self.layout = layout\n",
|
||||
" self.direction = direction\n",
|
||||
" self.nodes = nodes\n",
|
||||
" self.bc = bc\n",
|
||||
" self.blocks = []\n",
|
||||
" self.nodesMap = OrderedDict()\n",
|
||||
" self.blockSeq = []\n",
|
||||
" self.ecGroups = []\n",
|
||||
" self.rb = 0\n",
|
||||
"\n",
|
||||
" def reset(self):\n",
|
||||
" self.k = DEFAULT_K\n",
|
||||
" self.m = DEFAULT_M\n",
|
||||
" self.layout = DEFAULT_LAYOUT\n",
|
||||
" self.direction = DEFAULT_DIRECTION\n",
|
||||
" self.nodes = DEFAULT_NODES\n",
|
||||
" self.bc = DEFAULT_BLOCKS\n",
|
||||
" self.rb = 0\n",
|
||||
"\n",
|
||||
" self.blocks = list(range(1, self.bc + 1))\n",
|
||||
" self.nodesMap = OrderedDict()\n",
|
||||
" self.blockSeq = []\n",
|
||||
" self.ecGroups = []\n",
|
||||
"\n",
|
||||
" def update(self, k, m, bc, direction, layout, nodes, append):\n",
|
||||
" print(k, m, bc, direction, layout, nodes, append)\n",
|
||||
"\n",
|
||||
" if append:\n",
|
||||
" if bc > self.bc:\n",
|
||||
" appendBlocks = bc - self.bc\n",
|
||||
" appendBlocks = appendBlocks + roundBlocks(k, appendBlocks)\n",
|
||||
" self.blocks += list(\n",
|
||||
" range(self.bc + 1, bc + appendBlocks + 1))\n",
|
||||
"\n",
|
||||
" if len(self.blocks) > 0:\n",
|
||||
" blockSeqLen = len(self.blockSeq)\n",
|
||||
" print(blockSeqLen)\n",
|
||||
"\n",
|
||||
" newBlocks = list(\n",
|
||||
" range(blockSeqLen + 1, blockSeqLen + appendBlocks + 1))\n",
|
||||
"\n",
|
||||
" self.ecGroups += makeEcGroups(\n",
|
||||
" newBlocks, k, m, direction, blockSeqLen + appendBlocks) # Calculate EC groups\n",
|
||||
"\n",
|
||||
" self.blockSeq = makeReadSeq(\n",
|
||||
" k, m, self.ecGroups, direction, layout)\n",
|
||||
"\n",
|
||||
" self.nodesMap = mapNodes(\n",
|
||||
" self.blockSeq, nodes).items()\n",
|
||||
" else:\n",
|
||||
" # Calculate geometry\n",
|
||||
" self.rb = roundBlocks(k, bc)\n",
|
||||
" # Generate source blocks\n",
|
||||
" self.blocks = list(\n",
|
||||
" range(1, bc + 1 + self.rb))\n",
|
||||
"\n",
|
||||
" if len(self.blocks) > 0:\n",
|
||||
" self.ecGroups = makeEcGroups(\n",
|
||||
" self.blocks, k, m, direction) # Calculate EC groups\n",
|
||||
"\n",
|
||||
" self.blockSeq = makeReadSeq(\n",
|
||||
" k, m, self.ecGroups, direction, layout)\n",
|
||||
"\n",
|
||||
" self.nodesMap = mapNodes(\n",
|
||||
" self.blockSeq, nodes).items()\n",
|
||||
"\n",
|
||||
" self.bc = bc\n",
|
||||
" self.k = k\n",
|
||||
" self.m = m\n",
|
||||
" self.direction = dir\n",
|
||||
" self.layout = layout\n",
|
||||
" self.nodes = nodes\n",
|
||||
"\n",
|
||||
" self.render()\n",
|
||||
"\n",
|
||||
" def render(self):\n",
|
||||
" width = getZeros(len(self.blockSeq) or self.bc)\n",
|
||||
" formatBlock = getBlockFormat(width)\n",
|
||||
"\n",
|
||||
" print(\"\\n\")\n",
|
||||
" print(\"Rounded Blocks:\", self.rb)\n",
|
||||
" print(\"Encoding Groups:\", len(self.ecGroups))\n",
|
||||
" print(\"K + M = \", self.k + self.m)\n",
|
||||
" print(\"Source Blocks:\", \", \".join(map(formatBlock, self.blocks)))\n",
|
||||
" print(\"Read Layout:\", \", \".join(map(formatBlock, self.blockSeq)))\n",
|
||||
"\n",
|
||||
" groupsTable = tb.tabulate(\n",
|
||||
" [[\", \".join(map(formatBlock, g[:self.k])), \", \".join(map(formatBlock, g[-self.m:]))]\n",
|
||||
" for g in self.ecGroups], [\"Source\", \"Parity\"])\n",
|
||||
" print(\"\\n\", groupsTable)\n",
|
||||
"\n",
|
||||
" nodesTable = tb.tabulate(\n",
|
||||
" list(map(list, self.nodesMap)), [\"Nodes\", \"Blocks\"])\n",
|
||||
" print(\"\\n\", nodesTable)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"ecPlacement = ECPlacement()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def init(k, m, bc, dir, layout, nodes, append):\n",
|
||||
" # K - The number of source blocks to encode\n",
|
||||
" # BC - Initial block count\n",
|
||||
" # Dir - Coding direction\n",
|
||||
" # Layout - Block layout\n",
|
||||
" # Nodes - Nodes count\n",
|
||||
" ##\n",
|
||||
"\n",
|
||||
" ecPlacement.update(k, m, bc, dir, layout, nodes, append)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"MAX_BLOCKS = 256\n",
|
||||
"K = widgets.IntSlider(description=\"K\", min=2, max=MAX_BLOCKS)\n",
|
||||
"M = widgets.IntSlider(description=\"M\", min=1, max=MAX_BLOCKS)\n",
|
||||
"BC = widgets.IntSlider(description=\"Origina blocks count\",\n",
|
||||
" min=1, max=10000, value=DEFAULT_BLOCKS)\n",
|
||||
"\n",
|
||||
"D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n",
|
||||
" description=\"EC Coding Direction\")\n",
|
||||
"\n",
|
||||
"BlocksLayout = widgets.Dropdown(\n",
|
||||
" options=['Inline', 'Append'],\n",
|
||||
" description=\"Blocks Manifest Layout\")\n",
|
||||
"\n",
|
||||
"Clear = widgets.Button(description='Clear')\n",
|
||||
"Nodes = widgets.IntSlider(\n",
|
||||
" description=\"Nodes\", min=K.value + M.value, max=MAX_BLOCKS)\n",
|
||||
"\n",
|
||||
"Append = widgets.Checkbox(\n",
|
||||
" value=False,\n",
|
||||
" description='Append',\n",
|
||||
" disabled=False,\n",
|
||||
" indent=False)\n",
|
||||
"\n",
|
||||
"ui = widgets.VBox([\n",
|
||||
" widgets.Label(value=\"EC Layout and Placement\"),\n",
|
||||
" widgets.VBox([K, M, BC, D, BlocksLayout, Nodes, Append]),\n",
|
||||
" Clear])\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def clearModelClick(button):\n",
|
||||
" ecPlacement.reset()\n",
|
||||
" K.value = K.min\n",
|
||||
" M.value = M.min\n",
|
||||
" Nodes.value = Nodes.min = K.value + M.value\n",
|
||||
" BC.value = 12\n",
|
||||
" D.value = \"Horizontal\"\n",
|
||||
" BlocksLayout.value = \"Inline\"\n",
|
||||
" appendBlocks.value = 0\n",
|
||||
" appendNodes = 0\n",
|
||||
" Append.value = False\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Clear.on_click(clearModelClick)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def updateKRange(*args):\n",
|
||||
" K.max = MAX_BLOCKS - M.value\n",
|
||||
" Nodes.value = K.value + M.value\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"M.observe(updateKRange, 'value')\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def updateMRange(*args):\n",
|
||||
" M.max = MAX_BLOCKS - K.value\n",
|
||||
" Nodes.value = K.value + M.value\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"K.observe(updateMRange, 'value')\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def updateNodesRange(*args):\n",
|
||||
" Nodes.min = K.value + M.value\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Nodes.observe(updateNodesRange, 'value')\n",
|
||||
"\n",
|
||||
"init = widgets.interactive_output(\n",
|
||||
" init,\n",
|
||||
" {\n",
|
||||
" 'k': K,\n",
|
||||
" 'm': M,\n",
|
||||
" 'bc': BC,\n",
|
||||
" 'dir': D,\n",
|
||||
" 'layout': BlocksLayout,\n",
|
||||
" 'nodes': Nodes,\n",
|
||||
" 'append': Append})\n",
|
||||
"\n",
|
||||
"display(ui, init)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.5 64-bit ('3.9.5')",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.5"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user