codex-storage
/
codex-research
mirror of https://github.com/codex-storage/codex-research.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

allow appending blocks and nodes

This commit is contained in:
Dmitriy Ryajov 2022-08-19 21:08:51 -06:00
parent 8cbeb52d9e
commit faacc0bc0a
No known key found for this signature in database
GPG Key ID: DA8C680CE7C657A4
1 changed files with 98 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
analysis/ec-placement.ipynb
View File

@ -2,18 +2,18 @@
"cells": [
{
"cell_type": "code",
"execution_count": 77,
"execution_count": 56,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "dd5a5f6038bc44d1ac4b044894b50f30",
"model_id": "0d654a31aa8842bea08f69d17e1e9283",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(IntSlider(value=2, description='K', max=256, min=2), IntSlider(value=1, description='M', max=25…"
"VBox(children=(Label(value='EC Layout and Placement'), VBox(children=(IntSlider(value=2, description='K', max=…"
]
},
"metadata": {},
@ -22,7 +22,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1dbdb23c07e4466e82d422c423af3bad",
"model_id": "577931878dbf419d85441e582a1760f8",
"version_major": 2,
"version_minor": 0
},
@ -54,15 +54,16 @@
"\n",
"\n",
"def roundBlocks(factor, ob):\n",
" return ob + (factor - (ob % factor)) if (ob % factor) > 0 else 0\n",
" return (factor - (ob % factor)) if (ob % factor) > 0 else 0\n",
"\n",
"\n",
"def makeEcGroups(\n",
" blocks,\n",
" k, m,\n",
" direction):\n",
" direction,\n",
" offset=0):\n",
"\n",
" offset = len(blocks)\n",
" offset = len(blocks) if offset == 0 else offset\n",
" res = []\n",
"\n",
" groups = (len(blocks) // k)\n",
@ -140,6 +141,11 @@
" self.direction = direction\n",
" self.nodes = nodes\n",
" self.bc = bc\n",
" self.blocks = []\n",
" self.nodesMap = OrderedDict()\n",
" self.blockSeq = []\n",
" self.ecGroups = []\n",
" self.rb = 0\n",
"\n",
" def reset(self):\n",
" self.k = DEFAULT_K\n",
@ -155,14 +161,47 @@
" self.blockSeq = []\n",
" self.ecGroups = []\n",
"\n",
" def update(self, k, m, bc, dir, layout, nodes):\n",
" if self.k != k or self.m != m or self.bc != bc:\n",
" def update(self, k, m, bc, direction, layout, nodes, append):\n",
" print(k, m, bc, direction, layout, nodes, append)\n",
"\n",
" if append:\n",
" if bc > self.bc:\n",
" appendBlocks = bc - self.bc\n",
" appendBlocks = appendBlocks + roundBlocks(k, appendBlocks)\n",
" self.blocks += list(\n",
" range(self.bc + 1, bc + appendBlocks + 1))\n",
"\n",
" if len(self.blocks) > 0:\n",
" blockSeqLen = len(self.blockSeq)\n",
" print(blockSeqLen)\n",
"\n",
" newBlocks = list(\n",
" range(blockSeqLen + 1, blockSeqLen + appendBlocks + 1))\n",
"\n",
" self.ecGroups += makeEcGroups(\n",
" newBlocks, k, m, direction, blockSeqLen + appendBlocks) # Calculate EC groups\n",
"\n",
" self.blockSeq = makeReadSeq(\n",
" k, m, self.ecGroups, direction, layout)\n",
"\n",
" self.nodesMap = mapNodes(\n",
" self.blockSeq, nodes).items()\n",
" else:\n",
" # Calculate geometry\n",
" self.rb = roundBlocks(k, bc)\n",
" if (self.rb + bc) > len(self.blocks):\n",
" # Generate source blocks\n",
" self.blocks += list(\n",
" range(len(self.blocks) + 1, self.rb + 1))\n",
" # Generate source blocks\n",
" self.blocks = list(\n",
" range(1, bc + 1 + self.rb))\n",
"\n",
" if len(self.blocks) > 0:\n",
" self.ecGroups = makeEcGroups(\n",
" self.blocks, k, m, direction) # Calculate EC groups\n",
"\n",
" self.blockSeq = makeReadSeq(\n",
" k, m, self.ecGroups, direction, layout)\n",
"\n",
" self.nodesMap = mapNodes(\n",
" self.blockSeq, nodes).items()\n",
"\n",
" self.bc = bc\n",
" self.k = k\n",
@ -171,20 +210,10 @@
" self.layout = layout\n",
" self.nodes = nodes\n",
"\n",
" if len(self.blocks) > 0:\n",
" self.ecGroups = makeEcGroups(\n",
" self.blocks, self.k, self.m, self.direction) # Calculate EC groups\n",
"\n",
" self.blockSeq = makeReadSeq(\n",
" self.k, self.m, self.ecGroups, self.direction, self.layout)\n",
"\n",
" self.nodesMap = mapNodes(\n",
" self.blockSeq, self.nodes).items()\n",
"\n",
" return self.blockSeq\n",
" self.render()\n",
"\n",
" def render(self):\n",
" width = getZeros(len(self.blockSeq) or BC)\n",
" width = getZeros(len(self.blockSeq) or self.bc)\n",
" formatBlock = getBlockFormat(width)\n",
"\n",
" print(\"\\n\")\n",
@ -206,56 +235,76 @@
"\n",
"ecPlacement = ECPlacement()\n",
"\n",
"def main(k, m, bc, dir, layout, nodes):\n",
" # K - The number of source blocks to encode # BC - Initial block count\n",
"\n",
"def init(k, m, bc, dir, layout, nodes, append):\n",
" # K - The number of source blocks to encode\n",
" # BC - Initial block count\n",
" # Dir - Coding direction\n",
" # Layout - Block layout\n",
" # Nodes - Nodes count\n",
" ##\n",
"\n",
" ecPlacement.update(k, m, bc, dir, layout, nodes)\n",
" ecPlacement.render()\n",
" ecPlacement.update(k, m, bc, dir, layout, nodes, append)\n",
"\n",
"\n",
"MaxBlocks = 256\n",
"K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n",
"M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n",
"MAX_BLOCKS = 256\n",
"K = widgets.IntSlider(description=\"K\", min=2, max=MAX_BLOCKS)\n",
"M = widgets.IntSlider(description=\"M\", min=1, max=MAX_BLOCKS)\n",
"BC = widgets.IntSlider(description=\"Origina blocks count\",\n",
" min=1, max=10000, value=12)\n",
" min=1, max=10000, value=DEFAULT_BLOCKS)\n",
"\n",
"D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n",
" description=\"EC Coding Direction\")\n",
"Layout = widgets.Dropdown(\n",
"\n",
"BlocksLayout = widgets.Dropdown(\n",
" options=['Inline', 'Append'],\n",
" description=\"Blocks Manifest Layout\")\n",
"\n",
"Clear = widgets.Button(description='Clear')\n",
"Nodes = widgets.IntSlider(\n",
" description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n",
" description=\"Nodes\", min=K.value + M.value, max=MAX_BLOCKS)\n",
"\n",
"ui = widgets.VBox([K, M, BC, D, Layout, Nodes, Clear])\n",
"Append = widgets.Checkbox(\n",
" value=False,\n",
" description='Append',\n",
" disabled=False,\n",
" indent=False)\n",
"\n",
"ui = widgets.VBox([\n",
" widgets.Label(value=\"EC Layout and Placement\"),\n",
" widgets.VBox([K, M, BC, D, BlocksLayout, Nodes, Append]),\n",
" Clear])\n",
"\n",
"\n",
"def clearModelClick(button):\n",
" ecPlacement.reset()\n",
" K.value = K.min\n",
" M.value = M.min\n",
" Nodes.value = Nodes.min = K.value + M.value\n",
" BC.value = 12\n",
" D.value = \"Horizontal\"\n",
" Layout.value = \"Inline\"\n",
" Nodes.value = 3\n",
" BlocksLayout.value = \"Inline\"\n",
" appendBlocks.value = 0\n",
" appendNodes = 0\n",
" Append.value = False\n",
"\n",
"\n",
"Clear.on_click(clearModelClick)\n",
"\n",
"\n",
"def updateKRange(*args):\n",
" K.max = MaxBlocks - M.value\n",
"\n",
"\n",
"def updateMRange(*args):\n",
" M.max = MaxBlocks - K.value\n",
" K.max = MAX_BLOCKS - M.value\n",
" Nodes.value = K.value + M.value\n",
"\n",
"\n",
"M.observe(updateKRange, 'value')\n",
"\n",
"\n",
"def updateMRange(*args):\n",
" M.max = MAX_BLOCKS - K.value\n",
" Nodes.value = K.value + M.value\n",
"\n",
"\n",
"K.observe(updateMRange, 'value')\n",
"\n",
"\n",
@ -265,17 +314,18 @@
"\n",
"Nodes.observe(updateNodesRange, 'value')\n",
"\n",
"out = widgets.interactive_output(\n",
" main,\n",
"init = widgets.interactive_output(\n",
" init,\n",
" {\n",
" 'k': K,\n",
" 'm': M,\n",
" 'bc': BC,\n",
" 'dir': D,\n",
" 'layout': Layout,\n",
" 'nodes': Nodes})\n",
" 'layout': BlocksLayout,\n",
" 'nodes': Nodes,\n",
" 'append': Append})\n",
"\n",
"display(ui, out)\n"
"display(ui, init)\n"
]
}
],