mirror of
https://github.com/logos-storage/logos-storage-research.git
synced 2026-03-29 07:13:37 +00:00
move to correct dir and cleanup
This commit is contained in:
Dmitriy Ryajov
parent
4c23ff0100
commit
ac3cf527aa
230
analysis/ec-placement.ipynb
Normal file
230
analysis/ec-placement.ipynb
Normal file
@ -0,0 +1,230 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 97,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "a58d0687e0b9452e93be2ef56ebc68ed",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"VBox(children=(IntSlider(value=2, description='K', max=256, min=2), IntSlider(value=1, description='M', max=25…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b3d1ec6e56d4467bb2ec8f202567ee58",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Output()"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from __future__ import print_function\n",
|
||||
"from ipywidgets import interact, interactive, fixed, interact_manual\n",
|
||||
"import ipywidgets as widgets\n",
|
||||
"import itertools as it\n",
|
||||
"import tabulate as tb\n",
|
||||
"from functools import reduce\n",
|
||||
"from operator import concat\n",
|
||||
"from copy import deepcopy\n",
|
||||
"import math\n",
|
||||
"from collections import OrderedDict\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def roundBlocks(factor, ob):\n",
|
||||
" return ob + (factor - (ob % factor))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def makeEcGroups(\n",
|
||||
" blocks,\n",
|
||||
" k, m,\n",
|
||||
" direction):\n",
|
||||
"\n",
|
||||
" offset = len(blocks)\n",
|
||||
" res = []\n",
|
||||
"\n",
|
||||
" groups = (len(blocks) // k)\n",
|
||||
" steps = len(blocks) // groups\n",
|
||||
" if direction == \"Horizontal\":\n",
|
||||
" for b in range(0, len(blocks), steps):\n",
|
||||
" res.append(\n",
|
||||
" blocks[b:b + steps] +\n",
|
||||
" list(range(offset + 1, (offset + 1 + m))))\n",
|
||||
" offset += m\n",
|
||||
" else:\n",
|
||||
" for s in range(0, groups):\n",
|
||||
" res.append(\n",
|
||||
" blocks[s:s+groups+offset:groups] +\n",
|
||||
" list(range(offset + 1, (offset + 1 + m))))\n",
|
||||
" offset += m\n",
|
||||
"\n",
|
||||
" return res\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def mapNodes(blocks, hosts, offset=0):\n",
|
||||
" width = getZeros(len(blocks) or BC.value)\n",
|
||||
" formatBlock = getBlockFormat(width)\n",
|
||||
" res = OrderedDict()\n",
|
||||
"\n",
|
||||
" for i, _ in enumerate(blocks):\n",
|
||||
" key = \"node\" + str((i % hosts)+1)\n",
|
||||
" res[key] = (key in res and res[key]) or []\n",
|
||||
" res[key].append(formatBlock(blocks[i]))\n",
|
||||
"\n",
|
||||
" return res\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def getBlockFormat(width):\n",
|
||||
" return lambda b: \"{b:0{fmt}}\".format(b=b, fmt=width)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def getZeros(width):\n",
|
||||
" return int(math.log(width, 10) + 1)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def makeReadSeq(K, M, ecGroups, direction, layout):\n",
|
||||
" readSequence = []\n",
|
||||
" if layout == \"Append\":\n",
|
||||
" if direction == \"Horizontal\":\n",
|
||||
" readSequence = reduce(\n",
|
||||
" concat,\n",
|
||||
" [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n",
|
||||
" else:\n",
|
||||
" readSequence = reduce(concat, zip(*ecGroups))\n",
|
||||
" else:\n",
|
||||
" readSequence = reduce(concat, ecGroups)\n",
|
||||
"\n",
|
||||
" return readSequence\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def buildModel(K, M, blocks, Dir, Layout, Nodes):\n",
|
||||
" return\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def main(K, M, BC, Dir, Layout, Nodes):\n",
|
||||
" # K - The number of source blocks to encode\n",
|
||||
" # M - The number of parity blocks produced\n",
|
||||
" # BC - Initial block count\n",
|
||||
" # Dir - Coding direction\n",
|
||||
" # Layout - Block layout\n",
|
||||
" # Nodes - Nodes count\n",
|
||||
" ##\n",
|
||||
"\n",
|
||||
" rb = roundBlocks(Nodes, BC) # Calculate geometry\n",
|
||||
" blocks = list(range(1, rb + 1)) # Generate source blocks\n",
|
||||
" ecGroups = makeEcGroups(blocks, K, M, Dir) # Calculate EC groups\n",
|
||||
" readSequence = makeReadSeq(K, M, ecGroups, Dir, Layout)\n",
|
||||
"\n",
|
||||
" width = getZeros(len(readSequence) or BC)\n",
|
||||
" formatBlock = getBlockFormat(width)\n",
|
||||
"\n",
|
||||
" print(\"\\n\")\n",
|
||||
" print(K, M, BC, Dir, Layout, Nodes)\n",
|
||||
"\n",
|
||||
" print(\"Rounded Blocks:\", rb)\n",
|
||||
" print(\"Encoding Groups:\", len(ecGroups))\n",
|
||||
" print(\"K + M = \", K + M)\n",
|
||||
" print(\"Source Blocks:\", \", \".join(map(formatBlock, blocks)))\n",
|
||||
" print(\"Read Layout:\", \", \".join(map(formatBlock, readSequence)))\n",
|
||||
"\n",
|
||||
" groupsTable = tb.tabulate(\n",
|
||||
" [[\", \".join(map(formatBlock, g[:K])), \", \".join(map(formatBlock, g[-M:]))]\n",
|
||||
" for g in ecGroups], [\"Source\", \"Parity\"])\n",
|
||||
" print(\"\\n\", groupsTable)\n",
|
||||
"\n",
|
||||
" nodesTable = tb.tabulate(\n",
|
||||
" list(map(list, mapNodes(readSequence, Nodes).items())), [\"Nodes\", \"Blocks\"])\n",
|
||||
" print(\"\\n\", nodesTable)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"MaxBlocks = 256\n",
|
||||
"K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n",
|
||||
"M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n",
|
||||
"BC = widgets.IntSlider(description=\"Origina blocks count\",\n",
|
||||
" min=1, max=10000, value=10)\n",
|
||||
"D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n",
|
||||
" description=\"EC Coding Direction\")\n",
|
||||
"Layout = widgets.Dropdown(\n",
|
||||
" options=['Inline', 'Append'], description=\"Blocks Manifest Layout\")\n",
|
||||
"Nodes = widgets.IntSlider(\n",
|
||||
" description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n",
|
||||
"\n",
|
||||
"ui = widgets.VBox([K, M, BC, D, Layout, Nodes])\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def updateKRange(*args):\n",
|
||||
" K.max = MaxBlocks - M.value\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def updateMRange(*args):\n",
|
||||
" M.max = MaxBlocks - K.value\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"M.observe(updateKRange, 'value')\n",
|
||||
"K.observe(updateMRange, 'value')\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def updateNodesRange(*args):\n",
|
||||
" Nodes.min = K.value + M.value\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Nodes.observe(updateNodesRange, 'value')\n",
|
||||
"\n",
|
||||
"out = widgets.interactive_output(\n",
|
||||
" main,\n",
|
||||
" {\n",
|
||||
" 'K': K,\n",
|
||||
" 'M': M,\n",
|
||||
" 'BC': BC,\n",
|
||||
" 'Dir': D,\n",
|
||||
" 'Layout': Layout,\n",
|
||||
" 'Nodes': Nodes})\n",
|
||||
"\n",
|
||||
"display(ui, out)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.5 64-bit ('3.9.5')",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.5"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@ -1,231 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b63223694adb45e2925572977bf8381c",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"VBox(children=(IntSlider(value=2, description='K', max=256, min=2), IntSlider(value=1, description='M', max=25…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "92669eb1c4484a36a93107690a4c5dc9",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Output()"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from __future__ import print_function\n",
|
||||
"from ipywidgets import interact, interactive, fixed, interact_manual\n",
|
||||
"import ipywidgets as widgets\n",
|
||||
"import itertools as it\n",
|
||||
"import tabulate as tb\n",
|
||||
"from functools import reduce\n",
|
||||
"from operator import concat\n",
|
||||
"from copy import deepcopy\n",
|
||||
"\n",
|
||||
"blocks = []\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def main(K, M, BC, Dir, Man, Nodes):\n",
|
||||
" ## K - The number of source blocks to encode\n",
|
||||
" ## M - The number of parity blocks produced\n",
|
||||
" ## BC - Initial block count\n",
|
||||
" ## Dir - Coding direction\n",
|
||||
" ## Man - Manifest block layout\n",
|
||||
" ## Nodes - Nodes count\n",
|
||||
" ##\n",
|
||||
"\n",
|
||||
" (rb, eg, n) = calcGeometry(K, M, BC) # Calculate geometry\n",
|
||||
" blocks = list(range(1, rb + 1)) # Generate source blocks\n",
|
||||
" ecGroups = makeEcGroups(blocks, K, M, Dir) # Calculate EC groups\n",
|
||||
" # manifestBlocks = makeManifestBlocks(blocks, K, M, Man)\n",
|
||||
" manifestBlocks = []\n",
|
||||
"\n",
|
||||
" if Man == \"Append\":\n",
|
||||
" if Dir == \"Horizontal\":\n",
|
||||
" manifestBlocks = reduce(\n",
|
||||
" concat,\n",
|
||||
" [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n",
|
||||
" else:\n",
|
||||
" manifestBlocks = reduce(concat, zip(*ecGroups))\n",
|
||||
" else:\n",
|
||||
" manifestBlocks = reduce(concat, ecGroups)\n",
|
||||
"\n",
|
||||
" print(\"\\n\")\n",
|
||||
" print(K, M, BC, Dir, Man, Nodes)\n",
|
||||
" print(\"Rounded Blocks:\", rb)\n",
|
||||
" print(\"Encoding Groups:\", eg)\n",
|
||||
" print(\"K + M = \", n)\n",
|
||||
" print(\"Source Blocks:\", blocks)\n",
|
||||
" print(\"Read Order:\", manifestBlocks)\n",
|
||||
" print(\"\\n\",\n",
|
||||
" tb.tabulate(\n",
|
||||
" [[\", \".join(map(str, g[:K])), \" ,\".join(map(str, g[-M:]))]\n",
|
||||
" for g in ecGroups],\n",
|
||||
" [\"Source\", \"Parity\"]))\n",
|
||||
"\n",
|
||||
" print(\"\\n\",\n",
|
||||
" tb.tabulate(list(map(list, mapNodes(manifestBlocks, Nodes).items())),\n",
|
||||
" [\"Nodes\", \"Blocks\"]))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def calcGeometry(K, M, OB):\n",
|
||||
" RB = OB + (K - (OB % K))\n",
|
||||
" EG = (RB // K)\n",
|
||||
" N = K + M\n",
|
||||
"\n",
|
||||
" return (RB, EG, N)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def makeManifestBlocks(\n",
|
||||
" blocks,\n",
|
||||
" k, m,\n",
|
||||
" layout):\n",
|
||||
"\n",
|
||||
" offset = len(blocks)\n",
|
||||
" res = []\n",
|
||||
"\n",
|
||||
" steps = len(blocks) // (len(blocks) // k)\n",
|
||||
" if layout == \"Inline\":\n",
|
||||
" for b in range(0, len(blocks), steps):\n",
|
||||
" res += blocks[b:b + steps] + list(range(offset + 1, (offset + 1 + m)))\n",
|
||||
" offset += m\n",
|
||||
" else:\n",
|
||||
" res = deepcopy(blocks)\n",
|
||||
" for b in range(0, len(blocks), steps):\n",
|
||||
" res += list(range(offset + 1, (offset + 1 + m)))\n",
|
||||
" offset += m\n",
|
||||
"\n",
|
||||
" return res\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def makeEcGroups(\n",
|
||||
" blocks,\n",
|
||||
" k, m,\n",
|
||||
" direction):\n",
|
||||
"\n",
|
||||
" offset = len(blocks)\n",
|
||||
" res = []\n",
|
||||
"\n",
|
||||
" groups = (len(blocks) // k)\n",
|
||||
" steps = len(blocks) // groups\n",
|
||||
" if direction == \"Horizontal\":\n",
|
||||
" for b in range(0, len(blocks), steps):\n",
|
||||
" res.append(\n",
|
||||
" blocks[b:b + steps] +\n",
|
||||
" list(range(offset + 1, (offset + 1 + m))))\n",
|
||||
" offset += m\n",
|
||||
" else:\n",
|
||||
" for s in range(0, groups):\n",
|
||||
" res.append(\n",
|
||||
" blocks[s:s+groups+offset:groups] +\n",
|
||||
" list(range(offset + 1, (offset + 1 + m))))\n",
|
||||
" offset += m\n",
|
||||
"\n",
|
||||
" return res\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def mapNodes(blocks, hosts, offset=0):\n",
|
||||
" res = {\"node\" + str((h % hosts)+1): [] for h in range(1, hosts+1)}\n",
|
||||
" for i, blk in enumerate(blocks):\n",
|
||||
" res[\"node\" + str((i % hosts)+1)].append(blocks[i])\n",
|
||||
"\n",
|
||||
" return res\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"MaxBlocks = 256\n",
|
||||
"K = widgets.IntSlider(description=\"K\", min=2, max=MaxBlocks)\n",
|
||||
"M = widgets.IntSlider(description=\"M\", min=1, max=MaxBlocks)\n",
|
||||
"BC = widgets.IntSlider(description=\"Origina blocks count\",\n",
|
||||
" min=1, max=10000, value=10)\n",
|
||||
"D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n",
|
||||
" description=\"EC Coding Direction\")\n",
|
||||
"Manifest = widgets.Dropdown(\n",
|
||||
" options=['Inline', 'Append'], description=\"Blocks Manifest Layout\")\n",
|
||||
"Nodes = widgets.IntSlider(\n",
|
||||
" description=\"Nodes\", min=K.value + M.value, max=MaxBlocks)\n",
|
||||
"\n",
|
||||
"ui = widgets.VBox([K, M, BC, D, Manifest, Nodes])\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def updateKRange(*args):\n",
|
||||
" K.max = MaxBlocks - M.value\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def updateMRange(*args):\n",
|
||||
" M.max = MaxBlocks - K.value\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"M.observe(updateKRange, 'value')\n",
|
||||
"K.observe(updateMRange, 'value')\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def updateNodesRange(*args):\n",
|
||||
" Nodes.min = K.value + M.value\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Nodes.observe(updateNodesRange, 'value')\n",
|
||||
"\n",
|
||||
"out = widgets.interactive_output(\n",
|
||||
" main,\n",
|
||||
" {\n",
|
||||
" 'K': K,\n",
|
||||
" 'M': M,\n",
|
||||
" 'BC': BC,\n",
|
||||
" 'Dir': D,\n",
|
||||
" 'Man': Manifest,\n",
|
||||
" 'Nodes': Nodes})\n",
|
||||
"\n",
|
||||
"display(ui, out)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.5 64-bit ('3.9.5')",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.5"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user