logos-storage/das-research
1
0
Fork 0
mirror of https://github.com/logos-storage/das-research.git synced 2026-01-03 13:43:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11 from status-im/doc

Browse Source 
Documentation with Sphinx
This commit is contained in:
Leo 2023-02-16 11:41:48 +01:00 committed by GitHub
commit 8f2052e1ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 254 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,2 +1,5 @@
*.swp
*.pyc
results/*
myenv
doc/_build

12
DAS/block.py
View File

@ -5,41 +5,51 @@ from bitarray import bitarray
from bitarray.util import zeros
class Block:
"""This class represents a block in the Ethereum blockchain."""
def __init__(self, blockSize):
"""Initialize the block with a data array of blocksize^2 zeros."""
self.blockSize = blockSize
self.data = zeros(self.blockSize*self.blockSize)
def fill(self):
"""It fills the block data with ones."""
self.data.setall(1)
def merge(self, merged):
"""It merges (OR) the existing block with the received one."""
self.data |= merged.data
def getColumn(self, columnID):
"""It returns the block column corresponding to columnID."""
return self.data[columnID::self.blockSize]
def mergeColumn(self, columnID, column):
"""It merges (OR) the existing column with the received one."""
self.data[columnID::self.blockSize] |= column
def repairColumn(self, id):
"""It repairs the entire column if it has at least blockSize/2 ones."""
success = self.data[id::self.blockSize].count(1)
if success >= self.blockSize/2:
self.data[id::self.blockSize] = 1
def getRow(self, rowID):
"""It returns the block row corresponding to rowID."""
return self.data[rowID*self.blockSize:(rowID+1)*self.blockSize]
def mergeRow(self, rowID, row):
"""It merges (OR) the existing row with the received one."""
self.data[rowID*self.blockSize:(rowID+1)*self.blockSize] |= row
def repairRow(self, id):
"""It repairs the entire row if it has at least blockSize/2 ones."""
success = self.data[id*self.blockSize:(id+1)*self.blockSize].count(1)
if success >= self.blockSize/2:
self.data[id*self.blockSize:(id+1)*self.blockSize] = 1
def print(self):
"""It prints the block in the terminal (outside of the logger rules))."""
dash = "-" * (self.blockSize+2)
print(dash)
for i in range(self.blockSize):

3
DAS/configuration.py
View File

@ -3,9 +3,10 @@
import configparser
class Configuration:
"""This class stores all the configuration parameters for the given run."""
def __init__(self, fileName):
"""It initializes the configuration based on the given configuration."""
config = configparser.RawConfigParser()
config.read(fileName)

7
DAS/observer.py
View File

@ -3,8 +3,10 @@
from DAS.block import *
class Observer:
"""This class gathers global data from the simulation, like an 'all-seen god'."""
def __init__(self, logger, config):
"""It initializes the observer with a logger and given configuration."""
self.config = config
self.format = {"entity": "Observer"}
self.logger = logger
@ -16,6 +18,7 @@ class Observer:
def reset(self):
"""It resets all the gathered data to zeros."""
self.block = [0] * self.config.blockSize * self.config.blockSize
self.goldenData = [0] * self.config.blockSize * self.config.blockSize
self.rows = [0] * self.config.blockSize
@ -23,6 +26,7 @@ class Observer:
self.broadcasted = Block(self.config.blockSize)
def checkRowsColumns(self, validators):
"""It checks how many validators have been assigned to each row and column."""
for val in validators:
if val.amIproposer == 0:
for r in val.rowIDs:
@ -36,10 +40,12 @@ class Observer:
self.logger.warning("There is a row/column that has not been assigned", extra=self.format)
def setGoldenData(self, block):
"""Stores the original real data to compare it with future situations."""
for i in range(self.config.blockSize*self.config.blockSize):
self.goldenData[i] = block.data[i]
def checkBroadcasted(self):
"""It checks how many broadcasted samples are still missing in the network."""
zeros = 0
for i in range(self.blockSize * self.blockSize):
if self.broadcasted.data[i] == 0:
@ -49,6 +55,7 @@ class Observer:
return zeros
def checkStatus(self, validators):
"""It checks the status of how many expected and arrived samples globally."""
arrived = 0
expected = 0
for val in validators:

5
DAS/results.py
View File

@ -5,15 +5,17 @@ from xml.dom import minidom
from dicttoxml import dicttoxml
class Result:
"""This class stores and process/store the results of a simulation."""
def __init__(self, shape):
"""It initializes the instance with a specific shape."""
self.shape = shape
self.blockAvailable = -1
self.tta = -1
self.missingVector = []
def populate(self, shape, missingVector):
"""It populates part of the result data inside a vector."""
self.shape = shape
self.missingVector = missingVector
missingSamples = missingVector[-1]
@ -25,6 +27,7 @@ class Result:
self.tta = -1
def dump(self, execID):
"""It dumps the results of the simulation in an XML file."""
if not os.path.exists("results"):
os.makedirs("results")
if not os.path.exists("results/"+execID):

2
DAS/shape.py
View File

@ -1,8 +1,10 @@
#!/bin/python3
class Shape:
"""This class represents a set of parameters for a specific simulation."""
def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, run):
"""Initializes the shape with the parameters passed in argument."""
self.run = run
self.numberValidators = numberValidators
self.blockSize = blockSize

10
DAS/simulator.py
View File

@ -9,9 +9,10 @@ from DAS.observer import *
from DAS.validator import *
class Simulator:
"""This class implements the main DAS simulator."""
def __init__(self, shape):
"""It initializes the simulation with a set of parameters (shape)."""
self.shape = shape
self.format = {"entity": "Simulator"}
self.result = Result(self.shape)
@ -22,6 +23,7 @@ class Simulator:
self.glob = []
def initValidators(self):
"""It initializes all the validators in the network."""
self.glob = Observer(self.logger, self.shape)
self.glob.reset()
self.validators = []
@ -39,6 +41,7 @@ class Simulator:
self.validators.append(val)
def initNetwork(self):
"""It initializes the simulated network."""
self.shape.netDegree = 6
rowChannels = [[] for i in range(self.shape.blockSize)]
columnChannels = [[] for i in range(self.shape.blockSize)]
@ -73,6 +76,7 @@ class Simulator:
val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, self.shape.blockSize)})
def initLogger(self):
"""It initializes the logger."""
logger = logging.getLogger("DAS")
logger.setLevel(self.logLevel)
ch = logging.StreamHandler()
@ -83,6 +87,7 @@ class Simulator:
def resetShape(self, shape):
"""It resets the parameters of the simulation."""
self.shape = shape
self.result = Result(self.shape)
for val in self.validators:
@ -91,6 +96,7 @@ class Simulator:
def run(self):
"""It runs the main simulation until the block is available or it gets stucked."""
self.glob.checkRowsColumns(self.validators)
self.validators[self.proposerID].broadcastBlock()
arrived, expected = self.glob.checkStatus(self.validators)
@ -118,7 +124,7 @@ class Simulator:
missingRate = missingSamples*100/expected
self.logger.debug("step %d, missing %d of %d (%0.02f %%)" % (steps, missingSamples, expected, missingRate), extra=self.format)
if missingSamples == oldMissingSamples:
#self.logger.info("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format)
self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format)
missingVector.append(missingSamples)
break
elif missingSamples == 0:

35
DAS/tools.py
View File

@ -2,25 +2,28 @@
import logging
class CustomFormatter(logging.Formatter):
class CustomFormatter():
"""This class defines the terminal output formatting."""
blue = "\x1b[34;20m"
grey = "\x1b[38;20m"
yellow = "\x1b[33;20m"
red = "\x1b[31;20m"
bold_red = "\x1b[31;1m"
reset = "\x1b[0m"
format = "%(levelname)s : %(entity)s : %(message)s"
FORMATS = {
logging.DEBUG: grey + format + reset,
logging.INFO: blue + format + reset,
logging.WARNING: yellow + format + reset,
logging.ERROR: red + format + reset,
logging.CRITICAL: bold_red + format + reset
}
def __init__(self):
"""Initializes 5 different formats for logging with different colors."""
self.blue = "\x1b[34;20m"
self.grey = "\x1b[38;20m"
self.yellow = "\x1b[33;20m"
self.red = "\x1b[31;20m"
self.bold_red = "\x1b[31;1m"
self.reset = "\x1b[0m"
self.reformat = "%(levelname)s : %(entity)s : %(message)s"
self.FORMATS = {
logging.DEBUG: self.grey + self.reformat + self.reset,
logging.INFO: self.blue + self.reformat + self.reset,
logging.WARNING: self.yellow + self.reformat + self.reset,
logging.ERROR: self.red + self.reformat + self.reset,
logging.CRITICAL: self.bold_red + self.reformat + self.reset
}
def format(self, record):
"""Returns the formatter with the format corresponding to record."""
log_fmt = self.FORMATS.get(record.levelno)
formatter = logging.Formatter(log_fmt)
return formatter.format(record)

41
DAS/validator.py
View File

@ -7,24 +7,31 @@ from DAS.block import *
from bitarray import bitarray
from bitarray.util import zeros
class Neighbor:
"""This class implements a node neighbor to monitor sent and received data."""
def __repr__(self):
"""It returns the amount of sent and received data."""
return "%d:%d/%d" % (self.node.ID, self.sent.count(1), self.received.count(1))
def __init__(self, v, blockSize):
"""It initializes the neighbor with the node and sets counters to zero."""
self.node = v
self.receiving = zeros(blockSize)
self.received = zeros(blockSize)
self.sent = zeros(blockSize)
class Validator:
class Validator:
"""This class implements a validator/node in the network."""
def __repr__(self):
"""It returns the validator ID."""
return str(self.ID)
def __init__(self, ID, amIproposer, logger, shape, rows, columns):
"""It initializes the validator with the logger, shape and assigned rows/columns."""
self.shape = shape
FORMAT = "%(levelname)s : %(entity)s : %(message)s"
self.ID = ID
@ -60,6 +67,7 @@ class Validator:
self.statsRxPerSlot = []
def logIDs(self):
"""It logs the assigned rows and columns."""
if self.amIproposer == 1:
self.logger.warning("I am a block proposer."% self.ID)
else:
@ -67,14 +75,19 @@ class Validator:
self.logger.debug("Selected columns: "+str(self.columnIDs), extra=self.format)
def initBlock(self):
self.logger.debug("I am a block proposer.", extra=self.format)
self.block = Block(self.shape.blockSize)
self.block.fill()
#self.block.print()
"""It initializes the block for the proposer."""
if self.amIproposer == 1:
self.logger.debug("I am a block proposer.", extra=self.format)
self.block = Block(self.shape.blockSize)
self.block.fill()
#self.block.print()
else:
self.logger.warning("I am not a block proposer."% self.ID)
def broadcastBlock(self):
"""The block proposer broadcasts the block to all validators."""
if self.amIproposer == 0:
self.logger.error("I am NOT a block proposer", extra=self.format)
self.logger.warning("I am not a block proposer", extra=self.format)
else:
self.logger.debug("Broadcasting my block...", extra=self.format)
order = [i for i in range(self.shape.blockSize * self.shape.blockSize)]
@ -95,12 +108,15 @@ class Validator:
#broadcasted.print()
def getColumn(self, index):
"""It returns a given column."""
return self.block.getColumn(index)
def getRow(self, index):
"""It returns a given row."""
return self.block.getRow(index)
def receiveColumn(self, id, column, src):
"""It receives the given column if it has been assigned to it."""
if id in self.columnIDs:
# register receive so that we are not sending back
self.columnNeighbors[id][src].receiving |= column
@ -110,6 +126,7 @@ class Validator:
pass
def receiveRow(self, id, row, src):
"""It receives the given row if it has been assigned to it."""
if id in self.rowIDs:
# register receive so that we are not sending back
self.rowNeighbors[id][src].receiving |= row
@ -120,6 +137,7 @@ class Validator:
def receiveRowsColumns(self):
"""It receives rows and columns."""
if self.amIproposer == 1:
self.logger.error("I am a block proposer", extra=self.format)
else:
@ -147,7 +165,9 @@ class Validator:
for neigh in neighs.values():
neigh.received |= neigh.receiving
neigh.receiving.setall(0)
def updateStats(self):
"""It updates the stats related to sent and received data."""
self.logger.debug("Stats: tx %d, rx %d", self.statsTxInSlot, self.statsRxInSlot, extra=self.format)
self.statsRxPerSlot.append(self.statsRxInSlot)
self.statsTxPerSlot.append(self.statsTxInSlot)
@ -156,6 +176,7 @@ class Validator:
def sendColumn(self, columnID):
"""It sends any new sample in the given column."""
line = self.getColumn(columnID)
if line.any():
self.logger.debug("col %d -> %s", columnID, self.columnNeighbors[columnID] , extra=self.format)
@ -169,6 +190,7 @@ class Validator:
self.statsTxInSlot += toSend.count(1)
def sendRow(self, rowID):
"""It sends any new sample in the given row."""
line = self.getRow(rowID)
if line.any():
self.logger.debug("row %d -> %s", rowID, self.rowNeighbors[rowID], extra=self.format)
@ -182,36 +204,43 @@ class Validator:
self.statsTxInSlot += toSend.count(1)
def sendRows(self):
"""It sends all restored rows."""
self.logger.debug("Sending restored rows...", extra=self.format)
for r in self.rowIDs:
if self.changedRow[r]:
self.sendRow(r)
def sendColumns(self):
"""It sends all restored columns."""
self.logger.debug("Sending restored columns...", extra=self.format)
for c in self.columnIDs:
if self.changedColumn[c]:
self.sendColumn(c)
def logRows(self):
"""It logs the rows assigned to the validator."""
if self.logger.isEnabledFor(logging.DEBUG):
for id in self.rowIDs:
self.logger.debug("Row %d: %s", id, self.getRow(id), extra=self.format)
def logColumns(self):
"""It logs the columns assigned to the validator."""
if self.logger.isEnabledFor(logging.DEBUG):
for id in self.columnIDs:
self.logger.debug("Column %d: %s", id, self.getColumn(id), extra=self.format)
def restoreRows(self):
"""It restores the rows assigned to the validator, that can be repaired."""
for id in self.rowIDs:
self.block.repairRow(id)
def restoreColumns(self):
"""It restores the columns assigned to the validator, that can be repaired."""
for id in self.columnIDs:
self.block.repairColumn(id)
def checkStatus(self):
"""It checks how many expected/arrived samples are for each assigned row/column."""
arrived = 0
expected = 0
for id in self.columnIDs:

20
doc/Makefile Normal file
View File

@ -0,0 +1,20 @@
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = .
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

64
doc/conf.py Normal file
View File

@ -0,0 +1,64 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('../DAS'))
# -- Project information -----------------------------------------------------
project = 'DAS simulator'
copyright = '2023, Leonardo A. Bautista-Gomez, Csaba Kiraly'
author = 'Leonardo A. Bautista-Gomez, Csaba Kiraly'
# The full version, including alpha/beta/rc tags
release = '1'
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc'
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'myenv']
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# -- Options for autodoc -------------------------------------------------
autodoc_mock_imports = ["django", "dicttoxml", "bitarray", "DAS", "networkx"]

44
doc/index.rst Normal file
View File

@ -0,0 +1,44 @@
.. DAS simulator documentation master file, created by
sphinx-quickstart on Wed Feb 8 20:56:44 2023.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to DAS simulator's documentation!
=========================================
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
.. toctree::
:maxdepth: 2
:caption: Contents:
.. automodule:: block
:members:
.. automodule:: configuration
:members:
.. automodule:: observer
:members:
.. automodule:: results
:members:
.. automodule:: shape
:members:
.. automodule:: simulator
:members:
.. automodule:: tools
:members:
.. automodule:: validator
:members:

35
doc/make.bat Normal file
View File

@ -0,0 +1,35 @@
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=.
set BUILDDIR=_build
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.https://www.sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
:end
popd