From 17b97c37c094fab542bb94c566ffac2e7b29dd82 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 8 Feb 2023 22:31:51 +0100 Subject: [PATCH 1/5] Update gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b948985..13d2a0a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.swp *.pyc +results/* +myenv From 5c55fd7854a9462715cbd93534a92c5b1d7205b2 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 8 Feb 2023 22:33:16 +0100 Subject: [PATCH 2/5] Add documentation files --- doc/Makefile | 20 ++++++++++++++++ doc/conf.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/index.rst | 42 +++++++++++++++++++++++++++++++++ doc/make.bat | 35 ++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+) create mode 100644 doc/Makefile create mode 100644 doc/conf.py create mode 100644 doc/index.rst create mode 100644 doc/make.bat diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..7524d9c --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,64 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../DAS')) + + +# -- Project information ----------------------------------------------------- + +project = 'DAS simulator' +copyright = '2023, Leonardo A. Bautista-Gomez, Csaba Kiraly' +author = 'Leonardo A. Bautista-Gomez, Csaba Kiraly' + +# The full version, including alpha/beta/rc tags +release = '1' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc' +] + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'myenv'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for autodoc ------------------------------------------------- + +autodoc_mock_imports = ["django", "dicttoxml", "bitarray", "DAS", "networkx"] + + + diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000..f6b5fc9 --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,42 @@ +.. DAS simulator documentation master file, created by + sphinx-quickstart on Wed Feb 8 20:56:44 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to DAS simulator's documentation! +========================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. automodule:: block + :members: + +.. automodule:: configuration + :members: + +.. automodule:: observer + :members: + +.. automodule:: results + :members: + +.. automodule:: shape + :members: + +.. automodule:: simulator + :members: + +.. automodule:: tools + :members: + +.. automodule:: validator + :members: + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 0000000..153be5e --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd From cf780b3ca3ca533c841d2f01ad39431cc6e0ba4b Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 8 Feb 2023 22:45:01 +0100 Subject: [PATCH 3/5] Starting code documentation --- DAS/block.py | 2 +- DAS/tools.py | 3 ++- doc/index.rst | 12 +++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/DAS/block.py b/DAS/block.py index 1cea8c0..a22b70c 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -5,7 +5,7 @@ from bitarray import bitarray from bitarray.util import zeros class Block: - + """This class represents a block in the Ethereum blockchain""" def __init__(self, blockSize): self.blockSize = blockSize diff --git a/DAS/tools.py b/DAS/tools.py index b9e4a8e..68469fe 100644 --- a/DAS/tools.py +++ b/DAS/tools.py @@ -2,7 +2,8 @@ import logging -class CustomFormatter(logging.Formatter): +class CustomFormatter(): + """This class defines the terminal output formatting.""" blue = "\x1b[34;20m" grey = "\x1b[38;20m" diff --git a/doc/index.rst b/doc/index.rst index f6b5fc9..e253201 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -6,6 +6,13 @@ Welcome to DAS simulator's documentation! ========================================= +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + .. toctree:: :maxdepth: 2 :caption: Contents: @@ -34,9 +41,4 @@ Welcome to DAS simulator's documentation! .. automodule:: validator :members: -Indices and tables -================== -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` From 9888d96e438f8785e96e8be6e489b93b5577f97b Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 8 Feb 2023 22:46:12 +0100 Subject: [PATCH 4/5] Update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 13d2a0a..1e5dc45 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.pyc results/* myenv +doc/_build From 09569422ab4fce1c57a1a972beac37ed68984b29 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 15 Feb 2023 15:06:42 +0100 Subject: [PATCH 5/5] Documenting the DAS package for automatic generation with sphinx. First version. --- DAS/block.py | 12 +++++++++++- DAS/configuration.py | 3 ++- DAS/observer.py | 7 +++++++ DAS/results.py | 5 ++++- DAS/shape.py | 2 ++ DAS/simulator.py | 10 ++++++++-- DAS/tools.py | 32 +++++++++++++++++--------------- DAS/validator.py | 41 +++++++++++++++++++++++++++++++++++------ 8 files changed, 86 insertions(+), 26 deletions(-) diff --git a/DAS/block.py b/DAS/block.py index a22b70c..693d7b6 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -5,41 +5,51 @@ from bitarray import bitarray from bitarray.util import zeros class Block: - """This class represents a block in the Ethereum blockchain""" + """This class represents a block in the Ethereum blockchain.""" def __init__(self, blockSize): + """Initialize the block with a data array of blocksize^2 zeros.""" self.blockSize = blockSize self.data = zeros(self.blockSize*self.blockSize) def fill(self): + """It fills the block data with ones.""" self.data.setall(1) def merge(self, merged): + """It merges (OR) the existing block with the received one.""" self.data |= merged.data def getColumn(self, columnID): + """It returns the block column corresponding to columnID.""" return self.data[columnID::self.blockSize] def mergeColumn(self, columnID, column): + """It merges (OR) the existing column with the received one.""" self.data[columnID::self.blockSize] |= column def repairColumn(self, id): + """It repairs the entire column if it has at least blockSize/2 ones.""" success = self.data[id::self.blockSize].count(1) if success >= self.blockSize/2: self.data[id::self.blockSize] = 1 def getRow(self, rowID): + """It returns the block row corresponding to rowID.""" return self.data[rowID*self.blockSize:(rowID+1)*self.blockSize] def mergeRow(self, rowID, row): + """It merges (OR) the existing row with the received one.""" self.data[rowID*self.blockSize:(rowID+1)*self.blockSize] |= row def repairRow(self, id): + """It repairs the entire row if it has at least blockSize/2 ones.""" success = self.data[id*self.blockSize:(id+1)*self.blockSize].count(1) if success >= self.blockSize/2: self.data[id*self.blockSize:(id+1)*self.blockSize] = 1 def print(self): + """It prints the block in the terminal (outside of the logger rules)).""" dash = "-" * (self.blockSize+2) print(dash) for i in range(self.blockSize): diff --git a/DAS/configuration.py b/DAS/configuration.py index d3647a4..c840366 100644 --- a/DAS/configuration.py +++ b/DAS/configuration.py @@ -3,9 +3,10 @@ import configparser class Configuration: - + """This class stores all the configuration parameters for the given run.""" def __init__(self, fileName): + """It initializes the configuration based on the given configuration.""" config = configparser.RawConfigParser() config.read(fileName) diff --git a/DAS/observer.py b/DAS/observer.py index 515fe09..af5866a 100644 --- a/DAS/observer.py +++ b/DAS/observer.py @@ -3,8 +3,10 @@ from DAS.block import * class Observer: + """This class gathers global data from the simulation, like an 'all-seen god'.""" def __init__(self, logger, config): + """It initializes the observer with a logger and given configuration.""" self.config = config self.format = {"entity": "Observer"} self.logger = logger @@ -16,6 +18,7 @@ class Observer: def reset(self): + """It resets all the gathered data to zeros.""" self.block = [0] * self.config.blockSize * self.config.blockSize self.goldenData = [0] * self.config.blockSize * self.config.blockSize self.rows = [0] * self.config.blockSize @@ -23,6 +26,7 @@ class Observer: self.broadcasted = Block(self.config.blockSize) def checkRowsColumns(self, validators): + """It checks how many validators have been assigned to each row and column.""" for val in validators: if val.amIproposer == 0: for r in val.rowIDs: @@ -36,10 +40,12 @@ class Observer: self.logger.warning("There is a row/column that has not been assigned", extra=self.format) def setGoldenData(self, block): + """Stores the original real data to compare it with future situations.""" for i in range(self.config.blockSize*self.config.blockSize): self.goldenData[i] = block.data[i] def checkBroadcasted(self): + """It checks how many broadcasted samples are still missing in the network.""" zeros = 0 for i in range(self.blockSize * self.blockSize): if self.broadcasted.data[i] == 0: @@ -49,6 +55,7 @@ class Observer: return zeros def checkStatus(self, validators): + """It checks the status of how many expected and arrived samples globally.""" arrived = 0 expected = 0 for val in validators: diff --git a/DAS/results.py b/DAS/results.py index 174e5e9..e20cdec 100644 --- a/DAS/results.py +++ b/DAS/results.py @@ -5,15 +5,17 @@ from xml.dom import minidom from dicttoxml import dicttoxml class Result: - + """This class stores and process/store the results of a simulation.""" def __init__(self, shape): + """It initializes the instance with a specific shape.""" self.shape = shape self.blockAvailable = -1 self.tta = -1 self.missingVector = [] def populate(self, shape, missingVector): + """It populates part of the result data inside a vector.""" self.shape = shape self.missingVector = missingVector missingSamples = missingVector[-1] @@ -25,6 +27,7 @@ class Result: self.tta = -1 def dump(self, execID): + """It dumps the results of the simulation in an XML file.""" if not os.path.exists("results"): os.makedirs("results") if not os.path.exists("results/"+execID): diff --git a/DAS/shape.py b/DAS/shape.py index 84efc1e..2f99ebf 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -1,8 +1,10 @@ #!/bin/python3 class Shape: + """This class represents a set of parameters for a specific simulation.""" def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, run): + """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberValidators = numberValidators self.blockSize = blockSize diff --git a/DAS/simulator.py b/DAS/simulator.py index cc02a6b..3e6e496 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -9,9 +9,10 @@ from DAS.observer import * from DAS.validator import * class Simulator: - + """This class implements the main DAS simulator.""" def __init__(self, shape): + """It initializes the simulation with a set of parameters (shape).""" self.shape = shape self.format = {"entity": "Simulator"} self.result = Result(self.shape) @@ -22,6 +23,7 @@ class Simulator: self.glob = [] def initValidators(self): + """It initializes all the validators in the network.""" self.glob = Observer(self.logger, self.shape) self.glob.reset() self.validators = [] @@ -39,6 +41,7 @@ class Simulator: self.validators.append(val) def initNetwork(self): + """It initializes the simulated network.""" self.shape.netDegree = 6 rowChannels = [[] for i in range(self.shape.blockSize)] columnChannels = [[] for i in range(self.shape.blockSize)] @@ -73,6 +76,7 @@ class Simulator: val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, self.shape.blockSize)}) def initLogger(self): + """It initializes the logger.""" logger = logging.getLogger("DAS") logger.setLevel(self.logLevel) ch = logging.StreamHandler() @@ -83,6 +87,7 @@ class Simulator: def resetShape(self, shape): + """It resets the parameters of the simulation.""" self.shape = shape self.result = Result(self.shape) for val in self.validators: @@ -91,6 +96,7 @@ class Simulator: def run(self): + """It runs the main simulation until the block is available or it gets stucked.""" self.glob.checkRowsColumns(self.validators) self.validators[self.proposerID].broadcastBlock() arrived, expected = self.glob.checkStatus(self.validators) @@ -118,7 +124,7 @@ class Simulator: missingRate = missingSamples*100/expected self.logger.debug("step %d, missing %d of %d (%0.02f %%)" % (steps, missingSamples, expected, missingRate), extra=self.format) if missingSamples == oldMissingSamples: - #self.logger.info("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format) + self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format) missingVector.append(missingSamples) break elif missingSamples == 0: diff --git a/DAS/tools.py b/DAS/tools.py index 68469fe..fb40c71 100644 --- a/DAS/tools.py +++ b/DAS/tools.py @@ -5,23 +5,25 @@ import logging class CustomFormatter(): """This class defines the terminal output formatting.""" - blue = "\x1b[34;20m" - grey = "\x1b[38;20m" - yellow = "\x1b[33;20m" - red = "\x1b[31;20m" - bold_red = "\x1b[31;1m" - reset = "\x1b[0m" - format = "%(levelname)s : %(entity)s : %(message)s" - - FORMATS = { - logging.DEBUG: grey + format + reset, - logging.INFO: blue + format + reset, - logging.WARNING: yellow + format + reset, - logging.ERROR: red + format + reset, - logging.CRITICAL: bold_red + format + reset - } + def __init__(self): + """Initializes 5 different formats for logging with different colors.""" + self.blue = "\x1b[34;20m" + self.grey = "\x1b[38;20m" + self.yellow = "\x1b[33;20m" + self.red = "\x1b[31;20m" + self.bold_red = "\x1b[31;1m" + self.reset = "\x1b[0m" + self.reformat = "%(levelname)s : %(entity)s : %(message)s" + self.FORMATS = { + logging.DEBUG: self.grey + self.reformat + self.reset, + logging.INFO: self.blue + self.reformat + self.reset, + logging.WARNING: self.yellow + self.reformat + self.reset, + logging.ERROR: self.red + self.reformat + self.reset, + logging.CRITICAL: self.bold_red + self.reformat + self.reset + } def format(self, record): + """Returns the formatter with the format corresponding to record.""" log_fmt = self.FORMATS.get(record.levelno) formatter = logging.Formatter(log_fmt) return formatter.format(record) diff --git a/DAS/validator.py b/DAS/validator.py index 491f3da..6344925 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -7,24 +7,31 @@ from DAS.block import * from bitarray import bitarray from bitarray.util import zeros + class Neighbor: + """This class implements a node neighbor to monitor sent and received data.""" def __repr__(self): + """It returns the amount of sent and received data.""" return "%d:%d/%d" % (self.node.ID, self.sent.count(1), self.received.count(1)) def __init__(self, v, blockSize): + """It initializes the neighbor with the node and sets counters to zero.""" self.node = v self.receiving = zeros(blockSize) self.received = zeros(blockSize) self.sent = zeros(blockSize) -class Validator: +class Validator: + """This class implements a validator/node in the network.""" def __repr__(self): + """It returns the validator ID.""" return str(self.ID) def __init__(self, ID, amIproposer, logger, shape, rows, columns): + """It initializes the validator with the logger, shape and assigned rows/columns.""" self.shape = shape FORMAT = "%(levelname)s : %(entity)s : %(message)s" self.ID = ID @@ -60,6 +67,7 @@ class Validator: self.statsRxPerSlot = [] def logIDs(self): + """It logs the assigned rows and columns.""" if self.amIproposer == 1: self.logger.warning("I am a block proposer."% self.ID) else: @@ -67,14 +75,19 @@ class Validator: self.logger.debug("Selected columns: "+str(self.columnIDs), extra=self.format) def initBlock(self): - self.logger.debug("I am a block proposer.", extra=self.format) - self.block = Block(self.shape.blockSize) - self.block.fill() - #self.block.print() + """It initializes the block for the proposer.""" + if self.amIproposer == 1: + self.logger.debug("I am a block proposer.", extra=self.format) + self.block = Block(self.shape.blockSize) + self.block.fill() + #self.block.print() + else: + self.logger.warning("I am not a block proposer."% self.ID) def broadcastBlock(self): + """The block proposer broadcasts the block to all validators.""" if self.amIproposer == 0: - self.logger.error("I am NOT a block proposer", extra=self.format) + self.logger.warning("I am not a block proposer", extra=self.format) else: self.logger.debug("Broadcasting my block...", extra=self.format) order = [i for i in range(self.shape.blockSize * self.shape.blockSize)] @@ -95,12 +108,15 @@ class Validator: #broadcasted.print() def getColumn(self, index): + """It returns a given column.""" return self.block.getColumn(index) def getRow(self, index): + """It returns a given row.""" return self.block.getRow(index) def receiveColumn(self, id, column, src): + """It receives the given column if it has been assigned to it.""" if id in self.columnIDs: # register receive so that we are not sending back self.columnNeighbors[id][src].receiving |= column @@ -110,6 +126,7 @@ class Validator: pass def receiveRow(self, id, row, src): + """It receives the given row if it has been assigned to it.""" if id in self.rowIDs: # register receive so that we are not sending back self.rowNeighbors[id][src].receiving |= row @@ -120,6 +137,7 @@ class Validator: def receiveRowsColumns(self): + """It receives rows and columns.""" if self.amIproposer == 1: self.logger.error("I am a block proposer", extra=self.format) else: @@ -147,7 +165,9 @@ class Validator: for neigh in neighs.values(): neigh.received |= neigh.receiving neigh.receiving.setall(0) + def updateStats(self): + """It updates the stats related to sent and received data.""" self.logger.debug("Stats: tx %d, rx %d", self.statsTxInSlot, self.statsRxInSlot, extra=self.format) self.statsRxPerSlot.append(self.statsRxInSlot) self.statsTxPerSlot.append(self.statsTxInSlot) @@ -156,6 +176,7 @@ class Validator: def sendColumn(self, columnID): + """It sends any new sample in the given column.""" line = self.getColumn(columnID) if line.any(): self.logger.debug("col %d -> %s", columnID, self.columnNeighbors[columnID] , extra=self.format) @@ -169,6 +190,7 @@ class Validator: self.statsTxInSlot += toSend.count(1) def sendRow(self, rowID): + """It sends any new sample in the given row.""" line = self.getRow(rowID) if line.any(): self.logger.debug("row %d -> %s", rowID, self.rowNeighbors[rowID], extra=self.format) @@ -182,36 +204,43 @@ class Validator: self.statsTxInSlot += toSend.count(1) def sendRows(self): + """It sends all restored rows.""" self.logger.debug("Sending restored rows...", extra=self.format) for r in self.rowIDs: if self.changedRow[r]: self.sendRow(r) def sendColumns(self): + """It sends all restored columns.""" self.logger.debug("Sending restored columns...", extra=self.format) for c in self.columnIDs: if self.changedColumn[c]: self.sendColumn(c) def logRows(self): + """It logs the rows assigned to the validator.""" if self.logger.isEnabledFor(logging.DEBUG): for id in self.rowIDs: self.logger.debug("Row %d: %s", id, self.getRow(id), extra=self.format) def logColumns(self): + """It logs the columns assigned to the validator.""" if self.logger.isEnabledFor(logging.DEBUG): for id in self.columnIDs: self.logger.debug("Column %d: %s", id, self.getColumn(id), extra=self.format) def restoreRows(self): + """It restores the rows assigned to the validator, that can be repaired.""" for id in self.rowIDs: self.block.repairRow(id) def restoreColumns(self): + """It restores the columns assigned to the validator, that can be repaired.""" for id in self.columnIDs: self.block.repairColumn(id) def checkStatus(self): + """It checks how many expected/arrived samples are for each assigned row/column.""" arrived = 0 expected = 0 for id in self.columnIDs: