From 6334d1c779db3138a20e8432b742b0d4b208e21b Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 8 Feb 2023 20:10:26 +0100 Subject: [PATCH 01/89] Removing static variables --- DAS/block.py | 2 -- DAS/configuration.py | 1 - DAS/observer.py | 14 ++++++-------- DAS/results.py | 4 ---- DAS/shape.py | 6 ------ DAS/simulator.py | 13 +++++-------- DAS/validator.py | 5 ----- config.das => config.ini | 0 8 files changed, 11 insertions(+), 34 deletions(-) rename config.das => config.ini (100%) diff --git a/DAS/block.py b/DAS/block.py index aa5fe01..1cea8c0 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -6,8 +6,6 @@ from bitarray.util import zeros class Block: - blockSize = 0 - data = bitarray() def __init__(self, blockSize): self.blockSize = blockSize diff --git a/DAS/configuration.py b/DAS/configuration.py index a2e614e..d3647a4 100644 --- a/DAS/configuration.py +++ b/DAS/configuration.py @@ -4,7 +4,6 @@ import configparser class Configuration: - deterministic = 0 def __init__(self, fileName): diff --git a/DAS/observer.py b/DAS/observer.py index ad9052b..515fe09 100644 --- a/DAS/observer.py +++ b/DAS/observer.py @@ -4,18 +4,16 @@ from DAS.block import * class Observer: - block = [] - rows = [] - columns = [] - goldenData = [] - broadcasted = [] - config = [] - logger = [] - def __init__(self, logger, config): self.config = config self.format = {"entity": "Observer"} self.logger = logger + self.block = [] + self.rows = [] + self.columns = [] + self.goldenData = [] + self.broadcasted = [] + def reset(self): self.block = [0] * self.config.blockSize * self.config.blockSize diff --git a/DAS/results.py b/DAS/results.py index 48b6cbc..174e5e9 100644 --- a/DAS/results.py +++ b/DAS/results.py @@ -6,10 +6,6 @@ from dicttoxml import dicttoxml class Result: - shape = [] - missingVector = [] - blockAvailable = -1 - tta = -1 def __init__(self, shape): self.shape = shape diff --git a/DAS/shape.py b/DAS/shape.py index 243ae8e..84efc1e 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -1,12 +1,6 @@ #!/bin/python3 class Shape: - run = 0 - numberValidators = 0 - blockSize = 0 - failureRate = 0 - netDegree = 0 - chi = 0 def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, run): self.run = run diff --git a/DAS/simulator.py b/DAS/simulator.py index cba63c8..cc02a6b 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -10,19 +10,16 @@ from DAS.validator import * class Simulator: - proposerID = 0 - logLevel = logging.INFO - validators = [] - glob = [] - result = [] - shape = [] - logger = [] - format = {} def __init__(self, shape): self.shape = shape self.format = {"entity": "Simulator"} self.result = Result(self.shape) + self.validators = [] + self.logger = [] + self.logLevel = logging.INFO + self.proposerID = 0 + self.glob = [] def initValidators(self): self.glob = Observer(self.logger, self.shape) diff --git a/DAS/validator.py b/DAS/validator.py index 950fdea..491f3da 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -20,11 +20,6 @@ class Neighbor: class Validator: - ID = 0 - amIproposer = 0 - shape = [] - format = {} - logger = [] def __repr__(self): return str(self.ID) diff --git a/config.das b/config.ini similarity index 100% rename from config.das rename to config.ini From 17b97c37c094fab542bb94c566ffac2e7b29dd82 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 8 Feb 2023 22:31:51 +0100 Subject: [PATCH 02/89] Update gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b948985..13d2a0a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.swp *.pyc +results/* +myenv From 5c55fd7854a9462715cbd93534a92c5b1d7205b2 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 8 Feb 2023 22:33:16 +0100 Subject: [PATCH 03/89] Add documentation files --- doc/Makefile | 20 ++++++++++++++++ doc/conf.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/index.rst | 42 +++++++++++++++++++++++++++++++++ doc/make.bat | 35 ++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+) create mode 100644 doc/Makefile create mode 100644 doc/conf.py create mode 100644 doc/index.rst create mode 100644 doc/make.bat diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..7524d9c --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,64 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../DAS')) + + +# -- Project information ----------------------------------------------------- + +project = 'DAS simulator' +copyright = '2023, Leonardo A. Bautista-Gomez, Csaba Kiraly' +author = 'Leonardo A. Bautista-Gomez, Csaba Kiraly' + +# The full version, including alpha/beta/rc tags +release = '1' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc' +] + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'myenv'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for autodoc ------------------------------------------------- + +autodoc_mock_imports = ["django", "dicttoxml", "bitarray", "DAS", "networkx"] + + + diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000..f6b5fc9 --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,42 @@ +.. DAS simulator documentation master file, created by + sphinx-quickstart on Wed Feb 8 20:56:44 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to DAS simulator's documentation! +========================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. automodule:: block + :members: + +.. automodule:: configuration + :members: + +.. automodule:: observer + :members: + +.. automodule:: results + :members: + +.. automodule:: shape + :members: + +.. automodule:: simulator + :members: + +.. automodule:: tools + :members: + +.. automodule:: validator + :members: + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 0000000..153be5e --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd From cf780b3ca3ca533c841d2f01ad39431cc6e0ba4b Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 8 Feb 2023 22:45:01 +0100 Subject: [PATCH 04/89] Starting code documentation --- DAS/block.py | 2 +- DAS/tools.py | 3 ++- doc/index.rst | 12 +++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/DAS/block.py b/DAS/block.py index 1cea8c0..a22b70c 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -5,7 +5,7 @@ from bitarray import bitarray from bitarray.util import zeros class Block: - + """This class represents a block in the Ethereum blockchain""" def __init__(self, blockSize): self.blockSize = blockSize diff --git a/DAS/tools.py b/DAS/tools.py index b9e4a8e..68469fe 100644 --- a/DAS/tools.py +++ b/DAS/tools.py @@ -2,7 +2,8 @@ import logging -class CustomFormatter(logging.Formatter): +class CustomFormatter(): + """This class defines the terminal output formatting.""" blue = "\x1b[34;20m" grey = "\x1b[38;20m" diff --git a/doc/index.rst b/doc/index.rst index f6b5fc9..e253201 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -6,6 +6,13 @@ Welcome to DAS simulator's documentation! ========================================= +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + .. toctree:: :maxdepth: 2 :caption: Contents: @@ -34,9 +41,4 @@ Welcome to DAS simulator's documentation! .. automodule:: validator :members: -Indices and tables -================== -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` From 9888d96e438f8785e96e8be6e489b93b5577f97b Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 8 Feb 2023 22:46:12 +0100 Subject: [PATCH 05/89] Update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 13d2a0a..1e5dc45 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.pyc results/* myenv +doc/_build From 09569422ab4fce1c57a1a972beac37ed68984b29 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 15 Feb 2023 15:06:42 +0100 Subject: [PATCH 06/89] Documenting the DAS package for automatic generation with sphinx. First version. --- DAS/block.py | 12 +++++++++++- DAS/configuration.py | 3 ++- DAS/observer.py | 7 +++++++ DAS/results.py | 5 ++++- DAS/shape.py | 2 ++ DAS/simulator.py | 10 ++++++++-- DAS/tools.py | 32 +++++++++++++++++--------------- DAS/validator.py | 41 +++++++++++++++++++++++++++++++++++------ 8 files changed, 86 insertions(+), 26 deletions(-) diff --git a/DAS/block.py b/DAS/block.py index a22b70c..693d7b6 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -5,41 +5,51 @@ from bitarray import bitarray from bitarray.util import zeros class Block: - """This class represents a block in the Ethereum blockchain""" + """This class represents a block in the Ethereum blockchain.""" def __init__(self, blockSize): + """Initialize the block with a data array of blocksize^2 zeros.""" self.blockSize = blockSize self.data = zeros(self.blockSize*self.blockSize) def fill(self): + """It fills the block data with ones.""" self.data.setall(1) def merge(self, merged): + """It merges (OR) the existing block with the received one.""" self.data |= merged.data def getColumn(self, columnID): + """It returns the block column corresponding to columnID.""" return self.data[columnID::self.blockSize] def mergeColumn(self, columnID, column): + """It merges (OR) the existing column with the received one.""" self.data[columnID::self.blockSize] |= column def repairColumn(self, id): + """It repairs the entire column if it has at least blockSize/2 ones.""" success = self.data[id::self.blockSize].count(1) if success >= self.blockSize/2: self.data[id::self.blockSize] = 1 def getRow(self, rowID): + """It returns the block row corresponding to rowID.""" return self.data[rowID*self.blockSize:(rowID+1)*self.blockSize] def mergeRow(self, rowID, row): + """It merges (OR) the existing row with the received one.""" self.data[rowID*self.blockSize:(rowID+1)*self.blockSize] |= row def repairRow(self, id): + """It repairs the entire row if it has at least blockSize/2 ones.""" success = self.data[id*self.blockSize:(id+1)*self.blockSize].count(1) if success >= self.blockSize/2: self.data[id*self.blockSize:(id+1)*self.blockSize] = 1 def print(self): + """It prints the block in the terminal (outside of the logger rules)).""" dash = "-" * (self.blockSize+2) print(dash) for i in range(self.blockSize): diff --git a/DAS/configuration.py b/DAS/configuration.py index d3647a4..c840366 100644 --- a/DAS/configuration.py +++ b/DAS/configuration.py @@ -3,9 +3,10 @@ import configparser class Configuration: - + """This class stores all the configuration parameters for the given run.""" def __init__(self, fileName): + """It initializes the configuration based on the given configuration.""" config = configparser.RawConfigParser() config.read(fileName) diff --git a/DAS/observer.py b/DAS/observer.py index 515fe09..af5866a 100644 --- a/DAS/observer.py +++ b/DAS/observer.py @@ -3,8 +3,10 @@ from DAS.block import * class Observer: + """This class gathers global data from the simulation, like an 'all-seen god'.""" def __init__(self, logger, config): + """It initializes the observer with a logger and given configuration.""" self.config = config self.format = {"entity": "Observer"} self.logger = logger @@ -16,6 +18,7 @@ class Observer: def reset(self): + """It resets all the gathered data to zeros.""" self.block = [0] * self.config.blockSize * self.config.blockSize self.goldenData = [0] * self.config.blockSize * self.config.blockSize self.rows = [0] * self.config.blockSize @@ -23,6 +26,7 @@ class Observer: self.broadcasted = Block(self.config.blockSize) def checkRowsColumns(self, validators): + """It checks how many validators have been assigned to each row and column.""" for val in validators: if val.amIproposer == 0: for r in val.rowIDs: @@ -36,10 +40,12 @@ class Observer: self.logger.warning("There is a row/column that has not been assigned", extra=self.format) def setGoldenData(self, block): + """Stores the original real data to compare it with future situations.""" for i in range(self.config.blockSize*self.config.blockSize): self.goldenData[i] = block.data[i] def checkBroadcasted(self): + """It checks how many broadcasted samples are still missing in the network.""" zeros = 0 for i in range(self.blockSize * self.blockSize): if self.broadcasted.data[i] == 0: @@ -49,6 +55,7 @@ class Observer: return zeros def checkStatus(self, validators): + """It checks the status of how many expected and arrived samples globally.""" arrived = 0 expected = 0 for val in validators: diff --git a/DAS/results.py b/DAS/results.py index 174e5e9..e20cdec 100644 --- a/DAS/results.py +++ b/DAS/results.py @@ -5,15 +5,17 @@ from xml.dom import minidom from dicttoxml import dicttoxml class Result: - + """This class stores and process/store the results of a simulation.""" def __init__(self, shape): + """It initializes the instance with a specific shape.""" self.shape = shape self.blockAvailable = -1 self.tta = -1 self.missingVector = [] def populate(self, shape, missingVector): + """It populates part of the result data inside a vector.""" self.shape = shape self.missingVector = missingVector missingSamples = missingVector[-1] @@ -25,6 +27,7 @@ class Result: self.tta = -1 def dump(self, execID): + """It dumps the results of the simulation in an XML file.""" if not os.path.exists("results"): os.makedirs("results") if not os.path.exists("results/"+execID): diff --git a/DAS/shape.py b/DAS/shape.py index 84efc1e..2f99ebf 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -1,8 +1,10 @@ #!/bin/python3 class Shape: + """This class represents a set of parameters for a specific simulation.""" def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, run): + """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberValidators = numberValidators self.blockSize = blockSize diff --git a/DAS/simulator.py b/DAS/simulator.py index cc02a6b..3e6e496 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -9,9 +9,10 @@ from DAS.observer import * from DAS.validator import * class Simulator: - + """This class implements the main DAS simulator.""" def __init__(self, shape): + """It initializes the simulation with a set of parameters (shape).""" self.shape = shape self.format = {"entity": "Simulator"} self.result = Result(self.shape) @@ -22,6 +23,7 @@ class Simulator: self.glob = [] def initValidators(self): + """It initializes all the validators in the network.""" self.glob = Observer(self.logger, self.shape) self.glob.reset() self.validators = [] @@ -39,6 +41,7 @@ class Simulator: self.validators.append(val) def initNetwork(self): + """It initializes the simulated network.""" self.shape.netDegree = 6 rowChannels = [[] for i in range(self.shape.blockSize)] columnChannels = [[] for i in range(self.shape.blockSize)] @@ -73,6 +76,7 @@ class Simulator: val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, self.shape.blockSize)}) def initLogger(self): + """It initializes the logger.""" logger = logging.getLogger("DAS") logger.setLevel(self.logLevel) ch = logging.StreamHandler() @@ -83,6 +87,7 @@ class Simulator: def resetShape(self, shape): + """It resets the parameters of the simulation.""" self.shape = shape self.result = Result(self.shape) for val in self.validators: @@ -91,6 +96,7 @@ class Simulator: def run(self): + """It runs the main simulation until the block is available or it gets stucked.""" self.glob.checkRowsColumns(self.validators) self.validators[self.proposerID].broadcastBlock() arrived, expected = self.glob.checkStatus(self.validators) @@ -118,7 +124,7 @@ class Simulator: missingRate = missingSamples*100/expected self.logger.debug("step %d, missing %d of %d (%0.02f %%)" % (steps, missingSamples, expected, missingRate), extra=self.format) if missingSamples == oldMissingSamples: - #self.logger.info("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format) + self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format) missingVector.append(missingSamples) break elif missingSamples == 0: diff --git a/DAS/tools.py b/DAS/tools.py index 68469fe..fb40c71 100644 --- a/DAS/tools.py +++ b/DAS/tools.py @@ -5,23 +5,25 @@ import logging class CustomFormatter(): """This class defines the terminal output formatting.""" - blue = "\x1b[34;20m" - grey = "\x1b[38;20m" - yellow = "\x1b[33;20m" - red = "\x1b[31;20m" - bold_red = "\x1b[31;1m" - reset = "\x1b[0m" - format = "%(levelname)s : %(entity)s : %(message)s" - - FORMATS = { - logging.DEBUG: grey + format + reset, - logging.INFO: blue + format + reset, - logging.WARNING: yellow + format + reset, - logging.ERROR: red + format + reset, - logging.CRITICAL: bold_red + format + reset - } + def __init__(self): + """Initializes 5 different formats for logging with different colors.""" + self.blue = "\x1b[34;20m" + self.grey = "\x1b[38;20m" + self.yellow = "\x1b[33;20m" + self.red = "\x1b[31;20m" + self.bold_red = "\x1b[31;1m" + self.reset = "\x1b[0m" + self.reformat = "%(levelname)s : %(entity)s : %(message)s" + self.FORMATS = { + logging.DEBUG: self.grey + self.reformat + self.reset, + logging.INFO: self.blue + self.reformat + self.reset, + logging.WARNING: self.yellow + self.reformat + self.reset, + logging.ERROR: self.red + self.reformat + self.reset, + logging.CRITICAL: self.bold_red + self.reformat + self.reset + } def format(self, record): + """Returns the formatter with the format corresponding to record.""" log_fmt = self.FORMATS.get(record.levelno) formatter = logging.Formatter(log_fmt) return formatter.format(record) diff --git a/DAS/validator.py b/DAS/validator.py index 491f3da..6344925 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -7,24 +7,31 @@ from DAS.block import * from bitarray import bitarray from bitarray.util import zeros + class Neighbor: + """This class implements a node neighbor to monitor sent and received data.""" def __repr__(self): + """It returns the amount of sent and received data.""" return "%d:%d/%d" % (self.node.ID, self.sent.count(1), self.received.count(1)) def __init__(self, v, blockSize): + """It initializes the neighbor with the node and sets counters to zero.""" self.node = v self.receiving = zeros(blockSize) self.received = zeros(blockSize) self.sent = zeros(blockSize) -class Validator: +class Validator: + """This class implements a validator/node in the network.""" def __repr__(self): + """It returns the validator ID.""" return str(self.ID) def __init__(self, ID, amIproposer, logger, shape, rows, columns): + """It initializes the validator with the logger, shape and assigned rows/columns.""" self.shape = shape FORMAT = "%(levelname)s : %(entity)s : %(message)s" self.ID = ID @@ -60,6 +67,7 @@ class Validator: self.statsRxPerSlot = [] def logIDs(self): + """It logs the assigned rows and columns.""" if self.amIproposer == 1: self.logger.warning("I am a block proposer."% self.ID) else: @@ -67,14 +75,19 @@ class Validator: self.logger.debug("Selected columns: "+str(self.columnIDs), extra=self.format) def initBlock(self): - self.logger.debug("I am a block proposer.", extra=self.format) - self.block = Block(self.shape.blockSize) - self.block.fill() - #self.block.print() + """It initializes the block for the proposer.""" + if self.amIproposer == 1: + self.logger.debug("I am a block proposer.", extra=self.format) + self.block = Block(self.shape.blockSize) + self.block.fill() + #self.block.print() + else: + self.logger.warning("I am not a block proposer."% self.ID) def broadcastBlock(self): + """The block proposer broadcasts the block to all validators.""" if self.amIproposer == 0: - self.logger.error("I am NOT a block proposer", extra=self.format) + self.logger.warning("I am not a block proposer", extra=self.format) else: self.logger.debug("Broadcasting my block...", extra=self.format) order = [i for i in range(self.shape.blockSize * self.shape.blockSize)] @@ -95,12 +108,15 @@ class Validator: #broadcasted.print() def getColumn(self, index): + """It returns a given column.""" return self.block.getColumn(index) def getRow(self, index): + """It returns a given row.""" return self.block.getRow(index) def receiveColumn(self, id, column, src): + """It receives the given column if it has been assigned to it.""" if id in self.columnIDs: # register receive so that we are not sending back self.columnNeighbors[id][src].receiving |= column @@ -110,6 +126,7 @@ class Validator: pass def receiveRow(self, id, row, src): + """It receives the given row if it has been assigned to it.""" if id in self.rowIDs: # register receive so that we are not sending back self.rowNeighbors[id][src].receiving |= row @@ -120,6 +137,7 @@ class Validator: def receiveRowsColumns(self): + """It receives rows and columns.""" if self.amIproposer == 1: self.logger.error("I am a block proposer", extra=self.format) else: @@ -147,7 +165,9 @@ class Validator: for neigh in neighs.values(): neigh.received |= neigh.receiving neigh.receiving.setall(0) + def updateStats(self): + """It updates the stats related to sent and received data.""" self.logger.debug("Stats: tx %d, rx %d", self.statsTxInSlot, self.statsRxInSlot, extra=self.format) self.statsRxPerSlot.append(self.statsRxInSlot) self.statsTxPerSlot.append(self.statsTxInSlot) @@ -156,6 +176,7 @@ class Validator: def sendColumn(self, columnID): + """It sends any new sample in the given column.""" line = self.getColumn(columnID) if line.any(): self.logger.debug("col %d -> %s", columnID, self.columnNeighbors[columnID] , extra=self.format) @@ -169,6 +190,7 @@ class Validator: self.statsTxInSlot += toSend.count(1) def sendRow(self, rowID): + """It sends any new sample in the given row.""" line = self.getRow(rowID) if line.any(): self.logger.debug("row %d -> %s", rowID, self.rowNeighbors[rowID], extra=self.format) @@ -182,36 +204,43 @@ class Validator: self.statsTxInSlot += toSend.count(1) def sendRows(self): + """It sends all restored rows.""" self.logger.debug("Sending restored rows...", extra=self.format) for r in self.rowIDs: if self.changedRow[r]: self.sendRow(r) def sendColumns(self): + """It sends all restored columns.""" self.logger.debug("Sending restored columns...", extra=self.format) for c in self.columnIDs: if self.changedColumn[c]: self.sendColumn(c) def logRows(self): + """It logs the rows assigned to the validator.""" if self.logger.isEnabledFor(logging.DEBUG): for id in self.rowIDs: self.logger.debug("Row %d: %s", id, self.getRow(id), extra=self.format) def logColumns(self): + """It logs the columns assigned to the validator.""" if self.logger.isEnabledFor(logging.DEBUG): for id in self.columnIDs: self.logger.debug("Column %d: %s", id, self.getColumn(id), extra=self.format) def restoreRows(self): + """It restores the rows assigned to the validator, that can be repaired.""" for id in self.rowIDs: self.block.repairRow(id) def restoreColumns(self): + """It restores the columns assigned to the validator, that can be repaired.""" for id in self.columnIDs: self.block.repairColumn(id) def checkStatus(self): + """It checks how many expected/arrived samples are for each assigned row/column.""" arrived = 0 expected = 0 for id in self.columnIDs: From 22e4c89989eb42df6fe5eea244d68a3a45b5dd89 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 16 Feb 2023 17:32:57 +0100 Subject: [PATCH 07/89] fix: column IDs matching row IDs Fixes a simple copy paste error. Signed-off-by: Csaba Kiraly --- DAS/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DAS/validator.py b/DAS/validator.py index 6344925..3270c38 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -50,7 +50,7 @@ class Validator: self.columnIDs = range(shape.blockSize) else: self.rowIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] - self.columnIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] + self.columnIDs = columns[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] #if shape.deterministic: # random.seed(self.ID) #self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi) From 8c9ddcca5598bb312db3b84b5fd7c5e53cc086dc Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 02:51:47 +0100 Subject: [PATCH 08/89] add debug logging of topology Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DAS/simulator.py b/DAS/simulator.py index 3e6e496..33d7794 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -75,6 +75,11 @@ class Simulator: val1.columnNeighbors[id].update({val2.ID : Neighbor(val2, self.shape.blockSize)}) val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, self.shape.blockSize)}) + if self.logger.isEnabledFor(logging.DEBUG): + for i in range(1, self.shape.numberValidators): + self.logger.debug("Val %d : rowN %s", i, self.validators[i].rowNeighbors, extra=self.format) + self.logger.debug("Val %d : colN %s", i, self.validators[i].columnNeighbors, extra=self.format) + def initLogger(self): """It initializes the logger.""" logger = logging.getLogger("DAS") From 2775d180f11be032047fdc709555a2549ffc68c7 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 16 Feb 2023 17:33:31 +0100 Subject: [PATCH 09/89] debug log neighborhood of producer as well Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 33d7794..07cd5d4 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -76,7 +76,7 @@ class Simulator: val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, self.shape.blockSize)}) if self.logger.isEnabledFor(logging.DEBUG): - for i in range(1, self.shape.numberValidators): + for i in range(0, self.shape.numberValidators): self.logger.debug("Val %d : rowN %s", i, self.validators[i].rowNeighbors, extra=self.format) self.logger.debug("Val %d : colN %s", i, self.validators[i].columnNeighbors, extra=self.format) From 91a4b489479825aa95885d0b5cb6b787728e59b0 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 3 Feb 2023 23:05:37 +0100 Subject: [PATCH 10/89] log number of steps as well Signed-off-by: Csaba Kiraly --- study.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/study.py b/study.py index 68663b8..25e8735 100644 --- a/study.py +++ b/study.py @@ -37,7 +37,7 @@ def study(): sim.initValidators() sim.initNetwork() result = sim.run() - sim.logger.info("Shape: %s ... Block Available: %d" % (str(sim.shape.__dict__), result.blockAvailable), extra=sim.format) + sim.logger.info("Shape: %s ... Block Available: %d in %d steps" % (str(sim.shape.__dict__), result.blockAvailable, len(result.missingVector)), extra=sim.format) results.append(copy.deepcopy(result)) simCnt += 1 From e5c657e31e6642cca88af4f393331ff9a0d12132 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 02:55:06 +0100 Subject: [PATCH 11/89] more debug logging Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DAS/simulator.py b/DAS/simulator.py index 3e6e496..a8cec35 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -106,14 +106,18 @@ class Simulator: while(True): missingVector.append(missingSamples) oldMissingSamples = missingSamples + self.logger.debug("PHASE SEND %d" % steps, extra=self.format) for i in range(0,self.shape.numberValidators): self.validators[i].sendRows() self.validators[i].sendColumns() + self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format) for i in range(1,self.shape.numberValidators): self.validators[i].receiveRowsColumns() + self.logger.debug("PHASE RESTORE %d" % steps, extra=self.format) for i in range(1,self.shape.numberValidators): self.validators[i].restoreRows() self.validators[i].restoreColumns() + self.logger.debug("PHASE LOG %d" % steps, extra=self.format) for i in range(0,self.shape.numberValidators): self.validators[i].logRows() self.validators[i].logColumns() From 763ebfe1369891d6ba7bc03197bffd8d1fe97561 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 02:50:03 +0100 Subject: [PATCH 12/89] set up complete graph if n<=d If the number of nodes in a channel is smaller or equal than the requested degree, a fully connected graph is used. For n>d, a random d-regular graph is set up. (For n=d+1, the two are the same.) Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 3e6e496..5553087 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -53,9 +53,14 @@ class Simulator: for id in range(self.shape.blockSize): - if (len(rowChannels[id]) < self.shape.netDegree): - self.logger.error("Graph degree higher than %d" % len(rowChannels[id]), extra=self.format) - G = nx.random_regular_graph(self.shape.netDegree, len(rowChannels[id])) + # If the number of nodes in a channel is smaller or equal to the + # requested degree, a fully connected graph is used. For n>d, a random + # d-regular graph is set up. (For n=d+1, the two are the same.) + if (len(rowChannels[id]) <= self.shape.netDegree): + self.logger.debug("Graph fully connected with degree %d !" % (len(rowChannels[id]) - 1), extra=self.format) + G = nx.complete_graph(len(rowChannels[id])) + else: + G = nx.random_regular_graph(self.shape.netDegree, len(rowChannels[id])) if not nx.is_connected(G): self.logger.error("Graph not connected for row %d !" % id, extra=self.format) for u, v in G.edges: @@ -64,9 +69,11 @@ class Simulator: val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, self.shape.blockSize)}) val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, self.shape.blockSize)}) - if (len(columnChannels[id]) < self.shape.netDegree): - self.logger.error("Graph degree higher than %d" % len(columnChannels[id]), extra=self.format) - G = nx.random_regular_graph(self.shape.netDegree, len(columnChannels[id])) + if (len(columnChannels[id]) <= self.shape.netDegree): + self.logger.debug("Graph fully connected with degree %d !" % (len(columnChannels[id]) - 1), extra=self.format) + G = nx.complete_graph(len(columnChannels[id])) + else: + G = nx.random_regular_graph(self.shape.netDegree, len(columnChannels[id])) if not nx.is_connected(G): self.logger.error("Graph not connected for column %d !" % id, extra=self.format) for u, v in G.edges: From 6e42055cb9c768de5653d192fc4a973411f51d4e Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Thu, 23 Feb 2023 12:16:43 +0100 Subject: [PATCH 13/89] Add some more configuration parameters and some more testing --- DAS/configuration.py | 90 +++++++++++++++++++++++++++++++------------- DAS/simulator.py | 4 +- config.ini | 4 +- study.py | 2 +- 4 files changed, 69 insertions(+), 31 deletions(-) diff --git a/DAS/configuration.py b/DAS/configuration.py index c840366..2ff9d68 100644 --- a/DAS/configuration.py +++ b/DAS/configuration.py @@ -1,6 +1,6 @@ #!/bin/python3 -import configparser +import configparser, logging, sys class Configuration: """This class stores all the configuration parameters for the given run.""" @@ -11,39 +11,75 @@ class Configuration: config = configparser.RawConfigParser() config.read(fileName) - self.nvStart = int(config.get("Simulation Space", "numberValidatorStart")) - self.nvStop = int(config.get("Simulation Space", "numberValidatorStop")) - self.nvStep = int(config.get("Simulation Space", "numberValidatorStep")) + try: + self.nvStart = int(config.get("Simulation Space", "numberValidatorStart")) + self.nvStop = int(config.get("Simulation Space", "numberValidatorStop")) + self.nvStep = int(config.get("Simulation Space", "numberValidatorStep")) - self.blockSizeStart = int(config.get("Simulation Space", "blockSizeStart")) - self.blockSizeStop = int(config.get("Simulation Space", "blockSizeStop")) - self.blockSizeStep = int(config.get("Simulation Space", "blockSizeStep")) + self.blockSizeStart = int(config.get("Simulation Space", "blockSizeStart")) + self.blockSizeStop = int(config.get("Simulation Space", "blockSizeStop")) + self.blockSizeStep = int(config.get("Simulation Space", "blockSizeStep")) - self.netDegreeStart = int(config.get("Simulation Space", "netDegreeStart")) - self.netDegreeStop = int(config.get("Simulation Space", "netDegreeStop")) - self.netDegreeStep = int(config.get("Simulation Space", "netDegreeStep")) + self.netDegreeStart = int(config.get("Simulation Space", "netDegreeStart")) + self.netDegreeStop = int(config.get("Simulation Space", "netDegreeStop")) + self.netDegreeStep = int(config.get("Simulation Space", "netDegreeStep")) - self.failureRateStart = int(config.get("Simulation Space", "failureRateStart")) - self.failureRateStop = int(config.get("Simulation Space", "failureRateStop")) - self.failureRateStep = int(config.get("Simulation Space", "failureRateStep")) + self.failureRateStart = int(config.get("Simulation Space", "failureRateStart")) + self.failureRateStop = int(config.get("Simulation Space", "failureRateStop")) + self.failureRateStep = int(config.get("Simulation Space", "failureRateStep")) - self.chiStart = int(config.get("Simulation Space", "chiStart")) - self.chiStop = int(config.get("Simulation Space", "chiStop")) - self.chiStep = int(config.get("Simulation Space", "chiStep")) + self.chiStart = int(config.get("Simulation Space", "chiStart")) + self.chiStop = int(config.get("Simulation Space", "chiStop")) + self.chiStep = int(config.get("Simulation Space", "chiStep")) + except: + sys.exit("Configuration Error: It seems some of the [Simulation Space] parameters are missing. Cannot continue :( ") - self.numberRuns = int(config.get("Advanced", "numberRuns")) - self.deterministic = config.get("Advanced", "deterministic") - self.dumpXML = config.get("Advanced", "dumpXML") - if self.nvStop < (self.blockSizeStart*4): - print("ERROR: The number of validators cannot be lower than the block size * 4") - exit(1) - if self.chiStart < 1: - print("Chi has to be greater than 0") - exit(1) + + try: + self.numberRuns = int(config.get("Advanced", "numberRuns")) + self.deterministic = config.get("Advanced", "deterministic") + self.dumpXML = config.get("Advanced", "dumpXML") + self.logLevel = config.get("Advanced", "logLevel") + self.visualization = config.get("Advanced", "visualization") + except: + sys.exit("Configuration Error: It seems some of the [Advanced] parameters are missing. Cannot continue :( ") + self.test() + + def test(self): + + print("Testing configuration...") + if self.logLevel == "INFO": + self.logLevel = logging.INFO + elif self.logLevel == "DEBUG": + self.logLevel = logging.DEBUG + else: + self.logLevel = logging.INFO + + if self.nvStart >= self.nvStop: + sys.exit("Configuration Error: numberValidatorStart has to be smaller than numberValidatorStop") + + if self.failureRateStart >= self.failureRateStop: + sys.exit("Configuration Error: failureRateStart has to be smaller than failureRateStop") + + if self.blockSizeStart >= self.blockSizeStop: + sys.exit("Configuration Error: blockSizeStart has to be smaller than blockSizeStop") + + if self.netDegreeStart >= self.netDegreeStop: + sys.exit("Configuration Error: netDegreeStart has to be smaller than netDegreeStop") + + if self.chiStart >= self.chiStop: + sys.exit("Configuration Error: chiStart has to be smaller than chiStop") + + + if self.nvStart < self.blockSizeStop: + sys.exit("Configuration Error: numberValidatorStart hast to be larger than blockSizeStop.") + + if self.chiStart < 2: + sys.exit("Configuration Error: Chi has to be greater than 1.") + if self.chiStop > self.blockSizeStart: - print("Chi (%d) has to be smaller or equal to block the size (%d)" % (self.chiStop, self.blockSizeStart)) - exit(1) + sys.exit("Configuration Error: Chi (%d) has to be smaller or equal to block the size (%d)" % (self.chiStop, self.blockSizeStart)) diff --git a/DAS/simulator.py b/DAS/simulator.py index 267fa82..b0902d9 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -11,14 +11,14 @@ from DAS.validator import * class Simulator: """This class implements the main DAS simulator.""" - def __init__(self, shape): + def __init__(self, shape, config): """It initializes the simulation with a set of parameters (shape).""" self.shape = shape self.format = {"entity": "Simulator"} self.result = Result(self.shape) self.validators = [] self.logger = [] - self.logLevel = logging.INFO + self.logLevel = config.logLevel self.proposerID = 0 self.glob = [] diff --git a/config.ini b/config.ini index 052f754..12bd749 100644 --- a/config.ini +++ b/config.ini @@ -14,7 +14,7 @@ blockSizeStep = 16 netDegreeStart = 6 netDegreeStop = 8 -netDegreeStep = 1 +netDegreeStep = 2 chiStart = 4 chiStop = 8 @@ -26,3 +26,5 @@ chiStep = 2 deterministic = 0 numberRuns = 2 dumpXML = 1 +visualization = 1 +logLevel = INFO diff --git a/study.py b/study.py index 25e8735..0df9365 100644 --- a/study.py +++ b/study.py @@ -11,7 +11,7 @@ def study(): config = Configuration(sys.argv[1]) shape = Shape(0, 0, 0, 0, 0, 0) - sim = Simulator(shape) + sim = Simulator(shape, config) sim.initLogger() results = [] simCnt = 0 From bb8d05257bfe80c1ec2a97dbd6b4bd5e132680d0 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 26 Jan 2023 14:29:12 +0100 Subject: [PATCH 14/89] WIP: initial implementation of uplink bandwidth limit - approximate: BW is not handled strict, entire rows are sent and can go over limit - WIP: work in progress implementation Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 3 +- DAS/validator.py | 126 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 88 insertions(+), 41 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index b0902d9..5b60413 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -120,8 +120,7 @@ class Simulator: oldMissingSamples = missingSamples self.logger.debug("PHASE SEND %d" % steps, extra=self.format) for i in range(0,self.shape.numberValidators): - self.validators[i].sendRows() - self.validators[i].sendColumns() + self.validators[i].send() self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format) for i in range(1,self.shape.numberValidators): self.validators[i].receiveRowsColumns() diff --git a/DAS/validator.py b/DAS/validator.py index 3270c38..89ca316 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -7,6 +7,17 @@ from DAS.block import * from bitarray import bitarray from bitarray.util import zeros +def shuffled(lis): + # based on https://stackoverflow.com/a/60342323 + for index in random.sample(range(len(lis)), len(lis)): + yield lis[index] + +class NextToSend: + def __init__(self, neigh, toSend, id, dim): + self.neigh = neigh + self.toSend = toSend + self.id = id + self.dim = dim class Neighbor: """This class implements a node neighbor to monitor sent and received data.""" @@ -55,8 +66,6 @@ class Validator: # random.seed(self.ID) #self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi) #self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi) - self.changedRow = {id:False for id in self.rowIDs} - self.changedColumn = {id:False for id in self.columnIDs} self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) @@ -66,6 +75,13 @@ class Validator: self.statsRxInSlot = 0 self.statsRxPerSlot = [] + # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) + # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 + # TODO: this should be a parameter + self.bwUplink = 1100 if not self.amIproposer else 22000 # approx. 100Mbps and 2Gbps + + self.sched = self.nextToSend() + def logIDs(self): """It logs the assigned rows and columns.""" if self.amIproposer == 1: @@ -99,9 +115,6 @@ class Validator: else: self.block.data[i] = 0 - self.changedRow = {id:True for id in self.rowIDs} - self.changedColumn = {id:True for id in self.columnIDs} - nbFailures = self.block.data.count(0) measuredFailureRate = nbFailures * 100 / (self.shape.blockSize * self.shape.blockSize) self.logger.debug("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format) @@ -144,16 +157,6 @@ class Validator: self.logger.debug("Receiving the data...", extra=self.format) #self.logger.debug("%s -> %s", self.block.data, self.receivedBlock.data, extra=self.format) - self.changedRow = { id: - self.getRow(id) != self.receivedBlock.getRow(id) - for id in self.rowIDs - } - - self.changedColumn = { id: - self.getColumn(id) != self.receivedBlock.getColumn(id) - for id in self.columnIDs - } - self.block.merge(self.receivedBlock) for neighs in self.rowNeighbors.values(): @@ -175,47 +178,92 @@ class Validator: self.statsTxInSlot = 0 - def sendColumn(self, columnID): - """It sends any new sample in the given column.""" + def nextColumnToSend(self, columnID): line = self.getColumn(columnID) if line.any(): self.logger.debug("col %d -> %s", columnID, self.columnNeighbors[columnID] , extra=self.format) - for n in self.columnNeighbors[columnID].values(): + for n in shuffled(list(self.columnNeighbors[columnID].values())): # if there is anything new to send, send it toSend = line & ~n.sent & ~n.received if (toSend).any(): - n.sent |= toSend; - n.node.receiveColumn(columnID, toSend, self.ID) - self.statsTxInSlot += toSend.count(1) + yield NextToSend(n, toSend, columnID, 1) - def sendRow(self, rowID): - """It sends any new sample in the given row.""" + def nextRowToSend(self, rowID): line = self.getRow(rowID) if line.any(): self.logger.debug("row %d -> %s", rowID, self.rowNeighbors[rowID], extra=self.format) - for n in self.rowNeighbors[rowID].values(): + for n in shuffled(list(self.rowNeighbors[rowID].values())): # if there is anything new to send, send it toSend = line & ~n.sent & ~n.received if (toSend).any(): - n.sent |= toSend; - n.node.receiveRow(rowID, toSend, self.ID) - self.statsTxInSlot += toSend.count(1) + yield NextToSend(n, toSend, rowID, 0) - def sendRows(self): - """It sends all restored rows.""" - self.logger.debug("Sending restored rows...", extra=self.format) - for r in self.rowIDs: - if self.changedRow[r]: - self.sendRow(r) + def nextToSend(self): + """ Send scheduler as a generator function + + Yields next segment(s) to send when asked for it. + Generates an infinite flow, returning with exit only when + there is nothing more to send. - def sendColumns(self): - """It sends all restored columns.""" - self.logger.debug("Sending restored columns...", extra=self.format) - for c in self.columnIDs: - if self.changedColumn[c]: - self.sendColumn(c) + Generates a randomized order of columns and rows, sending to one neighbor + at each before sending to another neighbor. + Generates a new randomized ordering once all columns, rows, and neighbors + are processed once. + """ + + while True: + perLine = [] + for c in self.columnIDs: + perLine.append(self.nextColumnToSend(c)) + + for r in self.rowIDs: + perLine.append(self.nextRowToSend(r)) + + count = 0 + random.shuffle(perLine) + while (perLine): + for g in perLine.copy(): # we need a shallow copy to allow remove + n = next(g, None) + if not n: + perLine.remove(g) + continue + count += 1 + yield n + + # return if there is nothing more to send + if not count: + return + + def send(self): + """ Send as much as we can in the timeslot, limited by bwUplink + """ + + for n in self.sched: + neigh = n.neigh + toSend = n.toSend + id = n.id + dim = n.dim + + neigh.sent |= toSend; + if dim == 0: + neigh.node.receiveRow(id, toSend, self.ID) + else: + neigh.node.receiveColumn(id, toSend, self.ID) + + sent = toSend.count(1) + self.statsTxInSlot += sent + self.logger.debug("sending %s %d to %d (%d)", + "col" if dim else "row", id, neigh.node.ID, sent, extra=self.format) + + # until we exhaust capacity + # TODO: use exact limit + if self.statsTxInSlot >self.bwUplink: + return + + # Scheduler exited, nothing to send. Create new one for next round. + self.sched = self.nextToSend() def logRows(self): """It logs the rows assigned to the validator.""" From 07437ddde8cfca782d51bf8cac9f4f2704e2e643 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 7 Feb 2023 14:44:33 +0100 Subject: [PATCH 15/89] fixup bwUplink check (still approximate) Signed-off-by: Csaba Kiraly --- DAS/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DAS/validator.py b/DAS/validator.py index 89ca316..ba33170 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -259,7 +259,7 @@ class Validator: # until we exhaust capacity # TODO: use exact limit - if self.statsTxInSlot >self.bwUplink: + if self.statsTxInSlot >= self.bwUplink: return # Scheduler exited, nothing to send. Create new one for next round. From eb277d9b4307c7596436f4c49036968182157845 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 3 Feb 2023 23:07:55 +0100 Subject: [PATCH 16/89] limit batchsize of sending from a line Signed-off-by: Csaba Kiraly --- DAS/validator.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index ba33170..88ef42a 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -3,6 +3,7 @@ import random import collections import logging +import sys from DAS.block import * from bitarray import bitarray from bitarray.util import zeros @@ -12,6 +13,34 @@ def shuffled(lis): for index in random.sample(range(len(lis)), len(lis)): yield lis[index] +def sampleLine(line, limit): + """ sample up to 'limit' bits from a bitarray + + Since this is quite expensive, we use a number of heuristics to get it fast. + """ + if limit == sys.maxsize : + return line + else: + w = line.count(1) + if limit >= w : + return line + else: + l = len(line) + r = zeros(l) + if w < l/10 or limit > l/2 : + indices = [ i for i in range(l) if line[i] ] + sample = random.sample(indices, limit) + for i in sample: + r[i] = 1 + return r + else: + while limit: + i = random.randrange(0, l) + if line[i] and not r[i]: + r[i] = 1 + limit -= 1 + return r + class NextToSend: def __init__(self, neigh, toSend, id, dim): self.neigh = neigh @@ -177,8 +206,7 @@ class Validator: self.statsRxInSlot = 0 self.statsTxInSlot = 0 - - def nextColumnToSend(self, columnID): + def nextColumnToSend(self, columnID, limit = sys.maxsize): line = self.getColumn(columnID) if line.any(): self.logger.debug("col %d -> %s", columnID, self.columnNeighbors[columnID] , extra=self.format) @@ -187,9 +215,10 @@ class Validator: # if there is anything new to send, send it toSend = line & ~n.sent & ~n.received if (toSend).any(): + toSend = sampleLine(toSend, limit) yield NextToSend(n, toSend, columnID, 1) - def nextRowToSend(self, rowID): + def nextRowToSend(self, rowID, limit = sys.maxsize): line = self.getRow(rowID) if line.any(): self.logger.debug("row %d -> %s", rowID, self.rowNeighbors[rowID], extra=self.format) @@ -198,6 +227,7 @@ class Validator: # if there is anything new to send, send it toSend = line & ~n.sent & ~n.received if (toSend).any(): + toSend = sampleLine(toSend, limit) yield NextToSend(n, toSend, rowID, 0) def nextToSend(self): From 3917001e6ae8b6eac661c22dc8c6548b93545742 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 3 Feb 2023 23:08:53 +0100 Subject: [PATCH 17/89] send one segment at a time Signed-off-by: Csaba Kiraly --- DAS/validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 88ef42a..244f2a0 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -246,10 +246,10 @@ class Validator: while True: perLine = [] for c in self.columnIDs: - perLine.append(self.nextColumnToSend(c)) + perLine.append(self.nextColumnToSend(c, 1)) for r in self.rowIDs: - perLine.append(self.nextRowToSend(r)) + perLine.append(self.nextRowToSend(r, 1)) count = 0 random.shuffle(perLine) From 3fc7455c0beafcae28ae9ca54404d5b68e392114 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 14 Feb 2023 02:09:28 +0100 Subject: [PATCH 18/89] reduce default BW to more interesting values Signed-off-by: Csaba Kiraly --- DAS/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DAS/validator.py b/DAS/validator.py index 244f2a0..fcdebf4 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -107,7 +107,7 @@ class Validator: # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 # TODO: this should be a parameter - self.bwUplink = 1100 if not self.amIproposer else 22000 # approx. 100Mbps and 2Gbps + self.bwUplink = 110 if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps self.sched = self.nextToSend() From 382954de027071fb6ffc4a8aa35e880dbc521971 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 14 Feb 2023 02:11:44 +0100 Subject: [PATCH 19/89] add segment level send/receive Signed-off-by: Csaba Kiraly --- DAS/block.py | 6 ++++++ DAS/validator.py | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/DAS/block.py b/DAS/block.py index 693d7b6..e052fcb 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -20,6 +20,12 @@ class Block: """It merges (OR) the existing block with the received one.""" self.data |= merged.data + def getSegment(self, rowID, columnID): + return self.data[rowID*self.blockSize + columnID] + + def setSegment(self, rowID, columnID, v = 1): + self.data[rowID*self.blockSize + columnID] = v + def getColumn(self, columnID): """It returns the block column corresponding to columnID.""" return self.data[columnID::self.blockSize] diff --git a/DAS/validator.py b/DAS/validator.py index fcdebf4..5cec9e3 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -177,6 +177,20 @@ class Validator: else: pass + def receiveSegment(self, rID, cID, src): + # register receive so that we are not sending back + if rID in self.rowIDs: + if src in self.rowNeighbors[rID]: + self.rowNeighbors[rID][src].receiving[cID] = 1 + if cID in self.columnIDs: + if src in self.columnNeighbors[cID]: + self.columnNeighbors[cID][src].receiving[rID] = 1 + if not self.receivedBlock.getSegment(rID, cID): + self.receivedBlock.setSegment(rID, cID) + # else: + # self.statsRxDuplicateInSlot += 1 + self.statsRxInSlot += 1 + def receiveRowsColumns(self): """It receives rows and columns.""" @@ -266,6 +280,15 @@ class Validator: if not count: return + def sendSegmentToNeigh(self, rID, cID, neigh): + if not neigh.sent[cID] and not neigh.receiving[cID] : + neigh.sent[cID] = 1 + neigh.node.receiveSegment(rID, cID, self.ID) + self.statsTxInSlot += 1 + return True + else: + return False # received or already sent + def send(self): """ Send as much as we can in the timeslot, limited by bwUplink """ From 0f4883bf268c36e6a4668b9bf64dc72ea28b663a Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 14 Feb 2023 02:13:00 +0100 Subject: [PATCH 20/89] add node level send queue Signed-off-by: Csaba Kiraly --- DAS/validator.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/DAS/validator.py b/DAS/validator.py index 5cec9e3..a887bb5 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -7,6 +7,8 @@ import sys from DAS.block import * from bitarray import bitarray from bitarray.util import zeros +from collections import deque + def shuffled(lis): # based on https://stackoverflow.com/a/60342323 @@ -78,6 +80,8 @@ class Validator: self.format = {"entity": "Val "+str(self.ID)} self.block = Block(self.shape.blockSize) self.receivedBlock = Block(self.shape.blockSize) + self.receivedQueue = [] + self.sendQueue = deque() self.amIproposer = amIproposer self.logger = logger if self.shape.chi < 1: @@ -187,6 +191,7 @@ class Validator: self.columnNeighbors[cID][src].receiving[rID] = 1 if not self.receivedBlock.getSegment(rID, cID): self.receivedBlock.setSegment(rID, cID) + self.receivedQueue.append((rID, cID)) # else: # self.statsRxDuplicateInSlot += 1 self.statsRxInSlot += 1 @@ -212,6 +217,10 @@ class Validator: neigh.received |= neigh.receiving neigh.receiving.setall(0) + # add newly received segments to the send queue + self.sendQueue.extend(self.receivedQueue) + self.receivedQueue.clear() + def updateStats(self): """It updates the stats related to sent and received data.""" self.logger.debug("Stats: tx %d, rx %d", self.statsTxInSlot, self.statsRxInSlot, extra=self.format) @@ -293,6 +302,25 @@ class Validator: """ Send as much as we can in the timeslot, limited by bwUplink """ + while self.sendQueue: + (rID, cID) = self.sendQueue[0] + + if rID in self.rowIDs: + for neigh in self.rowNeighbors[rID].values(): + self.sendSegmentToNeigh(rID, cID, neigh) + + if self.statsTxInSlot >= self.bwUplink: + return + + if cID in self.columnIDs: + for neigh in self.columnNeighbors[cID].values(): + self.sendSegmentToNeigh(rID, cID, neigh) + + if self.statsTxInSlot >= self.bwUplink: + return + + self.sendQueue.popleft() + for n in self.sched: neigh = n.neigh toSend = n.toSend From 1403ca7ad03b4128475e20d99ef5e62084da3e1f Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 14 Feb 2023 02:18:37 +0100 Subject: [PATCH 21/89] add random scheduler --- DAS/validator.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/DAS/validator.py b/DAS/validator.py index a887bb5..67ef141 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -321,6 +321,35 @@ class Validator: self.sendQueue.popleft() + tries = 100 + while tries: + if self.rowIDs: + rID = random.choice(self.rowIDs) + cID = random.randrange(0, self.shape.blockSize) + if self.block.getSegment(rID, cID) : + neigh = random.choice(list(self.rowNeighbors[rID].values())) + if not neigh.sent[cID] and not neigh.receiving[cID] : + neigh.sent[cID] = 1 + neigh.node.receiveSegment(rID, cID, self.ID) + self.statsTxInSlot += 1 + tries = 100 + if self.statsTxInSlot >= self.bwUplink: + return + if self.columnIDs: + cID = random.choice(self.columnIDs) + rID = random.randrange(0, self.shape.blockSize) + if self.block.getSegment(rID, cID) : + neigh = random.choice(list(self.columnNeighbors[cID].values())) + if not neigh.sent[rID] and not neigh.receiving[rID] : + neigh.sent[rID] = 1 + neigh.node.receiveSegment(rID, cID, self.ID) + self.statsTxInSlot += 1 + tries = 100 + tries -= 1 + if self.statsTxInSlot >= self.bwUplink: + return + return + for n in self.sched: neigh = n.neigh toSend = n.toSend From 9ab51278c880ed52473e257cb5b6089d72d4258d Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 14 Feb 2023 11:45:55 +0100 Subject: [PATCH 22/89] add shuffledDict helper Signed-off-by: Csaba Kiraly --- DAS/validator.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 67ef141..311205d 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -15,6 +15,12 @@ def shuffled(lis): for index in random.sample(range(len(lis)), len(lis)): yield lis[index] +def shuffledDict(d): + lis = list(d.values()) + # based on https://stackoverflow.com/a/60342323 + for index in random.sample(range(len(d)), len(d)): + yield lis[index] + def sampleLine(line, limit): """ sample up to 'limit' bits from a bitarray @@ -233,7 +239,7 @@ class Validator: line = self.getColumn(columnID) if line.any(): self.logger.debug("col %d -> %s", columnID, self.columnNeighbors[columnID] , extra=self.format) - for n in shuffled(list(self.columnNeighbors[columnID].values())): + for n in shuffledDict(self.columnNeighbors[columnID]): # if there is anything new to send, send it toSend = line & ~n.sent & ~n.received @@ -245,7 +251,7 @@ class Validator: line = self.getRow(rowID) if line.any(): self.logger.debug("row %d -> %s", rowID, self.rowNeighbors[rowID], extra=self.format) - for n in shuffled(list(self.rowNeighbors[rowID].values())): + for n in shuffledDict(self.rowNeighbors[rowID]): # if there is anything new to send, send it toSend = line & ~n.sent & ~n.received From f67c70896c8965f36b3be8a3a67edfaa5c36fe58 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 14 Feb 2023 11:48:18 +0100 Subject: [PATCH 23/89] add to receivedQueue also in row/column receive code Signed-off-by: Csaba Kiraly --- DAS/validator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DAS/validator.py b/DAS/validator.py index 311205d..6c5575c 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -173,6 +173,9 @@ class Validator: # register receive so that we are not sending back self.columnNeighbors[id][src].receiving |= column self.receivedBlock.mergeColumn(id, column) + for i in range(len(column)): + if column[i]: + self.receivedQueue.append((i, id)) self.statsRxInSlot += column.count(1) else: pass @@ -183,6 +186,9 @@ class Validator: # register receive so that we are not sending back self.rowNeighbors[id][src].receiving |= row self.receivedBlock.mergeRow(id, row) + for i in range(len(row)): + if row[i]: + self.receivedQueue.append((id, i)) self.statsRxInSlot += row.count(1) else: pass From 7c0fcaba7891efe6ff73956cc0270d49007fb1a4 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 14 Feb 2023 11:51:04 +0100 Subject: [PATCH 24/89] add validator.perNodeQueue conf option Signed-off-by: Csaba Kiraly --- DAS/validator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DAS/validator.py b/DAS/validator.py index 6c5575c..88e32d6 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -119,6 +119,7 @@ class Validator: # TODO: this should be a parameter self.bwUplink = 110 if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps + self.perNodeQueue = False # keep a global queue of incoming messages for later sequential dispatch self.sched = self.nextToSend() def logIDs(self): @@ -230,7 +231,8 @@ class Validator: neigh.receiving.setall(0) # add newly received segments to the send queue - self.sendQueue.extend(self.receivedQueue) + if self.perNeighborQueue: + self.sendQueue.extend(self.receivedQueue) self.receivedQueue.clear() def updateStats(self): From 23e40693f18cde9cb5aac8de03e8e2bc529545b7 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 14 Feb 2023 11:52:47 +0100 Subject: [PATCH 25/89] add perNeighborQueue option If enabled, queue incoming messages to outgoing connections on arrival, as typical in some GossipSub implementations. Signed-off-by: Csaba Kiraly --- DAS/validator.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/DAS/validator.py b/DAS/validator.py index 88e32d6..9c43967 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -69,6 +69,7 @@ class Neighbor: self.receiving = zeros(blockSize) self.received = zeros(blockSize) self.sent = zeros(blockSize) + self.sendQueue = deque() class Validator: @@ -86,7 +87,7 @@ class Validator: self.format = {"entity": "Val "+str(self.ID)} self.block = Block(self.shape.blockSize) self.receivedBlock = Block(self.shape.blockSize) - self.receivedQueue = [] + self.receivedQueue = deque() self.sendQueue = deque() self.amIproposer = amIproposer self.logger = logger @@ -119,6 +120,7 @@ class Validator: # TODO: this should be a parameter self.bwUplink = 110 if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps + self.perNeighborQueue = False # queue incoming messages to outgoing connections on arrival (as typical GossipSub impl) self.perNodeQueue = False # keep a global queue of incoming messages for later sequential dispatch self.sched = self.nextToSend() @@ -233,6 +235,19 @@ class Validator: # add newly received segments to the send queue if self.perNeighborQueue: self.sendQueue.extend(self.receivedQueue) + + if self.perNodeQueue: + while self.receivedQueue: + (rID, cID) = self.receivedQueue.popleft() + + if rID in self.rowIDs: + for neigh in self.rowNeighbors[rID].values(): + neigh.sendQueue.append(cID) + + if cID in self.columnIDs: + for neigh in self.columnNeighbors[cID].values(): + neigh.sendQueue.append(rID) + self.receivedQueue.clear() def updateStats(self): @@ -335,6 +350,25 @@ class Validator: self.sendQueue.popleft() + progress = True + while (progress): + progress = False + for rID, neighs in self.rowNeighbors.items(): + for neigh in neighs.values(): + if (neigh.sendQueue): + self.sendSegmentToNeigh(rID, neigh.sendQueue.popleft(), neigh) + progress = True + if self.statsTxInSlot >= self.bwUplink: + return + + for cID, neighs in self.columnNeighbors.items(): + for neigh in neighs.values(): + if (neigh.sendQueue): + self.sendSegmentToNeigh(neigh.sendQueue.popleft(), cID, neigh) + progress = True + if self.statsTxInSlot >= self.bwUplink: + return + tries = 100 while tries: if self.rowIDs: From b7dab5bad9c8e291cdc619239e5b0620f159fe7a Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 03:10:55 +0100 Subject: [PATCH 26/89] fix sendSegmentToNeigh: specify dimension Specify along which dimension (row/column) a segment was sent. Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 8 ++++---- DAS/validator.py | 11 ++++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 5b60413..3261ad7 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -66,8 +66,8 @@ class Simulator: for u, v in G.edges: val1=rowChannels[id][u] val2=rowChannels[id][v] - val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, self.shape.blockSize)}) - val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, self.shape.blockSize)}) + val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, 0, self.shape.blockSize)}) + val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, 0, self.shape.blockSize)}) if (len(columnChannels[id]) <= self.shape.netDegree): self.logger.debug("Graph fully connected with degree %d !" % (len(columnChannels[id]) - 1), extra=self.format) @@ -79,8 +79,8 @@ class Simulator: for u, v in G.edges: val1=columnChannels[id][u] val2=columnChannels[id][v] - val1.columnNeighbors[id].update({val2.ID : Neighbor(val2, self.shape.blockSize)}) - val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, self.shape.blockSize)}) + val1.columnNeighbors[id].update({val2.ID : Neighbor(val2, 1, self.shape.blockSize)}) + val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, 1, self.shape.blockSize)}) if self.logger.isEnabledFor(logging.DEBUG): for i in range(0, self.shape.numberValidators): diff --git a/DAS/validator.py b/DAS/validator.py index 9c43967..aecbc37 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -63,9 +63,10 @@ class Neighbor: """It returns the amount of sent and received data.""" return "%d:%d/%d" % (self.node.ID, self.sent.count(1), self.received.count(1)) - def __init__(self, v, blockSize): + def __init__(self, v, dim, blockSize): """It initializes the neighbor with the node and sets counters to zero.""" self.node = v + self.dim = dim # 0:row 1:col self.receiving = zeros(blockSize) self.received = zeros(blockSize) self.sent = zeros(blockSize) @@ -319,8 +320,12 @@ class Validator: return def sendSegmentToNeigh(self, rID, cID, neigh): - if not neigh.sent[cID] and not neigh.receiving[cID] : - neigh.sent[cID] = 1 + if neigh.dim == 0: #row + i = cID + else: + i = rID + if not neigh.sent[i] and not neigh.receiving[i] : + neigh.sent[i] = 1 neigh.node.receiveSegment(rID, cID, self.ID) self.statsTxInSlot += 1 return True From 0c91eff67be2594f1625446cb25b80d8fd8f9481 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 03:15:00 +0100 Subject: [PATCH 27/89] add dumbRandomScheduler parameter Signed-off-by: Csaba Kiraly --- DAS/validator.py | 60 ++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index aecbc37..8e52f93 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -123,6 +123,7 @@ class Validator: self.perNeighborQueue = False # queue incoming messages to outgoing connections on arrival (as typical GossipSub impl) self.perNodeQueue = False # keep a global queue of incoming messages for later sequential dispatch + self.dumbRandomScheduler = False # dumb random scheduler self.sched = self.nextToSend() def logIDs(self): @@ -374,34 +375,37 @@ class Validator: if self.statsTxInSlot >= self.bwUplink: return - tries = 100 - while tries: - if self.rowIDs: - rID = random.choice(self.rowIDs) - cID = random.randrange(0, self.shape.blockSize) - if self.block.getSegment(rID, cID) : - neigh = random.choice(list(self.rowNeighbors[rID].values())) - if not neigh.sent[cID] and not neigh.receiving[cID] : - neigh.sent[cID] = 1 - neigh.node.receiveSegment(rID, cID, self.ID) - self.statsTxInSlot += 1 - tries = 100 - if self.statsTxInSlot >= self.bwUplink: - return - if self.columnIDs: - cID = random.choice(self.columnIDs) - rID = random.randrange(0, self.shape.blockSize) - if self.block.getSegment(rID, cID) : - neigh = random.choice(list(self.columnNeighbors[cID].values())) - if not neigh.sent[rID] and not neigh.receiving[rID] : - neigh.sent[rID] = 1 - neigh.node.receiveSegment(rID, cID, self.ID) - self.statsTxInSlot += 1 - tries = 100 - tries -= 1 - if self.statsTxInSlot >= self.bwUplink: - return - return + if self.dumbRandomScheduler: + # dumb random scheduler picking segments at random and trying to send it + tries = 100 + t = tries + while t: + if self.rowIDs: + rID = random.choice(self.rowIDs) + cID = random.randrange(0, self.shape.blockSize) + if self.block.getSegment(rID, cID) : + neigh = random.choice(list(self.rowNeighbors[rID].values())) + if not neigh.sent[cID] and not neigh.receiving[cID] : + neigh.sent[cID] = 1 + neigh.node.receiveSegment(rID, cID, self.ID) + self.statsTxInSlot += 1 + t = tries + if self.statsTxInSlot >= self.bwUplink: + return + if self.columnIDs: + cID = random.choice(self.columnIDs) + rID = random.randrange(0, self.shape.blockSize) + if self.block.getSegment(rID, cID) : + neigh = random.choice(list(self.columnNeighbors[cID].values())) + if not neigh.sent[rID] and not neigh.receiving[rID] : + neigh.sent[rID] = 1 + neigh.node.receiveSegment(rID, cID, self.ID) + self.statsTxInSlot += 1 + t = tries + t -= 1 + if self.statsTxInSlot >= self.bwUplink: + return + return for n in self.sched: neigh = n.neigh From dff0e5523af3598242284a5de4a8cd73f0129321 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 03:18:00 +0100 Subject: [PATCH 28/89] factorize addToSendQueue Signed-off-by: Csaba Kiraly --- DAS/validator.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 8e52f93..d152472 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -213,6 +213,18 @@ class Validator: # self.statsRxDuplicateInSlot += 1 self.statsRxInSlot += 1 + def addToSendQueue(self, rID, cID): + if self.perNodeQueue: + self.sendQueue.append((rID, cID)) + + if self.perNeighborQueue: + if rID in self.rowIDs: + for neigh in self.rowNeighbors[rID].values(): + neigh.sendQueue.append(cID) + + if cID in self.columnIDs: + for neigh in self.columnNeighbors[cID].values(): + neigh.sendQueue.append(rID) def receiveRowsColumns(self): """It receives rows and columns.""" @@ -235,22 +247,9 @@ class Validator: neigh.receiving.setall(0) # add newly received segments to the send queue - if self.perNeighborQueue: - self.sendQueue.extend(self.receivedQueue) - - if self.perNodeQueue: - while self.receivedQueue: - (rID, cID) = self.receivedQueue.popleft() - - if rID in self.rowIDs: - for neigh in self.rowNeighbors[rID].values(): - neigh.sendQueue.append(cID) - - if cID in self.columnIDs: - for neigh in self.columnNeighbors[cID].values(): - neigh.sendQueue.append(rID) - - self.receivedQueue.clear() + while self.receivedQueue: + (rID, cID) = self.receivedQueue.popleft() + self.addToSendQueue(rID, cID) def updateStats(self): """It updates the stats related to sent and received data.""" From f05c3cd233e2c3d462a7ce4e65a8381064fe32ee Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 03:22:37 +0100 Subject: [PATCH 29/89] fix queuing: should queue after repair If operation is based on send queues, segments should be queued after successful repair. Signed-off-by: Csaba Kiraly --- DAS/block.py | 22 ++++++++++++++++++---- DAS/validator.py | 20 ++++++++++++++++++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/DAS/block.py b/DAS/block.py index e052fcb..228e743 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -35,10 +35,17 @@ class Block: self.data[columnID::self.blockSize] |= column def repairColumn(self, id): - """It repairs the entire column if it has at least blockSize/2 ones.""" - success = self.data[id::self.blockSize].count(1) + """It repairs the entire column if it has at least blockSize/2 ones. + Returns: list of repaired segments + """ + line = self.data[id::self.blockSize] + success = line.count(1) if success >= self.blockSize/2: + ret = ~line self.data[id::self.blockSize] = 1 + else: + ret = zeros(self.blockSize) + return ret def getRow(self, rowID): """It returns the block row corresponding to rowID.""" @@ -49,10 +56,17 @@ class Block: self.data[rowID*self.blockSize:(rowID+1)*self.blockSize] |= row def repairRow(self, id): - """It repairs the entire row if it has at least blockSize/2 ones.""" - success = self.data[id*self.blockSize:(id+1)*self.blockSize].count(1) + """It repairs the entire row if it has at least blockSize/2 ones. + Returns: list of repaired segments + """ + line = self.data[id*self.blockSize:(id+1)*self.blockSize] + success = line.count(1) if success >= self.blockSize/2: + ret = ~line self.data[id*self.blockSize:(id+1)*self.blockSize] = 1 + else: + ret = zeros(self.blockSize) + return ret def print(self): """It prints the block in the terminal (outside of the logger rules)).""" diff --git a/DAS/validator.py b/DAS/validator.py index d152472..5065e16 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -446,12 +446,28 @@ class Validator: def restoreRows(self): """It restores the rows assigned to the validator, that can be repaired.""" for id in self.rowIDs: - self.block.repairRow(id) + rep = self.block.repairRow(id) + if (rep.any()): + # If operation is based on send queues, segments should + # be queued after successful repair. + for i in range(len(rep)): + if rep[i]: + self.logger.debug("Rep: %d,%d", id, i, extra=self.format) + self.addToSendQueue(id, i) + # self.statsRepairInSlot += rep.count(1) def restoreColumns(self): """It restores the columns assigned to the validator, that can be repaired.""" for id in self.columnIDs: - self.block.repairColumn(id) + rep = self.block.repairColumn(id) + if (rep.any()): + # If operation is based on send queues, segments should + # be queued after successful repair. + for i in range(len(rep)): + if rep[i]: + self.logger.debug("Rep: %d,%d", i, id, extra=self.format) + self.addToSendQueue(i, id) + # self.statsRepairInSlot += rep.count(1) def checkStatus(self): """It checks how many expected/arrived samples are for each assigned row/column.""" From d0641e4568f605cddfa0adcd2d80117d899bd07f Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 03:23:39 +0100 Subject: [PATCH 30/89] add repairOnTheFly parameter Signed-off-by: Csaba Kiraly --- DAS/validator.py | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 5065e16..f447848 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -121,6 +121,7 @@ class Validator: # TODO: this should be a parameter self.bwUplink = 110 if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps + self.repairOnTheFly = True self.perNeighborQueue = False # queue incoming messages to outgoing connections on arrival (as typical GossipSub impl) self.perNodeQueue = False # keep a global queue of incoming messages for later sequential dispatch self.dumbRandomScheduler = False # dumb random scheduler @@ -445,29 +446,31 @@ class Validator: def restoreRows(self): """It restores the rows assigned to the validator, that can be repaired.""" - for id in self.rowIDs: - rep = self.block.repairRow(id) - if (rep.any()): - # If operation is based on send queues, segments should - # be queued after successful repair. - for i in range(len(rep)): - if rep[i]: - self.logger.debug("Rep: %d,%d", id, i, extra=self.format) - self.addToSendQueue(id, i) - # self.statsRepairInSlot += rep.count(1) + if self.repairOnTheFly: + for id in self.rowIDs: + rep = self.block.repairRow(id) + if (rep.any()): + # If operation is based on send queues, segments should + # be queued after successful repair. + for i in range(len(rep)): + if rep[i]: + self.logger.debug("Rep: %d,%d", id, i, extra=self.format) + self.addToSendQueue(id, i) + # self.statsRepairInSlot += rep.count(1) def restoreColumns(self): """It restores the columns assigned to the validator, that can be repaired.""" - for id in self.columnIDs: - rep = self.block.repairColumn(id) - if (rep.any()): - # If operation is based on send queues, segments should - # be queued after successful repair. - for i in range(len(rep)): - if rep[i]: - self.logger.debug("Rep: %d,%d", i, id, extra=self.format) - self.addToSendQueue(i, id) - # self.statsRepairInSlot += rep.count(1) + if self.repairOnTheFly: + for id in self.columnIDs: + rep = self.block.repairColumn(id) + if (rep.any()): + # If operation is based on send queues, segments should + # be queued after successful repair. + for i in range(len(rep)): + if rep[i]: + self.logger.debug("Rep: %d,%d", i, id, extra=self.format) + self.addToSendQueue(i, id) + # self.statsRepairInSlot += rep.count(1) def checkStatus(self): """It checks how many expected/arrived samples are for each assigned row/column.""" From 5383c59f6f8e55b172e23c477ce1a9cd7fdc7f63 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 03:25:52 +0100 Subject: [PATCH 31/89] add shuffleLines and shuffleNeighbors params Signed-off-by: Csaba Kiraly --- DAS/validator.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index f447848..dcd6779 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -11,15 +11,24 @@ from collections import deque def shuffled(lis): + ''' Generator yielding list in shuffled order + ''' # based on https://stackoverflow.com/a/60342323 for index in random.sample(range(len(lis)), len(lis)): yield lis[index] -def shuffledDict(d): - lis = list(d.values()) - # based on https://stackoverflow.com/a/60342323 - for index in random.sample(range(len(d)), len(d)): - yield lis[index] +def shuffledDict(d, shuffle=True): + ''' Generator yielding dictionary in shuffled order + + Shuffle, except if not (optional parameter useful for experiment setup) + ''' + if shuffle: + lis = list(d.items()) + for index in random.sample(range(len(d)), len(d)): + yield lis[index] + else: + for kv in d.items(): + yield kv def sampleLine(line, limit): """ sample up to 'limit' bits from a bitarray @@ -124,6 +133,8 @@ class Validator: self.repairOnTheFly = True self.perNeighborQueue = False # queue incoming messages to outgoing connections on arrival (as typical GossipSub impl) self.perNodeQueue = False # keep a global queue of incoming messages for later sequential dispatch + self.shuffleLines = True # shuffle the order of rows/columns in each iteration while trying to send + self.shuffleNeighbors = True # shuffle the order of neighbors when sending the same segment to each neighbor self.dumbRandomScheduler = False # dumb random scheduler self.sched = self.nextToSend() @@ -264,7 +275,7 @@ class Validator: line = self.getColumn(columnID) if line.any(): self.logger.debug("col %d -> %s", columnID, self.columnNeighbors[columnID] , extra=self.format) - for n in shuffledDict(self.columnNeighbors[columnID]): + for _, n in shuffledDict(self.columnNeighbors[columnID]): # if there is anything new to send, send it toSend = line & ~n.sent & ~n.received @@ -276,7 +287,7 @@ class Validator: line = self.getRow(rowID) if line.any(): self.logger.debug("row %d -> %s", rowID, self.rowNeighbors[rowID], extra=self.format) - for n in shuffledDict(self.rowNeighbors[rowID]): + for _, n in shuffledDict(self.rowNeighbors[rowID]): # if there is anything new to send, send it toSend = line & ~n.sent & ~n.received @@ -341,14 +352,14 @@ class Validator: (rID, cID) = self.sendQueue[0] if rID in self.rowIDs: - for neigh in self.rowNeighbors[rID].values(): + for _, neigh in shuffledDict(self.rowNeighbors[rID], self.shuffleNeighbors): self.sendSegmentToNeigh(rID, cID, neigh) if self.statsTxInSlot >= self.bwUplink: return if cID in self.columnIDs: - for neigh in self.columnNeighbors[cID].values(): + for _, neigh in shuffledDict(self.columnNeighbors[cID], self.shuffleNeighbors): self.sendSegmentToNeigh(rID, cID, neigh) if self.statsTxInSlot >= self.bwUplink: @@ -359,16 +370,16 @@ class Validator: progress = True while (progress): progress = False - for rID, neighs in self.rowNeighbors.items(): - for neigh in neighs.values(): + for rID, neighs in shuffledDict(self.rowNeighbors, self.shuffleLines): + for _, neigh in shuffledDict(neighs, self.shuffleNeighbors): if (neigh.sendQueue): self.sendSegmentToNeigh(rID, neigh.sendQueue.popleft(), neigh) progress = True if self.statsTxInSlot >= self.bwUplink: return - for cID, neighs in self.columnNeighbors.items(): - for neigh in neighs.values(): + for cID, neighs in shuffledDict(self.columnNeighbors, self.shuffleLines): + for _, neigh in shuffledDict(neighs, self.shuffleNeighbors): if (neigh.sendQueue): self.sendSegmentToNeigh(neigh.sendQueue.popleft(), cID, neigh) progress = True From 1669ec92366f5b4f994643ba4ad7e1db7d975a05 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 03:26:31 +0100 Subject: [PATCH 32/89] more debug logging Signed-off-by: Csaba Kiraly --- DAS/validator.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index dcd6779..e2264b7 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -70,7 +70,7 @@ class Neighbor: def __repr__(self): """It returns the amount of sent and received data.""" - return "%d:%d/%d" % (self.node.ID, self.sent.count(1), self.received.count(1)) + return "%d:%d/%d, q:%d" % (self.node.ID, self.sent.count(1), self.received.count(1), len(self.sendQueue)) def __init__(self, v, dim, blockSize): """It initializes the neighbor with the node and sets counters to zero.""" @@ -192,6 +192,7 @@ class Validator: self.receivedBlock.mergeColumn(id, column) for i in range(len(column)): if column[i]: + self.logger.debug("Recv: %d->%d: %d,%d", src, self.ID, i, id, extra=self.format) self.receivedQueue.append((i, id)) self.statsRxInSlot += column.count(1) else: @@ -205,6 +206,7 @@ class Validator: self.receivedBlock.mergeRow(id, row) for i in range(len(row)): if row[i]: + self.logger.debug("Recv %d->%d: %d,%d", src, self.ID, id, i, extra=self.format) self.receivedQueue.append((id, i)) self.statsRxInSlot += row.count(1) else: @@ -219,9 +221,11 @@ class Validator: if src in self.columnNeighbors[cID]: self.columnNeighbors[cID][src].receiving[rID] = 1 if not self.receivedBlock.getSegment(rID, cID): + self.logger.debug("Recv new: %d->%d: %d,%d", src, self.ID, rID, cID, extra=self.format) self.receivedBlock.setSegment(rID, cID) self.receivedQueue.append((rID, cID)) - # else: + else: + self.logger.debug("Recv DUP: %d->%d: %d,%d", src, self.ID, rID, cID, extra=self.format) # self.statsRxDuplicateInSlot += 1 self.statsRxInSlot += 1 From af72e58d08d003d533cecbe5a2ed1a9c50ad8350 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 14:17:17 +0100 Subject: [PATCH 33/89] collect receivedQueue only if it is used later Signed-off-by: Csaba Kiraly --- DAS/validator.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index e2264b7..5af772a 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -193,7 +193,8 @@ class Validator: for i in range(len(column)): if column[i]: self.logger.debug("Recv: %d->%d: %d,%d", src, self.ID, i, id, extra=self.format) - self.receivedQueue.append((i, id)) + if self.perNodeQueue or self.perNeighborQueue: + self.receivedQueue.append((i, id)) self.statsRxInSlot += column.count(1) else: pass @@ -207,7 +208,8 @@ class Validator: for i in range(len(row)): if row[i]: self.logger.debug("Recv %d->%d: %d,%d", src, self.ID, id, i, extra=self.format) - self.receivedQueue.append((id, i)) + if self.perNodeQueue or self.perNeighborQueue: + self.receivedQueue.append((id, i)) self.statsRxInSlot += row.count(1) else: pass @@ -223,7 +225,8 @@ class Validator: if not self.receivedBlock.getSegment(rID, cID): self.logger.debug("Recv new: %d->%d: %d,%d", src, self.ID, rID, cID, extra=self.format) self.receivedBlock.setSegment(rID, cID) - self.receivedQueue.append((rID, cID)) + if self.perNodeQueue or self.perNeighborQueue: + self.receivedQueue.append((rID, cID)) else: self.logger.debug("Recv DUP: %d->%d: %d,%d", src, self.ID, rID, cID, extra=self.format) # self.statsRxDuplicateInSlot += 1 @@ -263,9 +266,10 @@ class Validator: neigh.receiving.setall(0) # add newly received segments to the send queue - while self.receivedQueue: - (rID, cID) = self.receivedQueue.popleft() - self.addToSendQueue(rID, cID) + if self.perNodeQueue or self.perNeighborQueue: + while self.receivedQueue: + (rID, cID) = self.receivedQueue.popleft() + self.addToSendQueue(rID, cID) def updateStats(self): """It updates the stats related to sent and received data.""" From 655f3a6642ec54d5181d281c266f50bc03226576 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 14:19:21 +0100 Subject: [PATCH 34/89] fix: avoid looking into the future Checking neigh.receiving is cheating in the current model. If the timeslot is small, information can't propagate that fast. Signed-off-by: Csaba Kiraly --- DAS/validator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 5af772a..8923a9a 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -344,7 +344,7 @@ class Validator: i = cID else: i = rID - if not neigh.sent[i] and not neigh.receiving[i] : + if not neigh.sent[i] and not neigh.received[i] : neigh.sent[i] = 1 neigh.node.receiveSegment(rID, cID, self.ID) self.statsTxInSlot += 1 @@ -404,7 +404,7 @@ class Validator: cID = random.randrange(0, self.shape.blockSize) if self.block.getSegment(rID, cID) : neigh = random.choice(list(self.rowNeighbors[rID].values())) - if not neigh.sent[cID] and not neigh.receiving[cID] : + if not neigh.sent[cID] and not neigh.received[cID] : neigh.sent[cID] = 1 neigh.node.receiveSegment(rID, cID, self.ID) self.statsTxInSlot += 1 @@ -416,7 +416,7 @@ class Validator: rID = random.randrange(0, self.shape.blockSize) if self.block.getSegment(rID, cID) : neigh = random.choice(list(self.columnNeighbors[cID].values())) - if not neigh.sent[rID] and not neigh.receiving[rID] : + if not neigh.sent[rID] and not neigh.received[rID] : neigh.sent[rID] = 1 neigh.node.receiveSegment(rID, cID, self.ID) self.statsTxInSlot += 1 From cd2b69149bda81692ee37c056749b0ec6bb7af29 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 14:22:55 +0100 Subject: [PATCH 35/89] add segmentShuffleScheduler This scheduler check which owned segments needs sending (at least one neighbor needing it). Then it sends each segment that's worth sending once, in shuffled order. This is repeated until bw limit. Signed-off-by: Csaba Kiraly --- DAS/validator.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/DAS/validator.py b/DAS/validator.py index 8923a9a..f9571da 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -65,6 +65,12 @@ class NextToSend: self.id = id self.dim = dim +class SegmentToSend: + def __init__(self, dim, id, i): + self.dim = dim + self.id = id + self.i = i + class Neighbor: """This class implements a node neighbor to monitor sent and received data.""" @@ -136,6 +142,7 @@ class Validator: self.shuffleLines = True # shuffle the order of rows/columns in each iteration while trying to send self.shuffleNeighbors = True # shuffle the order of neighbors when sending the same segment to each neighbor self.dumbRandomScheduler = False # dumb random scheduler + self.segmentShuffleScheduler = True # send each segment that's worth sending once in shuffled order, then repeat self.sched = self.nextToSend() def logIDs(self): @@ -394,6 +401,56 @@ class Validator: if self.statsTxInSlot >= self.bwUplink: return + # process possible segments to send in shuffled breadth-first order + if self.segmentShuffleScheduler: + while True: + self.segmentsToSend = [] + for rID, neighs in self.rowNeighbors.items(): + line = self.getRow(rID) + needed = zeros(self.shape.blockSize) + for neigh in neighs.values(): + needed |= ~(neigh.received | neigh.sent) + needed &= line + if (needed).any(): + for i in range(len(needed)): + if needed[i]: + self.segmentsToSend.append(SegmentToSend(0, rID, i)) + + for cID, neighs in self.columnNeighbors.items(): + line = self.getColumn(cID) + needed = zeros(self.shape.blockSize) + for neigh in neighs.values(): + needed |= ~(neigh.received | neigh.sent) + needed &= line + if (needed).any(): + for i in range(len(needed)): + if needed[i]: + self.segmentsToSend.append(SegmentToSend(1, cID, i)) + + if self.segmentsToSend: + self.logger.debug("TX:%d q:%d", self.statsTxInSlot, len(self.segmentsToSend), extra=self.format) + for s in shuffled(self.segmentsToSend): + self.logger.debug("%d:%d/%d", s.dim, s.id, s.i, extra=self.format) + if s.dim == 0: + for _, neigh in shuffledDict(self.rowNeighbors[s.id], self.shuffleNeighbors): + self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) + if not neigh.sent[s.i] and not neigh.received[s.i]: + self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) + self.sendSegmentToNeigh(s.id, s.i, neigh) + break + else: + for _, neigh in shuffledDict(self.columnNeighbors[s.id], self.shuffleNeighbors): + self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) + if not neigh.sent[s.i] and not neigh.received[s.i]: + self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) + self.sendSegmentToNeigh(s.i, s.id, neigh) + break + + if self.statsTxInSlot >= self.bwUplink: + return + else: + break + if self.dumbRandomScheduler: # dumb random scheduler picking segments at random and trying to send it tries = 100 From 54d101e5b875e8337fd996b88cd1bb4369d4f7b0 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 14:28:38 +0100 Subject: [PATCH 36/89] segmentShuffleScheduler: persist scheduler state between timesteps Signed-off-by: Csaba Kiraly --- DAS/validator.py | 52 ++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index f9571da..725bc98 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -143,6 +143,7 @@ class Validator: self.shuffleNeighbors = True # shuffle the order of neighbors when sending the same segment to each neighbor self.dumbRandomScheduler = False # dumb random scheduler self.segmentShuffleScheduler = True # send each segment that's worth sending once in shuffled order, then repeat + self.segmentShuffleSchedulerPersist = True # Persist scheduler state between timesteps self.sched = self.nextToSend() def logIDs(self): @@ -403,7 +404,35 @@ class Validator: # process possible segments to send in shuffled breadth-first order if self.segmentShuffleScheduler: + # This scheduler check which owned segments needs sending (at least + # one neighbor needing it). Then it sends each segment that's worth sending + # once, in shuffled order. This is repeated until bw limit. while True: + if hasattr(self, 'segmentsToSend') and self.segmentsToSend: + self.logger.debug("TX:%d q:%d", self.statsTxInSlot, len(self.segmentsToSend), extra=self.format) + for s in shuffled(self.segmentsToSend): + self.logger.debug("%d:%d/%d", s.dim, s.id, s.i, extra=self.format) + if s.dim == 0: + for _, neigh in shuffledDict(self.rowNeighbors[s.id], self.shuffleNeighbors): + self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) + if not neigh.sent[s.i] and not neigh.received[s.i]: + self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) + self.sendSegmentToNeigh(s.id, s.i, neigh) + break + else: + for _, neigh in shuffledDict(self.columnNeighbors[s.id], self.shuffleNeighbors): + self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) + if not neigh.sent[s.i] and not neigh.received[s.i]: + self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) + self.sendSegmentToNeigh(s.i, s.id, neigh) + break + + if self.statsTxInSlot >= self.bwUplink: + if not self.segmentShuffleSchedulerPersist: + # remove scheduler state before leaving + self.segmentsToSend = [] + return + self.segmentsToSend = [] for rID, neighs in self.rowNeighbors.items(): line = self.getRow(rID) @@ -427,28 +456,7 @@ class Validator: if needed[i]: self.segmentsToSend.append(SegmentToSend(1, cID, i)) - if self.segmentsToSend: - self.logger.debug("TX:%d q:%d", self.statsTxInSlot, len(self.segmentsToSend), extra=self.format) - for s in shuffled(self.segmentsToSend): - self.logger.debug("%d:%d/%d", s.dim, s.id, s.i, extra=self.format) - if s.dim == 0: - for _, neigh in shuffledDict(self.rowNeighbors[s.id], self.shuffleNeighbors): - self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) - if not neigh.sent[s.i] and not neigh.received[s.i]: - self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) - self.sendSegmentToNeigh(s.id, s.i, neigh) - break - else: - for _, neigh in shuffledDict(self.columnNeighbors[s.id], self.shuffleNeighbors): - self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) - if not neigh.sent[s.i] and not neigh.received[s.i]: - self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) - self.sendSegmentToNeigh(s.i, s.id, neigh) - break - - if self.statsTxInSlot >= self.bwUplink: - return - else: + if not self.segmentsToSend: break if self.dumbRandomScheduler: From bb55abe2b0fc921e382583f0acea23229a75a15d Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Feb 2023 14:34:24 +0100 Subject: [PATCH 37/89] comments only Signed-off-by: Csaba Kiraly --- DAS/validator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DAS/validator.py b/DAS/validator.py index 725bc98..26d4e6c 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -364,6 +364,7 @@ class Validator: """ Send as much as we can in the timeslot, limited by bwUplink """ + # process node level send queue while self.sendQueue: (rID, cID) = self.sendQueue[0] @@ -383,6 +384,7 @@ class Validator: self.sendQueue.popleft() + # process neighbor level send queues in shuffled breadth-first order progress = True while (progress): progress = False From f91f3da5d2e6eeeffcb6acc147570f515e56d9d0 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 16 Feb 2023 09:19:45 +0100 Subject: [PATCH 38/89] fixup: segmentShuffleScheduler Signed-off-by: Csaba Kiraly --- DAS/validator.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 26d4e6c..0d10425 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -410,9 +410,9 @@ class Validator: # one neighbor needing it). Then it sends each segment that's worth sending # once, in shuffled order. This is repeated until bw limit. while True: - if hasattr(self, 'segmentsToSend') and self.segmentsToSend: - self.logger.debug("TX:%d q:%d", self.statsTxInSlot, len(self.segmentsToSend), extra=self.format) - for s in shuffled(self.segmentsToSend): + if hasattr(self, 'segmentShuffleGen') and self.segmentShuffleGen is not None: + #self.logger.debug("TX:%d queue:%d", self.statsTxInSlot, len(self.segmentsToSend), extra=self.format) + for s in self.segmentShuffleGen: self.logger.debug("%d:%d/%d", s.dim, s.id, s.i, extra=self.format) if s.dim == 0: for _, neigh in shuffledDict(self.rowNeighbors[s.id], self.shuffleNeighbors): @@ -433,6 +433,7 @@ class Validator: if not self.segmentShuffleSchedulerPersist: # remove scheduler state before leaving self.segmentsToSend = [] + self.segmentShuffleGen = None return self.segmentsToSend = [] @@ -460,6 +461,8 @@ class Validator: if not self.segmentsToSend: break + else: + self.segmentShuffleGen = shuffled(self.segmentsToSend) if self.dumbRandomScheduler: # dumb random scheduler picking segments at random and trying to send it From e70740f530a2724455cd7a28fe3fb0dce86861f7 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 16 Feb 2023 17:34:11 +0100 Subject: [PATCH 39/89] handle duplicates in receiveRow/Column Signed-off-by: Csaba Kiraly --- DAS/validator.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 0d10425..7e02da3 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -197,12 +197,17 @@ class Validator: if id in self.columnIDs: # register receive so that we are not sending back self.columnNeighbors[id][src].receiving |= column - self.receivedBlock.mergeColumn(id, column) + #check for duplicates + old = self.receivedBlock.getColumn(id) for i in range(len(column)): if column[i]: - self.logger.debug("Recv: %d->%d: %d,%d", src, self.ID, i, id, extra=self.format) - if self.perNodeQueue or self.perNeighborQueue: - self.receivedQueue.append((i, id)) + if old[i]: + self.logger.debug("Recv DUP: %d->%d: %d,%d", src, self.ID, i, id, extra=self.format) + else: + self.logger.debug("Recv new: %d->%d: %d,%d", src, self.ID, i, id, extra=self.format) + if self.perNodeQueue or self.perNeighborQueue: + self.receivedQueue.append((i, id)) + self.receivedBlock.mergeColumn(id, column) self.statsRxInSlot += column.count(1) else: pass @@ -212,12 +217,17 @@ class Validator: if id in self.rowIDs: # register receive so that we are not sending back self.rowNeighbors[id][src].receiving |= row - self.receivedBlock.mergeRow(id, row) + #check for duplicates + old = self.receivedBlock.getRow(id) for i in range(len(row)): if row[i]: - self.logger.debug("Recv %d->%d: %d,%d", src, self.ID, id, i, extra=self.format) - if self.perNodeQueue or self.perNeighborQueue: - self.receivedQueue.append((id, i)) + if old[i]: + self.logger.debug("Recv DUP: %d->%d: %d,%d", src, self.ID, id, i, extra=self.format) + else: + self.logger.debug("Recv new: %d->%d: %d,%d", src, self.ID, id, i, extra=self.format) + if self.perNodeQueue or self.perNeighborQueue: + self.receivedQueue.append((id, i)) + self.receivedBlock.mergeRow(id, row) self.statsRxInSlot += row.count(1) else: pass From c0650bf75a4b797b91a2b8b94c4177a0738f6580 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 23 Feb 2023 11:21:58 +0100 Subject: [PATCH 40/89] implement partial line sending logic On any given p2p link, it only makes sense to send up to k messages, after that repair kicks in. Signed-off-by: Csaba Kiraly --- DAS/validator.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 7e02da3..f71277e 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -137,6 +137,7 @@ class Validator: self.bwUplink = 110 if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps self.repairOnTheFly = True + self.sendLineUntil = (self.shape.blockSize + 1) // 2 # stop sending on a p2p link if at least this amount of samples passed self.perNeighborQueue = False # queue incoming messages to outgoing connections on arrival (as typical GossipSub impl) self.perNodeQueue = False # keep a global queue of incoming messages for later sequential dispatch self.shuffleLines = True # shuffle the order of rows/columns in each iteration while trying to send @@ -358,6 +359,8 @@ class Validator: return def sendSegmentToNeigh(self, rID, cID, neigh): + if (neigh.sent | neigh.received).count(1) >= self.sendLineUntil: + return False # sent enough, other side can restore if neigh.dim == 0: #row i = cID else: @@ -451,7 +454,9 @@ class Validator: line = self.getRow(rID) needed = zeros(self.shape.blockSize) for neigh in neighs.values(): - needed |= ~(neigh.received | neigh.sent) + sentOrReceived = neigh.received | neigh.sent + if sentOrReceived.count(1) < self.sendLineUntil: + needed |= ~sentOrReceived needed &= line if (needed).any(): for i in range(len(needed)): @@ -462,7 +467,9 @@ class Validator: line = self.getColumn(cID) needed = zeros(self.shape.blockSize) for neigh in neighs.values(): - needed |= ~(neigh.received | neigh.sent) + sentOrReceived = neigh.received | neigh.sent + if sentOrReceived.count(1) < self.sendLineUntil: + needed |= ~sentOrReceived needed &= line if (needed).any(): for i in range(len(needed)): From a1a8a4282d9b876c4129b49438aab59409afd187 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 23 Feb 2023 11:22:28 +0100 Subject: [PATCH 41/89] fix scheduler to check result of endSegmentToNeigh Signed-off-by: Csaba Kiraly --- DAS/validator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index f71277e..bfd1109 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -431,16 +431,16 @@ class Validator: for _, neigh in shuffledDict(self.rowNeighbors[s.id], self.shuffleNeighbors): self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) if not neigh.sent[s.i] and not neigh.received[s.i]: - self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) - self.sendSegmentToNeigh(s.id, s.i, neigh) - break + if self.sendSegmentToNeigh(s.id, s.i, neigh): + self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) + break else: for _, neigh in shuffledDict(self.columnNeighbors[s.id], self.shuffleNeighbors): self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) if not neigh.sent[s.i] and not neigh.received[s.i]: - self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) - self.sendSegmentToNeigh(s.i, s.id, neigh) - break + if self.sendSegmentToNeigh(s.i, s.id, neigh): + self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) + break if self.statsTxInSlot >= self.bwUplink: if not self.segmentShuffleSchedulerPersist: From 186d430ad12f3b8a6e51e4d05545fb2c0c550b59 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 23 Feb 2023 12:55:19 +0100 Subject: [PATCH 42/89] consider shuffleLines in segmentShuffleScheduler Signed-off-by: Csaba Kiraly --- DAS/validator.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index bfd1109..dc19ed7 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -10,13 +10,16 @@ from bitarray.util import zeros from collections import deque -def shuffled(lis): +def shuffled(lis, shuffle=True): ''' Generator yielding list in shuffled order ''' # based on https://stackoverflow.com/a/60342323 - for index in random.sample(range(len(lis)), len(lis)): - yield lis[index] - + if shuffle: + for index in random.sample(range(len(lis)), len(lis)): + yield lis[index] + else: + for v in lis: + yield v def shuffledDict(d, shuffle=True): ''' Generator yielding dictionary in shuffled order @@ -479,7 +482,7 @@ class Validator: if not self.segmentsToSend: break else: - self.segmentShuffleGen = shuffled(self.segmentsToSend) + self.segmentShuffleGen = shuffled(self.segmentsToSend, self.shuffleLines) if self.dumbRandomScheduler: # dumb random scheduler picking segments at random and trying to send it From b33f829b0e42547d6e0ddabaef7e93484f662fcb Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 23 Feb 2023 20:58:16 +0100 Subject: [PATCH 43/89] proposer might push segments without participating in mesh Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 3261ad7..506775c 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -22,6 +22,18 @@ class Simulator: self.proposerID = 0 self.glob = [] + # In GossipSub the initiator might push messages without participating in the mesh. + # proposerPublishOnly regulates this behavior. If set to true, the proposer is not + # part of the p2p distribution graph, only pushes segments to it. If false, the proposer + # might get back segments from other peers since links are symmetric. + self.proposerPublishOnly = True + + # If proposerPublishOnly == True, this regulates how many copies of each segment are + # pushed out by the proposer. + # 1: the data is sent out exactly once on rows and once on columns (2 copies in total) + # self.shape.netDegree: default behavior similar (but not same) to previous code + self.proposerPublishTo = self.shape.netDegree + def initValidators(self): """It initializes all the validators in the network.""" self.glob = Observer(self.logger, self.shape) @@ -46,10 +58,11 @@ class Simulator: rowChannels = [[] for i in range(self.shape.blockSize)] columnChannels = [[] for i in range(self.shape.blockSize)] for v in self.validators: - for id in v.rowIDs: - rowChannels[id].append(v) - for id in v.columnIDs: - columnChannels[id].append(v) + if not (self.proposerPublishOnly and v.amIproposer): + for id in v.rowIDs: + rowChannels[id].append(v) + for id in v.columnIDs: + columnChannels[id].append(v) for id in range(self.shape.blockSize): @@ -82,6 +95,19 @@ class Simulator: val1.columnNeighbors[id].update({val2.ID : Neighbor(val2, 1, self.shape.blockSize)}) val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, 1, self.shape.blockSize)}) + for v in self.validators: + if (self.proposerPublishOnly and v.amIproposer): + for id in v.rowIDs: + count = min(self.proposerPublishTo, len(rowChannels[id])) + publishTo = random.sample(rowChannels[id], count) + for vi in publishTo: + v.rowNeighbors[id].update({vi.ID : Neighbor(vi, 0, self.shape.blockSize)}) + for id in v.columnIDs: + count = min(self.proposerPublishTo, len(columnChannels[id])) + publishTo = random.sample(columnChannels[id], count) + for vi in publishTo: + v.columnNeighbors[id].update({vi.ID : Neighbor(vi, 1, self.shape.blockSize)}) + if self.logger.isEnabledFor(logging.DEBUG): for i in range(0, self.shape.numberValidators): self.logger.debug("Val %d : rowN %s", i, self.validators[i].rowNeighbors, extra=self.format) From dfacd6bb18b72cddaa5577329f8a45aeaca7d428 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 23 Feb 2023 20:58:55 +0100 Subject: [PATCH 44/89] allow push from non-neighbor Signed-off-by: Csaba Kiraly --- DAS/validator.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index dc19ed7..5dfd99c 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -200,7 +200,10 @@ class Validator: """It receives the given column if it has been assigned to it.""" if id in self.columnIDs: # register receive so that we are not sending back - self.columnNeighbors[id][src].receiving |= column + if src in self.columnNeighbors[id]: # (check if peer or initial publish) + self.columnNeighbors[id][src].receiving |= column + else: + pass #check for duplicates old = self.receivedBlock.getColumn(id) for i in range(len(column)): @@ -220,7 +223,10 @@ class Validator: """It receives the given row if it has been assigned to it.""" if id in self.rowIDs: # register receive so that we are not sending back - self.rowNeighbors[id][src].receiving |= row + if src in self.rowNeighbors[id]: # (check if peer or initial publish) + self.rowNeighbors[id][src].receiving |= row + else: + pass #check for duplicates old = self.receivedBlock.getRow(id) for i in range(len(row)): From 2707269836abe0542fc5ba87d3d44e2f2761362c Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 23 Feb 2023 22:21:16 +0100 Subject: [PATCH 45/89] fixup: moving simulator config to resetShape Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 506775c..74d310e 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -22,18 +22,6 @@ class Simulator: self.proposerID = 0 self.glob = [] - # In GossipSub the initiator might push messages without participating in the mesh. - # proposerPublishOnly regulates this behavior. If set to true, the proposer is not - # part of the p2p distribution graph, only pushes segments to it. If false, the proposer - # might get back segments from other peers since links are symmetric. - self.proposerPublishOnly = True - - # If proposerPublishOnly == True, this regulates how many copies of each segment are - # pushed out by the proposer. - # 1: the data is sent out exactly once on rows and once on columns (2 copies in total) - # self.shape.netDegree: default behavior similar (but not same) to previous code - self.proposerPublishTo = self.shape.netDegree - def initValidators(self): """It initializes all the validators in the network.""" self.glob = Observer(self.logger, self.shape) @@ -132,6 +120,18 @@ class Simulator: val.shape.failureRate = shape.failureRate val.shape.chi = shape.chi + # In GossipSub the initiator might push messages without participating in the mesh. + # proposerPublishOnly regulates this behavior. If set to true, the proposer is not + # part of the p2p distribution graph, only pushes segments to it. If false, the proposer + # might get back segments from other peers since links are symmetric. + self.proposerPublishOnly = True + + # If proposerPublishOnly == True, this regulates how many copies of each segment are + # pushed out by the proposer. + # 1: the data is sent out exactly once on rows and once on columns (2 copies in total) + # self.shape.netDegree: default behavior similar (but not same) to previous code + self.proposerPublishTo = self.shape.netDegree + def run(self): """It runs the main simulation until the block is available or it gets stucked.""" From 89a6b1cdf70f24553191a6a1ee99c241f5165319 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 24 Feb 2023 00:35:17 +0100 Subject: [PATCH 46/89] remove old scheduler Signed-off-by: Csaba Kiraly --- DAS/validator.py | 132 ----------------------------------------------- 1 file changed, 132 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 5dfd99c..1b15fb4 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -148,7 +148,6 @@ class Validator: self.dumbRandomScheduler = False # dumb random scheduler self.segmentShuffleScheduler = True # send each segment that's worth sending once in shuffled order, then repeat self.segmentShuffleSchedulerPersist = True # Persist scheduler state between timesteps - self.sched = self.nextToSend() def logIDs(self): """It logs the assigned rows and columns.""" @@ -196,52 +195,6 @@ class Validator: """It returns a given row.""" return self.block.getRow(index) - def receiveColumn(self, id, column, src): - """It receives the given column if it has been assigned to it.""" - if id in self.columnIDs: - # register receive so that we are not sending back - if src in self.columnNeighbors[id]: # (check if peer or initial publish) - self.columnNeighbors[id][src].receiving |= column - else: - pass - #check for duplicates - old = self.receivedBlock.getColumn(id) - for i in range(len(column)): - if column[i]: - if old[i]: - self.logger.debug("Recv DUP: %d->%d: %d,%d", src, self.ID, i, id, extra=self.format) - else: - self.logger.debug("Recv new: %d->%d: %d,%d", src, self.ID, i, id, extra=self.format) - if self.perNodeQueue or self.perNeighborQueue: - self.receivedQueue.append((i, id)) - self.receivedBlock.mergeColumn(id, column) - self.statsRxInSlot += column.count(1) - else: - pass - - def receiveRow(self, id, row, src): - """It receives the given row if it has been assigned to it.""" - if id in self.rowIDs: - # register receive so that we are not sending back - if src in self.rowNeighbors[id]: # (check if peer or initial publish) - self.rowNeighbors[id][src].receiving |= row - else: - pass - #check for duplicates - old = self.receivedBlock.getRow(id) - for i in range(len(row)): - if row[i]: - if old[i]: - self.logger.debug("Recv DUP: %d->%d: %d,%d", src, self.ID, id, i, extra=self.format) - else: - self.logger.debug("Recv new: %d->%d: %d,%d", src, self.ID, id, i, extra=self.format) - if self.perNodeQueue or self.perNeighborQueue: - self.receivedQueue.append((id, i)) - self.receivedBlock.mergeRow(id, row) - self.statsRxInSlot += row.count(1) - else: - pass - def receiveSegment(self, rID, cID, src): # register receive so that we are not sending back if rID in self.rowIDs: @@ -307,66 +260,6 @@ class Validator: self.statsRxInSlot = 0 self.statsTxInSlot = 0 - def nextColumnToSend(self, columnID, limit = sys.maxsize): - line = self.getColumn(columnID) - if line.any(): - self.logger.debug("col %d -> %s", columnID, self.columnNeighbors[columnID] , extra=self.format) - for _, n in shuffledDict(self.columnNeighbors[columnID]): - - # if there is anything new to send, send it - toSend = line & ~n.sent & ~n.received - if (toSend).any(): - toSend = sampleLine(toSend, limit) - yield NextToSend(n, toSend, columnID, 1) - - def nextRowToSend(self, rowID, limit = sys.maxsize): - line = self.getRow(rowID) - if line.any(): - self.logger.debug("row %d -> %s", rowID, self.rowNeighbors[rowID], extra=self.format) - for _, n in shuffledDict(self.rowNeighbors[rowID]): - - # if there is anything new to send, send it - toSend = line & ~n.sent & ~n.received - if (toSend).any(): - toSend = sampleLine(toSend, limit) - yield NextToSend(n, toSend, rowID, 0) - - def nextToSend(self): - """ Send scheduler as a generator function - - Yields next segment(s) to send when asked for it. - Generates an infinite flow, returning with exit only when - there is nothing more to send. - - Generates a randomized order of columns and rows, sending to one neighbor - at each before sending to another neighbor. - Generates a new randomized ordering once all columns, rows, and neighbors - are processed once. - """ - - while True: - perLine = [] - for c in self.columnIDs: - perLine.append(self.nextColumnToSend(c, 1)) - - for r in self.rowIDs: - perLine.append(self.nextRowToSend(r, 1)) - - count = 0 - random.shuffle(perLine) - while (perLine): - for g in perLine.copy(): # we need a shallow copy to allow remove - n = next(g, None) - if not n: - perLine.remove(g) - continue - count += 1 - yield n - - # return if there is nothing more to send - if not count: - return - def sendSegmentToNeigh(self, rID, cID, neigh): if (neigh.sent | neigh.received).count(1) >= self.sendLineUntil: return False # sent enough, other side can restore @@ -521,31 +414,6 @@ class Validator: if self.statsTxInSlot >= self.bwUplink: return return - - for n in self.sched: - neigh = n.neigh - toSend = n.toSend - id = n.id - dim = n.dim - - neigh.sent |= toSend; - if dim == 0: - neigh.node.receiveRow(id, toSend, self.ID) - else: - neigh.node.receiveColumn(id, toSend, self.ID) - - sent = toSend.count(1) - self.statsTxInSlot += sent - self.logger.debug("sending %s %d to %d (%d)", - "col" if dim else "row", id, neigh.node.ID, sent, extra=self.format) - - # until we exhaust capacity - # TODO: use exact limit - if self.statsTxInSlot >= self.bwUplink: - return - - # Scheduler exited, nothing to send. Create new one for next round. - self.sched = self.nextToSend() def logRows(self): """It logs the rows assigned to the validator.""" From ead127e73edc124b8b833e7dfcd274822b54057e Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 24 Feb 2023 08:42:35 +0100 Subject: [PATCH 47/89] change defaults to queue per p2p link Signed-off-by: Csaba Kiraly --- DAS/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DAS/validator.py b/DAS/validator.py index 1b15fb4..00acc91 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -141,7 +141,7 @@ class Validator: self.repairOnTheFly = True self.sendLineUntil = (self.shape.blockSize + 1) // 2 # stop sending on a p2p link if at least this amount of samples passed - self.perNeighborQueue = False # queue incoming messages to outgoing connections on arrival (as typical GossipSub impl) + self.perNeighborQueue = True # queue incoming messages to outgoing connections on arrival (as typical GossipSub impl) self.perNodeQueue = False # keep a global queue of incoming messages for later sequential dispatch self.shuffleLines = True # shuffle the order of rows/columns in each iteration while trying to send self.shuffleNeighbors = True # shuffle the order of neighbors when sending the same segment to each neighbor From fa1818a43b314d67035c025a8b409ad389912c00 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 24 Feb 2023 08:43:18 +0100 Subject: [PATCH 48/89] simplify code Signed-off-by: Csaba Kiraly --- DAS/validator.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 00acc91..25ca426 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -8,7 +8,7 @@ from DAS.block import * from bitarray import bitarray from bitarray.util import zeros from collections import deque - +from itertools import chain def shuffled(lis, shuffle=True): ''' Generator yielding list in shuffled order @@ -236,12 +236,7 @@ class Validator: self.block.merge(self.receivedBlock) - for neighs in self.rowNeighbors.values(): - for neigh in neighs.values(): - neigh.received |= neigh.receiving - neigh.receiving.setall(0) - - for neighs in self.columnNeighbors.values(): + for neighs in chain (self.rowNeighbors.values(), self.columnNeighbors.values()): for neigh in neighs.values(): neigh.received |= neigh.receiving neigh.receiving.setall(0) @@ -332,17 +327,15 @@ class Validator: if s.dim == 0: for _, neigh in shuffledDict(self.rowNeighbors[s.id], self.shuffleNeighbors): self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) - if not neigh.sent[s.i] and not neigh.received[s.i]: - if self.sendSegmentToNeigh(s.id, s.i, neigh): - self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) - break + if self.sendSegmentToNeigh(s.id, s.i, neigh): + self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) + break else: for _, neigh in shuffledDict(self.columnNeighbors[s.id], self.shuffleNeighbors): self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) - if not neigh.sent[s.i] and not neigh.received[s.i]: - if self.sendSegmentToNeigh(s.i, s.id, neigh): - self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) - break + if self.sendSegmentToNeigh(s.i, s.id, neigh): + self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) + break if self.statsTxInSlot >= self.bwUplink: if not self.segmentShuffleSchedulerPersist: From 300bc19c677b3bedb4a611698b23d06dc56a3f8d Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 24 Feb 2023 08:55:26 +0100 Subject: [PATCH 49/89] factorize send code Signed-off-by: Csaba Kiraly --- DAS/validator.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 25ca426..8f0436d 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -270,11 +270,7 @@ class Validator: else: return False # received or already sent - def send(self): - """ Send as much as we can in the timeslot, limited by bwUplink - """ - - # process node level send queue + def processSendQueue(self): while self.sendQueue: (rID, cID) = self.sendQueue[0] @@ -294,7 +290,7 @@ class Validator: self.sendQueue.popleft() - # process neighbor level send queues in shuffled breadth-first order + def processPerNeighborSendQueue(self): progress = True while (progress): progress = False @@ -314,8 +310,7 @@ class Validator: if self.statsTxInSlot >= self.bwUplink: return - # process possible segments to send in shuffled breadth-first order - if self.segmentShuffleScheduler: + def runSegmentShuffleScheduler(self): # This scheduler check which owned segments needs sending (at least # one neighbor needing it). Then it sends each segment that's worth sending # once, in shuffled order. This is repeated until bw limit. @@ -376,7 +371,7 @@ class Validator: else: self.segmentShuffleGen = shuffled(self.segmentsToSend, self.shuffleLines) - if self.dumbRandomScheduler: + def runDumbRandomScheduler(self): # dumb random scheduler picking segments at random and trying to send it tries = 100 t = tries @@ -408,6 +403,23 @@ class Validator: return return + def send(self): + """ Send as much as we can in the timeslot, limited by bwUplink + """ + + # process node level send queue + self.processSendQueue() + + # process neighbor level send queues in shuffled breadth-first order + self.processPerNeighborSendQueue() + + # process possible segments to send in shuffled breadth-first order + if self.segmentShuffleScheduler: + self.runSegmentShuffleScheduler() + + if self.dumbRandomScheduler: + self.runDumbRandomScheduler() + def logRows(self): """It logs the rows assigned to the validator.""" if self.logger.isEnabledFor(logging.DEBUG): From a03371cf4eb34c8948e1c5a5d1d6420e975f44fe Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 24 Feb 2023 10:21:28 +0100 Subject: [PATCH 50/89] add logging of TX and RX statistics Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/DAS/simulator.py b/DAS/simulator.py index 74d310e..2fed35f 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -3,6 +3,7 @@ import networkx as nx import logging, random from datetime import datetime +from statistics import mean from DAS.tools import * from DAS.results import * from DAS.observer import * @@ -158,6 +159,15 @@ class Simulator: for i in range(0,self.shape.numberValidators): self.validators[i].logRows() self.validators[i].logColumns() + + # log TX and RX statistics + statsTxInSlot = [v.statsTxInSlot for v in self.validators] + statsRxInSlot = [v.statsRxInSlot for v in self.validators] + self.logger.debug("step %d: TX_prod=%.1f, RX_prod=%.1f, TX_avg=%.1f, TX_max=%.1f, Rx_avg=%.1f, Rx_max=%.1f" % + (steps, statsTxInSlot[0], statsRxInSlot[0], + mean(statsTxInSlot[1:]), max(statsTxInSlot[1:]), + mean(statsRxInSlot[1:]), max(statsRxInSlot[1:])), extra=self.format) + for i in range(0,self.shape.numberValidators): self.validators[i].updateStats() arrived, expected = self.glob.checkStatus(self.validators) From 0a418b35b2c1ebb63f21d9a9d40b7d2d76ef2ee3 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 24 Feb 2023 10:24:19 +0100 Subject: [PATCH 51/89] parametrize dumbRandomScheduler Signed-off-by: Csaba Kiraly --- DAS/validator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 8f0436d..2f4bf8e 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -371,9 +371,8 @@ class Validator: else: self.segmentShuffleGen = shuffled(self.segmentsToSend, self.shuffleLines) - def runDumbRandomScheduler(self): + def runDumbRandomScheduler(self, tries = 100): # dumb random scheduler picking segments at random and trying to send it - tries = 100 t = tries while t: if self.rowIDs: From d9a2d5d606f8738cd7a76793742ee5009925fa00 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 24 Feb 2023 12:04:07 +0100 Subject: [PATCH 52/89] fixup: ensure bw limit is respected Lost meaning of return while factorizing schedulers. Fix it by checking limits after each call. Signed-off-by: Csaba Kiraly --- DAS/validator.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DAS/validator.py b/DAS/validator.py index 2f4bf8e..4d5aa27 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -408,16 +408,24 @@ class Validator: # process node level send queue self.processSendQueue() + if self.statsTxInSlot >= self.bwUplink: + return # process neighbor level send queues in shuffled breadth-first order self.processPerNeighborSendQueue() + if self.statsTxInSlot >= self.bwUplink: + return # process possible segments to send in shuffled breadth-first order if self.segmentShuffleScheduler: self.runSegmentShuffleScheduler() + if self.statsTxInSlot >= self.bwUplink: + return if self.dumbRandomScheduler: self.runDumbRandomScheduler() + if self.statsTxInSlot >= self.bwUplink: + return def logRows(self): """It logs the rows assigned to the validator.""" From f95a393068f03fd27b117d830267091b01167c48 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 24 Feb 2023 12:10:34 +0100 Subject: [PATCH 53/89] improve perNeighborSendQueue - improve shuffling between rows and columns - speed up code execution Signed-off-by: Csaba Kiraly --- DAS/validator.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 4d5aa27..6410422 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -142,6 +142,7 @@ class Validator: self.repairOnTheFly = True self.sendLineUntil = (self.shape.blockSize + 1) // 2 # stop sending on a p2p link if at least this amount of samples passed self.perNeighborQueue = True # queue incoming messages to outgoing connections on arrival (as typical GossipSub impl) + self.shuffleQueues = True # shuffle the order of picking from active queues of a sender node self.perNodeQueue = False # keep a global queue of incoming messages for later sequential dispatch self.shuffleLines = True # shuffle the order of rows/columns in each iteration while trying to send self.shuffleNeighbors = True # shuffle the order of neighbors when sending the same segment to each neighbor @@ -294,21 +295,27 @@ class Validator: progress = True while (progress): progress = False - for rID, neighs in shuffledDict(self.rowNeighbors, self.shuffleLines): - for _, neigh in shuffledDict(neighs, self.shuffleNeighbors): - if (neigh.sendQueue): - self.sendSegmentToNeigh(rID, neigh.sendQueue.popleft(), neigh) - progress = True - if self.statsTxInSlot >= self.bwUplink: - return - for cID, neighs in shuffledDict(self.columnNeighbors, self.shuffleLines): - for _, neigh in shuffledDict(neighs, self.shuffleNeighbors): + queues = [] + # collect and shuffle + for rID, neighs in self.rowNeighbors.items(): + for neigh in neighs.values(): if (neigh.sendQueue): - self.sendSegmentToNeigh(neigh.sendQueue.popleft(), cID, neigh) - progress = True - if self.statsTxInSlot >= self.bwUplink: - return + queues.append((0, rID, neigh)) + + for cID, neighs in self.columnNeighbors.items(): + for neigh in neighs.values(): + if (neigh.sendQueue): + queues.append((1, cID, neigh)) + + for dim, lineID, neigh in shuffled(queues, self.shuffleQueues): + if dim == 0: + self.sendSegmentToNeigh(lineID, neigh.sendQueue.popleft(), neigh) + else: + self.sendSegmentToNeigh(neigh.sendQueue.popleft(), lineID, neigh) + progress = True + if self.statsTxInSlot >= self.bwUplink: + return def runSegmentShuffleScheduler(self): # This scheduler check which owned segments needs sending (at least From 82ee2b5189876ceaf619aead2c551740ea696d1f Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 28 Feb 2023 09:35:18 +0100 Subject: [PATCH 54/89] simplify dumbRandomScheduler code Signed-off-by: Csaba Kiraly --- DAS/validator.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 6410422..89930f6 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -61,13 +61,6 @@ def sampleLine(line, limit): limit -= 1 return r -class NextToSend: - def __init__(self, neigh, toSend, id, dim): - self.neigh = neigh - self.toSend = toSend - self.id = id - self.dim = dim - class SegmentToSend: def __init__(self, dim, id, i): self.dim = dim @@ -387,10 +380,7 @@ class Validator: cID = random.randrange(0, self.shape.blockSize) if self.block.getSegment(rID, cID) : neigh = random.choice(list(self.rowNeighbors[rID].values())) - if not neigh.sent[cID] and not neigh.received[cID] : - neigh.sent[cID] = 1 - neigh.node.receiveSegment(rID, cID, self.ID) - self.statsTxInSlot += 1 + if self.sendSegmentToNeigh(rID, cID, neigh): t = tries if self.statsTxInSlot >= self.bwUplink: return @@ -399,10 +389,7 @@ class Validator: rID = random.randrange(0, self.shape.blockSize) if self.block.getSegment(rID, cID) : neigh = random.choice(list(self.columnNeighbors[cID].values())) - if not neigh.sent[rID] and not neigh.received[rID] : - neigh.sent[rID] = 1 - neigh.node.receiveSegment(rID, cID, self.ID) - self.statsTxInSlot += 1 + if self.sendSegmentToNeigh(rID, cID, neigh): t = tries t -= 1 if self.statsTxInSlot >= self.bwUplink: From b5368b4e43a737fbdebb7002dc7fd4291888c312 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 28 Feb 2023 12:24:37 +0100 Subject: [PATCH 55/89] factorize restore Signed-off-by: Csaba Kiraly --- DAS/validator.py | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 89930f6..2e1b737 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -437,29 +437,35 @@ class Validator: """It restores the rows assigned to the validator, that can be repaired.""" if self.repairOnTheFly: for id in self.rowIDs: - rep = self.block.repairRow(id) - if (rep.any()): - # If operation is based on send queues, segments should - # be queued after successful repair. - for i in range(len(rep)): - if rep[i]: - self.logger.debug("Rep: %d,%d", id, i, extra=self.format) - self.addToSendQueue(id, i) - # self.statsRepairInSlot += rep.count(1) + self.restoreRow(id) + + def restoreRow(self, id): + rep = self.block.repairRow(id) + if (rep.any()): + # If operation is based on send queues, segments should + # be queued after successful repair. + for i in range(len(rep)): + if rep[i]: + self.logger.debug("Rep: %d,%d", id, i, extra=self.format) + self.addToSendQueue(id, i) + # self.statsRepairInSlot += rep.count(1) def restoreColumns(self): """It restores the columns assigned to the validator, that can be repaired.""" if self.repairOnTheFly: for id in self.columnIDs: - rep = self.block.repairColumn(id) - if (rep.any()): - # If operation is based on send queues, segments should - # be queued after successful repair. - for i in range(len(rep)): - if rep[i]: - self.logger.debug("Rep: %d,%d", i, id, extra=self.format) - self.addToSendQueue(i, id) - # self.statsRepairInSlot += rep.count(1) + self.restoreColumn(id) + + def restoreColumn(self, id): + rep = self.block.repairColumn(id) + if (rep.any()): + # If operation is based on send queues, segments should + # be queued after successful repair. + for i in range(len(rep)): + if rep[i]: + self.logger.debug("Rep: %d,%d", i, id, extra=self.format) + self.addToSendQueue(i, id) + # self.statsRepairInSlot += rep.count(1) def checkStatus(self): """It checks how many expected/arrived samples are for each assigned row/column.""" From 2bf85c41a24eb58c8e1403023f72cdf4b9a77e65 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 1 Mar 2023 09:53:13 +0100 Subject: [PATCH 56/89] factorize send code Signed-off-by: Csaba Kiraly --- DAS/validator.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 2e1b737..560d0d1 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -249,35 +249,43 @@ class Validator: self.statsRxInSlot = 0 self.statsTxInSlot = 0 - def sendSegmentToNeigh(self, rID, cID, neigh): + def checkSegmentToNeigh(self, rID, cID, neigh): if (neigh.sent | neigh.received).count(1) >= self.sendLineUntil: return False # sent enough, other side can restore - if neigh.dim == 0: #row - i = cID - else: - i = rID + i = rID if neigh.dim else cID if not neigh.sent[i] and not neigh.received[i] : - neigh.sent[i] = 1 - neigh.node.receiveSegment(rID, cID, self.ID) - self.statsTxInSlot += 1 return True else: return False # received or already sent + def sendSegmentToNeigh(self, rID, cID, neigh): + self.logger.debug("sending %d/%d to %d", rID, cID, neigh.node.ID, extra=self.format) + i = rID if neigh.dim else cID + neigh.sent[i] = 1 + neigh.node.receiveSegment(rID, cID, self.ID) + self.statsTxInSlot += 1 + + def checkSendSegmentToNeigh(self, rID, cID, neigh): + if self.checkSegmentToNeigh(rID, cID, neigh): + self.sendSegmentToNeigh(rID, cID, neigh) + return True + else: + return False + def processSendQueue(self): while self.sendQueue: (rID, cID) = self.sendQueue[0] if rID in self.rowIDs: for _, neigh in shuffledDict(self.rowNeighbors[rID], self.shuffleNeighbors): - self.sendSegmentToNeigh(rID, cID, neigh) + self.checkSendSegmentToNeigh(rID, cID, neigh) if self.statsTxInSlot >= self.bwUplink: return if cID in self.columnIDs: for _, neigh in shuffledDict(self.columnNeighbors[cID], self.shuffleNeighbors): - self.sendSegmentToNeigh(rID, cID, neigh) + self.checkSendSegmentToNeigh(rID, cID, neigh) if self.statsTxInSlot >= self.bwUplink: return @@ -303,9 +311,9 @@ class Validator: for dim, lineID, neigh in shuffled(queues, self.shuffleQueues): if dim == 0: - self.sendSegmentToNeigh(lineID, neigh.sendQueue.popleft(), neigh) + self.checkSendSegmentToNeigh(lineID, neigh.sendQueue.popleft(), neigh) else: - self.sendSegmentToNeigh(neigh.sendQueue.popleft(), lineID, neigh) + self.checkSendSegmentToNeigh(neigh.sendQueue.popleft(), lineID, neigh) progress = True if self.statsTxInSlot >= self.bwUplink: return @@ -322,13 +330,13 @@ class Validator: if s.dim == 0: for _, neigh in shuffledDict(self.rowNeighbors[s.id], self.shuffleNeighbors): self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) - if self.sendSegmentToNeigh(s.id, s.i, neigh): + if self.checkSendSegmentToNeigh(s.id, s.i, neigh): self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) break else: for _, neigh in shuffledDict(self.columnNeighbors[s.id], self.shuffleNeighbors): self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) - if self.sendSegmentToNeigh(s.i, s.id, neigh): + if self.checkSendSegmentToNeigh(s.i, s.id, neigh): self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) break @@ -380,7 +388,7 @@ class Validator: cID = random.randrange(0, self.shape.blockSize) if self.block.getSegment(rID, cID) : neigh = random.choice(list(self.rowNeighbors[rID].values())) - if self.sendSegmentToNeigh(rID, cID, neigh): + if self.checkSendSegmentToNeigh(rID, cID, neigh): t = tries if self.statsTxInSlot >= self.bwUplink: return @@ -389,7 +397,7 @@ class Validator: rID = random.randrange(0, self.shape.blockSize) if self.block.getSegment(rID, cID) : neigh = random.choice(list(self.columnNeighbors[cID].values())) - if self.sendSegmentToNeigh(rID, cID, neigh): + if self.checkSendSegmentToNeigh(rID, cID, neigh): t = tries t -= 1 if self.statsTxInSlot >= self.bwUplink: From 3095e440c678ff7226fec3380649437a491c048a Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 1 Mar 2023 10:41:47 +0100 Subject: [PATCH 57/89] factorize segmentShuffleScheduler code Signed-off-by: Csaba Kiraly --- DAS/validator.py | 80 +++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 560d0d1..33b9cf9 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -61,12 +61,6 @@ def sampleLine(line, limit): limit -= 1 return r -class SegmentToSend: - def __init__(self, dim, id, i): - self.dim = dim - self.id = id - self.i = i - class Neighbor: """This class implements a node neighbor to monitor sent and received data.""" @@ -319,35 +313,13 @@ class Validator: return def runSegmentShuffleScheduler(self): - # This scheduler check which owned segments needs sending (at least - # one neighbor needing it). Then it sends each segment that's worth sending - # once, in shuffled order. This is repeated until bw limit. - while True: - if hasattr(self, 'segmentShuffleGen') and self.segmentShuffleGen is not None: - #self.logger.debug("TX:%d queue:%d", self.statsTxInSlot, len(self.segmentsToSend), extra=self.format) - for s in self.segmentShuffleGen: - self.logger.debug("%d:%d/%d", s.dim, s.id, s.i, extra=self.format) - if s.dim == 0: - for _, neigh in shuffledDict(self.rowNeighbors[s.id], self.shuffleNeighbors): - self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) - if self.checkSendSegmentToNeigh(s.id, s.i, neigh): - self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) - break - else: - for _, neigh in shuffledDict(self.columnNeighbors[s.id], self.shuffleNeighbors): - self.logger.debug("%d or %d", neigh.sent[s.i], neigh.received[s.i], extra=self.format) - if self.checkSendSegmentToNeigh(s.i, s.id, neigh): - self.logger.debug("sending to %d", neigh.node.ID, extra=self.format) - break + # This scheduler check which owned segments needs sending (at least + # one neighbor needing it). Then it sends each segment that's worth sending + # once, in shuffled order. This is repeated until bw limit. - if self.statsTxInSlot >= self.bwUplink: - if not self.segmentShuffleSchedulerPersist: - # remove scheduler state before leaving - self.segmentsToSend = [] - self.segmentShuffleGen = None - return - - self.segmentsToSend = [] + def collectSegmentsToSend(): + # yields list of segments to send as (dim, lineID, id) + segmentsToSend = [] for rID, neighs in self.rowNeighbors.items(): line = self.getRow(rID) needed = zeros(self.shape.blockSize) @@ -359,7 +331,7 @@ class Validator: if (needed).any(): for i in range(len(needed)): if needed[i]: - self.segmentsToSend.append(SegmentToSend(0, rID, i)) + segmentsToSend.append((0, rID, i)) for cID, neighs in self.columnNeighbors.items(): line = self.getColumn(cID) @@ -372,12 +344,44 @@ class Validator: if (needed).any(): for i in range(len(needed)): if needed[i]: - self.segmentsToSend.append(SegmentToSend(1, cID, i)) + segmentsToSend.append((1, cID, i)) - if not self.segmentsToSend: + return segmentsToSend + + def nextSegment(): + while True: + # send each collected segment once + if hasattr(self, 'segmentShuffleGen') and self.segmentShuffleGen is not None: + for dim, lineID, id in self.segmentShuffleGen: + if dim == 0: + for _, neigh in shuffledDict(self.rowNeighbors[lineID], self.shuffleNeighbors): + if self.checkSegmentToNeigh(lineID, id, neigh): + yield((lineID, id, neigh)) + break + else: + for _, neigh in shuffledDict(self.columnNeighbors[lineID], self.shuffleNeighbors): + if self.checkSegmentToNeigh(id, lineID, neigh): + yield((id, lineID, neigh)) + break + + # collect segments for next round + segmentsToSend = collectSegmentsToSend() + + # finish if empty or set up shuffled generator based on collected segments + if not segmentsToSend: break else: - self.segmentShuffleGen = shuffled(self.segmentsToSend, self.shuffleLines) + self.segmentShuffleGen = shuffled(segmentsToSend, self.shuffleLines) + + for rid, cid, neigh in nextSegment(): + # segments are checked just before yield, so we can send directly + self.sendSegmentToNeigh(rid, cid, neigh) + + if self.statsTxInSlot >= self.bwUplink: + if not self.segmentShuffleSchedulerPersist: + # remove scheduler state before leaving + self.segmentShuffleGen = None + return def runDumbRandomScheduler(self, tries = 100): # dumb random scheduler picking segments at random and trying to send it From e611b5143c231d6a3d24686caede0fa043ddd76a Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 1 Mar 2023 10:55:04 +0100 Subject: [PATCH 58/89] refactor dumbRandomScheduler Signed-off-by: Csaba Kiraly --- DAS/validator.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 33b9cf9..02bf738 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -384,7 +384,9 @@ class Validator: return def runDumbRandomScheduler(self, tries = 100): - # dumb random scheduler picking segments at random and trying to send it + # dumb random scheduler picking segments at random and trying to send it + + def nextSegment(): t = tries while t: if self.rowIDs: @@ -392,21 +394,25 @@ class Validator: cID = random.randrange(0, self.shape.blockSize) if self.block.getSegment(rID, cID) : neigh = random.choice(list(self.rowNeighbors[rID].values())) - if self.checkSendSegmentToNeigh(rID, cID, neigh): + if self.checkSegmentToNeigh(rID, cID, neigh): + yield(rID, cID, neigh) t = tries - if self.statsTxInSlot >= self.bwUplink: - return if self.columnIDs: cID = random.choice(self.columnIDs) rID = random.randrange(0, self.shape.blockSize) if self.block.getSegment(rID, cID) : neigh = random.choice(list(self.columnNeighbors[cID].values())) - if self.checkSendSegmentToNeigh(rID, cID, neigh): + if self.checkSegmentToNeigh(rID, cID, neigh): + yield(rID, cID, neigh) t = tries t -= 1 - if self.statsTxInSlot >= self.bwUplink: - return - return + + for rid, cid, neigh in nextSegment(): + # segments are checked just before yield, so we can send directly + self.sendSegmentToNeigh(rid, cid, neigh) + + if self.statsTxInSlot >= self.bwUplink: + return def send(self): """ Send as much as we can in the timeslot, limited by bwUplink From 68fdaf3572729b41b5d548a1cf9e72d3a4f400e2 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 1 Mar 2023 22:21:31 +0100 Subject: [PATCH 59/89] add method descriptions Signed-off-by: Csaba Kiraly --- DAS/block.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DAS/block.py b/DAS/block.py index 228e743..10ff30d 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -21,10 +21,12 @@ class Block: self.data |= merged.data def getSegment(self, rowID, columnID): + """Check whether a segment is included""" return self.data[rowID*self.blockSize + columnID] - def setSegment(self, rowID, columnID, v = 1): - self.data[rowID*self.blockSize + columnID] = v + def setSegment(self, rowID, columnID, value = 1): + """Set value for a segment (default 1)""" + self.data[rowID*self.blockSize + columnID] = value def getColumn(self, columnID): """It returns the block column corresponding to columnID.""" From 66a9d66dc6ee6c63973814fa84437b2ffe140a49 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 1 Mar 2023 22:34:16 +0100 Subject: [PATCH 60/89] moving helper functions to tools.py Signed-off-by: Csaba Kiraly --- DAS/tools.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++- DAS/validator.py | 54 +----------------------------------------------- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/DAS/tools.py b/DAS/tools.py index fb40c71..6852a9d 100644 --- a/DAS/tools.py +++ b/DAS/tools.py @@ -1,7 +1,9 @@ #!/bin/python3 import logging - +import sys +import random +from bitarray.util import zeros class CustomFormatter(): """This class defines the terminal output formatting.""" @@ -28,3 +30,53 @@ class CustomFormatter(): formatter = logging.Formatter(log_fmt) return formatter.format(record) +def shuffled(lis, shuffle=True): + ''' Generator yielding list in shuffled order + ''' + # based on https://stackoverflow.com/a/60342323 + if shuffle: + for index in random.sample(range(len(lis)), len(lis)): + yield lis[index] + else: + for v in lis: + yield v +def shuffledDict(d, shuffle=True): + ''' Generator yielding dictionary in shuffled order + + Shuffle, except if not (optional parameter useful for experiment setup) + ''' + if shuffle: + lis = list(d.items()) + for index in random.sample(range(len(d)), len(d)): + yield lis[index] + else: + for kv in d.items(): + yield kv + +def sampleLine(line, limit): + """ sample up to 'limit' bits from a bitarray + + Since this is quite expensive, we use a number of heuristics to get it fast. + """ + if limit == sys.maxsize : + return line + else: + w = line.count(1) + if limit >= w : + return line + else: + l = len(line) + r = zeros(l) + if w < l/10 or limit > l/2 : + indices = [ i for i in range(l) if line[i] ] + sample = random.sample(indices, limit) + for i in sample: + r[i] = 1 + return r + else: + while limit: + i = random.randrange(0, l) + if line[i] and not r[i]: + r[i] = 1 + limit -= 1 + return r diff --git a/DAS/validator.py b/DAS/validator.py index 02bf738..5fa6aee 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -3,64 +3,12 @@ import random import collections import logging -import sys from DAS.block import * -from bitarray import bitarray +from DAS.tools import shuffled, shuffledDict from bitarray.util import zeros from collections import deque from itertools import chain -def shuffled(lis, shuffle=True): - ''' Generator yielding list in shuffled order - ''' - # based on https://stackoverflow.com/a/60342323 - if shuffle: - for index in random.sample(range(len(lis)), len(lis)): - yield lis[index] - else: - for v in lis: - yield v -def shuffledDict(d, shuffle=True): - ''' Generator yielding dictionary in shuffled order - - Shuffle, except if not (optional parameter useful for experiment setup) - ''' - if shuffle: - lis = list(d.items()) - for index in random.sample(range(len(d)), len(d)): - yield lis[index] - else: - for kv in d.items(): - yield kv - -def sampleLine(line, limit): - """ sample up to 'limit' bits from a bitarray - - Since this is quite expensive, we use a number of heuristics to get it fast. - """ - if limit == sys.maxsize : - return line - else: - w = line.count(1) - if limit >= w : - return line - else: - l = len(line) - r = zeros(l) - if w < l/10 or limit > l/2 : - indices = [ i for i in range(l) if line[i] ] - sample = random.sample(indices, limit) - for i in sample: - r[i] = 1 - return r - else: - while limit: - i = random.randrange(0, l) - if line[i] and not r[i]: - r[i] = 1 - limit -= 1 - return r - class Neighbor: """This class implements a node neighbor to monitor sent and received data.""" From daee84b9ea00c41d5c9e33097afd50b850028f3b Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 1 Mar 2023 23:59:35 +0100 Subject: [PATCH 61/89] add more function docustrings Signed-off-by: Csaba Kiraly --- DAS/validator.py | 49 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 5fa6aee..72f2610 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -10,7 +10,12 @@ from collections import deque from itertools import chain class Neighbor: - """This class implements a node neighbor to monitor sent and received data.""" + """This class implements a node neighbor to monitor sent and received data. + + It represents one side of a P2P link in the overlay. Sent and received + segments are monitored to avoid sending twice or sending back what was + received from a link. + """ def __repr__(self): """It returns the amount of sent and received data.""" @@ -132,6 +137,7 @@ class Validator: return self.block.getRow(index) def receiveSegment(self, rID, cID, src): + """Receive a segment, register it, and queue for forwarding as needed""" # register receive so that we are not sending back if rID in self.rowIDs: if src in self.rowNeighbors[rID]: @@ -150,6 +156,7 @@ class Validator: self.statsRxInSlot += 1 def addToSendQueue(self, rID, cID): + """Queue a segment for forwarding""" if self.perNodeQueue: self.sendQueue.append((rID, cID)) @@ -163,7 +170,7 @@ class Validator: neigh.sendQueue.append(rID) def receiveRowsColumns(self): - """It receives rows and columns.""" + """Finalize time step by merging newly received segments in state""" if self.amIproposer == 1: self.logger.error("I am a block proposer", extra=self.format) else: @@ -192,6 +199,7 @@ class Validator: self.statsTxInSlot = 0 def checkSegmentToNeigh(self, rID, cID, neigh): + """Check if a segment should be sent to a neighbor""" if (neigh.sent | neigh.received).count(1) >= self.sendLineUntil: return False # sent enough, other side can restore i = rID if neigh.dim else cID @@ -201,6 +209,7 @@ class Validator: return False # received or already sent def sendSegmentToNeigh(self, rID, cID, neigh): + """Send segment to a neighbor (without checks)""" self.logger.debug("sending %d/%d to %d", rID, cID, neigh.node.ID, extra=self.format) i = rID if neigh.dim else cID neigh.sent[i] = 1 @@ -208,6 +217,7 @@ class Validator: self.statsTxInSlot += 1 def checkSendSegmentToNeigh(self, rID, cID, neigh): + """Check and send a segment to a neighbor if needed""" if self.checkSegmentToNeigh(rID, cID, neigh): self.sendSegmentToNeigh(rID, cID, neigh) return True @@ -215,6 +225,11 @@ class Validator: return False def processSendQueue(self): + """Send out segments from queue until bandwidth limit reached + + SendQueue is a centralized queue from which segments are sent out + in FIFO order to all interested neighbors. + """ while self.sendQueue: (rID, cID) = self.sendQueue[0] @@ -235,6 +250,15 @@ class Validator: self.sendQueue.popleft() def processPerNeighborSendQueue(self): + """Send out segments from per-neighbor queues until bandwidth limit reached + + Segments are dispatched from per-neighbor transmission queues in a shuffled + round-robin order, emulating a type of fair queuing. Since neighborhood is + handled at the topic (column or row) level, fair queuing is also at the level + of flows per topic and per peer. A per-peer model might be closer to the + reality of libp2p implementations where topics between two nodes are + multiplexed over the same transport. + """ progress = True while (progress): progress = False @@ -261,9 +285,12 @@ class Validator: return def runSegmentShuffleScheduler(self): - # This scheduler check which owned segments needs sending (at least - # one neighbor needing it). Then it sends each segment that's worth sending - # once, in shuffled order. This is repeated until bw limit. + """ Schedule chunks for sending + + This scheduler check which owned segments needs sending (at least + one neighbor needing it). Then it sends each segment that's worth sending + once, in shuffled order. This is repeated until bw limit. + """ def collectSegmentsToSend(): # yields list of segments to send as (dim, lineID, id) @@ -332,7 +359,13 @@ class Validator: return def runDumbRandomScheduler(self, tries = 100): - # dumb random scheduler picking segments at random and trying to send it + """Random scheduler picking segments at random + + This scheduler implements a simple random scheduling order picking + segments at random and peers potentially interested in that segment + also at random. + It server more as a performance baseline than as a realistic model. + """ def nextSegment(): t = tries @@ -363,7 +396,7 @@ class Validator: return def send(self): - """ Send as much as we can in the timeslot, limited by bwUplink + """ Send as much as we can in the timestep, limited by bwUplink """ # process node level send queue @@ -406,6 +439,7 @@ class Validator: self.restoreRow(id) def restoreRow(self, id): + """Restore a given row if repairable.""" rep = self.block.repairRow(id) if (rep.any()): # If operation is based on send queues, segments should @@ -423,6 +457,7 @@ class Validator: self.restoreColumn(id) def restoreColumn(self, id): + """Restore a given column if repairable.""" rep = self.block.repairColumn(id) if (rep.any()): # If operation is based on send queues, segments should From b4348b0005488c4dd04223b3f2b463086cc7ff67 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Fri, 3 Mar 2023 11:47:27 +0100 Subject: [PATCH 62/89] Cosmetic changes for documentation --- DAS/block.py | 2 +- DAS/tools.py | 13 ++++++------- DAS/validator.py | 41 ++++++++++++++++++++--------------------- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/DAS/block.py b/DAS/block.py index 10ff30d..f76a944 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -59,7 +59,7 @@ class Block: def repairRow(self, id): """It repairs the entire row if it has at least blockSize/2 ones. - Returns: list of repaired segments + Returns: list of repaired segments. """ line = self.data[id*self.blockSize:(id+1)*self.blockSize] success = line.count(1) diff --git a/DAS/tools.py b/DAS/tools.py index 6852a9d..cd26850 100644 --- a/DAS/tools.py +++ b/DAS/tools.py @@ -31,8 +31,7 @@ class CustomFormatter(): return formatter.format(record) def shuffled(lis, shuffle=True): - ''' Generator yielding list in shuffled order - ''' + """Generator yielding list in shuffled order.""" # based on https://stackoverflow.com/a/60342323 if shuffle: for index in random.sample(range(len(lis)), len(lis)): @@ -41,10 +40,10 @@ def shuffled(lis, shuffle=True): for v in lis: yield v def shuffledDict(d, shuffle=True): - ''' Generator yielding dictionary in shuffled order + """Generator yielding dictionary in shuffled order. - Shuffle, except if not (optional parameter useful for experiment setup) - ''' + Shuffle, except if not (optional parameter useful for experiment setup). + """ if shuffle: lis = list(d.items()) for index in random.sample(range(len(d)), len(d)): @@ -54,9 +53,9 @@ def shuffledDict(d, shuffle=True): yield kv def sampleLine(line, limit): - """ sample up to 'limit' bits from a bitarray + """Sample up to 'limit' bits from a bitarray. - Since this is quite expensive, we use a number of heuristics to get it fast. + Since this is quite expensive, we use a number of heuristics to get it fast. """ if limit == sys.maxsize : return line diff --git a/DAS/validator.py b/DAS/validator.py index 72f2610..7b52e58 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -14,7 +14,7 @@ class Neighbor: It represents one side of a P2P link in the overlay. Sent and received segments are monitored to avoid sending twice or sending back what was - received from a link. + received from a link. """ def __repr__(self): @@ -137,7 +137,7 @@ class Validator: return self.block.getRow(index) def receiveSegment(self, rID, cID, src): - """Receive a segment, register it, and queue for forwarding as needed""" + """Receive a segment, register it, and queue for forwarding as needed.""" # register receive so that we are not sending back if rID in self.rowIDs: if src in self.rowNeighbors[rID]: @@ -156,7 +156,7 @@ class Validator: self.statsRxInSlot += 1 def addToSendQueue(self, rID, cID): - """Queue a segment for forwarding""" + """Queue a segment for forwarding.""" if self.perNodeQueue: self.sendQueue.append((rID, cID)) @@ -170,7 +170,7 @@ class Validator: neigh.sendQueue.append(rID) def receiveRowsColumns(self): - """Finalize time step by merging newly received segments in state""" + """Finalize time step by merging newly received segments in state.""" if self.amIproposer == 1: self.logger.error("I am a block proposer", extra=self.format) else: @@ -199,7 +199,7 @@ class Validator: self.statsTxInSlot = 0 def checkSegmentToNeigh(self, rID, cID, neigh): - """Check if a segment should be sent to a neighbor""" + """Check if a segment should be sent to a neighbor.""" if (neigh.sent | neigh.received).count(1) >= self.sendLineUntil: return False # sent enough, other side can restore i = rID if neigh.dim else cID @@ -209,7 +209,7 @@ class Validator: return False # received or already sent def sendSegmentToNeigh(self, rID, cID, neigh): - """Send segment to a neighbor (without checks)""" + """Send segment to a neighbor (without checks).""" self.logger.debug("sending %d/%d to %d", rID, cID, neigh.node.ID, extra=self.format) i = rID if neigh.dim else cID neigh.sent[i] = 1 @@ -217,7 +217,7 @@ class Validator: self.statsTxInSlot += 1 def checkSendSegmentToNeigh(self, rID, cID, neigh): - """Check and send a segment to a neighbor if needed""" + """Check and send a segment to a neighbor if needed.""" if self.checkSegmentToNeigh(rID, cID, neigh): self.sendSegmentToNeigh(rID, cID, neigh) return True @@ -225,10 +225,10 @@ class Validator: return False def processSendQueue(self): - """Send out segments from queue until bandwidth limit reached - + """Send out segments from queue until bandwidth limit reached. + SendQueue is a centralized queue from which segments are sent out - in FIFO order to all interested neighbors. + in FIFO order to all interested neighbors. """ while self.sendQueue: (rID, cID) = self.sendQueue[0] @@ -250,13 +250,13 @@ class Validator: self.sendQueue.popleft() def processPerNeighborSendQueue(self): - """Send out segments from per-neighbor queues until bandwidth limit reached - + """Send out segments from per-neighbor queues until bandwidth limit reached. + Segments are dispatched from per-neighbor transmission queues in a shuffled round-robin order, emulating a type of fair queuing. Since neighborhood is handled at the topic (column or row) level, fair queuing is also at the level of flows per topic and per peer. A per-peer model might be closer to the - reality of libp2p implementations where topics between two nodes are + reality of libp2p implementations where topics between two nodes are multiplexed over the same transport. """ progress = True @@ -285,8 +285,8 @@ class Validator: return def runSegmentShuffleScheduler(self): - """ Schedule chunks for sending - + """ Schedule chunks for sending. + This scheduler check which owned segments needs sending (at least one neighbor needing it). Then it sends each segment that's worth sending once, in shuffled order. This is repeated until bw limit. @@ -359,12 +359,12 @@ class Validator: return def runDumbRandomScheduler(self, tries = 100): - """Random scheduler picking segments at random - - This scheduler implements a simple random scheduling order picking + """Random scheduler picking segments at random. + + This scheduler implements a simple random scheduling order picking segments at random and peers potentially interested in that segment also at random. - It server more as a performance baseline than as a realistic model. + It serves more as a performance baseline than as a realistic model. """ def nextSegment(): @@ -396,8 +396,7 @@ class Validator: return def send(self): - """ Send as much as we can in the timestep, limited by bwUplink - """ + """ Send as much as we can in the timestep, limited by bwUplink.""" # process node level send queue self.processSendQueue() From 4a4f02427c640dcdcaea046736d0161280ecb7d3 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 24 Feb 2023 19:08:41 +0100 Subject: [PATCH 63/89] change config language to simple (or complex) code Signed-off-by: Csaba Kiraly --- config_example.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++ study.py | 32 +++++++++++----------------- 2 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 config_example.py diff --git a/config_example.py b/config_example.py new file mode 100644 index 0000000..0ac65ad --- /dev/null +++ b/config_example.py @@ -0,0 +1,53 @@ +"""Example configuration file + +This file illustrates how to define options and simulation parameter ranges. +It also defines the traversal order of the simulation space. As the file +extension suggests, configuration is pure python code, allowing complex +setups. Use at your own risk. + +To use this example, run + python3 study.py config_example + +Otherwise copy it and modify as needed. The default traversal order defined +in the nested loop of nextShape() is good for most cases, but customizable +if needed. +""" + +import logging +from DAS.shape import Shape + +dumpXML = 1 +visualization = 1 +logLevel = logging.INFO + +# Number of simulation runs with the same parameters for statistical relevance +runs = range(10) + +# Number of validators +numberValidators = range(256, 513, 128) + +# Percentage of block not released by producer +failureRates = range(10, 91, 40) + +# Block size in one dimension in segments. Block is blockSizes * blockSizes segments. +blockSizes = range(32,65,16) + +# Per-topic mesh neighborhood size +netDegrees = range(6, 9, 2) + +# number of rows and columns a validator is interested in +chis = range(4, 9, 2) + +deterministic = False + +def nextShape(): + for run in runs: + for fr in failureRates: + for chi in chis: + for blockSize in blockSizes: + for nv in numberValidators: + for netDegree in netDegrees: + # Network Degree has to be an even number + if netDegree % 2 == 0: + shape = Shape(blockSize, nv, fr, chi, netDegree, run) + yield shape diff --git a/study.py b/study.py index 028d327..20c1475 100644 --- a/study.py +++ b/study.py @@ -1,6 +1,7 @@ #! /bin/python3 import time, sys, random, copy +import importlib from DAS import * @@ -9,7 +10,7 @@ def study(): print("You need to pass a configuration file in parameter") exit(1) - config = Configuration(sys.argv[1]) + config = importlib.import_module(sys.argv[1]) shape = Shape(0, 0, 0, 0, 0, 0) sim = Simulator(shape, config) sim.initLogger() @@ -22,26 +23,17 @@ def study(): sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() - for run in range(config.numberRuns): - for nv in range(config.nvStart, config.nvStop+1, config.nvStep): - for blockSize in range(config.blockSizeStart, config.blockSizeStop+1, config.blockSizeStep): - for fr in range(config.failureRateStart, config.failureRateStop+1, config.failureRateStep): - for netDegree in range(config.netDegreeStart, config.netDegreeStop+1, config.netDegreeStep): - for chi in range(config.chiStart, config.chiStop+1, config.chiStep): + for shape in config.nextShape(): + if not config.deterministic: + random.seed(datetime.now()) - if not config.deterministic: - random.seed(datetime.now()) - - # Network Degree has to be an even number - if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, chi, netDegree, run) - sim.resetShape(shape) - sim.initValidators() - sim.initNetwork() - result = sim.run() - sim.logger.info("Shape: %s ... Block Available: %d in %d steps" % (str(sim.shape.__dict__), result.blockAvailable, len(result.missingVector)), extra=sim.format) - results.append(copy.deepcopy(result)) - simCnt += 1 + sim.resetShape(shape) + sim.initValidators() + sim.initNetwork() + result = sim.run() + sim.logger.info("Shape: %s ... Block Available: %d in %d steps" % (str(sim.shape.__dict__), result.blockAvailable, len(result.missingVector)), extra=sim.format) + results.append(copy.deepcopy(result)) + simCnt += 1 end = time.time() sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format) From 84e5482a6d99fe133ab3b663b74fcc7826ea2321 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 2 Mar 2023 01:01:52 +0100 Subject: [PATCH 64/89] improve config filename error handling Signed-off-by: Csaba Kiraly --- study.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/study.py b/study.py index 20c1475..52433c5 100644 --- a/study.py +++ b/study.py @@ -10,7 +10,16 @@ def study(): print("You need to pass a configuration file in parameter") exit(1) - config = importlib.import_module(sys.argv[1]) + try: + config = importlib.import_module(sys.argv[1]) + except ModuleNotFoundError as e: + try: + config = importlib.import_module(str(sys.argv[1]).replace(".py", "")) + except ModuleNotFoundError as e: + print(e) + print("You need to pass a configuration file in parameter") + exit(1) + shape = Shape(0, 0, 0, 0, 0, 0) sim = Simulator(shape, config) sim.initLogger() From 0a5afd97de348c4e069f112da717c1feb8a6cd02 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 7 Mar 2023 12:10:36 +0100 Subject: [PATCH 65/89] remove old config files Signed-off-by: Csaba Kiraly --- DAS/__init__.py | 1 - DAS/configuration.py | 85 -------------------------------------------- config.ini | 30 ---------------- 3 files changed, 116 deletions(-) delete mode 100644 DAS/configuration.py delete mode 100644 config.ini diff --git a/DAS/__init__.py b/DAS/__init__.py index c7fc7c0..67af3ae 100644 --- a/DAS/__init__.py +++ b/DAS/__init__.py @@ -1,4 +1,3 @@ from DAS.simulator import * -from DAS.configuration import * from DAS.shape import * from DAS.visualizer import * diff --git a/DAS/configuration.py b/DAS/configuration.py deleted file mode 100644 index 2ff9d68..0000000 --- a/DAS/configuration.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/python3 - -import configparser, logging, sys - -class Configuration: - """This class stores all the configuration parameters for the given run.""" - - def __init__(self, fileName): - """It initializes the configuration based on the given configuration.""" - - config = configparser.RawConfigParser() - config.read(fileName) - - try: - self.nvStart = int(config.get("Simulation Space", "numberValidatorStart")) - self.nvStop = int(config.get("Simulation Space", "numberValidatorStop")) - self.nvStep = int(config.get("Simulation Space", "numberValidatorStep")) - - self.blockSizeStart = int(config.get("Simulation Space", "blockSizeStart")) - self.blockSizeStop = int(config.get("Simulation Space", "blockSizeStop")) - self.blockSizeStep = int(config.get("Simulation Space", "blockSizeStep")) - - self.netDegreeStart = int(config.get("Simulation Space", "netDegreeStart")) - self.netDegreeStop = int(config.get("Simulation Space", "netDegreeStop")) - self.netDegreeStep = int(config.get("Simulation Space", "netDegreeStep")) - - self.failureRateStart = int(config.get("Simulation Space", "failureRateStart")) - self.failureRateStop = int(config.get("Simulation Space", "failureRateStop")) - self.failureRateStep = int(config.get("Simulation Space", "failureRateStep")) - - self.chiStart = int(config.get("Simulation Space", "chiStart")) - self.chiStop = int(config.get("Simulation Space", "chiStop")) - self.chiStep = int(config.get("Simulation Space", "chiStep")) - except: - sys.exit("Configuration Error: It seems some of the [Simulation Space] parameters are missing. Cannot continue :( ") - - - - try: - self.numberRuns = int(config.get("Advanced", "numberRuns")) - self.deterministic = config.get("Advanced", "deterministic") - self.dumpXML = config.get("Advanced", "dumpXML") - self.logLevel = config.get("Advanced", "logLevel") - self.visualization = config.get("Advanced", "visualization") - except: - sys.exit("Configuration Error: It seems some of the [Advanced] parameters are missing. Cannot continue :( ") - self.test() - - def test(self): - - print("Testing configuration...") - if self.logLevel == "INFO": - self.logLevel = logging.INFO - elif self.logLevel == "DEBUG": - self.logLevel = logging.DEBUG - else: - self.logLevel = logging.INFO - - if self.nvStart >= self.nvStop: - sys.exit("Configuration Error: numberValidatorStart has to be smaller than numberValidatorStop") - - if self.failureRateStart >= self.failureRateStop: - sys.exit("Configuration Error: failureRateStart has to be smaller than failureRateStop") - - if self.blockSizeStart >= self.blockSizeStop: - sys.exit("Configuration Error: blockSizeStart has to be smaller than blockSizeStop") - - if self.netDegreeStart >= self.netDegreeStop: - sys.exit("Configuration Error: netDegreeStart has to be smaller than netDegreeStop") - - if self.chiStart >= self.chiStop: - sys.exit("Configuration Error: chiStart has to be smaller than chiStop") - - - if self.nvStart < self.blockSizeStop: - sys.exit("Configuration Error: numberValidatorStart hast to be larger than blockSizeStop.") - - if self.chiStart < 2: - sys.exit("Configuration Error: Chi has to be greater than 1.") - - if self.chiStop > self.blockSizeStart: - sys.exit("Configuration Error: Chi (%d) has to be smaller or equal to block the size (%d)" % (self.chiStop, self.blockSizeStart)) - - - diff --git a/config.ini b/config.ini deleted file mode 100644 index 12bd749..0000000 --- a/config.ini +++ /dev/null @@ -1,30 +0,0 @@ -[Simulation Space] - -numberValidatorStart = 256 -numberValidatorStop = 512 -numberValidatorStep = 128 - -failureRateStart = 10 -failureRateStop = 90 -failureRateStep = 40 - -blockSizeStart = 32 -blockSizeStop = 64 -blockSizeStep = 16 - -netDegreeStart = 6 -netDegreeStop = 8 -netDegreeStep = 2 - -chiStart = 4 -chiStop = 8 -chiStep = 2 - - -[Advanced] - -deterministic = 0 -numberRuns = 2 -dumpXML = 1 -visualization = 1 -logLevel = INFO From 0b6cfad9675d3e2b12d1b379f750610ab6a61d74 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 3 Mar 2023 11:02:07 +0100 Subject: [PATCH 66/89] factorize study code Signed-off-by: Csaba Kiraly --- study.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/study.py b/study.py index 52433c5..2823fa1 100644 --- a/study.py +++ b/study.py @@ -4,6 +4,16 @@ import time, sys, random, copy import importlib from DAS import * +def runOnce(sim, config, shape): + if not config.deterministic: + random.seed(datetime.now()) + + sim.resetShape(shape) + sim.initValidators() + sim.initNetwork() + result = sim.run() + sim.logger.info("Shape: %s ... Block Available: %d in %d steps" % (str(sim.shape.__dict__), result.blockAvailable, len(result.missingVector)), extra=sim.format) + return result def study(): if len(sys.argv) < 2: @@ -24,7 +34,6 @@ def study(): sim = Simulator(shape, config) sim.initLogger() results = [] - simCnt = 0 now = datetime.now() execID = now.strftime("%Y-%m-%d_%H-%M-%S_")+str(random.randint(100,999)) @@ -33,19 +42,11 @@ def study(): start = time.time() for shape in config.nextShape(): - if not config.deterministic: - random.seed(datetime.now()) - - sim.resetShape(shape) - sim.initValidators() - sim.initNetwork() - result = sim.run() - sim.logger.info("Shape: %s ... Block Available: %d in %d steps" % (str(sim.shape.__dict__), result.blockAvailable, len(result.missingVector)), extra=sim.format) + result = runOnce(sim, config, shape) results.append(copy.deepcopy(result)) - simCnt += 1 end = time.time() - sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format) + sim.logger.info("A total of %d simulations ran in %d seconds" % (len(results), end-start), extra=sim.format) if config.dumpXML: for res in results: From f5ffb0a07b0ac0bece35e0b701ae7a5f9e851a68 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 3 Mar 2023 11:15:55 +0100 Subject: [PATCH 67/89] use joblib to run in parallel Signed-off-by: Csaba Kiraly --- study.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/study.py b/study.py index 2823fa1..4cd4b53 100644 --- a/study.py +++ b/study.py @@ -2,8 +2,13 @@ import time, sys, random, copy import importlib +from joblib import Parallel, delayed from DAS import * +# Parallel execution: +# The code currently uses 'joblib' to execute on multiple cores. For other options such as 'ray', see +# https://stackoverflow.com/questions/9786102/how-do-i-parallelize-a-simple-python-loop + def runOnce(sim, config, shape): if not config.deterministic: random.seed(datetime.now()) @@ -41,9 +46,7 @@ def study(): sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() - for shape in config.nextShape(): - result = runOnce(sim, config, shape) - results.append(copy.deepcopy(result)) + results = Parallel(-1)(delayed(runOnce)(sim, config, shape) for shape in config.nextShape()) end = time.time() sim.logger.info("A total of %d simulations ran in %d seconds" % (len(results), end-start), extra=sim.format) From 16b670e9165d824dd0c3ac970585957d6a53bfff Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 3 Mar 2023 11:26:00 +0100 Subject: [PATCH 68/89] fix issues with logging in parallel execution For fixing logging issues see https://stackoverflow.com/questions/58026381/logging-nested-functions-using-joblib-parallel-and-delayed-calls and https://github.com/joblib/joblib/issues/1017 Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 11 ++++++----- study.py | 4 ++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index dbd1b37..8b79e2c 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -104,11 +104,12 @@ class Simulator: def initLogger(self): """It initializes the logger.""" logger = logging.getLogger("DAS") - logger.setLevel(self.logLevel) - ch = logging.StreamHandler() - ch.setLevel(self.logLevel) - ch.setFormatter(CustomFormatter()) - logger.addHandler(ch) + if len(logger.handlers) == 0: + logger.setLevel(self.logLevel) + ch = logging.StreamHandler() + ch.setLevel(self.logLevel) + ch.setFormatter(CustomFormatter()) + logger.addHandler(ch) self.logger = logger diff --git a/study.py b/study.py index 4cd4b53..914acfe 100644 --- a/study.py +++ b/study.py @@ -8,11 +8,15 @@ from DAS import * # Parallel execution: # The code currently uses 'joblib' to execute on multiple cores. For other options such as 'ray', see # https://stackoverflow.com/questions/9786102/how-do-i-parallelize-a-simple-python-loop +# For fixing logging issues in parallel execution, see +# https://stackoverflow.com/questions/58026381/logging-nested-functions-using-joblib-parallel-and-delayed-calls +# and https://github.com/joblib/joblib/issues/1017 def runOnce(sim, config, shape): if not config.deterministic: random.seed(datetime.now()) + sim.initLogger() sim.resetShape(shape) sim.initValidators() sim.initNetwork() From 567d13e37083ba7bc7f8ebf4c64ffb2cbff1d1cb Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 13 Mar 2023 14:22:14 +0100 Subject: [PATCH 69/89] add numJobs parameter to config Signed-off-by: Csaba Kiraly --- config_example.py | 4 ++++ study.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config_example.py b/config_example.py index 0ac65ad..c2f7fcf 100644 --- a/config_example.py +++ b/config_example.py @@ -20,6 +20,10 @@ dumpXML = 1 visualization = 1 logLevel = logging.INFO +# number of parallel workers. -1: all cores; 1: sequential +# for more details, see joblib.Parallel +numJobs = 3 + # Number of simulation runs with the same parameters for statistical relevance runs = range(10) diff --git a/study.py b/study.py index 914acfe..f557fc2 100644 --- a/study.py +++ b/study.py @@ -50,7 +50,8 @@ def study(): sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() - results = Parallel(-1)(delayed(runOnce)(sim, config, shape) for shape in config.nextShape()) + results = Parallel(config.numJobs)(delayed(runOnce)(sim, config, shape) for shape in config.nextShape()) + end = time.time() sim.logger.info("A total of %d simulations ran in %d seconds" % (len(results), end-start), extra=sim.format) From ba94cc8da116c327a7b28049f86bd04fd15c75ed Mon Sep 17 00:00:00 2001 From: leobago Date: Tue, 14 Mar 2023 08:26:37 +0100 Subject: [PATCH 70/89] Update requirements --- DAS/requirements.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DAS/requirements.txt b/DAS/requirements.txt index d0bb457..da7dcc7 100644 --- a/DAS/requirements.txt +++ b/DAS/requirements.txt @@ -1,3 +1,7 @@ bitarray==2.6.0 -DAS==0.28.7 +DAS==0.29.0 +dicttoxml==1.7.16 +matplotlib==3.6.2 networkx==3.0 +numpy==1.23.5 +seaborn==0.12.2 From 377072ef7951c1d0a064869a247871a767af1801 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Tue, 14 Mar 2023 11:07:59 +0100 Subject: [PATCH 71/89] Fix deterministic feature --- study.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/study.py b/study.py index 52433c5..67dac5f 100644 --- a/study.py +++ b/study.py @@ -33,8 +33,11 @@ def study(): start = time.time() for shape in config.nextShape(): - if not config.deterministic: - random.seed(datetime.now()) + if config.deterministic: + random.seed("DASsimulator") + else: + random.seed(random.randint(0, 100)) + sim.resetShape(shape) sim.initValidators() From 45bdcc3308dc41bf1b944788f4d8821a4e63929d Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Tue, 14 Mar 2023 11:25:17 +0100 Subject: [PATCH 72/89] Seed by microseconds. Determinisme checked. --- study.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/study.py b/study.py index 67dac5f..e87b12f 100644 --- a/study.py +++ b/study.py @@ -27,17 +27,16 @@ def study(): simCnt = 0 now = datetime.now() + if config.deterministic: + random.seed("DASsimulator") + else: + random.seed(str(now).split(".")[1]) execID = now.strftime("%Y-%m-%d_%H-%M-%S_")+str(random.randint(100,999)) sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() for shape in config.nextShape(): - if config.deterministic: - random.seed("DASsimulator") - else: - random.seed(random.randint(0, 100)) - sim.resetShape(shape) sim.initValidators() From 833ae76097f8bd2655124d28baab51bb14df86f7 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 15 Mar 2023 12:37:23 +0100 Subject: [PATCH 73/89] add shape repr --- DAS/shape.py | 11 ++++++++++- study.py | 35 +++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/DAS/shape.py b/DAS/shape.py index 2f99ebf..a27b6ca 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -12,6 +12,15 @@ class Shape: self.netDegree = netDegree self.chi = chi - + def __repr__(self): + """Returns a printable representation of the shape""" + shastr = "" + shastr += "bs-"+str(self.blockSize) + shastr += "-nbv-"+str(self.numberValidators) + shastr += "-fr-"+str(self.failureRate) + shastr += "-chi-"+str(self.chi) + shastr += "-nd-"+str(self.netDegree) + shastr += "-r-"+str(self.run) + return repr(shastr) diff --git a/study.py b/study.py index e87b12f..6d223b8 100644 --- a/study.py +++ b/study.py @@ -2,8 +2,27 @@ import time, sys, random, copy import importlib +from joblib import Parallel, delayed from DAS import * +# Parallel execution: +# The code currently uses 'joblib' to execute on multiple cores. For other options such as 'ray', see +# https://stackoverflow.com/questions/9786102/how-do-i-parallelize-a-simple-python-loop +# For fixing logging issues in parallel execution, see +# https://stackoverflow.com/questions/58026381/logging-nested-functions-using-joblib-parallel-and-delayed-calls +# and https://github.com/joblib/joblib/issues/1017 + +def runOnce(sim, config, shape): + if config.deterministic: + random.seed(repr(shape)) + + sim.initLogger() + sim.resetShape(shape) + sim.initValidators() + sim.initNetwork() + result = sim.run() + sim.logger.info("Shape: %s ... Block Available: %d in %d steps" % (str(sim.shape.__dict__), result.blockAvailable, len(result.missingVector)), extra=sim.format) + return result def study(): if len(sys.argv) < 2: @@ -24,30 +43,18 @@ def study(): sim = Simulator(shape, config) sim.initLogger() results = [] - simCnt = 0 now = datetime.now() - if config.deterministic: - random.seed("DASsimulator") - else: - random.seed(str(now).split(".")[1]) execID = now.strftime("%Y-%m-%d_%H-%M-%S_")+str(random.randint(100,999)) sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() - for shape in config.nextShape(): + results = Parallel(config.numJobs)(delayed(runOnce)(sim, config, shape) for shape in config.nextShape()) - sim.resetShape(shape) - sim.initValidators() - sim.initNetwork() - result = sim.run() - sim.logger.info("Shape: %s ... Block Available: %d in %d steps" % (str(sim.shape.__dict__), result.blockAvailable, len(result.missingVector)), extra=sim.format) - results.append(copy.deepcopy(result)) - simCnt += 1 end = time.time() - sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format) + sim.logger.info("A total of %d simulations ran in %d seconds" % (len(results), end-start), extra=sim.format) if config.dumpXML: for res in results: From 0a92ef70713f6c7b7b97c3c1244308a2e7b76b09 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 15 Mar 2023 13:18:02 +0100 Subject: [PATCH 74/89] Adding and logging random seed in the shape for each simulation --- DAS/results.py | 7 +------ DAS/shape.py | 6 +++++- config_example.py | 6 +++++- study.py | 9 +++------ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/DAS/results.py b/DAS/results.py index e20cdec..0efe26d 100644 --- a/DAS/results.py +++ b/DAS/results.py @@ -39,11 +39,6 @@ class Result: resXml = dicttoxml(resd1) xmlstr = minidom.parseString(resXml) xmlPretty = xmlstr.toprettyxml() - filePath = "results/"+execID+"/nbv-"+str(self.shape.numberValidators)+\ - "-bs-"+str(self.shape.blockSize)+\ - "-nd-"+str(self.shape.netDegree)+\ - "-fr-"+str(self.shape.failureRate)+\ - "-chi-"+str(self.shape.chi)+\ - "-r-"+str(self.shape.run)+".xml" + filePath = "results/"+execID+"/"+str(self.shape)+".xml" with open(filePath, "w") as f: f.write(xmlPretty) diff --git a/DAS/shape.py b/DAS/shape.py index a27b6ca..1dd19b2 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -11,6 +11,7 @@ class Shape: self.failureRate = failureRate self.netDegree = netDegree self.chi = chi + self.randomSeed = "" def __repr__(self): """Returns a printable representation of the shape""" @@ -21,6 +22,9 @@ class Shape: shastr += "-chi-"+str(self.chi) shastr += "-nd-"+str(self.netDegree) shastr += "-r-"+str(self.run) - return repr(shastr) + return shastr + def setSeed(self, seed): + """Adds the random seed to the shape""" + self.randomSeed = seed diff --git a/config_example.py b/config_example.py index 0ac65ad..1a4db01 100644 --- a/config_example.py +++ b/config_example.py @@ -35,11 +35,15 @@ blockSizes = range(32,65,16) # Per-topic mesh neighborhood size netDegrees = range(6, 9, 2) -# number of rows and columns a validator is interested in +# Number of rows and columns a validator is interested in chis = range(4, 9, 2) +# Set to True if you want your run to be deterministic, False if not deterministic = False +# If your run is deterministic you can decide the random seed. This is ignore otherwise. +randomSeed = "DAS" + def nextShape(): for run in runs: for fr in failureRates: diff --git a/study.py b/study.py index 6d223b8..e24fb46 100644 --- a/study.py +++ b/study.py @@ -14,7 +14,8 @@ from DAS import * def runOnce(sim, config, shape): if config.deterministic: - random.seed(repr(shape)) + shape.setSeed(config.randomSeed+"-"+str(shape)) + random.seed(shape.randomSeed) sim.initLogger() sim.resetShape(shape) @@ -49,10 +50,7 @@ def study(): sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() - results = Parallel(config.numJobs)(delayed(runOnce)(sim, config, shape) for shape in config.nextShape()) - - end = time.time() sim.logger.info("A total of %d simulations ran in %d seconds" % (len(results), end-start), extra=sim.format) @@ -61,8 +59,7 @@ def study(): res.dump(execID) sim.logger.info("Results dumped into results/%s/" % (execID), extra=sim.format) - visualization = 1 - if visualization: + if config.visualization: vis = Visualizer(execID) vis.plotHeatmaps() From 99e051e6fc669b27e791a99b8971f4eaca7063cb Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 3 Mar 2023 18:41:12 +0100 Subject: [PATCH 75/89] adding uplinkBw configuration Signed-off-by: Csaba Kiraly --- DAS/shape.py | 3 ++- DAS/validator.py | 2 +- config_example.py | 13 +++++++++---- study.py | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/DAS/shape.py b/DAS/shape.py index 1dd19b2..de22c90 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,7 +3,7 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" - def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, run): + def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, bwUplink, run): """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberValidators = numberValidators @@ -12,6 +12,7 @@ class Shape: self.netDegree = netDegree self.chi = chi self.randomSeed = "" + self.bwUplink = bwUplink def __repr__(self): """Returns a printable representation of the shape""" diff --git a/DAS/validator.py b/DAS/validator.py index 7b52e58..4647df5 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -77,7 +77,7 @@ class Validator: # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 # TODO: this should be a parameter - self.bwUplink = 110 if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps + self.bwUplink = shape.bwUplink if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps self.repairOnTheFly = True self.sendLineUntil = (self.shape.blockSize + 1) // 2 # stop sending on a p2p link if at least this amount of samples passed diff --git a/config_example.py b/config_example.py index 248a8e9..0967a4f 100644 --- a/config_example.py +++ b/config_example.py @@ -42,6 +42,10 @@ netDegrees = range(6, 9, 2) # Number of rows and columns a validator is interested in chis = range(4, 9, 2) +# Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) +# 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 +bwUplinks = [11, 110] + # Set to True if you want your run to be deterministic, False if not deterministic = False @@ -55,7 +59,8 @@ def nextShape(): for blockSize in blockSizes: for nv in numberValidators: for netDegree in netDegrees: - # Network Degree has to be an even number - if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, chi, netDegree, run) - yield shape + for bwUplink in bwUplinks: + # Network Degree has to be an even number + if netDegree % 2 == 0: + shape = Shape(blockSize, nv, fr, chi, netDegree, bwUplink, run) + yield shape diff --git a/study.py b/study.py index e24fb46..2503895 100644 --- a/study.py +++ b/study.py @@ -40,7 +40,7 @@ def study(): print("You need to pass a configuration file in parameter") exit(1) - shape = Shape(0, 0, 0, 0, 0, 0) + shape = Shape(0, 0, 0, 0, 0, 0, 0) sim = Simulator(shape, config) sim.initLogger() results = [] From 4a5d410f6a86b48bf3688773f379ab1ff64ce66e Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 6 Mar 2023 11:42:45 +0100 Subject: [PATCH 76/89] fixup: add bwUplink to save file name Signed-off-by: Csaba Kiraly --- DAS/shape.py | 1 + 1 file changed, 1 insertion(+) diff --git a/DAS/shape.py b/DAS/shape.py index de22c90..7b24f6e 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -21,6 +21,7 @@ class Shape: shastr += "-nbv-"+str(self.numberValidators) shastr += "-fr-"+str(self.failureRate) shastr += "-chi-"+str(self.chi) + shastr += "-bwu-"+str(self.bwUplink) shastr += "-nd-"+str(self.netDegree) shastr += "-r-"+str(self.run) return shastr From dc7a4d3c032160762fe5191025bf8e3c7e087beb Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 6 Mar 2023 15:15:57 +0100 Subject: [PATCH 77/89] generate row/column interest locally in validator Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 17 ++++++++++++----- DAS/validator.py | 21 +++++++++++++++------ config_example.py | 4 ++++ 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 8b79e2c..f0195cd 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -15,6 +15,7 @@ class Simulator: def __init__(self, shape, config): """It initializes the simulation with a set of parameters (shape).""" self.shape = shape + self.config = config self.format = {"entity": "Simulator"} self.result = Result(self.shape) self.validators = [] @@ -28,12 +29,18 @@ class Simulator: self.glob = Observer(self.logger, self.shape) self.glob.reset() self.validators = [] - rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) - columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) - random.shuffle(rows) - random.shuffle(columns) + if self.config.evenLineDistribution: + rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) + columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) + random.shuffle(rows) + random.shuffle(columns) for i in range(self.shape.numberValidators): - val = Validator(i, int(not i!=0), self.logger, self.shape, rows, columns) + if self.config.evenLineDistribution: + val = Validator(i, int(not i!=0), self.logger, self.shape, + rows[(i*self.shape.chi):((i+1)*self.shape.chi)], + columns[(i*self.shape.chi):((i+1)*self.shape.chi)]) + else: + val = Validator(i, int(not i!=0), self.logger, self.shape) if i == self.proposerID: val.initBlock() self.glob.setGoldenData(val.block) diff --git a/DAS/validator.py b/DAS/validator.py index 4647df5..6a1b80a 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -38,8 +38,13 @@ class Validator: """It returns the validator ID.""" return str(self.ID) - def __init__(self, ID, amIproposer, logger, shape, rows, columns): - """It initializes the validator with the logger, shape and assigned rows/columns.""" + def __init__(self, ID, amIproposer, logger, shape, rows = None, columns = None): + """It initializes the validator with the logger shape and rows/columns. + + If rows/columns are specified these are observed, otherwise (default) + chi rows and columns are selected randomly. + """ + self.shape = shape FORMAT = "%(levelname)s : %(entity)s : %(message)s" self.ID = ID @@ -59,12 +64,16 @@ class Validator: self.rowIDs = range(shape.blockSize) self.columnIDs = range(shape.blockSize) else: - self.rowIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] - self.columnIDs = columns[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] #if shape.deterministic: # random.seed(self.ID) - #self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi) - #self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi) + if rows: + self.rowIDs = rows + else: + self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi) + if columns: + self.columnIDs = columns + else: + self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) diff --git a/config_example.py b/config_example.py index 0967a4f..9235f10 100644 --- a/config_example.py +++ b/config_example.py @@ -24,6 +24,10 @@ logLevel = logging.INFO # for more details, see joblib.Parallel numJobs = 3 +# distribute rows/columns evenly between validators (True) +# or generate it using local randomness (False) +evenLineDistribution = False + # Number of simulation runs with the same parameters for statistical relevance runs = range(10) From 49b1c239d79c1eae2f332cbc6388c1c22091303a Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 6 Mar 2023 22:46:17 +0100 Subject: [PATCH 78/89] use itertools Signed-off-by: Csaba Kiraly --- config_example.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/config_example.py b/config_example.py index 9235f10..59d8fef 100644 --- a/config_example.py +++ b/config_example.py @@ -14,6 +14,7 @@ if needed. """ import logging +import itertools from DAS.shape import Shape dumpXML = 1 @@ -57,14 +58,9 @@ deterministic = False randomSeed = "DAS" def nextShape(): - for run in runs: - for fr in failureRates: - for chi in chis: - for blockSize in blockSizes: - for nv in numberValidators: - for netDegree in netDegrees: - for bwUplink in bwUplinks: - # Network Degree has to be an even number - if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, chi, netDegree, bwUplink, run) - yield shape + for run, fr, chi, blockSize, nv, netDegree, bwUplink in itertools.product( + runs, failureRates, chis, blockSizes, numberValidators, netDegrees, bwUplinks): + # Network Degree has to be an even number + if netDegree % 2 == 0: + shape = Shape(blockSize, nv, fr, chi, netDegree, bwUplink, run) + yield shape From ef4e32ed53a3a0793f42a0451448b62429e879bc Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 7 Mar 2023 13:24:11 +0100 Subject: [PATCH 79/89] introduce node classes Signed-off-by: Csaba Kiraly --- DAS/shape.py | 18 +++++++++++++----- DAS/simulator.py | 3 ++- DAS/validator.py | 21 ++++++++++++++++----- config_example.py | 16 +++++++++++----- study.py | 2 +- 5 files changed, 43 insertions(+), 17 deletions(-) diff --git a/DAS/shape.py b/DAS/shape.py index 7b24f6e..b707e51 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,16 +3,20 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" - def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, bwUplink, run): + def __init__(self, blockSize, numberValidators, failureRate, class1ratio, chi1, chi2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberValidators = numberValidators self.blockSize = blockSize self.failureRate = failureRate self.netDegree = netDegree - self.chi = chi + self.class1ratio = class1ratio + self.chi1 = chi1 + self.chi2 = chi2 + self.bwUplinkProd = bwUplinkProd + self.bwUplink1 = bwUplink1 + self.bwUplink2 = bwUplink2 self.randomSeed = "" - self.bwUplink = bwUplink def __repr__(self): """Returns a printable representation of the shape""" @@ -20,8 +24,12 @@ class Shape: shastr += "bs-"+str(self.blockSize) shastr += "-nbv-"+str(self.numberValidators) shastr += "-fr-"+str(self.failureRate) - shastr += "-chi-"+str(self.chi) - shastr += "-bwu-"+str(self.bwUplink) + shastr += "-c1r-"+str(self.class1ratio) + shastr += "-chi1-"+str(self.chi1) + shastr += "-chi2-"+str(self.chi2) + shastr += "-bwupprod-"+str(self.bwUplinkProd) + shastr += "-bwup1-"+str(self.bwUplink1) + shastr += "-bwup2-"+str(self.bwUplink2) shastr += "-nd-"+str(self.netDegree) shastr += "-r-"+str(self.run) return shastr diff --git a/DAS/simulator.py b/DAS/simulator.py index f0195cd..2d6e635 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -126,7 +126,8 @@ class Simulator: self.result = Result(self.shape) for val in self.validators: val.shape.failureRate = shape.failureRate - val.shape.chi = shape.chi + val.shape.chi1 = shape.chi1 + val.shape.chi2 = shape.chi2 # In GossipSub the initiator might push messages without participating in the mesh. # proposerPublishOnly regulates this behavior. If set to true, the proposer is not diff --git a/DAS/validator.py b/DAS/validator.py index 6a1b80a..71e5791 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -55,11 +55,17 @@ class Validator: self.sendQueue = deque() self.amIproposer = amIproposer self.logger = logger - if self.shape.chi < 1: + if self.shape.chi1 < 1 or self.shape.chi2 < 1: self.logger.error("Chi has to be greater than 0", extra=self.format) - elif self.shape.chi > self.shape.blockSize: + elif self.shape.chi1 > self.shape.blockSize or self.shape.chi2 > self.shape.blockSize: self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) else: + if self.amIproposer: + self.chi = 1 # not used + elif self.ID <= shape.numberValidators * shape.class1ratio: + self.chi = shape.chi1 + else: + self.chi = shape.chi2 if amIproposer: self.rowIDs = range(shape.blockSize) self.columnIDs = range(shape.blockSize) @@ -69,11 +75,11 @@ class Validator: if rows: self.rowIDs = rows else: - self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi) + self.rowIDs = random.sample(range(self.shape.blockSize), self.chi) if columns: self.columnIDs = columns else: - self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi) + self.columnIDs = random.sample(range(self.shape.blockSize), self.chi) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) @@ -86,7 +92,12 @@ class Validator: # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 # TODO: this should be a parameter - self.bwUplink = shape.bwUplink if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps + if self.amIproposer: + self.bwUplink = shape.bwUplinkProd + elif self.ID <= shape.numberValidators * shape.class1ratio: + self.bwUplink = shape.bwUplink1 + else: + self.bwUplink = shape.bwUplink2 self.repairOnTheFly = True self.sendLineUntil = (self.shape.blockSize + 1) // 2 # stop sending on a p2p link if at least this amount of samples passed diff --git a/config_example.py b/config_example.py index 59d8fef..b718528 100644 --- a/config_example.py +++ b/config_example.py @@ -15,6 +15,7 @@ if needed. import logging import itertools +import numpy as np from DAS.shape import Shape dumpXML = 1 @@ -44,12 +45,17 @@ blockSizes = range(32,65,16) # Per-topic mesh neighborhood size netDegrees = range(6, 9, 2) +class1ratios = np.arange(0, 1, .2) + # Number of rows and columns a validator is interested in -chis = range(4, 9, 2) +chis1 = range(1, 5, 2) +chis2 = range(10, 30, 20) # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 -bwUplinks = [11, 110] +bwUplinksProd = [2200] +bwUplinks1 = [2200] +bwUplinks2 = [110] # Set to True if you want your run to be deterministic, False if not deterministic = False @@ -58,9 +64,9 @@ deterministic = False randomSeed = "DAS" def nextShape(): - for run, fr, chi, blockSize, nv, netDegree, bwUplink in itertools.product( - runs, failureRates, chis, blockSizes, numberValidators, netDegrees, bwUplinks): + for run, fr, class1ratio, chi1, chi2, blockSize, nv, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( + runs, failureRates, class1ratios, chis1, chis2, blockSizes, numberValidators, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): # Network Degree has to be an even number if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, chi, netDegree, bwUplink, run) + shape = Shape(blockSize, nv, fr, class1ratio, chi1, chi2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) yield shape diff --git a/study.py b/study.py index 2503895..3efd83e 100644 --- a/study.py +++ b/study.py @@ -40,7 +40,7 @@ def study(): print("You need to pass a configuration file in parameter") exit(1) - shape = Shape(0, 0, 0, 0, 0, 0, 0) + shape = Shape(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) sim = Simulator(shape, config) sim.initLogger() results = [] From 45fe4542045a082b369a5012a1d6aa9e75b103ac Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 7 Mar 2023 14:36:13 +0100 Subject: [PATCH 80/89] WIP: extend Vis with new parameters Current code assumes 6 parameters in some places. Still needs some work Signed-off-by: Csaba Kiraly --- DAS/visualizer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/DAS/visualizer.py b/DAS/visualizer.py index 047e513..e0579e2 100644 --- a/DAS/visualizer.py +++ b/DAS/visualizer.py @@ -12,7 +12,8 @@ class Visualizer: def __init__(self, execID): self.execID = execID self.folderPath = "results/"+self.execID - self.parameters = ['run', 'blockSize', 'failureRate', 'numberValidators', 'netDegree', 'chi'] + self.parameters = ['run', 'blockSize', 'failureRate', 'numberValidators', 'netDegree', + 'chi1', 'chi2', 'bwUplinkProd', 'bwUplink1', 'bwUplink2'] self.minimumDataPoints = 2 def plottingData(self): @@ -29,20 +30,24 @@ class Visualizer: failureRate = int(root.find('failureRate').text) numberValidators = int(root.find('numberValidators').text) netDegree = int(root.find('netDegree').text) - chi = int(root.find('chi').text) + chi1 = int(root.find('chi1').text) + chi2 = int(root.find('chi2').text) + bwUplinkProd = int(root.find('bwUplinkProd').text) + bwUplink1 = int(root.find('bwUplink1').text) + bwUplink2 = int(root.find('bwUplink2').text) tta = int(root.find('tta').text) # Loop over all possible combinations of length 4 of the parameters for combination in combinations(self.parameters, 4): # Get the indices and values of the parameters in the combination indices = [self.parameters.index(element) for element in combination] - selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi] + selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi1, chi2, bwUplinkProd, bwUplink1, bwUplink2] values = [selectedValues[index] for index in indices] names = [self.parameters[i] for i in indices] keyComponents = [f"{name}_{value}" for name, value in zip(names, values)] key = tuple(keyComponents[:4]) #Get the names of the other 2 parameters that are not included in the key - otherParams = [self.parameters[i] for i in range(6) if i not in indices] + otherParams = [self.parameters[i] for i in range(len(self.parameters)) if i not in indices] #Append the values of the other 2 parameters and the ttas to the lists for the key otherIndices = [i for i in range(len(self.parameters)) if i not in indices] From 065086f88caadca7fde37d9d0c1c3976361e08cc Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 13 Mar 2023 15:00:43 +0100 Subject: [PATCH 81/89] configure using validatorsPerNode1/2 instead of chi1/2 Signed-off-by: Csaba Kiraly --- DAS/shape.py | 12 +++++++----- DAS/simulator.py | 5 +++-- DAS/validator.py | 10 +++++----- DAS/visualizer.py | 9 +++++---- config_example.py | 16 ++++++++++------ study.py | 2 +- 6 files changed, 31 insertions(+), 23 deletions(-) diff --git a/DAS/shape.py b/DAS/shape.py index b707e51..484b3eb 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,7 +3,7 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" - def __init__(self, blockSize, numberValidators, failureRate, class1ratio, chi1, chi2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): + def __init__(self, blockSize, numberValidators, failureRate, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberValidators = numberValidators @@ -11,8 +11,9 @@ class Shape: self.failureRate = failureRate self.netDegree = netDegree self.class1ratio = class1ratio - self.chi1 = chi1 - self.chi2 = chi2 + self.chi = chi + self.vpn1 = vpn1 + self.vpn2 = vpn2 self.bwUplinkProd = bwUplinkProd self.bwUplink1 = bwUplink1 self.bwUplink2 = bwUplink2 @@ -25,8 +26,9 @@ class Shape: shastr += "-nbv-"+str(self.numberValidators) shastr += "-fr-"+str(self.failureRate) shastr += "-c1r-"+str(self.class1ratio) - shastr += "-chi1-"+str(self.chi1) - shastr += "-chi2-"+str(self.chi2) + shastr += "-chi-"+str(self.chi) + shastr += "-vpn1-"+str(self.vpn1) + shastr += "-vpn2-"+str(self.vpn2) shastr += "-bwupprod-"+str(self.bwUplinkProd) shastr += "-bwup1-"+str(self.bwUplink1) shastr += "-bwup2-"+str(self.bwUplink2) diff --git a/DAS/simulator.py b/DAS/simulator.py index 2d6e635..65f8ef2 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -126,8 +126,9 @@ class Simulator: self.result = Result(self.shape) for val in self.validators: val.shape.failureRate = shape.failureRate - val.shape.chi1 = shape.chi1 - val.shape.chi2 = shape.chi2 + val.shape.chi = shape.chi + val.shape.vpn1 = shape.vpn1 + val.shape.vpn2 = shape.vpn2 # In GossipSub the initiator might push messages without participating in the mesh. # proposerPublishOnly regulates this behavior. If set to true, the proposer is not diff --git a/DAS/validator.py b/DAS/validator.py index 71e5791..ae7be43 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -55,17 +55,17 @@ class Validator: self.sendQueue = deque() self.amIproposer = amIproposer self.logger = logger - if self.shape.chi1 < 1 or self.shape.chi2 < 1: + if self.shape.chi < 1: self.logger.error("Chi has to be greater than 0", extra=self.format) - elif self.shape.chi1 > self.shape.blockSize or self.shape.chi2 > self.shape.blockSize: - self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) + elif self.shape.chi > self.shape.blockSize: + self.logger.error("Chi has to be smaller than %d" % self.shape.blockSize, extra=self.format) else: if self.amIproposer: self.chi = 1 # not used elif self.ID <= shape.numberValidators * shape.class1ratio: - self.chi = shape.chi1 + self.chi = shape.chi * shape.vpn1 else: - self.chi = shape.chi2 + self.chi = shape.chi * shape.vpn2 # TODO: union of random subsets vpn2 times if amIproposer: self.rowIDs = range(shape.blockSize) self.columnIDs = range(shape.blockSize) diff --git a/DAS/visualizer.py b/DAS/visualizer.py index e0579e2..7004c87 100644 --- a/DAS/visualizer.py +++ b/DAS/visualizer.py @@ -13,7 +13,7 @@ class Visualizer: self.execID = execID self.folderPath = "results/"+self.execID self.parameters = ['run', 'blockSize', 'failureRate', 'numberValidators', 'netDegree', - 'chi1', 'chi2', 'bwUplinkProd', 'bwUplink1', 'bwUplink2'] + 'chi', 'vpn1', 'vpn2', 'bwUplinkProd', 'bwUplink1', 'bwUplink2'] self.minimumDataPoints = 2 def plottingData(self): @@ -30,8 +30,9 @@ class Visualizer: failureRate = int(root.find('failureRate').text) numberValidators = int(root.find('numberValidators').text) netDegree = int(root.find('netDegree').text) - chi1 = int(root.find('chi1').text) - chi2 = int(root.find('chi2').text) + chi = int(root.find('chi').text) + vpn1 = int(root.find('vpn1').text) + vpn2 = int(root.find('vpn2').text) bwUplinkProd = int(root.find('bwUplinkProd').text) bwUplink1 = int(root.find('bwUplink1').text) bwUplink2 = int(root.find('bwUplink2').text) @@ -41,7 +42,7 @@ class Visualizer: for combination in combinations(self.parameters, 4): # Get the indices and values of the parameters in the combination indices = [self.parameters.index(element) for element in combination] - selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi1, chi2, bwUplinkProd, bwUplink1, bwUplink2] + selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi, vpn1, vpn2, bwUplinkProd, bwUplink1, bwUplink2] values = [selectedValues[index] for index in indices] names = [self.parameters[i] for i in indices] keyComponents = [f"{name}_{value}" for name, value in zip(names, values)] diff --git a/config_example.py b/config_example.py index b718528..0bd509c 100644 --- a/config_example.py +++ b/config_example.py @@ -45,11 +45,15 @@ blockSizes = range(32,65,16) # Per-topic mesh neighborhood size netDegrees = range(6, 9, 2) +# number of rows and columns a validator is interested in +chis = range(1, 5, 2) + +# ratio of class1 nodes (see below for parameters per class) class1ratios = np.arange(0, 1, .2) -# Number of rows and columns a validator is interested in -chis1 = range(1, 5, 2) -chis2 = range(10, 30, 20) +# Number of validators per beacon node +validatorsPerNode1 = [1] +validatorsPerNode2 = [2, 4, 8, 16, 32] # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 @@ -64,9 +68,9 @@ deterministic = False randomSeed = "DAS" def nextShape(): - for run, fr, class1ratio, chi1, chi2, blockSize, nv, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( - runs, failureRates, class1ratios, chis1, chis2, blockSizes, numberValidators, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): + for run, fr, class1ratio, chi, vpn1, vpn2, blockSize, nv, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( + runs, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberValidators, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): # Network Degree has to be an even number if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, class1ratio, chi1, chi2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) + shape = Shape(blockSize, nv, fr, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) yield shape diff --git a/study.py b/study.py index 3efd83e..2b2aab6 100644 --- a/study.py +++ b/study.py @@ -40,7 +40,7 @@ def study(): print("You need to pass a configuration file in parameter") exit(1) - shape = Shape(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + shape = Shape(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) sim = Simulator(shape, config) sim.initLogger() results = [] From 9d9612fd34dbdee32f398f5c8be03e48b8708248 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 13 Mar 2023 15:03:55 +0100 Subject: [PATCH 82/89] rename numberValidators to numberNodes Signed-off-by: Csaba Kiraly --- DAS/shape.py | 6 +++--- DAS/simulator.py | 18 +++++++++--------- DAS/validator.py | 4 ++-- DAS/visualizer.py | 6 +++--- config_example.py | 8 ++++---- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/DAS/shape.py b/DAS/shape.py index 484b3eb..d83351d 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,10 +3,10 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" - def __init__(self, blockSize, numberValidators, failureRate, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): + def __init__(self, blockSize, numberNodes, failureRate, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): """Initializes the shape with the parameters passed in argument.""" self.run = run - self.numberValidators = numberValidators + self.numberNodes = numberNodes self.blockSize = blockSize self.failureRate = failureRate self.netDegree = netDegree @@ -23,7 +23,7 @@ class Shape: """Returns a printable representation of the shape""" shastr = "" shastr += "bs-"+str(self.blockSize) - shastr += "-nbv-"+str(self.numberValidators) + shastr += "-nn-"+str(self.numberNodes) shastr += "-fr-"+str(self.failureRate) shastr += "-c1r-"+str(self.class1ratio) shastr += "-chi-"+str(self.chi) diff --git a/DAS/simulator.py b/DAS/simulator.py index 65f8ef2..4cadf9f 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -30,11 +30,11 @@ class Simulator: self.glob.reset() self.validators = [] if self.config.evenLineDistribution: - rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) - columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) + rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberNodes/self.shape.blockSize) + columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberNodes/self.shape.blockSize) random.shuffle(rows) random.shuffle(columns) - for i in range(self.shape.numberValidators): + for i in range(self.shape.numberNodes): if self.config.evenLineDistribution: val = Validator(i, int(not i!=0), self.logger, self.shape, rows[(i*self.shape.chi):((i+1)*self.shape.chi)], @@ -104,7 +104,7 @@ class Simulator: v.columnNeighbors[id].update({vi.ID : Neighbor(vi, 1, self.shape.blockSize)}) if self.logger.isEnabledFor(logging.DEBUG): - for i in range(0, self.shape.numberValidators): + for i in range(0, self.shape.numberNodes): self.logger.debug("Val %d : rowN %s", i, self.validators[i].rowNeighbors, extra=self.format) self.logger.debug("Val %d : colN %s", i, self.validators[i].columnNeighbors, extra=self.format) @@ -155,17 +155,17 @@ class Simulator: missingVector.append(missingSamples) oldMissingSamples = missingSamples self.logger.debug("PHASE SEND %d" % steps, extra=self.format) - for i in range(0,self.shape.numberValidators): + for i in range(0,self.shape.numberNodes): self.validators[i].send() self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format) - for i in range(1,self.shape.numberValidators): + for i in range(1,self.shape.numberNodes): self.validators[i].receiveRowsColumns() self.logger.debug("PHASE RESTORE %d" % steps, extra=self.format) - for i in range(1,self.shape.numberValidators): + for i in range(1,self.shape.numberNodes): self.validators[i].restoreRows() self.validators[i].restoreColumns() self.logger.debug("PHASE LOG %d" % steps, extra=self.format) - for i in range(0,self.shape.numberValidators): + for i in range(0,self.shape.numberNodes): self.validators[i].logRows() self.validators[i].logColumns() @@ -176,7 +176,7 @@ class Simulator: (steps, statsTxInSlot[0], statsRxInSlot[0], mean(statsTxInSlot[1:]), max(statsTxInSlot[1:]), mean(statsRxInSlot[1:]), max(statsRxInSlot[1:])), extra=self.format) - for i in range(0,self.shape.numberValidators): + for i in range(0,self.shape.numberNodes): self.validators[i].updateStats() arrived, expected = self.glob.checkStatus(self.validators) diff --git a/DAS/validator.py b/DAS/validator.py index ae7be43..b0fbede 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -62,7 +62,7 @@ class Validator: else: if self.amIproposer: self.chi = 1 # not used - elif self.ID <= shape.numberValidators * shape.class1ratio: + elif self.ID <= shape.numberNodes * shape.class1ratio: self.chi = shape.chi * shape.vpn1 else: self.chi = shape.chi * shape.vpn2 # TODO: union of random subsets vpn2 times @@ -94,7 +94,7 @@ class Validator: # TODO: this should be a parameter if self.amIproposer: self.bwUplink = shape.bwUplinkProd - elif self.ID <= shape.numberValidators * shape.class1ratio: + elif self.ID <= shape.numberNodes * shape.class1ratio: self.bwUplink = shape.bwUplink1 else: self.bwUplink = shape.bwUplink2 diff --git a/DAS/visualizer.py b/DAS/visualizer.py index 7004c87..d165f0d 100644 --- a/DAS/visualizer.py +++ b/DAS/visualizer.py @@ -12,7 +12,7 @@ class Visualizer: def __init__(self, execID): self.execID = execID self.folderPath = "results/"+self.execID - self.parameters = ['run', 'blockSize', 'failureRate', 'numberValidators', 'netDegree', + self.parameters = ['run', 'blockSize', 'failureRate', 'numberNodes', 'netDegree', 'chi', 'vpn1', 'vpn2', 'bwUplinkProd', 'bwUplink1', 'bwUplink2'] self.minimumDataPoints = 2 @@ -28,7 +28,7 @@ class Visualizer: run = int(root.find('run').text) blockSize = int(root.find('blockSize').text) failureRate = int(root.find('failureRate').text) - numberValidators = int(root.find('numberValidators').text) + numberNodes = int(root.find('numberNodes').text) netDegree = int(root.find('netDegree').text) chi = int(root.find('chi').text) vpn1 = int(root.find('vpn1').text) @@ -42,7 +42,7 @@ class Visualizer: for combination in combinations(self.parameters, 4): # Get the indices and values of the parameters in the combination indices = [self.parameters.index(element) for element in combination] - selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi, vpn1, vpn2, bwUplinkProd, bwUplink1, bwUplink2] + selectedValues = [run, blockSize, failureRate, numberNodes, netDegree, chi, vpn1, vpn2, bwUplinkProd, bwUplink1, bwUplink2] values = [selectedValues[index] for index in indices] names = [self.parameters[i] for i in indices] keyComponents = [f"{name}_{value}" for name, value in zip(names, values)] diff --git a/config_example.py b/config_example.py index 0bd509c..5272555 100644 --- a/config_example.py +++ b/config_example.py @@ -34,7 +34,7 @@ evenLineDistribution = False runs = range(10) # Number of validators -numberValidators = range(256, 513, 128) +numberNodes = range(256, 513, 128) # Percentage of block not released by producer failureRates = range(10, 91, 40) @@ -68,9 +68,9 @@ deterministic = False randomSeed = "DAS" def nextShape(): - for run, fr, class1ratio, chi, vpn1, vpn2, blockSize, nv, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( - runs, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberValidators, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): + for run, fr, class1ratio, chi, vpn1, vpn2, blockSize, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( + runs, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): # Network Degree has to be an even number if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) + shape = Shape(blockSize, nn, fr, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) yield shape From 56139965477e9281b5bb75f4c218e98fce3313d8 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 13 Mar 2023 15:27:23 +0100 Subject: [PATCH 83/89] fixup: make validatorsPerNode and bandwidth consistent Signed-off-by: Csaba Kiraly --- config_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config_example.py b/config_example.py index 5272555..b54dff1 100644 --- a/config_example.py +++ b/config_example.py @@ -58,8 +58,8 @@ validatorsPerNode2 = [2, 4, 8, 16, 32] # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 bwUplinksProd = [2200] -bwUplinks1 = [2200] -bwUplinks2 = [110] +bwUplinks1 = [110] +bwUplinks2 = [2200] # Set to True if you want your run to be deterministic, False if not deterministic = False From c366c05616fa67b2634fd4f894a7b8039870132c Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 14 Mar 2023 15:26:48 +0100 Subject: [PATCH 84/89] handle overlap for multiple validators per node correctly Signed-off-by: Csaba Kiraly --- DAS/tools.py | 6 ++++++ DAS/validator.py | 13 ++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/DAS/tools.py b/DAS/tools.py index cd26850..2b47b11 100644 --- a/DAS/tools.py +++ b/DAS/tools.py @@ -79,3 +79,9 @@ def sampleLine(line, limit): r[i] = 1 limit -= 1 return r + +def unionOfSamples(population, sampleSize, times): + selected = set() + for t in range(times): + selected |= set(random.sample(population, sampleSize)) + return selected diff --git a/DAS/validator.py b/DAS/validator.py index b0fbede..9abb125 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -4,7 +4,7 @@ import random import collections import logging from DAS.block import * -from DAS.tools import shuffled, shuffledDict +from DAS.tools import shuffled, shuffledDict, unionOfSamples from bitarray.util import zeros from collections import deque from itertools import chain @@ -60,12 +60,7 @@ class Validator: elif self.shape.chi > self.shape.blockSize: self.logger.error("Chi has to be smaller than %d" % self.shape.blockSize, extra=self.format) else: - if self.amIproposer: - self.chi = 1 # not used - elif self.ID <= shape.numberNodes * shape.class1ratio: - self.chi = shape.chi * shape.vpn1 - else: - self.chi = shape.chi * shape.vpn2 # TODO: union of random subsets vpn2 times + self.chi = shape.chi if amIproposer: self.rowIDs = range(shape.blockSize) self.columnIDs = range(shape.blockSize) @@ -75,11 +70,11 @@ class Validator: if rows: self.rowIDs = rows else: - self.rowIDs = random.sample(range(self.shape.blockSize), self.chi) + self.rowIDs = unionOfSamples(range(self.shape.blockSize), self.chi, self.shape.vpn1) if columns: self.columnIDs = columns else: - self.columnIDs = random.sample(range(self.shape.blockSize), self.chi) + self.columnIDs = unionOfSamples(range(self.shape.blockSize), self.chi, self.shape.vpn1) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) From 353efec1412cf9350c81b011a242df806b90a2a7 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Mar 2023 12:53:43 +0100 Subject: [PATCH 85/89] fixup: use vpn1 and vpn2 correctly Signed-off-by: Csaba Kiraly --- DAS/validator.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 9abb125..f869171 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -60,21 +60,15 @@ class Validator: elif self.shape.chi > self.shape.blockSize: self.logger.error("Chi has to be smaller than %d" % self.shape.blockSize, extra=self.format) else: - self.chi = shape.chi if amIproposer: self.rowIDs = range(shape.blockSize) self.columnIDs = range(shape.blockSize) else: #if shape.deterministic: # random.seed(self.ID) - if rows: - self.rowIDs = rows - else: - self.rowIDs = unionOfSamples(range(self.shape.blockSize), self.chi, self.shape.vpn1) - if columns: - self.columnIDs = columns - else: - self.columnIDs = unionOfSamples(range(self.shape.blockSize), self.chi, self.shape.vpn1) + vpn = self.shape.vpn1 if (self.ID <= shape.numberNodes * shape.class1ratio) else self.shape.vpn2 + self.rowIDs = rows if rows else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) + self.columnIDs = columns if columns else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) From 64fcbff0ef1bd46e5721f527ebff8efed01b1707 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 20 Mar 2023 17:09:17 +0100 Subject: [PATCH 86/89] warn if no nodes for a row/column Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 4cadf9f..06ebd8e 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -64,7 +64,10 @@ class Simulator: # If the number of nodes in a channel is smaller or equal to the # requested degree, a fully connected graph is used. For n>d, a random # d-regular graph is set up. (For n=d+1, the two are the same.) - if (len(rowChannels[id]) <= self.shape.netDegree): + if not rowChannels[id]: + self.logger.error("No nodes for row %d !" % id, extra=self.format) + continue + elif (len(rowChannels[id]) <= self.shape.netDegree): self.logger.debug("Graph fully connected with degree %d !" % (len(rowChannels[id]) - 1), extra=self.format) G = nx.complete_graph(len(rowChannels[id])) else: @@ -77,7 +80,10 @@ class Simulator: val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, 0, self.shape.blockSize)}) val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, 0, self.shape.blockSize)}) - if (len(columnChannels[id]) <= self.shape.netDegree): + if not columnChannels[id]: + self.logger.error("No nodes for column %d !" % id, extra=self.format) + continue + elif (len(columnChannels[id]) <= self.shape.netDegree): self.logger.debug("Graph fully connected with degree %d !" % (len(columnChannels[id]) - 1), extra=self.format) G = nx.complete_graph(len(columnChannels[id])) else: From 125ba2ad4a64303b4d42e684cfdbfa4046046b1d Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Tue, 21 Mar 2023 10:41:52 +0100 Subject: [PATCH 87/89] Fix visualizer to loop over all parameters --- DAS/visualizer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/DAS/visualizer.py b/DAS/visualizer.py index d165f0d..8afd006 100644 --- a/DAS/visualizer.py +++ b/DAS/visualizer.py @@ -38,15 +38,15 @@ class Visualizer: bwUplink2 = int(root.find('bwUplink2').text) tta = int(root.find('tta').text) - # Loop over all possible combinations of length 4 of the parameters - for combination in combinations(self.parameters, 4): + # Loop over all possible combinations of of the parameters minus two + for combination in combinations(self.parameters, len(self.parameters)-2): # Get the indices and values of the parameters in the combination indices = [self.parameters.index(element) for element in combination] selectedValues = [run, blockSize, failureRate, numberNodes, netDegree, chi, vpn1, vpn2, bwUplinkProd, bwUplink1, bwUplink2] values = [selectedValues[index] for index in indices] names = [self.parameters[i] for i in indices] keyComponents = [f"{name}_{value}" for name, value in zip(names, values)] - key = tuple(keyComponents[:4]) + key = tuple(keyComponents[:len(self.parameters)-2]) #Get the names of the other 2 parameters that are not included in the key otherParams = [self.parameters[i] for i in range(len(self.parameters)) if i not in indices] #Append the values of the other 2 parameters and the ttas to the lists for the key @@ -92,7 +92,7 @@ class Visualizer: #Label formatting for the figures result = ''.join([f" {char}" if char.isupper() else char for char in label]) return result.title() - + def formatTitle(self, key): #Title formatting for the figures name = ''.join([f" {char}" if char.isupper() else char for char in key.split('_')[0]]) @@ -104,7 +104,7 @@ class Visualizer: data = self.plottingData() filteredKeys = self.similarKeys(data) print("Plotting heatmaps...") - + #Create the directory if it doesn't exist already heatmapsFolder = self.folderPath + '/heatmaps' if not os.path.exists(heatmapsFolder): From 3642083f2235d1355b5f777324f0b6a19a097bf9 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Tue, 21 Mar 2023 15:16:19 +0100 Subject: [PATCH 88/89] Fixing the global random uniform distribution of topics across validators --- DAS/simulator.py | 33 +++++++++++++++++++++++++++------ DAS/validator.py | 12 ++++++++---- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 06ebd8e..de3adff 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -30,15 +30,27 @@ class Simulator: self.glob.reset() self.validators = [] if self.config.evenLineDistribution: - rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberNodes/self.shape.blockSize) - columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberNodes/self.shape.blockSize) + + lightVal = int(self.shape.numberNodes * self.shape.class1ratio * self.shape.vpn1) + heavyVal = int(self.shape.numberNodes * (1-self.shape.class1ratio) * self.shape.vpn2) + totalValidators = lightVal + heavyVal + rows = list(range(self.shape.blockSize)) * (int(totalValidators/self.shape.blockSize)+1) + columns = list(range(self.shape.blockSize)) * (int(totalValidators/self.shape.blockSize)+1) + offset = heavyVal*self.shape.chi random.shuffle(rows) random.shuffle(columns) for i in range(self.shape.numberNodes): if self.config.evenLineDistribution: - val = Validator(i, int(not i!=0), self.logger, self.shape, - rows[(i*self.shape.chi):((i+1)*self.shape.chi)], - columns[(i*self.shape.chi):((i+1)*self.shape.chi)]) + if i < int(heavyVal/self.shape.vpn2): # First start with the heavy nodes + start = i *self.shape.chi*self.shape.vpn2 + end = (i+1)*self.shape.chi*self.shape.vpn2 + else: # Then the solo stakers + j = i - int(heavyVal/self.shape.vpn2) + start = offset+( j *self.shape.chi) + end = offset+((j+1)*self.shape.chi) + r = rows[start:end] + c = columns[start:end] + val = Validator(i, int(not i!=0), self.logger, self.shape, r, c, self.config.evenLineDistribution) else: val = Validator(i, int(not i!=0), self.logger, self.shape) if i == self.proposerID: @@ -47,6 +59,7 @@ class Simulator: else: val.logIDs() self.validators.append(val) + self.logger.debug("Validators initialized.", extra=self.format) def initNetwork(self): """It initializes the simulated network.""" @@ -59,6 +72,14 @@ class Simulator: for id in v.columnIDs: columnChannels[id].append(v) + # Check rows/columns distribution + #totalR = 0 + #totalC = 0 + #for r in rowChannels: + # totalR += len(r) + #for c in columnChannels: + # totalC += len(c) + for id in range(self.shape.blockSize): # If the number of nodes in a channel is smaller or equal to the @@ -178,7 +199,7 @@ class Simulator: # log TX and RX statistics statsTxInSlot = [v.statsTxInSlot for v in self.validators] statsRxInSlot = [v.statsRxInSlot for v in self.validators] - self.logger.debug("step %d: TX_prod=%.1f, RX_prod=%.1f, TX_avg=%.1f, TX_max=%.1f, Rx_avg=%.1f, Rx_max=%.1f" % + self.logger.debug("step %d: TX_prod=%.1f, RX_prod=%.1f, TX_avg=%.1f, TX_max=%.1f, Rx_avg=%.1f, Rx_max=%.1f" % (steps, statsTxInSlot[0], statsRxInSlot[0], mean(statsTxInSlot[1:]), max(statsTxInSlot[1:]), mean(statsRxInSlot[1:]), max(statsRxInSlot[1:])), extra=self.format) diff --git a/DAS/validator.py b/DAS/validator.py index f869171..6b3904e 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -38,7 +38,7 @@ class Validator: """It returns the validator ID.""" return str(self.ID) - def __init__(self, ID, amIproposer, logger, shape, rows = None, columns = None): + def __init__(self, ID, amIproposer, logger, shape, rows = None, columns = None, globalRandomness = True): """It initializes the validator with the logger shape and rows/columns. If rows/columns are specified these are observed, otherwise (default) @@ -66,9 +66,13 @@ class Validator: else: #if shape.deterministic: # random.seed(self.ID) - vpn = self.shape.vpn1 if (self.ID <= shape.numberNodes * shape.class1ratio) else self.shape.vpn2 - self.rowIDs = rows if rows else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) - self.columnIDs = columns if columns else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) + if globalRandomness: + self.rowIDs = rows + self.columnIDs = columns + else: + vpn = self.shape.vpn1 if (self.ID <= shape.numberNodes * shape.class1ratio) else self.shape.vpn2 + self.rowIDs = rows if rows else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) + self.columnIDs = columns if columns else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) From 680817b97be0a5b161dfa8475f8743f5edd6333c Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Thu, 23 Mar 2023 20:10:27 +0100 Subject: [PATCH 89/89] Remove global randomness parameter for validator --- DAS/simulator.py | 2 +- DAS/validator.py | 12 ++++-------- config_example.py | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index de3adff..30c0d86 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -50,7 +50,7 @@ class Simulator: end = offset+((j+1)*self.shape.chi) r = rows[start:end] c = columns[start:end] - val = Validator(i, int(not i!=0), self.logger, self.shape, r, c, self.config.evenLineDistribution) + val = Validator(i, int(not i!=0), self.logger, self.shape, r, c) else: val = Validator(i, int(not i!=0), self.logger, self.shape) if i == self.proposerID: diff --git a/DAS/validator.py b/DAS/validator.py index 6b3904e..f869171 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -38,7 +38,7 @@ class Validator: """It returns the validator ID.""" return str(self.ID) - def __init__(self, ID, amIproposer, logger, shape, rows = None, columns = None, globalRandomness = True): + def __init__(self, ID, amIproposer, logger, shape, rows = None, columns = None): """It initializes the validator with the logger shape and rows/columns. If rows/columns are specified these are observed, otherwise (default) @@ -66,13 +66,9 @@ class Validator: else: #if shape.deterministic: # random.seed(self.ID) - if globalRandomness: - self.rowIDs = rows - self.columnIDs = columns - else: - vpn = self.shape.vpn1 if (self.ID <= shape.numberNodes * shape.class1ratio) else self.shape.vpn2 - self.rowIDs = rows if rows else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) - self.columnIDs = columns if columns else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) + vpn = self.shape.vpn1 if (self.ID <= shape.numberNodes * shape.class1ratio) else self.shape.vpn2 + self.rowIDs = rows if rows else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) + self.columnIDs = columns if columns else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) diff --git a/config_example.py b/config_example.py index b54dff1..af55fc2 100644 --- a/config_example.py +++ b/config_example.py @@ -28,7 +28,7 @@ numJobs = 3 # distribute rows/columns evenly between validators (True) # or generate it using local randomness (False) -evenLineDistribution = False +evenLineDistribution = True # Number of simulation runs with the same parameters for statistical relevance runs = range(10)