From e8e82d746007978415b5c1abb1f1f65bc5b83281 Mon Sep 17 00:00:00 2001 From: leobago Date: Fri, 13 Jan 2023 16:51:27 +0100 Subject: [PATCH 1/4] Adding configuration class --- DAS/__init__.py | 1 + DAS/block.py | 4 ++-- DAS/configuration.py | 30 ++++++++++++++++++++++++ DAS/observer.py | 24 +++++++++---------- DAS/simulator.py | 38 +++++++++++++----------------- DAS/validator.py | 56 ++++++++++++++++++++------------------------ study.py | 11 ++++----- 7 files changed, 92 insertions(+), 72 deletions(-) create mode 100644 DAS/configuration.py diff --git a/DAS/__init__.py b/DAS/__init__.py index ef4ca24..abe299a 100644 --- a/DAS/__init__.py +++ b/DAS/__init__.py @@ -1 +1,2 @@ from DAS.simulator import * +from DAS.configuration import * diff --git a/DAS/block.py b/DAS/block.py index 76379d8..aa5fe01 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -9,8 +9,8 @@ class Block: blockSize = 0 data = bitarray() - def __init__(self, size): - self.blockSize = size + def __init__(self, blockSize): + self.blockSize = blockSize self.data = zeros(self.blockSize*self.blockSize) def fill(self): diff --git a/DAS/configuration.py b/DAS/configuration.py new file mode 100644 index 0000000..ded9547 --- /dev/null +++ b/DAS/configuration.py @@ -0,0 +1,30 @@ +#!/bin/python3 + +class Configuration: + + blockSize = 0 + failureRateStep = 0 + maxTries = 0 + numberValidators = 0 + chi = 0 + failureRate = 0 + deterministic = 0 + + def __init__(self, blockSize, failureRateStep, maxTries, numberValidators, chi, failureRate, deterministic): + if numberValidators < (blockSize*4): + print("ERROR: The number of validators cannot be lower than the block size * 4") + exit(1) + if chi < 1: + print("Chi has to be greater than 0") + exit(1) + if chi > blockSize: + print("Chi has to be smaller than %d" % blockSize) + exit(1) + + self.blockSize = blockSize + self.failureRateStep = failureRateStep + self.maxTries = maxTries + self.numberValidators = numberValidators + self.chi = chi + self.failureRate = failureRate + self.deterministic = deterministic diff --git a/DAS/observer.py b/DAS/observer.py index f905af3..ad9052b 100644 --- a/DAS/observer.py +++ b/DAS/observer.py @@ -5,40 +5,40 @@ from DAS.block import * class Observer: block = [] - blockSize = 0 rows = [] columns = [] goldenData = [] broadcasted = [] + config = [] logger = [] - def __init__(self, blockSize, logger): + def __init__(self, logger, config): + self.config = config self.format = {"entity": "Observer"} - self.blockSize = blockSize self.logger = logger def reset(self): - self.block = [0] * self.blockSize * self.blockSize - self.goldenData = [0] * self.blockSize * self.blockSize - self.rows = [0] * self.blockSize - self.columns = [0] * self.blockSize - self.broadcasted = Block(self.blockSize) + self.block = [0] * self.config.blockSize * self.config.blockSize + self.goldenData = [0] * self.config.blockSize * self.config.blockSize + self.rows = [0] * self.config.blockSize + self.columns = [0] * self.config.blockSize + self.broadcasted = Block(self.config.blockSize) def checkRowsColumns(self, validators): for val in validators: - if val.proposer == 0: + if val.amIproposer == 0: for r in val.rowIDs: self.rows[r] += 1 for c in val.columnIDs: self.columns[c] += 1 - for i in range(self.blockSize): + for i in range(self.config.blockSize): self.logger.debug("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format) if self.rows[i] == 0 or self.columns[i] == 0: self.logger.warning("There is a row/column that has not been assigned", extra=self.format) def setGoldenData(self, block): - for i in range(self.blockSize*self.blockSize): + for i in range(self.config.blockSize*self.config.blockSize): self.goldenData[i] = block.data[i] def checkBroadcasted(self): @@ -54,7 +54,7 @@ class Observer: arrived = 0 expected = 0 for val in validators: - if val.proposer == 0: + if val.amIproposer == 0: (a, e) = val.checkStatus() arrived += a expected += e diff --git a/DAS/simulator.py b/DAS/simulator.py index fcfeb2e..25adcc3 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -9,32 +9,26 @@ from DAS.validator import * class Simulator: - chi = 8 - blockSize = 256 - numberValidators = 8192 - failureRate = 0 proposerID = 0 logLevel = logging.INFO - deterministic = 0 validators = [] glob = [] + config = [] logger = [] format = {} - steps = 0 - def __init__(self, failureRate): - self.failureRate = failureRate + def __init__(self, config): + self.config = config self.format = {"entity": "Simulator"} - self.steps = 0 def initValidators(self): - if not self.deterministic: + if not self.config.deterministic: random.seed(datetime.now()) - self.glob = Observer(self.blockSize, self.logger) + self.glob = Observer(self.logger, self.config) self.glob.reset() self.validators = [] - for i in range(self.numberValidators): - val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic, self.logger) + for i in range(self.config.numberValidators): + val = Validator(i, int(not i!=0), self.logger, self.config) if i == self.proposerID: val.initBlock() self.glob.setGoldenData(val.block) @@ -43,15 +37,15 @@ class Simulator: self.validators.append(val) def initNetwork(self, d=6): - rowChannels = [[] for i in range(self.blockSize)] - columnChannels = [[] for i in range(self.blockSize)] + rowChannels = [[] for i in range(self.config.blockSize)] + columnChannels = [[] for i in range(self.config.blockSize)] for v in self.validators: for id in v.rowIDs: rowChannels[id].append(v) for id in v.columnIDs: columnChannels[id].append(v) - for id in range(self.blockSize): + for id in range(self.config.blockSize): G = nx.random_regular_graph(d, len(rowChannels[id])) if not nx.is_connected(G): self.logger.error("graph not connected for row %d !" % id, extra=self.format) @@ -86,12 +80,12 @@ class Simulator: self.validators[self.proposerID].broadcastBlock() arrived, expected = self.glob.checkStatus(self.validators) missingSamples = expected - arrived - self.steps = 0 + steps = 0 while(missingSamples > 0): oldMissingSamples = missingSamples - for i in range(1,self.numberValidators): + for i in range(1,self.config.numberValidators): self.validators[i].receiveRowsColumns() - for i in range(1,self.numberValidators): + for i in range(1,self.config.numberValidators): self.validators[i].restoreRows() self.validators[i].restoreColumns() self.validators[i].sendRows() @@ -102,16 +96,16 @@ class Simulator: arrived, expected = self.glob.checkStatus(self.validators) missingSamples = expected - arrived missingRate = missingSamples*100/expected - self.logger.info("step %d, missing %d of %d (%0.02f %%)" % (self.steps, missingSamples, expected, missingRate), extra=self.format) + self.logger.info("step %d, missing %d of %d (%0.02f %%)" % (steps, missingSamples, expected, missingRate), extra=self.format) if missingSamples == oldMissingSamples: break elif missingSamples == 0: break else: - self.steps += 1 + steps += 1 if missingSamples == 0: - self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (self.steps, self.failureRate), extra=self.format) + self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.failureRate), extra=self.format) return 0 else: self.logger.debug("The block cannot be recovered, failure rate %d!" % self.failureRate, extra=self.format) diff --git a/DAS/validator.py b/DAS/validator.py index eaeb0a0..7d29a8f 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -10,44 +10,40 @@ from bitarray.util import zeros class Validator: ID = 0 - chi = 0 + amIproposer = 0 + config = [] format = {} - blocksize = 0 - proposer = 0 - failureRate = 0 logger = [] - def __init__(self, ID, chi, blockSize, proposer, failureRate, deterministic, logger): + def __init__(self, ID, amIproposer, logger, config): + self.config = config FORMAT = "%(levelname)s : %(entity)s : %(message)s" self.ID = ID self.format = {"entity": "Val "+str(self.ID)} - self.blockSize = blockSize - self.block = Block(blockSize) - self.receivedBlock = Block(blockSize) - self.proposer = proposer - self.failureRate = failureRate + self.block = Block(self.config.blockSize) + self.receivedBlock = Block(self.config.blockSize) + self.amIproposer = amIproposer self.logger = logger - if chi < 1: + if self.config.chi < 1: self.logger.error("Chi has to be greater than 0", extra=self.format) - elif chi > blockSize: + elif self.config.chi > self.config.blockSize: self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) else: - self.chi = chi - if proposer: - self.rowIDs = range(blockSize) - self.columnIDs = range(blockSize) + if amIproposer: + self.rowIDs = range(config.blockSize) + self.columnIDs = range(config.blockSize) else: self.rowIDs = [] self.columnIDs = [] - if deterministic: + if config.deterministic: random.seed(self.ID) - self.rowIDs = random.sample(range(self.blockSize), self.chi) - self.columnIDs = random.sample(range(self.blockSize), self.chi) + self.rowIDs = random.sample(range(self.config.blockSize), self.config.chi) + self.columnIDs = random.sample(range(self.config.blockSize), self.config.chi) self.rowNeighbors = collections.defaultdict(list) self.columnNeighbors = collections.defaultdict(list) def logIDs(self): - if self.proposer == 1: + if self.amIproposer == 1: self.logger.warning("I am a block proposer."% self.ID) else: self.logger.debug("Selected rows: "+str(self.rowIDs), extra=self.format) @@ -55,30 +51,30 @@ class Validator: def initBlock(self): self.logger.debug("I am a block proposer.", extra=self.format) - self.block = Block(self.blockSize) + self.block = Block(self.config.blockSize) self.block.fill() #self.block.print() def broadcastBlock(self): - if self.proposer == 0: + if self.amIproposer == 0: self.logger.error("I am NOT a block proposer", extra=self.format) else: self.logger.debug("Broadcasting my block...", extra=self.format) - order = [i for i in range(self.blockSize * self.blockSize)] + order = [i for i in range(self.config.blockSize * self.config.blockSize)] random.shuffle(order) while(order): i = order.pop() - if (random.randint(0,99) >= self.failureRate): + if (random.randint(0,99) >= self.config.failureRate): self.block.data[i] = 1 else: self.block.data[i] = 0 nbFailures = self.block.data.count(0) - measuredFailureRate = nbFailures * 100 / (self.blockSize * self.blockSize) + measuredFailureRate = nbFailures * 100 / (self.config.blockSize * self.config.blockSize) self.logger.info("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format) #broadcasted.print() - for id in range(self.blockSize): + for id in range(self.config.blockSize): self.sendColumn(id) - for id in range(self.blockSize): + for id in range(self.config.blockSize): self.sendRow(id) def getColumn(self, index): @@ -101,7 +97,7 @@ class Validator: def receiveRowsColumns(self): - if self.proposer == 1: + if self.amIproposer == 1: self.logger.error("I am a block proposer", extra=self.format) else: self.logger.debug("Receiving the data...", extra=self.format) @@ -124,7 +120,7 @@ class Validator: n.receiveRow(rowID, line) def sendRows(self): - if self.proposer == 1: + if self.amIproposer == 1: self.logger.error("I am a block proposer", extra=self.format) else: self.logger.debug("Sending restored rows...", extra=self.format) @@ -132,7 +128,7 @@ class Validator: self.sendRow(r) def sendColumns(self): - if self.proposer == 1: + if self.amIproposer == 1: self.logger.error("I am a block proposer", extra=self.format) else: self.logger.debug("Sending restored columns...", extra=self.format) diff --git a/study.py b/study.py index 04578de..0b2fa0a 100644 --- a/study.py +++ b/study.py @@ -5,27 +5,26 @@ from DAS import * def study(): - sim = Simulator(0) + config = Configuration(64, 20, 10, 256, 8, 0, 0) + sim = Simulator(config) sim.initLogger() - maxTries = 10 - step = 20 frRange = [] resultRange = [] simCnt = 0 sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() - for fr in range(0, 100, step): + for fr in range(0, 100, config.failureRateStep): if fr % 10 == 0: sim.logger.info("Failure rate %d %% ..." % fr, extra=sim.format) sim.resetFailureRate(fr) result = 0 - for i in range(maxTries): + for i in range(config.maxTries): sim.initValidators() sim.initNetwork() result += sim.run() simCnt += 1 frRange.append(fr) - resultRange.append((maxTries-result)*100/maxTries) + resultRange.append((config.maxTries-result)*100/config.maxTries) end = time.time() sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format) for i in range(len(frRange)): From bf1a5a60e4671d9f8adb675f95aad71aeb7e01bd Mon Sep 17 00:00:00 2001 From: leobago Date: Mon, 16 Jan 2023 22:43:52 +0100 Subject: [PATCH 2/4] Adding multidimensional simulator and results module to gather data for visualization. --- DAS/configuration.py | 50 +++++++++++++++++++++++++++++++------------- DAS/results.py | 17 +++++++++++++++ DAS/simulator.py | 32 +++++++++++++++++++++------- DAS/validator.py | 2 +- study.py | 30 ++++++++++++++------------ 5 files changed, 94 insertions(+), 37 deletions(-) create mode 100644 DAS/results.py diff --git a/DAS/configuration.py b/DAS/configuration.py index ded9547..949723b 100644 --- a/DAS/configuration.py +++ b/DAS/configuration.py @@ -2,29 +2,49 @@ class Configuration: - blockSize = 0 - failureRateStep = 0 - maxTries = 0 - numberValidators = 0 - chi = 0 - failureRate = 0 deterministic = 0 + blockSize = 0 + numberValidators = 0 + failureRate = 0 + failureRateStart = 0 + failureRateStop = 0 + failureRateStep = 0 + chio = 0 + chiStart = 0 + chiStop = 0 + chiStep = 0 + run =0 + runStart = 0 + runStop = 0 + runStep = 0 + + def __init__(self, deterministic, blockSize, numberValidators,\ + failureRateStart, failureRateStop, failureRateStep,\ + chiStart, chiStop, chiStep,\ + runStart, runStop, runStep): - def __init__(self, blockSize, failureRateStep, maxTries, numberValidators, chi, failureRate, deterministic): if numberValidators < (blockSize*4): print("ERROR: The number of validators cannot be lower than the block size * 4") exit(1) - if chi < 1: + if chiStart < 1: print("Chi has to be greater than 0") exit(1) - if chi > blockSize: + if chiStop > blockSize: print("Chi has to be smaller than %d" % blockSize) exit(1) - self.blockSize = blockSize - self.failureRateStep = failureRateStep - self.maxTries = maxTries - self.numberValidators = numberValidators - self.chi = chi - self.failureRate = failureRate self.deterministic = deterministic + self.blockSize = blockSize + self.numberValidators = numberValidators + self.failureRateStart = failureRateStart + self.failureRateStop = failureRateStop + self.failureRateStep = failureRateStep + self.failureRate = failureRateStart + self.chiStart = chiStart + self.chiStop = chiStop + self.chiStep = chiStep + self.chi = chiStart + self.runStart = runStart + self.runStop = runStop + self.runStep = runStep + self.run = runStart diff --git a/DAS/results.py b/DAS/results.py new file mode 100644 index 0000000..8468b09 --- /dev/null +++ b/DAS/results.py @@ -0,0 +1,17 @@ +#!/bin/python3 + + +class Result: + + config = [] + missingVector = [] + blockAvailable = -1 + + def __init__(self, config): + self.config = config + self.blockAvailable = -1 + self.missingVector = [] + + + def addMissing(self, missingVector): + self.missingVector = missingVector diff --git a/DAS/simulator.py b/DAS/simulator.py index 25adcc3..42b0e1f 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -4,6 +4,7 @@ import networkx as nx import logging from datetime import datetime from DAS.tools import * +from DAS.results import * from DAS.observer import * from DAS.validator import * @@ -13,6 +14,7 @@ class Simulator: logLevel = logging.INFO validators = [] glob = [] + result = [] config = [] logger = [] format = {} @@ -20,6 +22,7 @@ class Simulator: def __init__(self, config): self.config = config self.format = {"entity": "Simulator"} + self.result = Result(self.config) def initValidators(self): if not self.config.deterministic: @@ -48,7 +51,7 @@ class Simulator: for id in range(self.config.blockSize): G = nx.random_regular_graph(d, len(rowChannels[id])) if not nx.is_connected(G): - self.logger.error("graph not connected for row %d !" % id, extra=self.format) + self.logger.error("Graph not connected for row %d !" % id, extra=self.format) for u, v in G.edges: val1=rowChannels[id][u] val2=rowChannels[id][v] @@ -56,7 +59,7 @@ class Simulator: val2.rowNeighbors[id].append(val1) G = nx.random_regular_graph(d, len(columnChannels[id])) if not nx.is_connected(G): - self.logger.error("graph not connected for column %d !" % id, extra=self.format) + self.logger.error("Graph not connected for column %d !" % id, extra=self.format) for u, v in G.edges: val1=columnChannels[id][u] val2=columnChannels[id][v] @@ -73,15 +76,25 @@ class Simulator: self.logger = logger def resetFailureRate(self, failureRate): - self.failureRate = failureRate + self.config.failureRate = failureRate + for val in self.validators: + val.config.failureRate = failureRate + + def resetChi(self, chi): + self.config.chi = chi + for val in self.validators: + val.config.chi = chi + def run(self): self.glob.checkRowsColumns(self.validators) self.validators[self.proposerID].broadcastBlock() arrived, expected = self.glob.checkStatus(self.validators) missingSamples = expected - arrived + missingVector = [] steps = 0 while(missingSamples > 0): + missingVector.append(missingSamples) oldMissingSamples = missingSamples for i in range(1,self.config.numberValidators): self.validators[i].receiveRowsColumns() @@ -96,7 +109,7 @@ class Simulator: arrived, expected = self.glob.checkStatus(self.validators) missingSamples = expected - arrived missingRate = missingSamples*100/expected - self.logger.info("step %d, missing %d of %d (%0.02f %%)" % (steps, missingSamples, expected, missingRate), extra=self.format) + self.logger.debug("step %d, missing %d of %d (%0.02f %%)" % (steps, missingSamples, expected, missingRate), extra=self.format) if missingSamples == oldMissingSamples: break elif missingSamples == 0: @@ -104,10 +117,13 @@ class Simulator: else: steps += 1 + self.result.addMissing(missingVector) if missingSamples == 0: - self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.failureRate), extra=self.format) - return 0 + self.result.blockAvailable = 1 + self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.config.failureRate), extra=self.format) + return self.result else: - self.logger.debug("The block cannot be recovered, failure rate %d!" % self.failureRate, extra=self.format) - return 1 + self.result.blockAvailable = 0 + self.logger.debug("The block cannot be recovered, failure rate %d!" % self.config.failureRate, extra=self.format) + return self.result diff --git a/DAS/validator.py b/DAS/validator.py index 7d29a8f..7a4b2fe 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -70,7 +70,7 @@ class Validator: self.block.data[i] = 0 nbFailures = self.block.data.count(0) measuredFailureRate = nbFailures * 100 / (self.config.blockSize * self.config.blockSize) - self.logger.info("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format) + self.logger.debug("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format) #broadcasted.print() for id in range(self.config.blockSize): self.sendColumn(id) diff --git a/study.py b/study.py index 0b2fa0a..fcd1444 100644 --- a/study.py +++ b/study.py @@ -5,30 +5,34 @@ from DAS import * def study(): - config = Configuration(64, 20, 10, 256, 8, 0, 0) + config = Configuration(0, 64, 256, 0, 100, 20, 8, 16, 4, 0, 10, 1) sim = Simulator(config) sim.initLogger() frRange = [] + results = [] resultRange = [] simCnt = 0 sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() - for fr in range(0, 100, config.failureRateStep): - if fr % 10 == 0: - sim.logger.info("Failure rate %d %% ..." % fr, extra=sim.format) + for fr in range(config.failureRateStart, config.failureRateStop+1, config.failureRateStep): sim.resetFailureRate(fr) - result = 0 - for i in range(config.maxTries): - sim.initValidators() - sim.initNetwork() - result += sim.run() - simCnt += 1 + for chi in range(config.chiStart, config.chiStop+1, config.chiStep): + sim.resetChi(chi) + blockAvailable = 0 + for run in range(config.runStart, config.runStop, config.runStep): + sim.logger.info("FR: %d %%, Chi: %d %%, Run: %d ..." % (fr, chi, run), extra=sim.format) + sim.initValidators() + sim.initNetwork() + result = sim.run() + blockAvailable += result.blockAvailable + results.append(result) + simCnt += 1 frRange.append(fr) - resultRange.append((config.maxTries-result)*100/config.maxTries) + resultRange.append((blockAvailable)*100/config.runStop) end = time.time() sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format) - for i in range(len(frRange)): - sim.logger.info("For failure rate of %d we got %d %% success rate in DAS!" % (frRange[i], resultRange[i]), extra=sim.format) + #for i in range(len(frRange)): + # sim.logger.info("For failure rate of %d we got %d %% success rate in DAS!" % (frRange[i], resultRange[i]), extra=sim.format) study() From fc7339dc9187b6514f34a5e1dbfdba5c7155a298 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Mon, 23 Jan 2023 18:04:54 +0100 Subject: [PATCH 3/4] Add configuration file, split configuration from simulation shape, fix bug about network degree and unbalanced row/column verification --- DAS/__init__.py | 1 + DAS/configuration.py | 72 +++++++++++++++++++++----------------------- DAS/shape.py | 19 ++++++++++++ DAS/simulator.py | 60 +++++++++++++++++++----------------- DAS/validator.py | 42 +++++++++++++------------- README.md | 8 ++--- config.das | 27 +++++++++++++++++ study.py | 49 +++++++++++++++++------------- 8 files changed, 168 insertions(+), 110 deletions(-) create mode 100644 DAS/shape.py create mode 100644 config.das diff --git a/DAS/__init__.py b/DAS/__init__.py index abe299a..8dd11bf 100644 --- a/DAS/__init__.py +++ b/DAS/__init__.py @@ -1,2 +1,3 @@ from DAS.simulator import * from DAS.configuration import * +from DAS.shape import * diff --git a/DAS/configuration.py b/DAS/configuration.py index 949723b..91df16f 100644 --- a/DAS/configuration.py +++ b/DAS/configuration.py @@ -1,50 +1,48 @@ #!/bin/python3 +import configparser + class Configuration: deterministic = 0 - blockSize = 0 - numberValidators = 0 - failureRate = 0 - failureRateStart = 0 - failureRateStop = 0 - failureRateStep = 0 - chio = 0 - chiStart = 0 - chiStop = 0 - chiStep = 0 - run =0 - runStart = 0 - runStop = 0 - runStep = 0 - def __init__(self, deterministic, blockSize, numberValidators,\ - failureRateStart, failureRateStop, failureRateStep,\ - chiStart, chiStop, chiStep,\ - runStart, runStop, runStep): + def __init__(self, fileName): - if numberValidators < (blockSize*4): + config = configparser.RawConfigParser() + config.read(fileName) + + self.nvStart = int(config.get("Simulation Space", "numberValidatorStart")) + self.nvStop = int(config.get("Simulation Space", "numberValidatorStop")) + self.nvStep = int(config.get("Simulation Space", "numberValidatorStep")) + + self.blockSizeStart = int(config.get("Simulation Space", "blockSizeStart")) + self.blockSizeStop = int(config.get("Simulation Space", "blockSizeStop")) + self.blockSizeStep = int(config.get("Simulation Space", "blockSizeStep")) + + self.netDegreeStart = int(config.get("Simulation Space", "netDegreeStart")) + self.netDegreeStop = int(config.get("Simulation Space", "netDegreeStop")) + self.netDegreeStep = int(config.get("Simulation Space", "netDegreeStep")) + + self.failureRateStart = int(config.get("Simulation Space", "failureRateStart")) + self.failureRateStop = int(config.get("Simulation Space", "failureRateStop")) + self.failureRateStep = int(config.get("Simulation Space", "failureRateStep")) + + self.chiStart = int(config.get("Simulation Space", "chiStart")) + self.chiStop = int(config.get("Simulation Space", "chiStop")) + self.chiStep = int(config.get("Simulation Space", "chiStep")) + + self.numberRuns = int(config.get("Advanced", "numberRuns")) + self.deterministic = config.get("Advanced", "deterministic") + + if self.nvStop < (self.blockSizeStart*4): print("ERROR: The number of validators cannot be lower than the block size * 4") exit(1) - if chiStart < 1: + if self.chiStart < 1: print("Chi has to be greater than 0") exit(1) - if chiStop > blockSize: - print("Chi has to be smaller than %d" % blockSize) + if self.chiStop > self.blockSizeStart: + print("Chi (%d) has to be smaller or equal to block the size (%d)" % (self.chiStop, self.blockSizeStart)) exit(1) - self.deterministic = deterministic - self.blockSize = blockSize - self.numberValidators = numberValidators - self.failureRateStart = failureRateStart - self.failureRateStop = failureRateStop - self.failureRateStep = failureRateStep - self.failureRate = failureRateStart - self.chiStart = chiStart - self.chiStop = chiStop - self.chiStep = chiStep - self.chi = chiStart - self.runStart = runStart - self.runStop = runStop - self.runStep = runStep - self.run = runStart + + diff --git a/DAS/shape.py b/DAS/shape.py new file mode 100644 index 0000000..2918422 --- /dev/null +++ b/DAS/shape.py @@ -0,0 +1,19 @@ +#!/bin/python3 + +class Shape: + numberValidators = 0 + failureRate = 0 + blockSize = 0 + netDegree = 0 + chi = 0 + + def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree): + self.numberValidators = numberValidators + self.failureRate = failureRate + self.blockSize = blockSize + self.netDegree = netDegree + self.chi = chi + + + + diff --git a/DAS/simulator.py b/DAS/simulator.py index 42b0e1f..3701167 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -1,7 +1,7 @@ #!/bin/python import networkx as nx -import logging +import logging, random from datetime import datetime from DAS.tools import * from DAS.results import * @@ -15,23 +15,25 @@ class Simulator: validators = [] glob = [] result = [] - config = [] + shape = [] logger = [] format = {} - def __init__(self, config): - self.config = config + def __init__(self, shape): + self.shape = shape self.format = {"entity": "Simulator"} - self.result = Result(self.config) + self.result = Result(self.shape) def initValidators(self): - if not self.config.deterministic: - random.seed(datetime.now()) - self.glob = Observer(self.logger, self.config) + self.glob = Observer(self.logger, self.shape) self.glob.reset() self.validators = [] - for i in range(self.config.numberValidators): - val = Validator(i, int(not i!=0), self.logger, self.config) + rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) + columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) + random.shuffle(rows) + random.shuffle(columns) + for i in range(self.shape.numberValidators): + val = Validator(i, int(not i!=0), self.logger, self.shape, rows, columns) if i == self.proposerID: val.initBlock() self.glob.setGoldenData(val.block) @@ -39,17 +41,21 @@ class Simulator: val.logIDs() self.validators.append(val) - def initNetwork(self, d=6): - rowChannels = [[] for i in range(self.config.blockSize)] - columnChannels = [[] for i in range(self.config.blockSize)] + def initNetwork(self): + self.shape.netDegree = 6 + rowChannels = [[] for i in range(self.shape.blockSize)] + columnChannels = [[] for i in range(self.shape.blockSize)] for v in self.validators: for id in v.rowIDs: rowChannels[id].append(v) for id in v.columnIDs: columnChannels[id].append(v) - for id in range(self.config.blockSize): - G = nx.random_regular_graph(d, len(rowChannels[id])) + for id in range(self.shape.blockSize): + + if (len(rowChannels[id]) < self.shape.netDegree): + self.logger.error("Graph degree higher than %d" % len(rowChannels[id]), extra=self.format) + G = nx.random_regular_graph(self.shape.netDegree, len(rowChannels[id])) if not nx.is_connected(G): self.logger.error("Graph not connected for row %d !" % id, extra=self.format) for u, v in G.edges: @@ -57,7 +63,10 @@ class Simulator: val2=rowChannels[id][v] val1.rowNeighbors[id].append(val2) val2.rowNeighbors[id].append(val1) - G = nx.random_regular_graph(d, len(columnChannels[id])) + + if (len(columnChannels[id]) < self.shape.netDegree): + self.logger.error("Graph degree higher than %d" % len(columnChannels[id]), extra=self.format) + G = nx.random_regular_graph(self.shape.netDegree, len(columnChannels[id])) if not nx.is_connected(G): self.logger.error("Graph not connected for column %d !" % id, extra=self.format) for u, v in G.edges: @@ -75,15 +84,12 @@ class Simulator: logger.addHandler(ch) self.logger = logger - def resetFailureRate(self, failureRate): - self.config.failureRate = failureRate - for val in self.validators: - val.config.failureRate = failureRate - def resetChi(self, chi): - self.config.chi = chi + def resetShape(self, shape): + self.shape = shape for val in self.validators: - val.config.chi = chi + val.shape.failureRate = shape.failureRate + val.shape.chi = shape.chi def run(self): @@ -96,9 +102,9 @@ class Simulator: while(missingSamples > 0): missingVector.append(missingSamples) oldMissingSamples = missingSamples - for i in range(1,self.config.numberValidators): + for i in range(1,self.shape.numberValidators): self.validators[i].receiveRowsColumns() - for i in range(1,self.config.numberValidators): + for i in range(1,self.shape.numberValidators): self.validators[i].restoreRows() self.validators[i].restoreColumns() self.validators[i].sendRows() @@ -120,10 +126,10 @@ class Simulator: self.result.addMissing(missingVector) if missingSamples == 0: self.result.blockAvailable = 1 - self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.config.failureRate), extra=self.format) + self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.shape.failureRate), extra=self.format) return self.result else: self.result.blockAvailable = 0 - self.logger.debug("The block cannot be recovered, failure rate %d!" % self.config.failureRate, extra=self.format) + self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format) return self.result diff --git a/DAS/validator.py b/DAS/validator.py index 7a4b2fe..af16df5 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -11,34 +11,34 @@ class Validator: ID = 0 amIproposer = 0 - config = [] + shape = [] format = {} logger = [] - def __init__(self, ID, amIproposer, logger, config): - self.config = config + def __init__(self, ID, amIproposer, logger, shape, rows, columns): + self.shape = shape FORMAT = "%(levelname)s : %(entity)s : %(message)s" self.ID = ID self.format = {"entity": "Val "+str(self.ID)} - self.block = Block(self.config.blockSize) - self.receivedBlock = Block(self.config.blockSize) + self.block = Block(self.shape.blockSize) + self.receivedBlock = Block(self.shape.blockSize) self.amIproposer = amIproposer self.logger = logger - if self.config.chi < 1: + if self.shape.chi < 1: self.logger.error("Chi has to be greater than 0", extra=self.format) - elif self.config.chi > self.config.blockSize: + elif self.shape.chi > self.shape.blockSize: self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) else: if amIproposer: - self.rowIDs = range(config.blockSize) - self.columnIDs = range(config.blockSize) + self.rowIDs = range(shape.blockSize) + self.columnIDs = range(shape.blockSize) else: - self.rowIDs = [] - self.columnIDs = [] - if config.deterministic: - random.seed(self.ID) - self.rowIDs = random.sample(range(self.config.blockSize), self.config.chi) - self.columnIDs = random.sample(range(self.config.blockSize), self.config.chi) + self.rowIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] + self.columnIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] + #if shape.deterministic: + # random.seed(self.ID) + #self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi) + #self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi) self.rowNeighbors = collections.defaultdict(list) self.columnNeighbors = collections.defaultdict(list) @@ -51,7 +51,7 @@ class Validator: def initBlock(self): self.logger.debug("I am a block proposer.", extra=self.format) - self.block = Block(self.config.blockSize) + self.block = Block(self.shape.blockSize) self.block.fill() #self.block.print() @@ -60,21 +60,21 @@ class Validator: self.logger.error("I am NOT a block proposer", extra=self.format) else: self.logger.debug("Broadcasting my block...", extra=self.format) - order = [i for i in range(self.config.blockSize * self.config.blockSize)] + order = [i for i in range(self.shape.blockSize * self.shape.blockSize)] random.shuffle(order) while(order): i = order.pop() - if (random.randint(0,99) >= self.config.failureRate): + if (random.randint(0,99) >= self.shape.failureRate): self.block.data[i] = 1 else: self.block.data[i] = 0 nbFailures = self.block.data.count(0) - measuredFailureRate = nbFailures * 100 / (self.config.blockSize * self.config.blockSize) + measuredFailureRate = nbFailures * 100 / (self.shape.blockSize * self.shape.blockSize) self.logger.debug("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format) #broadcasted.print() - for id in range(self.config.blockSize): + for id in range(self.shape.blockSize): self.sendColumn(id) - for id in range(self.config.blockSize): + for id in range(self.shape.blockSize): self.sendRow(id) def getColumn(self, index): diff --git a/README.md b/README.md index 1b41851..97846ce 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# DAS Research +# DAS Research -This repository hosts all the research on DAS for the collaboration between Codex and the EF. +This repository hosts all the research on DAS for the collaboration between Codex and the EF. ## Prepare the environment @@ -16,11 +16,11 @@ $ cd das-research ``` $ python3 -m venv myenv $ source myenv/bin/activate -$ pip3 install -r DAS/requeriments.txt +$ pip3 install -r DAS/requirements.txt ``` ## Run the simulator ``` -$ python3 study.py +$ python3 study.py config.das ``` diff --git a/config.das b/config.das new file mode 100644 index 0000000..f5d608e --- /dev/null +++ b/config.das @@ -0,0 +1,27 @@ +[Simulation Space] + +numberValidatorStart = 256 +numberValidatorStop = 512 +numberValidatorStep = 128 + +failureRateStart = 10 +failureRateStop = 90 +failureRateStep = 40 + +blockSizeStart = 32 +blockSizeStop = 64 +blockSizeStep = 16 + +netDegreeStart = 6 +netDegreeStop = 8 +netDegreeStep = 1 + +chiStart = 4 +chiStop = 8 +chiStep = 2 + + +[Advanced] + +deterministic = 0 +numberRuns = 2 diff --git a/study.py b/study.py index fcd1444..d74cecc 100644 --- a/study.py +++ b/study.py @@ -1,38 +1,45 @@ #! /bin/python3 -import time +import time, sys from DAS import * def study(): - config = Configuration(0, 64, 256, 0, 100, 20, 8, 16, 4, 0, 10, 1) + if len(sys.argv) < 2: + print("You need to pass a configuration file in parameter") + exit(1) + + config = Configuration(sys.argv[1]) sim = Simulator(config) sim.initLogger() - frRange = [] results = [] - resultRange = [] simCnt = 0 + sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() - for fr in range(config.failureRateStart, config.failureRateStop+1, config.failureRateStep): - sim.resetFailureRate(fr) - for chi in range(config.chiStart, config.chiStop+1, config.chiStep): - sim.resetChi(chi) - blockAvailable = 0 - for run in range(config.runStart, config.runStop, config.runStep): - sim.logger.info("FR: %d %%, Chi: %d %%, Run: %d ..." % (fr, chi, run), extra=sim.format) - sim.initValidators() - sim.initNetwork() - result = sim.run() - blockAvailable += result.blockAvailable - results.append(result) - simCnt += 1 - frRange.append(fr) - resultRange.append((blockAvailable)*100/config.runStop) + + for run in range(config.numberRuns): + for fr in range(config.failureRateStart, config.failureRateStop+1, config.failureRateStep): + for chi in range(config.chiStart, config.chiStop+1, config.chiStep): + for blockSize in range(config.blockSizeStart, config.blockSizeStop+1, config.blockSizeStep): + for nv in range(config.nvStart, config.nvStop+1, config.nvStep): + for netDegree in range(config.netDegreeStart, config.netDegreeStop, config.netDegreeStep): + + if not config.deterministic: + random.seed(datetime.now()) + + shape = Shape(blockSize, nv, fr, chi, netDegree) + sim.resetShape(shape) + sim.initValidators() + sim.initNetwork() + result = sim.run() + sim.logger.info("Run %d, FR: %d %%, Chi: %d, BlockSize: %d, Nb.Val: %d, netDegree: %d ... Block Available: %d" % (run, fr, chi, blockSize, nv, netDegree, result.blockAvailable), extra=sim.format) + results.append(result) + simCnt += 1 + end = time.time() sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format) - #for i in range(len(frRange)): - # sim.logger.info("For failure rate of %d we got %d %% success rate in DAS!" % (frRange[i], resultRange[i]), extra=sim.format) + study() From ef3714054b6ad474a2d6dbf1b02fb3aca336d226 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Thu, 26 Jan 2023 14:30:56 +0100 Subject: [PATCH 4/4] Fix netDegree loop --- study.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/study.py b/study.py index d74cecc..ae27e5c 100644 --- a/study.py +++ b/study.py @@ -23,7 +23,7 @@ def study(): for chi in range(config.chiStart, config.chiStop+1, config.chiStep): for blockSize in range(config.blockSizeStart, config.blockSizeStop+1, config.blockSizeStep): for nv in range(config.nvStart, config.nvStop+1, config.nvStep): - for netDegree in range(config.netDegreeStart, config.netDegreeStop, config.netDegreeStep): + for netDegree in range(config.netDegreeStart, config.netDegreeStop+1, config.netDegreeStep): if not config.deterministic: random.seed(datetime.now())