From fc7339dc9187b6514f34a5e1dbfdba5c7155a298 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Mon, 23 Jan 2023 18:04:54 +0100 Subject: [PATCH] Add configuration file, split configuration from simulation shape, fix bug about network degree and unbalanced row/column verification --- DAS/__init__.py | 1 + DAS/configuration.py | 72 +++++++++++++++++++++----------------------- DAS/shape.py | 19 ++++++++++++ DAS/simulator.py | 60 +++++++++++++++++++----------------- DAS/validator.py | 42 +++++++++++++------------- README.md | 8 ++--- config.das | 27 +++++++++++++++++ study.py | 49 +++++++++++++++++------------- 8 files changed, 168 insertions(+), 110 deletions(-) create mode 100644 DAS/shape.py create mode 100644 config.das diff --git a/DAS/__init__.py b/DAS/__init__.py index abe299a..8dd11bf 100644 --- a/DAS/__init__.py +++ b/DAS/__init__.py @@ -1,2 +1,3 @@ from DAS.simulator import * from DAS.configuration import * +from DAS.shape import * diff --git a/DAS/configuration.py b/DAS/configuration.py index 949723b..91df16f 100644 --- a/DAS/configuration.py +++ b/DAS/configuration.py @@ -1,50 +1,48 @@ #!/bin/python3 +import configparser + class Configuration: deterministic = 0 - blockSize = 0 - numberValidators = 0 - failureRate = 0 - failureRateStart = 0 - failureRateStop = 0 - failureRateStep = 0 - chio = 0 - chiStart = 0 - chiStop = 0 - chiStep = 0 - run =0 - runStart = 0 - runStop = 0 - runStep = 0 - def __init__(self, deterministic, blockSize, numberValidators,\ - failureRateStart, failureRateStop, failureRateStep,\ - chiStart, chiStop, chiStep,\ - runStart, runStop, runStep): + def __init__(self, fileName): - if numberValidators < (blockSize*4): + config = configparser.RawConfigParser() + config.read(fileName) + + self.nvStart = int(config.get("Simulation Space", "numberValidatorStart")) + self.nvStop = int(config.get("Simulation Space", "numberValidatorStop")) + self.nvStep = int(config.get("Simulation Space", "numberValidatorStep")) + + self.blockSizeStart = int(config.get("Simulation Space", "blockSizeStart")) + self.blockSizeStop = int(config.get("Simulation Space", "blockSizeStop")) + self.blockSizeStep = int(config.get("Simulation Space", "blockSizeStep")) + + self.netDegreeStart = int(config.get("Simulation Space", "netDegreeStart")) + self.netDegreeStop = int(config.get("Simulation Space", "netDegreeStop")) + self.netDegreeStep = int(config.get("Simulation Space", "netDegreeStep")) + + self.failureRateStart = int(config.get("Simulation Space", "failureRateStart")) + self.failureRateStop = int(config.get("Simulation Space", "failureRateStop")) + self.failureRateStep = int(config.get("Simulation Space", "failureRateStep")) + + self.chiStart = int(config.get("Simulation Space", "chiStart")) + self.chiStop = int(config.get("Simulation Space", "chiStop")) + self.chiStep = int(config.get("Simulation Space", "chiStep")) + + self.numberRuns = int(config.get("Advanced", "numberRuns")) + self.deterministic = config.get("Advanced", "deterministic") + + if self.nvStop < (self.blockSizeStart*4): print("ERROR: The number of validators cannot be lower than the block size * 4") exit(1) - if chiStart < 1: + if self.chiStart < 1: print("Chi has to be greater than 0") exit(1) - if chiStop > blockSize: - print("Chi has to be smaller than %d" % blockSize) + if self.chiStop > self.blockSizeStart: + print("Chi (%d) has to be smaller or equal to block the size (%d)" % (self.chiStop, self.blockSizeStart)) exit(1) - self.deterministic = deterministic - self.blockSize = blockSize - self.numberValidators = numberValidators - self.failureRateStart = failureRateStart - self.failureRateStop = failureRateStop - self.failureRateStep = failureRateStep - self.failureRate = failureRateStart - self.chiStart = chiStart - self.chiStop = chiStop - self.chiStep = chiStep - self.chi = chiStart - self.runStart = runStart - self.runStop = runStop - self.runStep = runStep - self.run = runStart + + diff --git a/DAS/shape.py b/DAS/shape.py new file mode 100644 index 0000000..2918422 --- /dev/null +++ b/DAS/shape.py @@ -0,0 +1,19 @@ +#!/bin/python3 + +class Shape: + numberValidators = 0 + failureRate = 0 + blockSize = 0 + netDegree = 0 + chi = 0 + + def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree): + self.numberValidators = numberValidators + self.failureRate = failureRate + self.blockSize = blockSize + self.netDegree = netDegree + self.chi = chi + + + + diff --git a/DAS/simulator.py b/DAS/simulator.py index 42b0e1f..3701167 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -1,7 +1,7 @@ #!/bin/python import networkx as nx -import logging +import logging, random from datetime import datetime from DAS.tools import * from DAS.results import * @@ -15,23 +15,25 @@ class Simulator: validators = [] glob = [] result = [] - config = [] + shape = [] logger = [] format = {} - def __init__(self, config): - self.config = config + def __init__(self, shape): + self.shape = shape self.format = {"entity": "Simulator"} - self.result = Result(self.config) + self.result = Result(self.shape) def initValidators(self): - if not self.config.deterministic: - random.seed(datetime.now()) - self.glob = Observer(self.logger, self.config) + self.glob = Observer(self.logger, self.shape) self.glob.reset() self.validators = [] - for i in range(self.config.numberValidators): - val = Validator(i, int(not i!=0), self.logger, self.config) + rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) + columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) + random.shuffle(rows) + random.shuffle(columns) + for i in range(self.shape.numberValidators): + val = Validator(i, int(not i!=0), self.logger, self.shape, rows, columns) if i == self.proposerID: val.initBlock() self.glob.setGoldenData(val.block) @@ -39,17 +41,21 @@ class Simulator: val.logIDs() self.validators.append(val) - def initNetwork(self, d=6): - rowChannels = [[] for i in range(self.config.blockSize)] - columnChannels = [[] for i in range(self.config.blockSize)] + def initNetwork(self): + self.shape.netDegree = 6 + rowChannels = [[] for i in range(self.shape.blockSize)] + columnChannels = [[] for i in range(self.shape.blockSize)] for v in self.validators: for id in v.rowIDs: rowChannels[id].append(v) for id in v.columnIDs: columnChannels[id].append(v) - for id in range(self.config.blockSize): - G = nx.random_regular_graph(d, len(rowChannels[id])) + for id in range(self.shape.blockSize): + + if (len(rowChannels[id]) < self.shape.netDegree): + self.logger.error("Graph degree higher than %d" % len(rowChannels[id]), extra=self.format) + G = nx.random_regular_graph(self.shape.netDegree, len(rowChannels[id])) if not nx.is_connected(G): self.logger.error("Graph not connected for row %d !" % id, extra=self.format) for u, v in G.edges: @@ -57,7 +63,10 @@ class Simulator: val2=rowChannels[id][v] val1.rowNeighbors[id].append(val2) val2.rowNeighbors[id].append(val1) - G = nx.random_regular_graph(d, len(columnChannels[id])) + + if (len(columnChannels[id]) < self.shape.netDegree): + self.logger.error("Graph degree higher than %d" % len(columnChannels[id]), extra=self.format) + G = nx.random_regular_graph(self.shape.netDegree, len(columnChannels[id])) if not nx.is_connected(G): self.logger.error("Graph not connected for column %d !" % id, extra=self.format) for u, v in G.edges: @@ -75,15 +84,12 @@ class Simulator: logger.addHandler(ch) self.logger = logger - def resetFailureRate(self, failureRate): - self.config.failureRate = failureRate - for val in self.validators: - val.config.failureRate = failureRate - def resetChi(self, chi): - self.config.chi = chi + def resetShape(self, shape): + self.shape = shape for val in self.validators: - val.config.chi = chi + val.shape.failureRate = shape.failureRate + val.shape.chi = shape.chi def run(self): @@ -96,9 +102,9 @@ class Simulator: while(missingSamples > 0): missingVector.append(missingSamples) oldMissingSamples = missingSamples - for i in range(1,self.config.numberValidators): + for i in range(1,self.shape.numberValidators): self.validators[i].receiveRowsColumns() - for i in range(1,self.config.numberValidators): + for i in range(1,self.shape.numberValidators): self.validators[i].restoreRows() self.validators[i].restoreColumns() self.validators[i].sendRows() @@ -120,10 +126,10 @@ class Simulator: self.result.addMissing(missingVector) if missingSamples == 0: self.result.blockAvailable = 1 - self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.config.failureRate), extra=self.format) + self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.shape.failureRate), extra=self.format) return self.result else: self.result.blockAvailable = 0 - self.logger.debug("The block cannot be recovered, failure rate %d!" % self.config.failureRate, extra=self.format) + self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format) return self.result diff --git a/DAS/validator.py b/DAS/validator.py index 7a4b2fe..af16df5 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -11,34 +11,34 @@ class Validator: ID = 0 amIproposer = 0 - config = [] + shape = [] format = {} logger = [] - def __init__(self, ID, amIproposer, logger, config): - self.config = config + def __init__(self, ID, amIproposer, logger, shape, rows, columns): + self.shape = shape FORMAT = "%(levelname)s : %(entity)s : %(message)s" self.ID = ID self.format = {"entity": "Val "+str(self.ID)} - self.block = Block(self.config.blockSize) - self.receivedBlock = Block(self.config.blockSize) + self.block = Block(self.shape.blockSize) + self.receivedBlock = Block(self.shape.blockSize) self.amIproposer = amIproposer self.logger = logger - if self.config.chi < 1: + if self.shape.chi < 1: self.logger.error("Chi has to be greater than 0", extra=self.format) - elif self.config.chi > self.config.blockSize: + elif self.shape.chi > self.shape.blockSize: self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) else: if amIproposer: - self.rowIDs = range(config.blockSize) - self.columnIDs = range(config.blockSize) + self.rowIDs = range(shape.blockSize) + self.columnIDs = range(shape.blockSize) else: - self.rowIDs = [] - self.columnIDs = [] - if config.deterministic: - random.seed(self.ID) - self.rowIDs = random.sample(range(self.config.blockSize), self.config.chi) - self.columnIDs = random.sample(range(self.config.blockSize), self.config.chi) + self.rowIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] + self.columnIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] + #if shape.deterministic: + # random.seed(self.ID) + #self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi) + #self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi) self.rowNeighbors = collections.defaultdict(list) self.columnNeighbors = collections.defaultdict(list) @@ -51,7 +51,7 @@ class Validator: def initBlock(self): self.logger.debug("I am a block proposer.", extra=self.format) - self.block = Block(self.config.blockSize) + self.block = Block(self.shape.blockSize) self.block.fill() #self.block.print() @@ -60,21 +60,21 @@ class Validator: self.logger.error("I am NOT a block proposer", extra=self.format) else: self.logger.debug("Broadcasting my block...", extra=self.format) - order = [i for i in range(self.config.blockSize * self.config.blockSize)] + order = [i for i in range(self.shape.blockSize * self.shape.blockSize)] random.shuffle(order) while(order): i = order.pop() - if (random.randint(0,99) >= self.config.failureRate): + if (random.randint(0,99) >= self.shape.failureRate): self.block.data[i] = 1 else: self.block.data[i] = 0 nbFailures = self.block.data.count(0) - measuredFailureRate = nbFailures * 100 / (self.config.blockSize * self.config.blockSize) + measuredFailureRate = nbFailures * 100 / (self.shape.blockSize * self.shape.blockSize) self.logger.debug("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format) #broadcasted.print() - for id in range(self.config.blockSize): + for id in range(self.shape.blockSize): self.sendColumn(id) - for id in range(self.config.blockSize): + for id in range(self.shape.blockSize): self.sendRow(id) def getColumn(self, index): diff --git a/README.md b/README.md index 1b41851..97846ce 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# DAS Research +# DAS Research -This repository hosts all the research on DAS for the collaboration between Codex and the EF. +This repository hosts all the research on DAS for the collaboration between Codex and the EF. ## Prepare the environment @@ -16,11 +16,11 @@ $ cd das-research ``` $ python3 -m venv myenv $ source myenv/bin/activate -$ pip3 install -r DAS/requeriments.txt +$ pip3 install -r DAS/requirements.txt ``` ## Run the simulator ``` -$ python3 study.py +$ python3 study.py config.das ``` diff --git a/config.das b/config.das new file mode 100644 index 0000000..f5d608e --- /dev/null +++ b/config.das @@ -0,0 +1,27 @@ +[Simulation Space] + +numberValidatorStart = 256 +numberValidatorStop = 512 +numberValidatorStep = 128 + +failureRateStart = 10 +failureRateStop = 90 +failureRateStep = 40 + +blockSizeStart = 32 +blockSizeStop = 64 +blockSizeStep = 16 + +netDegreeStart = 6 +netDegreeStop = 8 +netDegreeStep = 1 + +chiStart = 4 +chiStop = 8 +chiStep = 2 + + +[Advanced] + +deterministic = 0 +numberRuns = 2 diff --git a/study.py b/study.py index fcd1444..d74cecc 100644 --- a/study.py +++ b/study.py @@ -1,38 +1,45 @@ #! /bin/python3 -import time +import time, sys from DAS import * def study(): - config = Configuration(0, 64, 256, 0, 100, 20, 8, 16, 4, 0, 10, 1) + if len(sys.argv) < 2: + print("You need to pass a configuration file in parameter") + exit(1) + + config = Configuration(sys.argv[1]) sim = Simulator(config) sim.initLogger() - frRange = [] results = [] - resultRange = [] simCnt = 0 + sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() - for fr in range(config.failureRateStart, config.failureRateStop+1, config.failureRateStep): - sim.resetFailureRate(fr) - for chi in range(config.chiStart, config.chiStop+1, config.chiStep): - sim.resetChi(chi) - blockAvailable = 0 - for run in range(config.runStart, config.runStop, config.runStep): - sim.logger.info("FR: %d %%, Chi: %d %%, Run: %d ..." % (fr, chi, run), extra=sim.format) - sim.initValidators() - sim.initNetwork() - result = sim.run() - blockAvailable += result.blockAvailable - results.append(result) - simCnt += 1 - frRange.append(fr) - resultRange.append((blockAvailable)*100/config.runStop) + + for run in range(config.numberRuns): + for fr in range(config.failureRateStart, config.failureRateStop+1, config.failureRateStep): + for chi in range(config.chiStart, config.chiStop+1, config.chiStep): + for blockSize in range(config.blockSizeStart, config.blockSizeStop+1, config.blockSizeStep): + for nv in range(config.nvStart, config.nvStop+1, config.nvStep): + for netDegree in range(config.netDegreeStart, config.netDegreeStop, config.netDegreeStep): + + if not config.deterministic: + random.seed(datetime.now()) + + shape = Shape(blockSize, nv, fr, chi, netDegree) + sim.resetShape(shape) + sim.initValidators() + sim.initNetwork() + result = sim.run() + sim.logger.info("Run %d, FR: %d %%, Chi: %d, BlockSize: %d, Nb.Val: %d, netDegree: %d ... Block Available: %d" % (run, fr, chi, blockSize, nv, netDegree, result.blockAvailable), extra=sim.format) + results.append(result) + simCnt += 1 + end = time.time() sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format) - #for i in range(len(frRange)): - # sim.logger.info("For failure rate of %d we got %d %% success rate in DAS!" % (frRange[i], resultRange[i]), extra=sim.format) + study()