From 71cef6671cd5575d65b9751b84df234b9fd53ff7 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Fri, 25 Nov 2022 16:12:32 +0100 Subject: [PATCH 1/7] Improved logging system --- simulator/DAS.py | 128 ++++++++++++++++++++++++++++++----------------- 1 file changed, 82 insertions(+), 46 deletions(-) diff --git a/simulator/DAS.py b/simulator/DAS.py index 1a13718..ce3a659 100644 --- a/simulator/DAS.py +++ b/simulator/DAS.py @@ -1,6 +1,6 @@ #! /bin/python3 -import random +import random, logging from datetime import datetime class Block: @@ -17,16 +17,43 @@ class Block: self.data[i] = random.randint(1, 9) def print(self): + dash = "-" * (self.blockSize+2) + print(dash) for i in range(self.blockSize): + line = "|" for j in range(self.blockSize): - print("%i" % self.data[(i*self.blockSize)+j], end="") - print("") + line += "%i" % self.data[(i*self.blockSize)+j] + print(line+"|") + print(dash) +class CustomFormatter(logging.Formatter): + + blue = "\x1b[34;20m" + grey = "\x1b[38;20m" + yellow = "\x1b[33;20m" + red = "\x1b[31;20m" + bold_red = "\x1b[31;1m" + reset = "\x1b[0m" + format = "%(levelname)s : %(entity)s : %(message)s" + + FORMATS = { + logging.DEBUG: grey + format + reset, + logging.INFO: blue + format + reset, + logging.WARNING: yellow + format + reset, + logging.ERROR: red + format + reset, + logging.CRITICAL: bold_red + format + reset + } + + def format(self, record): + log_fmt = self.FORMATS.get(record.levelno) + formatter = logging.Formatter(log_fmt) + return formatter.format(record) class Validator: ID = 0 chi = 0 + format = {} blocksize = 0 block = [] rowIDs = [] @@ -35,16 +62,20 @@ class Validator: columns = [] proposer = 0 failureRate = 0 + logger = [] - def __init__(self, ID, chi, blockSize, proposer, failureRate, deterministic): + def __init__(self, ID, chi, blockSize, proposer, failureRate, deterministic, logger): + FORMAT = "%(levelname)s : %(entity)s : %(message)s" self.ID = ID + self.format = {"entity": "Val "+str(self.ID)} self.blockSize = blockSize self.proposer = proposer self.failureRate = failureRate + self.logger = logger if chi < 1: - print("ERROR: chi has to be greater than 0") + self.logger.error("Chi has to be greater than 0", extra=self.format) elif chi > blockSize: - print("ERROR: chi has to be smaller than %d" % blockSize) + self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) else: self.chi = chi self.rowIDs = [] @@ -55,31 +86,24 @@ class Validator: self.rowIDs.append(random.randint(0,blockSize-1)) self.columnIDs.append(random.randint(0,blockSize-1)) - def printIDs(self): + def logIDs(self): if self.proposer == 1: - print("Hi! I am validator %d and I am a block proposer."% self.ID) + self.logger.warning("I am a block proposer."% self.ID) else: - print("Hi! I am validator %d and these are my rows and columns."% self.ID) - print("Selected rows: ", end="") - for i in range(self.chi): - print("%d " % self.rowIDs[i], end="") - print("") - print("Selected columns: ", end="") - for i in range(self.chi): - print("%d " % self.columnIDs[i], end="") - print("") + self.logger.info("Selected rows: "+str(self.rowIDs), extra=self.format) + self.logger.info("Selected columns: "+str(self.columnIDs), extra=self.format) def initBlock(self): - print("Hi! I am validator %d and I am a block proposer."% self.ID) + self.logger.info("I am a block proposer.", extra=self.format) self.block = Block(self.blockSize) self.block.fill() self.block.print() def broadcastBlock(self, broadcasted): if self.proposer == 0: - print("ERROR: I am validator %d and I am NOT a block proposer" % self.ID) + self.logger.error("I am NOT a block proposer", extra=self.format) else: - print("I am validator %d and I am broadcasting my block..." % self.ID) + self.logger.info("Broadcasting my block...", extra=self.format) tempBlock = self.block order = [i for i in range(self.blockSize * self.blockSize)] random.shuffle(order) @@ -105,21 +129,19 @@ class Validator: self.rows = [] self.columns = [] if self.proposer == 1: - print("ERROR: I am validator %d and I am a block proposer" % self.ID) + self.logger.error("I am a block proposer", extra=self.format) else: - print("I am validator %d and I am receiving the data..." % self.ID) + self.logger.debug("Receiving the data...", extra=self.format) for r in self.rowIDs: self.getRow(r, broadcasted) for c in self.columnIDs: self.getColumn(c, broadcasted) - def printRows(self): - print("Val %d - Rows: " % self.ID, end="") - print(self.rows) + def logRows(self): + self.logger.info("Rows: "+str(self.rows), extra=self.format) - def printColumns(self): - print("Val %d - Columns: " % self.ID, end="") - print(self.columns) + def logColumns(self): + self.logger.info("Columns: "+str(self.columns), extra=self.format) def checkRestoreRows(self, goldenData): for rid in range(len(self.rows)): @@ -132,15 +154,15 @@ class Validator: elif i > 0 and i < 10: success += 1 else: - print("ERROR: Data has been corrupted") + self.logger.error("Data has been corrupted") if failures > 0: if success >= len(row)/2: for i in range(len(row)): self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i] - print("Val %d: Row %d data restored" % (self.ID, self.rowIDs[rid])) + self.logger.info("Row %d data restored" % (self.rowIDs[rid]), extra=self.format ) else: - print("WARNING Val %d: Row %d cannot be restored" % (self.ID, self.rowIDs[rid])) + self.logger.warning("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) def checkRestoreColumns(self, goldenData): for cid in range(len(self.columns)): @@ -153,15 +175,15 @@ class Validator: elif i > 0 and i < 10: success += 1 else: - print("ERROR: Data has been corrupted") + self.logger.error("Data has been corrupted", extra=self.format) if failures > 0: if success >= len(column)/2: for i in range(len(column)): self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]] - print("Val %d: Column %d data restored" % (self.ID, self.columnIDs[cid])) + self.logger.info("Column %d data restored" % (self.columnIDs[cid]), extra=self.format) else: - print("Val %d: Column %d cannot be restored" % (self.ID, self.columnIDs[cid])) + self.logger.info("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format) class Observer: @@ -171,12 +193,15 @@ class Observer: rows = [] columns = [] goldenData = [] + logger = [] - def __init__(self, blockSize): + def __init__(self, blockSize, logger): + self.format = {"entity": "Observer"} self.blockSize = blockSize self.block = [0] * self.blockSize * self.blockSize self.rows = [0] * self.blockSize self.columns = [0] * self.blockSize + self.logger = logger def checkRowsColumns(self, validators): for val in validators: @@ -187,9 +212,9 @@ class Observer: self.columns[c] += 1 for i in range(self.blockSize): - print("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i])) + self.logger.info("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format) if self.rows[i] == 0 or self.columns[i] == 0: - print("WARNING: There is a row/column that has not been assigned") + logging.warning("There is a row/column that has not been assigned", extra=self.format) def setGoldenData(self, block): self.goldenData = [0] * self.blockSize * self.blockSize @@ -201,37 +226,48 @@ class Simulator: chi = 4 blockSize = 16 numberValidators = 32 - failureRate = 10 + failureRate = 40 proposerID = 0 - deterministic = 1 + logLevel = logging.DEBUG + deterministic = 0 validators = [] glob = [] + logger = [] def __init__(self): + logger = logging.getLogger("DAS") + logger.setLevel(self.logLevel) + ch = logging.StreamHandler() + ch.setLevel(self.logLevel) + ch.setFormatter(CustomFormatter()) + logger.addHandler(ch) + self.logger = logger + if not self.deterministic: random.seed(datetime.now()) - self.glob = Observer(self.blockSize) + self.glob = Observer(self.blockSize, self.logger) for i in range(self.numberValidators): - val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic) + val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic, self.logger) if i == self.proposerID: val.initBlock() self.glob.setGoldenData(val.block) else: - val.printIDs() + val.logIDs() self.validators.append(val) + def run(self): broadcasted = Block(self.blockSize) self.glob.checkRowsColumns(self.validators) self.validators[self.proposerID].broadcastBlock(broadcasted) for i in range(1,self.numberValidators): self.validators[i].receiveRowsColumns(broadcasted) - self.validators[i].printRows() - self.validators[i].printColumns() + self.validators[i].logRows() + self.validators[i].logColumns() self.validators[i].checkRestoreRows(self.glob.goldenData) self.validators[i].checkRestoreColumns(self.glob.goldenData) - self.validators[i].printRows() - self.validators[i].printColumns() + self.validators[i].logRows() + self.validators[i].logColumns() sim = Simulator() From 4707084a010966c9aca2cb4b987f8965d080e3b5 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Fri, 25 Nov 2022 16:17:42 +0100 Subject: [PATCH 2/7] Adding gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b948985 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.swp +*.pyc From 2df3ef45f7317ddeafda77c783dc6d3587c5abbb Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Fri, 25 Nov 2022 18:50:55 +0100 Subject: [PATCH 3/7] Add iterative round based simulation and stop conditions --- simulator/DAS.py | 108 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 88 insertions(+), 20 deletions(-) diff --git a/simulator/DAS.py b/simulator/DAS.py index ce3a659..762711b 100644 --- a/simulator/DAS.py +++ b/simulator/DAS.py @@ -82,9 +82,13 @@ class Validator: self.columnIDs = [] if deterministic: random.seed(self.ID) - for i in range(self.chi): - self.rowIDs.append(random.randint(0,blockSize-1)) - self.columnIDs.append(random.randint(0,blockSize-1)) + lr = [i for i in range(self.blockSize)] + lc = [i for i in range(self.blockSize)] + random.shuffle(lr) + random.shuffle(lc) + for i in range(self.chi): # TODO : Avoid doubles + self.rowIDs.append(lr.pop()) + self.columnIDs.append(lc.pop()) def logIDs(self): if self.proposer == 1: @@ -137,11 +141,38 @@ class Validator: for c in self.columnIDs: self.getColumn(c, broadcasted) + def sendColumn(self, c, columnID, broadcasted): + column = [0] * self.blockSize + for i in range(self.blockSize): + if broadcasted.data[(i*self.blockSize)+columnID] == 0: + broadcasted.data[(i*self.blockSize)+columnID] = self.columns[c][i] + + def sendRow(self, r, rowID, broadcasted): + for i in range(self.blockSize): + if broadcasted.data[(rowID*self.blockSize)+i] == 0: + broadcasted.data[(rowID*self.blockSize)+i] = self.rows[r][i] + + def sendRows(self, broadcasted): + if self.proposer == 1: + self.logger.error("I am a block proposer", extra=self.format) + else: + self.logger.debug("Sending restored rows...", extra=self.format) + for r in range(len(self.rowIDs)): + self.sendRow(r, self.rowIDs[r], broadcasted) + + def sendColumns(self, broadcasted): + if self.proposer == 1: + self.logger.error("I am a block proposer", extra=self.format) + else: + self.logger.debug("Sending restored columns...", extra=self.format) + for c in range(len(self.columnIDs)): + self.sendColumn(c, self.columnIDs[c], broadcasted) + def logRows(self): - self.logger.info("Rows: "+str(self.rows), extra=self.format) + self.logger.debug("Rows: "+str(self.rows), extra=self.format) def logColumns(self): - self.logger.info("Columns: "+str(self.columns), extra=self.format) + self.logger.debug("Columns: "+str(self.columns), extra=self.format) def checkRestoreRows(self, goldenData): for rid in range(len(self.rows)): @@ -160,7 +191,7 @@ class Validator: if success >= len(row)/2: for i in range(len(row)): self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i] - self.logger.info("Row %d data restored" % (self.rowIDs[rid]), extra=self.format ) + self.logger.info("%d samples restored in row %d" % (failures, self.rowIDs[rid]), extra=self.format ) else: self.logger.warning("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) @@ -181,7 +212,7 @@ class Validator: if success >= len(column)/2: for i in range(len(column)): self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]] - self.logger.info("Column %d data restored" % (self.columnIDs[cid]), extra=self.format) + self.logger.info("%d samples restored in column %d" % (failures, self.columnIDs[cid]), extra=self.format) else: self.logger.info("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format) @@ -193,15 +224,18 @@ class Observer: rows = [] columns = [] goldenData = [] + broadcasted = [] logger = [] def __init__(self, blockSize, logger): self.format = {"entity": "Observer"} self.blockSize = blockSize self.block = [0] * self.blockSize * self.blockSize + self.goldenData = [0] * self.blockSize * self.blockSize self.rows = [0] * self.blockSize self.columns = [0] * self.blockSize self.logger = logger + self.broadcasted = Block(self.blockSize) def checkRowsColumns(self, validators): for val in validators: @@ -217,31 +251,44 @@ class Observer: logging.warning("There is a row/column that has not been assigned", extra=self.format) def setGoldenData(self, block): - self.goldenData = [0] * self.blockSize * self.blockSize for i in range(self.blockSize*self.blockSize): self.goldenData[i] = block.data[i] + def checkBroadcasted(self): + zeros = 0 + for i in range(self.blockSize * self.blockSize): + if self.broadcasted.data[i] == 0: + zeros += 1 + if zeros > 0: + self.logger.warning("There are %d missing samples in the network" % zeros, extra=self.format) + return zeros + + class Simulator: chi = 4 blockSize = 16 numberValidators = 32 - failureRate = 40 + failureRate = 66 proposerID = 0 - logLevel = logging.DEBUG + logLevel = logging.INFO deterministic = 0 validators = [] glob = [] logger = [] + format = {} + steps = 0 def __init__(self): logger = logging.getLogger("DAS") logger.setLevel(self.logLevel) ch = logging.StreamHandler() ch.setLevel(self.logLevel) + self.format = {"entity": "Simulator"} ch.setFormatter(CustomFormatter()) logger.addHandler(ch) self.logger = logger + self.steps = 0 if not self.deterministic: random.seed(datetime.now()) @@ -257,17 +304,38 @@ class Simulator: def run(self): - broadcasted = Block(self.blockSize) self.glob.checkRowsColumns(self.validators) - self.validators[self.proposerID].broadcastBlock(broadcasted) - for i in range(1,self.numberValidators): - self.validators[i].receiveRowsColumns(broadcasted) - self.validators[i].logRows() - self.validators[i].logColumns() - self.validators[i].checkRestoreRows(self.glob.goldenData) - self.validators[i].checkRestoreColumns(self.glob.goldenData) - self.validators[i].logRows() - self.validators[i].logColumns() + self.validators[self.proposerID].broadcastBlock(self.glob.broadcasted) + missingSamples = self.glob.checkBroadcasted() + while(missingSamples > 0): + oldMissingSamples = missingSamples + self.logger.info("Step %d:" % self.steps, extra=self.format) + for i in range(1,self.numberValidators): + self.validators[i].receiveRowsColumns(self.glob.broadcasted) + #Rows + self.validators[i].checkRestoreRows(self.glob.goldenData) + self.validators[i].sendRows(self.glob.broadcasted) + self.validators[i].logRows() + self.validators[i].logColumns() + # Columns + self.validators[i].checkRestoreColumns(self.glob.goldenData) + self.validators[i].sendColumns(self.glob.broadcasted) + self.validators[i].logRows() + self.validators[i].logColumns() + + missingSamples = self.glob.checkBroadcasted() + if missingSamples == oldMissingSamples: + break + elif missingSamples == 0: + break + else: + self.steps += 1 + + if missingSamples == 0: + self.logger.info("The entire block is available at step %d!" % self.steps, extra=self.format) + else: + self.logger.warning("The block is CANNOT be recovered!", extra=self.format) + self.logger.info("End of simulation", extra=self.format) sim = Simulator() From 8221490196b3a844198aa5ee9c273a3863c1984a Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Tue, 29 Nov 2022 18:12:02 +0100 Subject: [PATCH 4/7] Add logic for multi-simulation studies --- simulator/DAS.py | 105 +++++++++++++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 35 deletions(-) diff --git a/simulator/DAS.py b/simulator/DAS.py index 762711b..856a10f 100644 --- a/simulator/DAS.py +++ b/simulator/DAS.py @@ -1,6 +1,6 @@ #! /bin/python3 -import random, logging +import random, logging, time from datetime import datetime class Block: @@ -94,20 +94,20 @@ class Validator: if self.proposer == 1: self.logger.warning("I am a block proposer."% self.ID) else: - self.logger.info("Selected rows: "+str(self.rowIDs), extra=self.format) - self.logger.info("Selected columns: "+str(self.columnIDs), extra=self.format) + self.logger.debug("Selected rows: "+str(self.rowIDs), extra=self.format) + self.logger.debug("Selected columns: "+str(self.columnIDs), extra=self.format) def initBlock(self): - self.logger.info("I am a block proposer.", extra=self.format) + self.logger.debug("I am a block proposer.", extra=self.format) self.block = Block(self.blockSize) self.block.fill() - self.block.print() + #self.block.print() def broadcastBlock(self, broadcasted): if self.proposer == 0: self.logger.error("I am NOT a block proposer", extra=self.format) else: - self.logger.info("Broadcasting my block...", extra=self.format) + self.logger.debug("Broadcasting my block...", extra=self.format) tempBlock = self.block order = [i for i in range(self.blockSize * self.blockSize)] random.shuffle(order) @@ -115,7 +115,7 @@ class Validator: i = order.pop() if (random.randint(0,99) > self.failureRate): broadcasted.data[i] = self.block.data[i] - broadcasted.print() + #broadcasted.print() def getColumn(self, columnID, broadcasted): column = [0] * self.blockSize @@ -191,9 +191,9 @@ class Validator: if success >= len(row)/2: for i in range(len(row)): self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i] - self.logger.info("%d samples restored in row %d" % (failures, self.rowIDs[rid]), extra=self.format ) + self.logger.debug("%d samples restored in row %d" % (failures, self.rowIDs[rid]), extra=self.format ) else: - self.logger.warning("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) + self.logger.debug("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) def checkRestoreColumns(self, goldenData): for cid in range(len(self.columns)): @@ -212,9 +212,9 @@ class Validator: if success >= len(column)/2: for i in range(len(column)): self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]] - self.logger.info("%d samples restored in column %d" % (failures, self.columnIDs[cid]), extra=self.format) + self.logger.debug("%d samples restored in column %d" % (failures, self.columnIDs[cid]), extra=self.format) else: - self.logger.info("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format) + self.logger.debug("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format) class Observer: @@ -230,11 +230,13 @@ class Observer: def __init__(self, blockSize, logger): self.format = {"entity": "Observer"} self.blockSize = blockSize + self.logger = logger + + def reset(self): self.block = [0] * self.blockSize * self.blockSize self.goldenData = [0] * self.blockSize * self.blockSize self.rows = [0] * self.blockSize self.columns = [0] * self.blockSize - self.logger = logger self.broadcasted = Block(self.blockSize) def checkRowsColumns(self, validators): @@ -246,9 +248,9 @@ class Observer: self.columns[c] += 1 for i in range(self.blockSize): - self.logger.info("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format) + self.logger.debug("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format) if self.rows[i] == 0 or self.columns[i] == 0: - logging.warning("There is a row/column that has not been assigned", extra=self.format) + self.logger.warning("There is a row/column that has not been assigned", extra=self.format) def setGoldenData(self, block): for i in range(self.blockSize*self.blockSize): @@ -260,16 +262,16 @@ class Observer: if self.broadcasted.data[i] == 0: zeros += 1 if zeros > 0: - self.logger.warning("There are %d missing samples in the network" % zeros, extra=self.format) + self.logger.debug("There are %d missing samples in the network" % zeros, extra=self.format) return zeros class Simulator: - chi = 4 - blockSize = 16 - numberValidators = 32 - failureRate = 66 + chi = 8 + blockSize = 256 + numberValidators = 8192 + failureRate = 0 proposerID = 0 logLevel = logging.INFO deterministic = 0 @@ -279,20 +281,17 @@ class Simulator: format = {} steps = 0 - def __init__(self): - logger = logging.getLogger("DAS") - logger.setLevel(self.logLevel) - ch = logging.StreamHandler() - ch.setLevel(self.logLevel) + def __init__(self, failureRate): + self.failureRate = failureRate self.format = {"entity": "Simulator"} - ch.setFormatter(CustomFormatter()) - logger.addHandler(ch) - self.logger = logger self.steps = 0 + def initValidators(self): if not self.deterministic: random.seed(datetime.now()) self.glob = Observer(self.blockSize, self.logger) + self.glob.reset() + self.validators = [] for i in range(self.numberValidators): val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic, self.logger) if i == self.proposerID: @@ -302,14 +301,26 @@ class Simulator: val.logIDs() self.validators.append(val) + def initLogger(self): + logger = logging.getLogger("DAS") + logger.setLevel(self.logLevel) + ch = logging.StreamHandler() + ch.setLevel(self.logLevel) + ch.setFormatter(CustomFormatter()) + logger.addHandler(ch) + self.logger = logger + + def resetFailureRate(self, failureRate): + self.failureRate = failureRate def run(self): self.glob.checkRowsColumns(self.validators) self.validators[self.proposerID].broadcastBlock(self.glob.broadcasted) missingSamples = self.glob.checkBroadcasted() + self.steps = 0 while(missingSamples > 0): oldMissingSamples = missingSamples - self.logger.info("Step %d:" % self.steps, extra=self.format) + self.logger.debug("Step %d:" % self.steps, extra=self.format) for i in range(1,self.numberValidators): self.validators[i].receiveRowsColumns(self.glob.broadcasted) #Rows @@ -332,14 +343,38 @@ class Simulator: self.steps += 1 if missingSamples == 0: - self.logger.info("The entire block is available at step %d!" % self.steps, extra=self.format) + self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (self.steps, self.failureRate), extra=self.format) + return 0 else: - self.logger.warning("The block is CANNOT be recovered!", extra=self.format) - self.logger.info("End of simulation", extra=self.format) - - -sim = Simulator() -sim.run() + self.logger.debug("The block cannot be recovered, failure rate %d!" % self.failureRate, extra=self.format) + return 1 + +def study(): + sim = Simulator(0) + sim.initLogger() + maxTries = 5 + step = 25 + frRange = [] + resultRange = [] + simCnt = 0 + sim.logger.info("Starting %d simulations:" % (maxTries*100), extra=sim.format) + start = time.time() + for fr in range(0, 100, step): + if fr % 10 == 0: + sim.logger.info("Failure rate %d %% ..." % fr, extra=sim.format) + sim.resetFailureRate(fr) + result = 0 + for i in range(maxTries): + sim.initValidators() + result += sim.run() + simCnt += 1 + frRange.append(fr) + resultRange.append(100-result) + end = time.time() + sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format) + for i in range(len(frRange)): + sim.logger.info("For failure rate of %d we got %d %% success rate in DAS!" % (frRange[i], resultRange[i]), extra=sim.format) +study() From 6cbe751b8413945ee1bdbc87c8296da2087f88ea Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Tue, 29 Nov 2022 19:32:05 +0100 Subject: [PATCH 5/7] Split code in multiple modules --- simulator/DAS.py | 353 +---------------------------------------- simulator/block.py | 27 ++++ simulator/observer.py | 52 ++++++ simulator/simulator.py | 91 +++++++++++ simulator/tools.py | 27 ++++ simulator/validator.py | 173 ++++++++++++++++++++ 6 files changed, 374 insertions(+), 349 deletions(-) create mode 100644 simulator/block.py create mode 100644 simulator/observer.py create mode 100644 simulator/simulator.py create mode 100644 simulator/tools.py create mode 100644 simulator/validator.py diff --git a/simulator/DAS.py b/simulator/DAS.py index 856a10f..fe4f4e3 100644 --- a/simulator/DAS.py +++ b/simulator/DAS.py @@ -1,363 +1,18 @@ #! /bin/python3 -import random, logging, time -from datetime import datetime +import time +from simulator import * -class Block: - - blockSize = 0 - data = [] - - def __init__(self, size): - self.blockSize = size - self.data = [0] * (self.blockSize*self.blockSize) - - def fill(self): - for i in range(self.blockSize*self.blockSize): - self.data[i] = random.randint(1, 9) - - def print(self): - dash = "-" * (self.blockSize+2) - print(dash) - for i in range(self.blockSize): - line = "|" - for j in range(self.blockSize): - line += "%i" % self.data[(i*self.blockSize)+j] - print(line+"|") - print(dash) - -class CustomFormatter(logging.Formatter): - - blue = "\x1b[34;20m" - grey = "\x1b[38;20m" - yellow = "\x1b[33;20m" - red = "\x1b[31;20m" - bold_red = "\x1b[31;1m" - reset = "\x1b[0m" - format = "%(levelname)s : %(entity)s : %(message)s" - - FORMATS = { - logging.DEBUG: grey + format + reset, - logging.INFO: blue + format + reset, - logging.WARNING: yellow + format + reset, - logging.ERROR: red + format + reset, - logging.CRITICAL: bold_red + format + reset - } - - def format(self, record): - log_fmt = self.FORMATS.get(record.levelno) - formatter = logging.Formatter(log_fmt) - return formatter.format(record) - -class Validator: - - ID = 0 - chi = 0 - format = {} - blocksize = 0 - block = [] - rowIDs = [] - columnIDs = [] - rows = [] - columns = [] - proposer = 0 - failureRate = 0 - logger = [] - - def __init__(self, ID, chi, blockSize, proposer, failureRate, deterministic, logger): - FORMAT = "%(levelname)s : %(entity)s : %(message)s" - self.ID = ID - self.format = {"entity": "Val "+str(self.ID)} - self.blockSize = blockSize - self.proposer = proposer - self.failureRate = failureRate - self.logger = logger - if chi < 1: - self.logger.error("Chi has to be greater than 0", extra=self.format) - elif chi > blockSize: - self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) - else: - self.chi = chi - self.rowIDs = [] - self.columnIDs = [] - if deterministic: - random.seed(self.ID) - lr = [i for i in range(self.blockSize)] - lc = [i for i in range(self.blockSize)] - random.shuffle(lr) - random.shuffle(lc) - for i in range(self.chi): # TODO : Avoid doubles - self.rowIDs.append(lr.pop()) - self.columnIDs.append(lc.pop()) - - def logIDs(self): - if self.proposer == 1: - self.logger.warning("I am a block proposer."% self.ID) - else: - self.logger.debug("Selected rows: "+str(self.rowIDs), extra=self.format) - self.logger.debug("Selected columns: "+str(self.columnIDs), extra=self.format) - - def initBlock(self): - self.logger.debug("I am a block proposer.", extra=self.format) - self.block = Block(self.blockSize) - self.block.fill() - #self.block.print() - - def broadcastBlock(self, broadcasted): - if self.proposer == 0: - self.logger.error("I am NOT a block proposer", extra=self.format) - else: - self.logger.debug("Broadcasting my block...", extra=self.format) - tempBlock = self.block - order = [i for i in range(self.blockSize * self.blockSize)] - random.shuffle(order) - while(order): - i = order.pop() - if (random.randint(0,99) > self.failureRate): - broadcasted.data[i] = self.block.data[i] - #broadcasted.print() - - def getColumn(self, columnID, broadcasted): - column = [0] * self.blockSize - for i in range(self.blockSize): - column[i] = broadcasted.data[(i*self.blockSize)+columnID] - self.columns.append(column) - - def getRow(self, rowID, broadcasted): - row = [0] * self.blockSize - for i in range(self.blockSize): - row[i] = broadcasted.data[(rowID*self.blockSize)+i] - self.rows.append(row) - - def receiveRowsColumns(self, broadcasted): - self.rows = [] - self.columns = [] - if self.proposer == 1: - self.logger.error("I am a block proposer", extra=self.format) - else: - self.logger.debug("Receiving the data...", extra=self.format) - for r in self.rowIDs: - self.getRow(r, broadcasted) - for c in self.columnIDs: - self.getColumn(c, broadcasted) - - def sendColumn(self, c, columnID, broadcasted): - column = [0] * self.blockSize - for i in range(self.blockSize): - if broadcasted.data[(i*self.blockSize)+columnID] == 0: - broadcasted.data[(i*self.blockSize)+columnID] = self.columns[c][i] - - def sendRow(self, r, rowID, broadcasted): - for i in range(self.blockSize): - if broadcasted.data[(rowID*self.blockSize)+i] == 0: - broadcasted.data[(rowID*self.blockSize)+i] = self.rows[r][i] - - def sendRows(self, broadcasted): - if self.proposer == 1: - self.logger.error("I am a block proposer", extra=self.format) - else: - self.logger.debug("Sending restored rows...", extra=self.format) - for r in range(len(self.rowIDs)): - self.sendRow(r, self.rowIDs[r], broadcasted) - - def sendColumns(self, broadcasted): - if self.proposer == 1: - self.logger.error("I am a block proposer", extra=self.format) - else: - self.logger.debug("Sending restored columns...", extra=self.format) - for c in range(len(self.columnIDs)): - self.sendColumn(c, self.columnIDs[c], broadcasted) - - def logRows(self): - self.logger.debug("Rows: "+str(self.rows), extra=self.format) - - def logColumns(self): - self.logger.debug("Columns: "+str(self.columns), extra=self.format) - - def checkRestoreRows(self, goldenData): - for rid in range(len(self.rows)): - row = self.rows[rid] - failures = 0 - success = 0 - for i in row: - if i == 0: - failures += 1 - elif i > 0 and i < 10: - success += 1 - else: - self.logger.error("Data has been corrupted") - - if failures > 0: - if success >= len(row)/2: - for i in range(len(row)): - self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i] - self.logger.debug("%d samples restored in row %d" % (failures, self.rowIDs[rid]), extra=self.format ) - else: - self.logger.debug("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) - - def checkRestoreColumns(self, goldenData): - for cid in range(len(self.columns)): - column = self.columns[cid] - failures = 0 - success = 0 - for i in column: - if i == 0: - failures += 1 - elif i > 0 and i < 10: - success += 1 - else: - self.logger.error("Data has been corrupted", extra=self.format) - - if failures > 0: - if success >= len(column)/2: - for i in range(len(column)): - self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]] - self.logger.debug("%d samples restored in column %d" % (failures, self.columnIDs[cid]), extra=self.format) - else: - self.logger.debug("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format) - - -class Observer: - - block = [] - blockSize = 0 - rows = [] - columns = [] - goldenData = [] - broadcasted = [] - logger = [] - - def __init__(self, blockSize, logger): - self.format = {"entity": "Observer"} - self.blockSize = blockSize - self.logger = logger - - def reset(self): - self.block = [0] * self.blockSize * self.blockSize - self.goldenData = [0] * self.blockSize * self.blockSize - self.rows = [0] * self.blockSize - self.columns = [0] * self.blockSize - self.broadcasted = Block(self.blockSize) - - def checkRowsColumns(self, validators): - for val in validators: - if val.proposer == 0: - for r in val.rowIDs: - self.rows[r] += 1 - for c in val.columnIDs: - self.columns[c] += 1 - - for i in range(self.blockSize): - self.logger.debug("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format) - if self.rows[i] == 0 or self.columns[i] == 0: - self.logger.warning("There is a row/column that has not been assigned", extra=self.format) - - def setGoldenData(self, block): - for i in range(self.blockSize*self.blockSize): - self.goldenData[i] = block.data[i] - - def checkBroadcasted(self): - zeros = 0 - for i in range(self.blockSize * self.blockSize): - if self.broadcasted.data[i] == 0: - zeros += 1 - if zeros > 0: - self.logger.debug("There are %d missing samples in the network" % zeros, extra=self.format) - return zeros - - -class Simulator: - - chi = 8 - blockSize = 256 - numberValidators = 8192 - failureRate = 0 - proposerID = 0 - logLevel = logging.INFO - deterministic = 0 - validators = [] - glob = [] - logger = [] - format = {} - steps = 0 - - def __init__(self, failureRate): - self.failureRate = failureRate - self.format = {"entity": "Simulator"} - self.steps = 0 - - def initValidators(self): - if not self.deterministic: - random.seed(datetime.now()) - self.glob = Observer(self.blockSize, self.logger) - self.glob.reset() - self.validators = [] - for i in range(self.numberValidators): - val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic, self.logger) - if i == self.proposerID: - val.initBlock() - self.glob.setGoldenData(val.block) - else: - val.logIDs() - self.validators.append(val) - - def initLogger(self): - logger = logging.getLogger("DAS") - logger.setLevel(self.logLevel) - ch = logging.StreamHandler() - ch.setLevel(self.logLevel) - ch.setFormatter(CustomFormatter()) - logger.addHandler(ch) - self.logger = logger - - def resetFailureRate(self, failureRate): - self.failureRate = failureRate - - def run(self): - self.glob.checkRowsColumns(self.validators) - self.validators[self.proposerID].broadcastBlock(self.glob.broadcasted) - missingSamples = self.glob.checkBroadcasted() - self.steps = 0 - while(missingSamples > 0): - oldMissingSamples = missingSamples - self.logger.debug("Step %d:" % self.steps, extra=self.format) - for i in range(1,self.numberValidators): - self.validators[i].receiveRowsColumns(self.glob.broadcasted) - #Rows - self.validators[i].checkRestoreRows(self.glob.goldenData) - self.validators[i].sendRows(self.glob.broadcasted) - self.validators[i].logRows() - self.validators[i].logColumns() - # Columns - self.validators[i].checkRestoreColumns(self.glob.goldenData) - self.validators[i].sendColumns(self.glob.broadcasted) - self.validators[i].logRows() - self.validators[i].logColumns() - - missingSamples = self.glob.checkBroadcasted() - if missingSamples == oldMissingSamples: - break - elif missingSamples == 0: - break - else: - self.steps += 1 - - if missingSamples == 0: - self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (self.steps, self.failureRate), extra=self.format) - return 0 - else: - self.logger.debug("The block cannot be recovered, failure rate %d!" % self.failureRate, extra=self.format) - return 1 def study(): sim = Simulator(0) sim.initLogger() - maxTries = 5 + maxTries = 2 step = 25 frRange = [] resultRange = [] simCnt = 0 - sim.logger.info("Starting %d simulations:" % (maxTries*100), extra=sim.format) + sim.logger.info("Starting simulations:", extra=sim.format) start = time.time() for fr in range(0, 100, step): if fr % 10 == 0: diff --git a/simulator/block.py b/simulator/block.py new file mode 100644 index 0000000..5539865 --- /dev/null +++ b/simulator/block.py @@ -0,0 +1,27 @@ +#!/bin/python3 + +import random + +class Block: + + blockSize = 0 + data = [] + + def __init__(self, size): + self.blockSize = size + self.data = [0] * (self.blockSize*self.blockSize) + + def fill(self): + for i in range(self.blockSize*self.blockSize): + self.data[i] = random.randint(1, 9) + + def print(self): + dash = "-" * (self.blockSize+2) + print(dash) + for i in range(self.blockSize): + line = "|" + for j in range(self.blockSize): + line += "%i" % self.data[(i*self.blockSize)+j] + print(line+"|") + print(dash) + diff --git a/simulator/observer.py b/simulator/observer.py new file mode 100644 index 0000000..3a13d47 --- /dev/null +++ b/simulator/observer.py @@ -0,0 +1,52 @@ +#!/bin/python3 + +from block import * + +class Observer: + + block = [] + blockSize = 0 + rows = [] + columns = [] + goldenData = [] + broadcasted = [] + logger = [] + + def __init__(self, blockSize, logger): + self.format = {"entity": "Observer"} + self.blockSize = blockSize + self.logger = logger + + def reset(self): + self.block = [0] * self.blockSize * self.blockSize + self.goldenData = [0] * self.blockSize * self.blockSize + self.rows = [0] * self.blockSize + self.columns = [0] * self.blockSize + self.broadcasted = Block(self.blockSize) + + def checkRowsColumns(self, validators): + for val in validators: + if val.proposer == 0: + for r in val.rowIDs: + self.rows[r] += 1 + for c in val.columnIDs: + self.columns[c] += 1 + + for i in range(self.blockSize): + self.logger.debug("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format) + if self.rows[i] == 0 or self.columns[i] == 0: + self.logger.warning("There is a row/column that has not been assigned", extra=self.format) + + def setGoldenData(self, block): + for i in range(self.blockSize*self.blockSize): + self.goldenData[i] = block.data[i] + + def checkBroadcasted(self): + zeros = 0 + for i in range(self.blockSize * self.blockSize): + if self.broadcasted.data[i] == 0: + zeros += 1 + if zeros > 0: + self.logger.debug("There are %d missing samples in the network" % zeros, extra=self.format) + return zeros + diff --git a/simulator/simulator.py b/simulator/simulator.py new file mode 100644 index 0000000..7781080 --- /dev/null +++ b/simulator/simulator.py @@ -0,0 +1,91 @@ +#!/bin/python + +import logging +from datetime import datetime +from tools import * +from observer import * +from validator import * + +class Simulator: + + chi = 8 + blockSize = 256 + numberValidators = 8192 + failureRate = 0 + proposerID = 0 + logLevel = logging.INFO + deterministic = 0 + validators = [] + glob = [] + logger = [] + format = {} + steps = 0 + + def __init__(self, failureRate): + self.failureRate = failureRate + self.format = {"entity": "Simulator"} + self.steps = 0 + + def initValidators(self): + if not self.deterministic: + random.seed(datetime.now()) + self.glob = Observer(self.blockSize, self.logger) + self.glob.reset() + self.validators = [] + for i in range(self.numberValidators): + val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic, self.logger) + if i == self.proposerID: + val.initBlock() + self.glob.setGoldenData(val.block) + else: + val.logIDs() + self.validators.append(val) + + def initLogger(self): + logger = logging.getLogger("DAS") + logger.setLevel(self.logLevel) + ch = logging.StreamHandler() + ch.setLevel(self.logLevel) + ch.setFormatter(CustomFormatter()) + logger.addHandler(ch) + self.logger = logger + + def resetFailureRate(self, failureRate): + self.failureRate = failureRate + + def run(self): + self.glob.checkRowsColumns(self.validators) + self.validators[self.proposerID].broadcastBlock(self.glob.broadcasted) + missingSamples = self.glob.checkBroadcasted() + self.steps = 0 + while(missingSamples > 0): + oldMissingSamples = missingSamples + self.logger.debug("Step %d:" % self.steps, extra=self.format) + for i in range(1,self.numberValidators): + self.validators[i].receiveRowsColumns(self.glob.broadcasted) + #Rows + self.validators[i].checkRestoreRows(self.glob.goldenData) + self.validators[i].sendRows(self.glob.broadcasted) + self.validators[i].logRows() + self.validators[i].logColumns() + # Columns + self.validators[i].checkRestoreColumns(self.glob.goldenData) + self.validators[i].sendColumns(self.glob.broadcasted) + self.validators[i].logRows() + self.validators[i].logColumns() + + missingSamples = self.glob.checkBroadcasted() + if missingSamples == oldMissingSamples: + break + elif missingSamples == 0: + break + else: + self.steps += 1 + + if missingSamples == 0: + self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (self.steps, self.failureRate), extra=self.format) + return 0 + else: + self.logger.debug("The block cannot be recovered, failure rate %d!" % self.failureRate, extra=self.format) + return 1 + diff --git a/simulator/tools.py b/simulator/tools.py new file mode 100644 index 0000000..b9e4a8e --- /dev/null +++ b/simulator/tools.py @@ -0,0 +1,27 @@ +#!/bin/python3 + +import logging + +class CustomFormatter(logging.Formatter): + + blue = "\x1b[34;20m" + grey = "\x1b[38;20m" + yellow = "\x1b[33;20m" + red = "\x1b[31;20m" + bold_red = "\x1b[31;1m" + reset = "\x1b[0m" + format = "%(levelname)s : %(entity)s : %(message)s" + + FORMATS = { + logging.DEBUG: grey + format + reset, + logging.INFO: blue + format + reset, + logging.WARNING: yellow + format + reset, + logging.ERROR: red + format + reset, + logging.CRITICAL: bold_red + format + reset + } + + def format(self, record): + log_fmt = self.FORMATS.get(record.levelno) + formatter = logging.Formatter(log_fmt) + return formatter.format(record) + diff --git a/simulator/validator.py b/simulator/validator.py new file mode 100644 index 0000000..e2c005d --- /dev/null +++ b/simulator/validator.py @@ -0,0 +1,173 @@ +#!/bin/python3 + +import random +from block import * + +class Validator: + + ID = 0 + chi = 0 + format = {} + blocksize = 0 + block = [] + rowIDs = [] + columnIDs = [] + rows = [] + columns = [] + proposer = 0 + failureRate = 0 + logger = [] + + def __init__(self, ID, chi, blockSize, proposer, failureRate, deterministic, logger): + FORMAT = "%(levelname)s : %(entity)s : %(message)s" + self.ID = ID + self.format = {"entity": "Val "+str(self.ID)} + self.blockSize = blockSize + self.proposer = proposer + self.failureRate = failureRate + self.logger = logger + if chi < 1: + self.logger.error("Chi has to be greater than 0", extra=self.format) + elif chi > blockSize: + self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) + else: + self.chi = chi + self.rowIDs = [] + self.columnIDs = [] + if deterministic: + random.seed(self.ID) + lr = [i for i in range(self.blockSize)] + lc = [i for i in range(self.blockSize)] + random.shuffle(lr) + random.shuffle(lc) + for i in range(self.chi): # TODO : Avoid doubles + self.rowIDs.append(lr.pop()) + self.columnIDs.append(lc.pop()) + + def logIDs(self): + if self.proposer == 1: + self.logger.warning("I am a block proposer."% self.ID) + else: + self.logger.debug("Selected rows: "+str(self.rowIDs), extra=self.format) + self.logger.debug("Selected columns: "+str(self.columnIDs), extra=self.format) + + def initBlock(self): + self.logger.debug("I am a block proposer.", extra=self.format) + self.block = Block(self.blockSize) + self.block.fill() + #self.block.print() + + def broadcastBlock(self, broadcasted): + if self.proposer == 0: + self.logger.error("I am NOT a block proposer", extra=self.format) + else: + self.logger.debug("Broadcasting my block...", extra=self.format) + tempBlock = self.block + order = [i for i in range(self.blockSize * self.blockSize)] + random.shuffle(order) + while(order): + i = order.pop() + if (random.randint(0,99) > self.failureRate): + broadcasted.data[i] = self.block.data[i] + #broadcasted.print() + + def getColumn(self, columnID, broadcasted): + column = [0] * self.blockSize + for i in range(self.blockSize): + column[i] = broadcasted.data[(i*self.blockSize)+columnID] + self.columns.append(column) + + def getRow(self, rowID, broadcasted): + row = [0] * self.blockSize + for i in range(self.blockSize): + row[i] = broadcasted.data[(rowID*self.blockSize)+i] + self.rows.append(row) + + def receiveRowsColumns(self, broadcasted): + self.rows = [] + self.columns = [] + if self.proposer == 1: + self.logger.error("I am a block proposer", extra=self.format) + else: + self.logger.debug("Receiving the data...", extra=self.format) + for r in self.rowIDs: + self.getRow(r, broadcasted) + for c in self.columnIDs: + self.getColumn(c, broadcasted) + + def sendColumn(self, c, columnID, broadcasted): + column = [0] * self.blockSize + for i in range(self.blockSize): + if broadcasted.data[(i*self.blockSize)+columnID] == 0: + broadcasted.data[(i*self.blockSize)+columnID] = self.columns[c][i] + + def sendRow(self, r, rowID, broadcasted): + for i in range(self.blockSize): + if broadcasted.data[(rowID*self.blockSize)+i] == 0: + broadcasted.data[(rowID*self.blockSize)+i] = self.rows[r][i] + + def sendRows(self, broadcasted): + if self.proposer == 1: + self.logger.error("I am a block proposer", extra=self.format) + else: + self.logger.debug("Sending restored rows...", extra=self.format) + for r in range(len(self.rowIDs)): + self.sendRow(r, self.rowIDs[r], broadcasted) + + def sendColumns(self, broadcasted): + if self.proposer == 1: + self.logger.error("I am a block proposer", extra=self.format) + else: + self.logger.debug("Sending restored columns...", extra=self.format) + for c in range(len(self.columnIDs)): + self.sendColumn(c, self.columnIDs[c], broadcasted) + + def logRows(self): + self.logger.debug("Rows: "+str(self.rows), extra=self.format) + + def logColumns(self): + self.logger.debug("Columns: "+str(self.columns), extra=self.format) + + def checkRestoreRows(self, goldenData): + for rid in range(len(self.rows)): + row = self.rows[rid] + failures = 0 + success = 0 + for i in row: + if i == 0: + failures += 1 + elif i > 0 and i < 10: + success += 1 + else: + self.logger.error("Data has been corrupted") + + if failures > 0: + if success >= len(row)/2: + for i in range(len(row)): + self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i] + self.logger.debug("%d samples restored in row %d" % (failures, self.rowIDs[rid]), extra=self.format ) + else: + self.logger.debug("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) + + def checkRestoreColumns(self, goldenData): + for cid in range(len(self.columns)): + column = self.columns[cid] + failures = 0 + success = 0 + for i in column: + if i == 0: + failures += 1 + elif i > 0 and i < 10: + success += 1 + else: + self.logger.error("Data has been corrupted", extra=self.format) + + if failures > 0: + if success >= len(column)/2: + for i in range(len(column)): + self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]] + self.logger.debug("%d samples restored in column %d" % (failures, self.columnIDs[cid]), extra=self.format) + else: + self.logger.debug("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format) + + From 0c79ddbf6feef67312677d3d78f60ff28daec39e Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 30 Nov 2022 15:28:27 +0100 Subject: [PATCH 6/7] Restructure package --- DAS/__init__.py | 1 + DAS/block.py | 27 ++++++++ DAS/observer.py | 52 ++++++++++++++ DAS/simulator.py | 91 +++++++++++++++++++++++++ DAS/tools.py | 27 ++++++++ DAS/validator.py | 173 +++++++++++++++++++++++++++++++++++++++++++++++ study.py | 35 ++++++++++ 7 files changed, 406 insertions(+) create mode 100644 DAS/__init__.py create mode 100644 DAS/block.py create mode 100644 DAS/observer.py create mode 100644 DAS/simulator.py create mode 100644 DAS/tools.py create mode 100644 DAS/validator.py create mode 100644 study.py diff --git a/DAS/__init__.py b/DAS/__init__.py new file mode 100644 index 0000000..ef4ca24 --- /dev/null +++ b/DAS/__init__.py @@ -0,0 +1 @@ +from DAS.simulator import * diff --git a/DAS/block.py b/DAS/block.py new file mode 100644 index 0000000..5539865 --- /dev/null +++ b/DAS/block.py @@ -0,0 +1,27 @@ +#!/bin/python3 + +import random + +class Block: + + blockSize = 0 + data = [] + + def __init__(self, size): + self.blockSize = size + self.data = [0] * (self.blockSize*self.blockSize) + + def fill(self): + for i in range(self.blockSize*self.blockSize): + self.data[i] = random.randint(1, 9) + + def print(self): + dash = "-" * (self.blockSize+2) + print(dash) + for i in range(self.blockSize): + line = "|" + for j in range(self.blockSize): + line += "%i" % self.data[(i*self.blockSize)+j] + print(line+"|") + print(dash) + diff --git a/DAS/observer.py b/DAS/observer.py new file mode 100644 index 0000000..cbbbf8d --- /dev/null +++ b/DAS/observer.py @@ -0,0 +1,52 @@ +#!/bin/python3 + +from DAS.block import * + +class Observer: + + block = [] + blockSize = 0 + rows = [] + columns = [] + goldenData = [] + broadcasted = [] + logger = [] + + def __init__(self, blockSize, logger): + self.format = {"entity": "Observer"} + self.blockSize = blockSize + self.logger = logger + + def reset(self): + self.block = [0] * self.blockSize * self.blockSize + self.goldenData = [0] * self.blockSize * self.blockSize + self.rows = [0] * self.blockSize + self.columns = [0] * self.blockSize + self.broadcasted = Block(self.blockSize) + + def checkRowsColumns(self, validators): + for val in validators: + if val.proposer == 0: + for r in val.rowIDs: + self.rows[r] += 1 + for c in val.columnIDs: + self.columns[c] += 1 + + for i in range(self.blockSize): + self.logger.debug("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format) + if self.rows[i] == 0 or self.columns[i] == 0: + self.logger.warning("There is a row/column that has not been assigned", extra=self.format) + + def setGoldenData(self, block): + for i in range(self.blockSize*self.blockSize): + self.goldenData[i] = block.data[i] + + def checkBroadcasted(self): + zeros = 0 + for i in range(self.blockSize * self.blockSize): + if self.broadcasted.data[i] == 0: + zeros += 1 + if zeros > 0: + self.logger.debug("There are %d missing samples in the network" % zeros, extra=self.format) + return zeros + diff --git a/DAS/simulator.py b/DAS/simulator.py new file mode 100644 index 0000000..5d35f14 --- /dev/null +++ b/DAS/simulator.py @@ -0,0 +1,91 @@ +#!/bin/python + +import logging +from datetime import datetime +from DAS.tools import * +from DAS.observer import * +from DAS.validator import * + +class Simulator: + + chi = 8 + blockSize = 256 + numberValidators = 8192 + failureRate = 0 + proposerID = 0 + logLevel = logging.INFO + deterministic = 0 + validators = [] + glob = [] + logger = [] + format = {} + steps = 0 + + def __init__(self, failureRate): + self.failureRate = failureRate + self.format = {"entity": "Simulator"} + self.steps = 0 + + def initValidators(self): + if not self.deterministic: + random.seed(datetime.now()) + self.glob = Observer(self.blockSize, self.logger) + self.glob.reset() + self.validators = [] + for i in range(self.numberValidators): + val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic, self.logger) + if i == self.proposerID: + val.initBlock() + self.glob.setGoldenData(val.block) + else: + val.logIDs() + self.validators.append(val) + + def initLogger(self): + logger = logging.getLogger("DAS") + logger.setLevel(self.logLevel) + ch = logging.StreamHandler() + ch.setLevel(self.logLevel) + ch.setFormatter(CustomFormatter()) + logger.addHandler(ch) + self.logger = logger + + def resetFailureRate(self, failureRate): + self.failureRate = failureRate + + def run(self): + self.glob.checkRowsColumns(self.validators) + self.validators[self.proposerID].broadcastBlock(self.glob.broadcasted) + missingSamples = self.glob.checkBroadcasted() + self.steps = 0 + while(missingSamples > 0): + oldMissingSamples = missingSamples + self.logger.debug("Step %d:" % self.steps, extra=self.format) + for i in range(1,self.numberValidators): + self.validators[i].receiveRowsColumns(self.glob.broadcasted) + #Rows + self.validators[i].checkRestoreRows(self.glob.goldenData) + self.validators[i].sendRows(self.glob.broadcasted) + self.validators[i].logRows() + self.validators[i].logColumns() + # Columns + self.validators[i].checkRestoreColumns(self.glob.goldenData) + self.validators[i].sendColumns(self.glob.broadcasted) + self.validators[i].logRows() + self.validators[i].logColumns() + + missingSamples = self.glob.checkBroadcasted() + if missingSamples == oldMissingSamples: + break + elif missingSamples == 0: + break + else: + self.steps += 1 + + if missingSamples == 0: + self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (self.steps, self.failureRate), extra=self.format) + return 0 + else: + self.logger.debug("The block cannot be recovered, failure rate %d!" % self.failureRate, extra=self.format) + return 1 + diff --git a/DAS/tools.py b/DAS/tools.py new file mode 100644 index 0000000..b9e4a8e --- /dev/null +++ b/DAS/tools.py @@ -0,0 +1,27 @@ +#!/bin/python3 + +import logging + +class CustomFormatter(logging.Formatter): + + blue = "\x1b[34;20m" + grey = "\x1b[38;20m" + yellow = "\x1b[33;20m" + red = "\x1b[31;20m" + bold_red = "\x1b[31;1m" + reset = "\x1b[0m" + format = "%(levelname)s : %(entity)s : %(message)s" + + FORMATS = { + logging.DEBUG: grey + format + reset, + logging.INFO: blue + format + reset, + logging.WARNING: yellow + format + reset, + logging.ERROR: red + format + reset, + logging.CRITICAL: bold_red + format + reset + } + + def format(self, record): + log_fmt = self.FORMATS.get(record.levelno) + formatter = logging.Formatter(log_fmt) + return formatter.format(record) + diff --git a/DAS/validator.py b/DAS/validator.py new file mode 100644 index 0000000..97f071e --- /dev/null +++ b/DAS/validator.py @@ -0,0 +1,173 @@ +#!/bin/python3 + +import random +from DAS.block import * + +class Validator: + + ID = 0 + chi = 0 + format = {} + blocksize = 0 + block = [] + rowIDs = [] + columnIDs = [] + rows = [] + columns = [] + proposer = 0 + failureRate = 0 + logger = [] + + def __init__(self, ID, chi, blockSize, proposer, failureRate, deterministic, logger): + FORMAT = "%(levelname)s : %(entity)s : %(message)s" + self.ID = ID + self.format = {"entity": "Val "+str(self.ID)} + self.blockSize = blockSize + self.proposer = proposer + self.failureRate = failureRate + self.logger = logger + if chi < 1: + self.logger.error("Chi has to be greater than 0", extra=self.format) + elif chi > blockSize: + self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) + else: + self.chi = chi + self.rowIDs = [] + self.columnIDs = [] + if deterministic: + random.seed(self.ID) + lr = [i for i in range(self.blockSize)] + lc = [i for i in range(self.blockSize)] + random.shuffle(lr) + random.shuffle(lc) + for i in range(self.chi): # TODO : Avoid doubles + self.rowIDs.append(lr.pop()) + self.columnIDs.append(lc.pop()) + + def logIDs(self): + if self.proposer == 1: + self.logger.warning("I am a block proposer."% self.ID) + else: + self.logger.debug("Selected rows: "+str(self.rowIDs), extra=self.format) + self.logger.debug("Selected columns: "+str(self.columnIDs), extra=self.format) + + def initBlock(self): + self.logger.debug("I am a block proposer.", extra=self.format) + self.block = Block(self.blockSize) + self.block.fill() + #self.block.print() + + def broadcastBlock(self, broadcasted): + if self.proposer == 0: + self.logger.error("I am NOT a block proposer", extra=self.format) + else: + self.logger.debug("Broadcasting my block...", extra=self.format) + tempBlock = self.block + order = [i for i in range(self.blockSize * self.blockSize)] + random.shuffle(order) + while(order): + i = order.pop() + if (random.randint(0,99) > self.failureRate): + broadcasted.data[i] = self.block.data[i] + #broadcasted.print() + + def getColumn(self, columnID, broadcasted): + column = [0] * self.blockSize + for i in range(self.blockSize): + column[i] = broadcasted.data[(i*self.blockSize)+columnID] + self.columns.append(column) + + def getRow(self, rowID, broadcasted): + row = [0] * self.blockSize + for i in range(self.blockSize): + row[i] = broadcasted.data[(rowID*self.blockSize)+i] + self.rows.append(row) + + def receiveRowsColumns(self, broadcasted): + self.rows = [] + self.columns = [] + if self.proposer == 1: + self.logger.error("I am a block proposer", extra=self.format) + else: + self.logger.debug("Receiving the data...", extra=self.format) + for r in self.rowIDs: + self.getRow(r, broadcasted) + for c in self.columnIDs: + self.getColumn(c, broadcasted) + + def sendColumn(self, c, columnID, broadcasted): + column = [0] * self.blockSize + for i in range(self.blockSize): + if broadcasted.data[(i*self.blockSize)+columnID] == 0: + broadcasted.data[(i*self.blockSize)+columnID] = self.columns[c][i] + + def sendRow(self, r, rowID, broadcasted): + for i in range(self.blockSize): + if broadcasted.data[(rowID*self.blockSize)+i] == 0: + broadcasted.data[(rowID*self.blockSize)+i] = self.rows[r][i] + + def sendRows(self, broadcasted): + if self.proposer == 1: + self.logger.error("I am a block proposer", extra=self.format) + else: + self.logger.debug("Sending restored rows...", extra=self.format) + for r in range(len(self.rowIDs)): + self.sendRow(r, self.rowIDs[r], broadcasted) + + def sendColumns(self, broadcasted): + if self.proposer == 1: + self.logger.error("I am a block proposer", extra=self.format) + else: + self.logger.debug("Sending restored columns...", extra=self.format) + for c in range(len(self.columnIDs)): + self.sendColumn(c, self.columnIDs[c], broadcasted) + + def logRows(self): + self.logger.debug("Rows: "+str(self.rows), extra=self.format) + + def logColumns(self): + self.logger.debug("Columns: "+str(self.columns), extra=self.format) + + def checkRestoreRows(self, goldenData): + for rid in range(len(self.rows)): + row = self.rows[rid] + failures = 0 + success = 0 + for i in row: + if i == 0: + failures += 1 + elif i > 0 and i < 10: + success += 1 + else: + self.logger.error("Data has been corrupted") + + if failures > 0: + if success >= len(row)/2: + for i in range(len(row)): + self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i] + self.logger.debug("%d samples restored in row %d" % (failures, self.rowIDs[rid]), extra=self.format ) + else: + self.logger.debug("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) + + def checkRestoreColumns(self, goldenData): + for cid in range(len(self.columns)): + column = self.columns[cid] + failures = 0 + success = 0 + for i in column: + if i == 0: + failures += 1 + elif i > 0 and i < 10: + success += 1 + else: + self.logger.error("Data has been corrupted", extra=self.format) + + if failures > 0: + if success >= len(column)/2: + for i in range(len(column)): + self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]] + self.logger.debug("%d samples restored in column %d" % (failures, self.columnIDs[cid]), extra=self.format) + else: + self.logger.debug("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format) + + diff --git a/study.py b/study.py new file mode 100644 index 0000000..b06e21b --- /dev/null +++ b/study.py @@ -0,0 +1,35 @@ +#! /bin/python3 + +import time +from DAS import * + + +def study(): + sim = Simulator(0) + sim.initLogger() + maxTries = 2 + step = 25 + frRange = [] + resultRange = [] + simCnt = 0 + sim.logger.info("Starting simulations:", extra=sim.format) + start = time.time() + for fr in range(0, 100, step): + if fr % 10 == 0: + sim.logger.info("Failure rate %d %% ..." % fr, extra=sim.format) + sim.resetFailureRate(fr) + result = 0 + for i in range(maxTries): + sim.initValidators() + result += sim.run() + simCnt += 1 + frRange.append(fr) + resultRange.append(100-result) + end = time.time() + sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format) + for i in range(len(frRange)): + sim.logger.info("For failure rate of %d we got %d %% success rate in DAS!" % (frRange[i], resultRange[i]), extra=sim.format) + + +study() + From eb8d9b9095c9c663f17dea616d5bb98b9faf308b Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 30 Nov 2022 15:31:21 +0100 Subject: [PATCH 7/7] Remove old files --- simulator/DAS.py | 35 --------- simulator/block.py | 27 ------- simulator/observer.py | 52 ------------- simulator/simulator.py | 91 ---------------------- simulator/tools.py | 27 ------- simulator/validator.py | 173 ----------------------------------------- 6 files changed, 405 deletions(-) delete mode 100644 simulator/DAS.py delete mode 100644 simulator/block.py delete mode 100644 simulator/observer.py delete mode 100644 simulator/simulator.py delete mode 100644 simulator/tools.py delete mode 100644 simulator/validator.py diff --git a/simulator/DAS.py b/simulator/DAS.py deleted file mode 100644 index fe4f4e3..0000000 --- a/simulator/DAS.py +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/python3 - -import time -from simulator import * - - -def study(): - sim = Simulator(0) - sim.initLogger() - maxTries = 2 - step = 25 - frRange = [] - resultRange = [] - simCnt = 0 - sim.logger.info("Starting simulations:", extra=sim.format) - start = time.time() - for fr in range(0, 100, step): - if fr % 10 == 0: - sim.logger.info("Failure rate %d %% ..." % fr, extra=sim.format) - sim.resetFailureRate(fr) - result = 0 - for i in range(maxTries): - sim.initValidators() - result += sim.run() - simCnt += 1 - frRange.append(fr) - resultRange.append(100-result) - end = time.time() - sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format) - for i in range(len(frRange)): - sim.logger.info("For failure rate of %d we got %d %% success rate in DAS!" % (frRange[i], resultRange[i]), extra=sim.format) - - -study() - diff --git a/simulator/block.py b/simulator/block.py deleted file mode 100644 index 5539865..0000000 --- a/simulator/block.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/python3 - -import random - -class Block: - - blockSize = 0 - data = [] - - def __init__(self, size): - self.blockSize = size - self.data = [0] * (self.blockSize*self.blockSize) - - def fill(self): - for i in range(self.blockSize*self.blockSize): - self.data[i] = random.randint(1, 9) - - def print(self): - dash = "-" * (self.blockSize+2) - print(dash) - for i in range(self.blockSize): - line = "|" - for j in range(self.blockSize): - line += "%i" % self.data[(i*self.blockSize)+j] - print(line+"|") - print(dash) - diff --git a/simulator/observer.py b/simulator/observer.py deleted file mode 100644 index 3a13d47..0000000 --- a/simulator/observer.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/python3 - -from block import * - -class Observer: - - block = [] - blockSize = 0 - rows = [] - columns = [] - goldenData = [] - broadcasted = [] - logger = [] - - def __init__(self, blockSize, logger): - self.format = {"entity": "Observer"} - self.blockSize = blockSize - self.logger = logger - - def reset(self): - self.block = [0] * self.blockSize * self.blockSize - self.goldenData = [0] * self.blockSize * self.blockSize - self.rows = [0] * self.blockSize - self.columns = [0] * self.blockSize - self.broadcasted = Block(self.blockSize) - - def checkRowsColumns(self, validators): - for val in validators: - if val.proposer == 0: - for r in val.rowIDs: - self.rows[r] += 1 - for c in val.columnIDs: - self.columns[c] += 1 - - for i in range(self.blockSize): - self.logger.debug("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format) - if self.rows[i] == 0 or self.columns[i] == 0: - self.logger.warning("There is a row/column that has not been assigned", extra=self.format) - - def setGoldenData(self, block): - for i in range(self.blockSize*self.blockSize): - self.goldenData[i] = block.data[i] - - def checkBroadcasted(self): - zeros = 0 - for i in range(self.blockSize * self.blockSize): - if self.broadcasted.data[i] == 0: - zeros += 1 - if zeros > 0: - self.logger.debug("There are %d missing samples in the network" % zeros, extra=self.format) - return zeros - diff --git a/simulator/simulator.py b/simulator/simulator.py deleted file mode 100644 index 7781080..0000000 --- a/simulator/simulator.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/python - -import logging -from datetime import datetime -from tools import * -from observer import * -from validator import * - -class Simulator: - - chi = 8 - blockSize = 256 - numberValidators = 8192 - failureRate = 0 - proposerID = 0 - logLevel = logging.INFO - deterministic = 0 - validators = [] - glob = [] - logger = [] - format = {} - steps = 0 - - def __init__(self, failureRate): - self.failureRate = failureRate - self.format = {"entity": "Simulator"} - self.steps = 0 - - def initValidators(self): - if not self.deterministic: - random.seed(datetime.now()) - self.glob = Observer(self.blockSize, self.logger) - self.glob.reset() - self.validators = [] - for i in range(self.numberValidators): - val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic, self.logger) - if i == self.proposerID: - val.initBlock() - self.glob.setGoldenData(val.block) - else: - val.logIDs() - self.validators.append(val) - - def initLogger(self): - logger = logging.getLogger("DAS") - logger.setLevel(self.logLevel) - ch = logging.StreamHandler() - ch.setLevel(self.logLevel) - ch.setFormatter(CustomFormatter()) - logger.addHandler(ch) - self.logger = logger - - def resetFailureRate(self, failureRate): - self.failureRate = failureRate - - def run(self): - self.glob.checkRowsColumns(self.validators) - self.validators[self.proposerID].broadcastBlock(self.glob.broadcasted) - missingSamples = self.glob.checkBroadcasted() - self.steps = 0 - while(missingSamples > 0): - oldMissingSamples = missingSamples - self.logger.debug("Step %d:" % self.steps, extra=self.format) - for i in range(1,self.numberValidators): - self.validators[i].receiveRowsColumns(self.glob.broadcasted) - #Rows - self.validators[i].checkRestoreRows(self.glob.goldenData) - self.validators[i].sendRows(self.glob.broadcasted) - self.validators[i].logRows() - self.validators[i].logColumns() - # Columns - self.validators[i].checkRestoreColumns(self.glob.goldenData) - self.validators[i].sendColumns(self.glob.broadcasted) - self.validators[i].logRows() - self.validators[i].logColumns() - - missingSamples = self.glob.checkBroadcasted() - if missingSamples == oldMissingSamples: - break - elif missingSamples == 0: - break - else: - self.steps += 1 - - if missingSamples == 0: - self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (self.steps, self.failureRate), extra=self.format) - return 0 - else: - self.logger.debug("The block cannot be recovered, failure rate %d!" % self.failureRate, extra=self.format) - return 1 - diff --git a/simulator/tools.py b/simulator/tools.py deleted file mode 100644 index b9e4a8e..0000000 --- a/simulator/tools.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/python3 - -import logging - -class CustomFormatter(logging.Formatter): - - blue = "\x1b[34;20m" - grey = "\x1b[38;20m" - yellow = "\x1b[33;20m" - red = "\x1b[31;20m" - bold_red = "\x1b[31;1m" - reset = "\x1b[0m" - format = "%(levelname)s : %(entity)s : %(message)s" - - FORMATS = { - logging.DEBUG: grey + format + reset, - logging.INFO: blue + format + reset, - logging.WARNING: yellow + format + reset, - logging.ERROR: red + format + reset, - logging.CRITICAL: bold_red + format + reset - } - - def format(self, record): - log_fmt = self.FORMATS.get(record.levelno) - formatter = logging.Formatter(log_fmt) - return formatter.format(record) - diff --git a/simulator/validator.py b/simulator/validator.py deleted file mode 100644 index e2c005d..0000000 --- a/simulator/validator.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/python3 - -import random -from block import * - -class Validator: - - ID = 0 - chi = 0 - format = {} - blocksize = 0 - block = [] - rowIDs = [] - columnIDs = [] - rows = [] - columns = [] - proposer = 0 - failureRate = 0 - logger = [] - - def __init__(self, ID, chi, blockSize, proposer, failureRate, deterministic, logger): - FORMAT = "%(levelname)s : %(entity)s : %(message)s" - self.ID = ID - self.format = {"entity": "Val "+str(self.ID)} - self.blockSize = blockSize - self.proposer = proposer - self.failureRate = failureRate - self.logger = logger - if chi < 1: - self.logger.error("Chi has to be greater than 0", extra=self.format) - elif chi > blockSize: - self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) - else: - self.chi = chi - self.rowIDs = [] - self.columnIDs = [] - if deterministic: - random.seed(self.ID) - lr = [i for i in range(self.blockSize)] - lc = [i for i in range(self.blockSize)] - random.shuffle(lr) - random.shuffle(lc) - for i in range(self.chi): # TODO : Avoid doubles - self.rowIDs.append(lr.pop()) - self.columnIDs.append(lc.pop()) - - def logIDs(self): - if self.proposer == 1: - self.logger.warning("I am a block proposer."% self.ID) - else: - self.logger.debug("Selected rows: "+str(self.rowIDs), extra=self.format) - self.logger.debug("Selected columns: "+str(self.columnIDs), extra=self.format) - - def initBlock(self): - self.logger.debug("I am a block proposer.", extra=self.format) - self.block = Block(self.blockSize) - self.block.fill() - #self.block.print() - - def broadcastBlock(self, broadcasted): - if self.proposer == 0: - self.logger.error("I am NOT a block proposer", extra=self.format) - else: - self.logger.debug("Broadcasting my block...", extra=self.format) - tempBlock = self.block - order = [i for i in range(self.blockSize * self.blockSize)] - random.shuffle(order) - while(order): - i = order.pop() - if (random.randint(0,99) > self.failureRate): - broadcasted.data[i] = self.block.data[i] - #broadcasted.print() - - def getColumn(self, columnID, broadcasted): - column = [0] * self.blockSize - for i in range(self.blockSize): - column[i] = broadcasted.data[(i*self.blockSize)+columnID] - self.columns.append(column) - - def getRow(self, rowID, broadcasted): - row = [0] * self.blockSize - for i in range(self.blockSize): - row[i] = broadcasted.data[(rowID*self.blockSize)+i] - self.rows.append(row) - - def receiveRowsColumns(self, broadcasted): - self.rows = [] - self.columns = [] - if self.proposer == 1: - self.logger.error("I am a block proposer", extra=self.format) - else: - self.logger.debug("Receiving the data...", extra=self.format) - for r in self.rowIDs: - self.getRow(r, broadcasted) - for c in self.columnIDs: - self.getColumn(c, broadcasted) - - def sendColumn(self, c, columnID, broadcasted): - column = [0] * self.blockSize - for i in range(self.blockSize): - if broadcasted.data[(i*self.blockSize)+columnID] == 0: - broadcasted.data[(i*self.blockSize)+columnID] = self.columns[c][i] - - def sendRow(self, r, rowID, broadcasted): - for i in range(self.blockSize): - if broadcasted.data[(rowID*self.blockSize)+i] == 0: - broadcasted.data[(rowID*self.blockSize)+i] = self.rows[r][i] - - def sendRows(self, broadcasted): - if self.proposer == 1: - self.logger.error("I am a block proposer", extra=self.format) - else: - self.logger.debug("Sending restored rows...", extra=self.format) - for r in range(len(self.rowIDs)): - self.sendRow(r, self.rowIDs[r], broadcasted) - - def sendColumns(self, broadcasted): - if self.proposer == 1: - self.logger.error("I am a block proposer", extra=self.format) - else: - self.logger.debug("Sending restored columns...", extra=self.format) - for c in range(len(self.columnIDs)): - self.sendColumn(c, self.columnIDs[c], broadcasted) - - def logRows(self): - self.logger.debug("Rows: "+str(self.rows), extra=self.format) - - def logColumns(self): - self.logger.debug("Columns: "+str(self.columns), extra=self.format) - - def checkRestoreRows(self, goldenData): - for rid in range(len(self.rows)): - row = self.rows[rid] - failures = 0 - success = 0 - for i in row: - if i == 0: - failures += 1 - elif i > 0 and i < 10: - success += 1 - else: - self.logger.error("Data has been corrupted") - - if failures > 0: - if success >= len(row)/2: - for i in range(len(row)): - self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i] - self.logger.debug("%d samples restored in row %d" % (failures, self.rowIDs[rid]), extra=self.format ) - else: - self.logger.debug("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) - - def checkRestoreColumns(self, goldenData): - for cid in range(len(self.columns)): - column = self.columns[cid] - failures = 0 - success = 0 - for i in column: - if i == 0: - failures += 1 - elif i > 0 and i < 10: - success += 1 - else: - self.logger.error("Data has been corrupted", extra=self.format) - - if failures > 0: - if success >= len(column)/2: - for i in range(len(column)): - self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]] - self.logger.debug("%d samples restored in column %d" % (failures, self.columnIDs[cid]), extra=self.format) - else: - self.logger.debug("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format) - -