From 13580cd44d48314703659b62835d320b1db9606e Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 7 Dec 2022 10:59:41 +0100 Subject: [PATCH 1/7] pick row/column IDs without replacement Signed-off-by: Csaba Kiraly --- DAS/validator.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 97f071e..2d8ac39 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -36,13 +36,8 @@ class Validator: self.columnIDs = [] if deterministic: random.seed(self.ID) - lr = [i for i in range(self.blockSize)] - lc = [i for i in range(self.blockSize)] - random.shuffle(lr) - random.shuffle(lc) - for i in range(self.chi): # TODO : Avoid doubles - self.rowIDs.append(lr.pop()) - self.columnIDs.append(lc.pop()) + self.rowIDs = random.sample(range(self.blockSize), self.chi) + self.columnIDs = random.sample(range(self.blockSize), self.chi) def logIDs(self): if self.proposer == 1: From 8da447ac5a2dd2c987b5e1cbacc4d693d5617ef4 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 7 Dec 2022 15:10:35 +0100 Subject: [PATCH 2/7] add column and row accessors Signed-off-by: Csaba Kiraly --- DAS/block.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/DAS/block.py b/DAS/block.py index 5539865..5c09ca1 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -15,6 +15,18 @@ class Block: for i in range(self.blockSize*self.blockSize): self.data[i] = random.randint(1, 9) + def getColumn(self, columnID): + column = [0] * self.blockSize + for i in range(self.blockSize): + column[i] = self.data[(i*self.blockSize)+columnID] + return column + + def getRow(self, rowID): + row = [0] * self.blockSize + for i in range(self.blockSize): + row[i] = self.data[(rowID*self.blockSize)+i] + return row + def print(self): dash = "-" * (self.blockSize+2) print(dash) From b48fb6f791ad7f129aa9495bdf426f67874bc88f Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 7 Dec 2022 15:14:36 +0100 Subject: [PATCH 3/7] use column and row accessors Signed-off-by: Csaba Kiraly --- DAS/validator.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 2d8ac39..bfe0adb 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -67,15 +67,11 @@ class Validator: #broadcasted.print() def getColumn(self, columnID, broadcasted): - column = [0] * self.blockSize - for i in range(self.blockSize): - column[i] = broadcasted.data[(i*self.blockSize)+columnID] + column = broadcasted.getColumn(columnID) self.columns.append(column) def getRow(self, rowID, broadcasted): - row = [0] * self.blockSize - for i in range(self.blockSize): - row[i] = broadcasted.data[(rowID*self.blockSize)+i] + row = broadcasted.getRow(rowID) self.rows.append(row) def receiveRowsColumns(self, broadcasted): From a07be50727af0e98decca498fc5e111046ae9aef Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 7 Dec 2022 15:25:48 +0100 Subject: [PATCH 4/7] change to bitmaps It is faster to store and process data availability as bitmaps. It is also enough, as we will not do anything with the data itself. Signed-off-by: Csaba Kiraly --- DAS/block.py | 18 +++++++----------- DAS/validator.py | 11 ++++------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/DAS/block.py b/DAS/block.py index 5c09ca1..3119167 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -1,31 +1,27 @@ #!/bin/python3 import random +from bitarray import bitarray +from bitarray.util import zeros class Block: blockSize = 0 - data = [] + data = bitarray() def __init__(self, size): self.blockSize = size - self.data = [0] * (self.blockSize*self.blockSize) + self.data = zeros(self.blockSize*self.blockSize) def fill(self): for i in range(self.blockSize*self.blockSize): - self.data[i] = random.randint(1, 9) + self.data[i] = 1 def getColumn(self, columnID): - column = [0] * self.blockSize - for i in range(self.blockSize): - column[i] = self.data[(i*self.blockSize)+columnID] - return column + return self.data[columnID::self.blockSize] def getRow(self, rowID): - row = [0] * self.blockSize - for i in range(self.blockSize): - row[i] = self.data[(rowID*self.blockSize)+i] - return row + return self.data[rowID*self.blockSize:(rowID+1)*self.blockSize] def print(self): dash = "-" * (self.blockSize+2) diff --git a/DAS/validator.py b/DAS/validator.py index bfe0adb..a570e88 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -2,6 +2,8 @@ import random from DAS.block import * +from bitarray import bitarray +from bitarray.util import zeros class Validator: @@ -87,15 +89,10 @@ class Validator: self.getColumn(c, broadcasted) def sendColumn(self, c, columnID, broadcasted): - column = [0] * self.blockSize - for i in range(self.blockSize): - if broadcasted.data[(i*self.blockSize)+columnID] == 0: - broadcasted.data[(i*self.blockSize)+columnID] = self.columns[c][i] + broadcasted.data[columnID::self.blockSize] |= self.columns[c] def sendRow(self, r, rowID, broadcasted): - for i in range(self.blockSize): - if broadcasted.data[(rowID*self.blockSize)+i] == 0: - broadcasted.data[(rowID*self.blockSize)+i] = self.rows[r][i] + broadcasted.data[rowID*self.blockSize:(rowID+1)*self.blockSize] |= self.rows[r] def sendRows(self, broadcasted): if self.proposer == 1: From 26ba6a38f738db1fa26c763d7f3beca67c555373 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 7 Dec 2022 15:39:11 +0100 Subject: [PATCH 5/7] speed up restoration Signed-off-by: Csaba Kiraly --- DAS/validator.py | 45 ++++++++++++--------------------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index a570e88..c0a5c6d 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -119,43 +119,22 @@ class Validator: def checkRestoreRows(self, goldenData): for rid in range(len(self.rows)): row = self.rows[rid] - failures = 0 - success = 0 - for i in row: - if i == 0: - failures += 1 - elif i > 0 and i < 10: - success += 1 - else: - self.logger.error("Data has been corrupted") + success = row.count(1) - if failures > 0: - if success >= len(row)/2: - for i in range(len(row)): - self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i] - self.logger.debug("%d samples restored in row %d" % (failures, self.rowIDs[rid]), extra=self.format ) - else: - self.logger.debug("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) + if success >= len(row)/2: + self.rows[rid].setall(1) + self.logger.debug("%d samples restored in row %d" % (len(row)-success, self.rowIDs[rid]), extra=self.format ) + else: + self.logger.debug("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) def checkRestoreColumns(self, goldenData): for cid in range(len(self.columns)): column = self.columns[cid] - failures = 0 - success = 0 - for i in column: - if i == 0: - failures += 1 - elif i > 0 and i < 10: - success += 1 - else: - self.logger.error("Data has been corrupted", extra=self.format) - - if failures > 0: - if success >= len(column)/2: - for i in range(len(column)): - self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]] - self.logger.debug("%d samples restored in column %d" % (failures, self.columnIDs[cid]), extra=self.format) - else: - self.logger.debug("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format) + success = column.count(1) + if success >= len(column)/2: + self.columns[cid].setall(1) + self.logger.debug("%d samples restored in column %d" % (len(column)-success, self.columnIDs[cid]), extra=self.format) + else: + self.logger.debug("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format) From 4179841c89ae142f8540b0f8117ba85a10144899 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 7 Dec 2022 15:46:45 +0100 Subject: [PATCH 6/7] change checkRestore to restore We are not simulating data errors, so no need to check. Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 4 ++-- DAS/validator.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 5d35f14..a7f2b64 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -64,12 +64,12 @@ class Simulator: for i in range(1,self.numberValidators): self.validators[i].receiveRowsColumns(self.glob.broadcasted) #Rows - self.validators[i].checkRestoreRows(self.glob.goldenData) + self.validators[i].restoreRows() self.validators[i].sendRows(self.glob.broadcasted) self.validators[i].logRows() self.validators[i].logColumns() # Columns - self.validators[i].checkRestoreColumns(self.glob.goldenData) + self.validators[i].restoreColumns() self.validators[i].sendColumns(self.glob.broadcasted) self.validators[i].logRows() self.validators[i].logColumns() diff --git a/DAS/validator.py b/DAS/validator.py index c0a5c6d..d6c9684 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -116,7 +116,7 @@ class Validator: def logColumns(self): self.logger.debug("Columns: "+str(self.columns), extra=self.format) - def checkRestoreRows(self, goldenData): + def restoreRows(self): for rid in range(len(self.rows)): row = self.rows[rid] success = row.count(1) @@ -127,7 +127,7 @@ class Validator: else: self.logger.debug("Row %d cannot be restored" % (self.rowIDs[rid]), extra=self.format) - def checkRestoreColumns(self, goldenData): + def restoreColumns(self): for cid in range(len(self.columns)): column = self.columns[cid] success = column.count(1) From accbc3eff99c558bd144f65d45148e30c3e6f7b0 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 7 Dec 2022 18:44:02 +0100 Subject: [PATCH 7/7] simplify code Signed-off-by: Csaba Kiraly --- DAS/block.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DAS/block.py b/DAS/block.py index 3119167..0ec24a4 100644 --- a/DAS/block.py +++ b/DAS/block.py @@ -14,8 +14,7 @@ class Block: self.data = zeros(self.blockSize*self.blockSize) def fill(self): - for i in range(self.blockSize*self.blockSize): - self.data[i] = 1 + self.data.setall(1) def getColumn(self, columnID): return self.data[columnID::self.blockSize]