Merge pull request #4 from status-im/bitmaps

Bitmaps optimizations

DAS/block.py

@@ -1,19 +1,26 @@
 #!/bin/python3
 
 import random
+from bitarray import bitarray
+from bitarray.util import zeros
 
 class Block:
 
     blockSize = 0
-    data = []
+    data = bitarray()
 
     def __init__(self, size):
         self.blockSize = size
-        self.data = [0] * (self.blockSize*self.blockSize)
+        self.data = zeros(self.blockSize*self.blockSize)
 
     def fill(self):
-        for i in range(self.blockSize*self.blockSize):
+        self.data.setall(1)
-            self.data[i] = random.randint(1, 9)
+
+    def getColumn(self, columnID):
+        return self.data[columnID::self.blockSize]
+
+    def getRow(self, rowID):
+        return self.data[rowID*self.blockSize:(rowID+1)*self.blockSize]
 
     def print(self):
         dash = "-" * (self.blockSize+2)

DAS/simulator.py

@@ -64,12 +64,12 @@ class Simulator:
             for i in range(1,self.numberValidators):
                 self.validators[i].receiveRowsColumns(self.glob.broadcasted)
                 #Rows
-                self.validators[i].checkRestoreRows(self.glob.goldenData)
+                self.validators[i].restoreRows()
                 self.validators[i].sendRows(self.glob.broadcasted)
                 self.validators[i].logRows()
                 self.validators[i].logColumns()
                 # Columns
-                self.validators[i].checkRestoreColumns(self.glob.goldenData)
+                self.validators[i].restoreColumns()
                 self.validators[i].sendColumns(self.glob.broadcasted)
                 self.validators[i].logRows()
                 self.validators[i].logColumns()

DAS/validator.py

@@ -2,6 +2,8 @@
 
 import random
 from DAS.block import *
+from bitarray import bitarray
+from bitarray.util import zeros
 
 class Validator:
 
@@ -36,13 +38,8 @@ class Validator:
             self.columnIDs = []
             if deterministic:
                 random.seed(self.ID)
-            lr = [i for i in range(self.blockSize)]
+            self.rowIDs = random.sample(range(self.blockSize), self.chi)
-            lc = [i for i in range(self.blockSize)]
+            self.columnIDs = random.sample(range(self.blockSize), self.chi)
-            random.shuffle(lr)
-            random.shuffle(lc)
-            for i in range(self.chi): # TODO : Avoid doubles
-                self.rowIDs.append(lr.pop())
-                self.columnIDs.append(lc.pop())
 
     def logIDs(self):
         if self.proposer == 1:
@@ -72,15 +69,11 @@ class Validator:
             #broadcasted.print()
 
     def getColumn(self, columnID, broadcasted):
-        column = [0] * self.blockSize
+        column = broadcasted.getColumn(columnID)
-        for i in range(self.blockSize):
-            column[i] = broadcasted.data[(i*self.blockSize)+columnID]
         self.columns.append(column)
 
     def getRow(self, rowID, broadcasted):
-        row = [0] * self.blockSize
+        row = broadcasted.getRow(rowID)
-        for i in range(self.blockSize):
-            row[i] = broadcasted.data[(rowID*self.blockSize)+i]
         self.rows.append(row)
 
     def receiveRowsColumns(self, broadcasted):
@@ -96,15 +89,10 @@ class Validator:
                 self.getColumn(c, broadcasted)
 
     def sendColumn(self, c, columnID, broadcasted):
-        column = [0] * self.blockSize
+        broadcasted.data[columnID::self.blockSize] |= self.columns[c]
-        for i in range(self.blockSize):
-            if broadcasted.data[(i*self.blockSize)+columnID] == 0:
-                broadcasted.data[(i*self.blockSize)+columnID] = self.columns[c][i]
 
     def sendRow(self, r, rowID, broadcasted):
-        for i in range(self.blockSize):
+        broadcasted.data[rowID*self.blockSize:(rowID+1)*self.blockSize] |=  self.rows[r]
-            if broadcasted.data[(rowID*self.blockSize)+i] == 0:
-                broadcasted.data[(rowID*self.blockSize)+i] = self.rows[r][i]
 
     def sendRows(self, broadcasted):
         if self.proposer == 1:
@@ -128,46 +116,25 @@ class Validator:
     def logColumns(self):
         self.logger.debug("Columns: "+str(self.columns), extra=self.format)
 
-    def checkRestoreRows(self, goldenData):
+    def restoreRows(self):
         for rid in range(len(self.rows)):
             row = self.rows[rid]
-            failures = 0
+            success = row.count(1)
-            success = 0
-            for i in row:
-                if i == 0:
-                    failures += 1
-                elif i > 0 and i < 10:
-                    success += 1
-                else:
-                    self.logger.error("Data has been corrupted")
 
-            if failures > 0:
+            if success >= len(row)/2:
-                if success >= len(row)/2:
+                self.rows[rid].setall(1)
-                    for i in range(len(row)):
+                self.logger.debug("%d samples restored in row %d" % (len(row)-success, self.rowIDs[rid]), extra=self.format )
-                        self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i]
+            else:
-                    self.logger.debug("%d samples restored in row %d" % (failures, self.rowIDs[rid]), extra=self.format )
+                self.logger.debug("Row %d cannot be restored" %  (self.rowIDs[rid]), extra=self.format)
-                else:
-                    self.logger.debug("Row %d cannot be restored" %  (self.rowIDs[rid]), extra=self.format)
 
-    def checkRestoreColumns(self, goldenData):
+    def restoreColumns(self):
         for cid in range(len(self.columns)):
             column = self.columns[cid]
-            failures = 0
+            success = column.count(1)
-            success = 0
+            if success >= len(column)/2:
-            for i in column:
+                self.columns[cid].setall(1)
-                if i == 0:
+                self.logger.debug("%d samples restored in column %d" % (len(column)-success, self.columnIDs[cid]), extra=self.format)
-                    failures += 1
+            else:
-                elif i > 0 and i < 10:
+                self.logger.debug("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format)
-                    success += 1
-                else:
-                    self.logger.error("Data has been corrupted", extra=self.format)
-
-            if failures > 0:
-                if success >= len(column)/2:
-                    for i in range(len(column)):
-                        self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]]
-                    self.logger.debug("%d samples restored in column %d" % (failures, self.columnIDs[cid]), extra=self.format)
-                else:
-                    self.logger.debug("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format)