Merge pull request #3 from status-im/simulator-initial

Simulator initial. First package structure.

.gitignore

@@ -0,0 +1,2 @@
+*.swp
+*.pyc

DAS/__init__.py

@@ -0,0 +1 @@
+from DAS.simulator import *

DAS/block.py

@@ -0,0 +1,27 @@
+#!/bin/python3
+
+import random
+
+class Block:
+
+    blockSize = 0
+    data = []
+
+    def __init__(self, size):
+        self.blockSize = size
+        self.data = [0] * (self.blockSize*self.blockSize)
+
+    def fill(self):
+        for i in range(self.blockSize*self.blockSize):
+            self.data[i] = random.randint(1, 9)
+
+    def print(self):
+        dash = "-" * (self.blockSize+2)
+        print(dash)
+        for i in range(self.blockSize):
+            line = "|"
+            for j in range(self.blockSize):
+                line += "%i" % self.data[(i*self.blockSize)+j]
+            print(line+"|")
+        print(dash)
+

DAS/observer.py

@@ -0,0 +1,52 @@
+#!/bin/python3
+
+from DAS.block import *
+
+class Observer:
+
+    block = []
+    blockSize = 0
+    rows = []
+    columns = []
+    goldenData = []
+    broadcasted = []
+    logger = []
+
+    def __init__(self, blockSize, logger):
+        self.format = {"entity": "Observer"}
+        self.blockSize = blockSize
+        self.logger = logger
+
+    def reset(self):
+        self.block = [0] * self.blockSize * self.blockSize
+        self.goldenData = [0] * self.blockSize * self.blockSize
+        self.rows = [0] * self.blockSize
+        self.columns = [0] * self.blockSize
+        self.broadcasted = Block(self.blockSize)
+
+    def checkRowsColumns(self, validators):
+        for val in validators:
+            if val.proposer == 0:
+                for r in val.rowIDs:
+                    self.rows[r] += 1
+                for c in val.columnIDs:
+                    self.columns[c] += 1
+
+        for i in range(self.blockSize):
+            self.logger.debug("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format)
+            if self.rows[i] == 0 or self.columns[i] == 0:
+                self.logger.warning("There is a row/column that has not been assigned", extra=self.format)
+
+    def setGoldenData(self, block):
+        for i in range(self.blockSize*self.blockSize):
+            self.goldenData[i] = block.data[i]
+
+    def checkBroadcasted(self):
+        zeros = 0
+        for i in range(self.blockSize * self.blockSize):
+            if self.broadcasted.data[i] == 0:
+                zeros += 1
+        if zeros > 0:
+            self.logger.debug("There are %d missing samples in the network" % zeros, extra=self.format)
+        return zeros
+

DAS/simulator.py

@@ -0,0 +1,91 @@
+#!/bin/python
+
+import logging
+from datetime import datetime
+from DAS.tools import *
+from DAS.observer import *
+from DAS.validator import *
+
+class Simulator:
+
+    chi = 8
+    blockSize = 256
+    numberValidators = 8192
+    failureRate = 0
+    proposerID = 0
+    logLevel = logging.INFO
+    deterministic = 0
+    validators = []
+    glob = []
+    logger = []
+    format = {}
+    steps = 0
+
+    def __init__(self, failureRate):
+        self.failureRate = failureRate
+        self.format = {"entity": "Simulator"}
+        self.steps = 0
+
+    def initValidators(self):
+        if not self.deterministic:
+            random.seed(datetime.now())
+        self.glob = Observer(self.blockSize, self.logger)
+        self.glob.reset()
+        self.validators = []
+        for i in range(self.numberValidators):
+            val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic, self.logger)
+            if i == self.proposerID:
+                val.initBlock()
+                self.glob.setGoldenData(val.block)
+            else:
+                val.logIDs()
+            self.validators.append(val)
+
+    def initLogger(self):
+        logger = logging.getLogger("DAS")
+        logger.setLevel(self.logLevel)
+        ch = logging.StreamHandler()
+        ch.setLevel(self.logLevel)
+        ch.setFormatter(CustomFormatter())
+        logger.addHandler(ch)
+        self.logger = logger
+
+    def resetFailureRate(self, failureRate):
+        self.failureRate = failureRate
+
+    def run(self):
+        self.glob.checkRowsColumns(self.validators)
+        self.validators[self.proposerID].broadcastBlock(self.glob.broadcasted)
+        missingSamples = self.glob.checkBroadcasted()
+        self.steps = 0
+        while(missingSamples > 0):
+            oldMissingSamples = missingSamples
+            self.logger.debug("Step %d:" % self.steps, extra=self.format)
+            for i in range(1,self.numberValidators):
+                self.validators[i].receiveRowsColumns(self.glob.broadcasted)
+                #Rows
+                self.validators[i].checkRestoreRows(self.glob.goldenData)
+                self.validators[i].sendRows(self.glob.broadcasted)
+                self.validators[i].logRows()
+                self.validators[i].logColumns()
+                # Columns
+                self.validators[i].checkRestoreColumns(self.glob.goldenData)
+                self.validators[i].sendColumns(self.glob.broadcasted)
+                self.validators[i].logRows()
+                self.validators[i].logColumns()
+
+            missingSamples = self.glob.checkBroadcasted()
+            if missingSamples == oldMissingSamples:
+                break
+            elif missingSamples == 0:
+                break
+            else:
+                self.steps += 1
+
+        if missingSamples == 0:
+            self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (self.steps, self.failureRate), extra=self.format)
+            return 0
+        else:
+            self.logger.debug("The block cannot be recovered, failure rate %d!" % self.failureRate, extra=self.format)
+            return 1
+

DAS/tools.py

@@ -0,0 +1,27 @@
+#!/bin/python3
+
+import logging
+
+class CustomFormatter(logging.Formatter):
+
+    blue = "\x1b[34;20m"
+    grey = "\x1b[38;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    format = "%(levelname)s : %(entity)s : %(message)s"
+
+    FORMATS = {
+        logging.DEBUG: grey + format + reset,
+        logging.INFO: blue + format + reset,
+        logging.WARNING: yellow + format + reset,
+        logging.ERROR: red + format + reset,
+        logging.CRITICAL: bold_red + format + reset
+    }
+
+    def format(self, record):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)
+

DAS/validator.py

@@ -0,0 +1,173 @@
+#!/bin/python3
+
+import random
+from DAS.block import *
+
+class Validator:
+
+    ID = 0
+    chi = 0
+    format = {}
+    blocksize = 0
+    block = []
+    rowIDs = []
+    columnIDs = []
+    rows = []
+    columns = []
+    proposer = 0
+    failureRate = 0
+    logger = []
+
+    def __init__(self, ID, chi, blockSize, proposer, failureRate, deterministic, logger):
+        FORMAT = "%(levelname)s : %(entity)s : %(message)s"
+        self.ID = ID
+        self.format = {"entity": "Val "+str(self.ID)}
+        self.blockSize = blockSize
+        self.proposer = proposer
+        self.failureRate = failureRate
+        self.logger = logger
+        if chi < 1:
+            self.logger.error("Chi has to be greater than 0", extra=self.format)
+        elif chi > blockSize:
+            self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format)
+        else:
+            self.chi = chi
+            self.rowIDs = []
+            self.columnIDs = []
+            if deterministic:
+                random.seed(self.ID)
+            lr = [i for i in range(self.blockSize)]
+            lc = [i for i in range(self.blockSize)]
+            random.shuffle(lr)
+            random.shuffle(lc)
+            for i in range(self.chi): # TODO : Avoid doubles
+                self.rowIDs.append(lr.pop())
+                self.columnIDs.append(lc.pop())
+
+    def logIDs(self):
+        if self.proposer == 1:
+            self.logger.warning("I am a block proposer."% self.ID)
+        else:
+            self.logger.debug("Selected rows: "+str(self.rowIDs), extra=self.format)
+            self.logger.debug("Selected columns: "+str(self.columnIDs), extra=self.format)
+
+    def initBlock(self):
+        self.logger.debug("I am a block proposer.", extra=self.format)
+        self.block = Block(self.blockSize)
+        self.block.fill()
+        #self.block.print()
+
+    def broadcastBlock(self, broadcasted):
+        if self.proposer == 0:
+            self.logger.error("I am NOT a block proposer", extra=self.format)
+        else:
+            self.logger.debug("Broadcasting my block...", extra=self.format)
+            tempBlock = self.block
+            order = [i for i in range(self.blockSize * self.blockSize)]
+            random.shuffle(order)
+            while(order):
+                i = order.pop()
+                if (random.randint(0,99) > self.failureRate):
+                    broadcasted.data[i] = self.block.data[i]
+            #broadcasted.print()
+
+    def getColumn(self, columnID, broadcasted):
+        column = [0] * self.blockSize
+        for i in range(self.blockSize):
+            column[i] = broadcasted.data[(i*self.blockSize)+columnID]
+        self.columns.append(column)
+
+    def getRow(self, rowID, broadcasted):
+        row = [0] * self.blockSize
+        for i in range(self.blockSize):
+            row[i] = broadcasted.data[(rowID*self.blockSize)+i]
+        self.rows.append(row)
+
+    def receiveRowsColumns(self, broadcasted):
+        self.rows = []
+        self.columns = []
+        if self.proposer == 1:
+            self.logger.error("I am a block proposer", extra=self.format)
+        else:
+            self.logger.debug("Receiving the data...", extra=self.format)
+            for r in self.rowIDs:
+                self.getRow(r, broadcasted)
+            for c in self.columnIDs:
+                self.getColumn(c, broadcasted)
+
+    def sendColumn(self, c, columnID, broadcasted):
+        column = [0] * self.blockSize
+        for i in range(self.blockSize):
+            if broadcasted.data[(i*self.blockSize)+columnID] == 0:
+                broadcasted.data[(i*self.blockSize)+columnID] = self.columns[c][i]
+
+    def sendRow(self, r, rowID, broadcasted):
+        for i in range(self.blockSize):
+            if broadcasted.data[(rowID*self.blockSize)+i] == 0:
+                broadcasted.data[(rowID*self.blockSize)+i] = self.rows[r][i]
+
+    def sendRows(self, broadcasted):
+        if self.proposer == 1:
+            self.logger.error("I am a block proposer", extra=self.format)
+        else:
+            self.logger.debug("Sending restored rows...", extra=self.format)
+            for r in range(len(self.rowIDs)):
+                self.sendRow(r, self.rowIDs[r], broadcasted)
+
+    def sendColumns(self, broadcasted):
+        if self.proposer == 1:
+            self.logger.error("I am a block proposer", extra=self.format)
+        else:
+            self.logger.debug("Sending restored columns...", extra=self.format)
+            for c in range(len(self.columnIDs)):
+                self.sendColumn(c, self.columnIDs[c], broadcasted)
+
+    def logRows(self):
+        self.logger.debug("Rows: "+str(self.rows), extra=self.format)
+
+    def logColumns(self):
+        self.logger.debug("Columns: "+str(self.columns), extra=self.format)
+
+    def checkRestoreRows(self, goldenData):
+        for rid in range(len(self.rows)):
+            row = self.rows[rid]
+            failures = 0
+            success = 0
+            for i in row:
+                if i == 0:
+                    failures += 1
+                elif i > 0 and i < 10:
+                    success += 1
+                else:
+                    self.logger.error("Data has been corrupted")
+
+            if failures > 0:
+                if success >= len(row)/2:
+                    for i in range(len(row)):
+                        self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i]
+                    self.logger.debug("%d samples restored in row %d" % (failures, self.rowIDs[rid]), extra=self.format )
+                else:
+                    self.logger.debug("Row %d cannot be restored" %  (self.rowIDs[rid]), extra=self.format)
+
+    def checkRestoreColumns(self, goldenData):
+        for cid in range(len(self.columns)):
+            column = self.columns[cid]
+            failures = 0
+            success = 0
+            for i in column:
+                if i == 0:
+                    failures += 1
+                elif i > 0 and i < 10:
+                    success += 1
+                else:
+                    self.logger.error("Data has been corrupted", extra=self.format)
+
+            if failures > 0:
+                if success >= len(column)/2:
+                    for i in range(len(column)):
+                        self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]]
+                    self.logger.debug("%d samples restored in column %d" % (failures, self.columnIDs[cid]), extra=self.format)
+                else:
+                    self.logger.debug("Column %d cannot be restored" % (self.columnIDs[cid]), extra=self.format)
+
+

simulator/DAS.py

@@ -1,241 +0,0 @@
-#! /bin/python3
-
-import random
-from datetime import datetime
-
-class Block:
-
-    blockSize = 0
-    data = []
-
-    def __init__(self, size):
-        self.blockSize = size
-        self.data = [0] * (self.blockSize*self.blockSize)
-
-    def fill(self):
-        for i in range(self.blockSize*self.blockSize):
-            self.data[i] = random.randint(1, 9)
-
-    def print(self):
-        for i in range(self.blockSize):
-            for j in range(self.blockSize):
-                print("%i" % self.data[(i*self.blockSize)+j], end="")
-            print("")
-
-
-class Validator:
-
-    ID = 0
-    chi = 0
-    blocksize = 0
-    block = []
-    rowIDs = []
-    columnIDs = []
-    rows = []
-    columns = []
-    proposer = 0
-    failureRate = 0
-
-    def __init__(self, ID, chi, blockSize, proposer, failureRate, deterministic):
-        self.ID = ID
-        self.blockSize = blockSize
-        self.proposer = proposer
-        self.failureRate = failureRate
-        if chi < 1:
-            print("ERROR: chi has to be greater than 0")
-        elif chi > blockSize:
-            print("ERROR: chi has to be smaller than %d" % blockSize)
-        else:
-            self.chi = chi
-            self.rowIDs = []
-            self.columnIDs = []
-            if deterministic:
-                random.seed(self.ID)
-            for i in range(self.chi):
-                self.rowIDs.append(random.randint(0,blockSize-1))
-                self.columnIDs.append(random.randint(0,blockSize-1))
-
-    def printIDs(self):
-        if self.proposer == 1:
-            print("Hi! I am validator %d and I am a block proposer."% self.ID)
-        else:
-            print("Hi! I am validator %d and these are my rows and columns."% self.ID)
-            print("Selected rows: ", end="")
-            for i in range(self.chi):
-                print("%d " % self.rowIDs[i], end="")
-            print("")
-            print("Selected columns: ", end="")
-            for i in range(self.chi):
-                print("%d " % self.columnIDs[i], end="")
-            print("")
-
-    def initBlock(self):
-        print("Hi! I am validator %d and I am a block proposer."% self.ID)
-        self.block = Block(self.blockSize)
-        self.block.fill()
-        self.block.print()
-
-    def broadcastBlock(self, broadcasted):
-        if self.proposer == 0:
-            print("ERROR: I am validator %d and I am NOT a block proposer" % self.ID)
-        else:
-            print("I am validator %d and I am broadcasting my block..." % self.ID)
-            tempBlock = self.block
-            order = [i for i in range(self.blockSize * self.blockSize)]
-            random.shuffle(order)
-            while(order):
-                i = order.pop()
-                if (random.randint(0,99) > self.failureRate):
-                    broadcasted.data[i] = self.block.data[i]
-            broadcasted.print()
-
-    def getColumn(self, columnID, broadcasted):
-        column = [0] * self.blockSize
-        for i in range(self.blockSize):
-            column[i] = broadcasted.data[(i*self.blockSize)+columnID]
-        self.columns.append(column)
-
-    def getRow(self, rowID, broadcasted):
-        row = [0] * self.blockSize
-        for i in range(self.blockSize):
-            row[i] = broadcasted.data[(rowID*self.blockSize)+i]
-        self.rows.append(row)
-
-    def receiveRowsColumns(self, broadcasted):
-        self.rows = []
-        self.columns = []
-        if self.proposer == 1:
-            print("ERROR: I am validator %d and I am a block proposer" % self.ID)
-        else:
-            print("I am validator %d and I am receiving the data..." % self.ID)
-            for r in self.rowIDs:
-                self.getRow(r, broadcasted)
-            for c in self.columnIDs:
-                self.getColumn(c, broadcasted)
-
-    def printRows(self):
-        print("Val %d - Rows: " % self.ID, end="")
-        print(self.rows)
-
-    def printColumns(self):
-        print("Val %d - Columns: " % self.ID, end="")
-        print(self.columns)
-
-    def checkRestoreRows(self, goldenData):
-        for rid in range(len(self.rows)):
-            row = self.rows[rid]
-            failures = 0
-            success = 0
-            for i in row:
-                if i == 0:
-                    failures += 1
-                elif i > 0 and i < 10:
-                    success += 1
-                else:
-                    print("ERROR: Data has been corrupted")
-
-            if failures > 0:
-                if success >= len(row)/2:
-                    for i in range(len(row)):
-                        self.rows[rid][i] = goldenData[(self.rowIDs[rid]*self.blockSize)+i]
-                    print("Val %d: Row %d data restored" % (self.ID, self.rowIDs[rid]))
-                else:
-                    print("WARNING Val %d: Row %d cannot be restored" %  (self.ID, self.rowIDs[rid]))
-
-    def checkRestoreColumns(self, goldenData):
-        for cid in range(len(self.columns)):
-            column = self.columns[cid]
-            failures = 0
-            success = 0
-            for i in column:
-                if i == 0:
-                    failures += 1
-                elif i > 0 and i < 10:
-                    success += 1
-                else:
-                    print("ERROR: Data has been corrupted")
-
-            if failures > 0:
-                if success >= len(column)/2:
-                    for i in range(len(column)):
-                        self.columns[cid][i] = goldenData[(i*self.blockSize)+self.columnIDs[cid]]
-                    print("Val %d: Column %d data restored" % (self.ID, self.columnIDs[cid]))
-                else:
-                    print("Val %d: Column %d cannot be restored" % (self.ID, self.columnIDs[cid]))
-
-
-class Observer:
-
-    block = []
-    blockSize = 0
-    rows = []
-    columns = []
-    goldenData = []
-
-    def __init__(self, blockSize):
-        self.blockSize = blockSize
-        self.block = [0] * self.blockSize * self.blockSize
-        self.rows = [0] * self.blockSize
-        self.columns = [0] * self.blockSize
-
-    def checkRowsColumns(self, validators):
-        for val in validators:
-            if val.proposer == 0:
-                for r in val.rowIDs:
-                    self.rows[r] += 1
-                for c in val.columnIDs:
-                    self.columns[c] += 1
-
-        for i in range(self.blockSize):
-            print("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]))
-            if self.rows[i] == 0 or self.columns[i] == 0:
-                print("WARNING: There is a row/column that has not been assigned")
-
-    def setGoldenData(self, block):
-        self.goldenData = [0] * self.blockSize * self.blockSize
-        for i in range(self.blockSize*self.blockSize):
-            self.goldenData[i] = block.data[i]
-
-class Simulator:
-
-    chi = 4
-    blockSize = 16
-    numberValidators = 32
-    failureRate = 10
-    proposerID = 0
-    deterministic = 1
-    validators = []
-    glob = []
-
-    def __init__(self):
-        if not self.deterministic:
-            random.seed(datetime.now())
-        self.glob = Observer(self.blockSize)
-        for i in range(self.numberValidators):
-            val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic)
-            if i == self.proposerID:
-                val.initBlock()
-                self.glob.setGoldenData(val.block)
-            else:
-                val.printIDs()
-            self.validators.append(val)
-
-    def run(self):
-        broadcasted = Block(self.blockSize)
-        self.glob.checkRowsColumns(self.validators)
-        self.validators[self.proposerID].broadcastBlock(broadcasted)
-        for i in range(1,self.numberValidators):
-            self.validators[i].receiveRowsColumns(broadcasted)
-            self.validators[i].printRows()
-            self.validators[i].printColumns()
-            self.validators[i].checkRestoreRows(self.glob.goldenData)
-            self.validators[i].checkRestoreColumns(self.glob.goldenData)
-            self.validators[i].printRows()
-            self.validators[i].printColumns()
-
-
-sim = Simulator()
-sim.run()
-
-
-

study.py

@@ -0,0 +1,35 @@
+#! /bin/python3
+
+import time
+from DAS import *
+
+
+def study():
+    sim = Simulator(0)
+    sim.initLogger()
+    maxTries = 2
+    step = 25
+    frRange = []
+    resultRange = []
+    simCnt = 0
+    sim.logger.info("Starting simulations:", extra=sim.format)
+    start = time.time()
+    for fr in range(0, 100, step):
+        if fr % 10 == 0:
+            sim.logger.info("Failure rate %d %% ..." % fr, extra=sim.format)
+        sim.resetFailureRate(fr)
+        result = 0
+        for i in range(maxTries):
+            sim.initValidators()
+            result += sim.run()
+            simCnt += 1
+        frRange.append(fr)
+        resultRange.append(100-result)
+    end = time.time()
+    sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format)
+    for i in range(len(frRange)):
+        sim.logger.info("For failure rate of %d we got %d %% success rate in DAS!" % (frRange[i], resultRange[i]), extra=sim.format)
+
+
+study()
+