Merge pull request #8 from status-im/config

Adding configuration and shape classes

DAS/__init__.py

@@ -1 +1,3 @@
 from DAS.simulator import *
+from DAS.configuration import *
+from DAS.shape import *

DAS/block.py

@@ -9,8 +9,8 @@ class Block:
     blockSize = 0
     data = bitarray()
 
-    def __init__(self, size):
+    def __init__(self, blockSize):
-        self.blockSize = size
+        self.blockSize = blockSize
         self.data = zeros(self.blockSize*self.blockSize)
 
     def fill(self):

DAS/configuration.py

@@ -0,0 +1,48 @@
+#!/bin/python3
+
+import configparser
+
+class Configuration:
+
+    deterministic = 0
+
+    def __init__(self, fileName):
+
+        config = configparser.RawConfigParser()
+        config.read(fileName)
+
+        self.nvStart = int(config.get("Simulation Space", "numberValidatorStart"))
+        self.nvStop = int(config.get("Simulation Space", "numberValidatorStop"))
+        self.nvStep = int(config.get("Simulation Space", "numberValidatorStep"))
+
+        self.blockSizeStart = int(config.get("Simulation Space", "blockSizeStart"))
+        self.blockSizeStop = int(config.get("Simulation Space", "blockSizeStop"))
+        self.blockSizeStep = int(config.get("Simulation Space", "blockSizeStep"))
+
+        self.netDegreeStart = int(config.get("Simulation Space", "netDegreeStart"))
+        self.netDegreeStop = int(config.get("Simulation Space", "netDegreeStop"))
+        self.netDegreeStep = int(config.get("Simulation Space", "netDegreeStep"))
+
+        self.failureRateStart = int(config.get("Simulation Space", "failureRateStart"))
+        self.failureRateStop = int(config.get("Simulation Space", "failureRateStop"))
+        self.failureRateStep = int(config.get("Simulation Space", "failureRateStep"))
+
+        self.chiStart = int(config.get("Simulation Space", "chiStart"))
+        self.chiStop = int(config.get("Simulation Space", "chiStop"))
+        self.chiStep = int(config.get("Simulation Space", "chiStep"))
+
+        self.numberRuns = int(config.get("Advanced", "numberRuns"))
+        self.deterministic = config.get("Advanced", "deterministic")
+
+        if self.nvStop < (self.blockSizeStart*4):
+            print("ERROR: The number of validators cannot be lower than the block size * 4")
+            exit(1)
+        if self.chiStart < 1:
+            print("Chi has to be greater than 0")
+            exit(1)
+        if self.chiStop > self.blockSizeStart:
+            print("Chi (%d) has to be smaller or equal to block the size (%d)" % (self.chiStop, self.blockSizeStart))
+            exit(1)
+
+
+

DAS/observer.py

@@ -5,40 +5,40 @@ from DAS.block import *
 class Observer:
 
     block = []
-    blockSize = 0
     rows = []
     columns = []
     goldenData = []
     broadcasted = []
+    config = []
     logger = []
 
-    def __init__(self, blockSize, logger):
+    def __init__(self, logger, config):
+        self.config = config
         self.format = {"entity": "Observer"}
-        self.blockSize = blockSize
         self.logger = logger
 
     def reset(self):
-        self.block = [0] * self.blockSize * self.blockSize
+        self.block = [0] * self.config.blockSize * self.config.blockSize
-        self.goldenData = [0] * self.blockSize * self.blockSize
+        self.goldenData = [0] * self.config.blockSize * self.config.blockSize
-        self.rows = [0] * self.blockSize
+        self.rows = [0] * self.config.blockSize
-        self.columns = [0] * self.blockSize
+        self.columns = [0] * self.config.blockSize
-        self.broadcasted = Block(self.blockSize)
+        self.broadcasted = Block(self.config.blockSize)
 
     def checkRowsColumns(self, validators):
         for val in validators:
-            if val.proposer == 0:
+            if val.amIproposer == 0:
                 for r in val.rowIDs:
                     self.rows[r] += 1
                 for c in val.columnIDs:
                     self.columns[c] += 1
 
-        for i in range(self.blockSize):
+        for i in range(self.config.blockSize):
             self.logger.debug("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format)
             if self.rows[i] == 0 or self.columns[i] == 0:
                 self.logger.warning("There is a row/column that has not been assigned", extra=self.format)
 
     def setGoldenData(self, block):
-        for i in range(self.blockSize*self.blockSize):
+        for i in range(self.config.blockSize*self.config.blockSize):
             self.goldenData[i] = block.data[i]
 
     def checkBroadcasted(self):
@@ -54,7 +54,7 @@ class Observer:
         arrived = 0
         expected = 0
         for val in validators:
-            if val.proposer == 0:
+            if val.amIproposer == 0:
                 (a, e) = val.checkStatus()
                 arrived += a
                 expected += e

DAS/results.py

@@ -0,0 +1,17 @@
+#!/bin/python3
+
+
+class Result:
+
+    config = []
+    missingVector = []
+    blockAvailable = -1
+
+    def __init__(self, config):
+        self.config = config
+        self.blockAvailable = -1
+        self.missingVector = []
+
+
+    def addMissing(self, missingVector):
+        self.missingVector = missingVector

DAS/shape.py

@@ -0,0 +1,19 @@
+#!/bin/python3
+
+class Shape:
+    numberValidators = 0
+    failureRate = 0
+    blockSize = 0
+    netDegree = 0
+    chi = 0
+
+    def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree):
+        self.numberValidators = numberValidators
+        self.failureRate = failureRate
+        self.blockSize = blockSize
+        self.netDegree = netDegree
+        self.chi = chi
+
+
+
+

DAS/simulator.py

@@ -1,40 +1,39 @@
 #!/bin/python
 
 import networkx as nx
-import logging
+import logging, random
 from datetime import datetime
 from DAS.tools import *
+from DAS.results import *
 from DAS.observer import *
 from DAS.validator import *
 
 class Simulator:
 
-    chi = 8
-    blockSize = 256
-    numberValidators = 8192
-    failureRate = 0
     proposerID = 0
     logLevel = logging.INFO
-    deterministic = 0
     validators = []
     glob = []
+    result = []
+    shape = []
     logger = []
     format = {}
-    steps = 0
 
-    def __init__(self, failureRate):
+    def __init__(self, shape):
-        self.failureRate = failureRate
+        self.shape = shape
         self.format = {"entity": "Simulator"}
-        self.steps = 0
+        self.result = Result(self.shape)
 
     def initValidators(self):
-        if not self.deterministic:
+        self.glob = Observer(self.logger, self.shape)
-            random.seed(datetime.now())
-        self.glob = Observer(self.blockSize, self.logger)
         self.glob.reset()
         self.validators = []
-        for i in range(self.numberValidators):
+        rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize)
-            val = Validator(i, self.chi, self.blockSize, int(not i!=0), self.failureRate, self.deterministic, self.logger)
+        columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize)
+        random.shuffle(rows)
+        random.shuffle(columns)
+        for i in range(self.shape.numberValidators):
+            val = Validator(i, int(not i!=0), self.logger, self.shape, rows, columns)
             if i == self.proposerID:
                 val.initBlock()
                 self.glob.setGoldenData(val.block)
@@ -42,27 +41,34 @@ class Simulator:
                 val.logIDs()
             self.validators.append(val)
 
-    def initNetwork(self, d=6):
+    def initNetwork(self):
-        rowChannels = [[] for i in range(self.blockSize)]
+        self.shape.netDegree = 6
-        columnChannels = [[] for i in range(self.blockSize)]
+        rowChannels = [[] for i in range(self.shape.blockSize)]
+        columnChannels = [[] for i in range(self.shape.blockSize)]
         for v in self.validators:
             for id in v.rowIDs:
                 rowChannels[id].append(v)
             for id in v.columnIDs:
                 columnChannels[id].append(v)
 
-        for id in range(self.blockSize):
+        for id in range(self.shape.blockSize):
-            G = nx.random_regular_graph(d, len(rowChannels[id]))
+
+            if (len(rowChannels[id]) < self.shape.netDegree):
+                self.logger.error("Graph degree higher than %d" % len(rowChannels[id]), extra=self.format)
+            G = nx.random_regular_graph(self.shape.netDegree, len(rowChannels[id]))
             if not nx.is_connected(G):
-                self.logger.error("graph not connected for row %d !" % id, extra=self.format)
+                self.logger.error("Graph not connected for row %d !" % id, extra=self.format)
             for u, v in G.edges:
                 val1=rowChannels[id][u]
                 val2=rowChannels[id][v]
                 val1.rowNeighbors[id].append(val2)
                 val2.rowNeighbors[id].append(val1)
-            G = nx.random_regular_graph(d, len(columnChannels[id]))
+
+            if (len(columnChannels[id]) < self.shape.netDegree):
+                self.logger.error("Graph degree higher than %d" % len(columnChannels[id]), extra=self.format)
+            G = nx.random_regular_graph(self.shape.netDegree, len(columnChannels[id]))
             if not nx.is_connected(G):
-                self.logger.error("graph not connected for column %d !" % id, extra=self.format)
+                self.logger.error("Graph not connected for column %d !" % id, extra=self.format)
             for u, v in G.edges:
                 val1=columnChannels[id][u]
                 val2=columnChannels[id][v]
@@ -78,20 +84,27 @@ class Simulator:
         logger.addHandler(ch)
         self.logger = logger
 
-    def resetFailureRate(self, failureRate):
+
-        self.failureRate = failureRate
+    def resetShape(self, shape):
+        self.shape = shape
+        for val in self.validators:
+            val.shape.failureRate = shape.failureRate
+            val.shape.chi = shape.chi
+
 
     def run(self):
         self.glob.checkRowsColumns(self.validators)
         self.validators[self.proposerID].broadcastBlock()
         arrived, expected = self.glob.checkStatus(self.validators)
         missingSamples = expected - arrived
-        self.steps = 0
+        missingVector = []
+        steps = 0
         while(missingSamples > 0):
+            missingVector.append(missingSamples)
             oldMissingSamples = missingSamples
-            for i in range(1,self.numberValidators):
+            for i in range(1,self.shape.numberValidators):
                 self.validators[i].receiveRowsColumns()
-            for i in range(1,self.numberValidators):
+            for i in range(1,self.shape.numberValidators):
                 self.validators[i].restoreRows()
                 self.validators[i].restoreColumns()
                 self.validators[i].sendRows()
@@ -102,18 +115,21 @@ class Simulator:
             arrived, expected = self.glob.checkStatus(self.validators)
             missingSamples = expected - arrived
             missingRate = missingSamples*100/expected
-            self.logger.info("step %d, missing %d of %d (%0.02f %%)" % (self.steps, missingSamples, expected, missingRate), extra=self.format)
+            self.logger.debug("step %d, missing %d of %d (%0.02f %%)" % (steps, missingSamples, expected, missingRate), extra=self.format)
             if missingSamples == oldMissingSamples:
                 break
             elif missingSamples == 0:
                 break
             else:
-                self.steps += 1
+                steps += 1
 
+        self.result.addMissing(missingVector)
         if missingSamples == 0:
-            self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (self.steps, self.failureRate), extra=self.format)
+            self.result.blockAvailable = 1
-            return 0
+            self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.shape.failureRate), extra=self.format)
+            return self.result
         else:
-            self.logger.debug("The block cannot be recovered, failure rate %d!" % self.failureRate, extra=self.format)
+            self.result.blockAvailable = 0
-            return 1
+            self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format)
+            return self.result
 

DAS/validator.py

@@ -10,44 +10,40 @@ from bitarray.util import zeros
 class Validator:
 
     ID = 0
-    chi = 0
+    amIproposer = 0
+    shape = []
     format = {}
-    blocksize = 0
-    proposer = 0
-    failureRate = 0
     logger = []
 
-    def __init__(self, ID, chi, blockSize, proposer, failureRate, deterministic, logger):
+    def __init__(self, ID, amIproposer, logger, shape, rows, columns):
+        self.shape = shape
         FORMAT = "%(levelname)s : %(entity)s : %(message)s"
         self.ID = ID
         self.format = {"entity": "Val "+str(self.ID)}
-        self.blockSize = blockSize
+        self.block = Block(self.shape.blockSize)
-        self.block = Block(blockSize)
+        self.receivedBlock = Block(self.shape.blockSize)
-        self.receivedBlock = Block(blockSize)
+        self.amIproposer = amIproposer
-        self.proposer = proposer
-        self.failureRate = failureRate
         self.logger = logger
-        if chi < 1:
+        if self.shape.chi < 1:
             self.logger.error("Chi has to be greater than 0", extra=self.format)
-        elif chi > blockSize:
+        elif self.shape.chi > self.shape.blockSize:
             self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format)
         else:
-            self.chi = chi
+            if amIproposer:
-            if proposer:
+                self.rowIDs = range(shape.blockSize)
-                self.rowIDs = range(blockSize)
+                self.columnIDs = range(shape.blockSize)
-                self.columnIDs = range(blockSize)
             else:
-                self.rowIDs = []
+                self.rowIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)]
-                self.columnIDs = []
+                self.columnIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)]
-                if deterministic:
+                #if shape.deterministic:
-                    random.seed(self.ID)
+                #    random.seed(self.ID)
-                self.rowIDs = random.sample(range(self.blockSize), self.chi)
+                #self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi)
-                self.columnIDs = random.sample(range(self.blockSize), self.chi)
+                #self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi)
         self.rowNeighbors = collections.defaultdict(list)
         self.columnNeighbors = collections.defaultdict(list)
 
     def logIDs(self):
-        if self.proposer == 1:
+        if self.amIproposer == 1:
             self.logger.warning("I am a block proposer."% self.ID)
         else:
             self.logger.debug("Selected rows: "+str(self.rowIDs), extra=self.format)
@@ -55,30 +51,30 @@ class Validator:
 
     def initBlock(self):
         self.logger.debug("I am a block proposer.", extra=self.format)
-        self.block = Block(self.blockSize)
+        self.block = Block(self.shape.blockSize)
         self.block.fill()
         #self.block.print()
 
     def broadcastBlock(self):
-        if self.proposer == 0:
+        if self.amIproposer == 0:
             self.logger.error("I am NOT a block proposer", extra=self.format)
         else:
             self.logger.debug("Broadcasting my block...", extra=self.format)
-            order = [i for i in range(self.blockSize * self.blockSize)]
+            order = [i for i in range(self.shape.blockSize * self.shape.blockSize)]
             random.shuffle(order)
             while(order):
                 i = order.pop()
-                if (random.randint(0,99) >= self.failureRate):
+                if (random.randint(0,99) >= self.shape.failureRate):
                     self.block.data[i] = 1
                 else:
                     self.block.data[i] = 0
             nbFailures = self.block.data.count(0)
-            measuredFailureRate = nbFailures * 100 / (self.blockSize * self.blockSize)
+            measuredFailureRate = nbFailures * 100 / (self.shape.blockSize * self.shape.blockSize)
-            self.logger.info("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format)
+            self.logger.debug("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format)
             #broadcasted.print()
-            for id in range(self.blockSize):
+            for id in range(self.shape.blockSize):
                 self.sendColumn(id)
-            for id in range(self.blockSize):
+            for id in range(self.shape.blockSize):
                 self.sendRow(id)
 
     def getColumn(self, index):
@@ -101,7 +97,7 @@ class Validator:
 
 
     def receiveRowsColumns(self):
-        if self.proposer == 1:
+        if self.amIproposer == 1:
             self.logger.error("I am a block proposer", extra=self.format)
         else:
             self.logger.debug("Receiving the data...", extra=self.format)
@@ -124,7 +120,7 @@ class Validator:
                 n.receiveRow(rowID, line)
 
     def sendRows(self):
-        if self.proposer == 1:
+        if self.amIproposer == 1:
             self.logger.error("I am a block proposer", extra=self.format)
         else:
             self.logger.debug("Sending restored rows...", extra=self.format)
@@ -132,7 +128,7 @@ class Validator:
                 self.sendRow(r)
 
     def sendColumns(self):
-        if self.proposer == 1:
+        if self.amIproposer == 1:
             self.logger.error("I am a block proposer", extra=self.format)
         else:
             self.logger.debug("Sending restored columns...", extra=self.format)

README.md

@@ -1,6 +1,6 @@
-# DAS Research 
+# DAS Research
 
-This repository hosts all the research on DAS for the collaboration between Codex and the EF. 
+This repository hosts all the research on DAS for the collaboration between Codex and the EF.
 
 ## Prepare the environment
 
@@ -16,11 +16,11 @@ $ cd das-research
 ```
 $ python3 -m venv myenv
 $ source myenv/bin/activate
-$ pip3 install -r DAS/requeriments.txt
+$ pip3 install -r DAS/requirements.txt
 ```
 
 ## Run the simulator
 
 ```
-$ python3 study.py
+$ python3 study.py config.das
 ```

config.das

@@ -0,0 +1,27 @@
+[Simulation Space]
+
+numberValidatorStart = 256
+numberValidatorStop  = 512
+numberValidatorStep  = 128
+
+failureRateStart = 10
+failureRateStop  = 90
+failureRateStep  = 40
+
+blockSizeStart = 32
+blockSizeStop  = 64
+blockSizeStep  = 16
+
+netDegreeStart = 6
+netDegreeStop  = 8
+netDegreeStep  = 1
+
+chiStart = 4
+chiStop  = 8
+chiStep  = 2
+
+
+[Advanced]
+
+deterministic = 0
+numberRuns = 2

study.py

@@ -1,35 +1,45 @@
 #! /bin/python3
 
-import time
+import time, sys
 from DAS import *
 
 
 def study():
-    sim = Simulator(0)
+    if len(sys.argv) < 2:
+        print("You need to pass a configuration file in parameter")
+        exit(1)
+
+    config = Configuration(sys.argv[1])
+    sim = Simulator(config)
     sim.initLogger()
-    maxTries = 10
+    results = []
-    step = 20
-    frRange = []
-    resultRange = []
     simCnt = 0
+
     sim.logger.info("Starting simulations:", extra=sim.format)
     start = time.time()
-    for fr in range(0, 100, step):
+
-        if fr % 10 == 0:
+    for run in range(config.numberRuns):
-            sim.logger.info("Failure rate %d %% ..." % fr, extra=sim.format)
+        for fr in range(config.failureRateStart, config.failureRateStop+1, config.failureRateStep):
-        sim.resetFailureRate(fr)
+            for chi in range(config.chiStart, config.chiStop+1, config.chiStep):
-        result = 0
+                for blockSize in range(config.blockSizeStart, config.blockSizeStop+1, config.blockSizeStep):
-        for i in range(maxTries):
+                    for nv in range(config.nvStart, config.nvStop+1, config.nvStep):
-            sim.initValidators()
+                        for netDegree in range(config.netDegreeStart, config.netDegreeStop+1, config.netDegreeStep):
-            sim.initNetwork()
+
-            result += sim.run()
+                            if not config.deterministic:
-            simCnt += 1
+                                random.seed(datetime.now())
-        frRange.append(fr)
+
-        resultRange.append((maxTries-result)*100/maxTries)
+                            shape = Shape(blockSize, nv, fr, chi, netDegree)
+                            sim.resetShape(shape)
+                            sim.initValidators()
+                            sim.initNetwork()
+                            result = sim.run()
+                            sim.logger.info("Run %d, FR: %d %%, Chi: %d, BlockSize: %d, Nb.Val: %d, netDegree: %d ... Block Available: %d" % (run, fr, chi, blockSize, nv, netDegree, result.blockAvailable), extra=sim.format)
+                            results.append(result)
+                            simCnt += 1
+
     end = time.time()
     sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format)
-    for i in range(len(frRange)):
+
-        sim.logger.info("For failure rate of %d we got %d %% success rate in DAS!" % (frRange[i], resultRange[i]), extra=sim.format)
 
 
 study()