Merge pull request #22 from status-im/twoClasses

Two classes

DAS/shape.py

@@ -3,23 +3,35 @@
 class Shape:
     """This class represents a set of parameters for a specific simulation."""
 
-    def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, run):
+    def __init__(self, blockSize, numberNodes, failureRate, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run):
         """Initializes the shape with the parameters passed in argument."""
         self.run = run
-        self.numberValidators = numberValidators
+        self.numberNodes = numberNodes
         self.blockSize = blockSize
         self.failureRate = failureRate
         self.netDegree = netDegree
+        self.class1ratio = class1ratio
         self.chi = chi
+        self.vpn1 = vpn1
+        self.vpn2 = vpn2
+        self.bwUplinkProd = bwUplinkProd
+        self.bwUplink1 = bwUplink1
+        self.bwUplink2 = bwUplink2
         self.randomSeed = ""
 
     def __repr__(self):
         """Returns a printable representation of the shape"""
         shastr = ""
         shastr += "bs-"+str(self.blockSize)
-        shastr += "-nbv-"+str(self.numberValidators)
+        shastr += "-nn-"+str(self.numberNodes)
         shastr += "-fr-"+str(self.failureRate)
+        shastr += "-c1r-"+str(self.class1ratio)
         shastr += "-chi-"+str(self.chi)
+        shastr += "-vpn1-"+str(self.vpn1)
+        shastr += "-vpn2-"+str(self.vpn2)
+        shastr += "-bwupprod-"+str(self.bwUplinkProd)
+        shastr += "-bwup1-"+str(self.bwUplink1)
+        shastr += "-bwup2-"+str(self.bwUplink2)
         shastr += "-nd-"+str(self.netDegree)
         shastr += "-r-"+str(self.run)
         return shastr

DAS/simulator.py

@@ -15,6 +15,7 @@ class Simulator:
     def __init__(self, shape, config):
         """It initializes the simulation with a set of parameters (shape)."""
         self.shape = shape
+        self.config = config
         self.format = {"entity": "Simulator"}
         self.result = Result(self.shape)
         self.validators = []
@@ -28,12 +29,18 @@ class Simulator:
         self.glob = Observer(self.logger, self.shape)
         self.glob.reset()
         self.validators = []
-        rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize)
+        if self.config.evenLineDistribution:
-        columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize)
+            rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberNodes/self.shape.blockSize)
-        random.shuffle(rows)
+            columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberNodes/self.shape.blockSize)
-        random.shuffle(columns)
+            random.shuffle(rows)
-        for i in range(self.shape.numberValidators):
+            random.shuffle(columns)
-            val = Validator(i, int(not i!=0), self.logger, self.shape, rows, columns)
+        for i in range(self.shape.numberNodes):
+            if self.config.evenLineDistribution:
+                val = Validator(i, int(not i!=0), self.logger, self.shape,
+                            rows[(i*self.shape.chi):((i+1)*self.shape.chi)],
+                            columns[(i*self.shape.chi):((i+1)*self.shape.chi)])
+            else:
+                val = Validator(i, int(not i!=0), self.logger, self.shape)
             if i == self.proposerID:
                 val.initBlock()
                 self.glob.setGoldenData(val.block)
@@ -57,7 +64,10 @@ class Simulator:
             # If the number of nodes in a channel is smaller or equal to the
             # requested degree, a fully connected graph is used. For n>d, a random
             # d-regular graph is set up. (For n=d+1, the two are the same.)
-            if (len(rowChannels[id]) <= self.shape.netDegree):
+            if not rowChannels[id]:
+                self.logger.error("No nodes for row %d !" % id, extra=self.format)
+                continue
+            elif (len(rowChannels[id]) <= self.shape.netDegree):
                 self.logger.debug("Graph fully connected with degree %d !" % (len(rowChannels[id]) - 1), extra=self.format)
                 G = nx.complete_graph(len(rowChannels[id]))
             else:
@@ -70,7 +80,10 @@ class Simulator:
                 val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, 0, self.shape.blockSize)})
                 val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, 0, self.shape.blockSize)})
 
-            if (len(columnChannels[id]) <= self.shape.netDegree):
+            if not columnChannels[id]:
+                self.logger.error("No nodes for column %d !" % id, extra=self.format)
+                continue
+            elif (len(columnChannels[id]) <= self.shape.netDegree):
                 self.logger.debug("Graph fully connected with degree %d !" % (len(columnChannels[id]) - 1), extra=self.format)
                 G = nx.complete_graph(len(columnChannels[id]))
             else:
@@ -97,7 +110,7 @@ class Simulator:
                         v.columnNeighbors[id].update({vi.ID : Neighbor(vi, 1, self.shape.blockSize)})
 
         if self.logger.isEnabledFor(logging.DEBUG):
-            for i in range(0, self.shape.numberValidators):
+            for i in range(0, self.shape.numberNodes):
                 self.logger.debug("Val %d : rowN %s", i, self.validators[i].rowNeighbors, extra=self.format)
                 self.logger.debug("Val %d : colN %s", i, self.validators[i].columnNeighbors, extra=self.format)
 
@@ -120,6 +133,8 @@ class Simulator:
         for val in self.validators:
             val.shape.failureRate = shape.failureRate
             val.shape.chi = shape.chi
+            val.shape.vpn1 = shape.vpn1
+            val.shape.vpn2 = shape.vpn2
 
         # In GossipSub the initiator might push messages without participating in the mesh.
         # proposerPublishOnly regulates this behavior. If set to true, the proposer is not
@@ -146,17 +161,17 @@ class Simulator:
             missingVector.append(missingSamples)
             oldMissingSamples = missingSamples
             self.logger.debug("PHASE SEND %d" % steps, extra=self.format)
-            for i in range(0,self.shape.numberValidators):
+            for i in range(0,self.shape.numberNodes):
                 self.validators[i].send()
             self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format)
-            for i in range(1,self.shape.numberValidators):
+            for i in range(1,self.shape.numberNodes):
                 self.validators[i].receiveRowsColumns()
             self.logger.debug("PHASE RESTORE %d" % steps, extra=self.format)
-            for i in range(1,self.shape.numberValidators):
+            for i in range(1,self.shape.numberNodes):
                 self.validators[i].restoreRows()
                 self.validators[i].restoreColumns()
             self.logger.debug("PHASE LOG %d" % steps, extra=self.format)
-            for i in range(0,self.shape.numberValidators):
+            for i in range(0,self.shape.numberNodes):
                 self.validators[i].logRows()
                 self.validators[i].logColumns()
 
@@ -167,7 +182,7 @@ class Simulator:
                 (steps, statsTxInSlot[0], statsRxInSlot[0],
                  mean(statsTxInSlot[1:]), max(statsTxInSlot[1:]),
                  mean(statsRxInSlot[1:]), max(statsRxInSlot[1:])), extra=self.format)
-            for i in range(0,self.shape.numberValidators):
+            for i in range(0,self.shape.numberNodes):
                 self.validators[i].updateStats()
 
             arrived, expected = self.glob.checkStatus(self.validators)

DAS/tools.py

@@ -79,3 +79,9 @@ def sampleLine(line, limit):
                         r[i] = 1
                         limit -= 1
                 return r
+
+def unionOfSamples(population, sampleSize, times):
+    selected = set()
+    for t in range(times):
+        selected |= set(random.sample(population, sampleSize))
+    return selected

DAS/validator.py

@@ -4,7 +4,7 @@ import random
 import collections
 import logging
 from DAS.block import *
-from DAS.tools import shuffled, shuffledDict
+from DAS.tools import shuffled, shuffledDict, unionOfSamples
 from bitarray.util import zeros
 from collections import deque
 from itertools import chain
@@ -38,8 +38,13 @@ class Validator:
         """It returns the validator ID."""
         return str(self.ID)
 
-    def __init__(self, ID, amIproposer, logger, shape, rows, columns):
+    def __init__(self, ID, amIproposer, logger, shape, rows = None, columns = None):
-        """It initializes the validator with the logger, shape and assigned rows/columns."""
+        """It initializes the validator with the logger shape and rows/columns.
+
+            If rows/columns are specified these are observed, otherwise (default)
+            chi rows and columns are selected randomly.
+        """
+
         self.shape = shape
         FORMAT = "%(levelname)s : %(entity)s : %(message)s"
         self.ID = ID
@@ -53,18 +58,17 @@ class Validator:
         if self.shape.chi < 1:
             self.logger.error("Chi has to be greater than 0", extra=self.format)
         elif self.shape.chi > self.shape.blockSize:
-            self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format)
+            self.logger.error("Chi has to be smaller than %d" % self.shape.blockSize, extra=self.format)
         else:
             if amIproposer:
                 self.rowIDs = range(shape.blockSize)
                 self.columnIDs = range(shape.blockSize)
             else:
-                self.rowIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)]
-                self.columnIDs = columns[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)]
                 #if shape.deterministic:
                 #    random.seed(self.ID)
-                #self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi)
+                vpn = self.shape.vpn1 if (self.ID <= shape.numberNodes * shape.class1ratio) else self.shape.vpn2
-                #self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi)
+                self.rowIDs = rows if rows else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn)
+                self.columnIDs = columns if columns else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn)
         self.rowNeighbors = collections.defaultdict(dict)
         self.columnNeighbors = collections.defaultdict(dict)
 
@@ -77,7 +81,12 @@ class Validator:
         # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?)
         # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11
         # TODO: this should be a parameter
-        self.bwUplink = 110 if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps
+        if self.amIproposer:
+            self.bwUplink = shape.bwUplinkProd
+        elif self.ID <= shape.numberNodes * shape.class1ratio:
+            self.bwUplink = shape.bwUplink1
+        else:
+            self.bwUplink = shape.bwUplink2
 
         self.repairOnTheFly = True
         self.sendLineUntil = (self.shape.blockSize + 1) // 2 # stop sending on a p2p link if at least this amount of samples passed

DAS/visualizer.py

@@ -12,7 +12,8 @@ class Visualizer:
     def __init__(self, execID):
         self.execID = execID
         self.folderPath = "results/"+self.execID
-        self.parameters = ['run', 'blockSize', 'failureRate', 'numberValidators', 'netDegree', 'chi']
+        self.parameters = ['run', 'blockSize', 'failureRate', 'numberNodes', 'netDegree',
+                           'chi', 'vpn1', 'vpn2', 'bwUplinkProd', 'bwUplink1', 'bwUplink2']
         self.minimumDataPoints = 2
 
     def plottingData(self):
@@ -27,22 +28,27 @@ class Visualizer:
                 run = int(root.find('run').text)
                 blockSize = int(root.find('blockSize').text)
                 failureRate = int(root.find('failureRate').text)
-                numberValidators = int(root.find('numberValidators').text)
+                numberNodes = int(root.find('numberNodes').text)
                 netDegree = int(root.find('netDegree').text)
                 chi = int(root.find('chi').text)
+                vpn1 = int(root.find('vpn1').text)
+                vpn2 = int(root.find('vpn2').text)
+                bwUplinkProd = int(root.find('bwUplinkProd').text)
+                bwUplink1 = int(root.find('bwUplink1').text)
+                bwUplink2 = int(root.find('bwUplink2').text)
                 tta = int(root.find('tta').text)
 
                 # Loop over all possible combinations of length 4 of the parameters
                 for combination in combinations(self.parameters, 4):
                     # Get the indices and values of the parameters in the combination
                     indices = [self.parameters.index(element) for element in combination]
-                    selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi]
+                    selectedValues = [run, blockSize, failureRate, numberNodes, netDegree, chi, vpn1, vpn2, bwUplinkProd, bwUplink1, bwUplink2]
                     values = [selectedValues[index] for index in indices]
                     names = [self.parameters[i] for i in indices]
                     keyComponents = [f"{name}_{value}" for name, value in zip(names, values)]
                     key = tuple(keyComponents[:4])
                     #Get the names of the other 2 parameters that are not included in the key
-                    otherParams = [self.parameters[i] for i in range(6) if i not in indices]
+                    otherParams = [self.parameters[i] for i in range(len(self.parameters)) if i not in indices]
                     #Append the values of the other 2 parameters and the ttas to the lists for the key
                     otherIndices = [i for i in range(len(self.parameters)) if i not in indices]
 

config_example.py

@@ -14,6 +14,8 @@ if needed.
 """
 
 import logging
+import itertools
+import numpy as np
 from DAS.shape import Shape
 
 dumpXML = 1
@@ -24,11 +26,15 @@ logLevel = logging.INFO
 # for more details, see joblib.Parallel
 numJobs = 3
 
+# distribute rows/columns evenly between validators (True)
+# or generate it using local randomness (False)
+evenLineDistribution = False
+
 # Number of simulation runs with the same parameters for statistical relevance
 runs = range(10)
 
 # Number of validators
-numberValidators = range(256, 513, 128)
+numberNodes = range(256, 513, 128)
 
 # Percentage of block not released by producer
 failureRates = range(10, 91, 40)
@@ -39,8 +45,21 @@ blockSizes = range(32,65,16)
 # Per-topic mesh neighborhood size
 netDegrees = range(6, 9, 2)
 
-# Number of rows and columns a validator is interested in
+# number of rows and columns a validator is interested in
-chis = range(4, 9, 2)
+chis = range(1, 5, 2)
+
+# ratio of class1 nodes (see below for parameters per class)
+class1ratios = np.arange(0, 1, .2)
+
+# Number of validators per beacon node
+validatorsPerNode1 = [1]
+validatorsPerNode2 = [2, 4, 8, 16, 32]
+
+# Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?)
+# 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11
+bwUplinksProd = [2200]
+bwUplinks1 = [110]
+bwUplinks2 = [2200]
 
 # Set to True if you want your run to be deterministic, False if not
 deterministic = False
@@ -49,13 +68,9 @@ deterministic = False
 randomSeed = "DAS"
 
 def nextShape():
-    for run in runs:
+    for run, fr, class1ratio, chi, vpn1, vpn2, blockSize, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product(
-        for fr in failureRates:
+        runs, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2):
-            for chi in chis:
+        # Network Degree has to be an even number
-                for blockSize in blockSizes:
+        if netDegree % 2 == 0:
-                    for nv in numberValidators:
+            shape = Shape(blockSize, nn, fr, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run)
-                        for netDegree in netDegrees:
+            yield shape
-                            # Network Degree has to be an even number
-                            if netDegree % 2 == 0:
-                                shape = Shape(blockSize, nv, fr, chi, netDegree, run)
-                                yield shape

study.py

@@ -40,7 +40,7 @@ def study():
             print("You need to pass a configuration file in parameter")
             exit(1)
 
-    shape = Shape(0, 0, 0, 0, 0, 0)
+    shape = Shape(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
     sim = Simulator(shape, config)
     sim.initLogger()
     results = []