From 99e051e6fc669b27e791a99b8971f4eaca7063cb Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Fri, 3 Mar 2023 18:41:12 +0100 Subject: [PATCH 01/12] adding uplinkBw configuration Signed-off-by: Csaba Kiraly --- DAS/shape.py | 3 ++- DAS/validator.py | 2 +- config_example.py | 13 +++++++++---- study.py | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/DAS/shape.py b/DAS/shape.py index 1dd19b2..de22c90 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,7 +3,7 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" - def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, run): + def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, bwUplink, run): """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberValidators = numberValidators @@ -12,6 +12,7 @@ class Shape: self.netDegree = netDegree self.chi = chi self.randomSeed = "" + self.bwUplink = bwUplink def __repr__(self): """Returns a printable representation of the shape""" diff --git a/DAS/validator.py b/DAS/validator.py index 7b52e58..4647df5 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -77,7 +77,7 @@ class Validator: # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 # TODO: this should be a parameter - self.bwUplink = 110 if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps + self.bwUplink = shape.bwUplink if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps self.repairOnTheFly = True self.sendLineUntil = (self.shape.blockSize + 1) // 2 # stop sending on a p2p link if at least this amount of samples passed diff --git a/config_example.py b/config_example.py index 248a8e9..0967a4f 100644 --- a/config_example.py +++ b/config_example.py @@ -42,6 +42,10 @@ netDegrees = range(6, 9, 2) # Number of rows and columns a validator is interested in chis = range(4, 9, 2) +# Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) +# 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 +bwUplinks = [11, 110] + # Set to True if you want your run to be deterministic, False if not deterministic = False @@ -55,7 +59,8 @@ def nextShape(): for blockSize in blockSizes: for nv in numberValidators: for netDegree in netDegrees: - # Network Degree has to be an even number - if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, chi, netDegree, run) - yield shape + for bwUplink in bwUplinks: + # Network Degree has to be an even number + if netDegree % 2 == 0: + shape = Shape(blockSize, nv, fr, chi, netDegree, bwUplink, run) + yield shape diff --git a/study.py b/study.py index e24fb46..2503895 100644 --- a/study.py +++ b/study.py @@ -40,7 +40,7 @@ def study(): print("You need to pass a configuration file in parameter") exit(1) - shape = Shape(0, 0, 0, 0, 0, 0) + shape = Shape(0, 0, 0, 0, 0, 0, 0) sim = Simulator(shape, config) sim.initLogger() results = [] From 4a5d410f6a86b48bf3688773f379ab1ff64ce66e Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 6 Mar 2023 11:42:45 +0100 Subject: [PATCH 02/12] fixup: add bwUplink to save file name Signed-off-by: Csaba Kiraly --- DAS/shape.py | 1 + 1 file changed, 1 insertion(+) diff --git a/DAS/shape.py b/DAS/shape.py index de22c90..7b24f6e 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -21,6 +21,7 @@ class Shape: shastr += "-nbv-"+str(self.numberValidators) shastr += "-fr-"+str(self.failureRate) shastr += "-chi-"+str(self.chi) + shastr += "-bwu-"+str(self.bwUplink) shastr += "-nd-"+str(self.netDegree) shastr += "-r-"+str(self.run) return shastr From dc7a4d3c032160762fe5191025bf8e3c7e087beb Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 6 Mar 2023 15:15:57 +0100 Subject: [PATCH 03/12] generate row/column interest locally in validator Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 17 ++++++++++++----- DAS/validator.py | 21 +++++++++++++++------ config_example.py | 4 ++++ 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 8b79e2c..f0195cd 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -15,6 +15,7 @@ class Simulator: def __init__(self, shape, config): """It initializes the simulation with a set of parameters (shape).""" self.shape = shape + self.config = config self.format = {"entity": "Simulator"} self.result = Result(self.shape) self.validators = [] @@ -28,12 +29,18 @@ class Simulator: self.glob = Observer(self.logger, self.shape) self.glob.reset() self.validators = [] - rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) - columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) - random.shuffle(rows) - random.shuffle(columns) + if self.config.evenLineDistribution: + rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) + columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) + random.shuffle(rows) + random.shuffle(columns) for i in range(self.shape.numberValidators): - val = Validator(i, int(not i!=0), self.logger, self.shape, rows, columns) + if self.config.evenLineDistribution: + val = Validator(i, int(not i!=0), self.logger, self.shape, + rows[(i*self.shape.chi):((i+1)*self.shape.chi)], + columns[(i*self.shape.chi):((i+1)*self.shape.chi)]) + else: + val = Validator(i, int(not i!=0), self.logger, self.shape) if i == self.proposerID: val.initBlock() self.glob.setGoldenData(val.block) diff --git a/DAS/validator.py b/DAS/validator.py index 4647df5..6a1b80a 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -38,8 +38,13 @@ class Validator: """It returns the validator ID.""" return str(self.ID) - def __init__(self, ID, amIproposer, logger, shape, rows, columns): - """It initializes the validator with the logger, shape and assigned rows/columns.""" + def __init__(self, ID, amIproposer, logger, shape, rows = None, columns = None): + """It initializes the validator with the logger shape and rows/columns. + + If rows/columns are specified these are observed, otherwise (default) + chi rows and columns are selected randomly. + """ + self.shape = shape FORMAT = "%(levelname)s : %(entity)s : %(message)s" self.ID = ID @@ -59,12 +64,16 @@ class Validator: self.rowIDs = range(shape.blockSize) self.columnIDs = range(shape.blockSize) else: - self.rowIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] - self.columnIDs = columns[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)] #if shape.deterministic: # random.seed(self.ID) - #self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi) - #self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi) + if rows: + self.rowIDs = rows + else: + self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi) + if columns: + self.columnIDs = columns + else: + self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) diff --git a/config_example.py b/config_example.py index 0967a4f..9235f10 100644 --- a/config_example.py +++ b/config_example.py @@ -24,6 +24,10 @@ logLevel = logging.INFO # for more details, see joblib.Parallel numJobs = 3 +# distribute rows/columns evenly between validators (True) +# or generate it using local randomness (False) +evenLineDistribution = False + # Number of simulation runs with the same parameters for statistical relevance runs = range(10) From 49b1c239d79c1eae2f332cbc6388c1c22091303a Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 6 Mar 2023 22:46:17 +0100 Subject: [PATCH 04/12] use itertools Signed-off-by: Csaba Kiraly --- config_example.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/config_example.py b/config_example.py index 9235f10..59d8fef 100644 --- a/config_example.py +++ b/config_example.py @@ -14,6 +14,7 @@ if needed. """ import logging +import itertools from DAS.shape import Shape dumpXML = 1 @@ -57,14 +58,9 @@ deterministic = False randomSeed = "DAS" def nextShape(): - for run in runs: - for fr in failureRates: - for chi in chis: - for blockSize in blockSizes: - for nv in numberValidators: - for netDegree in netDegrees: - for bwUplink in bwUplinks: - # Network Degree has to be an even number - if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, chi, netDegree, bwUplink, run) - yield shape + for run, fr, chi, blockSize, nv, netDegree, bwUplink in itertools.product( + runs, failureRates, chis, blockSizes, numberValidators, netDegrees, bwUplinks): + # Network Degree has to be an even number + if netDegree % 2 == 0: + shape = Shape(blockSize, nv, fr, chi, netDegree, bwUplink, run) + yield shape From ef4e32ed53a3a0793f42a0451448b62429e879bc Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 7 Mar 2023 13:24:11 +0100 Subject: [PATCH 05/12] introduce node classes Signed-off-by: Csaba Kiraly --- DAS/shape.py | 18 +++++++++++++----- DAS/simulator.py | 3 ++- DAS/validator.py | 21 ++++++++++++++++----- config_example.py | 16 +++++++++++----- study.py | 2 +- 5 files changed, 43 insertions(+), 17 deletions(-) diff --git a/DAS/shape.py b/DAS/shape.py index 7b24f6e..b707e51 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,16 +3,20 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" - def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, bwUplink, run): + def __init__(self, blockSize, numberValidators, failureRate, class1ratio, chi1, chi2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberValidators = numberValidators self.blockSize = blockSize self.failureRate = failureRate self.netDegree = netDegree - self.chi = chi + self.class1ratio = class1ratio + self.chi1 = chi1 + self.chi2 = chi2 + self.bwUplinkProd = bwUplinkProd + self.bwUplink1 = bwUplink1 + self.bwUplink2 = bwUplink2 self.randomSeed = "" - self.bwUplink = bwUplink def __repr__(self): """Returns a printable representation of the shape""" @@ -20,8 +24,12 @@ class Shape: shastr += "bs-"+str(self.blockSize) shastr += "-nbv-"+str(self.numberValidators) shastr += "-fr-"+str(self.failureRate) - shastr += "-chi-"+str(self.chi) - shastr += "-bwu-"+str(self.bwUplink) + shastr += "-c1r-"+str(self.class1ratio) + shastr += "-chi1-"+str(self.chi1) + shastr += "-chi2-"+str(self.chi2) + shastr += "-bwupprod-"+str(self.bwUplinkProd) + shastr += "-bwup1-"+str(self.bwUplink1) + shastr += "-bwup2-"+str(self.bwUplink2) shastr += "-nd-"+str(self.netDegree) shastr += "-r-"+str(self.run) return shastr diff --git a/DAS/simulator.py b/DAS/simulator.py index f0195cd..2d6e635 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -126,7 +126,8 @@ class Simulator: self.result = Result(self.shape) for val in self.validators: val.shape.failureRate = shape.failureRate - val.shape.chi = shape.chi + val.shape.chi1 = shape.chi1 + val.shape.chi2 = shape.chi2 # In GossipSub the initiator might push messages without participating in the mesh. # proposerPublishOnly regulates this behavior. If set to true, the proposer is not diff --git a/DAS/validator.py b/DAS/validator.py index 6a1b80a..71e5791 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -55,11 +55,17 @@ class Validator: self.sendQueue = deque() self.amIproposer = amIproposer self.logger = logger - if self.shape.chi < 1: + if self.shape.chi1 < 1 or self.shape.chi2 < 1: self.logger.error("Chi has to be greater than 0", extra=self.format) - elif self.shape.chi > self.shape.blockSize: + elif self.shape.chi1 > self.shape.blockSize or self.shape.chi2 > self.shape.blockSize: self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) else: + if self.amIproposer: + self.chi = 1 # not used + elif self.ID <= shape.numberValidators * shape.class1ratio: + self.chi = shape.chi1 + else: + self.chi = shape.chi2 if amIproposer: self.rowIDs = range(shape.blockSize) self.columnIDs = range(shape.blockSize) @@ -69,11 +75,11 @@ class Validator: if rows: self.rowIDs = rows else: - self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi) + self.rowIDs = random.sample(range(self.shape.blockSize), self.chi) if columns: self.columnIDs = columns else: - self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi) + self.columnIDs = random.sample(range(self.shape.blockSize), self.chi) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) @@ -86,7 +92,12 @@ class Validator: # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 # TODO: this should be a parameter - self.bwUplink = shape.bwUplink if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps + if self.amIproposer: + self.bwUplink = shape.bwUplinkProd + elif self.ID <= shape.numberValidators * shape.class1ratio: + self.bwUplink = shape.bwUplink1 + else: + self.bwUplink = shape.bwUplink2 self.repairOnTheFly = True self.sendLineUntil = (self.shape.blockSize + 1) // 2 # stop sending on a p2p link if at least this amount of samples passed diff --git a/config_example.py b/config_example.py index 59d8fef..b718528 100644 --- a/config_example.py +++ b/config_example.py @@ -15,6 +15,7 @@ if needed. import logging import itertools +import numpy as np from DAS.shape import Shape dumpXML = 1 @@ -44,12 +45,17 @@ blockSizes = range(32,65,16) # Per-topic mesh neighborhood size netDegrees = range(6, 9, 2) +class1ratios = np.arange(0, 1, .2) + # Number of rows and columns a validator is interested in -chis = range(4, 9, 2) +chis1 = range(1, 5, 2) +chis2 = range(10, 30, 20) # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 -bwUplinks = [11, 110] +bwUplinksProd = [2200] +bwUplinks1 = [2200] +bwUplinks2 = [110] # Set to True if you want your run to be deterministic, False if not deterministic = False @@ -58,9 +64,9 @@ deterministic = False randomSeed = "DAS" def nextShape(): - for run, fr, chi, blockSize, nv, netDegree, bwUplink in itertools.product( - runs, failureRates, chis, blockSizes, numberValidators, netDegrees, bwUplinks): + for run, fr, class1ratio, chi1, chi2, blockSize, nv, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( + runs, failureRates, class1ratios, chis1, chis2, blockSizes, numberValidators, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): # Network Degree has to be an even number if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, chi, netDegree, bwUplink, run) + shape = Shape(blockSize, nv, fr, class1ratio, chi1, chi2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) yield shape diff --git a/study.py b/study.py index 2503895..3efd83e 100644 --- a/study.py +++ b/study.py @@ -40,7 +40,7 @@ def study(): print("You need to pass a configuration file in parameter") exit(1) - shape = Shape(0, 0, 0, 0, 0, 0, 0) + shape = Shape(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) sim = Simulator(shape, config) sim.initLogger() results = [] From 45fe4542045a082b369a5012a1d6aa9e75b103ac Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 7 Mar 2023 14:36:13 +0100 Subject: [PATCH 06/12] WIP: extend Vis with new parameters Current code assumes 6 parameters in some places. Still needs some work Signed-off-by: Csaba Kiraly --- DAS/visualizer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/DAS/visualizer.py b/DAS/visualizer.py index 047e513..e0579e2 100644 --- a/DAS/visualizer.py +++ b/DAS/visualizer.py @@ -12,7 +12,8 @@ class Visualizer: def __init__(self, execID): self.execID = execID self.folderPath = "results/"+self.execID - self.parameters = ['run', 'blockSize', 'failureRate', 'numberValidators', 'netDegree', 'chi'] + self.parameters = ['run', 'blockSize', 'failureRate', 'numberValidators', 'netDegree', + 'chi1', 'chi2', 'bwUplinkProd', 'bwUplink1', 'bwUplink2'] self.minimumDataPoints = 2 def plottingData(self): @@ -29,20 +30,24 @@ class Visualizer: failureRate = int(root.find('failureRate').text) numberValidators = int(root.find('numberValidators').text) netDegree = int(root.find('netDegree').text) - chi = int(root.find('chi').text) + chi1 = int(root.find('chi1').text) + chi2 = int(root.find('chi2').text) + bwUplinkProd = int(root.find('bwUplinkProd').text) + bwUplink1 = int(root.find('bwUplink1').text) + bwUplink2 = int(root.find('bwUplink2').text) tta = int(root.find('tta').text) # Loop over all possible combinations of length 4 of the parameters for combination in combinations(self.parameters, 4): # Get the indices and values of the parameters in the combination indices = [self.parameters.index(element) for element in combination] - selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi] + selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi1, chi2, bwUplinkProd, bwUplink1, bwUplink2] values = [selectedValues[index] for index in indices] names = [self.parameters[i] for i in indices] keyComponents = [f"{name}_{value}" for name, value in zip(names, values)] key = tuple(keyComponents[:4]) #Get the names of the other 2 parameters that are not included in the key - otherParams = [self.parameters[i] for i in range(6) if i not in indices] + otherParams = [self.parameters[i] for i in range(len(self.parameters)) if i not in indices] #Append the values of the other 2 parameters and the ttas to the lists for the key otherIndices = [i for i in range(len(self.parameters)) if i not in indices] From 065086f88caadca7fde37d9d0c1c3976361e08cc Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 13 Mar 2023 15:00:43 +0100 Subject: [PATCH 07/12] configure using validatorsPerNode1/2 instead of chi1/2 Signed-off-by: Csaba Kiraly --- DAS/shape.py | 12 +++++++----- DAS/simulator.py | 5 +++-- DAS/validator.py | 10 +++++----- DAS/visualizer.py | 9 +++++---- config_example.py | 16 ++++++++++------ study.py | 2 +- 6 files changed, 31 insertions(+), 23 deletions(-) diff --git a/DAS/shape.py b/DAS/shape.py index b707e51..484b3eb 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,7 +3,7 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" - def __init__(self, blockSize, numberValidators, failureRate, class1ratio, chi1, chi2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): + def __init__(self, blockSize, numberValidators, failureRate, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberValidators = numberValidators @@ -11,8 +11,9 @@ class Shape: self.failureRate = failureRate self.netDegree = netDegree self.class1ratio = class1ratio - self.chi1 = chi1 - self.chi2 = chi2 + self.chi = chi + self.vpn1 = vpn1 + self.vpn2 = vpn2 self.bwUplinkProd = bwUplinkProd self.bwUplink1 = bwUplink1 self.bwUplink2 = bwUplink2 @@ -25,8 +26,9 @@ class Shape: shastr += "-nbv-"+str(self.numberValidators) shastr += "-fr-"+str(self.failureRate) shastr += "-c1r-"+str(self.class1ratio) - shastr += "-chi1-"+str(self.chi1) - shastr += "-chi2-"+str(self.chi2) + shastr += "-chi-"+str(self.chi) + shastr += "-vpn1-"+str(self.vpn1) + shastr += "-vpn2-"+str(self.vpn2) shastr += "-bwupprod-"+str(self.bwUplinkProd) shastr += "-bwup1-"+str(self.bwUplink1) shastr += "-bwup2-"+str(self.bwUplink2) diff --git a/DAS/simulator.py b/DAS/simulator.py index 2d6e635..65f8ef2 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -126,8 +126,9 @@ class Simulator: self.result = Result(self.shape) for val in self.validators: val.shape.failureRate = shape.failureRate - val.shape.chi1 = shape.chi1 - val.shape.chi2 = shape.chi2 + val.shape.chi = shape.chi + val.shape.vpn1 = shape.vpn1 + val.shape.vpn2 = shape.vpn2 # In GossipSub the initiator might push messages without participating in the mesh. # proposerPublishOnly regulates this behavior. If set to true, the proposer is not diff --git a/DAS/validator.py b/DAS/validator.py index 71e5791..ae7be43 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -55,17 +55,17 @@ class Validator: self.sendQueue = deque() self.amIproposer = amIproposer self.logger = logger - if self.shape.chi1 < 1 or self.shape.chi2 < 1: + if self.shape.chi < 1: self.logger.error("Chi has to be greater than 0", extra=self.format) - elif self.shape.chi1 > self.shape.blockSize or self.shape.chi2 > self.shape.blockSize: - self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format) + elif self.shape.chi > self.shape.blockSize: + self.logger.error("Chi has to be smaller than %d" % self.shape.blockSize, extra=self.format) else: if self.amIproposer: self.chi = 1 # not used elif self.ID <= shape.numberValidators * shape.class1ratio: - self.chi = shape.chi1 + self.chi = shape.chi * shape.vpn1 else: - self.chi = shape.chi2 + self.chi = shape.chi * shape.vpn2 # TODO: union of random subsets vpn2 times if amIproposer: self.rowIDs = range(shape.blockSize) self.columnIDs = range(shape.blockSize) diff --git a/DAS/visualizer.py b/DAS/visualizer.py index e0579e2..7004c87 100644 --- a/DAS/visualizer.py +++ b/DAS/visualizer.py @@ -13,7 +13,7 @@ class Visualizer: self.execID = execID self.folderPath = "results/"+self.execID self.parameters = ['run', 'blockSize', 'failureRate', 'numberValidators', 'netDegree', - 'chi1', 'chi2', 'bwUplinkProd', 'bwUplink1', 'bwUplink2'] + 'chi', 'vpn1', 'vpn2', 'bwUplinkProd', 'bwUplink1', 'bwUplink2'] self.minimumDataPoints = 2 def plottingData(self): @@ -30,8 +30,9 @@ class Visualizer: failureRate = int(root.find('failureRate').text) numberValidators = int(root.find('numberValidators').text) netDegree = int(root.find('netDegree').text) - chi1 = int(root.find('chi1').text) - chi2 = int(root.find('chi2').text) + chi = int(root.find('chi').text) + vpn1 = int(root.find('vpn1').text) + vpn2 = int(root.find('vpn2').text) bwUplinkProd = int(root.find('bwUplinkProd').text) bwUplink1 = int(root.find('bwUplink1').text) bwUplink2 = int(root.find('bwUplink2').text) @@ -41,7 +42,7 @@ class Visualizer: for combination in combinations(self.parameters, 4): # Get the indices and values of the parameters in the combination indices = [self.parameters.index(element) for element in combination] - selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi1, chi2, bwUplinkProd, bwUplink1, bwUplink2] + selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi, vpn1, vpn2, bwUplinkProd, bwUplink1, bwUplink2] values = [selectedValues[index] for index in indices] names = [self.parameters[i] for i in indices] keyComponents = [f"{name}_{value}" for name, value in zip(names, values)] diff --git a/config_example.py b/config_example.py index b718528..0bd509c 100644 --- a/config_example.py +++ b/config_example.py @@ -45,11 +45,15 @@ blockSizes = range(32,65,16) # Per-topic mesh neighborhood size netDegrees = range(6, 9, 2) +# number of rows and columns a validator is interested in +chis = range(1, 5, 2) + +# ratio of class1 nodes (see below for parameters per class) class1ratios = np.arange(0, 1, .2) -# Number of rows and columns a validator is interested in -chis1 = range(1, 5, 2) -chis2 = range(10, 30, 20) +# Number of validators per beacon node +validatorsPerNode1 = [1] +validatorsPerNode2 = [2, 4, 8, 16, 32] # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 @@ -64,9 +68,9 @@ deterministic = False randomSeed = "DAS" def nextShape(): - for run, fr, class1ratio, chi1, chi2, blockSize, nv, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( - runs, failureRates, class1ratios, chis1, chis2, blockSizes, numberValidators, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): + for run, fr, class1ratio, chi, vpn1, vpn2, blockSize, nv, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( + runs, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberValidators, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): # Network Degree has to be an even number if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, class1ratio, chi1, chi2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) + shape = Shape(blockSize, nv, fr, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) yield shape diff --git a/study.py b/study.py index 3efd83e..2b2aab6 100644 --- a/study.py +++ b/study.py @@ -40,7 +40,7 @@ def study(): print("You need to pass a configuration file in parameter") exit(1) - shape = Shape(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + shape = Shape(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) sim = Simulator(shape, config) sim.initLogger() results = [] From 9d9612fd34dbdee32f398f5c8be03e48b8708248 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 13 Mar 2023 15:03:55 +0100 Subject: [PATCH 08/12] rename numberValidators to numberNodes Signed-off-by: Csaba Kiraly --- DAS/shape.py | 6 +++--- DAS/simulator.py | 18 +++++++++--------- DAS/validator.py | 4 ++-- DAS/visualizer.py | 6 +++--- config_example.py | 8 ++++---- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/DAS/shape.py b/DAS/shape.py index 484b3eb..d83351d 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,10 +3,10 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" - def __init__(self, blockSize, numberValidators, failureRate, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): + def __init__(self, blockSize, numberNodes, failureRate, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): """Initializes the shape with the parameters passed in argument.""" self.run = run - self.numberValidators = numberValidators + self.numberNodes = numberNodes self.blockSize = blockSize self.failureRate = failureRate self.netDegree = netDegree @@ -23,7 +23,7 @@ class Shape: """Returns a printable representation of the shape""" shastr = "" shastr += "bs-"+str(self.blockSize) - shastr += "-nbv-"+str(self.numberValidators) + shastr += "-nn-"+str(self.numberNodes) shastr += "-fr-"+str(self.failureRate) shastr += "-c1r-"+str(self.class1ratio) shastr += "-chi-"+str(self.chi) diff --git a/DAS/simulator.py b/DAS/simulator.py index 65f8ef2..4cadf9f 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -30,11 +30,11 @@ class Simulator: self.glob.reset() self.validators = [] if self.config.evenLineDistribution: - rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) - columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize) + rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberNodes/self.shape.blockSize) + columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberNodes/self.shape.blockSize) random.shuffle(rows) random.shuffle(columns) - for i in range(self.shape.numberValidators): + for i in range(self.shape.numberNodes): if self.config.evenLineDistribution: val = Validator(i, int(not i!=0), self.logger, self.shape, rows[(i*self.shape.chi):((i+1)*self.shape.chi)], @@ -104,7 +104,7 @@ class Simulator: v.columnNeighbors[id].update({vi.ID : Neighbor(vi, 1, self.shape.blockSize)}) if self.logger.isEnabledFor(logging.DEBUG): - for i in range(0, self.shape.numberValidators): + for i in range(0, self.shape.numberNodes): self.logger.debug("Val %d : rowN %s", i, self.validators[i].rowNeighbors, extra=self.format) self.logger.debug("Val %d : colN %s", i, self.validators[i].columnNeighbors, extra=self.format) @@ -155,17 +155,17 @@ class Simulator: missingVector.append(missingSamples) oldMissingSamples = missingSamples self.logger.debug("PHASE SEND %d" % steps, extra=self.format) - for i in range(0,self.shape.numberValidators): + for i in range(0,self.shape.numberNodes): self.validators[i].send() self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format) - for i in range(1,self.shape.numberValidators): + for i in range(1,self.shape.numberNodes): self.validators[i].receiveRowsColumns() self.logger.debug("PHASE RESTORE %d" % steps, extra=self.format) - for i in range(1,self.shape.numberValidators): + for i in range(1,self.shape.numberNodes): self.validators[i].restoreRows() self.validators[i].restoreColumns() self.logger.debug("PHASE LOG %d" % steps, extra=self.format) - for i in range(0,self.shape.numberValidators): + for i in range(0,self.shape.numberNodes): self.validators[i].logRows() self.validators[i].logColumns() @@ -176,7 +176,7 @@ class Simulator: (steps, statsTxInSlot[0], statsRxInSlot[0], mean(statsTxInSlot[1:]), max(statsTxInSlot[1:]), mean(statsRxInSlot[1:]), max(statsRxInSlot[1:])), extra=self.format) - for i in range(0,self.shape.numberValidators): + for i in range(0,self.shape.numberNodes): self.validators[i].updateStats() arrived, expected = self.glob.checkStatus(self.validators) diff --git a/DAS/validator.py b/DAS/validator.py index ae7be43..b0fbede 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -62,7 +62,7 @@ class Validator: else: if self.amIproposer: self.chi = 1 # not used - elif self.ID <= shape.numberValidators * shape.class1ratio: + elif self.ID <= shape.numberNodes * shape.class1ratio: self.chi = shape.chi * shape.vpn1 else: self.chi = shape.chi * shape.vpn2 # TODO: union of random subsets vpn2 times @@ -94,7 +94,7 @@ class Validator: # TODO: this should be a parameter if self.amIproposer: self.bwUplink = shape.bwUplinkProd - elif self.ID <= shape.numberValidators * shape.class1ratio: + elif self.ID <= shape.numberNodes * shape.class1ratio: self.bwUplink = shape.bwUplink1 else: self.bwUplink = shape.bwUplink2 diff --git a/DAS/visualizer.py b/DAS/visualizer.py index 7004c87..d165f0d 100644 --- a/DAS/visualizer.py +++ b/DAS/visualizer.py @@ -12,7 +12,7 @@ class Visualizer: def __init__(self, execID): self.execID = execID self.folderPath = "results/"+self.execID - self.parameters = ['run', 'blockSize', 'failureRate', 'numberValidators', 'netDegree', + self.parameters = ['run', 'blockSize', 'failureRate', 'numberNodes', 'netDegree', 'chi', 'vpn1', 'vpn2', 'bwUplinkProd', 'bwUplink1', 'bwUplink2'] self.minimumDataPoints = 2 @@ -28,7 +28,7 @@ class Visualizer: run = int(root.find('run').text) blockSize = int(root.find('blockSize').text) failureRate = int(root.find('failureRate').text) - numberValidators = int(root.find('numberValidators').text) + numberNodes = int(root.find('numberNodes').text) netDegree = int(root.find('netDegree').text) chi = int(root.find('chi').text) vpn1 = int(root.find('vpn1').text) @@ -42,7 +42,7 @@ class Visualizer: for combination in combinations(self.parameters, 4): # Get the indices and values of the parameters in the combination indices = [self.parameters.index(element) for element in combination] - selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi, vpn1, vpn2, bwUplinkProd, bwUplink1, bwUplink2] + selectedValues = [run, blockSize, failureRate, numberNodes, netDegree, chi, vpn1, vpn2, bwUplinkProd, bwUplink1, bwUplink2] values = [selectedValues[index] for index in indices] names = [self.parameters[i] for i in indices] keyComponents = [f"{name}_{value}" for name, value in zip(names, values)] diff --git a/config_example.py b/config_example.py index 0bd509c..5272555 100644 --- a/config_example.py +++ b/config_example.py @@ -34,7 +34,7 @@ evenLineDistribution = False runs = range(10) # Number of validators -numberValidators = range(256, 513, 128) +numberNodes = range(256, 513, 128) # Percentage of block not released by producer failureRates = range(10, 91, 40) @@ -68,9 +68,9 @@ deterministic = False randomSeed = "DAS" def nextShape(): - for run, fr, class1ratio, chi, vpn1, vpn2, blockSize, nv, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( - runs, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberValidators, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): + for run, fr, class1ratio, chi, vpn1, vpn2, blockSize, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( + runs, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): # Network Degree has to be an even number if netDegree % 2 == 0: - shape = Shape(blockSize, nv, fr, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) + shape = Shape(blockSize, nn, fr, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) yield shape From 56139965477e9281b5bb75f4c218e98fce3313d8 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 13 Mar 2023 15:27:23 +0100 Subject: [PATCH 09/12] fixup: make validatorsPerNode and bandwidth consistent Signed-off-by: Csaba Kiraly --- config_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config_example.py b/config_example.py index 5272555..b54dff1 100644 --- a/config_example.py +++ b/config_example.py @@ -58,8 +58,8 @@ validatorsPerNode2 = [2, 4, 8, 16, 32] # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 bwUplinksProd = [2200] -bwUplinks1 = [2200] -bwUplinks2 = [110] +bwUplinks1 = [110] +bwUplinks2 = [2200] # Set to True if you want your run to be deterministic, False if not deterministic = False From c366c05616fa67b2634fd4f894a7b8039870132c Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 14 Mar 2023 15:26:48 +0100 Subject: [PATCH 10/12] handle overlap for multiple validators per node correctly Signed-off-by: Csaba Kiraly --- DAS/tools.py | 6 ++++++ DAS/validator.py | 13 ++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/DAS/tools.py b/DAS/tools.py index cd26850..2b47b11 100644 --- a/DAS/tools.py +++ b/DAS/tools.py @@ -79,3 +79,9 @@ def sampleLine(line, limit): r[i] = 1 limit -= 1 return r + +def unionOfSamples(population, sampleSize, times): + selected = set() + for t in range(times): + selected |= set(random.sample(population, sampleSize)) + return selected diff --git a/DAS/validator.py b/DAS/validator.py index b0fbede..9abb125 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -4,7 +4,7 @@ import random import collections import logging from DAS.block import * -from DAS.tools import shuffled, shuffledDict +from DAS.tools import shuffled, shuffledDict, unionOfSamples from bitarray.util import zeros from collections import deque from itertools import chain @@ -60,12 +60,7 @@ class Validator: elif self.shape.chi > self.shape.blockSize: self.logger.error("Chi has to be smaller than %d" % self.shape.blockSize, extra=self.format) else: - if self.amIproposer: - self.chi = 1 # not used - elif self.ID <= shape.numberNodes * shape.class1ratio: - self.chi = shape.chi * shape.vpn1 - else: - self.chi = shape.chi * shape.vpn2 # TODO: union of random subsets vpn2 times + self.chi = shape.chi if amIproposer: self.rowIDs = range(shape.blockSize) self.columnIDs = range(shape.blockSize) @@ -75,11 +70,11 @@ class Validator: if rows: self.rowIDs = rows else: - self.rowIDs = random.sample(range(self.shape.blockSize), self.chi) + self.rowIDs = unionOfSamples(range(self.shape.blockSize), self.chi, self.shape.vpn1) if columns: self.columnIDs = columns else: - self.columnIDs = random.sample(range(self.shape.blockSize), self.chi) + self.columnIDs = unionOfSamples(range(self.shape.blockSize), self.chi, self.shape.vpn1) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) From 353efec1412cf9350c81b011a242df806b90a2a7 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 15 Mar 2023 12:53:43 +0100 Subject: [PATCH 11/12] fixup: use vpn1 and vpn2 correctly Signed-off-by: Csaba Kiraly --- DAS/validator.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/DAS/validator.py b/DAS/validator.py index 9abb125..f869171 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -60,21 +60,15 @@ class Validator: elif self.shape.chi > self.shape.blockSize: self.logger.error("Chi has to be smaller than %d" % self.shape.blockSize, extra=self.format) else: - self.chi = shape.chi if amIproposer: self.rowIDs = range(shape.blockSize) self.columnIDs = range(shape.blockSize) else: #if shape.deterministic: # random.seed(self.ID) - if rows: - self.rowIDs = rows - else: - self.rowIDs = unionOfSamples(range(self.shape.blockSize), self.chi, self.shape.vpn1) - if columns: - self.columnIDs = columns - else: - self.columnIDs = unionOfSamples(range(self.shape.blockSize), self.chi, self.shape.vpn1) + vpn = self.shape.vpn1 if (self.ID <= shape.numberNodes * shape.class1ratio) else self.shape.vpn2 + self.rowIDs = rows if rows else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) + self.columnIDs = columns if columns else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) From 64fcbff0ef1bd46e5721f527ebff8efed01b1707 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 20 Mar 2023 17:09:17 +0100 Subject: [PATCH 12/12] warn if no nodes for a row/column Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 4cadf9f..06ebd8e 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -64,7 +64,10 @@ class Simulator: # If the number of nodes in a channel is smaller or equal to the # requested degree, a fully connected graph is used. For n>d, a random # d-regular graph is set up. (For n=d+1, the two are the same.) - if (len(rowChannels[id]) <= self.shape.netDegree): + if not rowChannels[id]: + self.logger.error("No nodes for row %d !" % id, extra=self.format) + continue + elif (len(rowChannels[id]) <= self.shape.netDegree): self.logger.debug("Graph fully connected with degree %d !" % (len(rowChannels[id]) - 1), extra=self.format) G = nx.complete_graph(len(rowChannels[id])) else: @@ -77,7 +80,10 @@ class Simulator: val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, 0, self.shape.blockSize)}) val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, 0, self.shape.blockSize)}) - if (len(columnChannels[id]) <= self.shape.netDegree): + if not columnChannels[id]: + self.logger.error("No nodes for column %d !" % id, extra=self.format) + continue + elif (len(columnChannels[id]) <= self.shape.netDegree): self.logger.debug("Graph fully connected with degree %d !" % (len(columnChannels[id]) - 1), extra=self.format) G = nx.complete_graph(len(columnChannels[id])) else: