From dc51727b329ede185e7008e6d5a0f88b4e611025 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 27 Mar 2023 23:16:05 +0200 Subject: [PATCH 01/10] plot progress per run Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 14 +++++++++++++- config_example.py | 4 ++++ study.py | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 677a260..c2f4654 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -3,6 +3,7 @@ import networkx as nx import logging, random import pandas as pd +import matplotlib from functools import partial, partialmethod from datetime import datetime from DAS.tools import * @@ -13,7 +14,7 @@ from DAS.validator import * class Simulator: """This class implements the main DAS simulator.""" - def __init__(self, shape, config): + def __init__(self, shape, config, execID): """It initializes the simulation with a set of parameters (shape).""" self.shape = shape self.config = config @@ -24,6 +25,7 @@ class Simulator: self.logLevel = config.logLevel self.proposerID = 0 self.glob = [] + self.execID = execID # In GossipSub the initiator might push messages without participating in the mesh. # proposerPublishOnly regulates this behavior. If set to true, the proposer is not @@ -240,6 +242,16 @@ class Simulator: progress = pd.DataFrame(progressVector) if self.config.saveProgress: self.result.addMetric("progress", progress.to_dict(orient='list')) + if self.config.plotProgress: + progress.plot.line(subplots = [[cnS, cnN, cnV], [cnT0], [cnT1, cnR1, cnD1], [cnT2, cnR2, cnD2]], + title = str(self.shape)) + if not os.path.exists("results"): + os.makedirs("results") + if not os.path.exists("results/"+self.execID): + os.makedirs("results/"+self.execID) + filePath = "results/"+self.execID+"/"+str(self.shape)+".png" + matplotlib.pyplot.savefig(filePath) + self.result.populate(self.shape, missingVector) return self.result diff --git a/config_example.py b/config_example.py index 5b1a396..461ee36 100644 --- a/config_example.py +++ b/config_example.py @@ -22,6 +22,10 @@ dumpXML = 1 # save progress vectors to XML saveProgress = 1 + +# plot progress for each run to PNG +plotProgress = 1 + visualization = 1 logLevel = logging.INFO diff --git a/study.py b/study.py index fde8099..aa27a5f 100644 --- a/study.py +++ b/study.py @@ -28,7 +28,7 @@ def runOnce(config, shape, execID): shape.setSeed(config.randomSeed+"-"+str(shape)) random.seed(shape.randomSeed) - sim = Simulator(shape, config) + sim = Simulator(shape, config, execID) sim.initLogger() sim.initValidators() sim.initNetwork() From 04ad03f1751168368d17eaadd6da70e04d24d2dd Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 27 Mar 2023 23:27:49 +0200 Subject: [PATCH 02/10] fixup: avoid warning on mean if empty Signed-off-by: Csaba Kiraly --- DAS/observer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/DAS/observer.py b/DAS/observer.py index 1bdf4d0..18c0461 100644 --- a/DAS/observer.py +++ b/DAS/observer.py @@ -70,6 +70,8 @@ class Observer: def getTrafficStats(self, validators): def maxOrNan(l): return np.max(l) if l else np.NaN + def meanOrNan(l): + return np.mean(l) if l else np.NaN trafficStats = {} for cl in range(0,3): @@ -77,9 +79,9 @@ class Observer: Rx = [v.statsRxInSlot for v in validators if v.nodeClass == cl] RxDup = [v.statsRxDupInSlot for v in validators if v.nodeClass == cl] trafficStats[cl] = { - "Tx": {"mean": np.mean(Tx), "max": maxOrNan(Tx)}, - "Rx": {"mean": np.mean(Rx), "max": maxOrNan(Rx)}, - "RxDup": {"mean": np.mean(RxDup), "max": maxOrNan(RxDup)}, + "Tx": {"mean": meanOrNan(Tx), "max": maxOrNan(Tx)}, + "Rx": {"mean": meanOrNan(Rx), "max": maxOrNan(Rx)}, + "RxDup": {"mean": meanOrNan(RxDup), "max": maxOrNan(RxDup)}, } return trafficStats From 75a9b484e915f5a804486936a50d25acc1c54793 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Mon, 27 Mar 2023 23:28:12 +0200 Subject: [PATCH 03/10] fixup: close plot to release memory Signed-off-by: Csaba Kiraly --- DAS/simulator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/DAS/simulator.py b/DAS/simulator.py index c2f4654..04d1eca 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -251,6 +251,7 @@ class Simulator: os.makedirs("results/"+self.execID) filePath = "results/"+self.execID+"/"+str(self.shape)+".png" matplotlib.pyplot.savefig(filePath) + matplotlib.pyplot.close() self.result.populate(self.shape, missingVector) return self.result From e285890fa734fcc2f9db7e06a4155a88de34071a Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 29 Mar 2023 16:42:09 +0200 Subject: [PATCH 04/10] Fixes allocation bug, remove duplicates in rowIDs and columnIDs, add diagnostics when the block is not available. Add number of steps without progress to stop condition. --- DAS/simulator.py | 78 ++++++++++++++++++++++++++++++++++++++--------- config_example.py | 6 ++++ 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 04d1eca..6ec3bfa 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -48,11 +48,19 @@ class Simulator: lightVal = int(self.shape.numberNodes * self.shape.class1ratio * self.shape.vpn1) heavyVal = int(self.shape.numberNodes * (1-self.shape.class1ratio) * self.shape.vpn2) totalValidators = lightVal + heavyVal - rows = list(range(self.shape.blockSize)) * (int(totalValidators/self.shape.blockSize)+1) - columns = list(range(self.shape.blockSize)) * (int(totalValidators/self.shape.blockSize)+1) + totalRows = totalValidators * self.shape.chi + rows = list(range(self.shape.blockSize)) * (int(totalRows/self.shape.blockSize)+1) + columns = list(range(self.shape.blockSize)) * (int(totalRows/self.shape.blockSize)+1) offset = heavyVal*self.shape.chi random.shuffle(rows) random.shuffle(columns) + self.logger.debug("There is a total of %d validators" % totalValidators, extra=self.format) + self.logger.debug("Shuffling a total of %d rows/columns" % len(rows), extra=self.format) + self.logger.debug("Shuffled rows: %s" % str(rows), extra=self.format) + self.logger.debug("Shuffled columns: %s" % str(columns), extra=self.format) + + assignatedRows = [] + assignatedCols = [] for i in range(self.shape.numberNodes): if self.config.evenLineDistribution: if i < int(heavyVal/self.shape.vpn2): # First start with the heavy nodes @@ -62,9 +70,17 @@ class Simulator: j = i - int(heavyVal/self.shape.vpn2) start = offset+( j *self.shape.chi) end = offset+((j+1)*self.shape.chi) - r = rows[start:end] - c = columns[start:end] + # Remove duplicates + r = list(dict.fromkeys(rows[start:end])) + c = list(dict.fromkeys(columns[start:end])) + r.sort() + c.sort() val = Validator(i, int(not i!=0), self.logger, self.shape, r, c) + self.logger.debug("Validators %d row IDs: %s" % (val.ID, val.rowIDs), extra=self.format) + self.logger.debug("Validators %d column IDs: %s" % (val.ID, val.columnIDs), extra=self.format) + assignatedRows = assignatedRows + r + assignatedCols = assignatedCols + c + else: val = Validator(i, int(not i!=0), self.logger, self.shape) if i == self.proposerID: @@ -72,6 +88,11 @@ class Simulator: else: val.logIDs() self.validators.append(val) + + assignatedRows.sort() + assignatedCols.sort() + self.logger.debug("Rows assignated: %s" % str(assignatedRows), extra=self.format) + self.logger.debug("Columns assignated: %s" % str(assignatedCols), extra=self.format) self.logger.debug("Validators initialized.", extra=self.format) def initNetwork(self): @@ -86,12 +107,14 @@ class Simulator: columnChannels[id].append(v) # Check rows/columns distribution - #totalR = 0 - #totalC = 0 - #for r in rowChannels: - # totalR += len(r) - #for c in columnChannels: - # totalC += len(c) + distR = [] + distC = [] + for r in rowChannels: + distR.append(len(r)) + for c in columnChannels: + distC.append(len(c)) + self.logger.debug("Number of validators per row; Min: %d, Max: %d" % (min(distR), max(distR)), extra=self.format) + self.logger.debug("Number of validators per column; Min: %d, Max: %d" % (min(distC), max(distC)), extra=self.format) for id in range(self.shape.blockSize): @@ -164,6 +187,29 @@ class Simulator: logger.addHandler(ch) self.logger = logger + def printDiagnostics(self): + """Print all required diagnostics to check when a block does not become available""" + for val in self.validators: + (a, e) = val.checkStatus() + if e-a > 0 and val.ID != 0: + self.logger.warning("Node %d is missing %d samples" % (val.ID, e-a), extra=self.format) + for r in val.rowIDs: + row = val.getRow(r) + if row.count() < len(row): + self.logger.debug("Row %d: %s" % (r, str(row)), extra=self.format) + neiR = val.rowNeighbors[r] + for nr in neiR: + self.logger.debug("Row %d, Neighbor %d sent: %s" % (r, val.rowNeighbors[r][nr].node.ID, val.rowNeighbors[r][nr].received), extra=self.format) + self.logger.debug("Row %d, Neighbor %d has: %s" % (r, val.rowNeighbors[r][nr].node.ID, self.validators[val.rowNeighbors[r][nr].node.ID].getRow(r)), extra=self.format) + for c in val.columnIDs: + col = val.getColumn(c) + if col.count() < len(col): + self.logger.debug("Column %d: %s" % (c, str(col)), extra=self.format) + neiC = val.columnNeighbors[c] + for nc in neiC: + self.logger.debug("Column %d, Neighbor %d sent: %s" % (c, val.columnNeighbors[c][nc].node.ID, val.columnNeighbors[c][nc].received), extra=self.format) + self.logger.debug("Column %d, Neighbor %d has: %s" % (c, val.columnNeighbors[c][nc].node.ID, self.validators[val.columnNeighbors[c][nc].node.ID].getColumn(c)), extra=self.format) + def run(self): """It runs the main simulation until the block is available or it gets stucked.""" self.glob.checkRowsColumns(self.validators) @@ -194,14 +240,14 @@ class Simulator: # log TX and RX statistics trafficStats = self.glob.getTrafficStats(self.validators) - self.logger.debug("step %d: %s" % + self.logger.debug("step %d: %s" % (steps, trafficStats), extra=self.format) for i in range(0,self.shape.numberNodes): self.validators[i].updateStats() trafficStatsVector.append(trafficStats) missingSamples, sampleProgress, nodeProgress, validatorProgress = self.glob.getProgress(self.validators) - self.logger.debug("step %d, arrived %0.02f %%, ready %0.02f %%, validated %0.02f %%" + self.logger.debug("step %d, arrived %0.02f %%, ready %0.02f %%, validated %0.02f %%" % (steps, sampleProgress*100, nodeProgress*100, validatorProgress*100), extra=self.format) cnS = "samples received" @@ -229,9 +275,13 @@ class Simulator: }) if missingSamples == oldMissingSamples: - self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format) + if len(missingVector) > self.config.steps4StopCondition: + if missingSamples == missingVector[len(missingVector)-1-self.config.steps4StopCondition]: + self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format) + if self.config.diagnostics: + self.printDiagnostics() + break missingVector.append(missingSamples) - break elif missingSamples == 0: #self.logger.info("The entire block is available at step %d, with failure rate %d !" % (steps, self.shape.failureRate), extra=self.format) missingVector.append(missingSamples) diff --git a/config_example.py b/config_example.py index 461ee36..d07a899 100644 --- a/config_example.py +++ b/config_example.py @@ -74,6 +74,12 @@ deterministic = False # If your run is deterministic you can decide the random seed. This is ignore otherwise. randomSeed = "DAS" +# If True, print diagnostics when the block is not available +diagnostics = False + +# Number of steps without progress to stop simulation +steps4StopCondition = 7 + def nextShape(): for run, fr, class1ratio, chi, vpn1, vpn2, blockSize, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( runs, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): From 9800161ac91576a63ce99121c454c2361400d9ed Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Mon, 27 Mar 2023 15:29:39 +0200 Subject: [PATCH 05/10] Switch from time steps to miliseconds. Rebase to traffic progress. --- DAS/results.py | 15 ++++++++------- DAS/simulator.py | 13 +++++++------ DAS/visualizer.py | 6 ++++-- config_example.py | 15 +++++++++------ study.py | 4 ++-- 5 files changed, 30 insertions(+), 23 deletions(-) diff --git a/DAS/results.py b/DAS/results.py index c5687a5..48512cd 100644 --- a/DAS/results.py +++ b/DAS/results.py @@ -7,22 +7,23 @@ from dicttoxml import dicttoxml class Result: """This class stores and process/store the results of a simulation.""" - def __init__(self, shape): + def __init__(self, shape, execID): """It initializes the instance with a specific shape.""" self.shape = shape + self.execID = execID self.blockAvailable = -1 self.tta = -1 self.missingVector = [] self.metrics = {} - def populate(self, shape, missingVector): + def populate(self, shape, config, missingVector): """It populates part of the result data inside a vector.""" self.shape = shape self.missingVector = missingVector missingSamples = missingVector[-1] if missingSamples == 0: self.blockAvailable = 1 - self.tta = len(missingVector) + self.tta = len(missingVector) * (1000/config.stepDuration) else: self.blockAvailable = 0 self.tta = -1 @@ -31,12 +32,12 @@ class Result: """Generic function to add a metric to the results.""" self.metrics[name] = metric - def dump(self, execID): + def dump(self): """It dumps the results of the simulation in an XML file.""" if not os.path.exists("results"): os.makedirs("results") - if not os.path.exists("results/"+execID): - os.makedirs("results/"+execID) + if not os.path.exists("results/"+self.execID): + os.makedirs("results/"+self.execID) resd1 = self.shape.__dict__ resd2 = self.__dict__.copy() resd2.pop("shape") @@ -44,6 +45,6 @@ class Result: resXml = dicttoxml(resd1) xmlstr = minidom.parseString(resXml) xmlPretty = xmlstr.toprettyxml() - filePath = "results/"+execID+"/"+str(self.shape)+".xml" + filePath = "results/"+self.execID+"/"+str(self.shape)+".xml" with open(filePath, "w") as f: f.write(xmlPretty) diff --git a/DAS/simulator.py b/DAS/simulator.py index 677a260..4acf3f5 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -13,12 +13,13 @@ from DAS.validator import * class Simulator: """This class implements the main DAS simulator.""" - def __init__(self, shape, config): + def __init__(self, shape, config, execID): """It initializes the simulation with a set of parameters (shape).""" self.shape = shape self.config = config self.format = {"entity": "Simulator"} - self.result = Result(self.shape) + self.execID = execID + self.result = Result(self.shape, self.execID) self.validators = [] self.logger = [] self.logLevel = config.logLevel @@ -192,14 +193,14 @@ class Simulator: # log TX and RX statistics trafficStats = self.glob.getTrafficStats(self.validators) - self.logger.debug("step %d: %s" % + self.logger.debug("step %d: %s" % (steps, trafficStats), extra=self.format) for i in range(0,self.shape.numberNodes): self.validators[i].updateStats() trafficStatsVector.append(trafficStats) missingSamples, sampleProgress, nodeProgress, validatorProgress = self.glob.getProgress(self.validators) - self.logger.debug("step %d, arrived %0.02f %%, ready %0.02f %%, validated %0.02f %%" + self.logger.debug("step %d, arrived %0.02f %%, ready %0.02f %%, validated %0.02f %%" % (steps, sampleProgress*100, nodeProgress*100, validatorProgress*100), extra=self.format) cnS = "samples received" @@ -231,7 +232,7 @@ class Simulator: missingVector.append(missingSamples) break elif missingSamples == 0: - #self.logger.info("The entire block is available at step %d, with failure rate %d !" % (steps, self.shape.failureRate), extra=self.format) + self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.shape.failureRate), extra=self.format) missingVector.append(missingSamples) break else: @@ -240,6 +241,6 @@ class Simulator: progress = pd.DataFrame(progressVector) if self.config.saveProgress: self.result.addMetric("progress", progress.to_dict(orient='list')) - self.result.populate(self.shape, missingVector) + self.result.populate(self.shape, self.config, missingVector) return self.result diff --git a/DAS/visualizer.py b/DAS/visualizer.py index 8afd006..db4b2d5 100644 --- a/DAS/visualizer.py +++ b/DAS/visualizer.py @@ -36,7 +36,7 @@ class Visualizer: bwUplinkProd = int(root.find('bwUplinkProd').text) bwUplink1 = int(root.find('bwUplink1').text) bwUplink2 = int(root.find('bwUplink2').text) - tta = int(root.find('tta').text) + tta = float(root.find('tta').text) # Loop over all possible combinations of of the parameters minus two for combination in combinations(self.parameters, len(self.parameters)-2): @@ -120,7 +120,7 @@ class Visualizer: hist, xedges, yedges = np.histogram2d(data[key][labels[0]], data[key][labels[1]], bins=(len(xlabels), len(ylabels)), weights=data[key]['ttas']) hist = hist.T fig, ax = plt.subplots(figsize=(10, 6)) - sns.heatmap(hist, xticklabels=xlabels, yticklabels=ylabels, cmap='Purples', cbar_kws={'label': 'Time to block availability'}, linecolor='black', linewidths=0.3, annot=True, fmt=".2f", ax=ax) + sns.heatmap(hist, xticklabels=xlabels, yticklabels=ylabels, cmap='Purples', cbar_kws={'label': 'Time to block availability (ms)'}, linecolor='black', linewidths=0.3, annot=True, fmt=".2f", ax=ax) plt.xlabel(self.formatLabel(labels[0])) plt.ylabel(self.formatLabel(labels[1])) filename = "" @@ -131,6 +131,8 @@ class Visualizer: filename += f"{key[paramValueCnt]}" formattedTitle = self.formatTitle(key[paramValueCnt]) title += formattedTitle + if (paramValueCnt+1) % 5 == 0: + title += "\n" paramValueCnt += 1 title_obj = plt.title(title) font_size = 16 * fig.get_size_inches()[0] / 10 diff --git a/config_example.py b/config_example.py index 5b1a396..3ff5ae8 100644 --- a/config_example.py +++ b/config_example.py @@ -27,14 +27,14 @@ logLevel = logging.INFO # number of parallel workers. -1: all cores; 1: sequential # for more details, see joblib.Parallel -numJobs = 3 +numJobs = -1 # distribute rows/columns evenly between validators (True) # or generate it using local randomness (False) evenLineDistribution = True # Number of simulation runs with the same parameters for statistical relevance -runs = range(10) +runs = range(2) # Number of validators numberNodes = range(256, 513, 128) @@ -49,14 +49,14 @@ blockSizes = range(32,65,16) netDegrees = range(6, 9, 2) # number of rows and columns a validator is interested in -chis = range(1, 5, 2) +chis = range(2, 5, 2) # ratio of class1 nodes (see below for parameters per class) -class1ratios = np.arange(0, 1, .2) +class1ratios = [0.8, 0.9] # Number of validators per beacon node validatorsPerNode1 = [1] -validatorsPerNode2 = [2, 4, 8, 16, 32] +validatorsPerNode2 = [500] # Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?) # 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11 @@ -64,8 +64,11 @@ bwUplinksProd = [2200] bwUplinks1 = [110] bwUplinks2 = [2200] +# Step duration in miliseconds (Classic RTT is about 100ms) +stepDuration = 50 + # Set to True if you want your run to be deterministic, False if not -deterministic = False +deterministic = True # If your run is deterministic you can decide the random seed. This is ignore otherwise. randomSeed = "DAS" diff --git a/study.py b/study.py index fde8099..74dba63 100644 --- a/study.py +++ b/study.py @@ -28,7 +28,7 @@ def runOnce(config, shape, execID): shape.setSeed(config.randomSeed+"-"+str(shape)) random.seed(shape.randomSeed) - sim = Simulator(shape, config) + sim = Simulator(shape, config, execID) sim.initLogger() sim.initValidators() sim.initNetwork() @@ -36,7 +36,7 @@ def runOnce(config, shape, execID): sim.logger.info("Shape: %s ... Block Available: %d in %d steps" % (str(sim.shape.__dict__), result.blockAvailable, len(result.missingVector)), extra=sim.format) if config.dumpXML: - result.dump(execID) + result.dump() return result From 41e83991591eab9d9d8ea2e787c0064628c10cd0 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Mon, 27 Mar 2023 21:24:25 +0200 Subject: [PATCH 06/10] Add Tx and Rx stats to resultsi. Rebase to traffic progress. --- DAS/results.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/DAS/results.py b/DAS/results.py index 48512cd..469810c 100644 --- a/DAS/results.py +++ b/DAS/results.py @@ -16,10 +16,16 @@ class Result: self.missingVector = [] self.metrics = {} - def populate(self, shape, config, missingVector): + def populate(self, shape, config, missingVector, bandwidthVector): """It populates part of the result data inside a vector.""" self.shape = shape self.missingVector = missingVector + self.proTx = bandwidthVector[0] + self.proRx = bandwidthVector[1] + self.aveTx = bandwidthVector[2] + self.maxTx = bandwidthVector[3] + self.aveRx = bandwidthVector[4] + self.maxRx = bandwidthVector[5] missingSamples = missingVector[-1] if missingSamples == 0: self.blockAvailable = 1 From b5390b9f1b8d777f8df75a1174fe757e759255a8 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Wed, 29 Mar 2023 17:01:28 +0200 Subject: [PATCH 07/10] Remove traffic statsi. Rebase to traffic progress. --- DAS/results.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/DAS/results.py b/DAS/results.py index 469810c..48512cd 100644 --- a/DAS/results.py +++ b/DAS/results.py @@ -16,16 +16,10 @@ class Result: self.missingVector = [] self.metrics = {} - def populate(self, shape, config, missingVector, bandwidthVector): + def populate(self, shape, config, missingVector): """It populates part of the result data inside a vector.""" self.shape = shape self.missingVector = missingVector - self.proTx = bandwidthVector[0] - self.proRx = bandwidthVector[1] - self.aveTx = bandwidthVector[2] - self.maxTx = bandwidthVector[3] - self.aveRx = bandwidthVector[4] - self.maxRx = bandwidthVector[5] missingSamples = missingVector[-1] if missingSamples == 0: self.blockAvailable = 1 From 98423d29c056ee3424d65850be5551d0cf7e9072 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Thu, 30 Mar 2023 13:49:01 +0200 Subject: [PATCH 08/10] Typos and sets --- DAS/simulator.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 6ec3bfa..83ba876 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -59,8 +59,8 @@ class Simulator: self.logger.debug("Shuffled rows: %s" % str(rows), extra=self.format) self.logger.debug("Shuffled columns: %s" % str(columns), extra=self.format) - assignatedRows = [] - assignatedCols = [] + assignedRows = [] + assignedCols = [] for i in range(self.shape.numberNodes): if self.config.evenLineDistribution: if i < int(heavyVal/self.shape.vpn2): # First start with the heavy nodes @@ -71,15 +71,13 @@ class Simulator: start = offset+( j *self.shape.chi) end = offset+((j+1)*self.shape.chi) # Remove duplicates - r = list(dict.fromkeys(rows[start:end])) - c = list(dict.fromkeys(columns[start:end])) - r.sort() - c.sort() + r = set(rows[start:end]) + c = set(columns[start:end]) val = Validator(i, int(not i!=0), self.logger, self.shape, r, c) self.logger.debug("Validators %d row IDs: %s" % (val.ID, val.rowIDs), extra=self.format) self.logger.debug("Validators %d column IDs: %s" % (val.ID, val.columnIDs), extra=self.format) - assignatedRows = assignatedRows + r - assignatedCols = assignatedCols + c + assignedRows = assignedRows + list(r) + assignedCols = assignedCols + list(c) else: val = Validator(i, int(not i!=0), self.logger, self.shape) @@ -89,10 +87,10 @@ class Simulator: val.logIDs() self.validators.append(val) - assignatedRows.sort() - assignatedCols.sort() - self.logger.debug("Rows assignated: %s" % str(assignatedRows), extra=self.format) - self.logger.debug("Columns assignated: %s" % str(assignatedCols), extra=self.format) + assignedRows.sort() + assignedCols.sort() + self.logger.debug("Rows assigned: %s" % str(assignedRows), extra=self.format) + self.logger.debug("Columns assigned: %s" % str(assignedCols), extra=self.format) self.logger.debug("Validators initialized.", extra=self.format) def initNetwork(self): From 4b7bf81cee8b7125facf544d3c1fa402db4e3fda Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Thu, 30 Mar 2023 13:52:40 +0200 Subject: [PATCH 09/10] Remove stop condition fix --- DAS/simulator.py | 10 ++++------ config_example.py | 3 --- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 83ba876..b7b7dff 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -273,13 +273,11 @@ class Simulator: }) if missingSamples == oldMissingSamples: - if len(missingVector) > self.config.steps4StopCondition: - if missingSamples == missingVector[len(missingVector)-1-self.config.steps4StopCondition]: - self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format) - if self.config.diagnostics: - self.printDiagnostics() - break + self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format) + if self.config.diagnostics: + self.printDiagnostics() missingVector.append(missingSamples) + break elif missingSamples == 0: #self.logger.info("The entire block is available at step %d, with failure rate %d !" % (steps, self.shape.failureRate), extra=self.format) missingVector.append(missingSamples) diff --git a/config_example.py b/config_example.py index d07a899..f799c20 100644 --- a/config_example.py +++ b/config_example.py @@ -77,9 +77,6 @@ randomSeed = "DAS" # If True, print diagnostics when the block is not available diagnostics = False -# Number of steps without progress to stop simulation -steps4StopCondition = 7 - def nextShape(): for run, fr, class1ratio, chi, vpn1, vpn2, blockSize, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( runs, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): From d2a653b07c637204929473da3ef9ea42a86d9b27 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Thu, 30 Mar 2023 14:11:12 +0200 Subject: [PATCH 10/10] Remove plotting config --- DAS/simulator.py | 1 - config_example.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 674c1e7..b0ea19f 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -3,7 +3,6 @@ import networkx as nx import logging, random import pandas as pd -import matplotlib from functools import partial, partialmethod from datetime import datetime from DAS.tools import * diff --git a/config_example.py b/config_example.py index 2187108..e048028 100644 --- a/config_example.py +++ b/config_example.py @@ -22,10 +22,6 @@ dumpXML = 1 # save progress vectors to XML saveProgress = 1 - -# plot progress for each run to PNG -plotProgress = 1 - visualization = 1 logLevel = logging.INFO