From 5718fe14b7954476e18d5df9d4cb557daecd4bab Mon Sep 17 00:00:00 2001 From: Arunima Chaudhuri Date: Mon, 20 May 2024 14:02:39 +0000 Subject: [PATCH 01/34] Gossipsub implementation Signed-off-by: Arunima Chaudhuri --- DAS/node.py | 34 +++++++++++++++++++++++++++++++++- DAS/simulator.py | 14 +++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/DAS/node.py b/DAS/node.py index 85aaf8a..5774242 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -246,6 +246,14 @@ class Node: self.statsRxDupInSlot += 1 self.statsRxInSlot += 1 + def receiveSegmentViaGossip(self, rID, cID): + """Receive a segment via gossipsub protocol.""" + if not self.amImalicious: + self.logger.trace("Recv via gossipsub: %d: %d,%d", self.ID, rID, cID, extra=self.format) + self.receivedBlock.setSegment(rID, cID) + self.sampleRecvCount += 1 + self.statsRxInSlot += 1 + def addToSendQueue(self, rID, cID): """Queue a segment for forwarding.""" if self.perNodeQueue and not self.amImalicious: @@ -504,7 +512,26 @@ class Node: if self.statsTxInSlot >= self.bwUplink: return - def send(self): + def gossipSub(self, rows, cols): + """ This function facilitates the Gossipsub protocol for segment distribution among nodes. + It ensures that each node receives any missing segments by checking other nodes in the network. + + Args: + rows (dict): A hash table where the keys are row IDs and the values are lists of nodes that contain these rows. + cols (dict): A hash table where the keys are column IDs and the values are lists of nodes that contain these columns. + + Description: + - The function iterates through all row IDs and column IDs. + - For each segment identified by a row ID (rID) and a column ID (cID): + - It checks if the current node (self) already has the segment. + - If the segment is missing, it attempts to receive the segment from other nodes using the Gossipsub protocol via the receiveSegmentViaGossip method. + """ + for rID in rows: + for cID in cols: + if not self.receivedBlock.getSegment(rID, cID): + self.receiveSegmentViaGossip(rID, cID) + + def send(self, rows, cols): """ Send as much as we can in the timestep, limited by bwUplink.""" # process node level send queue @@ -529,6 +556,11 @@ class Node: self.runDumbRandomScheduler() if self.statsTxInSlot >= self.bwUplink: return + + if not self.amImalicious: + self.gossipSub(rows, cols) + if self.statsTxInSlot >= self.bwUplink: + return def logRows(self): """It logs the rows assigned to the validator.""" diff --git a/DAS/simulator.py b/DAS/simulator.py index 7d4b341..9aed26d 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -278,10 +278,22 @@ class Simulator: self.logger.debug("Expected Samples: %d" % expected, extra=self.format) self.logger.debug("Missing Samples: %d" % missingSamples, extra=self.format) oldMissingSamples = missingSamples + rows = {} + cols = {} + for i in range(0, self.shape.numberNodes): + if not self.validators[i].amIproposer and not self.validators[i].amImalicious: + for id in self.validators[i].columnIDs: + if id not in cols: + cols[id] = [] + cols[id].append(self.validators[i]) + for id in self.validators[i].rowIDs: + if id not in rows: + rows[id] = [] + rows[id].append(self.validators[i]) self.logger.debug("PHASE SEND %d" % steps, extra=self.format) for i in range(0,self.shape.numberNodes): if not self.validators[i].amImalicious: - self.validators[i].send() + self.validators[i].send(rows, cols) self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format) for i in range(1,self.shape.numberNodes): self.validators[i].receiveRowsColumns() From 45f773b184a6ce7122d2b431c108081e64429731 Mon Sep 17 00:00:00 2001 From: Arunima Chaudhuri Date: Tue, 21 May 2024 08:34:58 +0000 Subject: [PATCH 02/34] add gossipsub as a parameter in the config file Signed-off-by: Arunima Chaudhuri --- DAS/node.py | 69 ++++++++++++++++++++++++++---------------------- DAS/simulator.py | 2 +- smallConf.py | 3 +++ 3 files changed, 42 insertions(+), 32 deletions(-) diff --git a/DAS/node.py b/DAS/node.py index 5774242..ef36930 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -246,10 +246,10 @@ class Node: self.statsRxDupInSlot += 1 self.statsRxInSlot += 1 - def receiveSegmentViaGossip(self, rID, cID): - """Receive a segment via gossipsub protocol.""" + def receiveSegmentViaGossip(self, rID, cID, source): + """Receive a segment via gossipsub protocol from a specific source.""" if not self.amImalicious: - self.logger.trace("Recv via gossipsub: %d: %d,%d", self.ID, rID, cID, extra=self.format) + self.logger.trace("Recv via gossipsub %d-> %d: %d,%d", source, self.ID, rID, cID, extra=self.format) self.receivedBlock.setSegment(rID, cID) self.sampleRecvCount += 1 self.statsRxInSlot += 1 @@ -526,41 +526,48 @@ class Node: - It checks if the current node (self) already has the segment. - If the segment is missing, it attempts to receive the segment from other nodes using the Gossipsub protocol via the receiveSegmentViaGossip method. """ - for rID in rows: - for cID in cols: + for rID, rSources in rows.items(): + for cID, cSources in cols.items(): if not self.receivedBlock.getSegment(rID, cID): - self.receiveSegmentViaGossip(rID, cID) + sources = list(set(rSources).intersection(cSources)) + if sources: + source = sources[0] # Pick the first source from the intersection + self.receiveSegmentViaGossip(rID, cID, source) + self.statsTxInSlot += 1 # request sent to receive segment via gossip + - def send(self, rows, cols): + def send(self, gossipsub, rows, cols): """ Send as much as we can in the timestep, limited by bwUplink.""" - # process node level send queue - if not self.amImalicious: - self.processSendQueue() - if self.statsTxInSlot >= self.bwUplink: - return + if gossipsub: + if not self.amImalicious: + self.gossipSub(rows, cols) + if self.statsTxInSlot >= self.bwUplink: + return - # process neighbor level send queues in shuffled breadth-first order - if not self.amImalicious: - self.processPerNeighborSendQueue() - if self.statsTxInSlot >= self.bwUplink: - return + else: + # process node level send queue + if not self.amImalicious: + self.processSendQueue() + if self.statsTxInSlot >= self.bwUplink: + return - # process possible segments to send in shuffled breadth-first order - if self.segmentShuffleScheduler and not self.amImalicious: - self.runSegmentShuffleScheduler() - if self.statsTxInSlot >= self.bwUplink: - return + # process neighbor level send queues in shuffled breadth-first order + if not self.amImalicious: + self.processPerNeighborSendQueue() + if self.statsTxInSlot >= self.bwUplink: + return - if self.dumbRandomScheduler and not self.amImalicious: - self.runDumbRandomScheduler() - if self.statsTxInSlot >= self.bwUplink: - return - - if not self.amImalicious: - self.gossipSub(rows, cols) - if self.statsTxInSlot >= self.bwUplink: - return + # process possible segments to send in shuffled breadth-first order + if self.segmentShuffleScheduler and not self.amImalicious: + self.runSegmentShuffleScheduler() + if self.statsTxInSlot >= self.bwUplink: + return + + if self.dumbRandomScheduler and not self.amImalicious: + self.runDumbRandomScheduler() + if self.statsTxInSlot >= self.bwUplink: + return def logRows(self): """It logs the rows assigned to the validator.""" diff --git a/DAS/simulator.py b/DAS/simulator.py index 9aed26d..a1b3cf4 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -293,7 +293,7 @@ class Simulator: self.logger.debug("PHASE SEND %d" % steps, extra=self.format) for i in range(0,self.shape.numberNodes): if not self.validators[i].amImalicious: - self.validators[i].send(rows, cols) + self.validators[i].send(self.config.gossipsub, rows, cols) self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format) for i in range(1,self.shape.numberNodes): self.validators[i].receiveRowsColumns() diff --git a/smallConf.py b/smallConf.py index 18a9c17..c2f2d9a 100644 --- a/smallConf.py +++ b/smallConf.py @@ -59,6 +59,9 @@ maliciousNodes = range(40,41,20) # If True, the malicious nodes will be assigned randomly; if False, a predefined pattern may be used randomizeMaliciousNodes = True +# When set to True, nodes will use the Gossipsub protocol for communication +gossipsub = False + # Per-topic mesh neighborhood size netDegrees = range(8, 9, 2) From d9f29dc5f29eb26c55ff54fe014a45103e506e63 Mon Sep 17 00:00:00 2001 From: Arunima Chaudhuri Date: Wed, 29 May 2024 11:48:09 +0000 Subject: [PATCH 03/34] --- DAS/node.py | 142 ++++++++++++++++++++++++++++------------------- DAS/simulator.py | 20 +++---- smallConf.py | 5 +- 3 files changed, 97 insertions(+), 70 deletions(-) diff --git a/DAS/node.py b/DAS/node.py index ef36930..e89575c 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -3,6 +3,8 @@ import random import collections import logging +from collections import defaultdict +import threading from DAS.block import * from DAS.tools import shuffled, shuffledDict, unionOfSamples from bitarray.util import zeros @@ -76,6 +78,7 @@ class Node: self.repairedSampleCount = 0 self.logger = logger self.validators = validators + self.received_gossip = defaultdict(list) if amIproposer: self.nodeClass = 0 @@ -246,14 +249,6 @@ class Node: self.statsRxDupInSlot += 1 self.statsRxInSlot += 1 - def receiveSegmentViaGossip(self, rID, cID, source): - """Receive a segment via gossipsub protocol from a specific source.""" - if not self.amImalicious: - self.logger.trace("Recv via gossipsub %d-> %d: %d,%d", source, self.ID, rID, cID, extra=self.format) - self.receivedBlock.setSegment(rID, cID) - self.sampleRecvCount += 1 - self.statsRxInSlot += 1 - def addToSendQueue(self, rID, cID): """Queue a segment for forwarding.""" if self.perNodeQueue and not self.amImalicious: @@ -512,62 +507,97 @@ class Node: if self.statsTxInSlot >= self.bwUplink: return - def gossipSub(self, rows, cols): - """ This function facilitates the Gossipsub protocol for segment distribution among nodes. - It ensures that each node receives any missing segments by checking other nodes in the network. + def sendGossip(self, neigh): + """Simulate sending row and column IDs to a peer.""" + have_info = {'source': self.ID, 'rowIDs': self.rowIDs, 'columnIDs': self.columnIDs} + neigh.node.received_gossip[self.ID].append(have_info) + neigh.node.msgRecvCount += 1 + self.logger.debug(f"Gossip sent to {neigh.node.ID}: {neigh.node.received_gossip}", extra=self.format) - Args: - rows (dict): A hash table where the keys are row IDs and the values are lists of nodes that contain these rows. - cols (dict): A hash table where the keys are column IDs and the values are lists of nodes that contain these columns. - - Description: - - The function iterates through all row IDs and column IDs. - - For each segment identified by a row ID (rID) and a column ID (cID): - - It checks if the current node (self) already has the segment. - - If the segment is missing, it attempts to receive the segment from other nodes using the Gossipsub protocol via the receiveSegmentViaGossip method. + def process_received_gossip(self, simulator): """ - for rID, rSources in rows.items(): - for cID, cSources in cols.items(): - if not self.receivedBlock.getSegment(rID, cID): - sources = list(set(rSources).intersection(cSources)) - if sources: - source = sources[0] # Pick the first source from the intersection - self.receiveSegmentViaGossip(rID, cID, source) - self.statsTxInSlot += 1 # request sent to receive segment via gossip + Processes received gossip messages to request and receive data segments. + For each segment not already received, it simulates requesting the segment, + logs the request and receipt, and updates the segment status and relevant counters. + """ + for sender, have_infos in self.received_gossip.items(): + for have_info in have_infos: + for rowID in have_info['rowIDs']: + for columnID in have_info['columnIDs']: + if not self.receivedBlock.getSegment(rowID, columnID): + # request for the segment + self.logger.debug(f"Requesting segment ({rowID}, {columnID}) from {have_info['source']}", extra=self.format) + self.msgSentCount += 1 + # source sends the segment + self.logger.debug(f"Sending segment ({rowID}, {columnID}) to {self.ID} from {have_info['source']}", extra=self.format) + simulator.validators[have_info['source']].sampleSentCount += 1 + simulator.validators[have_info['source']].statsTxInSlot += 1 + # receive the segment + self.receivedBlock.setSegment(rowID, columnID) + self.sampleRecvCount += 1 + self.logger.debug(f"Received segment ({rowID}, {columnID}) via gossip from {have_info['source']}", extra=self.format) + self.received_gossip.clear() - - def send(self, gossipsub, rows, cols): + def gossip(self, simulator): + """ + Periodically sends gossip messages to a random subset of neighbors to share information + about data segments (row and column IDs). The process involves: + 1. Selecting a random subset of row and column neighbors. + 2. Sending the node's current state (row and column IDs) to these neighbors. + 3. Neighbors process the received gossip and update their state accordingly. + + This ensures data dissemination across the network with minimal delay, + occurring at intervals defined by the HEARTBEAT timer. + """ + if self.rowIDs: + rID = random.choice(list(self.rowIDs)) + rowNeighs = list(self.rowNeighbors[rID].values()) + num_row_peers = random.randint(1, len(rowNeighs)) + selected_row_neighs = random.sample(rowNeighs, num_row_peers) + for rowNeigh in selected_row_neighs: + self.sendGossip(rowNeigh) + self.msgSentCount += 1 + rowNeigh.node.process_received_gossip(simulator) + if self.statsTxInSlot >= self.bwUplink: + return + + if self.columnIDs: + cID = random.choice(list(self.columnIDs)) + columnNeighs = list(self.columnNeighbors[cID].values()) + num_column_peers = random.randint(1, len(columnNeighs)) + selected_column_neighs = random.sample(columnNeighs, num_column_peers) + for columnNeigh in selected_column_neighs: + self.sendGossip(columnNeigh) + self.msgSentCount += 1 + columnNeigh.node.process_received_gossip(simulator) + if self.statsTxInSlot >= self.bwUplink: + return + + def send(self): """ Send as much as we can in the timestep, limited by bwUplink.""" - if gossipsub: - if not self.amImalicious: - self.gossipSub(rows, cols) - if self.statsTxInSlot >= self.bwUplink: - return + # process node level send queue + if not self.amImalicious: + self.processSendQueue() + if self.statsTxInSlot >= self.bwUplink: + return - else: - # process node level send queue - if not self.amImalicious: - self.processSendQueue() - if self.statsTxInSlot >= self.bwUplink: - return + # process neighbor level send queues in shuffled breadth-first order + if not self.amImalicious: + self.processPerNeighborSendQueue() + if self.statsTxInSlot >= self.bwUplink: + return - # process neighbor level send queues in shuffled breadth-first order - if not self.amImalicious: - self.processPerNeighborSendQueue() - if self.statsTxInSlot >= self.bwUplink: - return + # process possible segments to send in shuffled breadth-first order + if self.segmentShuffleScheduler and not self.amImalicious: + self.runSegmentShuffleScheduler() + if self.statsTxInSlot >= self.bwUplink: + return - # process possible segments to send in shuffled breadth-first order - if self.segmentShuffleScheduler and not self.amImalicious: - self.runSegmentShuffleScheduler() - if self.statsTxInSlot >= self.bwUplink: - return - - if self.dumbRandomScheduler and not self.amImalicious: - self.runDumbRandomScheduler() - if self.statsTxInSlot >= self.bwUplink: - return + if self.dumbRandomScheduler and not self.amImalicious: + self.runDumbRandomScheduler() + if self.statsTxInSlot >= self.bwUplink: + return def logRows(self): """It logs the rows assigned to the validator.""" diff --git a/DAS/simulator.py b/DAS/simulator.py index a1b3cf4..06c5510 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -278,22 +278,16 @@ class Simulator: self.logger.debug("Expected Samples: %d" % expected, extra=self.format) self.logger.debug("Missing Samples: %d" % missingSamples, extra=self.format) oldMissingSamples = missingSamples - rows = {} - cols = {} - for i in range(0, self.shape.numberNodes): - if not self.validators[i].amIproposer and not self.validators[i].amImalicious: - for id in self.validators[i].columnIDs: - if id not in cols: - cols[id] = [] - cols[id].append(self.validators[i]) - for id in self.validators[i].rowIDs: - if id not in rows: - rows[id] = [] - rows[id].append(self.validators[i]) + self.logger.debug("PHASE SEND %d" % steps, extra=self.format) for i in range(0,self.shape.numberNodes): if not self.validators[i].amImalicious: - self.validators[i].send(self.config.gossipsub, rows, cols) + self.validators[i].send() + if steps % self.config.heartbeat == 0 and self.config.gossip: + self.logger.debug("PHASE GOSSIP %d" % steps, extra=self.format) + for i in range(1,self.shape.numberNodes): + if not self.validators[i].amImalicious: + self.validators[i].gossip(self) self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format) for i in range(1,self.shape.numberNodes): self.validators[i].receiveRowsColumns() diff --git a/smallConf.py b/smallConf.py index c2f2d9a..7172013 100644 --- a/smallConf.py +++ b/smallConf.py @@ -60,7 +60,10 @@ maliciousNodes = range(40,41,20) randomizeMaliciousNodes = True # When set to True, nodes will use the Gossipsub protocol for communication -gossipsub = False +gossip = True + +# Heartbeat interval for gossip messages in simulation steps +heartbeat = 10 # Per-topic mesh neighborhood size netDegrees = range(8, 9, 2) From e5dd13bd97cd7cccff81bc6272ed6df9ff242980 Mon Sep 17 00:00:00 2001 From: Arunima Chaudhuri Date: Wed, 29 May 2024 11:51:02 +0000 Subject: [PATCH 04/34] set heartbeat to 20 steps Signed-off-by: Arunima Chaudhuri --- smallConf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smallConf.py b/smallConf.py index 7172013..1c60cc0 100644 --- a/smallConf.py +++ b/smallConf.py @@ -63,7 +63,7 @@ randomizeMaliciousNodes = True gossip = True # Heartbeat interval for gossip messages in simulation steps -heartbeat = 10 +heartbeat = 20 # Per-topic mesh neighborhood size netDegrees = range(8, 9, 2) From 1e4aefe2612575e89fe42c167e6884ea0ca537f2 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Thu, 30 May 2024 14:49:40 +0000 Subject: [PATCH 05/34] things are good before the plotting part --- DAS/node.py | 8 +++----- DAS/results.py | 1 - DAS/shape.py | 14 +++----------- DAS/simulator.py | 19 ++++++++++++++++--- smallConf.py | 24 ++++++++++++------------ 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/DAS/node.py b/DAS/node.py index 85aaf8a..1787af3 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -48,7 +48,7 @@ class Node: """It returns the node ID.""" return str(self.ID) - def __init__(self, ID, amIproposer, amImalicious, logger, shape, config, + def __init__(self, ID, amIproposer, nodeClass, amImalicious, logger, shape, config, validators, rows = set(), columns = set()): """It initializes the node, and eventual validators, following the simulation configuration in shape and config. @@ -82,7 +82,7 @@ class Node: self.rowIDs = range(shape.nbRows) self.columnIDs = range(shape.nbCols) else: - self.nodeClass = 1 if (self.ID <= shape.numberNodes * shape.class1ratio) else 2 + self.nodeClass = nodeClass self.vpn = len(validators) #TODO: needed by old code, change to fn self.rowIDs = set(rows) @@ -120,10 +120,8 @@ class Node: # 1 Mbps ~= 1e6 mbps * 0.050 s / (560*8) bits ~= 11 segments/timestep if self.amIproposer: self.bwUplink = shape.bwUplinkProd - elif self.nodeClass == 1: - self.bwUplink = shape.bwUplink1 else: - self.bwUplink = shape.bwUplink2 + self.bwUplink = shape.nodeTypes[self.nodeClass]['bwUplinks'] self.bwUplink *= 1e3 / 8 * config.stepDuration / config.segmentSize self.repairOnTheFly = config.evalConf(self, config.repairOnTheFly, shape) diff --git a/DAS/results.py b/DAS/results.py index f679702..2dc8db4 100644 --- a/DAS/results.py +++ b/DAS/results.py @@ -24,7 +24,6 @@ class Result: self.restoreColumnCount = [0] * shape.numberNodes self.repairedSampleCount = [0] * shape.numberNodes self.numberNodes = shape.numberNodes - self.class1ratio = shape.class1ratio def copyValidators(self, validators): """Copy information from simulator.validators to result.""" diff --git a/DAS/shape.py b/DAS/shape.py index ab17c4a..bb8ede2 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,7 +3,7 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" def __init__(self, nbCols, nbColsK, nbRows, nbRowsK, - numberNodes, failureModel, failureRate, maliciousNodes, class1ratio, custodyRows, custodyCols, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run): + numberNodes, failureModel, failureRate, maliciousNodes, custodyRows, custodyCols, netDegree, bwUplinkProd, run, nodeTypes): """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberNodes = numberNodes @@ -15,14 +15,10 @@ class Shape: self.failureRate = failureRate self.maliciousNodes = maliciousNodes self.netDegree = netDegree - self.class1ratio = class1ratio self.custodyRows = custodyRows self.custodyCols = custodyCols - self.vpn1 = vpn1 - self.vpn2 = vpn2 self.bwUplinkProd = bwUplinkProd - self.bwUplink1 = bwUplink1 - self.bwUplink2 = bwUplink2 + self.nodeTypes = nodeTypes self.randomSeed = "" def __repr__(self): @@ -35,17 +31,13 @@ class Shape: shastr += "-nn-"+str(self.numberNodes) shastr += "-fm-"+str(self.failureModel) shastr += "-fr-"+str(self.failureRate) - shastr += "-c1r-"+str(self.class1ratio) shastr += "-cusr-"+str(self.custodyRows) shastr += "-cusc-"+str(self.custodyCols) - shastr += "-vpn1-"+str(self.vpn1) - shastr += "-vpn2-"+str(self.vpn2) shastr += "-bwupprod-"+str(self.bwUplinkProd) - shastr += "-bwup1-"+str(self.bwUplink1) - shastr += "-bwup2-"+str(self.bwUplink2) shastr += "-nd-"+str(self.netDegree) shastr += "-r-"+str(self.run) shastr += "-mn-"+str(self.maliciousNodes) + shastr += "-ntypes-"+str(self.nodeTypes['group']) return shastr def setSeed(self, seed): diff --git a/DAS/simulator.py b/DAS/simulator.py index 3657b03..a62ddfc 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -44,6 +44,19 @@ class Simulator: self.proposerPublishToR = config.evalConf(self, config.proposerPublishToR, shape) self.proposerPublishToC = config.evalConf(self, config.proposerPublishToR, shape) + def getNodeClass(self, nodeIdx): + nodeRatios = [] + for _k, _v in self.shape.nodeTypes.items(): + if _k != "group": nodeRatios.append(_v['ratio']) + nodeCounts = [int(self.shape.numberNodes * ratio / sum(nodeRatios)) for ratio in nodeRatios] + commulativeSum = [nodeCounts[0]] + for count in nodeCounts[1: ]: + commulativeSum.append(commulativeSum[-1] + count) + commulativeSum[-1] = self.shape.numberNodes + for i, idx in enumerate(commulativeSum): + if nodeIdx < idx: + return i + 1 + def initValidators(self): """It initializes all the validators in the network.""" self.glob = Observer(self.logger, self.shape) @@ -125,11 +138,11 @@ class Simulator: self.logger.error("custodyRows has to be smaller than %d" % self.shape.nbRows) vs = [] - nodeClass = 1 if (i <= self.shape.numberNodes * self.shape.class1ratio) else 2 - vpn = self.shape.vpn1 if (nodeClass == 1) else self.shape.vpn2 + nodeClass = self.getNodeClass(i) + vpn = self.shape.nodeTypes[nodeClass]['validatorsPerNode'] for v in range(vpn): vs.append(initValidator(self.shape.nbRows, self.shape.custodyRows, self.shape.nbCols, self.shape.custodyCols)) - val = Node(i, int(not i!=0), amImalicious_value, self.logger, self.shape, self.config, vs) + val = Node(i, int(not i!=0), nodeClass, amImalicious_value, self.logger, self.shape, self.config, vs) if i == self.proposerID: val.initBlock() else: diff --git a/smallConf.py b/smallConf.py index 18a9c17..87334dd 100644 --- a/smallConf.py +++ b/smallConf.py @@ -77,17 +77,17 @@ validatorBasedCustody = False custodyRows = range(2, 3, 2) custodyCols = range(2, 3, 2) -# ratio of class1 nodes (see below for parameters per class) -class1ratios = [0.8] - -# Number of validators per beacon node -validatorsPerNode1 = [1] -validatorsPerNode2 = [5] - # Set uplink bandwidth in megabits/second bwUplinksProd = [200] -bwUplinks1 = [10] -bwUplinks2 = [200] + +nodeTypesGroup = [ + { + "group": "g1", + # nodeClass: node config + 1: {'validatorsPerNode': 1, 'bwUplinks': 10, 'ratio': 8}, + 2: {'validatorsPerNode': 5, 'bwUplinks': 200, 'ratio': 2} + } +] # Step duration in miliseconds (Classic RTT is about 100ms) stepDuration = 50 @@ -135,11 +135,11 @@ colsK = range(32, 65, 128) rowsK = range(32, 65, 128) def nextShape(): - for nbCols, nbColsK, nbRows, nbRowsK, run, fm, fr, mn, class1ratio, chR, chC, vpn1, vpn2, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product( - cols, colsK, rows, rowsK, runs, failureModels, failureRates, maliciousNodes, class1ratios, custodyRows, custodyCols, validatorsPerNode1, validatorsPerNode2, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2): + for nbCols, nbColsK, nbRows, nbRowsK, run, fm, fr, mn, chR, chC, nn, netDegree, bwUplinkProd, nodeTypes in itertools.product( + cols, colsK, rows, rowsK, runs, failureModels, failureRates, maliciousNodes, custodyRows, custodyCols, numberNodes, netDegrees, bwUplinksProd, nodeTypesGroup): # Network Degree has to be an even number if netDegree % 2 == 0: - shape = Shape(nbCols, nbColsK, nbRows, nbRowsK, nn, fm, fr, mn, class1ratio, chR, chC, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run) + shape = Shape(nbCols, nbColsK, nbRows, nbRowsK, nn, fm, fr, mn, chR, chC, netDegree, bwUplinkProd, run, nodeTypes) yield shape def evalConf(self, param, shape = None): From eb141deb5b7d69c4bd1a3b97af18e0e3c511c3c8 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sat, 1 Jun 2024 11:14:53 +0000 Subject: [PATCH 06/34] Compact code --- DAS/simulator.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index a62ddfc..6e7b74a 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -45,13 +45,9 @@ class Simulator: self.proposerPublishToC = config.evalConf(self, config.proposerPublishToR, shape) def getNodeClass(self, nodeIdx): - nodeRatios = [] - for _k, _v in self.shape.nodeTypes.items(): - if _k != "group": nodeRatios.append(_v['ratio']) + nodeRatios = [_v['ratio'] for _k, _v in self.shape.nodeTypes.items() if _k != "group"] nodeCounts = [int(self.shape.numberNodes * ratio / sum(nodeRatios)) for ratio in nodeRatios] - commulativeSum = [nodeCounts[0]] - for count in nodeCounts[1: ]: - commulativeSum.append(commulativeSum[-1] + count) + commulativeSum = [sum(nodeCounts[:i+1]) for i in range(len(nodeCounts))] commulativeSum[-1] = self.shape.numberNodes for i, idx in enumerate(commulativeSum): if nodeIdx < idx: From 0d4c74a01a2409acd1e5994a75085b3ae0c87e6d Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sun, 2 Jun 2024 07:39:26 +0000 Subject: [PATCH 07/34] Plot textbox updated for new config type --- DAS/visualizor.py | 268 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 231 insertions(+), 37 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index a95c9c4..473a900 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -63,7 +63,21 @@ class Visualizor: for i in range(0, len(text), 2): d[text[i]] = text[i + 1] return d - + + def __getNodeTypes__(self, group): + theGroup = dict() + for nt in self.config.nodeTypesGroup: + if nt['group'] == group: + for _k, _v in nt.items(): + if _k != 'group': + theGroup[_k] = { + "vpn": _v["validatorsPerNode"], + "bw": _v["bwUplinks"] + } + break + + return theGroup + def plotHeatmaps(self, x, y): """Plot the heatmap using the parameters given as x axis and y axis""" print("Plotting heatmap "+x+" vs "+y) @@ -169,10 +183,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Box Plot of Restore Row Count by Nodes" conf["xlabel"] = "Node Type" @@ -196,10 +215,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Box Plot of Restore Column Count by Nodes" conf["xlabel"] = "Node Type" @@ -223,10 +247,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Boxen Plot of Restore Row Count by Nodes" conf["xlabel"] = "Restore Row Count" @@ -248,10 +277,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Boxen Plot of Restore Column Count by Nodes" conf["xlabel"] = "Restore Column Count" @@ -273,10 +307,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Restore Row Count by Nodes" conf["xlabel"] = "Restore Row Count" @@ -302,10 +341,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Restore Column Count by Nodes" conf["xlabel"] = "Restore Column Count" @@ -331,10 +375,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Messages Sent by Nodes" conf["xlabel"] = "Number of Messages Sent" @@ -359,10 +408,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Messages Received by Nodes" conf["xlabel"] = "Number of Messages Received" @@ -387,10 +441,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Samples Received by Nodes" conf["xlabel"] = "Number of Samples Received" @@ -415,10 +474,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Row-Col Distribution by Nodes" conf["xlabel"] = "Row-Col Distribution" @@ -443,10 +507,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Samples Repaired by Nodes" conf["xlabel"] = "Number of Samples Repaired" @@ -471,10 +540,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Samples Received by Nodes" conf["xlabel"] = "Node Type" @@ -498,10 +572,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Samples Repaired by Nodes" conf["xlabel"] = "Node Type" @@ -525,10 +604,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Row/Column Distribution" conf["xlabel"] = "Row/Column Type" @@ -557,10 +641,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Sent by Nodes" conf["xlabel"] = "Node Type" @@ -582,10 +671,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Received by Nodes" conf["xlabel"] = "Node Type" @@ -607,10 +701,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Samples Repaired by Nodes" conf["type"] = "individual_bar" @@ -629,10 +728,15 @@ class Visualizor: plt.clf() conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Row/Column Distribution" conf["xlabel"] = "" @@ -653,10 +757,15 @@ class Visualizor: """Plots the restoreRowCount for each node""" conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Restore Row Count for Each Node" conf["type"] = "individual_bar" @@ -676,10 +785,15 @@ class Visualizor: """Plots the restoreColumnCount for each node""" conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Restore Column Count for Each Node" conf["type"] = "individual_bar" @@ -699,10 +813,15 @@ class Visualizor: """Plots the percentage sampleRecv for each node""" conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Percentage of Samples Received by Nodes" conf["type"] = "individual_bar_with_2line" @@ -735,10 +854,15 @@ class Visualizor: """Box Plot of the sampleRecv for each node""" conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Samples Received by Nodes" conf["type"] = "individual_bar_with_2line" @@ -757,10 +881,15 @@ class Visualizor: """Plots the missing segments in the network""" conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize)+"\nMissing Segment: "+str(round(min(result.missingVector) * 100 / max(result.missingVector), 3))+"%"\ +"\nMissing Segments: "+str(result.missingVector[-1]) conf["title"] = "Missing Segments" @@ -792,10 +921,15 @@ class Visualizor: vector3 = [x * 100 for x in result.metrics["progress"]["samples received"]] conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Nodes/validators ready" conf["type"] = "plot" @@ -823,10 +957,15 @@ class Visualizor: vector3[i] = (vector3[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Sent data" conf["type"] = "plot" @@ -856,10 +995,15 @@ class Visualizor: vector2[i] = (vector2[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Received data" conf["type"] = "plot" @@ -889,10 +1033,15 @@ class Visualizor: vector2[i] = (vector2[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Duplicated data" conf["type"] = "plot" @@ -923,10 +1072,15 @@ class Visualizor: vector1 += [np.nan] * (len(vector2) - len(vector1)) conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Row/Column distribution" conf["type"] = "grouped_bar" @@ -951,10 +1105,15 @@ class Visualizor: """Plots the number of messages sent by all nodes""" conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Sent by Nodes" conf["type"] = "individual_bar" @@ -974,10 +1133,15 @@ class Visualizor: """Box Plot of the number of messages sent by all nodes""" conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Sent by Nodes" conf["xlabel"] = "Node Type" @@ -992,10 +1156,15 @@ class Visualizor: """Plots the number of messages received by all nodes""" conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Received by Nodes" conf["type"] = "individual_bar" @@ -1015,10 +1184,15 @@ class Visualizor: """Plots the number of messages received by all nodes""" conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Received by Nodes" conf["type"] = "individual_bar" @@ -1039,10 +1213,15 @@ class Visualizor: """Plots the number of samples repaired by all nodes""" conf = {} attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2']\ + +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Samples Repaired by Nodes" conf["type"] = "individual_bar" @@ -1097,11 +1276,16 @@ class Visualizor: xyS = dict() for result in self.results: attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] textBox = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"\ +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']\ - +"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2'] + +"\n"+nodeTypesTxt filename = "bsrn_" + attrbs['bsrn'] +\ "_bsrk_" + attrbs['bsrk'] +\ "_bscn_" + attrbs['bscn' ] +\ @@ -1158,11 +1342,16 @@ class Visualizor: xyS = dict() for result in self.results: attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] textBox = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nFailure rate: "+attrbs['fr']+"%"+"\nNodes: "+attrbs['nn']\ +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']\ - +"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2'] + +"\n"+nodeTypesTxt filename = "bsrn_" + attrbs['bsrn'] +\ "_bsrk_" + attrbs['bsrk'] +\ "_bscn_" + attrbs['bscn' ] +\ @@ -1219,11 +1408,16 @@ class Visualizor: xyS = dict() for result in self.results: attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] textBox = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNodes: "+attrbs['nn']+"\nMalicious Node: "+attrbs['mn']+"%"\ +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']\ - +"\nCustody 1: "+attrbs['vpn1']+"\nCustody 2: "+attrbs['vpn2'] + +"\n"+nodeTypesTxt filename = "bsrn_" + attrbs['bsrn'] +\ "_bsrk_" + attrbs['bsrk'] +\ "_bscn_" + attrbs['bscn' ] +\ From c2925935b3838349191cbbfa80ba7707344d8c1e Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sun, 2 Jun 2024 13:40:34 +0000 Subject: [PATCH 08/34] plotMissingSegments fixed --- DAS/visualizor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index 473a900..7d54ac4 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -72,7 +72,8 @@ class Visualizor: if _k != 'group': theGroup[_k] = { "vpn": _v["validatorsPerNode"], - "bw": _v["bwUplinks"] + "bw": _v["bwUplinks"], + "r": _v["ratio"] } break @@ -909,7 +910,7 @@ class Visualizor: maxi = max(v) conf["yaxismax"] = maxi x = result.shape.nbCols * result.shape.custodyRows + result.shape.nbRows * result.shape.custodyCols - conf["expected_value"] = (result.shape.numberNodes - 1) * (result.shape.class1ratio * result.shape.vpn1 * x + (1 - result.shape.class1ratio) * result.shape.vpn2 * x) + conf["expected_value"] = (result.shape.numberNodes - 1) * x * sum([(_v['r'] * _v['vpn']) for _v in nodeTypes.values()]) / sum([_v['r'] for _v in nodeTypes.values()]) conf["line_label"] = "Total segments to deliver" plotData(conf) print("Plot %s created." % conf["path"]) From c5214093ead04f69012dd58afb285331cbc095a8 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 4 Jun 2024 07:02:31 +0000 Subject: [PATCH 09/34] Simulator progressVector updated --- DAS/observer.py | 2 +- DAS/shape.py | 1 + DAS/simulator.py | 36 +++++++++++++++++------------------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/DAS/observer.py b/DAS/observer.py index 7abc606..80b68dc 100644 --- a/DAS/observer.py +++ b/DAS/observer.py @@ -96,7 +96,7 @@ class Observer: return np.mean(l) if l else np.NaN trafficStats = {} - for cl in range(0,3): + for cl in self.config.nodeClasses: Tx = [v.statsTxInSlot for v in validators if v.nodeClass == cl] Rx = [v.statsRxInSlot for v in validators if v.nodeClass == cl] RxDup = [v.statsRxDupInSlot for v in validators if v.nodeClass == cl] diff --git a/DAS/shape.py b/DAS/shape.py index bb8ede2..dea8b7a 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -19,6 +19,7 @@ class Shape: self.custodyCols = custodyCols self.bwUplinkProd = bwUplinkProd self.nodeTypes = nodeTypes + self.nodeClasses = [0] + [_k for _k in nodeTypes.keys() if _k != "group"] self.randomSeed = "" def __repr__(self): diff --git a/DAS/simulator.py b/DAS/simulator.py index 6e7b74a..b801e77 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -320,12 +320,9 @@ class Simulator: cnN = "nodes ready" cnV = "validators ready" cnT0 = "TX builder mean" - cnT1 = "TX class1 mean" - cnT2 = "TX class2 mean" - cnR1 = "RX class1 mean" - cnR2 = "RX class2 mean" - cnD1 = "Dup class1 mean" - cnD2 = "Dup class2 mean" + cnT = lambda i: f"TX class{i} mean" + cnR = lambda i: f"RX class{i} mean" + cnD = lambda i: f"Dup class{i} mean" # if custody is based on the requirements of underlying individual # validators, we can get detailed data on how many validated. @@ -334,19 +331,20 @@ class Simulator: cnVv = validatorProgress else: cnVv = validatorAllProgress - - progressVector.append({ - cnS:sampleProgress, - cnN:nodeProgress, - cnV:cnVv, - cnT0: trafficStats[0]["Tx"]["mean"], - cnT1: trafficStats[1]["Tx"]["mean"], - cnT2: trafficStats[2]["Tx"]["mean"], - cnR1: trafficStats[1]["Rx"]["mean"], - cnR2: trafficStats[2]["Rx"]["mean"], - cnD1: trafficStats[1]["RxDup"]["mean"], - cnD2: trafficStats[2]["RxDup"]["mean"], - }) + + progressDict = { + cnS: sampleProgress, + cnN: nodeProgress, + cnV: cnVv, + cnT0: trafficStats[0]["Tx"]["mean"] + } + for nc in self.shape.nodeClasses: + if nc != 0: + progressDict[cnT(nc)] = trafficStats[nc]["Tx"]["mean"] + progressDict[cnR(nc)] = trafficStats[nc]["Rx"]["mean"] + progressDict[cnD(nc)] = trafficStats[nc]["RxDup"]["mean"] + + progressVector.append(progressDict) if missingSamples == oldMissingSamples: if len(missingVector) > self.config.steps4StopCondition: From 3ed4248aa024d2491e010475a26c9a1e44a13e9d Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 4 Jun 2024 08:43:19 +0000 Subject: [PATCH 10/34] Sent Data plot updated --- DAS/visualizor.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index 7d54ac4..cc917e2 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -14,7 +14,8 @@ def plotData(conf): plt.text(1.05, 0.05, conf["textBox"], fontsize=14, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props) if conf["type"] == "plot" or conf["type"] == "plot_with_1line": for i in range(len(conf["data"])): - plt.plot(conf["xdots"], conf["data"][i], conf["colors"][i], label=conf["labels"][i]) + # plt.plot(conf["xdots"], conf["data"][i], conf["colors"][i], label=conf["labels"][i]) + plt.plot(conf["xdots"], conf["data"][i], label=conf["labels"][i]) elif conf["type"] == "individual_bar" or conf["type"] == "individual_bar_with_2line": plt.bar(conf["xdots"], conf["data"]) elif conf["type"] == "grouped_bar": @@ -949,13 +950,12 @@ class Visualizor: def plotSentData(self, result, plotPath): """Plots the percentage of nodes ready in the network""" - vector1 = result.metrics["progress"]["TX builder mean"] - vector2 = result.metrics["progress"]["TX class1 mean"] - vector3 = result.metrics["progress"]["TX class2 mean"] - for i in range(len(vector1)): - vector1[i] = (vector1[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 - vector2[i] = (vector2[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 - vector3[i] = (vector3[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 + vectors = { 0: result.metrics["progress"]["TX builder mean"] } + for nc in result.shape.nodeClasses: + if nc != 0: vectors[nc] = result.metrics["progress"][f"TX class{nc} mean"] + for _k in vectors.keys(): + for i in range(len(vectors[0])): + vectors[_k][i] = (vectors[_k][i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 conf = {} attrbs = self.__get_attrbs__(result) nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) @@ -972,12 +972,16 @@ class Visualizor: conf["type"] = "plot" conf["legLoc"] = 2 conf["desLoc"] = 2 - conf["colors"] = ["y-", "c-", "m-"] - conf["labels"] = ["Block Builder", "Solo stakers", "Staking pools"] + # conf["colors"] = ["y-", "c-", "m-"] + conf["labels"] = ["Block Builder"] + conf["data"] = [vectors[0]] + for _k, _v in vectors.items(): + if _k != 0: + conf["labels"].append(f"Node Class: {_k}") + conf["data"].append(_v) conf["xlabel"] = "Time (ms)" conf["ylabel"] = "Bandwidth (MBits/s)" - conf["data"] = [vector1, vector2, vector3] - conf["xdots"] = [x*self.config.stepDuration for x in range(len(vector1))] + conf["xdots"] = [x*self.config.stepDuration for x in range(len(vectors[0]))] conf["path"] = plotPath+"/sentData.png" maxi = 0 for v in conf["data"]: From fe8cde9a6193a022023130d065060c3d758eafbf Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 4 Jun 2024 09:15:40 +0000 Subject: [PATCH 11/34] Received data plot updated --- DAS/visualizor.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index cc917e2..c5b2d35 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -954,7 +954,7 @@ class Visualizor: for nc in result.shape.nodeClasses: if nc != 0: vectors[nc] = result.metrics["progress"][f"TX class{nc} mean"] for _k in vectors.keys(): - for i in range(len(vectors[0])): + for i in range(len(list(vectors.values())[0])): vectors[_k][i] = (vectors[_k][i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 conf = {} attrbs = self.__get_attrbs__(result) @@ -981,7 +981,7 @@ class Visualizor: conf["data"].append(_v) conf["xlabel"] = "Time (ms)" conf["ylabel"] = "Bandwidth (MBits/s)" - conf["xdots"] = [x*self.config.stepDuration for x in range(len(vectors[0]))] + conf["xdots"] = [x*self.config.stepDuration for x in range(len(list(vectors.values())[0]))] conf["path"] = plotPath+"/sentData.png" maxi = 0 for v in conf["data"]: @@ -993,11 +993,12 @@ class Visualizor: def plotRecvData(self, result, plotPath): """Plots the percentage of nodes ready in the network""" - vector1 = result.metrics["progress"]["RX class1 mean"] - vector2 = result.metrics["progress"]["RX class2 mean"] - for i in range(len(vector1)): - vector1[i] = (vector1[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 - vector2[i] = (vector2[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 + vectors = {} + for nc in result.shape.nodeClasses: + if nc != 0: vectors[nc] = result.metrics["progress"][f"RX class{nc} mean"] + for _k in vectors.keys(): + for i in range(len(list(vectors.values())[0])): + vectors[_k][i] = (vectors[_k][i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 conf = {} attrbs = self.__get_attrbs__(result) nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) @@ -1014,12 +1015,15 @@ class Visualizor: conf["type"] = "plot" conf["legLoc"] = 2 conf["desLoc"] = 2 - conf["colors"] = ["c-", "m-"] - conf["labels"] = ["Solo stakers", "Staking pools"] + # conf["colors"] = ["c-", "m-"] + conf["labels"] = [] + conf["data"] = [] + for _k, _v in vectors.items(): + conf["labels"].append(f"Node Class: {_k}") + conf["data"].append(_v) conf["xlabel"] = "Time (ms)" conf["ylabel"] = "Bandwidth (MBits/s)" - conf["data"] = [vector1, vector2] - conf["xdots"] = [x*self.config.stepDuration for x in range(len(vector1))] + conf["xdots"] = [x*self.config.stepDuration for x in range(len(list(vectors.values())[0]))] conf["path"] = plotPath+"/recvData.png" maxi = 0 for v in conf["data"]: From 201fc08da025b1e2c37c98bc0c713cd27cdaa07b Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 4 Jun 2024 09:30:12 +0000 Subject: [PATCH 12/34] duplicate data plot fixed --- DAS/visualizor.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index c5b2d35..a4565fb 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -1035,11 +1035,12 @@ class Visualizor: def plotDupData(self, result, plotPath): """Plots the percentage of nodes ready in the network""" - vector1 = result.metrics["progress"]["Dup class1 mean"] - vector2 = result.metrics["progress"]["Dup class2 mean"] - for i in range(len(vector1)): - vector1[i] = (vector1[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 - vector2[i] = (vector2[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 + vectors = {} + for nc in result.shape.nodeClasses: + if nc != 0: vectors[nc] = result.metrics["progress"][f"Dup class{nc} mean"] + for _k in vectors.keys(): + for i in range(len(list(vectors.values())[0])): + vectors[_k][i] = (vectors[_k][i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000 conf = {} attrbs = self.__get_attrbs__(result) nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) @@ -1056,12 +1057,15 @@ class Visualizor: conf["type"] = "plot" conf["legLoc"] = 2 conf["desLoc"] = 2 - conf["colors"] = ["c-", "m-"] - conf["labels"] = ["Solo stakers", "Staking pools"] + # conf["colors"] = ["c-", "m-"] + conf["labels"] = [] + conf["data"] = [] + for _k, _v in vectors.items(): + conf["labels"].append(f"Node Class: {_k}") + conf["data"].append(_v) conf["xlabel"] = "Time (ms)" conf["ylabel"] = "Bandwidth (MBits/s)" - conf["data"] = [vector1, vector2] - conf["xdots"] = [x*self.config.stepDuration for x in range(len(vector1))] + conf["xdots"] = [x*self.config.stepDuration for x in range(len(list(vectors.values())[0]))] conf["path"] = plotPath+"/dupData.png" maxi = 0 for v in conf["data"]: From bdbe95c219c35328640886a9287f05e19073d71e Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 4 Jun 2024 15:20:36 +0000 Subject: [PATCH 13/34] Samples Repaired Boxen plot updated --- DAS/visualizor.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index a4565fb..1506f4b 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -80,6 +80,18 @@ class Visualizor: return theGroup + def __getNodeRanges(self, shape): + nodeClasses, nodeRatios = [], [] + for _k, _v in shape.nodeTypes.items(): + if _k != "group": + nodeClasses.append(_k) + nodeRatios.append(_v['ratio']) + nodeCounts = [int(shape.numberNodes * ratio / sum(nodeRatios)) for ratio in nodeRatios] + commulativeSum = [sum(nodeCounts[:i+1]) for i in range(len(nodeCounts))] + commulativeSum[-1] = shape.numberNodes + + return nodeClasses, commulativeSum + def plotHeatmaps(self, x, y): """Plot the heatmap using the parameters given as x axis and y axis""" print("Plotting heatmap "+x+" vs "+y) @@ -587,10 +599,22 @@ class Visualizor: conf["title"] = "Number of Samples Repaired by Nodes" conf["xlabel"] = "Node Type" conf["ylabel"] = "Number of Samples Repaired" - n1 = int(result.numberNodes * result.class1ratio) - data = [result.repairedSampleCount[1: n1], result.repairedSampleCount[n1+1: ]] + data = [] + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + _start = 1 + for _range in nodeRanges: + data.append(result.repairedSampleCount[_start: _range]) + _start = _range + _values, _categories = [], [] + for _d, _nc in zip(data, nodeClasses): + _values += _d + _categories += [f'Class {_nc}'] * len(_d) + data = pd.DataFrame({ + 'values': _values, + 'category': _categories + }) plt.figure(figsize=(8, 6)) - sns.boxenplot(data=data, width=0.8) + sns.boxenplot(x='category', y='values', data=data, width=0.8) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) From 92b67786220cbfe82ee5e3f71889e04e032df037 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 4 Jun 2024 21:05:34 +0000 Subject: [PATCH 14/34] Message sent boxen plot updated --- DAS/visualizor.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index 1506f4b..9fb0fd5 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -680,10 +680,21 @@ class Visualizor: conf["title"] = "Number of Messages Sent by Nodes" conf["xlabel"] = "Node Type" conf["ylabel"] = "Number of Messages Sent" - n1 = int(result.numberNodes * result.class1ratio) - data = [result.msgSentCount[1: n1], result.msgSentCount[n1+1: ]] - labels = ["Class 1", "Class 2"] - sns.boxenplot(data=data, palette="Set2", ax=plt.gca()) + data = [] + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + _start = 1 + for _range in nodeRanges: + data.append(result.msgSentCount[_start: _range]) + _start = _range + _values, _categories = [], [] + for _d, _nc in zip(data, nodeClasses): + _values += _d + _categories += [f'Class {_nc}'] * len(_d) + data = pd.DataFrame({ + 'values': _values, + 'category': _categories + }) + sns.boxenplot(x='category', y='values', data=data, width=0.8) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) From 346195bd8d8fc062458d4aa8b153fdcbffe8f3b3 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 4 Jun 2024 21:19:55 +0000 Subject: [PATCH 15/34] Messages Received boxen plot updated --- DAS/visualizor.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index 9fb0fd5..951c86b 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -614,7 +614,7 @@ class Visualizor: 'category': _categories }) plt.figure(figsize=(8, 6)) - sns.boxenplot(x='category', y='values', data=data, width=0.8) + sns.boxenplot(x='category', y='values', data=data, width=0.8, palette="Set2", ax=plt.gca()) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -694,7 +694,7 @@ class Visualizor: 'values': _values, 'category': _categories }) - sns.boxenplot(x='category', y='values', data=data, width=0.8) + sns.boxenplot(x='category', y='values', data=data, width=0.8, palette="Set2", ax=plt.gca()) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -721,10 +721,21 @@ class Visualizor: conf["title"] = "Number of Messages Received by Nodes" conf["xlabel"] = "Node Type" conf["ylabel"] = "Number of Messages Received" - n1 = int(result.numberNodes * result.class1ratio) - data = [result.msgRecvCount[1: n1], result.msgRecvCount[n1+1: ]] - labels = ["Class 1", "Class 2"] - sns.boxenplot(data=data, palette="Set2", ax=plt.gca()) + data = [] + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + _start = 1 + for _range in nodeRanges: + data.append(result.msgRecvCount[_start: _range]) + _start = _range + _values, _categories = [], [] + for _d, _nc in zip(data, nodeClasses): + _values += _d + _categories += [f'Class {_nc}'] * len(_d) + data = pd.DataFrame({ + 'values': _values, + 'category': _categories + }) + sns.boxenplot(x='category', y='values', data=data, palette="Set2", ax=plt.gca()) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) From 86e9a78f34ff0646d77646501445705eed278362 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 4 Jun 2024 21:23:33 +0000 Subject: [PATCH 16/34] Samples Received boxen plot updated --- DAS/visualizor.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index 951c86b..85c5abc 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -567,10 +567,22 @@ class Visualizor: conf["title"] = "Number of Samples Received by Nodes" conf["xlabel"] = "Node Type" conf["ylabel"] = "Number of Samples Received" - n1 = int(result.numberNodes * result.class1ratio) - data = [result.sampleRecvCount[1: n1], result.sampleRecvCount[n1+1: ]] + data = [] + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + _start = 1 + for _range in nodeRanges: + data.append(result.sampleRecvCount[_start: _range]) + _start = _range + _values, _categories = [], [] + for _d, _nc in zip(data, nodeClasses): + _values += _d + _categories += [f'Class {_nc}'] * len(_d) + data = pd.DataFrame({ + 'values': _values, + 'category': _categories + }) plt.figure(figsize=(8, 6)) - sns.boxenplot(data=data, width=0.8) + sns.boxenplot(x='category', y='values', data=data, palette="Set2", ax=plt.gca(), width=0.8) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) From bfd75feaa9ae0ba7f49a2515a8ac96389264cae3 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 4 Jun 2024 21:28:22 +0000 Subject: [PATCH 17/34] Restore row cout boxen plot updated --- DAS/visualizor.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index 85c5abc..c96ae10 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -274,10 +274,22 @@ class Visualizor: conf["title"] = "Boxen Plot of Restore Row Count by Nodes" conf["xlabel"] = "Restore Row Count" conf["ylabel"] = "Nodes" - n1 = int(result.numberNodes * result.class1ratio) - data = [result.restoreRowCount[1: n1], result.restoreRowCount[n1+1: ]] + data = [] + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + _start = 1 + for _range in nodeRanges: + data.append(result.restoreRowCount[_start: _range]) + _start = _range + _values, _categories = [], [] + for _d, _nc in zip(data, nodeClasses): + _values += _d + _categories += [f'Class {_nc}'] * len(_d) + data = pd.DataFrame({ + 'values': _values, + 'category': _categories + }) plt.figure(figsize=(8, 6)) - sns.boxenplot(data=data, width=0.8) + sns.boxenplot(x='category', y='values', data=data, palette="Set2", ax=plt.gca(), width=0.8) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) From 8aaaf57ed622fed7c273dd3c0e3ecd28d67f2989 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 4 Jun 2024 21:32:47 +0000 Subject: [PATCH 18/34] Restore column count boxen plot updated --- DAS/visualizor.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index c96ae10..83c96ab 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -316,10 +316,22 @@ class Visualizor: conf["title"] = "Boxen Plot of Restore Column Count by Nodes" conf["xlabel"] = "Restore Column Count" conf["ylabel"] = "Nodes" - n1 = int(result.numberNodes * result.class1ratio) - data = [result.restoreColumnCount[1: n1], result.restoreColumnCount[n1+1: ]] + data = [] + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + _start = 1 + for _range in nodeRanges: + data.append(result.restoreColumnCount[_start: _range]) + _start = _range + _values, _categories = [], [] + for _d, _nc in zip(data, nodeClasses): + _values += _d + _categories += [f'Class {_nc}'] * len(_d) + data = pd.DataFrame({ + 'values': _values, + 'category': _categories + }) plt.figure(figsize=(8, 6)) - sns.boxenplot(data=data, width=0.8) + sns.boxenplot(x='category', y='values', data=data, palette="Set2", ax=plt.gca(), width=0.8) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) From ae6f2a370db48d9e5638d5ef302ffdb722427a34 Mon Sep 17 00:00:00 2001 From: Arunima Chaudhuri Date: Sat, 8 Jun 2024 07:32:55 +0000 Subject: [PATCH 19/34] debug gossip Signed-off-by: Arunima Chaudhuri --- DAS/node.py | 83 +++++++++++++++++++++++----------------------------- smallConf.py | 2 +- 2 files changed, 38 insertions(+), 47 deletions(-) diff --git a/DAS/node.py b/DAS/node.py index e89575c..19f317b 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -507,14 +507,14 @@ class Node: if self.statsTxInSlot >= self.bwUplink: return - def sendGossip(self, neigh): + def sendGossip(self, peer, segments_to_send): """Simulate sending row and column IDs to a peer.""" - have_info = {'source': self.ID, 'rowIDs': self.rowIDs, 'columnIDs': self.columnIDs} - neigh.node.received_gossip[self.ID].append(have_info) - neigh.node.msgRecvCount += 1 - self.logger.debug(f"Gossip sent to {neigh.node.ID}: {neigh.node.received_gossip}", extra=self.format) + have_info = {'source': self.ID, 'segments': segments_to_send} + peer.received_gossip[self.ID].append(have_info) + peer.msgRecvCount += 1 + self.logger.debug(f"Gossip sent to {peer.ID}: {peer.received_gossip}", extra=self.format) - def process_received_gossip(self, simulator): + def processReceivedGossip(self, simulator): """ Processes received gossip messages to request and receive data segments. For each segment not already received, it simulates requesting the segment, @@ -522,54 +522,45 @@ class Node: """ for sender, have_infos in self.received_gossip.items(): for have_info in have_infos: - for rowID in have_info['rowIDs']: - for columnID in have_info['columnIDs']: - if not self.receivedBlock.getSegment(rowID, columnID): - # request for the segment - self.logger.debug(f"Requesting segment ({rowID}, {columnID}) from {have_info['source']}", extra=self.format) - self.msgSentCount += 1 - # source sends the segment - self.logger.debug(f"Sending segment ({rowID}, {columnID}) to {self.ID} from {have_info['source']}", extra=self.format) - simulator.validators[have_info['source']].sampleSentCount += 1 - simulator.validators[have_info['source']].statsTxInSlot += 1 - # receive the segment - self.receivedBlock.setSegment(rowID, columnID) - self.sampleRecvCount += 1 - self.logger.debug(f"Received segment ({rowID}, {columnID}) via gossip from {have_info['source']}", extra=self.format) + for rowID, columnID in have_info['segments']: + if not self.receivedBlock.getSegment(rowID, columnID): + # request for the segment + self.logger.debug(f"Requesting segment ({rowID}, {columnID}) from {have_info['source']}", extra=self.format) + self.msgSentCount += 1 + # source sends the segment + self.logger.debug(f"Sending segment ({rowID}, {columnID}) to {self.ID} from {have_info['source']}", extra=self.format) + simulator.validators[have_info['source']].sampleSentCount += 1 + simulator.validators[have_info['source']].statsTxInSlot += 1 + # receive the segment + self.receivedBlock.setSegment(rowID, columnID) + self.sampleRecvCount += 1 + self.logger.debug(f"Received segment ({rowID}, {columnID}) via gossip from {have_info['source']}", extra=self.format) self.received_gossip.clear() def gossip(self, simulator): """ - Periodically sends gossip messages to a random subset of neighbors to share information - about data segments (row and column IDs). The process involves: - 1. Selecting a random subset of row and column neighbors. - 2. Sending the node's current state (row and column IDs) to these neighbors. - 3. Neighbors process the received gossip and update their state accordingly. + Periodically sends gossip messages to a random subset of nodes to share information + about data segments. The process involves: + 1. Selecting a random subset of nodes. + 2. Sending the node's current state (row and column IDs) to these nodes. + 3. Process the received gossip and update their state accordingly. - This ensures data dissemination across the network with minimal delay, + This ensures data dissemination across the network, occurring at intervals defined by the HEARTBEAT timer. """ - if self.rowIDs: - rID = random.choice(list(self.rowIDs)) - rowNeighs = list(self.rowNeighbors[rID].values()) - num_row_peers = random.randint(1, len(rowNeighs)) - selected_row_neighs = random.sample(rowNeighs, num_row_peers) - for rowNeigh in selected_row_neighs: - self.sendGossip(rowNeigh) + total_nodes = simulator.shape.numberNodes + num_peers = random.randint(1, total_nodes - 1) + peers = random.sample(range(1, total_nodes), num_peers) + segments_to_send = [] + for rID in range(0, self.shape.nbRows): + for cID in range(0, self.shape.nbCols): + if self.block.getSegment(rID, cID): + segments_to_send.append((rID, cID)) + if segments_to_send: + for peer in peers: + self.sendGossip(simulator.validators[peer], segments_to_send) self.msgSentCount += 1 - rowNeigh.node.process_received_gossip(simulator) - if self.statsTxInSlot >= self.bwUplink: - return - - if self.columnIDs: - cID = random.choice(list(self.columnIDs)) - columnNeighs = list(self.columnNeighbors[cID].values()) - num_column_peers = random.randint(1, len(columnNeighs)) - selected_column_neighs = random.sample(columnNeighs, num_column_peers) - for columnNeigh in selected_column_neighs: - self.sendGossip(columnNeigh) - self.msgSentCount += 1 - columnNeigh.node.process_received_gossip(simulator) + simulator.validators[peer].processReceivedGossip(simulator) if self.statsTxInSlot >= self.bwUplink: return diff --git a/smallConf.py b/smallConf.py index 1c60cc0..74d5dd0 100644 --- a/smallConf.py +++ b/smallConf.py @@ -59,7 +59,7 @@ maliciousNodes = range(40,41,20) # If True, the malicious nodes will be assigned randomly; if False, a predefined pattern may be used randomizeMaliciousNodes = True -# When set to True, nodes will use the Gossipsub protocol for communication +# When set to True, nodes will use the Gossip for communication gossip = True # Heartbeat interval for gossip messages in simulation steps From 0e51082f71aa14ebca2dec409dc946fcc002bece Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Mon, 10 Jun 2024 07:07:14 +0000 Subject: [PATCH 20/34] Changed nodeType configuration structure --- DAS/node.py | 2 +- DAS/shape.py | 2 +- DAS/simulator.py | 4 ++-- DAS/visualizor.py | 26 ++++++++++++-------------- smallConf.py | 17 ++++++++++++++--- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/DAS/node.py b/DAS/node.py index 1787af3..40e8dce 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -121,7 +121,7 @@ class Node: if self.amIproposer: self.bwUplink = shape.bwUplinkProd else: - self.bwUplink = shape.nodeTypes[self.nodeClass]['bwUplinks'] + self.bwUplink = shape.nodeTypes["classes"][self.nodeClass]["def"]['bwUplinks'] self.bwUplink *= 1e3 / 8 * config.stepDuration / config.segmentSize self.repairOnTheFly = config.evalConf(self, config.repairOnTheFly, shape) diff --git a/DAS/shape.py b/DAS/shape.py index dea8b7a..02ec6a7 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -19,7 +19,7 @@ class Shape: self.custodyCols = custodyCols self.bwUplinkProd = bwUplinkProd self.nodeTypes = nodeTypes - self.nodeClasses = [0] + [_k for _k in nodeTypes.keys() if _k != "group"] + self.nodeClasses = [0] + [_k for _k in nodeTypes["classes"].keys()] self.randomSeed = "" def __repr__(self): diff --git a/DAS/simulator.py b/DAS/simulator.py index b801e77..afbf418 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -45,7 +45,7 @@ class Simulator: self.proposerPublishToC = config.evalConf(self, config.proposerPublishToR, shape) def getNodeClass(self, nodeIdx): - nodeRatios = [_v['ratio'] for _k, _v in self.shape.nodeTypes.items() if _k != "group"] + nodeRatios = [_v['weight'] for _k, _v in self.shape.nodeTypes["classes"].items()] nodeCounts = [int(self.shape.numberNodes * ratio / sum(nodeRatios)) for ratio in nodeRatios] commulativeSum = [sum(nodeCounts[:i+1]) for i in range(len(nodeCounts))] commulativeSum[-1] = self.shape.numberNodes @@ -135,7 +135,7 @@ class Simulator: vs = [] nodeClass = self.getNodeClass(i) - vpn = self.shape.nodeTypes[nodeClass]['validatorsPerNode'] + vpn = self.shape.nodeTypes["classes"][nodeClass]["def"]['validatorsPerNode'] for v in range(vpn): vs.append(initValidator(self.shape.nbRows, self.shape.custodyRows, self.shape.nbCols, self.shape.custodyCols)) val = Node(i, int(not i!=0), nodeClass, amImalicious_value, self.logger, self.shape, self.config, vs) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index 83c96ab..6fde77d 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -22,7 +22,7 @@ def plotData(conf): for i in range(len(conf["data"])): plt.bar(conf["xdots"], conf["data"][i], label=conf["labels"][i]) if conf["type"] == "individual_bar_with_2line": - plt.axhline(y = conf["expected_value1"], color='r', linestyle='--', label=conf["line_label1"]) + plt.axhline(y = conf["expected_value1"], color='w', linestyle='--', label=conf["line_label1"]) plt.axhline(y = conf["expected_value2"], color='g', linestyle='--', label=conf["line_label2"]) if conf["type"] == "plot_with_1line": plt.axhline(y = conf["expected_value"], color='g', linestyle='--', label=conf["line_label"]) @@ -69,23 +69,21 @@ class Visualizor: theGroup = dict() for nt in self.config.nodeTypesGroup: if nt['group'] == group: - for _k, _v in nt.items(): - if _k != 'group': - theGroup[_k] = { - "vpn": _v["validatorsPerNode"], - "bw": _v["bwUplinks"], - "r": _v["ratio"] - } + for _k, _v in nt["classes"].items(): + theGroup[_k] = { + "vpn": _v["def"]["validatorsPerNode"], + "bw": _v["def"]["bwUplinks"], + "w": _v["weight"] + } break return theGroup def __getNodeRanges(self, shape): nodeClasses, nodeRatios = [], [] - for _k, _v in shape.nodeTypes.items(): - if _k != "group": - nodeClasses.append(_k) - nodeRatios.append(_v['ratio']) + for _k, _v in shape.nodeTypes["classes"].items(): + nodeClasses.append(_k) + nodeRatios.append(_v['weight']) nodeCounts = [int(shape.numberNodes * ratio / sum(nodeRatios)) for ratio in nodeRatios] commulativeSum = [sum(nodeCounts[:i+1]) for i in range(len(nodeCounts))] commulativeSum[-1] = shape.numberNodes @@ -993,7 +991,7 @@ class Visualizor: maxi = max(v) conf["yaxismax"] = maxi x = result.shape.nbCols * result.shape.custodyRows + result.shape.nbRows * result.shape.custodyCols - conf["expected_value"] = (result.shape.numberNodes - 1) * x * sum([(_v['r'] * _v['vpn']) for _v in nodeTypes.values()]) / sum([_v['r'] for _v in nodeTypes.values()]) + conf["expected_value"] = (result.shape.numberNodes - 1) * x * sum([(_v['w'] * _v['vpn']) for _v in nodeTypes.values()]) / sum([_v['w'] for _v in nodeTypes.values()]) conf["line_label"] = "Total segments to deliver" plotData(conf) print("Plot %s created." % conf["path"]) @@ -1362,7 +1360,7 @@ class Visualizor: rs = [] for result in self.results: attrbs = self.__get_attrbs__(result) - rs.append(int(attrbs['r'])) + rs.append(int(attrbs['w'])) return max(rs) - min(rs) + 1 diff --git a/smallConf.py b/smallConf.py index 87334dd..c5c497a 100644 --- a/smallConf.py +++ b/smallConf.py @@ -83,9 +83,20 @@ bwUplinksProd = [200] nodeTypesGroup = [ { "group": "g1", - # nodeClass: node config - 1: {'validatorsPerNode': 1, 'bwUplinks': 10, 'ratio': 8}, - 2: {'validatorsPerNode': 5, 'bwUplinks': 200, 'ratio': 2} + "classes": { + 1: { + "weight": 70, + "def": {'validatorsPerNode': 1, 'bwUplinks': 10} + }, + 2: { + "weight": 20, + "def": {'validatorsPerNode': 5, 'bwUplinks': 200} + }, + 3: { + "weight": 10, + "def": {'validatorsPerNode': 10, 'bwUplinks': 500} + } + } } ] From f3f1e8664d1e9f8798b71d51373baa37a171eb59 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Mon, 10 Jun 2024 19:00:29 +0000 Subject: [PATCH 21/34] Fixed rounding error for number of nodes --- DAS/simulator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index afbf418..fe6f3f9 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -50,8 +50,8 @@ class Simulator: commulativeSum = [sum(nodeCounts[:i+1]) for i in range(len(nodeCounts))] commulativeSum[-1] = self.shape.numberNodes for i, idx in enumerate(commulativeSum): - if nodeIdx < idx: - return i + 1 + if nodeIdx <= idx: + return self.shape.nodeClasses[i + 1] def initValidators(self): """It initializes all the validators in the network.""" From 67c8b09295c1e6c20ee6ca13605f14f97559bc04 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 18 Jun 2024 07:57:03 +0000 Subject: [PATCH 22/34] Deterministic validator row column distributio --- DAS/node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DAS/node.py b/DAS/node.py index 40e8dce..a7bb4b9 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -32,11 +32,13 @@ class Neighbor: class Validator: + i = 0 def __init__(self, rowIDs, columnIDs): self.rowIDs = rowIDs self.columnIDs = columnIDs def initValidator(nbRows, custodyRows, nbCols, custodyCols): + random.seed(10 + Validator.i); Validator.i += 1 rowIDs = set(random.sample(range(nbRows), custodyRows)) columnIDs = set(random.sample(range(nbCols), custodyCols)) return Validator(rowIDs, columnIDs) From f4448e4a2a6dccb02e9bc1c009dd08f958aa5b76 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 18 Jun 2024 09:20:32 +0000 Subject: [PATCH 23/34] ECDF plots updated --- DAS/visualizor.py | 97 +++++++++++++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index 6fde77d..ad50414 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -356,11 +356,15 @@ class Visualizor: conf["title"] = "ECDF of Restore Row Count by Nodes" conf["xlabel"] = "Restore Row Count" conf["ylabel"] = "ECDF" - n1 = int(result.numberNodes * result.class1ratio) - class1_data = result.restoreRowCount[1: n1] - class2_data = result.restoreRowCount[n1+1: ] - sns.ecdfplot(data=class1_data, label='Class 1 Nodes') - sns.ecdfplot(data=class2_data, label='Class 2 Nodes') + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + start = 1 + labels = [] + for i, rng in enumerate(nodeRanges): + class_data = result.repairedSampleCount[start: rng + 1] + label = f"Class {nodeClasses[i]} Nodes" + labels.append(label) + start = rng + 1 + sns.ecdfplot(data=class_data, label=label) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -368,7 +372,7 @@ class Visualizor: plt.xlim(left=0, right=max_val if max_val > 0 else 1) props = dict(boxstyle='round', facecolor='wheat', alpha=0.5) plt.text(1.05, 0.05, conf["textBox"], fontsize=14, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props) - plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'], loc=1) + plt.legend(title='Node Class', labels=labels, loc=1) plt.savefig(plotPath + "/ecdf_restoreRowCount.png", bbox_inches="tight") print("Plot %s created." % (plotPath + "/ecdf_restoreRowCount.png")) @@ -390,11 +394,15 @@ class Visualizor: conf["title"] = "ECDF of Restore Column Count by Nodes" conf["xlabel"] = "Restore Column Count" conf["ylabel"] = "ECDF" - n1 = int(result.numberNodes * result.class1ratio) - class1_data = result.restoreColumnCount[1: n1] - class2_data = result.restoreColumnCount[n1+1: ] - sns.ecdfplot(data=class1_data, label='Class 1 Nodes') - sns.ecdfplot(data=class2_data, label='Class 2 Nodes') + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + start = 1 + labels = [] + for i, rng in enumerate(nodeRanges): + class_data = result.repairedSampleCount[start: rng + 1] + label = f"Class {nodeClasses[i]} Nodes" + labels.append(label) + start = rng + 1 + sns.ecdfplot(data=class_data, label=label) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -402,7 +410,7 @@ class Visualizor: plt.xlim(left=0, right=max_val if max_val > 0 else 1) props = dict(boxstyle='round', facecolor='wheat', alpha=0.5) plt.text(1.05, 0.05, conf["textBox"], fontsize=14, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props) - plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'], loc=1) + plt.legend(title='Node Class', labels=labels, loc=1) plt.savefig(plotPath + "/ecdf_restoreColumnCount.png", bbox_inches="tight") print("Plot %s created." % (plotPath + "/ecdf_restoreColumnCount.png")) @@ -424,12 +432,16 @@ class Visualizor: conf["title"] = "ECDF of Messages Sent by Nodes" conf["xlabel"] = "Number of Messages Sent" conf["ylabel"] = "ECDF" - n1 = int(result.numberNodes * result.class1ratio) - class1_data = result.msgSentCount[1: n1] - class2_data = result.msgSentCount[n1+1: ] - sns.ecdfplot(data=class1_data, label='Class 1 Nodes') - sns.ecdfplot(data=class2_data, label='Class 2 Nodes') - plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'], loc=1) + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + start = 1 + labels = [] + for i, rng in enumerate(nodeRanges): + class_data = result.msgSentCount[start: rng + 1] + label = f"Class {nodeClasses[i]} Nodes" + labels.append(label) + start = rng + 1 + sns.ecdfplot(data=class_data, label=label) + plt.legend(title='Node Class', labels=labels) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -457,12 +469,16 @@ class Visualizor: conf["title"] = "ECDF of Messages Received by Nodes" conf["xlabel"] = "Number of Messages Received" conf["ylabel"] = "ECDF" - n1 = int(result.numberNodes * result.class1ratio) - class1_data = result.msgRecvCount[1: n1] - class2_data = result.msgRecvCount[n1+1: ] - sns.ecdfplot(data=class1_data, label='Class 1 Nodes') - sns.ecdfplot(data=class2_data, label='Class 2 Nodes') - plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'], loc=1) + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + start = 1 + labels = [] + for i, rng in enumerate(nodeRanges): + class_data = result.msgRecvCount[start: rng + 1] + label = f"Class {nodeClasses[i]} Nodes" + labels.append(label) + start = rng + 1 + sns.ecdfplot(data=class_data, label=label) + plt.legend(title='Node Class', labels=labels) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -490,12 +506,16 @@ class Visualizor: conf["title"] = "ECDF of Samples Received by Nodes" conf["xlabel"] = "Number of Samples Received" conf["ylabel"] = "ECDF" - n1 = int(result.numberNodes * result.class1ratio) - class1_data = result.sampleRecvCount[1: n1] - class2_data = result.sampleRecvCount[n1+1: ] - sns.ecdfplot(data=class1_data, label='Class 1 Nodes') - sns.ecdfplot(data=class2_data, label='Class 2 Nodes') - plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'], loc=1) + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + start = 1 + labels = [] + for i, rng in enumerate(nodeRanges): + class_data = result.sampleRecvCount[start: rng + 1] + label = f"Class {nodeClasses[i]} Nodes" + labels.append(label) + start = rng + 1 + sns.ecdfplot(data=class_data, label=label) + plt.legend(title='Node Class', labels=labels) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -525,7 +545,6 @@ class Visualizor: conf["ylabel"] = "ECDF" vector1 = result.metrics["rowDist"] vector2 = result.metrics["columnDist"] - n1 = int(result.numberNodes * result.class1ratio) sns.ecdfplot(data=vector1, label='Rows') sns.ecdfplot(data=vector2, label='Columns') plt.xlabel(conf["xlabel"], fontsize=12) @@ -556,12 +575,16 @@ class Visualizor: conf["title"] = "ECDF of Samples Repaired by Nodes" conf["xlabel"] = "Number of Samples Repaired" conf["ylabel"] = "ECDF" - n1 = int(result.numberNodes * result.class1ratio) - class1_data = result.repairedSampleCount[1: n1] - class2_data = result.repairedSampleCount[n1+1: ] - sns.ecdfplot(data=class1_data, label='Class 1 Nodes') - sns.ecdfplot(data=class2_data, label='Class 2 Nodes') - plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes']) + nodeClasses, nodeRanges = self.__getNodeRanges(result.shape) + start = 1 + labels = [] + for i, rng in enumerate(nodeRanges): + class_data = result.repairedSampleCount[start: rng + 1] + label = f"Class {nodeClasses[i]} Nodes" + labels.append(label) + start = rng + 1 + sns.ecdfplot(data=class_data, label=label) + plt.legend(title='Node Class', labels=labels) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) From 4dc4f3a32a487190d3b7448c817229e969d5f857 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 18 Jun 2024 15:55:16 +0000 Subject: [PATCH 24/34] Heatmaps updated --- DAS/visualizor.py | 20 +++++++------------- study.py | 6 +++--- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index ad50414..deef58e 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -1383,7 +1383,7 @@ class Visualizor: rs = [] for result in self.results: attrbs = self.__get_attrbs__(result) - rs.append(int(attrbs['w'])) + rs.append(int(attrbs['r'])) return max(rs) - min(rs) + 1 @@ -1410,13 +1410,11 @@ class Visualizor: "_mn_" + attrbs['mn'] +\ "_cusr_" + attrbs['cusr'] +\ "_cusc_" + attrbs['cusc'] +\ - "_vpn1_" + attrbs['vpn1'] +\ - "_vpn2_" + attrbs['vpn2'] + "_ntypes_" + attrbs['ntypes'] identifier = ( attrbs['bsrn'], attrbs['bsrk'], attrbs['bscn'], attrbs['bsck'], attrbs['fr'], attrbs['mn'], - attrbs['cusr'], attrbs['cusc'], attrbs['vpn1'], - attrbs['vpn2'] + attrbs['cusr'], attrbs['cusc'], attrbs['ntypes'] ) if identifier in xyS.keys(): xyS[identifier]['x'].append(result.shape.netDegree) @@ -1476,13 +1474,11 @@ class Visualizor: "_fr_" + attrbs['fr'] +\ "_cusr_" + attrbs['cusr'] +\ "_cusc_" + attrbs['cusc'] +\ - "_vpn1_" + attrbs['vpn1'] +\ - "_vpn2_" + attrbs['vpn2'] + "_ntypes_" + attrbs['ntypes'] identifier = ( attrbs['bsrn'], attrbs['bsrk'], attrbs['bscn'], attrbs['bsck'], attrbs['fr'], attrbs['nn'], - attrbs['cusr'], attrbs['cusc'], attrbs['vpn1'], - attrbs['vpn2'] + attrbs['cusr'], attrbs['cusc'], attrbs['ntypes'] ) if identifier in xyS.keys(): xyS[identifier]['x'].append(result.shape.netDegree) @@ -1542,13 +1538,11 @@ class Visualizor: "_mn_" + attrbs['mn'] +\ "_cusr_" + attrbs['cusr'] +\ "_cusc_" + attrbs['cusc'] +\ - "_vpn1_" + attrbs['vpn1'] +\ - "_vpn2_" + attrbs['vpn2'] + "_ntypes_" + attrbs['ntypes'] identifier = ( attrbs['bsrn'], attrbs['bsrk'], attrbs['bscn'], attrbs['bsck'], attrbs['mn'], attrbs['nn'], - attrbs['cusr'], attrbs['cusc'], attrbs['vpn1'], - attrbs['vpn2'] + attrbs['cusr'], attrbs['cusc'], attrbs['ntypes'] ) if identifier in xyS.keys(): xyS[identifier]['x'].append(result.shape.netDegree) diff --git a/study.py b/study.py index a824e66..0e47a77 100644 --- a/study.py +++ b/study.py @@ -213,11 +213,11 @@ def study(): logger.info("A total of %d simulations ran in %d seconds" % (len(results), end-start), extra=format) if config.visualization: - vis = Visualizer(execID, config) - vis.plotHeatmaps() + # vis = Visualizer(execID, config) + # vis.plotHeatmaps() visual = Visualizor(execID, config, results) - visual.plotHeatmaps("nn", "fr") + # visual.plotHeatmaps("nn", "fr") visual.plotAllHeatMaps() if __name__ == "__main__": From 11d31d73504a12195ab75725d1bc026a30d2af82 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 18 Jun 2024 19:51:00 +0000 Subject: [PATCH 25/34] minimum custody feature added --- DAS/node.py | 4 ++-- DAS/shape.py | 6 +++++- smallConf.py | 8 +++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/DAS/node.py b/DAS/node.py index a7bb4b9..abc10fe 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -98,13 +98,13 @@ class Node: self.logger.warning("Row custody (*vpn) larger than number of rows!", extra=self.format) self.rowIDs = range(self.shape.nbRows) else: - self.rowIDs = set(random.sample(range(self.shape.nbRows), self.vpn*self.shape.custodyRows)) + self.rowIDs = set(random.sample(range(self.shape.nbRows), max(self.vpn*self.shape.custodyRows, self.shape.minCustodyRows))) if (self.vpn * self.shape.custodyCols) > self.shape.nbCols: self.logger.warning("Column custody (*vpn) larger than number of columns!", extra=self.format) self.columnIDs = range(self.shape.nbCols) else: - self.columnIDs = set(random.sample(range(self.shape.nbCols), self.vpn*self.shape.custodyCols)) + self.columnIDs = set(random.sample(range(self.shape.nbCols), max(self.vpn*self.shape.custodyCols, self.shape.minCustodyCols))) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) diff --git a/DAS/shape.py b/DAS/shape.py index 02ec6a7..f559176 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,7 +3,7 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" def __init__(self, nbCols, nbColsK, nbRows, nbRowsK, - numberNodes, failureModel, failureRate, maliciousNodes, custodyRows, custodyCols, netDegree, bwUplinkProd, run, nodeTypes): + numberNodes, failureModel, failureRate, maliciousNodes, custodyRows, custodyCols, minCustodyRows, minCustodyCols, netDegree, bwUplinkProd, run, nodeTypes): """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberNodes = numberNodes @@ -17,6 +17,8 @@ class Shape: self.netDegree = netDegree self.custodyRows = custodyRows self.custodyCols = custodyCols + self.minCustodyRows = minCustodyRows + self.minCustodyCols = minCustodyCols self.bwUplinkProd = bwUplinkProd self.nodeTypes = nodeTypes self.nodeClasses = [0] + [_k for _k in nodeTypes["classes"].keys()] @@ -34,6 +36,8 @@ class Shape: shastr += "-fr-"+str(self.failureRate) shastr += "-cusr-"+str(self.custodyRows) shastr += "-cusc-"+str(self.custodyCols) + shastr += "-mcusr-"+str(self.minCustodyRows) + shastr += "-mcusc-"+str(self.minCustodyCols) shastr += "-bwupprod-"+str(self.bwUplinkProd) shastr += "-nd-"+str(self.netDegree) shastr += "-r-"+str(self.run) diff --git a/smallConf.py b/smallConf.py index c5c497a..df3a546 100644 --- a/smallConf.py +++ b/smallConf.py @@ -76,6 +76,8 @@ proposerPublishToC = "shape.netDegree" validatorBasedCustody = False custodyRows = range(2, 3, 2) custodyCols = range(2, 3, 2) +minCustodyRows = range(2, 3, 2) +minCustodyCols = range(2, 3, 2) # Set uplink bandwidth in megabits/second bwUplinksProd = [200] @@ -146,11 +148,11 @@ colsK = range(32, 65, 128) rowsK = range(32, 65, 128) def nextShape(): - for nbCols, nbColsK, nbRows, nbRowsK, run, fm, fr, mn, chR, chC, nn, netDegree, bwUplinkProd, nodeTypes in itertools.product( - cols, colsK, rows, rowsK, runs, failureModels, failureRates, maliciousNodes, custodyRows, custodyCols, numberNodes, netDegrees, bwUplinksProd, nodeTypesGroup): + for nbCols, nbColsK, nbRows, nbRowsK, run, fm, fr, mn, chR, chC, minChR, minChC, nn, netDegree, bwUplinkProd, nodeTypes in itertools.product( + cols, colsK, rows, rowsK, runs, failureModels, failureRates, maliciousNodes, custodyRows, custodyCols, minCustodyRows, minCustodyCols, numberNodes, netDegrees, bwUplinksProd, nodeTypesGroup): # Network Degree has to be an even number if netDegree % 2 == 0: - shape = Shape(nbCols, nbColsK, nbRows, nbRowsK, nn, fm, fr, mn, chR, chC, netDegree, bwUplinkProd, run, nodeTypes) + shape = Shape(nbCols, nbColsK, nbRows, nbRowsK, nn, fm, fr, mn, chR, chC, minChR, minChC, netDegree, bwUplinkProd, run, nodeTypes) yield shape def evalConf(self, param, shape = None): From 23eacb31132e529ac68f0b3641ac59ccf0e06efe Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Mon, 24 Jun 2024 07:24:43 +0000 Subject: [PATCH 26/34] Fixed sns boxenplot warning --- DAS/visualizor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index deef58e..dce6122 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -287,7 +287,7 @@ class Visualizor: 'category': _categories }) plt.figure(figsize=(8, 6)) - sns.boxenplot(x='category', y='values', data=data, palette="Set2", ax=plt.gca(), width=0.8) + sns.boxenplot(x='category', y='values', hue='category', data=data, palette="Set2", ax=plt.gca(), width=0.8) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -329,7 +329,7 @@ class Visualizor: 'category': _categories }) plt.figure(figsize=(8, 6)) - sns.boxenplot(x='category', y='values', data=data, palette="Set2", ax=plt.gca(), width=0.8) + sns.boxenplot(x='category', y='values', hue='category', data=data, palette="Set2", ax=plt.gca(), width=0.8) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -627,7 +627,7 @@ class Visualizor: 'category': _categories }) plt.figure(figsize=(8, 6)) - sns.boxenplot(x='category', y='values', data=data, palette="Set2", ax=plt.gca(), width=0.8) + sns.boxenplot(x='category', y='values', hue='category', data=data, palette="Set2", ax=plt.gca(), width=0.8) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -671,7 +671,7 @@ class Visualizor: 'category': _categories }) plt.figure(figsize=(8, 6)) - sns.boxenplot(x='category', y='values', data=data, width=0.8, palette="Set2", ax=plt.gca()) + sns.boxenplot(x='category', y='values', hue='category', data=data, width=0.8, palette="Set2", ax=plt.gca()) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -751,7 +751,7 @@ class Visualizor: 'values': _values, 'category': _categories }) - sns.boxenplot(x='category', y='values', data=data, width=0.8, palette="Set2", ax=plt.gca()) + sns.boxenplot(x='category', y='values', hue='category', data=data, width=0.8, palette="Set2", ax=plt.gca()) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) @@ -792,7 +792,7 @@ class Visualizor: 'values': _values, 'category': _categories }) - sns.boxenplot(x='category', y='values', data=data, palette="Set2", ax=plt.gca()) + sns.boxenplot(x='category', y='values', hue='category', data=data, palette="Set2", ax=plt.gca()) plt.xlabel(conf["xlabel"], fontsize=12) plt.ylabel(conf["ylabel"], fontsize=12) plt.title(conf["title"], fontsize=14) From 588ead24f9b0256d045b6f0fd28e3830c3f8df7b Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 25 Jun 2024 09:50:03 +0000 Subject: [PATCH 27/34] Min custodies in textbox --- DAS/visualizor.py | 76 +++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index dce6122..0db4262 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -159,8 +159,8 @@ class Visualizor: # self.plotSampleRecv(result, plotPath) # self.plotRestoreRowCount(result, plotPath) # self.plotRestoreColumnCount(result, plotPath) - # if self.config.saveRCdist: - # self.plotRowCol(result, plotPath) + if self.config.saveRCdist: + self.plotRowCol(result, plotPath) # self.plotBoxSamplesRepaired(result, plotPath) # self.plotBoxMessagesSent(result, plotPath) @@ -203,7 +203,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Box Plot of Restore Row Count by Nodes" conf["xlabel"] = "Node Type" @@ -235,7 +235,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Box Plot of Restore Column Count by Nodes" conf["xlabel"] = "Node Type" @@ -267,7 +267,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Boxen Plot of Restore Row Count by Nodes" conf["xlabel"] = "Restore Row Count" @@ -309,7 +309,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Boxen Plot of Restore Column Count by Nodes" conf["xlabel"] = "Restore Column Count" @@ -351,7 +351,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Restore Row Count by Nodes" conf["xlabel"] = "Restore Row Count" @@ -389,7 +389,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Restore Column Count by Nodes" conf["xlabel"] = "Restore Column Count" @@ -427,7 +427,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Messages Sent by Nodes" conf["xlabel"] = "Number of Messages Sent" @@ -464,7 +464,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Messages Received by Nodes" conf["xlabel"] = "Number of Messages Received" @@ -501,7 +501,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Samples Received by Nodes" conf["xlabel"] = "Number of Samples Received" @@ -538,7 +538,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Row-Col Distribution by Nodes" conf["xlabel"] = "Row-Col Distribution" @@ -570,7 +570,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "ECDF of Samples Repaired by Nodes" conf["xlabel"] = "Number of Samples Repaired" @@ -607,7 +607,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Samples Received by Nodes" conf["xlabel"] = "Node Type" @@ -651,7 +651,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Samples Repaired by Nodes" conf["xlabel"] = "Node Type" @@ -695,7 +695,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Row/Column Distribution" conf["xlabel"] = "Row/Column Type" @@ -732,7 +732,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Sent by Nodes" conf["xlabel"] = "Node Type" @@ -773,7 +773,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Received by Nodes" conf["xlabel"] = "Node Type" @@ -814,7 +814,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Samples Repaired by Nodes" conf["type"] = "individual_bar" @@ -841,7 +841,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Row/Column Distribution" conf["xlabel"] = "" @@ -870,7 +870,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Restore Row Count for Each Node" conf["type"] = "individual_bar" @@ -898,7 +898,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Restore Column Count for Each Node" conf["type"] = "individual_bar" @@ -926,7 +926,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Percentage of Samples Received by Nodes" conf["type"] = "individual_bar_with_2line" @@ -967,7 +967,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Samples Received by Nodes" conf["type"] = "individual_bar_with_2line" @@ -994,7 +994,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize)+"\nMissing Segment: "+str(round(min(result.missingVector) * 100 / max(result.missingVector), 3))+"%"\ +"\nMissing Segments: "+str(result.missingVector[-1]) conf["title"] = "Missing Segments" @@ -1034,7 +1034,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Nodes/validators ready" conf["type"] = "plot" @@ -1069,7 +1069,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Sent data" conf["type"] = "plot" @@ -1112,7 +1112,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Received data" conf["type"] = "plot" @@ -1154,7 +1154,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Duplicated data" conf["type"] = "plot" @@ -1196,7 +1196,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Row/Column distribution" conf["type"] = "grouped_bar" @@ -1229,7 +1229,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Sent by Nodes" conf["type"] = "individual_bar" @@ -1257,7 +1257,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Sent by Nodes" conf["xlabel"] = "Node Type" @@ -1280,7 +1280,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Received by Nodes" conf["type"] = "individual_bar" @@ -1308,7 +1308,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Messages Received by Nodes" conf["type"] = "individual_bar" @@ -1337,7 +1337,7 @@ class Visualizor: conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']+"\n"+nodeTypesTxt\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ +"\nSegment Size: "+str(self.config.segmentSize) conf["title"] = "Number of Samples Repaired by Nodes" conf["type"] = "individual_bar" @@ -1400,7 +1400,7 @@ class Visualizor: textBox = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"\ +"\n"+nodeTypesTxt filename = "bsrn_" + attrbs['bsrn'] +\ "_bsrk_" + attrbs['bsrk'] +\ @@ -1464,7 +1464,7 @@ class Visualizor: textBox = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nFailure rate: "+attrbs['fr']+"%"+"\nNodes: "+attrbs['nn']\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"\ +"\n"+nodeTypesTxt filename = "bsrn_" + attrbs['bsrn'] +\ "_bsrk_" + attrbs['bsrk'] +\ @@ -1528,7 +1528,7 @@ class Visualizor: textBox = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ +"\nNodes: "+attrbs['nn']+"\nMalicious Node: "+attrbs['mn']+"%"\ - +"\nCustody Rows: "+attrbs['cusr']+"\nCustody Cols: "+attrbs['cusc']\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"\ +"\n"+nodeTypesTxt filename = "bsrn_" + attrbs['bsrn'] +\ "_bsrk_" + attrbs['bsrk'] +\ From f5f2131a45df4089b75ff5834bf39aacbf923b27 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 25 Jun 2024 10:08:08 +0000 Subject: [PATCH 28/34] Divide by 0 error fix for Node with 0 validators --- DAS/observer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DAS/observer.py b/DAS/observer.py index 80b68dc..04009de 100644 --- a/DAS/observer.py +++ b/DAS/observer.py @@ -83,8 +83,8 @@ class Observer: sampleProgress = arrived / expected nodeProgress = ready / (len(validators)-1) validatorCnt = sum([v.vpn for v in validators[1:]]) - validatorAllProgress = validatedall / validatorCnt - validatorProgress = validated / validatorCnt + validatorAllProgress = (validatedall / validatorCnt) if validatorCnt != 0 else 1 + validatorProgress = (validated / validatorCnt) if validatorCnt != 0 else 1 return missingSamples, sampleProgress, nodeProgress, validatorAllProgress, validatorProgress From a7b82e848a420e8e5be11c76491343b98395bb96 Mon Sep 17 00:00:00 2001 From: Arunima Chaudhuri Date: Fri, 5 Jul 2024 21:38:50 +0000 Subject: [PATCH 29/34] add check for rowID and columnID Signed-off-by: Arunima Chaudhuri --- DAS/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DAS/node.py b/DAS/node.py index aab003f..d282703 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -523,7 +523,7 @@ class Node: for sender, have_infos in self.received_gossip.items(): for have_info in have_infos: for rowID, columnID in have_info['segments']: - if not self.receivedBlock.getSegment(rowID, columnID): + if not self.receivedBlock.getSegment(rowID, columnID) and (rowID in self.rowIDs or columnID in self.columnIDs): # request for the segment self.logger.debug(f"Requesting segment ({rowID}, {columnID}) from {have_info['source']}", extra=self.format) self.msgSentCount += 1 From 5f1e007fbf20c77253a44d0095dfa6ae9970a0b5 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sat, 13 Jul 2024 05:59:03 +0000 Subject: [PATCH 30/34] Plots the min, max, avg percentage of samples recived by nodes --- DAS/simulator.py | 25 ++++++++++++++++++++----- DAS/visualizor.py | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/DAS/simulator.py b/DAS/simulator.py index 6d2bdd9..f8cd4ce 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -86,6 +86,7 @@ class Simulator: assignedCols = [] maliciousNodesCount = int((self.shape.maliciousNodes / 100) * self.shape.numberNodes) remainingMaliciousNodes = maliciousNodesCount + expectedSamples = [] for i in range(self.shape.numberNodes): if i == 0: @@ -139,12 +140,16 @@ class Simulator: for v in range(vpn): vs.append(initValidator(self.shape.nbRows, self.shape.custodyRows, self.shape.nbCols, self.shape.custodyCols)) val = Node(i, int(not i!=0), nodeClass, amImalicious_value, self.logger, self.shape, self.config, vs) + if i != 0: + i_expectedSamples = len(val.columnIDs) * self.shape.nbRows + len(val.rowIDs) * self.shape.nbCols - len(val.columnIDs) * len(val.rowIDs) + expectedSamples.append(i_expectedSamples) if i == self.proposerID: val.initBlock() else: val.logIDs() self.validators.append(val) - + + self.result.addMetric("expectedSamples", expectedSamples) assignedRows.sort() assignedCols.sort() self.logger.debug("Rows assigned: %s" % str(assignedRows), extra=self.format) @@ -282,12 +287,14 @@ class Simulator: trafficStatsVector = [] malicious_nodes_not_added_count = 0 steps = 0 - + samplesReceived = [] + while(True): missingVector.append(missingSamples) self.logger.debug("Expected Samples: %d" % expected, extra=self.format) self.logger.debug("Missing Samples: %d" % missingSamples, extra=self.format) oldMissingSamples = missingSamples + i_sampleReceived = [] self.logger.debug("PHASE SEND %d" % steps, extra=self.format) for i in range(0,self.shape.numberNodes): @@ -301,6 +308,9 @@ class Simulator: self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format) for i in range(1,self.shape.numberNodes): self.validators[i].receiveRowsColumns() + self.logger.debug("PHASE SAMPLE COUNT %d" % steps, extra=self.format) + for i in range(1,self.shape.numberNodes): + i_sampleReceived.append(self.validators[i].sampleRecvCount) self.logger.debug("PHASE RESTORE %d" % steps, extra=self.format) for i in range(1,self.shape.numberNodes): self.validators[i].restoreRows() @@ -309,7 +319,10 @@ class Simulator: for i in range(0,self.shape.numberNodes): self.validators[i].logRows() self.validators[i].logColumns() - + + # Store sample received count by each node in current step + samplesReceived.append(i_sampleReceived) + # log TX and RX statistics trafficStats = self.glob.getTrafficStats(self.validators) self.logger.debug("step %d: %s" % @@ -365,8 +378,10 @@ class Simulator: missingVector.append(missingSamples) break steps += 1 - - + + # Store sample received count by each node in each step + self.result.addMetric("samplesReceived", samplesReceived) + for i in range(0,self.shape.numberNodes): if not self.validators[i].amIaddedToQueue : malicious_nodes_not_added_count += 1 diff --git a/DAS/visualizor.py b/DAS/visualizor.py index 0db4262..d4932aa 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -149,6 +149,7 @@ class Visualizor: os.makedirs(plotPath, exist_ok=True) self.plotMissingSegments(result, plotPath) self.plotProgress(result, plotPath) + self.plotSamplesReceived(result, plotPath) self.plotSentData(result, plotPath) self.plotRecvData(result, plotPath) self.plotDupData(result, plotPath) @@ -1047,7 +1048,46 @@ class Visualizor: conf["data"] = [vector1, vector2, vector3] conf["xdots"] = [x*self.config.stepDuration for x in range(len(vector1))] conf["path"] = plotPath+"/nodesReady.png" - conf["yaxismax"] = 1 + conf["yaxismax"] = 100 + plotData(conf) + print("Plot %s created." % conf["path"]) + + def plotSamplesReceived(self, result, plotPath): + """Plots the min, max, avg percentage of samples recived by nodes""" + samplesReceived = result.metrics["samplesReceived"] + expectedSamples = result.metrics["expectedSamples"] + vector1, vector2, vector3 = [], [], [] + for i in range(len(samplesReceived)): + percentages = [] + for j in range(1, result.numberNodes): + percentages.append(samplesReceived[i][j - 1] * 100 / expectedSamples[j - 1]) + vector1.append(max(percentages)) + vector2.append(min(percentages)) + vector3.append(sum(percentages) / len(percentages)) + conf = {} + attrbs = self.__get_attrbs__(result) + nodeTypes = self.__getNodeTypes__(attrbs['ntypes']) + nodeTypesTxt = "" + for _k, _v in nodeTypes.items(): + nodeTypesTxt += f"Type ({_k}): " + str(_v) + "\n" + if nodeTypesTxt != "": nodeTypesTxt = nodeTypesTxt[: -1] + conf["textBox"] = "Row Size (N, K): "+attrbs['bsrn']+ ", "+attrbs['bsrk']\ + +"\nColumn Size: (N, K): "+attrbs['bscn']+ ", "+attrbs['bsck']\ + +"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"%"+"\nMalicious Node: "+attrbs['mn']+"%"+"\nNetwork degree: "+attrbs['nd']\ + +"\nCustody Rows: "+attrbs['cusr']+" (Min: "+attrbs['mcusr']+")"+"\nCustody Cols: "+attrbs['cusc']+" (Min: "+attrbs['mcusc']+")"+"\n"+nodeTypesTxt\ + +"\nSegment Size: "+str(self.config.segmentSize) + conf["title"] = "Percentages of Samples Received" + conf["type"] = "plot" + conf["legLoc"] = 2 + conf["desLoc"] = 2 + conf["colors"] = ["g-", "b-", "r-"] + conf["labels"] = ["Max", "Min", "Average"] + conf["xlabel"] = "Time (ms)" + conf["ylabel"] = "Percentage (%)" + conf["data"] = [vector1, vector2, vector3] + conf["xdots"] = [x*self.config.stepDuration for x in range(len(vector1))] + conf["path"] = plotPath+"/samplesReceivedPercentages.png" + conf["yaxismax"] = 100 plotData(conf) print("Plot %s created." % conf["path"]) From 059f452bad653c646bd2930f9fc6cb240d219571 Mon Sep 17 00:00:00 2001 From: Arunima Chaudhuri Date: Sun, 5 Jan 2025 23:06:11 +0530 Subject: [PATCH 31/34] sampling methods implementation --- DAS/node.py | 230 ++++++++++++++++++++++++++++++++++++++++++++++ DAS/results.py | 19 ++++ DAS/simulator.py | 9 ++ DAS/visualizor.py | 119 +++++++++++++++++++++++- 4 files changed, 376 insertions(+), 1 deletion(-) diff --git a/DAS/node.py b/DAS/node.py index d282703..690f130 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -82,6 +82,27 @@ class Node: self.validators = validators self.received_gossip = defaultdict(list) + # query methods + self.exponential_growth = False + self.linear_growth = True + self.linear_constant_growth = False + self.hybrid_growth = False + self.exponential_constant_growth = False + self.linear_growth_constant = 10 + + # query results + self.query_times = [] + self.query_total_time = 0 + self.all_original_retries = [] + self.query_results = 'success' + self.original_retries_sum = 0 + + # Cache latency values based on horizon level + self.latency_cache = { + "level_1": [random.uniform(0.1, 0.2) for _ in range(1000)], + "level_2": [random.uniform(0.2, 0.3) for _ in range(1000)], + } + if amIproposer: self.nodeClass = 0 self.rowIDs = range(shape.nbRows) @@ -564,6 +585,215 @@ class Node: if self.statsTxInSlot >= self.bwUplink: return + + + def get_latency(self, peer_to_query, original_peers_with_custody, original_peers_with_custody_level_2): + if peer_to_query in original_peers_with_custody: + return random.choice(self.latency_cache["level_1"]) + elif peer_to_query in original_peers_with_custody_level_2: + return random.choice(self.latency_cache["level_2"]) + return None + + def generate_random_samples(self, num_queries): + return [(random.randint(0, self.shape.nbRows-1), random.randint(0, self.shape.nbCols-1)) for _ in range(num_queries)] + + def query_peer(self, peer_to_query, original_peers_with_custody, original_peers_with_custody_level_2, simulator, sample_row, sample_col): + """Query peer with custody, simulate latency, and return the time taken.""" + if simulator.validators[peer_to_query].amImalicious: + return 'timeout', 0.5 + + elif sample_row in simulator.validators[peer_to_query].rowIDs or sample_col in simulator.validators[peer_to_query].columnIDs: + if not simulator.validators[peer_to_query].block.getSegment(sample_row, sample_col): + return 'timeout', 0.5 + + latency = self.get_latency(peer_to_query, original_peers_with_custody, original_peers_with_custody_level_2) + if latency: + return 'success', latency + return 'invalid', 0.5 + + + def generate_growth_series(self): + if self.exponential_growth: + return [2**i for i in range(1000)] + elif self.linear_growth: + linear_part = list(range(10, 201, self.linear_growth_constant)) + return [1] + linear_part + elif self.linear_constant_growth: + series = [1, 10, 20, 30, 40] + series.extend([40] * 1000) + return series + elif self.hybrid_growth: + exponential_part = [2**i for i in range(6)] # [1, 2, 4, 8, 16, 32] + linear_part = list(range(64, 105, 10)) # [64, 74, 84, 94, 104] + constant_part = [104] * 1000 + return exponential_part + linear_part + constant_part + elif self.exponential_constant_growth: + exponential_part = [2**i for i in range(6)] # [1, 2, 4, 8, 16, 32] + constant_part = [32] * 1000 + return exponential_part + constant_part + else: + raise ValueError("No growth method selected!") + + + def query_peer_with_retries(self, peers_with_custody, peers_with_custody_level_2, simulator, sample_row, sample_col, max_retries=10150): + + queried_peers = [] + retries = 0 + original_retries = 0 + + peers_with_custody = list(set(peers_with_custody)) + peers_with_custody_level_2 = list(set(peers_with_custody_level_2)) + + original_peers_with_custody = peers_with_custody[:] + original_peers_with_custody_level_2 = peers_with_custody_level_2[:] + + random.shuffle(peers_with_custody) + random.shuffle(peers_with_custody_level_2) + + growth_series = self.generate_growth_series() + + for num_peers_to_query in growth_series: + if retries >= max_retries: + break + + original_retries += num_peers_to_query + + # Query Level 1 peers + level_1_batch = peers_with_custody[:num_peers_to_query] + for peer_to_query in level_1_batch: + queried_peers.append(peer_to_query) + result, time_taken = self.query_peer(peer_to_query, original_peers_with_custody, original_peers_with_custody_level_2, simulator, sample_row, sample_col) + + if result == 'success': + if retries <= 24: + return 'success', time_taken + 0.5 * retries, queried_peers, original_retries + else: + return 'failure', time_taken + 0.5 * retries, queried_peers, original_retries + + elif result == 'timeout': + if retries >= max_retries: + return 'failure', 0.5 * max_retries, queried_peers, original_retries + + # Remove queried Level 1 peers + peers_with_custody = peers_with_custody[num_peers_to_query:] + + # If all Level 1 peers are queried, move to Level 2 peers + if not peers_with_custody: + level_2_batch = peers_with_custody_level_2[:num_peers_to_query] + for peer_to_query in level_2_batch: + queried_peers.append(peer_to_query) + result, time_taken = self.query_peer(peer_to_query, original_peers_with_custody, original_peers_with_custody_level_2, simulator, sample_row, sample_col) + + if result == 'success': + if retries <= 24: + return 'success', time_taken + 0.5 * retries, queried_peers, original_retries + else: + return 'failure', time_taken + 0.5 * retries, queried_peers, original_retries + + elif result == 'timeout': + if retries >= max_retries: + return 'failure', 0.5 * max_retries, queried_peers, original_retries + + # Remove queried Level 2 peers + peers_with_custody_level_2 = peers_with_custody_level_2[num_peers_to_query:] + + retries += 1 + + return 'failure', 0.5 * retries, queried_peers, original_retries + + + + def query_peer_for_samples(self, simulator): + if self.amImalicious: + return + + num_queries = 75 + samples = self.generate_random_samples(num_queries) + query_times = [] + all_original_retries = [] + results = 'success' + original_retries_sum = 0 + + for sample_row, sample_col in samples: + + if (sample_row in self.rowIDs or sample_col in self.columnIDs or + len(self.columnIDs) >= self.shape.nbColsK or + len(self.rowIDs) >= self.shape.nbRowsK): + query_times.append(0) + all_original_retries.append(0) + else: + row_neighbors_copy = {row: list(neighbors) for row, neighbors in self.rowNeighbors.items()} + column_neighbors_copy = {col: list(neighbors) for col, neighbors in self.columnNeighbors.items()} + + row_peer_ids = list({node_id for neighbors in row_neighbors_copy.values() for node_id in neighbors}) + col_peer_ids = list({node_id for neighbors in column_neighbors_copy.values() for node_id in neighbors}) + + peers_with_custody = set() + + for peer_id in row_peer_ids: + if (sample_row in simulator.validators[peer_id].rowIDs or + sample_col in simulator.validators[peer_id].columnIDs or + len(simulator.validators[peer_id].rowIDs) >= self.shape.nbRowsK or + len(simulator.validators[peer_id].columnIDs) >= self.shape.nbColsK): + peers_with_custody.update({peer_id}) + + for peer_id in col_peer_ids: + if (sample_row in simulator.validators[peer_id].rowIDs or + sample_col in simulator.validators[peer_id].columnIDs or + len(simulator.validators[peer_id].rowIDs) >= self.shape.nbRowsK or + len(simulator.validators[peer_id].columnIDs) >= self.shape.nbColsK): + peers_with_custody.update({peer_id}) + + peers_with_custody = list(peers_with_custody) + + peers_with_custody_level_2 = [] + + row_neighbors_l2 = set() + col_neighbors_l2 = set() + + for p in row_peer_ids: + for neighbors in simulator.validators[p].rowNeighbors.values(): + row_neighbors_l2.update(neighbors) + for neighbors in simulator.validators[p].columnNeighbors.values(): + row_neighbors_l2.update(neighbors) + + for p in col_peer_ids: + for neighbors in simulator.validators[p].rowNeighbors.values(): + col_neighbors_l2.update(neighbors) + for neighbors in simulator.validators[p].columnNeighbors.values(): + col_neighbors_l2.update(neighbors) + + + neighbors_level_2 = list(row_neighbors_l2.union(col_neighbors_l2)) + peers_with_custody_level_2 = set() + + for p in neighbors_level_2: + if (sample_row in simulator.validators[p].rowIDs or + sample_col in simulator.validators[p].columnIDs or + len(simulator.validators[p].rowIDs) >= self.shape.nbRowsK or + len(simulator.validators[p].columnIDs) >= self.shape.nbColsK): + peers_with_custody_level_2.update({p}) + peers_with_custody_level_2 = list(peers_with_custody_level_2) + + result, time_taken, queried_peers_list, original_retries = self.query_peer_with_retries( + peers_with_custody, peers_with_custody_level_2, simulator, sample_row, sample_col + ) + query_times.append(time_taken) + if result == 'failure': + results = 'failure' + original_retries_sum += original_retries + all_original_retries.append(original_retries) + + total_time = max(query_times) + + self.query_times = query_times[:] + self.query_total_time = total_time + self.all_original_retries = all_original_retries[:] + self.query_results = results + self.original_retries_sum = original_retries_sum + + + def send(self): """ Send as much as we can in the timestep, limited by bwUplink.""" diff --git a/DAS/results.py b/DAS/results.py index 2dc8db4..4b45377 100644 --- a/DAS/results.py +++ b/DAS/results.py @@ -23,6 +23,13 @@ class Result: self.restoreRowCount = [0] * shape.numberNodes self.restoreColumnCount = [0] * shape.numberNodes self.repairedSampleCount = [0] * shape.numberNodes + + self.query_times = [[] for _ in range(shape.numberNodes)] # List of empty lists + self.query_total_time = [None] * shape.numberNodes # List of None values, or empty lists if needed + self.all_original_retries = [[] for _ in range(shape.numberNodes)] # List of empty lists + self.query_results = [''] * shape.numberNodes # List of empty strings + self.original_retries_sum = [None] * shape.numberNodes # List of None values + self.numberNodes = shape.numberNodes def copyValidators(self, validators): @@ -35,6 +42,18 @@ class Result: self.restoreRowCount[i] = validators[i].restoreRowCount self.restoreColumnCount[i] = validators[i].restoreColumnCount self.repairedSampleCount[i] = validators[i].repairedSampleCount + if not validators[i].amImalicious or not validators[i].amIproposer: + self.query_times[i] = validators[i].query_times[:] + self.query_total_time[i] = validators[i].query_total_time + self.all_original_retries[i] = validators[i].all_original_retries[:] + self.query_results[i] = validators[i].query_results + self.original_retries_sum[i] = validators[i].original_retries_sum + else: + self.query_times[i] = None + self.query_total_time[i] = None + self.all_original_retries[i] = None + self.query_results[i] = None + self.original_retries_sum[i] = None def populate(self, shape, config, missingVector): """It populates part of the result data inside a vector.""" diff --git a/DAS/simulator.py b/DAS/simulator.py index f8cd4ce..1773e3b 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -158,6 +158,8 @@ class Simulator: def initNetwork(self): """It initializes the simulated network.""" + # rowChannels and columnChannels stores the nodes that have the custody of each row/col. + # rowChannel[rowID]->node ids that have the custody of that row rowChannels = [[] for i in range(self.shape.nbRows)] columnChannels = [[] for i in range(self.shape.nbCols)] for v in self.validators: @@ -168,6 +170,8 @@ class Simulator: columnChannels[id].append(v) # Check rows/columns distribution + # distR and distC has how many nodes have the custody of every row + # len(r) gives how many nodes have the custody of that row for r in rowChannels: self.distR.append(len(r)) for c in columnChannels: @@ -379,6 +383,11 @@ class Simulator: break steps += 1 + self.logger.debug("PHASE QUERY SAMPLE %d" % steps, extra=self.format) + for i in range(1,self.shape.numberNodes): + if not self.validators[i].amImalicious: + self.validators[i].query_peer_for_samples(self) + # Store sample received count by each node in each step self.result.addMetric("samplesReceived", samplesReceived) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index d4932aa..beea475 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -188,8 +188,125 @@ class Visualizor: self.plotECDFRestoreRowCount(result, plotPath) self.plotECDFRestoreColumnCount(result, plotPath) if self.config.saveRCdist: - self.plotECDFRowColDist(result, plotPath) + self.plotECDFRowColDist(result, plotPath) + # plots for query results + self.plot_query_times_boxplot_all(result, plotPath) + self.plot_query_results(result, plotPath) + self.plot_retries_boxplot(result, plotPath) + self.plot_retries_sum_per_node_boxplot(result, plotPath) + + + def plot_query_times_boxplot_all(self, result, plotPath): + """Plot boxplots for query times for all nodes.""" + attrbs = self.__get_attrbs__(result) + plt.figure(figsize=(14, 7)) + + all_query_times = [time for time in result.query_total_time if time is not None] + + plt.boxplot(all_query_times, patch_artist=True, boxprops=dict(facecolor="lightblue")) + plt.title(f"Query Times for Different Connection Ranges", fontsize=16) + plt.ylabel("Query Time (seconds)", fontsize=16) + plt.grid(True, axis='y', color='gray', linestyle='--', linewidth=0.5) + plt.tick_params(axis='both', which='major', labelsize=16) + plt.axhline(y=12, color='red', linestyle='--', linewidth=1) + plt.subplots_adjust(top=0.85) + plt.figtext( + 0.3, 0.96, + f"Custody Rows: {attrbs['cusr']}, Custody Columns: {attrbs['cusc']} Malicious nodes: {attrbs['mn']}%", + fontsize=16, + ha='center', + bbox=dict(facecolor='white', edgecolor='black', boxstyle='round') + ) + os.makedirs(plotPath, exist_ok=True) + plt.savefig(os.path.join(plotPath, 'query_times_all_connections_boxplot.png')) + plt.close() + + + + def plot_query_results(self, result, plotPath): + """Plot a single pie chart for block availability based on query results.""" + attrbs = self.__get_attrbs__(result) + query_results = result.query_results + + available_count = query_results.count('success') + not_available_count = query_results.count('failure') + + sizes = [available_count, not_available_count] + colors = ['lightgreen', 'salmon'] + + fig, ax = plt.subplots(figsize=(7, 7)) + wedges, texts, autotexts = ax.pie( + sizes, autopct='%1.1f%%', startangle=140, colors=colors, textprops={'fontsize': 16} + ) + for autotext in autotexts: + autotext.set_fontsize(16) + + ax.set_title("Block Availability", fontsize=16) + plt.figtext( + 0.5, 0.96, + f"Custody Rows: {attrbs['cusr']}, Custody Columns: {attrbs['cusc']} Malicious Nodes: {attrbs['mn']}%", + fontsize=16, + ha='center', + bbox=dict(facecolor='white', edgecolor='black', boxstyle='round') + ) + os.makedirs(plotPath, exist_ok=True) + output_path = os.path.join(plotPath, 'query_results_pie_chart.png') + plt.savefig(output_path) + plt.close() + + + + def plot_retries_boxplot(self, result, plotPath): + """Plot boxplots for original retries for all nodes.""" + attrbs = self.__get_attrbs__(result) + plt.figure(figsize=(14, 7)) + + all_original_retries = [ + retry for sublist in result.all_original_retries for retry in sublist if retry is not None + ] + + plt.boxplot(all_original_retries, patch_artist=True, boxprops=dict(facecolor="lightgreen")) + + plt.title("Number of peers queried by each node for a sample across connection ranges", fontsize=16) + plt.ylabel("Count of Queried Peers", fontsize=16) + plt.grid(True, axis='y', color='gray', linestyle='--', linewidth=0.5) + plt.tick_params(axis='both', which='major', labelsize=16) + plt.figtext( + 0.3, 0.96, + f"Custody Rows: {attrbs['cusr']}, Custody Columns: {attrbs['cusc']} Malicious nodes: {attrbs['mn']}%", + fontsize=16, + ha='center', + bbox=dict(facecolor='white', edgecolor='black', boxstyle='round') + ) + os.makedirs(plotPath, exist_ok=True) + plt.savefig(os.path.join(plotPath, 'original_retries_all_connections_boxplot.png')) + plt.close() + + + def plot_retries_sum_per_node_boxplot(self, result, plotPath): + attrbs = self.__get_attrbs__(result) + plt.figure(figsize=(14, 7)) + + all_retries_sum = [retries for retries in result.original_retries_sum if retries is not None] + + plt.boxplot(all_retries_sum, patch_artist=True, + boxprops=dict(facecolor="lightgreen")) + plt.title("Total Sampling Requests Sent by Each Node", fontsize=16) + plt.ylabel("Sum of all sampling requests for each node", fontsize=16) + plt.grid(True, axis='y', color='gray', linestyle='--', linewidth=0.5) + plt.tick_params(axis='both', which='major', labelsize=16) + plt.figtext( + 0.3, 0.96, + f"Custody Rows: {attrbs['cusr']}, Custody Columns: {attrbs['cusc']} Malicious nodes: {attrbs['mn']}%", + fontsize=16, + ha='center', + bbox=dict(facecolor='white', edgecolor='black', boxstyle='round') + ) + output_path = os.path.join(plotPath, 'retries_sum_boxplot_per_node.png') + plt.savefig(output_path) + plt.close() + def plotBoxRestoreRowCount(self, result, plotPath): """Box Plot of restoreRowCount for all nodes""" From 3cdc53960c53c94f1f4334327697e4598e81b76e Mon Sep 17 00:00:00 2001 From: Arunima Chaudhuri Date: Thu, 9 Jan 2025 16:19:49 +0530 Subject: [PATCH 32/34] debug --- DAS/node.py | 8 +++++++- DAS/visualizor.py | 6 ++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/DAS/node.py b/DAS/node.py index 690f130..223b0f2 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -653,7 +653,7 @@ class Node: growth_series = self.generate_growth_series() for num_peers_to_query in growth_series: - if retries >= max_retries: + if not peers_with_custody and not peers_with_custody_level_2: break original_retries += num_peers_to_query @@ -774,6 +774,12 @@ class Node: len(simulator.validators[p].columnIDs) >= self.shape.nbColsK): peers_with_custody_level_2.update({p}) peers_with_custody_level_2 = list(peers_with_custody_level_2) + + if self.ID in peers_with_custody: + peers_with_custody.remove(self.ID) + + if self.ID in peers_with_custody_level_2: + peers_with_custody_level_2.remove(self.ID) result, time_taken, queried_peers_list, original_retries = self.query_peer_with_retries( peers_with_custody, peers_with_custody_level_2, simulator, sample_row, sample_col diff --git a/DAS/visualizor.py b/DAS/visualizor.py index beea475..57ce869 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -205,7 +205,7 @@ class Visualizor: all_query_times = [time for time in result.query_total_time if time is not None] plt.boxplot(all_query_times, patch_artist=True, boxprops=dict(facecolor="lightblue")) - plt.title(f"Query Times for Different Connection Ranges", fontsize=16) + plt.title(f"Query Times", fontsize=16) plt.ylabel("Query Time (seconds)", fontsize=16) plt.grid(True, axis='y', color='gray', linestyle='--', linewidth=0.5) plt.tick_params(axis='both', which='major', labelsize=16) @@ -265,10 +265,8 @@ class Visualizor: all_original_retries = [ retry for sublist in result.all_original_retries for retry in sublist if retry is not None ] - plt.boxplot(all_original_retries, patch_artist=True, boxprops=dict(facecolor="lightgreen")) - - plt.title("Number of peers queried by each node for a sample across connection ranges", fontsize=16) + plt.title("Number of peers queried by each node for a sample", fontsize=16) plt.ylabel("Count of Queried Peers", fontsize=16) plt.grid(True, axis='y', color='gray', linestyle='--', linewidth=0.5) plt.tick_params(axis='both', which='major', labelsize=16) From 2a42c17a1fdb62581809d862fb61dfeecc6d9a06 Mon Sep 17 00:00:00 2001 From: Arunima Chaudhuri Date: Tue, 28 Jan 2025 00:32:31 +0530 Subject: [PATCH 33/34] debug --- DAS/node.py | 10 +++++----- DAS/results.py | 10 +++++----- DAS/simulator.py | 11 +++++++++++ DAS/visualizor.py | 2 +- smallConf.py | 23 +++++++++++++++++++++++ 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/DAS/node.py b/DAS/node.py index 223b0f2..15d29bd 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -83,8 +83,8 @@ class Node: self.received_gossip = defaultdict(list) # query methods - self.exponential_growth = False - self.linear_growth = True + self.exponential_growth = True + self.linear_growth = False self.linear_constant_growth = False self.hybrid_growth = False self.exponential_constant_growth = False @@ -92,10 +92,10 @@ class Node: # query results self.query_times = [] - self.query_total_time = 0 + self.query_total_time = None self.all_original_retries = [] - self.query_results = 'success' - self.original_retries_sum = 0 + self.query_results = None + self.original_retries_sum = None # Cache latency values based on horizon level self.latency_cache = { diff --git a/DAS/results.py b/DAS/results.py index 4b45377..6db69f0 100644 --- a/DAS/results.py +++ b/DAS/results.py @@ -24,11 +24,11 @@ class Result: self.restoreColumnCount = [0] * shape.numberNodes self.repairedSampleCount = [0] * shape.numberNodes - self.query_times = [[] for _ in range(shape.numberNodes)] # List of empty lists - self.query_total_time = [None] * shape.numberNodes # List of None values, or empty lists if needed - self.all_original_retries = [[] for _ in range(shape.numberNodes)] # List of empty lists - self.query_results = [''] * shape.numberNodes # List of empty strings - self.original_retries_sum = [None] * shape.numberNodes # List of None values + self.query_times = [[] for _ in range(shape.numberNodes)] + self.query_total_time = [None] * shape.numberNodes + self.all_original_retries = [[] for _ in range(shape.numberNodes)] + self.query_results = [''] * shape.numberNodes + self.original_retries_sum = [None] * shape.numberNodes self.numberNodes = shape.numberNodes diff --git a/DAS/simulator.py b/DAS/simulator.py index 1773e3b..0db8cbd 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -395,17 +395,28 @@ class Simulator: if not self.validators[i].amIaddedToQueue : malicious_nodes_not_added_count += 1 + valid_rows = set() + valid_columns = set() for i in range(0,self.shape.numberNodes): column_ids = [] row_ids = [] for rID in self.validators[i].rowIDs: row_ids.append(rID) + if not self.validators[i].amImalicious and not self.validators[i].amIproposer: + valid_rows.add(rID) for cID in self.validators[i].columnIDs: column_ids.append(cID) + if not self.validators[i].amImalicious and not self.validators[i].amIproposer: + valid_columns.add(cID) self.logger.debug("List of columnIDs for %d node: %s", i, column_ids, extra=self.format) self.logger.debug("List of rowIDs for %d node: %s", i, row_ids, extra=self.format) + if len(valid_rows) >= self.shape.nbRowsK or len(valid_columns) >= self.shape.nbColsK: + self.logger.debug("Block available within the non-malicious nodes.", extra=self.format) + else: + self.logger.debug("Block not available within the non-malicious nodes.", extra=self.format) + self.logger.debug("Number of malicious nodes not added to the send queue: %d" % malicious_nodes_not_added_count, extra=self.format) malicious_nodes_not_added_percentage = (malicious_nodes_not_added_count * 100)/(self.shape.numberNodes) self.logger.debug("Percentage of malicious nodes not added to the send queue: %d" % malicious_nodes_not_added_percentage, extra=self.format) diff --git a/DAS/visualizor.py b/DAS/visualizor.py index 57ce869..060e740 100644 --- a/DAS/visualizor.py +++ b/DAS/visualizor.py @@ -205,7 +205,7 @@ class Visualizor: all_query_times = [time for time in result.query_total_time if time is not None] plt.boxplot(all_query_times, patch_artist=True, boxprops=dict(facecolor="lightblue")) - plt.title(f"Query Times", fontsize=16) + plt.title(f"Total Query Time for each node", fontsize=16) plt.ylabel("Query Time (seconds)", fontsize=16) plt.grid(True, axis='y', color='gray', linestyle='--', linewidth=0.5) plt.tick_params(axis='both', which='major', labelsize=16) diff --git a/smallConf.py b/smallConf.py index f9e3b13..0b531c5 100644 --- a/smallConf.py +++ b/smallConf.py @@ -154,6 +154,29 @@ colsK = range(32, 65, 128) rowsK = range(32, 65, 128) def nextShape(): + params = { + "cols": cols, + "colsK": colsK, + "rows": rows, + "rowsK": rowsK, + "runs": runs, + "failureModels": failureModels, + "failureRates": failureRates, + "maliciousNodes": maliciousNodes, + "custodyRows": custodyRows, + "custodyCols": custodyCols, + "minCustodyRows": minCustodyRows, + "minCustodyCols": minCustodyCols, + "numberNodes": numberNodes, + "netDegrees": netDegrees, + "bwUplinksProd": bwUplinksProd, + "nodeTypesGroup": nodeTypesGroup, + } + for key, value in params.items(): + if not value: + logging.warning(f"The parameter '{key}' is empty. Please assign a value and start the simulation.") + exit(1) + for nbCols, nbColsK, nbRows, nbRowsK, run, fm, fr, mn, chR, chC, minChR, minChC, nn, netDegree, bwUplinkProd, nodeTypes in itertools.product( cols, colsK, rows, rowsK, runs, failureModels, failureRates, maliciousNodes, custodyRows, custodyCols, minCustodyRows, minCustodyCols, numberNodes, netDegrees, bwUplinksProd, nodeTypesGroup): # Network Degree has to be an even number From 254c6d67b2c764fc71916f35d1e67eb941a9013c Mon Sep 17 00:00:00 2001 From: Arunima Chaudhuri Date: Wed, 19 Feb 2025 22:28:23 +0530 Subject: [PATCH 34/34] add connect_peers() --- DAS/node.py | 54 ++++++++++++------------------------------------ DAS/shape.py | 4 +++- DAS/simulator.py | 22 ++++++++++++++++++++ smallConf.py | 24 +++++++++++++++++---- study.py | 1 + 5 files changed, 59 insertions(+), 46 deletions(-) diff --git a/DAS/node.py b/DAS/node.py index 15d29bd..93ade54 100644 --- a/DAS/node.py +++ b/DAS/node.py @@ -81,13 +81,14 @@ class Node: self.logger = logger self.validators = validators self.received_gossip = defaultdict(list) + self.peer_connections = set() # query methods - self.exponential_growth = True + self.exponential_growth = False self.linear_growth = False self.linear_constant_growth = False self.hybrid_growth = False - self.exponential_constant_growth = False + self.exponential_constant_growth = True self.linear_growth_constant = 10 # query results @@ -722,57 +723,28 @@ class Node: query_times.append(0) all_original_retries.append(0) else: - row_neighbors_copy = {row: list(neighbors) for row, neighbors in self.rowNeighbors.items()} - column_neighbors_copy = {col: list(neighbors) for col, neighbors in self.columnNeighbors.items()} - - row_peer_ids = list({node_id for neighbors in row_neighbors_copy.values() for node_id in neighbors}) - col_peer_ids = list({node_id for neighbors in column_neighbors_copy.values() for node_id in neighbors}) - + peers_with_custody = set() - for peer_id in row_peer_ids: + for peer_id in self.peer_connections: if (sample_row in simulator.validators[peer_id].rowIDs or sample_col in simulator.validators[peer_id].columnIDs or len(simulator.validators[peer_id].rowIDs) >= self.shape.nbRowsK or len(simulator.validators[peer_id].columnIDs) >= self.shape.nbColsK): peers_with_custody.update({peer_id}) - for peer_id in col_peer_ids: - if (sample_row in simulator.validators[peer_id].rowIDs or - sample_col in simulator.validators[peer_id].columnIDs or - len(simulator.validators[peer_id].rowIDs) >= self.shape.nbRowsK or - len(simulator.validators[peer_id].columnIDs) >= self.shape.nbColsK): - peers_with_custody.update({peer_id}) - peers_with_custody = list(peers_with_custody) - peers_with_custody_level_2 = [] - - row_neighbors_l2 = set() - col_neighbors_l2 = set() - - for p in row_peer_ids: - for neighbors in simulator.validators[p].rowNeighbors.values(): - row_neighbors_l2.update(neighbors) - for neighbors in simulator.validators[p].columnNeighbors.values(): - row_neighbors_l2.update(neighbors) - - for p in col_peer_ids: - for neighbors in simulator.validators[p].rowNeighbors.values(): - col_neighbors_l2.update(neighbors) - for neighbors in simulator.validators[p].columnNeighbors.values(): - col_neighbors_l2.update(neighbors) - - - neighbors_level_2 = list(row_neighbors_l2.union(col_neighbors_l2)) peers_with_custody_level_2 = set() - for p in neighbors_level_2: - if (sample_row in simulator.validators[p].rowIDs or - sample_col in simulator.validators[p].columnIDs or - len(simulator.validators[p].rowIDs) >= self.shape.nbRowsK or - len(simulator.validators[p].columnIDs) >= self.shape.nbColsK): - peers_with_custody_level_2.update({p}) + for p in self.peer_connections: + for peer_l2 in simulator.validators[p].peer_connections: + if (sample_row in simulator.validators[peer_l2].rowIDs or + sample_col in simulator.validators[peer_l2].rowIDs or + len(simulator.validators[peer_l2].rowIDs) >= self.shape.nbRowsK or + len(simulator.validators[peer_l2].columnIDs) >= self.shape.nbColsK): + peers_with_custody_level_2.update({peer_l2}) + peers_with_custody_level_2 = list(peers_with_custody_level_2) if self.ID in peers_with_custody: diff --git a/DAS/shape.py b/DAS/shape.py index f559176..49646ff 100644 --- a/DAS/shape.py +++ b/DAS/shape.py @@ -3,7 +3,7 @@ class Shape: """This class represents a set of parameters for a specific simulation.""" def __init__(self, nbCols, nbColsK, nbRows, nbRowsK, - numberNodes, failureModel, failureRate, maliciousNodes, custodyRows, custodyCols, minCustodyRows, minCustodyCols, netDegree, bwUplinkProd, run, nodeTypes): + numberNodes, failureModel, failureRate, maliciousNodes, custodyRows, custodyCols, minCustodyRows, minCustodyCols, netDegree, numPeersMin, numPeersMax, bwUplinkProd, run, nodeTypes): """Initializes the shape with the parameters passed in argument.""" self.run = run self.numberNodes = numberNodes @@ -15,6 +15,7 @@ class Shape: self.failureRate = failureRate self.maliciousNodes = maliciousNodes self.netDegree = netDegree + self.numPeers = [numPeersMin, numPeersMax] self.custodyRows = custodyRows self.custodyCols = custodyCols self.minCustodyRows = minCustodyRows @@ -43,6 +44,7 @@ class Shape: shastr += "-r-"+str(self.run) shastr += "-mn-"+str(self.maliciousNodes) shastr += "-ntypes-"+str(self.nodeTypes['group']) + shastr += "-np-"+str(self.numPeers) return shastr def setSeed(self, seed): diff --git a/DAS/simulator.py b/DAS/simulator.py index 0db8cbd..0d940dc 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -236,6 +236,27 @@ class Simulator: self.logger.debug("Val %d : rowN %s", i, self.validators[i].rowNeighbors, extra=self.format) self.logger.debug("Val %d : colN %s", i, self.validators[i].columnNeighbors, extra=self.format) + def connect_peers(self): + connections_range = self.shape.numPeers + + for peer in self.validators: + num_connections = random.randint(connections_range[0], connections_range[1]) + available_peers = [i for i in range(self.shape.numberNodes)] + + for neighbor_dict in [peer.rowNeighbors, peer.columnNeighbors]: + for inner_dict in neighbor_dict.values(): + for peers in inner_dict.values(): + peer.peer_connections.add(peers.node.ID) + + available_peers = list(set(available_peers) - peer.peer_connections) + random.shuffle(available_peers) + + while len(peer.peer_connections) < num_connections and available_peers: + other_peer = available_peers.pop() + if other_peer != peer.ID and len(self.validators[other_peer].peer_connections) < num_connections: + peer.peer_connections.add(other_peer) + self.validators[other_peer].peer_connections.add(peer.ID) + def initLogger(self): """It initializes the logger.""" logging.TRACE = 5 @@ -429,4 +450,5 @@ class Simulator: self.result.addMetric("progress", progress.to_dict(orient='list')) self.result.populate(self.shape, self.config, missingVector) self.result.copyValidators(self.validators) + print(self.validators[1].statsTxPerSlot) return self.result diff --git a/smallConf.py b/smallConf.py index 0b531c5..4188ed4 100644 --- a/smallConf.py +++ b/smallConf.py @@ -68,6 +68,9 @@ heartbeat = 20 # Per-topic mesh neighborhood size netDegrees = range(8, 9, 2) +# Number of peers for sampling +numPeers = [[50, 150]] + # How many copies are sent out by the block producer # Note, previously this was set to match netDegree proposerPublishToR = "shape.netDegree" @@ -169,19 +172,32 @@ def nextShape(): "minCustodyCols": minCustodyCols, "numberNodes": numberNodes, "netDegrees": netDegrees, + "numPeers": numPeers, "bwUplinksProd": bwUplinksProd, "nodeTypesGroup": nodeTypesGroup, } + for key, value in params.items(): if not value: logging.warning(f"The parameter '{key}' is empty. Please assign a value and start the simulation.") exit(1) - for nbCols, nbColsK, nbRows, nbRowsK, run, fm, fr, mn, chR, chC, minChR, minChC, nn, netDegree, bwUplinkProd, nodeTypes in itertools.product( - cols, colsK, rows, rowsK, runs, failureModels, failureRates, maliciousNodes, custodyRows, custodyCols, minCustodyRows, minCustodyCols, numberNodes, netDegrees, bwUplinksProd, nodeTypesGroup): - # Network Degree has to be an even number + for ( + nbCols, nbColsK, nbRows, nbRowsK, run, fm, fr, mn, chR, chC, minChR, minChC, + nn, netDegree, numPeersList, bwUplinkProd, nodeTypes + ) in itertools.product( + cols, colsK, rows, rowsK, runs, failureModels, failureRates, maliciousNodes, + custodyRows, custodyCols, minCustodyRows, minCustodyCols, numberNodes, + netDegrees, numPeers, bwUplinksProd, nodeTypesGroup + ): + numPeersMin, numPeersMax = numPeersList # Unpack here + + # Ensure netDegree is even if netDegree % 2 == 0: - shape = Shape(nbCols, nbColsK, nbRows, nbRowsK, nn, fm, fr, mn, chR, chC, minChR, minChC, netDegree, bwUplinkProd, run, nodeTypes) + shape = Shape( + nbCols, nbColsK, nbRows, nbRowsK, nn, fm, fr, mn, chR, chC, minChR, + minChC, netDegree, numPeersMin, numPeersMax, bwUplinkProd, run, nodeTypes + ) yield shape def evalConf(self, param, shape = None): diff --git a/study.py b/study.py index 0e47a77..52d9908 100644 --- a/study.py +++ b/study.py @@ -43,6 +43,7 @@ def runOnce(config, shape, execID): sim.initLogger() sim.initValidators() sim.initNetwork() + sim.connect_peers() result = sim.run() sim.logger.info("Shape: %s ... Block Available: %d in %d steps" % (str(sim.shape.__dict__), result.blockAvailable, len(result.missingVector)), extra=sim.format)