codex-storage/das-research
1
0
Fork 0
mirror of https://github.com/codex-storage/das-research.git synced 2025-02-23 00:08:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into config-scheduler

Browse Source 
# Conflicts:
#	DAS/node.py
#	smallConf.py

Signed-off-by: Csaba Kiraly <csaba.kiraly@gmail.com>
This commit is contained in:
Csaba Kiraly 2024-03-04 17:50:17 +01:00
commit bd5789e2e0
No known key found for this signature in database
GPG Key ID: 0FE274EE8C95166E
9 changed files with 1077 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
DAS/block.py
View File

@ -9,7 +9,6 @@ class Block:
def __init__(self, blockSizeR, blockSizeRK=0, blockSizeC=0, blockSizeCK=0):
"""Initialize the block with a data array of blocksize^2 zeros.
BlockSizeR: row size
BlockSizeRK: original row size, before erasure coding to BlocksSizeR
BlockSizeC: column size (i.e. number of rows)
@ -51,12 +50,14 @@ class Block:
"""
line = self.data[id::self.blockSizeR]
success = line.count(1)
repairedSamples = 0
if success >= self.blockSizeCK:
ret = ~line
self.data[id::self.blockSizeR] = 1
repairedSamples = len(line) - success
else:
ret = zeros(self.blockSizeC)
return ret
return ret, repairedSamples
def getRow(self, rowID):
"""It returns the block row corresponding to rowID."""
@ -72,12 +73,14 @@ class Block:
"""
line = self.data[id*self.blockSizeR:(id+1)*self.blockSizeR]
success = line.count(1)
repairedSamples = 0
if success >= self.blockSizeRK:
ret = ~line
self.data[id*self.blockSizeR:(id+1)*self.blockSizeR] = 1
repairedSamples = len(line) - success
else:
ret = zeros(self.blockSizeR)
return ret
return ret, repairedSamples
def print(self):
"""It prints the block in the terminal (outside of the logger rules))."""
@ -89,4 +92,3 @@ class Block:
line += "%i" % self.data[(i*self.blockSizeR)+j]
print(line+"|")
print(dash)

300
DAS/validator.py → DAS/node.py
View File

@ -32,52 +32,78 @@ class Neighbor:
class Validator:
"""This class implements a validator/node in the network."""
def __init__(self, rowIDs, columnIDs):
self.rowIDs = rowIDs
self.columnIDs = columnIDs
def initValidator(nbRows, custodyRows, nbCols, custodyCols):
rowIDs = set(random.sample(range(nbRows), custodyRows))
columnIDs = set(random.sample(range(nbCols), custodyCols))
return Validator(rowIDs, columnIDs)
class Node:
"""This class implements a node in the network."""
def __repr__(self):
"""It returns the validator ID."""
"""It returns the node ID."""
return str(self.ID)
def __init__(self, ID, amIproposer, logger, shape, config, rows = None, columns = None):
"""It initializes the validator with the logger shape and rows/columns.
def __init__(self, ID, amIproposer, amImalicious, logger, shape, config,
validators, rows = set(), columns = set()):
"""It initializes the node, and eventual validators, following the simulation configuration in shape and config.
If rows/columns are specified these are observed, otherwise (default)
chiR rows and chiC columns are selected randomly.
custodyRows rows and custodyCols columns are selected randomly.
"""
self.shape = shape
FORMAT = "%(levelname)s : %(entity)s : %(message)s"
self.ID = ID
self.format = {"entity": "Val "+str(self.ID)}
self.block = Block(self.shape.blockSizeR, self.shape.blockSizeRK, self.shape.blockSizeC, self.shape.blockSizeCK)
self.receivedBlock = Block(self.shape.blockSizeR, self.shape.blockSizeRK, self.shape.blockSizeC, self.shape.blockSizeCK)
self.block = Block(self.shape.nbCols, self.shape.nbColsK, self.shape.nbRows, self.shape.nbRowsK)
self.receivedBlock = Block(self.shape.nbCols, self.shape.nbColsK, self.shape.nbRows, self.shape.nbRowsK)
self.receivedQueue = deque()
self.sendQueue = deque()
self.amIproposer = amIproposer
self.amImalicious = amImalicious
self.amIaddedToQueue = 0
self.msgSentCount = 0
self.msgRecvCount = 0
self.sampleSentCount = 0
self.sampleRecvCount = 0
self.restoreRowCount = 0
self.restoreColumnCount = 0
self.repairedSampleCount = 0
self.logger = logger
if self.shape.chiR < 1 and self.shape.chiC < 1:
self.logger.error("Chi has to be greater than 0", extra=self.format)
elif self.shape.chiC > self.shape.blockSizeR:
self.logger.error("ChiC has to be smaller than %d" % self.shape.blockSizeR, extra=self.format)
elif self.shape.chiR > self.shape.blockSizeC:
self.logger.error("ChiR has to be smaller than %d" % self.shape.blockSizeC, extra=self.format)
self.validators = validators
if amIproposer:
self.nodeClass = 0
self.rowIDs = range(shape.nbRows)
self.columnIDs = range(shape.nbCols)
else:
if amIproposer:
self.nodeClass = 0
self.rowIDs = range(shape.blockSizeC)
self.columnIDs = range(shape.blockSizeR)
self.nodeClass = 1 if (self.ID <= shape.numberNodes * shape.class1ratio) else 2
self.vpn = len(validators) #TODO: needed by old code, change to fn
self.rowIDs = set(rows)
self.columnIDs = set(columns)
if config.validatorBasedCustody:
for v in validators:
self.rowIDs = self.rowIDs.union(v.rowIDs)
self.columnIDs = self.columnIDs.union(v.columnIDs)
else:
#if shape.deterministic:
# random.seed(self.ID)
self.nodeClass = 1 if (self.ID <= shape.numberNodes * shape.class1ratio) else 2
self.vpn = self.shape.vpn1 if (self.nodeClass == 1) else self.shape.vpn2
self.vRowIDs = []
self.vColumnIDs = []
for i in range(self.vpn):
self.vRowIDs.append(set(rows[i*self.shape.chiR:(i+1)*self.shape.chiR]) if rows else set(random.sample(range(self.shape.blockSizeC), self.shape.chiR)))
self.vColumnIDs.append(set(columns[i*self.shape.chiC:(i+1)*self.shape.chiC]) if columns else set(random.sample(range(self.shape.blockSizeR), self.shape.chiC)))
self.rowIDs = set.union(*self.vRowIDs)
self.columnIDs = set.union(*self.vColumnIDs)
if (self.vpn * self.shape.custodyRows) > self.shape.nbRows:
self.logger.warning("Row custody (*vpn) larger than number of rows!", extra=self.format)
self.rowIDs = range(self.shape.nbRows)
else:
self.rowIDs = set(random.sample(range(self.shape.nbRows), self.vpn*self.shape.custodyRows))
if (self.vpn * self.shape.custodyCols) > self.shape.nbCols:
self.logger.warning("Column custody (*vpn) larger than number of columns!", extra=self.format)
self.columnIDs = range(self.shape.nbCols)
else:
self.columnIDs = set(random.sample(range(self.shape.nbCols), self.vpn*self.shape.custodyCols))
self.rowNeighbors = collections.defaultdict(dict)
self.columnNeighbors = collections.defaultdict(dict)
@ -89,7 +115,7 @@ class Validator:
self.statsRxDupInSlot = 0
self.statsRxDupPerSlot = []
# Set uplink bandwidth.
# Set uplink bandwidth.
# Assuming segments of ~560 bytes and timesteps of 50ms, we get
# 1 Mbps ~= 1e6 mbps * 0.050 s / (560*8) bits ~= 11 segments/timestep
if self.amIproposer:
@ -118,7 +144,7 @@ class Validator:
def logIDs(self):
"""It logs the assigned rows and columns."""
if self.amIproposer == 1:
self.logger.warning("I am a block proposer."% self.ID)
self.logger.warning("I am a block proposer.", extra=self.format)
else:
self.logger.debug("Selected rows: "+str(self.rowIDs), extra=self.format)
self.logger.debug("Selected columns: "+str(self.columnIDs), extra=self.format)
@ -130,58 +156,58 @@ class Validator:
else:
self.logger.debug("Creating block...", extra=self.format)
if self.shape.failureModel == "random":
order = [i for i in range(self.shape.blockSizeR * self.shape.blockSizeC)]
order = [i for i in range(self.shape.nbCols * self.shape.nbRows)]
order = random.sample(order, int((1 - self.shape.failureRate/100) * len(order)))
for i in order:
self.block.data[i] = 1
elif self.shape.failureModel == "sequential":
order = [i for i in range(self.shape.blockSizeR * self.shape.blockSizeC)]
order = [i for i in range(self.shape.nbCols * self.shape.nbRows)]
order = order[:int((1 - self.shape.failureRate/100) * len(order))]
for i in order:
self.block.data[i] = 1
elif self.shape.failureModel == "MEP": # Minimal size non-recoverable Erasure Pattern
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if r > self.shape.blockSizeRK or c > self.shape.blockSizeCK:
for r in range(self.shape.nbCols):
for c in range(self.shape.nbRows):
if r > self.shape.nbColsK or c > self.shape.nbRowsK:
self.block.setSegment(r,c)
elif self.shape.failureModel == "MEP+1": # MEP +1 segment to make it recoverable
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if r > self.shape.blockSizeRK or c > self.shape.blockSizeCK:
for r in range(self.shape.nbCols):
for c in range(self.shape.nbRows):
if r > self.shape.nbColsK or c > self.shape.nbRowsK:
self.block.setSegment(r,c)
self.block.setSegment(0, 0)
elif self.shape.failureModel == "DEP":
assert(self.shape.blockSizeR == self.shape.blockSizeC and self.shape.blockSizeRK == self.shape.blockSizeCK)
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if (r+c) % self.shape.blockSizeR > self.shape.blockSizeRK:
assert(self.shape.nbCols == self.shape.nbRows and self.shape.nbColsK == self.shape.nbRowsK)
for r in range(self.shape.nbCols):
for c in range(self.shape.nbRows):
if (r+c) % self.shape.nbCols > self.shape.nbColsK:
self.block.setSegment(r,c)
elif self.shape.failureModel == "DEP+1":
assert(self.shape.blockSizeR == self.shape.blockSizeC and self.shape.blockSizeRK == self.shape.blockSizeCK)
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if (r+c) % self.shape.blockSizeR > self.shape.blockSizeRK:
assert(self.shape.nbCols == self.shape.nbRows and self.shape.nbColsK == self.shape.nbRowsK)
for r in range(self.shape.nbCols):
for c in range(self.shape.nbRows):
if (r+c) % self.shape.nbCols > self.shape.nbColsK:
self.block.setSegment(r,c)
self.block.setSegment(0, 0)
elif self.shape.failureModel == "MREP": # Minimum size Recoverable Erasure Pattern
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if r < self.shape.blockSizeRK or c < self.shape.blockSizeCK:
for r in range(self.shape.nbCols):
for c in range(self.shape.nbRows):
if r < self.shape.nbColsK or c < self.shape.nbRowsK:
self.block.setSegment(r,c)
elif self.shape.failureModel == "MREP-1": # make MREP non-recoverable
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if r < self.shape.blockSizeRK or c < self.shape.blockSizeCK:
for r in range(self.shape.nbCols):
for c in range(self.shape.nbRows):
if r < self.shape.nbColsK or c < self.shape.nbRowsK:
self.block.setSegment(r,c)
self.block.setSegment(0, 0, 0)
nbFailures = self.block.data.count(0)
measuredFailureRate = nbFailures * 100 / (self.shape.blockSizeR * self.shape.blockSizeC)
measuredFailureRate = nbFailures * 100 / (self.shape.nbCols * self.shape.nbRows)
self.logger.debug("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format)
if self.queueAllOnInit:
for r in range(self.shape.blockSizeC):
for c in range(self.shape.blockSizeR):
for r in range(self.shape.nbRows):
for c in range(self.shape.nbCols):
if self.block.getSegment(r,c):
if r in self.rowNeighbors:
for n in self.rowNeighbors[r].values():
@ -210,9 +236,11 @@ class Validator:
if not self.receivedBlock.getSegment(rID, cID):
self.logger.trace("Recv new: %d->%d: %d,%d", src, self.ID, rID, cID, extra=self.format)
self.receivedBlock.setSegment(rID, cID)
self.sampleRecvCount += 1
if self.forwardOnReceive:
if self.perNodeQueue or self.perNeighborQueue:
self.receivedQueue.append((rID, cID))
self.msgRecvCount += 1
else:
self.logger.trace("Recv DUP: %d->%d: %d,%d", src, self.ID, rID, cID, extra=self.format)
self.statsRxDupInSlot += 1
@ -220,17 +248,23 @@ class Validator:
def addToSendQueue(self, rID, cID):
"""Queue a segment for forwarding."""
if self.perNodeQueue:
if self.perNodeQueue and not self.amImalicious:
self.sendQueue.append((rID, cID))
self.amIaddedToQueue = 1
self.msgSentCount += 1
if self.perNeighborQueue:
if self.perNeighborQueue and not self.amImalicious:
if rID in self.rowIDs:
for neigh in self.rowNeighbors[rID].values():
neigh.sendQueue.append(cID)
self.amIaddedToQueue = 1
self.msgSentCount += 1
if cID in self.columnIDs:
for neigh in self.columnNeighbors[cID].values():
neigh.sendQueue.append(rID)
self.amIaddedToQueue = 1
self.msgSentCount += 1
def receiveRowsColumns(self):
"""Finalize time step by merging newly received segments in state."""
@ -247,11 +281,14 @@ class Validator:
neigh.received |= neigh.receiving
neigh.receiving.setall(0)
for rID, cID in self.receivedQueue:
self.msgRecvCount += 1
# add newly received segments to the send queue
if self.perNodeQueue or self.perNeighborQueue:
while self.receivedQueue:
(rID, cID) = self.receivedQueue.popleft()
self.addToSendQueue(rID, cID)
if not self.amImalicious:
self.addToSendQueue(rID, cID)
def updateStats(self):
"""It updates the stats related to sent and received data."""
@ -265,25 +302,27 @@ class Validator:
def checkSegmentToNeigh(self, rID, cID, neigh):
"""Check if a segment should be sent to a neighbor."""
if (neigh.sent | neigh.received).count(1) >= (self.sendLineUntilC if neigh.dim else self.sendLineUntilR):
return False # sent enough, other side can restore
i = rID if neigh.dim else cID
if not neigh.sent[i] and not neigh.received[i] :
return True
if not self.amImalicious:
if (neigh.sent | neigh.received).count(1) >= (self.sendLineUntilC if neigh.dim else self.sendLineUntilR):
return False # sent enough, other side can restore
i = rID if neigh.dim else cID
if not neigh.sent[i] and not neigh.received[i] :
return True
else:
return False # received or already sent
return False # received or already sent or malicious
def sendSegmentToNeigh(self, rID, cID, neigh):
"""Send segment to a neighbor (without checks)."""
self.logger.trace("sending %d/%d to %d", rID, cID, neigh.node.ID, extra=self.format)
i = rID if neigh.dim else cID
neigh.sent[i] = 1
neigh.node.receiveSegment(rID, cID, self.ID)
self.statsTxInSlot += 1
if not self.amImalicious:
self.logger.trace("sending %d/%d to %d", rID, cID, neigh.node.ID, extra=self.format)
i = rID if neigh.dim else cID
neigh.sent[i] = 1
neigh.node.receiveSegment(rID, cID, self.ID)
self.statsTxInSlot += 1
def checkSendSegmentToNeigh(self, rID, cID, neigh):
"""Check and send a segment to a neighbor if needed."""
if self.checkSegmentToNeigh(rID, cID, neigh):
if self.checkSegmentToNeigh(rID, cID, neigh) and not self.amImalicious:
self.sendSegmentToNeigh(rID, cID, neigh)
return True
else:
@ -298,16 +337,18 @@ class Validator:
while self.sendQueue:
(rID, cID) = self.sendQueue[0]
if rID in self.rowIDs:
if rID in self.rowIDs and not self.amImalicious:
for _, neigh in shuffledDict(self.rowNeighbors[rID], self.shuffleNeighbors):
self.checkSendSegmentToNeigh(rID, cID, neigh)
if not self.amImalicious:
self.checkSendSegmentToNeigh(rID, cID, neigh)
if self.statsTxInSlot >= self.bwUplink:
return
if cID in self.columnIDs:
if cID in self.columnIDs and not self.amImalicious:
for _, neigh in shuffledDict(self.columnNeighbors[cID], self.shuffleNeighbors):
self.checkSendSegmentToNeigh(rID, cID, neigh)
if not self.amImalicious:
self.checkSendSegmentToNeigh(rID, cID, neigh)
if self.statsTxInSlot >= self.bwUplink:
return
@ -332,19 +373,20 @@ class Validator:
# collect and shuffle
for rID, neighs in self.rowNeighbors.items():
for neigh in neighs.values():
if (neigh.sendQueue):
if (neigh.sendQueue) and not self.amImalicious:
queues.append((0, rID, neigh))
for cID, neighs in self.columnNeighbors.items():
for neigh in neighs.values():
if (neigh.sendQueue):
if (neigh.sendQueue) and not self.amImalicious:
queues.append((1, cID, neigh))
for dim, lineID, neigh in shuffled(queues, self.shuffleQueues):
if dim == 0:
self.checkSendSegmentToNeigh(lineID, neigh.sendQueue.popleft(), neigh)
else:
self.checkSendSegmentToNeigh(neigh.sendQueue.popleft(), lineID, neigh)
if not self.amImalicious:
if dim == 0:
self.checkSendSegmentToNeigh(lineID, neigh.sendQueue.popleft(), neigh)
else:
self.checkSendSegmentToNeigh(neigh.sendQueue.popleft(), lineID, neigh)
progress = True
if self.statsTxInSlot >= self.bwUplink:
return
@ -360,32 +402,32 @@ class Validator:
def collectSegmentsToSend():
# yields list of segments to send as (dim, lineID, id)
segmentsToSend = []
for rID, neighs in self.rowNeighbors.items():
line = self.getRow(rID)
needed = zeros(self.shape.blockSizeR)
for neigh in neighs.values():
sentOrReceived = neigh.received | neigh.sent
if sentOrReceived.count(1) < self.sendLineUntilR:
needed |= ~sentOrReceived
needed &= line
if (needed).any():
for i in range(len(needed)):
if needed[i]:
segmentsToSend.append((0, rID, i))
for cID, neighs in self.columnNeighbors.items():
line = self.getColumn(cID)
needed = zeros(self.shape.blockSizeC)
for neigh in neighs.values():
sentOrReceived = neigh.received | neigh.sent
if sentOrReceived.count(1) < self.sendLineUntilC:
needed |= ~sentOrReceived
needed &= line
if (needed).any():
for i in range(len(needed)):
if needed[i]:
segmentsToSend.append((1, cID, i))
if not self.amImalicious:
for rID, neighs in self.rowNeighbors.items():
line = self.getRow(rID)
needed = zeros(self.shape.nbCols)
for neigh in neighs.values():
sentOrReceived = neigh.received | neigh.sent
if sentOrReceived.count(1) < self.sendLineUntilR:
needed |= ~sentOrReceived
needed &= line
if (needed).any():
for i in range(len(needed)):
if needed[i]:
segmentsToSend.append((0, rID, i))
for cID, neighs in self.columnNeighbors.items():
line = self.getColumn(cID)
needed = zeros(self.shape.nbRows)
for neigh in neighs.values():
sentOrReceived = neigh.received | neigh.sent
if sentOrReceived.count(1) < self.sendLineUntilC:
needed |= ~sentOrReceived
needed &= line
if (needed).any():
for i in range(len(needed)):
if needed[i]:
segmentsToSend.append((1, cID, i))
return segmentsToSend
def nextSegment():
@ -395,12 +437,12 @@ class Validator:
for dim, lineID, id in self.segmentShuffleGen:
if dim == 0:
for _, neigh in shuffledDict(self.rowNeighbors[lineID], self.shuffleNeighbors):
if self.checkSegmentToNeigh(lineID, id, neigh):
if self.checkSegmentToNeigh(lineID, id, neigh) and not self.amImalicious:
yield((lineID, id, neigh))
break
else:
for _, neigh in shuffledDict(self.columnNeighbors[lineID], self.shuffleNeighbors):
if self.checkSegmentToNeigh(id, lineID, neigh):
if self.checkSegmentToNeigh(id, lineID, neigh) and not self.amImalicious:
yield((id, lineID, neigh))
break
@ -415,7 +457,8 @@ class Validator:
for rid, cid, neigh in nextSegment():
# segments are checked just before yield, so we can send directly
self.sendSegmentToNeigh(rid, cid, neigh)
if not self.amImalicious:
self.sendSegmentToNeigh(rid, cid, neigh)
if self.statsTxInSlot >= self.bwUplink:
if not self.segmentShuffleSchedulerPersist:
@ -437,25 +480,26 @@ class Validator:
while t:
if self.rowIDs:
rID = random.choice(self.rowIDs)
cID = random.randrange(0, self.shape.blockSizeR)
cID = random.randrange(0, self.shape.nbCols)
if self.block.getSegment(rID, cID) :
neigh = random.choice(list(self.rowNeighbors[rID].values()))
if self.checkSegmentToNeigh(rID, cID, neigh):
if self.checkSegmentToNeigh(rID, cID, neigh) and not self.amImalicious:
yield(rID, cID, neigh)
t = tries
if self.columnIDs:
cID = random.choice(self.columnIDs)
rID = random.randrange(0, self.shape.blockSizeC)
rID = random.randrange(0, self.shape.nbRows)
if self.block.getSegment(rID, cID) :
neigh = random.choice(list(self.columnNeighbors[cID].values()))
if self.checkSegmentToNeigh(rID, cID, neigh):
if self.checkSegmentToNeigh(rID, cID, neigh) and not self.amImalicious:
yield(rID, cID, neigh)
t = tries
t -= 1
for rid, cid, neigh in nextSegment():
# segments are checked just before yield, so we can send directly
self.sendSegmentToNeigh(rid, cid, neigh)
if not self.amImalicious:
self.sendSegmentToNeigh(rid, cid, neigh)
if self.statsTxInSlot >= self.bwUplink:
return
@ -464,22 +508,24 @@ class Validator:
""" Send as much as we can in the timestep, limited by bwUplink."""
# process node level send queue
self.processSendQueue()
if not self.amImalicious:
self.processSendQueue()
if self.statsTxInSlot >= self.bwUplink:
return
# process neighbor level send queues in shuffled breadth-first order
self.processPerNeighborSendQueue()
if not self.amImalicious:
self.processPerNeighborSendQueue()
if self.statsTxInSlot >= self.bwUplink:
return
# process possible segments to send in shuffled breadth-first order
if self.segmentShuffleScheduler:
if self.segmentShuffleScheduler and not self.amImalicious:
self.runSegmentShuffleScheduler()
if self.statsTxInSlot >= self.bwUplink:
return
if self.dumbRandomScheduler:
if self.dumbRandomScheduler and not self.amImalicious:
self.runDumbRandomScheduler()
if self.statsTxInSlot >= self.bwUplink:
return
@ -504,14 +550,17 @@ class Validator:
def restoreRow(self, id):
"""Restore a given row if repairable."""
rep = self.block.repairRow(id)
rep, repairedSamples = self.block.repairRow(id)
self.repairedSampleCount += repairedSamples
if (rep.any()):
# If operation is based on send queues, segments should
# be queued after successful repair.
self.restoreRowCount += 1
for i in range(len(rep)):
if rep[i] or self.forwardOnRepair:
self.logger.trace("Rep: %d,%d", id, i, extra=self.format)
self.addToSendQueue(id, i)
if not self.amImalicious:
self.addToSendQueue(id, i)
# self.statsRepairInSlot += rep.count(1)
def restoreColumns(self):
@ -522,14 +571,17 @@ class Validator:
def restoreColumn(self, id):
"""Restore a given column if repairable."""
rep = self.block.repairColumn(id)
rep, repairedSamples = self.block.repairColumn(id)
self.repairedSampleCount += repairedSamples
if (rep.any()):
# If operation is based on send queues, segments should
# be queued after successful repair.
self.restoreColumnCount += 1
for i in range(len(rep)):
if rep[i] or self.forwardOnRepair:
self.logger.trace("Rep: %d,%d", i, id, extra=self.format)
self.addToSendQueue(i, id)
if not self.amImalicious:
self.addToSendQueue(i, id)
# self.statsRepairInSlot += rep.count(1)
def checkStatus(self):
@ -552,9 +604,9 @@ class Validator:
self.logger.debug("status: %d / %d", arrived, expected, extra=self.format)
validated = 0
for i in range(self.vpn):
a, e = checkStatus(self.vColumnIDs[i], self.vRowIDs[i])
for v in self.validators:
a, e = checkStatus(v.columnIDs, v.rowIDs)
if a == e:
validated+=1
return arrived, expected, validated
return arrived, expected, validated

16
DAS/observer.py
View File

@ -11,11 +11,11 @@ class Observer:
self.config = config
self.format = {"entity": "Observer"}
self.logger = logger
self.block = [0] * self.config.blockSizeR * self.config.blockSizeC
self.rows = [0] * self.config.blockSizeC
self.columns = [0] * self.config.blockSizeR
self.broadcasted = Block(self.config.blockSizeR, self.config.blockSizeRK,
self.config.blockSizeC, self.config.blockSizeCK)
self.block = [0] * self.config.nbCols * self.config.nbRows
self.rows = [0] * self.config.nbRows
self.columns = [0] * self.config.nbCols
self.broadcasted = Block(self.config.nbCols, self.config.nbColsK,
self.config.nbRows, self.config.nbRowsK)
def checkRowsColumns(self, validators):
@ -27,7 +27,7 @@ class Observer:
for c in val.columnIDs:
self.columns[c] += 1
for i in range(self.config.blockSizeC):
for i in range(self.config.nbRows):
self.logger.debug("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format)
if self.rows[i] == 0 or self.columns[i] == 0:
self.logger.warning("There is a row/column that has not been assigned", extra=self.format)
@ -35,7 +35,7 @@ class Observer:
def checkBroadcasted(self):
"""It checks how many broadcasted samples are still missing in the network."""
zeros = 0
for i in range(self.blockSizeR * self.blockSizeC):
for i in range(self.nbCols * self.nbRows):
if self.broadcasted.data[i] == 0:
zeros += 1
if zeros > 0:
@ -100,4 +100,4 @@ class Observer:
"RxDup": {"mean": meanOrNan(RxDup), "max": maxOrNan(RxDup)},
}
return trafficStats
return trafficStats

20
DAS/results.py
View File

@ -16,6 +16,26 @@ class Result:
self.tta = -1
self.missingVector = []
self.metrics = {}
self.amImalicious = [0] * shape.numberNodes
self.msgSentCount = [0] * shape.numberNodes
self.msgRecvCount = [0] * shape.numberNodes
self.sampleRecvCount = [0] * shape.numberNodes
self.restoreRowCount = [0] * shape.numberNodes
self.restoreColumnCount = [0] * shape.numberNodes
self.repairedSampleCount = [0] * shape.numberNodes
self.numberNodes = shape.numberNodes
self.class1ratio = shape.class1ratio
def copyValidators(self, validators):
"""Copy information from simulator.validators to result."""
for i in range(0,self.shape.numberNodes):
self.amImalicious[i] = validators[i].amImalicious
self.msgSentCount[i] = validators[i].msgSentCount
self.msgRecvCount[i] = validators[i].msgRecvCount
self.sampleRecvCount[i] = validators[i].sampleRecvCount
self.restoreRowCount[i] = validators[i].restoreRowCount
self.restoreColumnCount[i] = validators[i].restoreColumnCount
self.repairedSampleCount[i] = validators[i].repairedSampleCount
def populate(self, shape, config, missingVector):
"""It populates part of the result data inside a vector."""

32
DAS/shape.py
View File

@ -2,22 +2,22 @@
class Shape:
"""This class represents a set of parameters for a specific simulation."""
def __init__(self, blockSizeR, blockSizeRK, blockSizeC, blockSizeCK,
numberNodes, failureModel, failureRate, class1ratio, chiR, chiC, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run):
def __init__(self, nbCols, nbColsK, nbRows, nbRowsK,
numberNodes, failureModel, failureRate, maliciousNodes, class1ratio, custodyRows, custodyCols, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run):
"""Initializes the shape with the parameters passed in argument."""
self.run = run
self.numberNodes = numberNodes
self.blockSizeR = blockSizeR
self.blockSizeRK = blockSizeRK
self.blockSizeC = blockSizeC
self.blockSizeCK = blockSizeCK
self.nbCols = nbCols
self.nbColsK = nbColsK
self.nbRows = nbRows
self.nbRowsK = nbRowsK
self.failureModel = failureModel
self.failureRate = failureRate
self.maliciousNodes = maliciousNodes
self.netDegree = netDegree
self.class1ratio = class1ratio
self.chiR = chiR
self.chiC = chiC
self.custodyRows = custodyRows
self.custodyCols = custodyCols
self.vpn1 = vpn1
self.vpn2 = vpn2
self.bwUplinkProd = bwUplinkProd
@ -28,16 +28,16 @@ class Shape:
def __repr__(self):
"""Returns a printable representation of the shape"""
shastr = ""
shastr += "bsrn-"+str(self.blockSizeR)
shastr += "-bsrk-"+str(self.blockSizeRK)
shastr += "-bscn-"+str(self.blockSizeC)
shastr += "-bsck-"+str(self.blockSizeCK)
shastr += "bsrn-"+str(self.nbCols)
shastr += "-bsrk-"+str(self.nbColsK)
shastr += "-bscn-"+str(self.nbRows)
shastr += "-bsck-"+str(self.nbRowsK)
shastr += "-nn-"+str(self.numberNodes)
shastr += "-fm-"+str(self.failureModel)
shastr += "-fr-"+str(self.failureRate)
shastr += "-c1r-"+str(self.class1ratio)
shastr += "-chir-"+str(self.chiR)
shastr += "-chic-"+str(self.chiC)
shastr += "-cusr-"+str(self.custodyRows)
shastr += "-cusc-"+str(self.custodyCols)
shastr += "-vpn1-"+str(self.vpn1)
shastr += "-vpn2-"+str(self.vpn2)
shastr += "-bwupprod-"+str(self.bwUplinkProd)
@ -45,9 +45,9 @@ class Shape:
shastr += "-bwup2-"+str(self.bwUplink2)
shastr += "-nd-"+str(self.netDegree)
shastr += "-r-"+str(self.run)
shastr += "-mn-"+str(self.maliciousNodes)
return shastr
def setSeed(self, seed):
"""Adds the random seed to the shape"""
self.randomSeed = seed

135
DAS/simulator.py
View File

@ -8,7 +8,7 @@ from datetime import datetime
from DAS.tools import *
from DAS.results import *
from DAS.observer import *
from DAS.validator import *
from DAS.node import *
class Simulator:
"""This class implements the main DAS simulator."""
@ -48,48 +48,69 @@ class Simulator:
"""It initializes all the validators in the network."""
self.glob = Observer(self.logger, self.shape)
self.validators = []
if self.config.evenLineDistribution:
evenLineDistribution = False
if evenLineDistribution:
lightNodes = int(self.shape.numberNodes * self.shape.class1ratio)
heavyNodes = self.shape.numberNodes - lightNodes
lightVal = lightNodes * self.shape.vpn1
heavyVal = heavyNodes * self.shape.vpn2
totalValidators = lightVal + heavyVal
totalRows = totalValidators * self.shape.chiR
totalColumns = totalValidators * self.shape.chiC
rows = list(range(self.shape.blockSizeC)) * (int(totalRows/self.shape.blockSizeC)+1)
columns = list(range(self.shape.blockSizeR)) * (int(totalColumns/self.shape.blockSizeR)+1)
totalRows = totalValidators * self.shape.custodyRows
totalColumns = totalValidators * self.shape.custodyCols
rows = list(range(self.shape.nbRows)) * (int(totalRows/self.shape.nbRows)+1)
columns = list(range(self.shape.nbCols)) * (int(totalColumns/self.shape.nbCols)+1)
rows = rows[0:totalRows]
columns = columns[0:totalRows]
columns = columns[0:totalColumns]
random.shuffle(rows)
random.shuffle(columns)
offsetR = lightVal*self.shape.chiR
offsetC = lightVal*self.shape.chiC
offsetR = lightVal*self.shape.custodyRows
offsetC = lightVal*self.shape.custodyCols
self.logger.debug("There is a total of %d nodes, %d light and %d heavy." % (self.shape.numberNodes, lightNodes, heavyNodes), extra=self.format)
self.logger.debug("There is a total of %d validators, %d in light nodes and %d in heavy nodes" % (totalValidators, lightVal, heavyVal), extra=self.format)
self.logger.debug("Shuffling a total of %d rows to be assigned (X=%d)" % (len(rows), self.shape.chiR), extra=self.format)
self.logger.debug("Shuffling a total of %d columns to be assigned (X=%d)" % (len(columns), self.shape.chiC), extra=self.format)
self.logger.debug("Shuffling a total of %d rows to be assigned (X=%d)" % (len(rows), self.shape.custodyRows), extra=self.format)
self.logger.debug("Shuffling a total of %d columns to be assigned (X=%d)" % (len(columns), self.shape.custodyCols), extra=self.format)
self.logger.debug("Shuffled rows: %s" % str(rows), extra=self.format)
self.logger.debug("Shuffled columns: %s" % str(columns), extra=self.format)
assignedRows = []
assignedCols = []
maliciousNodesCount = int((self.shape.maliciousNodes / 100) * self.shape.numberNodes)
remainingMaliciousNodes = maliciousNodesCount
for i in range(self.shape.numberNodes):
if self.config.evenLineDistribution:
if i == 0:
amImalicious_value = 0
else:
if not self.config.randomizeMaliciousNodes:
# Assign based on predefined pattern when randomization is turned off
if i < maliciousNodesCount + 1:
amImalicious_value = 1
else:
amImalicious_value = 0
else:
# Randomly assign amImalicious_value when randomization is turned on
if remainingMaliciousNodes > 0 and random.random() < (self.shape.maliciousNodes / 100):
amImalicious_value = 1
remainingMaliciousNodes -= 1
else:
amImalicious_value = 0
if evenLineDistribution:
if i < int(lightVal/self.shape.vpn1): # First start with the light nodes
startR = i *self.shape.chiR*self.shape.vpn1
endR = (i+1)*self.shape.chiR*self.shape.vpn1
startC = i *self.shape.chiC*self.shape.vpn1
endC = (i+1)*self.shape.chiC*self.shape.vpn1
startR = i *self.shape.custodyRows*self.shape.vpn1
endR = (i+1)*self.shape.custodyRows*self.shape.vpn1
startC = i *self.shape.custodyCols*self.shape.vpn1
endC = (i+1)*self.shape.custodyCols*self.shape.vpn1
else:
j = i - int(lightVal/self.shape.vpn1)
startR = offsetR+( j *self.shape.chiR*self.shape.vpn2)
endR = offsetR+((j+1)*self.shape.chiR*self.shape.vpn2)
startC = offsetC+( j *self.shape.chiC*self.shape.vpn2)
endC = offsetC+((j+1)*self.shape.chiC*self.shape.vpn2)
startR = offsetR+( j *self.shape.custodyRows*self.shape.vpn2)
endR = offsetR+((j+1)*self.shape.custodyRows*self.shape.vpn2)
startC = offsetC+( j *self.shape.custodyCols*self.shape.vpn2)
endC = offsetC+((j+1)*self.shape.custodyCols*self.shape.vpn2)
r = rows[startR:endR]
c = columns[startC:endC]
val = Validator(i, int(not i!=0), self.logger, self.shape, self.config, r, c)
val = Node(i, int(not i!=0), amImalicious_value, self.logger, self.shape, self.config, r, c)
self.logger.debug("Node %d has row IDs: %s" % (val.ID, val.rowIDs), extra=self.format)
self.logger.debug("Node %d has column IDs: %s" % (val.ID, val.columnIDs), extra=self.format)
assignedRows = assignedRows + list(r)
@ -98,7 +119,17 @@ class Simulator:
self.nodeColumns.append(val.columnIDs)
else:
val = Validator(i, int(not i!=0), self.logger, self.shape, self.config)
if self.shape.custodyCols > self.shape.nbCols:
self.logger.error("custodyCols has to be smaller than %d" % self.shape.nbCols)
elif self.shape.custodyRows > self.shape.nbRows:
self.logger.error("custodyRows has to be smaller than %d" % self.shape.nbRows)
vs = []
nodeClass = 1 if (i <= self.shape.numberNodes * self.shape.class1ratio) else 2
vpn = self.shape.vpn1 if (nodeClass == 1) else self.shape.vpn2
for v in range(vpn):
vs.append(initValidator(self.shape.nbRows, self.shape.custodyRows, self.shape.nbCols, self.shape.custodyCols))
val = Node(i, int(not i!=0), amImalicious_value, self.logger, self.shape, self.config, vs)
if i == self.proposerID:
val.initBlock()
else:
@ -113,8 +144,8 @@ class Simulator:
def initNetwork(self):
"""It initializes the simulated network."""
rowChannels = [[] for i in range(self.shape.blockSizeC)]
columnChannels = [[] for i in range(self.shape.blockSizeR)]
rowChannels = [[] for i in range(self.shape.nbRows)]
columnChannels = [[] for i in range(self.shape.nbCols)]
for v in self.validators:
if not (self.proposerPublishOnly and v.amIproposer):
for id in v.rowIDs:
@ -130,7 +161,7 @@ class Simulator:
self.logger.debug("Number of validators per row; Min: %d, Max: %d" % (min(self.distR), max(self.distR)), extra=self.format)
self.logger.debug("Number of validators per column; Min: %d, Max: %d" % (min(self.distC), max(self.distC)), extra=self.format)
for id in range(self.shape.blockSizeC):
for id in range(self.shape.nbRows):
# If the number of nodes in a channel is smaller or equal to the
# requested degree, a fully connected graph is used. For n>d, a random
@ -148,10 +179,10 @@ class Simulator:
for u, v in G.edges:
val1=rowChannels[id][u]
val2=rowChannels[id][v]
val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, 0, self.shape.blockSizeR)})
val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, 0, self.shape.blockSizeR)})
val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, 0, self.shape.nbCols)})
val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, 0, self.shape.nbCols)})
for id in range(self.shape.blockSizeR):
for id in range(self.shape.nbCols):
if not columnChannels[id]:
self.logger.error("No nodes for column %d !" % id, extra=self.format)
@ -166,8 +197,8 @@ class Simulator:
for u, v in G.edges:
val1=columnChannels[id][u]
val2=columnChannels[id][v]
val1.columnNeighbors[id].update({val2.ID : Neighbor(val2, 1, self.shape.blockSizeC)})
val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, 1, self.shape.blockSizeC)})
val1.columnNeighbors[id].update({val2.ID : Neighbor(val2, 1, self.shape.nbRows)})
val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, 1, self.shape.nbRows)})
for v in self.validators:
if (self.proposerPublishOnly and v.amIproposer):
@ -175,12 +206,12 @@ class Simulator:
count = min(self.proposerPublishToR, len(rowChannels[id]))
publishTo = random.sample(rowChannels[id], count)
for vi in publishTo:
v.rowNeighbors[id].update({vi.ID : Neighbor(vi, 0, self.shape.blockSizeR)})
v.rowNeighbors[id].update({vi.ID : Neighbor(vi, 0, self.shape.nbCols)})
for id in v.columnIDs:
count = min(self.proposerPublishToC, len(columnChannels[id]))
publishTo = random.sample(columnChannels[id], count)
for vi in publishTo:
v.columnNeighbors[id].update({vi.ID : Neighbor(vi, 1, self.shape.blockSizeC)})
v.columnNeighbors[id].update({vi.ID : Neighbor(vi, 1, self.shape.nbRows)})
if self.logger.isEnabledFor(logging.DEBUG):
for i in range(0, self.shape.numberNodes):
@ -231,7 +262,8 @@ class Simulator:
self.glob.checkRowsColumns(self.validators)
for i in range(0,self.shape.numberNodes):
if i == self.proposerID:
self.validators[i].initBlock()
# self.validators[i].initBlock()
self.logger.warning("I am a block proposer.", extra=self.format)
else:
self.validators[i].logIDs()
arrived, expected, ready, validatedall, validated = self.glob.checkStatus(self.validators)
@ -239,13 +271,17 @@ class Simulator:
missingVector = []
progressVector = []
trafficStatsVector = []
malicious_nodes_not_added_count = 0
steps = 0
while(True):
missingVector.append(missingSamples)
self.logger.debug("Expected Samples: %d" % expected, extra=self.format)
self.logger.debug("Missing Samples: %d" % missingSamples, extra=self.format)
oldMissingSamples = missingSamples
self.logger.debug("PHASE SEND %d" % steps, extra=self.format)
for i in range(0,self.shape.numberNodes):
self.validators[i].send()
if not self.validators[i].amImalicious:
self.validators[i].send()
self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format)
for i in range(1,self.shape.numberNodes):
self.validators[i].receiveRowsColumns()
@ -281,10 +317,18 @@ class Simulator:
cnD1 = "Dup class1 mean"
cnD2 = "Dup class2 mean"
# if custody is based on the requirements of underlying individual
# validators, we can get detailed data on how many validated.
# Otherwise, we can only use the weighted average.
if self.config.validatorBasedCustody:
cnVv = validatorProgress
else:
cnVv = validatorAllProgress
progressVector.append({
cnS:sampleProgress,
cnN:nodeProgress,
cnV:validatorProgress,
cnV:cnVv,
cnT0: trafficStats[0]["Tx"]["mean"],
cnT1: trafficStats[1]["Tx"]["mean"],
cnT2: trafficStats[2]["Tx"]["mean"],
@ -308,6 +352,25 @@ class Simulator:
break
steps += 1
for i in range(0,self.shape.numberNodes):
if not self.validators[i].amIaddedToQueue :
malicious_nodes_not_added_count += 1
for i in range(0,self.shape.numberNodes):
column_ids = []
row_ids = []
for rID in self.validators[i].rowIDs:
row_ids.append(rID)
for cID in self.validators[i].columnIDs:
column_ids.append(cID)
self.logger.debug("List of columnIDs for %d node: %s", i, column_ids, extra=self.format)
self.logger.debug("List of rowIDs for %d node: %s", i, row_ids, extra=self.format)
self.logger.debug("Number of malicious nodes not added to the send queue: %d" % malicious_nodes_not_added_count, extra=self.format)
malicious_nodes_not_added_percentage = (malicious_nodes_not_added_count * 100)/(self.shape.numberNodes)
self.logger.debug("Percentage of malicious nodes not added to the send queue: %d" % malicious_nodes_not_added_percentage, extra=self.format)
progress = pd.DataFrame(progressVector)
if self.config.saveRCdist:
self.result.addMetric("rowDist", self.distR)
@ -315,5 +378,5 @@ class Simulator:
if self.config.saveProgress:
self.result.addMetric("progress", progress.to_dict(orient='list'))
self.result.populate(self.shape, self.config, missingVector)
self.result.copyValidators(self.validators)
return self.result

25
DAS/visualizer.py
View File

@ -17,7 +17,7 @@ class Visualizer:
self.execID = execID
self.config = config
self.folderPath = "results/"+self.execID
self.parameters = ['run', 'blockSize', 'failureRate', 'numberNodes', 'netDegree', 'chi', 'vpn1', 'vpn2', 'class1ratio', 'bwUplinkProd', 'bwUplink1', 'bwUplink2']
self.parameters = ['run', 'blockSize', 'failureRate', 'numberNodes', 'netDegree', 'cus', 'vpn1', 'vpn2', 'class1ratio', 'bwUplinkProd', 'bwUplink1', 'bwUplink2']
self.minimumDataPoints = 2
self.maxTTA = 11000
@ -33,12 +33,12 @@ class Visualizer:
tree = ET.parse(os.path.join(self.folderPath, filename))
root = tree.getroot()
run = int(root.find('run').text)
blockSize = int(root.find('blockSizeR').text) # TODO: maybe we want both dimensions
blockSize = int(root.find('nbCols').text) # TODO: maybe we want both dimensions
failureRate = int(root.find('failureRate').text)
numberNodes = int(root.find('numberNodes').text)
class1ratio = float(root.find('class1ratio').text)
netDegree = int(root.find('netDegree').text)
chi = int(root.find('chiR').text) # TODO: maybe we want both dimensions
custodyRows = int(root.find('custodyRows').text) # TODO: maybe we want both dimensions
vpn1 = int(root.find('vpn1').text)
vpn2 = int(root.find('vpn2').text)
bwUplinkProd = int(root.find('bwUplinkProd').text)
@ -54,7 +54,7 @@ class Visualizer:
# Get the indices and values of the parameters in the combination
indices = [self.parameters.index(element) for element in combination]
selectedValues = [run, blockSize, failureRate, numberNodes, netDegree, chi, vpn1, vpn2, class1ratio, bwUplinkProd, bwUplink1, bwUplink2]
selectedValues = [run, blockSize, failureRate, numberNodes, netDegree, custodyRows, vpn1, vpn2, class1ratio, bwUplinkProd, bwUplink1, bwUplink2]
values = [selectedValues[index] for index in indices]
names = [self.parameters[i] for i in indices]
keyComponents = [f"{name}_{value}" for name, value in zip(names, values)]
@ -241,21 +241,6 @@ class Visualizer:
plt.savefig(filename)
plt.clf()
def plotHist(self, bandwidth):
"""Plot Bandwidth Frequency Histogram"""
plt.hist(bandwidth, bins=5)
plt.xlabel('Bandwidth')
plt.ylabel('Frequency')
plt.title('Bandwidth Histogram')
"""Create the directory if it doesn't exist already"""
histogramFolder = self.folderPath + '/histogram'
if not os.path.exists(histogramFolder):
os.makedirs(histogramFolder)
filename = os.path.join(histogramFolder, 'histogram.png')
plt.savefig(filename)
plt.clf()
def plotCandleStick(self, TX_prod, TX_avg, TX_max):
#x-axis corresponding to steps
steps = range(len(TX_prod))
@ -274,4 +259,4 @@ class Visualizer:
plt.ylabel('Price')
#Test
plt.show()
plt.show()

738
DAS/visualizor.py
View File

@ -1,23 +1,33 @@
#!/bin/python3
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import os
def plotData(conf):
plt.clf()
fig = plt.figure("9, 3")
plt.grid(True)
if conf["desLoc"] == 1:
xDes = 0
else:
xDes = conf["xdots"][-1] * 0.6
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(xDes, conf["yaxismax"]/4, conf["textBox"], fontsize=10, verticalalignment='top', bbox=props)
for i in range(len(conf["data"])):
if conf["type"] == "plot":
plt.text(xDes, conf["yaxismax"]/3, conf["textBox"], fontsize=10, verticalalignment='top', bbox=props)
if conf["type"] == "plot" or conf["type"] == "plot_with_1line":
for i in range(len(conf["data"])):
plt.plot(conf["xdots"], conf["data"][i], conf["colors"][i], label=conf["labels"][i])
if conf["type"] == "bar":
elif conf["type"] == "individual_bar" or conf["type"] == "individual_bar_with_2line":
plt.bar(conf["xdots"], conf["data"])
elif conf["type"] == "grouped_bar":
for i in range(len(conf["data"])):
plt.bar(conf["xdots"], conf["data"][i], label=conf["labels"][i])
if conf["type"] == "individual_bar_with_2line":
plt.axhline(y = conf["expected_value1"], color='r', linestyle='--', label=conf["line_label1"])
plt.axhline(y = conf["expected_value2"], color='g', linestyle='--', label=conf["line_label2"])
if conf["type"] == "plot_with_1line":
plt.axhline(y = conf["expected_value"], color='g', linestyle='--', label=conf["line_label"])
plt.title(conf["title"])
plt.ylabel(conf["ylabel"])
plt.xlabel(conf["xlabel"])
@ -25,6 +35,20 @@ def plotData(conf):
plt.legend(loc=conf["legLoc"])
plt.savefig(conf["path"], bbox_inches="tight")
def plotBoxData(conf):
plt.clf()
plt.grid(True)
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
num_boxes = len(conf["data"])
positions = np.arange(num_boxes)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.boxplot(conf["data"], patch_artist=True, showmeans=True, meanline=True, positions=positions)
plt.title(conf["title"], fontsize=14)
plt.ylabel(conf["ylabel"], fontsize=12)
plt.xlabel(conf["xlabel"], fontsize=12)
plt.xticks(fontsize=10)
plt.yticks(fontsize=10)
plt.savefig(conf["path"], bbox_inches="tight")
class Visualizor:
"""This class helps the visualization of the results"""
@ -69,6 +93,32 @@ class Visualizor:
def plotHeatmaps(self, x, y):
"""Plot the heatmap using the parameters given as x axis and y axis"""
print("Plotting heatmap "+x+" vs "+y)
#Find the location of x in shape
#Find the location of y in shape
#Find the location od r in shape
#Loop over all results
#Add unique values foir every parameter
#Find number of runs from r
#If number of values for x and y > 3 then plot heatmap, otherwise finish
#Create a 2D grid with the dimensions of the number of values for x and y
#For all values of x
#For all values of y
# For all values in r
#Fixing all other values to 1 (in the mean time)
#Add/sum TTA into 2D grid
#if last r divide by number of runs
#Plot 2D grid
def plotAll(self):
"""Plot all the important elements of each result"""
for result in self.results:
@ -79,17 +129,573 @@ class Visualizor:
self.plotSentData(result, plotPath)
self.plotRecvData(result, plotPath)
self.plotDupData(result, plotPath)
if self.config.saveRCdist:
self.plotRowCol(result, plotPath)
# self.plotSamplesRepaired(result, plotPath)
# self.plotMessagesSent(result, plotPath)
# self.plotMessagesRecv(result, plotPath)
# self.plotSampleRecv(result, plotPath)
# self.plotRestoreRowCount(result, plotPath)
# self.plotRestoreColumnCount(result, plotPath)
# if self.config.saveRCdist:
# self.plotRowCol(result, plotPath)
# self.plotBoxSamplesRepaired(result, plotPath)
# self.plotBoxMessagesSent(result, plotPath)
# self.plotBoxMessagesRecv(result, plotPath)
# self.plotBoxSampleRecv(result, plotPath)
# self.plotBoxRestoreColumnCount(result, plotPath)
# self.plotBoxRestoreRowCount(result, plotPath)
# if self.config.saveRCdist:
# self.plotBoxRowCol(result, plotPath)
self.plotBoxenSamplesRepaired(result, plotPath)
self.plotBoxenMessagesSent(result, plotPath)
self.plotBoxenMessagesRecv(result, plotPath)
self.plotBoxenSamplesRecv(result, plotPath)
self.plotBoxenRestoreRowCount(result, plotPath)
self.plotBoxenRestoreColumnCount(result, plotPath)
if self.config.saveRCdist:
self.plotBoxenRowColDist(result, plotPath)
self.plotECDFSamplesRepaired(result, plotPath)
self.plotECDFMessagesSent(result, plotPath)
self.plotECDFMessagesRecv(result, plotPath)
self.plotECDFSamplesReceived(result, plotPath)
self.plotECDFRestoreRowCount(result, plotPath)
self.plotECDFRestoreColumnCount(result, plotPath)
if self.config.saveRCdist:
self.plotECDFRowColDist(result, plotPath)
def plotBoxRestoreRowCount(self, result, plotPath):
"""Box Plot of restoreRowCount for all nodes"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Box Plot of Restore Row Count by Nodes"
conf["xlabel"] = "Node Type"
conf["ylabel"] = "Restore Row Count"
n1 = int(result.numberNodes * result.class1ratio)
class1_data = result.restoreRowCount[1: n1]
class2_data = result.restoreRowCount[n1+1: ]
data = [class1_data, class2_data]
plt.boxplot(data)
plt.xticks([1, 2], ['Class 1 Nodes', 'Class 2 Nodes'])
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/box_restoreRowCount.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/box_restoreRowCount.png"))
def plotBoxRestoreColumnCount(self, result, plotPath):
"""Box Plot of restoreColumnCount for all nodes"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Box Plot of Restore Column Count by Nodes"
conf["xlabel"] = "Node Type"
conf["ylabel"] = "Restore Column Count"
n1 = int(result.numberNodes * result.class1ratio)
class1_data = result.restoreColumnCount[1: n1]
class2_data = result.restoreColumnCount[n1+1: ]
data = [class1_data, class2_data]
plt.boxplot(data)
plt.xticks([1, 2], ['Class 1 Nodes', 'Class 2 Nodes'])
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/box_restoreColumnCount.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/box_restoreColumnCount.png"))
def plotBoxenRestoreRowCount(self, result, plotPath):
"""Plots the Boxen plot of restoreRowCount for all nodes"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Boxen Plot of Restore Row Count by Nodes"
conf["xlabel"] = "Restore Row Count"
conf["ylabel"] = "Nodes"
n1 = int(result.numberNodes * result.class1ratio)
data = [result.restoreRowCount[1: n1], result.restoreRowCount[n1+1: ]]
plt.figure(figsize=(8, 6))
sns.boxenplot(data=data, width=0.8)
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/boxen_restoreRowCount.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/boxen_restoreRowCount.png"))
def plotBoxenRestoreColumnCount(self, result, plotPath):
"""Plots the Boxen plot of restoreColumnCount for all nodes"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Boxen Plot of Restore Column Count by Nodes"
conf["xlabel"] = "Restore Column Count"
conf["ylabel"] = "Nodes"
n1 = int(result.numberNodes * result.class1ratio)
data = [result.restoreColumnCount[1: n1], result.restoreColumnCount[n1+1: ]]
plt.figure(figsize=(8, 6))
sns.boxenplot(data=data, width=0.8)
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/boxen_restoreColumnCount.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/boxen_restoreColumnCount.png"))
def plotECDFRestoreRowCount(self, result, plotPath):
"""Plots the ECDF of restoreRowCount for all nodes using seaborn's ecdfplot"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "ECDF of Restore Row Count by Nodes"
conf["xlabel"] = "Restore Row Count"
conf["ylabel"] = "ECDF"
n1 = int(result.numberNodes * result.class1ratio)
class1_data = result.restoreRowCount[1: n1]
class2_data = result.restoreRowCount[n1+1: ]
sns.ecdfplot(data=class1_data, label='Class 1 Nodes')
sns.ecdfplot(data=class2_data, label='Class 2 Nodes')
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
plt.xlim(left=0, right=max(result.restoreRowCount) * 1.1)
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'], loc=1)
plt.savefig(plotPath + "/ecdf_restoreRowCount.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/ecdf_restoreRowCount.png"))
def plotECDFRestoreColumnCount(self, result, plotPath):
"""Plots the ECDF of restoreColumnCount for all nodes using seaborn's ecdfplot"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "ECDF of Restore Column Count by Nodes"
conf["xlabel"] = "Restore Column Count"
conf["ylabel"] = "ECDF"
n1 = int(result.numberNodes * result.class1ratio)
class1_data = result.restoreColumnCount[1: n1]
class2_data = result.restoreColumnCount[n1+1: ]
sns.ecdfplot(data=class1_data, label='Class 1 Nodes')
sns.ecdfplot(data=class2_data, label='Class 2 Nodes')
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
plt.xlim(left=0, right=max(result.restoreColumnCount) * 1.1)
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'], loc=1)
plt.savefig(plotPath + "/ecdf_restoreColumnCount.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/ecdf_restoreColumnCount.png"))
def plotECDFMessagesSent(self, result, plotPath):
"""Plots the ECDF of messages sent by all nodes using seaborn's ecdfplot"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "ECDF of Messages Sent by Nodes"
conf["xlabel"] = "Number of Messages Sent"
conf["ylabel"] = "ECDF"
n1 = int(result.numberNodes * result.class1ratio)
class1_data = result.msgSentCount[1: n1]
class2_data = result.msgSentCount[n1+1: ]
sns.ecdfplot(data=class1_data, label='Class 1 Nodes')
sns.ecdfplot(data=class2_data, label='Class 2 Nodes')
plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'], loc=1)
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
plt.xlim(left=0, right=max(result.msgSentCount) * 1.1)
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/ecdf_messagesSent.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/ecdf_messagesSent.png"))
def plotECDFMessagesRecv(self, result, plotPath):
"""Plots the ECDF of messages received by all nodes using seaborn's ecdfplot"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "ECDF of Messages Received by Nodes"
conf["xlabel"] = "Number of Messages Received"
conf["ylabel"] = "ECDF"
n1 = int(result.numberNodes * result.class1ratio)
class1_data = result.msgRecvCount[1: n1]
class2_data = result.msgRecvCount[n1+1: ]
sns.ecdfplot(data=class1_data, label='Class 1 Nodes')
sns.ecdfplot(data=class2_data, label='Class 2 Nodes')
plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'], loc=1)
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
plt.xlim(left=0, right=max(result.msgRecvCount) * 1.1)
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/ecdf_messagesRecv.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/ecdf_messagesRecv.png"))
def plotECDFSamplesReceived(self, result, plotPath):
"""Plots the ECDF of samples received by all nodes using seaborn's ecdfplot"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "ECDF of Samples Received by Nodes"
conf["xlabel"] = "Number of Samples Received"
conf["ylabel"] = "ECDF"
n1 = int(result.numberNodes * result.class1ratio)
class1_data = result.sampleRecvCount[1: n1]
class2_data = result.sampleRecvCount[n1+1: ]
sns.ecdfplot(data=class1_data, label='Class 1 Nodes')
sns.ecdfplot(data=class2_data, label='Class 2 Nodes')
plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'], loc=1)
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
plt.xlim(left=0, right=max(result.sampleRecvCount) * 1.1)
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/ecdf_samplesReceived.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/ecdf_samplesReceived.png"))
def plotECDFRowColDist(self, result, plotPath):
"""Plots the ECDF of row col distance by all nodes using seaborn's ecdfplot"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "ECDF of Row-Col Distance by Nodes"
conf["xlabel"] = "Row-Col Distance"
conf["ylabel"] = "ECDF"
vector1 = result.metrics["rowDist"]
vector2 = result.metrics["columnDist"]
n1 = int(result.numberNodes * result.class1ratio)
sns.ecdfplot(data=vector1, label='Rows')
sns.ecdfplot(data=vector2, label='Columns')
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
plt.xlim(left=0, right=max(max(vector1), max(vector2)) * 1.1)
plt.legend(labels=['Row Dist', 'Column Dist'], loc=1)
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/ecdf_rowColDist.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/ecdf_rowColDist.png"))
def plotECDFSamplesRepaired(self, result, plotPath):
"""Plots the ECDF of samples repaired by all nodes using seaborn's ecdfplot"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "ECDF of Samples Repaired by Nodes"
conf["xlabel"] = "Number of Samples Repaired"
conf["ylabel"] = "ECDF"
n1 = int(result.numberNodes * result.class1ratio)
class1_data = result.repairedSampleCount[1: n1]
class2_data = result.repairedSampleCount[n1+1: ]
sns.ecdfplot(data=class1_data, label='Class 1 Nodes')
sns.ecdfplot(data=class2_data, label='Class 2 Nodes')
plt.legend(title='Node Class', labels=['Class 1 Nodes', 'Class 2 Nodes'])
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
plt.xlim(left=0, right=max(result.repairedSampleCount) * 1.1)
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/ecdf_samplesRepaired.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/ecdf_samplesRepaired.png"))
def plotBoxenSamplesRecv(self, result, plotPath):
"""Boxen Plot of the number of samples received by all nodes"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Samples Received by Nodes"
conf["xlabel"] = "Node Type"
conf["ylabel"] = "Number of Samples Received"
n1 = int(result.numberNodes * result.class1ratio)
data = [result.sampleRecvCount[1: n1], result.sampleRecvCount[n1+1: ]]
plt.figure(figsize=(8, 6))
sns.boxenplot(data=data, width=0.8)
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.tight_layout()
plt.savefig(plotPath + "/boxen_samplesRecv.png")
plt.close()
print("Plot %s created." % (plotPath + "/boxen_samplesRecv.png"))
def plotBoxenSamplesRepaired(self, result, plotPath):
"""Boxen Plot of the number of samples repaired by all nodes"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Samples Repaired by Nodes"
conf["xlabel"] = "Node Type"
conf["ylabel"] = "Number of Samples Repaired"
n1 = int(result.numberNodes * result.class1ratio)
data = [result.repairedSampleCount[1: n1], result.repairedSampleCount[n1+1: ]]
plt.figure(figsize=(8, 6))
sns.boxenplot(data=data, width=0.8)
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.tight_layout()
plt.savefig(plotPath + "/boxen_samplesRepaired.png")
plt.close()
print("Plot %s created." % (plotPath + "/boxen_samplesRepaired.png"))
def plotBoxenRowColDist(self, result, plotPath):
"""Boxen Plot of the Row/Column distribution"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Row/Column Distribution"
conf["xlabel"] = "Row/Column Type"
conf["ylabel"] = "Validators Subscribed"
vector1 = result.metrics["rowDist"]
vector2 = result.metrics["columnDist"]
if len(vector1) > len(vector2):
vector2 += [np.nan] * (len(vector1) - len(vector2))
elif len(vector1) < len(vector2):
vector1 += [np.nan] * (len(vector2) - len(vector1))
data = [vector1, vector2]
plt.figure(figsize=(8, 6))
sns.boxenplot(data=data, width=0.8)
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.tight_layout()
plt.savefig(plotPath + "/boxen_rowColDist.png")
plt.close()
print("Plot %s created." % (plotPath + "/boxen_rowColDist.png"))
def plotBoxenMessagesSent(self, result, plotPath):
"""Plots the number of messages sent by all nodes using seaborn's boxenplot"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Messages Sent by Nodes"
conf["xlabel"] = "Node Type"
conf["ylabel"] = "Number of Messages Sent"
n1 = int(result.numberNodes * result.class1ratio)
data = [result.msgSentCount[1: n1], result.msgSentCount[n1+1: ]]
labels = ["Class 1", "Class 2"]
sns.boxenplot(data=data, palette="Set2", ax=plt.gca())
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/boxen_messagesSent.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/boxen_messagesSent.png"))
def plotBoxenMessagesRecv(self, result, plotPath):
"""Plots the number of messages received by all nodes using seaborn's boxenplot"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Messages Received by Nodes"
conf["xlabel"] = "Node Type"
conf["ylabel"] = "Number of Messages Received"
n1 = int(result.numberNodes * result.class1ratio)
data = [result.msgRecvCount[1: n1], result.msgRecvCount[n1+1: ]]
labels = ["Class 1", "Class 2"]
sns.boxenplot(data=data, palette="Set2", ax=plt.gca())
plt.xlabel(conf["xlabel"])
plt.ylabel(conf["ylabel"])
plt.title(conf["title"])
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
plt.text(0.05, 0.05, conf["textBox"], fontsize=10, verticalalignment='bottom', transform=plt.gca().transAxes, bbox=props)
plt.savefig(plotPath + "/boxen_messagesRecv.png", bbox_inches="tight")
print("Plot %s created." % (plotPath + "/boxen_messagesRecv.png"))
def plotBoxSamplesRepaired(self, result, plotPath):
"""Box Plot of the number of samples repaired by all nodes"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Samples Repaired by Nodes"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["xlabel"] = "Node Type"
conf["ylabel"] = "Number of Samples Repaired"
n1 = int(result.numberNodes * result.class1ratio)
conf["data"] = [result.repairedSampleCount[1: n1], result.repairedSampleCount[n1+1: ]]
conf["path"] = plotPath + "/box_samplesRepaired.png"
plotBoxData(conf)
print("Plot %s created." % conf["path"])
def plotBoxRowCol(self, result, plotPath):
"""Box Plot of the Row/Column distribution"""
plt.clf()
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Row/Column Distribution"
conf["xlabel"] = ""
conf["ylabel"] = "Validators Subscribed"
vector1 = result.metrics["rowDist"]
vector2 = result.metrics["columnDist"]
if len(vector1) > len(vector2):
vector2 += [np.nan] * (len(vector1) - len(vector2))
elif len(vector1) < len(vector2):
vector1 += [np.nan] * (len(vector2) - len(vector1))
n1 = int(result.numberNodes * result.class1ratio)
conf["data"] = [vector1, vector2]
conf["path"] = plotPath + "/box_rowColDist.png"
plotBoxData(conf)
print("Plot %s created." % conf["path"])
def plotRestoreRowCount(self, result, plotPath):
"""Plots the restoreRowCount for each node"""
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Restore Row Count for Each Node"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["xlabel"] = "Nodes"
conf["ylabel"] = "Restore Row Count"
conf["data"] = result.restoreRowCount
conf["xdots"] = range(result.shape.numberNodes)
conf["path"] = plotPath + "/restoreRowCount.png"
maxi = max(conf["data"])
conf["yaxismax"] = maxi
plotData(conf)
print("Plot %s created." % conf["path"])
def plotRestoreColumnCount(self, result, plotPath):
"""Plots the restoreColumnCount for each node"""
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Restore Column Count for Each Node"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["xlabel"] = "Nodes"
conf["ylabel"] = "Restore Column Count"
conf["data"] = result.restoreColumnCount
conf["xdots"] = range(result.shape.numberNodes)
conf["path"] = plotPath + "/restoreColumnCount.png"
maxi = max(conf["data"])
conf["yaxismax"] = maxi
plotData(conf)
print("Plot %s created." % conf["path"])
def plotSampleRecv(self, result, plotPath):
"""Plots the percentage sampleRecv for each node"""
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Percentage of Samples Received by Nodes"
conf["type"] = "individual_bar_with_2line"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["xlabel"] = "Nodes"
conf["ylabel"] = "Percentage of samples received (%)"
total_samples = result.shape.nbCols * result.shape.nbRows
percentage_data = [(count / total_samples) * 100 for count in result.sampleRecvCount]
conf["data"] = percentage_data
conf["xdots"] = range(result.shape.numberNodes)
conf["path"] = plotPath + "/sampleRecv.png"
maxi = max(conf["data"])
# conf["yaxismax"] = maxi * 1.1
expected_percentage1 = (result.shape.vpn1 * (result.shape.nbCols * result.shape.custodyRows + result.shape.nbRows * result.shape.custodyCols) * 100)/total_samples
expected_percentage2 = (result.shape.vpn2 * (result.shape.nbCols * result.shape.custodyRows + result.shape.nbRows * result.shape.custodyCols) * 100)/total_samples
if expected_percentage1 > 100:
expected_percentage1 = 100
if expected_percentage2 > 100:
expected_percentage2 = 100
conf["expected_value1"] = expected_percentage1
conf["expected_value2"] = expected_percentage2
conf["line_label1"] = "Expected Value for class 1 nodes"
conf["line_label2"] = "Expected Value for class 2 nodes"
conf["yaxismax"] = max(expected_percentage1, expected_percentage2) * 1.05
plotData(conf)
print("Plot %s created." % conf["path"])
def plotBoxSampleRecv(self, result, plotPath):
"""Box Plot of the sampleRecv for each node"""
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Samples Received by Nodes"
conf["type"] = "individual_bar_with_2line"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["xlabel"] = "Node Type"
conf["ylabel"] = "Number of samples received (%)"
n1 = int(result.numberNodes * result.class1ratio)
conf["data"] = [result.sampleRecvCount[1: n1], result.sampleRecvCount[n1+1: ]]
conf["xdots"] = range(result.shape.numberNodes)
conf["path"] = plotPath + "/box_sampleRecv.png"
plotBoxData(conf)
print("Plot %s created." % conf["path"])
def plotMissingSamples(self, result, plotPath):
"""Plots the missing samples in the network"""
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+" \nNetwork degree: "+attrbs['nd']
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Missing Samples"
conf["type"] = "plot"
conf["type"] = "plot_with_1line"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["colors"] = ["m-"]
@ -104,18 +710,21 @@ class Visualizor:
if max(v) > maxi:
maxi = max(v)
conf["yaxismax"] = maxi
x = result.shape.nbCols * result.shape.custodyRows + result.shape.nbRows * result.shape.custodyCols
conf["expected_value"] = (result.shape.numberNodes - 1) * (result.shape.class1ratio * result.shape.vpn1 * x + (1 - result.shape.class1ratio) * result.shape.vpn2 * x)
conf["line_label"] = "Total samples to deliver"
plotData(conf)
print("Plot %s created." % conf["path"])
def plotProgress(self, result, plotPath):
"""Plots the percentage of nodes ready in the network"""
vector1 = result.metrics["progress"]["nodes ready"]
vector2 = result.metrics["progress"]["validators ready"]
vector3 = result.metrics["progress"]["samples received"]
vector1 = [x * 100 for x in result.metrics["progress"]["nodes ready"]]
vector2 = [x * 100 for x in result.metrics["progress"]["validators ready"]]
vector3 = [x * 100 for x in result.metrics["progress"]["samples received"]]
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+" \nNetwork degree: "+attrbs['nd']
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Nodes/validators ready"
conf["type"] = "plot"
conf["legLoc"] = 2
@ -147,7 +756,7 @@ class Visualizor:
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+" \nNetwork degree: "+attrbs['nd']
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Sent data"
conf["type"] = "plot"
conf["legLoc"] = 2
@ -177,7 +786,7 @@ class Visualizor:
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+" \nNetwork degree: "+attrbs['nd']
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Received data"
conf["type"] = "plot"
conf["legLoc"] = 2
@ -207,7 +816,7 @@ class Visualizor:
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+" \nNetwork degree: "+attrbs['nd']
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Duplicated data"
conf["type"] = "plot"
conf["legLoc"] = 2
@ -238,9 +847,9 @@ class Visualizor:
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+" \nNetwork degree: "+attrbs['nd']
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Row/Column distribution"
conf["type"] = "bar"
conf["type"] = "grouped_bar"
conf["legLoc"] = 2
conf["desLoc"] = 2
conf["colors"] = ["r+", "b+"]
@ -258,3 +867,98 @@ class Visualizor:
plotData(conf)
print("Plot %s created." % conf["path"])
def plotMessagesSent(self, result, plotPath):
"""Plots the number of messages sent by all nodes"""
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Messages Sent by Nodes"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["xlabel"] = "Nodes"
conf["ylabel"] = "Number of Messages Sent"
conf["data"] = result.msgSentCount
conf["xdots"] = range(result.shape.numberNodes)
conf["path"] = plotPath + "/messagesSent.png"
maxi = max(conf["data"])
conf["yaxismax"] = maxi
plotData(conf)
print("Plot %s created." % conf["path"])
def plotBoxMessagesSent(self, result, plotPath):
"""Box Plot of the number of messages sent by all nodes"""
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Messages Sent by Nodes"
conf["xlabel"] = "Node Type"
conf["ylabel"] = "Number of Messages Sent"
n1 = int(result.numberNodes * result.class1ratio)
conf["data"] = [result.msgSentCount[1: n1], result.msgSentCount[n1+1: ]]
conf["path"] = plotPath + "/box_messagesSent.png"
plotBoxData(conf)
print("Plot %s created." % conf["path"])
def plotMessagesRecv(self, result, plotPath):
"""Plots the number of messages received by all nodes"""
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Messages Received by Nodes"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["xlabel"] = "Nodes"
conf["ylabel"] = "Number of Messages Received"
conf["data"] = result.msgRecvCount
conf["xdots"] = range(result.shape.numberNodes)
conf["path"] = plotPath + "/messagesRecv.png"
maxi = max(conf["data"])
conf["yaxismax"] = maxi
plotData(conf)
print("Plot %s created." % conf["path"])
def plotBoxMessagesRecv(self, result, plotPath):
"""Plots the number of messages received by all nodes"""
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Messages Received by Nodes"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["xlabel"] = "Node Type"
conf["ylabel"] = "Number of Messages Received"
n1 = int(result.numberNodes * result.class1ratio)
conf["data"] = [result.msgRecvCount[1: n1], result.msgRecvCount[n1+1: ]]
conf["xdots"] = range(result.shape.numberNodes)
conf["path"] = plotPath + "/box_messagesRecv.png"
maxi = max(conf["data"])
conf["yaxismax"] = maxi
plotBoxData(conf)
print("Plot %s created." % conf["path"])
def plotSamplesRepaired(self, result, plotPath):
"""Plots the number of samples repaired by all nodes"""
conf = {}
attrbs = self.__get_attrbs__(result)
conf["textBox"] = "Block Size R: "+attrbs['bsrn']+"\nBlock Size C: "+attrbs['bscn']\
+"\nNumber of nodes: "+attrbs['nn']+"\nFailure rate: "+attrbs['fr']+"\nMalicious Node: "+attrbs['mn']+"\nNetwork degree: "+attrbs['nd']
conf["title"] = "Number of Samples Repaired by Nodes"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["xlabel"] = "Nodes"
conf["ylabel"] = "Number of Samples Repaired"
conf["data"] = result.repairedSampleCount
conf["xdots"] = range(result.shape.numberNodes)
conf["path"] = plotPath + "/repairedSampleCount.png"
maxi = max(conf["data"])
conf["yaxismax"] = maxi
plotData(conf)
print("Plot %s created." % conf["path"])

43
smallConf.py
View File

@ -40,10 +40,6 @@ logLevel = logging.INFO
# for more details, see joblib.Parallel
numJobs = -1
# distribute rows/columns evenly between validators (True)
# or generate it using local randomness (False)
evenLineDistribution = True
# Number of simulation runs with the same parameters for statistical relevance
runs = range(3)
@ -56,8 +52,12 @@ failureModels = ["random"]
# Percentage of block not released by producer
failureRates = range(40, 81, 20)
# Block size in one dimension in segments. Block is blockSizes * blockSizes segments.
blockSizes = range(64, 113, 128)
# Percentage of nodes that are considered malicious
maliciousNodes = range(40,41,20)
# Parameter to determine whether to randomly assign malicious nodes or not
# If True, the malicious nodes will be assigned randomly; if False, a predefined pattern may be used
randomizeMaliciousNodes = True
# Per-topic mesh neighborhood size
netDegrees = range(8, 9, 2)
@ -67,15 +67,22 @@ netDegrees = range(8, 9, 2)
proposerPublishToR = "shape.netDegree"
proposerPublishToC = "shape.netDegree"
# number of rows and columns a validator is interested in
chis = range(2, 3, 2)
# the overall number of row/columns taken into custody by a node is determined by
# a base number (custody) and a class specific multiplier (validatorsPerNode).
# We support two models:
# - validatorsBasedCustody: each validator has a unique subset of size custody,
# and custody is the union of these. I.e. VPN is a "probabilistic multiplier"
# - !validatorsBasedCustody: VPN is interpreted as a simple custody multiplier
validatorBasedCustody = False
custodyRows = range(2, 3, 2)
custodyCols = range(2, 3, 2)
# ratio of class1 nodes (see below for parameters per class)
class1ratios = [0.8]
# Number of validators per beacon node
validatorsPerNode1 = [1]
validatorsPerNode2 = [500]
validatorsPerNode2 = [5]
# Set uplink bandwidth in megabits/second
bwUplinksProd = [200]
@ -108,8 +115,8 @@ saveGit = False
# configure Node options
repairOnTheFly = True
sendLineUntilR = "shape.blockSizeRK" # stop sending on a p2p link if at least this amount of samples passed
sendLineUntilC = lambda shape : shape.blockSizeCK # stop sending on a p2p link if at least this amount of samples passed
sendLineUntilR = "shape.nbColsK" # stop sending on a p2p link if at least this amount of samples passed
sendLineUntilC = lambda shape : shape.nbRowsK # stop sending on a p2p link if at least this amount of samples passed
perNeighborQueue = True # queue incoming messages to outgoing connections on arrival (as typical GossipSub impl)
shuffleQueues = True # shuffle the order of picking from active queues of a sender node
perNodeQueue = False # keep a global queue of incoming messages for later sequential dispatch
@ -122,15 +129,17 @@ queueAllOnInit = False # queue up everything in the block producer, without shuf
forwardOnReceive = True # forward segments as soon as received
forwardOnRepair = False # forward all segments when full line available (repaired segments are always forwarded)
cols = range(64, 113, 128)
rows = range(32, 113, 128)
colsK = range(32, 65, 128)
rowsK = range(32, 65, 128)
def nextShape():
for run, fm, fr, class1ratio, chi, vpn1, vpn2, blockSize, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product(
runs, failureModels, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2):
for nbCols, nbColsK, nbRows, nbRowsK, run, fm, fr, mn, class1ratio, chR, chC, vpn1, vpn2, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product(
cols, colsK, rows, rowsK, runs, failureModels, failureRates, maliciousNodes, class1ratios, custodyRows, custodyCols, validatorsPerNode1, validatorsPerNode2, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2):
# Network Degree has to be an even number
if netDegree % 2 == 0:
blockSizeR = blockSizeC = blockSize
blockSizeRK = blockSizeCK = blockSize // 2
chiR = chiC = chi
shape = Shape(blockSizeR, blockSizeRK, blockSizeC, blockSizeCK, nn, fm, fr, class1ratio, chiR, chiC, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run)
shape = Shape(nbCols, nbColsK, nbRows, nbRowsK, nn, fm, fr, mn, class1ratio, chR, chC, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run)
yield shape
def evalConf(self, param, shape = None):