codex-storage
/
das-research
mirror of https://github.com/codex-storage/das-research.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Independent dimensions 

Signed-off-by: Arunima Chaudhuri <arunimachaudhuri2020@gmail.com>
This commit is contained in:
Arunima Chaudhuri 2024-02-15 23:48:30 +05:30
parent 7ed3d8c690
commit 7875023ef0
9 changed files with 176 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,7 +1,7 @@
*.swp
*.pyc
results/*
myenv
myenv*/
doc/_build
!results/plots.py
Frontend/

57
DAS/block.py
View File

@ -7,10 +7,18 @@ from bitarray.util import zeros
class Block:
"""This class represents a block in the Ethereum blockchain."""
def __init__(self, blockSize):
"""Initialize the block with a data array of blocksize^2 zeros."""
self.blockSize = blockSize
self.data = zeros(self.blockSize*self.blockSize)
def __init__(self, blockSizeR, blockSizeRK=0, blockSizeC=0, blockSizeCK=0):
"""Initialize the block with a data array of blocksize^2 zeros.
BlockSizeR: row size
BlockSizeRK: original row size, before erasure coding to BlocksSizeR
BlockSizeC: column size (i.e. number of rows)
BlockSizeCK: original column size, before erasure coding to BlocksSizeR
"""
self.blockSizeR = blockSizeR
self.blockSizeRK = blockSizeRK if blockSizeRK else blockSizeR/2
self.blockSizeC = blockSizeC if blockSizeC else blockSizeR
self.blockSizeCK = blockSizeCK if blockSizeCK else blockSizeRK
self.data = zeros(self.blockSizeR*self.blockSizeC)
def fill(self):
"""It fills the block data with ones."""
@ -22,62 +30,61 @@ class Block:
def getSegment(self, rowID, columnID):
"""Check whether a segment is included"""
return self.data[rowID*self.blockSize + columnID]
return self.data[rowID*self.blockSizeR + columnID]
def setSegment(self, rowID, columnID, value = 1):
"""Set value for a segment (default 1)"""
self.data[rowID*self.blockSize + columnID] = value
self.data[rowID*self.blockSizeR + columnID] = value
def getColumn(self, columnID):
"""It returns the block column corresponding to columnID."""
return self.data[columnID::self.blockSize]
return self.data[columnID::self.blockSizeR]
def mergeColumn(self, columnID, column):
"""It merges (OR) the existing column with the received one."""
self.data[columnID::self.blockSize] |= column
self.data[columnID::self.blockSizeR] |= column
def repairColumn(self, id):
"""It repairs the entire column if it has at least blockSize/2 ones.
"""It repairs the entire column if it has at least blockSizeCK ones.
Returns: list of repaired segments
"""
line = self.data[id::self.blockSize]
line = self.data[id::self.blockSizeR]
success = line.count(1)
if success >= self.blockSize/2:
if success >= self.blockSizeCK:
ret = ~line
self.data[id::self.blockSize] = 1
self.data[id::self.blockSizeR] = 1
else:
ret = zeros(self.blockSize)
ret = zeros(self.blockSizeC)
return ret
def getRow(self, rowID):
"""It returns the block row corresponding to rowID."""
return self.data[rowID*self.blockSize:(rowID+1)*self.blockSize]
return self.data[rowID*self.blockSizeR:(rowID+1)*self.blockSizeR]
def mergeRow(self, rowID, row):
"""It merges (OR) the existing row with the received one."""
self.data[rowID*self.blockSize:(rowID+1)*self.blockSize] |= row
self.data[rowID*self.blockSizeR:(rowID+1)*self.blockSizeR] |= row
def repairRow(self, id):
"""It repairs the entire row if it has at least blockSize/2 ones.
"""It repairs the entire row if it has at least blockSizeRK ones.
Returns: list of repaired segments.
"""
line = self.data[id*self.blockSize:(id+1)*self.blockSize]
line = self.data[id*self.blockSizeR:(id+1)*self.blockSizeR]
success = line.count(1)
if success >= self.blockSize/2:
if success >= self.blockSizeRK:
ret = ~line
self.data[id*self.blockSize:(id+1)*self.blockSize] = 1
self.data[id*self.blockSizeR:(id+1)*self.blockSizeR] = 1
else:
ret = zeros(self.blockSize)
ret = zeros(self.blockSizeR)
return ret
def print(self):
"""It prints the block in the terminal (outside of the logger rules))."""
dash = "-" * (self.blockSize+2)
dash = "-" * (self.blockSizeR+2)
print(dash)
for i in range(self.blockSize):
for i in range(self.blockSizeC):
line = "|"
for j in range(self.blockSize):
line += "%i" % self.data[(i*self.blockSize)+j]
for j in range(self.blockSizeR):
line += "%i" % self.data[(i*self.blockSizeR)+j]
print(line+"|")
print(dash)

15
DAS/observer.py
View File

@ -11,10 +11,11 @@ class Observer:
self.config = config
self.format = {"entity": "Observer"}
self.logger = logger
self.block = [0] * self.config.blockSize * self.config.blockSize
self.rows = [0] * self.config.blockSize
self.columns = [0] * self.config.blockSize
self.broadcasted = Block(self.config.blockSize)
self.block = [0] * self.config.blockSizeR * self.config.blockSizeC
self.rows = [0] * self.config.blockSizeC
self.columns = [0] * self.config.blockSizeR
self.broadcasted = Block(self.config.blockSizeR, self.config.blockSizeRK,
self.config.blockSizeC, self.config.blockSizeCK)
def checkRowsColumns(self, validators):
@ -26,7 +27,7 @@ class Observer:
for c in val.columnIDs:
self.columns[c] += 1
for i in range(self.config.blockSize):
for i in range(self.config.blockSizeC):
self.logger.debug("Row/Column %d have %d and %d validators assigned." % (i, self.rows[i], self.columns[i]), extra=self.format)
if self.rows[i] == 0 or self.columns[i] == 0:
self.logger.warning("There is a row/column that has not been assigned", extra=self.format)
@ -34,7 +35,7 @@ class Observer:
def checkBroadcasted(self):
"""It checks how many broadcasted samples are still missing in the network."""
zeros = 0
for i in range(self.blockSize * self.blockSize):
for i in range(self.blockSizeR * self.blockSizeC):
if self.broadcasted.data[i] == 0:
zeros += 1
if zeros > 0:
@ -99,4 +100,4 @@ class Observer:
"RxDup": {"mean": meanOrNan(RxDup), "max": maxOrNan(RxDup)},
}
return trafficStats
return trafficStats

20
DAS/shape.py
View File

@ -3,17 +3,22 @@
class Shape:
"""This class represents a set of parameters for a specific simulation."""
def __init__(self, blockSize, numberNodes, failureModel, failureRate, maliciousNodes, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run):
def __init__(self, blockSizeR, blockSizeRK, blockSizeC, blockSizeCK,
numberNodes, failureModel, failureRate, maliciousNodes, class1ratio, chiR, chiC, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run):
"""Initializes the shape with the parameters passed in argument."""
self.run = run
self.numberNodes = numberNodes
self.blockSize = blockSize
self.blockSizeR = blockSizeR
self.blockSizeRK = blockSizeRK
self.blockSizeC = blockSizeC
self.blockSizeCK = blockSizeCK
self.failureModel = failureModel
self.failureRate = failureRate
self.maliciousNodes = maliciousNodes
self.netDegree = netDegree
self.class1ratio = class1ratio
self.chi = chi
self.chiR = chiR
self.chiC = chiC
self.vpn1 = vpn1
self.vpn2 = vpn2
self.bwUplinkProd = bwUplinkProd
@ -24,12 +29,16 @@ class Shape:
def __repr__(self):
"""Returns a printable representation of the shape"""
shastr = ""
shastr += "bs-"+str(self.blockSize)
shastr += "-bsrn-"+str(self.blockSizeR)
shastr += "-bsrk-"+str(self.blockSizeRK)
shastr += "-bscn-"+str(self.blockSizeC)
shastr += "-bsck-"+str(self.blockSizeCK)
shastr += "-nn-"+str(self.numberNodes)
shastr += "-fm-"+str(self.failureModel)
shastr += "-fr-"+str(self.failureRate)
shastr += "-c1r-"+str(self.class1ratio)
shastr += "-chi-"+str(self.chi)
shastr += "-chir-"+str(self.chiR)
shastr += "-chic-"+str(self.chiC)
shastr += "-vpn1-"+str(self.vpn1)
shastr += "-vpn2-"+str(self.vpn2)
shastr += "-bwupprod-"+str(self.bwUplinkProd)
@ -43,4 +52,3 @@ class Shape:
def setSeed(self, seed):
"""Adds the random seed to the shape"""
self.randomSeed = seed

52
DAS/simulator.py
View File

@ -41,7 +41,7 @@ class Simulator:
# pushed out by the proposer.
# 1: the data is sent out exactly once on rows and once on columns (2 copies in total)
# self.shape.netDegree: default behavior similar (but not same) to previous code
self.proposerPublishTo = self.shape.netDegree
self.proposerPublishTo = self.shape.netDegree # TODO: make this an external parameter
def initValidators(self):
"""It initializes all the validators in the network."""
@ -54,17 +54,20 @@ class Simulator:
lightVal = lightNodes * self.shape.vpn1
heavyVal = heavyNodes * self.shape.vpn2
totalValidators = lightVal + heavyVal
totalRows = totalValidators * self.shape.chi
rows = list(range(self.shape.blockSize)) * (int(totalRows/self.shape.blockSize)+1)
columns = list(range(self.shape.blockSize)) * (int(totalRows/self.shape.blockSize)+1)
totalRows = totalValidators * self.shape.chiR
totalColumns = totalValidators * self.shape.chiC
rows = list(range(self.shape.blockSizeC)) * (int(totalRows/self.shape.blockSizeC)+1)
columns = list(range(self.shape.blockSizeR)) * (int(totalColumns/self.shape.blockSizeR)+1)
rows = rows[0:totalRows]
columns = columns[0:totalRows]
random.shuffle(rows)
random.shuffle(columns)
offset = lightVal*self.shape.chi
offsetR = lightVal*self.shape.chiR
offsetC = lightVal*self.shape.chiC
self.logger.debug("There is a total of %d nodes, %d light and %d heavy." % (self.shape.numberNodes, lightNodes, heavyNodes), extra=self.format)
self.logger.debug("There is a total of %d validators, %d in light nodes and %d in heavy nodes" % (totalValidators, lightVal, heavyVal), extra=self.format)
self.logger.debug("Shuffling a total of %d rows/columns to be assigned (X=%d)" % (len(rows), self.shape.chi), extra=self.format)
self.logger.debug("Shuffling a total of %d rows to be assigned (X=%d)" % (len(rows), self.shape.chiR), extra=self.format)
self.logger.debug("Shuffling a total of %d columns to be assigned (X=%d)" % (len(columns), self.shape.chiC), extra=self.format)
self.logger.debug("Shuffled rows: %s" % str(rows), extra=self.format)
self.logger.debug("Shuffled columns: %s" % str(columns), extra=self.format)
@ -93,14 +96,18 @@ class Simulator:
if self.config.evenLineDistribution:
if i < int(lightVal/self.shape.vpn1): # First start with the light nodes
start = i *self.shape.chi*self.shape.vpn1
end = (i+1)*self.shape.chi*self.shape.vpn1
startR = i *self.shape.chiR*self.shape.vpn1
endR = (i+1)*self.shape.chiR*self.shape.vpn1
startC = i *self.shape.chiC*self.shape.vpn1
endC = (i+1)*self.shape.chiC*self.shape.vpn1
else:
j = i - int(lightVal/self.shape.vpn1)
start = offset+( j *self.shape.chi*self.shape.vpn2)
end = offset+((j+1)*self.shape.chi*self.shape.vpn2)
r = rows[start:end]
c = columns[start:end]
startR = offsetR+( j *self.shape.chiR*self.shape.vpn2)
endR = offsetR+((j+1)*self.shape.chiR*self.shape.vpn2)
startC = offsetC+( j *self.shape.chiC*self.shape.vpn2)
endC = offsetC+((j+1)*self.shape.chiC*self.shape.vpn2)
r = rows[startR:endR]
c = columns[startC:endC]
val = Validator(i, int(not i!=0), amImalicious_value, self.logger, self.shape, self.config, r, c)
self.logger.debug("Node %d has row IDs: %s" % (val.ID, val.rowIDs), extra=self.format)
self.logger.debug("Node %d has column IDs: %s" % (val.ID, val.columnIDs), extra=self.format)
@ -125,8 +132,8 @@ class Simulator:
def initNetwork(self):
"""It initializes the simulated network."""
rowChannels = [[] for i in range(self.shape.blockSize)]
columnChannels = [[] for i in range(self.shape.blockSize)]
rowChannels = [[] for i in range(self.shape.blockSizeC)]
columnChannels = [[] for i in range(self.shape.blockSizeR)]
for v in self.validators:
if not (self.proposerPublishOnly and v.amIproposer):
for id in v.rowIDs:
@ -142,7 +149,7 @@ class Simulator:
self.logger.debug("Number of validators per row; Min: %d, Max: %d" % (min(self.distR), max(self.distR)), extra=self.format)
self.logger.debug("Number of validators per column; Min: %d, Max: %d" % (min(self.distC), max(self.distC)), extra=self.format)
for id in range(self.shape.blockSize):
for id in range(self.shape.blockSizeC):
# If the number of nodes in a channel is smaller or equal to the
# requested degree, a fully connected graph is used. For n>d, a random
@ -160,9 +167,11 @@ class Simulator:
for u, v in G.edges:
val1=rowChannels[id][u]
val2=rowChannels[id][v]
val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, 0, self.shape.blockSize)})
val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, 0, self.shape.blockSize)})
val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, 0, self.shape.blockSizeR)})
val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, 0, self.shape.blockSizeR)})
for id in range(self.shape.blockSizeR):
if not columnChannels[id]:
self.logger.error("No nodes for column %d !" % id, extra=self.format)
continue
@ -176,8 +185,8 @@ class Simulator:
for u, v in G.edges:
val1=columnChannels[id][u]
val2=columnChannels[id][v]
val1.columnNeighbors[id].update({val2.ID : Neighbor(val2, 1, self.shape.blockSize)})
val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, 1, self.shape.blockSize)})
val1.columnNeighbors[id].update({val2.ID : Neighbor(val2, 1, self.shape.blockSizeC)})
val2.columnNeighbors[id].update({val1.ID : Neighbor(val1, 1, self.shape.blockSizeC)})
for v in self.validators:
if (self.proposerPublishOnly and v.amIproposer):
@ -185,12 +194,12 @@ class Simulator:
count = min(self.proposerPublishTo, len(rowChannels[id]))
publishTo = random.sample(rowChannels[id], count)
for vi in publishTo:
v.rowNeighbors[id].update({vi.ID : Neighbor(vi, 0, self.shape.blockSize)})
v.rowNeighbors[id].update({vi.ID : Neighbor(vi, 0, self.shape.blockSizeR)})
for id in v.columnIDs:
count = min(self.proposerPublishTo, len(columnChannels[id]))
publishTo = random.sample(columnChannels[id], count)
for vi in publishTo:
v.columnNeighbors[id].update({vi.ID : Neighbor(vi, 1, self.shape.blockSize)})
v.columnNeighbors[id].update({vi.ID : Neighbor(vi, 1, self.shape.blockSizeC)})
if self.logger.isEnabledFor(logging.DEBUG):
for i in range(0, self.shape.numberNodes):
@ -349,4 +358,3 @@ class Simulator:
self.result.populate(self.shape, self.config, missingVector)
self.result.copyValidators(self.validators)
return self.result

91
DAS/validator.py
View File

@ -42,15 +42,15 @@ class Validator:
"""It initializes the validator with the logger shape and rows/columns.
If rows/columns are specified these are observed, otherwise (default)
chi rows and columns are selected randomly.
chiR rows and chiC columns are selected randomly.
"""
self.shape = shape
FORMAT = "%(levelname)s : %(entity)s : %(message)s"
self.ID = ID
self.format = {"entity": "Val "+str(self.ID)}
self.block = Block(self.shape.blockSize)
self.receivedBlock = Block(self.shape.blockSize)
self.block = Block(self.shape.blockSizeR, self.shape.blockSizeRK, self.shape.blockSizeC, self.shape.blockSizeCK)
self.receivedBlock = Block(self.shape.blockSizeR, self.shape.blockSizeRK, self.shape.blockSizeC, self.shape.blockSizeCK)
self.receivedQueue = deque()
self.sendQueue = deque()
self.amIproposer = amIproposer
@ -63,15 +63,17 @@ class Validator:
self.restoreRowCount = 0
self.restoreColumnCount = 0
self.logger = logger
if self.shape.chi < 1:
if self.shape.chiR < 1 and self.shape.chiC < 1:
self.logger.error("Chi has to be greater than 0", extra=self.format)
elif self.shape.chi > self.shape.blockSize:
self.logger.error("Chi has to be smaller than %d" % self.shape.blockSize, extra=self.format)
elif self.shape.chiC > self.shape.blockSizeR:
self.logger.error("ChiC has to be smaller than %d" % self.shape.blockSizeR, extra=self.format)
elif self.shape.chiR > self.shape.blockSizeC:
self.logger.error("ChiR has to be smaller than %d" % self.shape.blockSizeC, extra=self.format)
else:
if amIproposer:
self.nodeClass = 0
self.rowIDs = range(shape.blockSize)
self.columnIDs = range(shape.blockSize)
self.rowIDs = range(shape.blockSizeC)
self.columnIDs = range(shape.blockSizeR)
else:
#if shape.deterministic:
# random.seed(self.ID)
@ -80,8 +82,8 @@ class Validator:
self.vRowIDs = []
self.vColumnIDs = []
for i in range(self.vpn):
self.vRowIDs.append(set(rows[i*self.shape.chi:(i+1)*self.shape.chi]) if rows else set(random.sample(range(self.shape.blockSize), self.shape.chi)))
self.vColumnIDs.append(set(columns[i*self.shape.chi:(i+1)*self.shape.chi]) if columns else set(random.sample(range(self.shape.blockSize), self.shape.chi)))
self.vRowIDs.append(set(rows[i*self.shape.chiR:(i+1)*self.shape.chiR]) if rows else set(random.sample(range(self.shape.blockSizeC), self.shape.chiR)))
self.vColumnIDs.append(set(columns[i*self.shape.chiC:(i+1)*self.shape.chiC]) if columns else set(random.sample(range(self.shape.blockSizeR), self.shape.chiC)))
self.rowIDs = set.union(*self.vRowIDs)
self.columnIDs = set.union(*self.vColumnIDs)
self.rowNeighbors = collections.defaultdict(dict)
@ -107,7 +109,8 @@ class Validator:
self.bwUplink *= 1e3 / 8 * config.stepDuration / config.segmentSize
self.repairOnTheFly = True
self.sendLineUntil = (self.shape.blockSize + 1) // 2 # stop sending on a p2p link if at least this amount of samples passed
self.sendLineUntilR = self.shape.blockSizeRK # stop sending on a p2p link if at least this amount of samples passed
self.sendLineUntilC = self.shape.blockSizeCK # stop sending on a p2p link if at least this amount of samples passed
self.perNeighborQueue = True # queue incoming messages to outgoing connections on arrival (as typical GossipSub impl)
self.shuffleQueues = True # shuffle the order of picking from active queues of a sender node
self.perNodeQueue = False # keep a global queue of incoming messages for later sequential dispatch
@ -132,57 +135,53 @@ class Validator:
else:
self.logger.debug("Creating block...", extra=self.format)
if self.shape.failureModel == "random":
order = [i for i in range(self.shape.blockSize * self.shape.blockSize)]
order = [i for i in range(self.shape.blockSizeR * self.shape.blockSizeC)]
order = random.sample(order, int((1 - self.shape.failureRate/100) * len(order)))
for i in order:
self.block.data[i] = 1
elif self.shape.failureModel == "sequential":
order = [i for i in range(self.shape.blockSize * self.shape.blockSize)]
order = [i for i in range(self.shape.blockSizeR * self.shape.blockSizeC)]
order = order[:int((1 - self.shape.failureRate/100) * len(order))]
for i in order:
self.block.data[i] = 1
elif self.shape.failureModel == "MEP": # Minimal size non-recoverable Erasure Pattern
for r in range(self.shape.blockSize):
for c in range(self.shape.blockSize):
k = self.shape.blockSize/2
if r > k or c > k:
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if r > self.shape.blockSizeRK or c > self.shape.blockSizeCK:
self.block.setSegment(r,c)
elif self.shape.failureModel == "MEP+1": # MEP +1 segment to make it recoverable
for r in range(self.shape.blockSize):
for c in range(self.shape.blockSize):
k = self.shape.blockSize/2
if r > k or c > k:
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if r > self.shape.blockSizeRK or c > self.shape.blockSizeCK:
self.block.setSegment(r,c)
self.block.setSegment(0, 0)
elif self.shape.failureModel == "DEP":
for r in range(self.shape.blockSize):
for c in range(self.shape.blockSize):
k = self.shape.blockSize/2
if (r+c) % self.shape.blockSize > k:
assert(self.shape.blockSizeR == self.shape.blockSizeC and self.shape.blockSizeRK == self.shape.blockSizeCK)
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if (r+c) % self.shape.blockSizeR > self.shape.blockSizeRK:
self.block.setSegment(r,c)
elif self.shape.failureModel == "DEP+1":
for r in range(self.shape.blockSize):
for c in range(self.shape.blockSize):
k = self.shape.blockSize/2
if (r+c) % self.shape.blockSize > k:
assert(self.shape.blockSizeR == self.shape.blockSizeC and self.shape.blockSizeRK == self.shape.blockSizeCK)
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if (r+c) % self.shape.blockSizeR > self.shape.blockSizeRK:
self.block.setSegment(r,c)
self.block.setSegment(0, 0)
elif self.shape.failureModel == "MREP": # Minimum size Recoverable Erasure Pattern
for r in range(self.shape.blockSize):
for c in range(self.shape.blockSize):
k = self.shape.blockSize/2
if r < k and c < k:
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if r < self.shape.blockSizeRK or c < self.shape.blockSizeCK:
self.block.setSegment(r,c)
elif self.shape.failureModel == "MREP-1": # make MREP non-recoverable
for r in range(self.shape.blockSize):
for c in range(self.shape.blockSize):
k = self.shape.blockSize/2
if r < k and c < k:
for r in range(self.shape.blockSizeR):
for c in range(self.shape.blockSizeC):
if r < self.shape.blockSizeRK or c < self.shape.blockSizeCK:
self.block.setSegment(r,c)
self.block.setSegment(0, 0, 0)
nbFailures = self.block.data.count(0)
measuredFailureRate = nbFailures * 100 / (self.shape.blockSize * self.shape.blockSize)
measuredFailureRate = nbFailures * 100 / (self.shape.blockSizeR * self.shape.blockSizeC)
self.logger.debug("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format)
def getColumn(self, index):
@ -271,7 +270,7 @@ class Validator:
def checkSegmentToNeigh(self, rID, cID, neigh):
"""Check if a segment should be sent to a neighbor."""
if not self.amImalicious:
if (neigh.sent | neigh.received).count(1) >= self.sendLineUntil:
if (neigh.sent | neigh.received).count(1) >= (self.sendLineUntilC if neigh.dim else self.sendLineUntilR):
return False # sent enough, other side can restore
i = rID if neigh.dim else cID
if not neigh.sent[i] and not neigh.received[i] :
@ -373,10 +372,10 @@ class Validator:
if not self.amImalicious:
for rID, neighs in self.rowNeighbors.items():
line = self.getRow(rID)
needed = zeros(self.shape.blockSize)
needed = zeros(self.shape.blockSizeR)
for neigh in neighs.values():
sentOrReceived = neigh.received | neigh.sent
if sentOrReceived.count(1) < self.sendLineUntil:
if sentOrReceived.count(1) < self.sendLineUntilR:
needed |= ~sentOrReceived
needed &= line
if (needed).any():
@ -386,10 +385,10 @@ class Validator:
for cID, neighs in self.columnNeighbors.items():
line = self.getColumn(cID)
needed = zeros(self.shape.blockSize)
needed = zeros(self.shape.blockSizeC)
for neigh in neighs.values():
sentOrReceived = neigh.received | neigh.sent
if sentOrReceived.count(1) < self.sendLineUntil:
if sentOrReceived.count(1) < self.sendLineUntilC:
needed |= ~sentOrReceived
needed &= line
if (needed).any():
@ -449,7 +448,7 @@ class Validator:
while t:
if self.rowIDs:
rID = random.choice(self.rowIDs)
cID = random.randrange(0, self.shape.blockSize)
cID = random.randrange(0, self.shape.blockSizeR)
if self.block.getSegment(rID, cID) :
neigh = random.choice(list(self.rowNeighbors[rID].values()))
if self.checkSegmentToNeigh(rID, cID, neigh) and not self.amImalicious:
@ -457,7 +456,7 @@ class Validator:
t = tries
if self.columnIDs:
cID = random.choice(self.columnIDs)
rID = random.randrange(0, self.shape.blockSize)
rID = random.randrange(0, self.shape.blockSizeC)
if self.block.getSegment(rID, cID) :
neigh = random.choice(list(self.columnNeighbors[cID].values()))
if self.checkSegmentToNeigh(rID, cID, neigh) and not self.amImalicious:
@ -576,4 +575,4 @@ class Validator:
if a == e:
validated+=1
return arrived, expected, validated
return arrived, expected, validated

6
DAS/visualizer.py
View File

@ -33,12 +33,12 @@ class Visualizer:
tree = ET.parse(os.path.join(self.folderPath, filename))
root = tree.getroot()
run = int(root.find('run').text)
blockSize = int(root.find('blockSize').text)
blockSize = int(root.find('blockSizeR').text) # TODO: maybe we want both dimensions
failureRate = int(root.find('failureRate').text)
numberNodes = int(root.find('numberNodes').text)
class1ratio = float(root.find('class1ratio').text)
netDegree = int(root.find('netDegree').text)
chi = int(root.find('chi').text)
chi = int(root.find('chiR').text) # TODO: maybe we want both dimensions
vpn1 = int(root.find('vpn1').text)
vpn2 = int(root.find('vpn2').text)
bwUplinkProd = int(root.find('bwUplinkProd').text)
@ -274,4 +274,4 @@ class Visualizer:
plt.ylabel('Price')
#Test
plt.show()
plt.show()

54
DAS/visualizor.py
View File

@ -1,6 +1,7 @@
#!/bin/python3
import matplotlib.pyplot as plt
import numpy as np
import os
def plotData(conf):
@ -87,8 +88,8 @@ class Visualizor:
"""Plots the restoreRowCount for each node"""
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+"\nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Restore Row Count for Each Node"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
@ -107,8 +108,8 @@ class Visualizor:
"""Plots the restoreColumnCount for each node"""
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+"\nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Restore Column Count for Each Node"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
@ -127,15 +128,15 @@ class Visualizor:
"""Plots the percentage sampleRecv for each node"""
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+"\nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Percentage of Samples Received by Nodes"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
conf["desLoc"] = 1
conf["xlabel"] = "Nodes"
conf["ylabel"] = "Percentage of samples received (%)"
total_samples = result.shape.blockSize * result.shape.blockSize
total_samples = result.shape.blockSizeR * result.shape.blockSizeC
percentage_data = [(count / total_samples) * 100 for count in result.sampleRecvCount]
conf["data"] = percentage_data
conf["xdots"] = range(result.shape.numberNodes)
@ -149,8 +150,8 @@ class Visualizor:
"""Plots the missing samples in the network"""
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+"\nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Missing Samples"
conf["type"] = "plot"
conf["legLoc"] = 1
@ -177,8 +178,8 @@ class Visualizor:
vector3 = [x * 100 for x in result.metrics["progress"]["samples received"]]
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+"\nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Nodes/validators ready"
conf["type"] = "plot"
conf["legLoc"] = 2
@ -209,8 +210,8 @@ class Visualizor:
vector3[i] = (vector3[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+"\nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Sent data"
conf["type"] = "plot"
conf["legLoc"] = 2
@ -239,8 +240,8 @@ class Visualizor:
vector2[i] = (vector2[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+" \nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Received data"
conf["type"] = "plot"
conf["legLoc"] = 2
@ -269,8 +270,8 @@ class Visualizor:
vector2[i] = (vector2[i] * 8 * (1000/self.config.stepDuration) * self.config.segmentSize) / 1000000
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+" \nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Duplicated data"
conf["type"] = "plot"
conf["legLoc"] = 2
@ -294,10 +295,14 @@ class Visualizor:
"""Plots the percentage of nodes ready in the network"""
vector1 = result.metrics["rowDist"]
vector2 = result.metrics["columnDist"]
if len(vector1) > len(vector2):
vector2 += [np.nan] * (len(vector1) - len(vector2))
elif len(vector1) < len(vector2):
vector1 += [np.nan] * (len(vector2) - len(vector1))
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+" \nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Row/Column distribution"
conf["type"] = "grouped_bar"
conf["legLoc"] = 2
@ -311,7 +316,7 @@ class Visualizor:
conf["path"] = plotPath+"/RowColDist.png"
maxi = 0
for v in conf["data"]:
if max(v) > maxi:
if np.nanmax(v) > maxi:
maxi = max(v)
conf["yaxismax"] = maxi
plotData(conf)
@ -321,8 +326,8 @@ class Visualizor:
"""Plots the number of messages sent by all nodes"""
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+" \nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Number of Messages Sent by Nodes"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
@ -341,8 +346,8 @@ class Visualizor:
"""Plots the number of messages received by all nodes"""
conf = {}
text = str(result.shape).split("-")
conf["textBox"] = "Block Size: "+text[1]+"\nNumber of nodes: "+text[3]\
+"\nFailure rate: "+text[7]+" \nNetwork degree: "+text[23]+"\nX: "+text[11]+" rows/columns"+"\nMalicious Nodes: "+text[27]+"%"
conf["textBox"] = "Block SizeR: "+text[2]+"\nBlock SizeC: "+text[6]+"\nNumber of nodes: "+text[10]\
+"\nFailure rate: "+text[14]+"%"+"\nNetwork degree: "+text[32]+"\nMalicious Nodes: "+text[36]+"%"
conf["title"] = "Number of Messages Received by Nodes"
conf["type"] = "individual_bar"
conf["legLoc"] = 1
@ -356,4 +361,3 @@ class Visualizor:
conf["yaxismax"] = maxi
plotData(conf)
print("Plot %s created." % conf["path"])

27
smallConf.py
View File

@ -57,27 +57,21 @@ failureModels = ["random"]
failureRates = [0]
# Percentage of nodes that are considered malicious
maliciousNodes = [95]
maliciousNodes = [0]
# Parameter to determine whether to randomly assign malicious nodes or not
# If True, the malicious nodes will be assigned randomly; if False, a predefined pattern may be used
randomizeMaliciousNodes = True
# Block size in one dimension in segments. Block is blockSizes * blockSizes segments.
blockSizes = [128]
# Per-topic mesh neighborhood size
netDegrees = range(8, 9, 2)
# number of rows and columns a validator is interested in
chis = range(2, 3, 2)
# ratio of class1 nodes (see below for parameters per class)
class1ratios = [0.8]
# Number of validators per beacon node
validatorsPerNode1 = [10]
validatorsPerNode2 = [250]
validatorsPerNode1 = [1]
validatorsPerNode2 = [1]
# Set uplink bandwidth in megabits/second
bwUplinksProd = [200]
@ -108,10 +102,17 @@ diagnostics = False
# True to save git diff and git commit
saveGit = False
blockSizeR =[128]
blockSizeC = [64]
blockSizeRK = [64]
blockSizeCK = [64]
chiR = [2]
chiC = [2]
def nextShape():
for run, fm, fr, mn, class1ratio, chi, vpn1, vpn2, blockSize, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product(
runs, failureModels, failureRates, maliciousNodes, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2):
for blckSizeR, blckSizeRK, blckSizeC, blckSizeCK, run, fm, fr, mn, class1ratio, chR, chC, vpn1, vpn2, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product(
blockSizeR, blockSizeRK, blockSizeC, blockSizeCK, runs, failureModels, failureRates, maliciousNodes, class1ratios, chiR, chiC, validatorsPerNode1, validatorsPerNode2, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2):
# Network Degree has to be an even number
if netDegree % 2 == 0:
shape = Shape(blockSize, nn, fm, fr, mn, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run)
yield shape
shape = Shape(blckSizeR, blckSizeRK, blckSizeC, blckSizeCK, nn, fm, fr, mn, class1ratio, chR, chC, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run)
yield shape