logos-storage/das-research
1
0
Fork 0
mirror of https://github.com/logos-storage/das-research.git synced 2026-01-04 06:03:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge f5f2131a45df4089b75ff5834bf39aacbf923b27 into f36c3c85ba31ed0fd27c3650794a511ad994a661

Browse Source
This commit is contained in:
Sudipta Basak 2024-06-25 13:16:38 +02:00 committed by GitHub
commit bf97928e5b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 522 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
DAS/node.py
View File

@ -32,11 +32,13 @@ class Neighbor:
class Validator:
i = 0
def __init__(self, rowIDs, columnIDs):
self.rowIDs = rowIDs
self.columnIDs = columnIDs
def initValidator(nbRows, custodyRows, nbCols, custodyCols):
random.seed(10 + Validator.i); Validator.i += 1
rowIDs = set(random.sample(range(nbRows), custodyRows))
columnIDs = set(random.sample(range(nbCols), custodyCols))
return Validator(rowIDs, columnIDs)
@ -48,7 +50,7 @@ class Node:
"""It returns the node ID."""
return str(self.ID)
def __init__(self, ID, amIproposer, amImalicious, logger, shape, config,
def __init__(self, ID, amIproposer, nodeClass, amImalicious, logger, shape, config,
validators, rows = set(), columns = set()):
"""It initializes the node, and eventual validators, following the simulation configuration in shape and config.
@ -82,7 +84,7 @@ class Node:
self.rowIDs = range(shape.nbRows)
self.columnIDs = range(shape.nbCols)
else:
self.nodeClass = 1 if (self.ID <= shape.numberNodes * shape.class1ratio) else 2
self.nodeClass = nodeClass
self.vpn = len(validators) #TODO: needed by old code, change to fn
self.rowIDs = set(rows)
@ -96,13 +98,13 @@ class Node:
self.logger.warning("Row custody (*vpn) larger than number of rows!", extra=self.format)
self.rowIDs = range(self.shape.nbRows)
else:
self.rowIDs = set(random.sample(range(self.shape.nbRows), self.vpn*self.shape.custodyRows))
self.rowIDs = set(random.sample(range(self.shape.nbRows), max(self.vpn*self.shape.custodyRows, self.shape.minCustodyRows)))
if (self.vpn * self.shape.custodyCols) > self.shape.nbCols:
self.logger.warning("Column custody (*vpn) larger than number of columns!", extra=self.format)
self.columnIDs = range(self.shape.nbCols)
else:
self.columnIDs = set(random.sample(range(self.shape.nbCols), self.vpn*self.shape.custodyCols))
self.columnIDs = set(random.sample(range(self.shape.nbCols), max(self.vpn*self.shape.custodyCols, self.shape.minCustodyCols)))
self.rowNeighbors = collections.defaultdict(dict)
self.columnNeighbors = collections.defaultdict(dict)
@ -120,10 +122,8 @@ class Node:
# 1 Mbps ~= 1e6 mbps * 0.050 s / (560*8) bits ~= 11 segments/timestep
if self.amIproposer:
self.bwUplink = shape.bwUplinkProd
elif self.nodeClass == 1:
self.bwUplink = shape.bwUplink1
else:
self.bwUplink = shape.bwUplink2
self.bwUplink = shape.nodeTypes["classes"][self.nodeClass]["def"]['bwUplinks']
self.bwUplink *= 1e3 / 8 * config.stepDuration / config.segmentSize
self.repairOnTheFly = config.evalConf(self, config.repairOnTheFly, shape)

6
DAS/observer.py
View File

@ -83,8 +83,8 @@ class Observer:
sampleProgress = arrived / expected
nodeProgress = ready / (len(validators)-1)
validatorCnt = sum([v.vpn for v in validators[1:]])
validatorAllProgress = validatedall / validatorCnt
validatorProgress = validated / validatorCnt
validatorAllProgress = (validatedall / validatorCnt) if validatorCnt != 0 else 1
validatorProgress = (validated / validatorCnt) if validatorCnt != 0 else 1
return missingSamples, sampleProgress, nodeProgress, validatorAllProgress, validatorProgress
@ -96,7 +96,7 @@ class Observer:
return np.mean(l) if l else np.NaN
trafficStats = {}
for cl in range(0,3):
for cl in self.config.nodeClasses:
Tx = [v.statsTxInSlot for v in validators if v.nodeClass == cl]
Rx = [v.statsRxInSlot for v in validators if v.nodeClass == cl]
RxDup = [v.statsRxDupInSlot for v in validators if v.nodeClass == cl]

1
DAS/results.py
View File

@ -24,7 +24,6 @@ class Result:
self.restoreColumnCount = [0] * shape.numberNodes
self.repairedSampleCount = [0] * shape.numberNodes
self.numberNodes = shape.numberNodes
self.class1ratio = shape.class1ratio
def copyValidators(self, validators):
"""Copy information from simulator.validators to result."""

19
DAS/shape.py
View File

@ -3,7 +3,7 @@
class Shape:
"""This class represents a set of parameters for a specific simulation."""
def __init__(self, nbCols, nbColsK, nbRows, nbRowsK,
numberNodes, failureModel, failureRate, maliciousNodes, class1ratio, custodyRows, custodyCols, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run):
numberNodes, failureModel, failureRate, maliciousNodes, custodyRows, custodyCols, minCustodyRows, minCustodyCols, netDegree, bwUplinkProd, run, nodeTypes):
"""Initializes the shape with the parameters passed in argument."""
self.run = run
self.numberNodes = numberNodes
@ -15,14 +15,13 @@ class Shape:
self.failureRate = failureRate
self.maliciousNodes = maliciousNodes
self.netDegree = netDegree
self.class1ratio = class1ratio
self.custodyRows = custodyRows
self.custodyCols = custodyCols
self.vpn1 = vpn1
self.vpn2 = vpn2
self.minCustodyRows = minCustodyRows
self.minCustodyCols = minCustodyCols
self.bwUplinkProd = bwUplinkProd
self.bwUplink1 = bwUplink1
self.bwUplink2 = bwUplink2
self.nodeTypes = nodeTypes
self.nodeClasses = [0] + [_k for _k in nodeTypes["classes"].keys()]
self.randomSeed = ""
def __repr__(self):
@ -35,17 +34,15 @@ class Shape:
shastr += "-nn-"+str(self.numberNodes)
shastr += "-fm-"+str(self.failureModel)
shastr += "-fr-"+str(self.failureRate)
shastr += "-c1r-"+str(self.class1ratio)
shastr += "-cusr-"+str(self.custodyRows)
shastr += "-cusc-"+str(self.custodyCols)
shastr += "-vpn1-"+str(self.vpn1)
shastr += "-vpn2-"+str(self.vpn2)
shastr += "-mcusr-"+str(self.minCustodyRows)
shastr += "-mcusc-"+str(self.minCustodyCols)
shastr += "-bwupprod-"+str(self.bwUplinkProd)
shastr += "-bwup1-"+str(self.bwUplink1)
shastr += "-bwup2-"+str(self.bwUplink2)
shastr += "-nd-"+str(self.netDegree)
shastr += "-r-"+str(self.run)
shastr += "-mn-"+str(self.maliciousNodes)
shastr += "-ntypes-"+str(self.nodeTypes['group'])
return shastr
def setSeed(self, seed):

51
DAS/simulator.py
View File

@ -44,6 +44,15 @@ class Simulator:
self.proposerPublishToR = config.evalConf(self, config.proposerPublishToR, shape)
self.proposerPublishToC = config.evalConf(self, config.proposerPublishToR, shape)
def getNodeClass(self, nodeIdx):
nodeRatios = [_v['weight'] for _k, _v in self.shape.nodeTypes["classes"].items()]
nodeCounts = [int(self.shape.numberNodes * ratio / sum(nodeRatios)) for ratio in nodeRatios]
commulativeSum = [sum(nodeCounts[:i+1]) for i in range(len(nodeCounts))]
commulativeSum[-1] = self.shape.numberNodes
for i, idx in enumerate(commulativeSum):
if nodeIdx <= idx:
return self.shape.nodeClasses[i + 1]
def initValidators(self):
"""It initializes all the validators in the network."""
self.glob = Observer(self.logger, self.shape)
@ -125,11 +134,11 @@ class Simulator:
self.logger.error("custodyRows has to be smaller than %d" % self.shape.nbRows)
vs = []
nodeClass = 1 if (i <= self.shape.numberNodes * self.shape.class1ratio) else 2
vpn = self.shape.vpn1 if (nodeClass == 1) else self.shape.vpn2
nodeClass = self.getNodeClass(i)
vpn = self.shape.nodeTypes["classes"][nodeClass]["def"]['validatorsPerNode']
for v in range(vpn):
vs.append(initValidator(self.shape.nbRows, self.shape.custodyRows, self.shape.nbCols, self.shape.custodyCols))
val = Node(i, int(not i!=0), amImalicious_value, self.logger, self.shape, self.config, vs)
val = Node(i, int(not i!=0), nodeClass, amImalicious_value, self.logger, self.shape, self.config, vs)
if i == self.proposerID:
val.initBlock()
else:
@ -311,12 +320,9 @@ class Simulator:
cnN = "nodes ready"
cnV = "validators ready"
cnT0 = "TX builder mean"
cnT1 = "TX class1 mean"
cnT2 = "TX class2 mean"
cnR1 = "RX class1 mean"
cnR2 = "RX class2 mean"
cnD1 = "Dup class1 mean"
cnD2 = "Dup class2 mean"
cnT = lambda i: f"TX class{i} mean"
cnR = lambda i: f"RX class{i} mean"
cnD = lambda i: f"Dup class{i} mean"
# if custody is based on the requirements of underlying individual
# validators, we can get detailed data on how many validated.
@ -325,19 +331,20 @@ class Simulator:
cnVv = validatorProgress
else:
cnVv = validatorAllProgress
progressVector.append({
cnS:sampleProgress,
cnN:nodeProgress,
cnV:cnVv,
cnT0: trafficStats[0]["Tx"]["mean"],
cnT1: trafficStats[1]["Tx"]["mean"],
cnT2: trafficStats[2]["Tx"]["mean"],
cnR1: trafficStats[1]["Rx"]["mean"],
cnR2: trafficStats[2]["Rx"]["mean"],
cnD1: trafficStats[1]["RxDup"]["mean"],
cnD2: trafficStats[2]["RxDup"]["mean"],
})
progressDict = {
cnS: sampleProgress,
cnN: nodeProgress,
cnV: cnVv,
cnT0: trafficStats[0]["Tx"]["mean"]
}
for nc in self.shape.nodeClasses:
if nc != 0:
progressDict[cnT(nc)] = trafficStats[nc]["Tx"]["mean"]
progressDict[cnR(nc)] = trafficStats[nc]["Rx"]["mean"]
progressDict[cnD(nc)] = trafficStats[nc]["RxDup"]["mean"]
progressVector.append(progressDict)
if missingSamples == oldMissingSamples:
if len(missingVector) > self.config.steps4StopCondition:

590
DAS/visualizor.py
View File

File diff suppressed because it is too large Load Diff

37
smallConf.py
View File

@ -76,18 +76,31 @@ proposerPublishToC = "shape.netDegree"
validatorBasedCustody = False
custodyRows = range(2, 3, 2)
custodyCols = range(2, 3, 2)
# ratio of class1 nodes (see below for parameters per class)
class1ratios = [0.8]
# Number of validators per beacon node
validatorsPerNode1 = [1]
validatorsPerNode2 = [5]
minCustodyRows = range(2, 3, 2)
minCustodyCols = range(2, 3, 2)
# Set uplink bandwidth in megabits/second
bwUplinksProd = [200]
bwUplinks1 = [10]
bwUplinks2 = [200]
nodeTypesGroup = [
{
"group": "g1",
"classes": {
1: {
"weight": 70,
"def": {'validatorsPerNode': 1, 'bwUplinks': 10}
},
2: {
"weight": 20,
"def": {'validatorsPerNode': 5, 'bwUplinks': 200}
},
3: {
"weight": 10,
"def": {'validatorsPerNode': 10, 'bwUplinks': 500}
}
}
}
]
# Step duration in miliseconds (Classic RTT is about 100ms)
stepDuration = 50
@ -135,11 +148,11 @@ colsK = range(32, 65, 128)
rowsK = range(32, 65, 128)
def nextShape():
for nbCols, nbColsK, nbRows, nbRowsK, run, fm, fr, mn, class1ratio, chR, chC, vpn1, vpn2, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product(
cols, colsK, rows, rowsK, runs, failureModels, failureRates, maliciousNodes, class1ratios, custodyRows, custodyCols, validatorsPerNode1, validatorsPerNode2, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2):
for nbCols, nbColsK, nbRows, nbRowsK, run, fm, fr, mn, chR, chC, minChR, minChC, nn, netDegree, bwUplinkProd, nodeTypes in itertools.product(
cols, colsK, rows, rowsK, runs, failureModels, failureRates, maliciousNodes, custodyRows, custodyCols, minCustodyRows, minCustodyCols, numberNodes, netDegrees, bwUplinksProd, nodeTypesGroup):
# Network Degree has to be an even number
if netDegree % 2 == 0:
shape = Shape(nbCols, nbColsK, nbRows, nbRowsK, nn, fm, fr, mn, class1ratio, chR, chC, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run)
shape = Shape(nbCols, nbColsK, nbRows, nbRowsK, nn, fm, fr, mn, chR, chC, minChR, minChC, netDegree, bwUplinkProd, run, nodeTypes)
yield shape
def evalConf(self, param, shape = None):

6
study.py
View File

@ -213,11 +213,11 @@ def study():
logger.info("A total of %d simulations ran in %d seconds" % (len(results), end-start), extra=format)
if config.visualization:
vis = Visualizer(execID, config)
vis.plotHeatmaps()
# vis = Visualizer(execID, config)
# vis.plotHeatmaps()
visual = Visualizor(execID, config, results)
visual.plotHeatmaps("nn", "fr")
# visual.plotHeatmaps("nn", "fr")
visual.plotAllHeatMaps()
if __name__ == "__main__":