codex-storage
/
das-research
mirror of https://github.com/codex-storage/das-research.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #22 from status-im/twoClasses 

Two classes
This commit is contained in:
Leo 2023-03-20 21:01:28 +01:00 committed by GitHub
parent d9d79f9245 64fcbff0ef
commit fd532a6f17
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 107 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
DAS/shape.py
View File

@ -3,23 +3,35 @@
class Shape:
"""This class represents a set of parameters for a specific simulation."""
def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree, run):
def __init__(self, blockSize, numberNodes, failureRate, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run):
"""Initializes the shape with the parameters passed in argument."""
self.run = run
self.numberValidators = numberValidators
self.numberNodes = numberNodes
self.blockSize = blockSize
self.failureRate = failureRate
self.netDegree = netDegree
self.class1ratio = class1ratio
self.chi = chi
self.vpn1 = vpn1
self.vpn2 = vpn2
self.bwUplinkProd = bwUplinkProd
self.bwUplink1 = bwUplink1
self.bwUplink2 = bwUplink2
self.randomSeed = ""
def __repr__(self):
"""Returns a printable representation of the shape"""
shastr = ""
shastr += "bs-"+str(self.blockSize)
shastr += "-nbv-"+str(self.numberValidators)
shastr += "-nn-"+str(self.numberNodes)
shastr += "-fr-"+str(self.failureRate)
shastr += "-c1r-"+str(self.class1ratio)
shastr += "-chi-"+str(self.chi)
shastr += "-vpn1-"+str(self.vpn1)
shastr += "-vpn2-"+str(self.vpn2)
shastr += "-bwupprod-"+str(self.bwUplinkProd)
shastr += "-bwup1-"+str(self.bwUplink1)
shastr += "-bwup2-"+str(self.bwUplink2)
shastr += "-nd-"+str(self.netDegree)
shastr += "-r-"+str(self.run)
return shastr

43
DAS/simulator.py
View File

@ -15,6 +15,7 @@ class Simulator:
def __init__(self, shape, config):
"""It initializes the simulation with a set of parameters (shape)."""
self.shape = shape
self.config = config
self.format = {"entity": "Simulator"}
self.result = Result(self.shape)
self.validators = []
@ -28,12 +29,18 @@ class Simulator:
self.glob = Observer(self.logger, self.shape)
self.glob.reset()
self.validators = []
rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize)
columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize)
random.shuffle(rows)
random.shuffle(columns)
for i in range(self.shape.numberValidators):
val = Validator(i, int(not i!=0), self.logger, self.shape, rows, columns)
if self.config.evenLineDistribution:
rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberNodes/self.shape.blockSize)
columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberNodes/self.shape.blockSize)
random.shuffle(rows)
random.shuffle(columns)
for i in range(self.shape.numberNodes):
if self.config.evenLineDistribution:
val = Validator(i, int(not i!=0), self.logger, self.shape,
rows[(i*self.shape.chi):((i+1)*self.shape.chi)],
columns[(i*self.shape.chi):((i+1)*self.shape.chi)])
else:
val = Validator(i, int(not i!=0), self.logger, self.shape)
if i == self.proposerID:
val.initBlock()
self.glob.setGoldenData(val.block)
@ -57,7 +64,10 @@ class Simulator:
# If the number of nodes in a channel is smaller or equal to the
# requested degree, a fully connected graph is used. For n>d, a random
# d-regular graph is set up. (For n=d+1, the two are the same.)
if (len(rowChannels[id]) <= self.shape.netDegree):
if not rowChannels[id]:
self.logger.error("No nodes for row %d !" % id, extra=self.format)
continue
elif (len(rowChannels[id]) <= self.shape.netDegree):
self.logger.debug("Graph fully connected with degree %d !" % (len(rowChannels[id]) - 1), extra=self.format)
G = nx.complete_graph(len(rowChannels[id]))
else:
@ -70,7 +80,10 @@ class Simulator:
val1.rowNeighbors[id].update({val2.ID : Neighbor(val2, 0, self.shape.blockSize)})
val2.rowNeighbors[id].update({val1.ID : Neighbor(val1, 0, self.shape.blockSize)})
if (len(columnChannels[id]) <= self.shape.netDegree):
if not columnChannels[id]:
self.logger.error("No nodes for column %d !" % id, extra=self.format)
continue
elif (len(columnChannels[id]) <= self.shape.netDegree):
self.logger.debug("Graph fully connected with degree %d !" % (len(columnChannels[id]) - 1), extra=self.format)
G = nx.complete_graph(len(columnChannels[id]))
else:
@ -97,7 +110,7 @@ class Simulator:
v.columnNeighbors[id].update({vi.ID : Neighbor(vi, 1, self.shape.blockSize)})
if self.logger.isEnabledFor(logging.DEBUG):
for i in range(0, self.shape.numberValidators):
for i in range(0, self.shape.numberNodes):
self.logger.debug("Val %d : rowN %s", i, self.validators[i].rowNeighbors, extra=self.format)
self.logger.debug("Val %d : colN %s", i, self.validators[i].columnNeighbors, extra=self.format)
@ -120,6 +133,8 @@ class Simulator:
for val in self.validators:
val.shape.failureRate = shape.failureRate
val.shape.chi = shape.chi
val.shape.vpn1 = shape.vpn1
val.shape.vpn2 = shape.vpn2
# In GossipSub the initiator might push messages without participating in the mesh.
# proposerPublishOnly regulates this behavior. If set to true, the proposer is not
@ -146,17 +161,17 @@ class Simulator:
missingVector.append(missingSamples)
oldMissingSamples = missingSamples
self.logger.debug("PHASE SEND %d" % steps, extra=self.format)
for i in range(0,self.shape.numberValidators):
for i in range(0,self.shape.numberNodes):
self.validators[i].send()
self.logger.debug("PHASE RECEIVE %d" % steps, extra=self.format)
for i in range(1,self.shape.numberValidators):
for i in range(1,self.shape.numberNodes):
self.validators[i].receiveRowsColumns()
self.logger.debug("PHASE RESTORE %d" % steps, extra=self.format)
for i in range(1,self.shape.numberValidators):
for i in range(1,self.shape.numberNodes):
self.validators[i].restoreRows()
self.validators[i].restoreColumns()
self.logger.debug("PHASE LOG %d" % steps, extra=self.format)
for i in range(0,self.shape.numberValidators):
for i in range(0,self.shape.numberNodes):
self.validators[i].logRows()
self.validators[i].logColumns()
@ -167,7 +182,7 @@ class Simulator:
(steps, statsTxInSlot[0], statsRxInSlot[0],
mean(statsTxInSlot[1:]), max(statsTxInSlot[1:]),
mean(statsRxInSlot[1:]), max(statsRxInSlot[1:])), extra=self.format)
for i in range(0,self.shape.numberValidators):
for i in range(0,self.shape.numberNodes):
self.validators[i].updateStats()
arrived, expected = self.glob.checkStatus(self.validators)

6
DAS/tools.py
View File

@ -79,3 +79,9 @@ def sampleLine(line, limit):
r[i] = 1
limit -= 1
return r
def unionOfSamples(population, sampleSize, times):
selected = set()
for t in range(times):
selected |= set(random.sample(population, sampleSize))
return selected

27
DAS/validator.py
View File

@ -4,7 +4,7 @@ import random
import collections
import logging
from DAS.block import *
from DAS.tools import shuffled, shuffledDict
from DAS.tools import shuffled, shuffledDict, unionOfSamples
from bitarray.util import zeros
from collections import deque
from itertools import chain
@ -38,8 +38,13 @@ class Validator:
"""It returns the validator ID."""
return str(self.ID)
def __init__(self, ID, amIproposer, logger, shape, rows, columns):
"""It initializes the validator with the logger, shape and assigned rows/columns."""
def __init__(self, ID, amIproposer, logger, shape, rows = None, columns = None):
"""It initializes the validator with the logger shape and rows/columns.
If rows/columns are specified these are observed, otherwise (default)
chi rows and columns are selected randomly.
"""
self.shape = shape
FORMAT = "%(levelname)s : %(entity)s : %(message)s"
self.ID = ID
@ -53,18 +58,17 @@ class Validator:
if self.shape.chi < 1:
self.logger.error("Chi has to be greater than 0", extra=self.format)
elif self.shape.chi > self.shape.blockSize:
self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format)
self.logger.error("Chi has to be smaller than %d" % self.shape.blockSize, extra=self.format)
else:
if amIproposer:
self.rowIDs = range(shape.blockSize)
self.columnIDs = range(shape.blockSize)
else:
self.rowIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)]
self.columnIDs = columns[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)]
#if shape.deterministic:
# random.seed(self.ID)
#self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi)
#self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi)
vpn = self.shape.vpn1 if (self.ID <= shape.numberNodes * shape.class1ratio) else self.shape.vpn2
self.rowIDs = rows if rows else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn)
self.columnIDs = columns if columns else unionOfSamples(range(self.shape.blockSize), self.shape.chi, vpn)
self.rowNeighbors = collections.defaultdict(dict)
self.columnNeighbors = collections.defaultdict(dict)
@ -77,7 +81,12 @@ class Validator:
# Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?)
# 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11
# TODO: this should be a parameter
self.bwUplink = 110 if not self.amIproposer else 2200 # approx. 10Mbps and 200Mbps
if self.amIproposer:
self.bwUplink = shape.bwUplinkProd
elif self.ID <= shape.numberNodes * shape.class1ratio:
self.bwUplink = shape.bwUplink1
else:
self.bwUplink = shape.bwUplink2
self.repairOnTheFly = True
self.sendLineUntil = (self.shape.blockSize + 1) // 2 # stop sending on a p2p link if at least this amount of samples passed

14
DAS/visualizer.py
View File

@ -12,7 +12,8 @@ class Visualizer:
def __init__(self, execID):
self.execID = execID
self.folderPath = "results/"+self.execID
self.parameters = ['run', 'blockSize', 'failureRate', 'numberValidators', 'netDegree', 'chi']
self.parameters = ['run', 'blockSize', 'failureRate', 'numberNodes', 'netDegree',
'chi', 'vpn1', 'vpn2', 'bwUplinkProd', 'bwUplink1', 'bwUplink2']
self.minimumDataPoints = 2
def plottingData(self):
@ -27,22 +28,27 @@ class Visualizer:
run = int(root.find('run').text)
blockSize = int(root.find('blockSize').text)
failureRate = int(root.find('failureRate').text)
numberValidators = int(root.find('numberValidators').text)
numberNodes = int(root.find('numberNodes').text)
netDegree = int(root.find('netDegree').text)
chi = int(root.find('chi').text)
vpn1 = int(root.find('vpn1').text)
vpn2 = int(root.find('vpn2').text)
bwUplinkProd = int(root.find('bwUplinkProd').text)
bwUplink1 = int(root.find('bwUplink1').text)
bwUplink2 = int(root.find('bwUplink2').text)
tta = int(root.find('tta').text)
# Loop over all possible combinations of length 4 of the parameters
for combination in combinations(self.parameters, 4):
# Get the indices and values of the parameters in the combination
indices = [self.parameters.index(element) for element in combination]
selectedValues = [run, blockSize, failureRate, numberValidators, netDegree, chi]
selectedValues = [run, blockSize, failureRate, numberNodes, netDegree, chi, vpn1, vpn2, bwUplinkProd, bwUplink1, bwUplink2]
values = [selectedValues[index] for index in indices]
names = [self.parameters[i] for i in indices]
keyComponents = [f"{name}_{value}" for name, value in zip(names, values)]
key = tuple(keyComponents[:4])
#Get the names of the other 2 parameters that are not included in the key
otherParams = [self.parameters[i] for i in range(6) if i not in indices]
otherParams = [self.parameters[i] for i in range(len(self.parameters)) if i not in indices]
#Append the values of the other 2 parameters and the ttas to the lists for the key
otherIndices = [i for i in range(len(self.parameters)) if i not in indices]

41
config_example.py
View File

@ -14,6 +14,8 @@ if needed.
"""
import logging
import itertools
import numpy as np
from DAS.shape import Shape
dumpXML = 1
@ -24,11 +26,15 @@ logLevel = logging.INFO
# for more details, see joblib.Parallel
numJobs = 3
# distribute rows/columns evenly between validators (True)
# or generate it using local randomness (False)
evenLineDistribution = False
# Number of simulation runs with the same parameters for statistical relevance
runs = range(10)
# Number of validators
numberValidators = range(256, 513, 128)
numberNodes = range(256, 513, 128)
# Percentage of block not released by producer
failureRates = range(10, 91, 40)
@ -39,8 +45,21 @@ blockSizes = range(32,65,16)
# Per-topic mesh neighborhood size
netDegrees = range(6, 9, 2)
# Number of rows and columns a validator is interested in
chis = range(4, 9, 2)
# number of rows and columns a validator is interested in
chis = range(1, 5, 2)
# ratio of class1 nodes (see below for parameters per class)
class1ratios = np.arange(0, 1, .2)
# Number of validators per beacon node
validatorsPerNode1 = [1]
validatorsPerNode2 = [2, 4, 8, 16, 32]
# Set uplink bandwidth. In segments (~560 bytes) per timestep (50ms?)
# 1 Mbps ~= 1e6 / 20 / 8 / 560 ~= 11
bwUplinksProd = [2200]
bwUplinks1 = [110]
bwUplinks2 = [2200]
# Set to True if you want your run to be deterministic, False if not
deterministic = False
@ -49,13 +68,9 @@ deterministic = False
randomSeed = "DAS"
def nextShape():
for run in runs:
for fr in failureRates:
for chi in chis:
for blockSize in blockSizes:
for nv in numberValidators:
for netDegree in netDegrees:
# Network Degree has to be an even number
if netDegree % 2 == 0:
shape = Shape(blockSize, nv, fr, chi, netDegree, run)
yield shape
for run, fr, class1ratio, chi, vpn1, vpn2, blockSize, nn, netDegree, bwUplinkProd, bwUplink1, bwUplink2 in itertools.product(
runs, failureRates, class1ratios, chis, validatorsPerNode1, validatorsPerNode2, blockSizes, numberNodes, netDegrees, bwUplinksProd, bwUplinks1, bwUplinks2):
# Network Degree has to be an even number
if netDegree % 2 == 0:
shape = Shape(blockSize, nn, fr, class1ratio, chi, vpn1, vpn2, netDegree, bwUplinkProd, bwUplink1, bwUplink2, run)
yield shape

2
study.py
View File

@ -40,7 +40,7 @@ def study():
print("You need to pass a configuration file in parameter")
exit(1)
shape = Shape(0, 0, 0, 0, 0, 0)
shape = Shape(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
sim = Simulator(shape, config)
sim.initLogger()
results = []