mirror of
https://github.com/codex-storage/das-research.git
synced 2025-02-20 14:58:09 +00:00
Add configuration file, split configuration from simulation shape, fix bug about network degree and unbalanced row/column verification
This commit is contained in:
Leonardo Bautista-Gomez
parent
bf1a5a60e4
commit
fc7339dc91
@ -1,2 +1,3 @@
|
||||
from DAS.simulator import *
|
||||
from DAS.configuration import *
|
||||
from DAS.shape import *
|
||||
|
||||
@ -1,50 +1,48 @@
|
||||
#!/bin/python3
|
||||
|
||||
import configparser
|
||||
|
||||
class Configuration:
|
||||
|
||||
deterministic = 0
|
||||
blockSize = 0
|
||||
numberValidators = 0
|
||||
failureRate = 0
|
||||
failureRateStart = 0
|
||||
failureRateStop = 0
|
||||
failureRateStep = 0
|
||||
chio = 0
|
||||
chiStart = 0
|
||||
chiStop = 0
|
||||
chiStep = 0
|
||||
run =0
|
||||
runStart = 0
|
||||
runStop = 0
|
||||
runStep = 0
|
||||
|
||||
def __init__(self, deterministic, blockSize, numberValidators,\
|
||||
failureRateStart, failureRateStop, failureRateStep,\
|
||||
chiStart, chiStop, chiStep,\
|
||||
runStart, runStop, runStep):
|
||||
def __init__(self, fileName):
|
||||
|
||||
if numberValidators < (blockSize*4):
|
||||
config = configparser.RawConfigParser()
|
||||
config.read(fileName)
|
||||
|
||||
self.nvStart = int(config.get("Simulation Space", "numberValidatorStart"))
|
||||
self.nvStop = int(config.get("Simulation Space", "numberValidatorStop"))
|
||||
self.nvStep = int(config.get("Simulation Space", "numberValidatorStep"))
|
||||
|
||||
self.blockSizeStart = int(config.get("Simulation Space", "blockSizeStart"))
|
||||
self.blockSizeStop = int(config.get("Simulation Space", "blockSizeStop"))
|
||||
self.blockSizeStep = int(config.get("Simulation Space", "blockSizeStep"))
|
||||
|
||||
self.netDegreeStart = int(config.get("Simulation Space", "netDegreeStart"))
|
||||
self.netDegreeStop = int(config.get("Simulation Space", "netDegreeStop"))
|
||||
self.netDegreeStep = int(config.get("Simulation Space", "netDegreeStep"))
|
||||
|
||||
self.failureRateStart = int(config.get("Simulation Space", "failureRateStart"))
|
||||
self.failureRateStop = int(config.get("Simulation Space", "failureRateStop"))
|
||||
self.failureRateStep = int(config.get("Simulation Space", "failureRateStep"))
|
||||
|
||||
self.chiStart = int(config.get("Simulation Space", "chiStart"))
|
||||
self.chiStop = int(config.get("Simulation Space", "chiStop"))
|
||||
self.chiStep = int(config.get("Simulation Space", "chiStep"))
|
||||
|
||||
self.numberRuns = int(config.get("Advanced", "numberRuns"))
|
||||
self.deterministic = config.get("Advanced", "deterministic")
|
||||
|
||||
if self.nvStop < (self.blockSizeStart*4):
|
||||
print("ERROR: The number of validators cannot be lower than the block size * 4")
|
||||
exit(1)
|
||||
if chiStart < 1:
|
||||
if self.chiStart < 1:
|
||||
print("Chi has to be greater than 0")
|
||||
exit(1)
|
||||
if chiStop > blockSize:
|
||||
print("Chi has to be smaller than %d" % blockSize)
|
||||
if self.chiStop > self.blockSizeStart:
|
||||
print("Chi (%d) has to be smaller or equal to block the size (%d)" % (self.chiStop, self.blockSizeStart))
|
||||
exit(1)
|
||||
|
||||
self.deterministic = deterministic
|
||||
self.blockSize = blockSize
|
||||
self.numberValidators = numberValidators
|
||||
self.failureRateStart = failureRateStart
|
||||
self.failureRateStop = failureRateStop
|
||||
self.failureRateStep = failureRateStep
|
||||
self.failureRate = failureRateStart
|
||||
self.chiStart = chiStart
|
||||
self.chiStop = chiStop
|
||||
self.chiStep = chiStep
|
||||
self.chi = chiStart
|
||||
self.runStart = runStart
|
||||
self.runStop = runStop
|
||||
self.runStep = runStep
|
||||
self.run = runStart
|
||||
|
||||
|
||||
|
||||
19
DAS/shape.py
Normal file
19
DAS/shape.py
Normal file
@ -0,0 +1,19 @@
|
||||
#!/bin/python3
|
||||
|
||||
class Shape:
|
||||
numberValidators = 0
|
||||
failureRate = 0
|
||||
blockSize = 0
|
||||
netDegree = 0
|
||||
chi = 0
|
||||
|
||||
def __init__(self, blockSize, numberValidators, failureRate, chi, netDegree):
|
||||
self.numberValidators = numberValidators
|
||||
self.failureRate = failureRate
|
||||
self.blockSize = blockSize
|
||||
self.netDegree = netDegree
|
||||
self.chi = chi
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#!/bin/python
|
||||
|
||||
import networkx as nx
|
||||
import logging
|
||||
import logging, random
|
||||
from datetime import datetime
|
||||
from DAS.tools import *
|
||||
from DAS.results import *
|
||||
@ -15,23 +15,25 @@ class Simulator:
|
||||
validators = []
|
||||
glob = []
|
||||
result = []
|
||||
config = []
|
||||
shape = []
|
||||
logger = []
|
||||
format = {}
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
def __init__(self, shape):
|
||||
self.shape = shape
|
||||
self.format = {"entity": "Simulator"}
|
||||
self.result = Result(self.config)
|
||||
self.result = Result(self.shape)
|
||||
|
||||
def initValidators(self):
|
||||
if not self.config.deterministic:
|
||||
random.seed(datetime.now())
|
||||
self.glob = Observer(self.logger, self.config)
|
||||
self.glob = Observer(self.logger, self.shape)
|
||||
self.glob.reset()
|
||||
self.validators = []
|
||||
for i in range(self.config.numberValidators):
|
||||
val = Validator(i, int(not i!=0), self.logger, self.config)
|
||||
rows = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize)
|
||||
columns = list(range(self.shape.blockSize)) * int(self.shape.chi*self.shape.numberValidators/self.shape.blockSize)
|
||||
random.shuffle(rows)
|
||||
random.shuffle(columns)
|
||||
for i in range(self.shape.numberValidators):
|
||||
val = Validator(i, int(not i!=0), self.logger, self.shape, rows, columns)
|
||||
if i == self.proposerID:
|
||||
val.initBlock()
|
||||
self.glob.setGoldenData(val.block)
|
||||
@ -39,17 +41,21 @@ class Simulator:
|
||||
val.logIDs()
|
||||
self.validators.append(val)
|
||||
|
||||
def initNetwork(self, d=6):
|
||||
rowChannels = [[] for i in range(self.config.blockSize)]
|
||||
columnChannels = [[] for i in range(self.config.blockSize)]
|
||||
def initNetwork(self):
|
||||
self.shape.netDegree = 6
|
||||
rowChannels = [[] for i in range(self.shape.blockSize)]
|
||||
columnChannels = [[] for i in range(self.shape.blockSize)]
|
||||
for v in self.validators:
|
||||
for id in v.rowIDs:
|
||||
rowChannels[id].append(v)
|
||||
for id in v.columnIDs:
|
||||
columnChannels[id].append(v)
|
||||
|
||||
for id in range(self.config.blockSize):
|
||||
G = nx.random_regular_graph(d, len(rowChannels[id]))
|
||||
for id in range(self.shape.blockSize):
|
||||
|
||||
if (len(rowChannels[id]) < self.shape.netDegree):
|
||||
self.logger.error("Graph degree higher than %d" % len(rowChannels[id]), extra=self.format)
|
||||
G = nx.random_regular_graph(self.shape.netDegree, len(rowChannels[id]))
|
||||
if not nx.is_connected(G):
|
||||
self.logger.error("Graph not connected for row %d !" % id, extra=self.format)
|
||||
for u, v in G.edges:
|
||||
@ -57,7 +63,10 @@ class Simulator:
|
||||
val2=rowChannels[id][v]
|
||||
val1.rowNeighbors[id].append(val2)
|
||||
val2.rowNeighbors[id].append(val1)
|
||||
G = nx.random_regular_graph(d, len(columnChannels[id]))
|
||||
|
||||
if (len(columnChannels[id]) < self.shape.netDegree):
|
||||
self.logger.error("Graph degree higher than %d" % len(columnChannels[id]), extra=self.format)
|
||||
G = nx.random_regular_graph(self.shape.netDegree, len(columnChannels[id]))
|
||||
if not nx.is_connected(G):
|
||||
self.logger.error("Graph not connected for column %d !" % id, extra=self.format)
|
||||
for u, v in G.edges:
|
||||
@ -75,15 +84,12 @@ class Simulator:
|
||||
logger.addHandler(ch)
|
||||
self.logger = logger
|
||||
|
||||
def resetFailureRate(self, failureRate):
|
||||
self.config.failureRate = failureRate
|
||||
for val in self.validators:
|
||||
val.config.failureRate = failureRate
|
||||
|
||||
def resetChi(self, chi):
|
||||
self.config.chi = chi
|
||||
def resetShape(self, shape):
|
||||
self.shape = shape
|
||||
for val in self.validators:
|
||||
val.config.chi = chi
|
||||
val.shape.failureRate = shape.failureRate
|
||||
val.shape.chi = shape.chi
|
||||
|
||||
|
||||
def run(self):
|
||||
@ -96,9 +102,9 @@ class Simulator:
|
||||
while(missingSamples > 0):
|
||||
missingVector.append(missingSamples)
|
||||
oldMissingSamples = missingSamples
|
||||
for i in range(1,self.config.numberValidators):
|
||||
for i in range(1,self.shape.numberValidators):
|
||||
self.validators[i].receiveRowsColumns()
|
||||
for i in range(1,self.config.numberValidators):
|
||||
for i in range(1,self.shape.numberValidators):
|
||||
self.validators[i].restoreRows()
|
||||
self.validators[i].restoreColumns()
|
||||
self.validators[i].sendRows()
|
||||
@ -120,10 +126,10 @@ class Simulator:
|
||||
self.result.addMissing(missingVector)
|
||||
if missingSamples == 0:
|
||||
self.result.blockAvailable = 1
|
||||
self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.config.failureRate), extra=self.format)
|
||||
self.logger.debug("The entire block is available at step %d, with failure rate %d !" % (steps, self.shape.failureRate), extra=self.format)
|
||||
return self.result
|
||||
else:
|
||||
self.result.blockAvailable = 0
|
||||
self.logger.debug("The block cannot be recovered, failure rate %d!" % self.config.failureRate, extra=self.format)
|
||||
self.logger.debug("The block cannot be recovered, failure rate %d!" % self.shape.failureRate, extra=self.format)
|
||||
return self.result
|
||||
|
||||
|
||||
@ -11,34 +11,34 @@ class Validator:
|
||||
|
||||
ID = 0
|
||||
amIproposer = 0
|
||||
config = []
|
||||
shape = []
|
||||
format = {}
|
||||
logger = []
|
||||
|
||||
def __init__(self, ID, amIproposer, logger, config):
|
||||
self.config = config
|
||||
def __init__(self, ID, amIproposer, logger, shape, rows, columns):
|
||||
self.shape = shape
|
||||
FORMAT = "%(levelname)s : %(entity)s : %(message)s"
|
||||
self.ID = ID
|
||||
self.format = {"entity": "Val "+str(self.ID)}
|
||||
self.block = Block(self.config.blockSize)
|
||||
self.receivedBlock = Block(self.config.blockSize)
|
||||
self.block = Block(self.shape.blockSize)
|
||||
self.receivedBlock = Block(self.shape.blockSize)
|
||||
self.amIproposer = amIproposer
|
||||
self.logger = logger
|
||||
if self.config.chi < 1:
|
||||
if self.shape.chi < 1:
|
||||
self.logger.error("Chi has to be greater than 0", extra=self.format)
|
||||
elif self.config.chi > self.config.blockSize:
|
||||
elif self.shape.chi > self.shape.blockSize:
|
||||
self.logger.error("Chi has to be smaller than %d" % blockSize, extra=self.format)
|
||||
else:
|
||||
if amIproposer:
|
||||
self.rowIDs = range(config.blockSize)
|
||||
self.columnIDs = range(config.blockSize)
|
||||
self.rowIDs = range(shape.blockSize)
|
||||
self.columnIDs = range(shape.blockSize)
|
||||
else:
|
||||
self.rowIDs = []
|
||||
self.columnIDs = []
|
||||
if config.deterministic:
|
||||
random.seed(self.ID)
|
||||
self.rowIDs = random.sample(range(self.config.blockSize), self.config.chi)
|
||||
self.columnIDs = random.sample(range(self.config.blockSize), self.config.chi)
|
||||
self.rowIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)]
|
||||
self.columnIDs = rows[(self.ID*self.shape.chi):(self.ID*self.shape.chi + self.shape.chi)]
|
||||
#if shape.deterministic:
|
||||
# random.seed(self.ID)
|
||||
#self.rowIDs = random.sample(range(self.shape.blockSize), self.shape.chi)
|
||||
#self.columnIDs = random.sample(range(self.shape.blockSize), self.shape.chi)
|
||||
self.rowNeighbors = collections.defaultdict(list)
|
||||
self.columnNeighbors = collections.defaultdict(list)
|
||||
|
||||
@ -51,7 +51,7 @@ class Validator:
|
||||
|
||||
def initBlock(self):
|
||||
self.logger.debug("I am a block proposer.", extra=self.format)
|
||||
self.block = Block(self.config.blockSize)
|
||||
self.block = Block(self.shape.blockSize)
|
||||
self.block.fill()
|
||||
#self.block.print()
|
||||
|
||||
@ -60,21 +60,21 @@ class Validator:
|
||||
self.logger.error("I am NOT a block proposer", extra=self.format)
|
||||
else:
|
||||
self.logger.debug("Broadcasting my block...", extra=self.format)
|
||||
order = [i for i in range(self.config.blockSize * self.config.blockSize)]
|
||||
order = [i for i in range(self.shape.blockSize * self.shape.blockSize)]
|
||||
random.shuffle(order)
|
||||
while(order):
|
||||
i = order.pop()
|
||||
if (random.randint(0,99) >= self.config.failureRate):
|
||||
if (random.randint(0,99) >= self.shape.failureRate):
|
||||
self.block.data[i] = 1
|
||||
else:
|
||||
self.block.data[i] = 0
|
||||
nbFailures = self.block.data.count(0)
|
||||
measuredFailureRate = nbFailures * 100 / (self.config.blockSize * self.config.blockSize)
|
||||
measuredFailureRate = nbFailures * 100 / (self.shape.blockSize * self.shape.blockSize)
|
||||
self.logger.debug("Number of failures: %d (%0.02f %%)", nbFailures, measuredFailureRate, extra=self.format)
|
||||
#broadcasted.print()
|
||||
for id in range(self.config.blockSize):
|
||||
for id in range(self.shape.blockSize):
|
||||
self.sendColumn(id)
|
||||
for id in range(self.config.blockSize):
|
||||
for id in range(self.shape.blockSize):
|
||||
self.sendRow(id)
|
||||
|
||||
def getColumn(self, index):
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# DAS Research
|
||||
# DAS Research
|
||||
|
||||
This repository hosts all the research on DAS for the collaboration between Codex and the EF.
|
||||
This repository hosts all the research on DAS for the collaboration between Codex and the EF.
|
||||
|
||||
## Prepare the environment
|
||||
|
||||
@ -16,11 +16,11 @@ $ cd das-research
|
||||
```
|
||||
$ python3 -m venv myenv
|
||||
$ source myenv/bin/activate
|
||||
$ pip3 install -r DAS/requeriments.txt
|
||||
$ pip3 install -r DAS/requirements.txt
|
||||
```
|
||||
|
||||
## Run the simulator
|
||||
|
||||
```
|
||||
$ python3 study.py
|
||||
$ python3 study.py config.das
|
||||
```
|
||||
|
||||
27
config.das
Normal file
27
config.das
Normal file
@ -0,0 +1,27 @@
|
||||
[Simulation Space]
|
||||
|
||||
numberValidatorStart = 256
|
||||
numberValidatorStop = 512
|
||||
numberValidatorStep = 128
|
||||
|
||||
failureRateStart = 10
|
||||
failureRateStop = 90
|
||||
failureRateStep = 40
|
||||
|
||||
blockSizeStart = 32
|
||||
blockSizeStop = 64
|
||||
blockSizeStep = 16
|
||||
|
||||
netDegreeStart = 6
|
||||
netDegreeStop = 8
|
||||
netDegreeStep = 1
|
||||
|
||||
chiStart = 4
|
||||
chiStop = 8
|
||||
chiStep = 2
|
||||
|
||||
|
||||
[Advanced]
|
||||
|
||||
deterministic = 0
|
||||
numberRuns = 2
|
||||
49
study.py
49
study.py
@ -1,38 +1,45 @@
|
||||
#! /bin/python3
|
||||
|
||||
import time
|
||||
import time, sys
|
||||
from DAS import *
|
||||
|
||||
|
||||
def study():
|
||||
config = Configuration(0, 64, 256, 0, 100, 20, 8, 16, 4, 0, 10, 1)
|
||||
if len(sys.argv) < 2:
|
||||
print("You need to pass a configuration file in parameter")
|
||||
exit(1)
|
||||
|
||||
config = Configuration(sys.argv[1])
|
||||
sim = Simulator(config)
|
||||
sim.initLogger()
|
||||
frRange = []
|
||||
results = []
|
||||
resultRange = []
|
||||
simCnt = 0
|
||||
|
||||
sim.logger.info("Starting simulations:", extra=sim.format)
|
||||
start = time.time()
|
||||
for fr in range(config.failureRateStart, config.failureRateStop+1, config.failureRateStep):
|
||||
sim.resetFailureRate(fr)
|
||||
for chi in range(config.chiStart, config.chiStop+1, config.chiStep):
|
||||
sim.resetChi(chi)
|
||||
blockAvailable = 0
|
||||
for run in range(config.runStart, config.runStop, config.runStep):
|
||||
sim.logger.info("FR: %d %%, Chi: %d %%, Run: %d ..." % (fr, chi, run), extra=sim.format)
|
||||
sim.initValidators()
|
||||
sim.initNetwork()
|
||||
result = sim.run()
|
||||
blockAvailable += result.blockAvailable
|
||||
results.append(result)
|
||||
simCnt += 1
|
||||
frRange.append(fr)
|
||||
resultRange.append((blockAvailable)*100/config.runStop)
|
||||
|
||||
for run in range(config.numberRuns):
|
||||
for fr in range(config.failureRateStart, config.failureRateStop+1, config.failureRateStep):
|
||||
for chi in range(config.chiStart, config.chiStop+1, config.chiStep):
|
||||
for blockSize in range(config.blockSizeStart, config.blockSizeStop+1, config.blockSizeStep):
|
||||
for nv in range(config.nvStart, config.nvStop+1, config.nvStep):
|
||||
for netDegree in range(config.netDegreeStart, config.netDegreeStop, config.netDegreeStep):
|
||||
|
||||
if not config.deterministic:
|
||||
random.seed(datetime.now())
|
||||
|
||||
shape = Shape(blockSize, nv, fr, chi, netDegree)
|
||||
sim.resetShape(shape)
|
||||
sim.initValidators()
|
||||
sim.initNetwork()
|
||||
result = sim.run()
|
||||
sim.logger.info("Run %d, FR: %d %%, Chi: %d, BlockSize: %d, Nb.Val: %d, netDegree: %d ... Block Available: %d" % (run, fr, chi, blockSize, nv, netDegree, result.blockAvailable), extra=sim.format)
|
||||
results.append(result)
|
||||
simCnt += 1
|
||||
|
||||
end = time.time()
|
||||
sim.logger.info("A total of %d simulations ran in %d seconds" % (simCnt, end-start), extra=sim.format)
|
||||
#for i in range(len(frRange)):
|
||||
# sim.logger.info("For failure rate of %d we got %d %% success rate in DAS!" % (frRange[i], resultRange[i]), extra=sim.format)
|
||||
|
||||
|
||||
|
||||
study()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user