mirror of
https://github.com/codex-storage/codex-research.git
synced 2025-01-09 18:26:07 +00:00
Adding first part of codex simulator
This commit is contained in:
Leonardo Bautista-Gomez
parent
9b94462fe0
commit
cbab3166e9
287
simulator/codex.py
Normal file
287
simulator/codex.py
Normal file
@ -0,0 +1,287 @@
|
||||
#!/bin/python3
|
||||
|
||||
|
||||
import os, time, json, hashlib
|
||||
import random as random
|
||||
from datetime import datetime
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def plotHistoricData(data, xLabel, yLabel, filename):
|
||||
plt.clf()
|
||||
plt.plot(data)
|
||||
plt.grid(True)
|
||||
plt.ylabel(yLabel)
|
||||
plt.xlabel(xLabel)
|
||||
plt.savefig(filename)
|
||||
print("Figure %s created." % filename)
|
||||
|
||||
|
||||
def plotStackedBars(nbBars, data, legend, xLabel, yLabel, filename):
|
||||
plt.clf()
|
||||
for i in range(nbBars):
|
||||
if i == 0:
|
||||
plt.bar(range(len(data[0])), data[0], label=legend[i])
|
||||
else:
|
||||
plt.bar(range(len(data[i])), data[i], label=legend[i], bottom=data[i-1])
|
||||
plt.legend(loc="upper left")
|
||||
plt.grid(True)
|
||||
plt.ylabel(yLabel)
|
||||
plt.xlabel(xLabel)
|
||||
plt.savefig(filename)
|
||||
print("Figure %s created." % filename)
|
||||
|
||||
|
||||
def plotPieChart(data, legend, filename):
|
||||
plt.clf()
|
||||
plt.pie(data, labels=legend, autopct='%1.1f%%', shadow=True, startangle=90)
|
||||
plt.savefig(filename)
|
||||
plt.axis("equal")
|
||||
print("Figure %s created." % filename)
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self):
|
||||
self.maxStoragePerNode = 9 # in Terabytes
|
||||
self.maxFileSize = 111028 # in Megabytes
|
||||
self.minFileSize = 1 # in Megabytes
|
||||
self.defaultECN = 128 # in number of blocks
|
||||
self.minNbNodes = 32 # in nodes
|
||||
self.simulationLength = 9 # in days
|
||||
self.meanUsage = 99 # in %
|
||||
|
||||
|
||||
class Tracker:
|
||||
def __init__(self):
|
||||
self.storageUsed = []
|
||||
self.storageAvailable = []
|
||||
self.nbNodes = []
|
||||
self.nbFiles = []
|
||||
|
||||
|
||||
class Block():
|
||||
def __init__(self, fileID, blockID, size):
|
||||
self.fileID = fileID
|
||||
self.blockID = blockID
|
||||
self.size = size
|
||||
|
||||
|
||||
class Node():
|
||||
def __init__(self, ID, MaxStorageNode):
|
||||
self.storage = random.random() * MaxStorageNode * 1024 * 1024 # Storage in MBs
|
||||
self.available = self.storage
|
||||
self.ID = ID
|
||||
self.used = 0
|
||||
self.latency = random.random() * 100 # Latency score
|
||||
self.blocks = []
|
||||
|
||||
def storeBlock(self, fileID, blockID, size):
|
||||
self.used = self.used + size
|
||||
self.available = self.storage - self.used
|
||||
#block = Block(fileID, blockID, size)
|
||||
#self.blocks.append(block) # Too much memory
|
||||
|
||||
|
||||
class File:
|
||||
def __init__(self, ID, network):
|
||||
self.ID = ID #hashlib.sha256(str(ID).encode('utf-8')).hexdigest()
|
||||
self.size = random.random() * network.config.maxFileSize
|
||||
ECN = network.config.defaultECN
|
||||
if network.nbNodes < ECN or self.size < 1:
|
||||
self.ECK = int(ECN/8)
|
||||
self.ECM = int(ECN/8)
|
||||
self.ECN = int(ECN/4)
|
||||
elif network.nbNodes < ECN*2:
|
||||
self.ECK = int(ECN/4)
|
||||
self.ECM = int(ECN/4)
|
||||
self.ECN = int(ECN/2)
|
||||
else:
|
||||
self.ECK = int(ECN/2)
|
||||
self.ECM = int(ECN/2)
|
||||
self.ECN = int(ECN)
|
||||
|
||||
def disperse(self, network):
|
||||
contracted = []
|
||||
full = []
|
||||
random.seed(time.time())
|
||||
for blockID in range(self.ECN):
|
||||
stored = 0
|
||||
while(not stored):
|
||||
r = random.randint(0, network.nbNodes-1)
|
||||
if (r not in contracted):
|
||||
if (network.nodes[r].available > self.size/self.ECK):
|
||||
contracted.append(r)
|
||||
stored = 1
|
||||
else:
|
||||
if (r not in full):
|
||||
full.append(r)
|
||||
#print("Not enough space on %i: %f available for %f" % (r, network.nodes[r].available, self.size/self.ECK))
|
||||
else:
|
||||
#print("Node %i already used for this file %s || contracted => %i full => %i" % (r, str(self.ID), len(contracted), len(full)))
|
||||
if (len(contracted) + len(full) >= network.nbNodes):
|
||||
break
|
||||
if len(contracted) == self.ECN:
|
||||
for r in contracted:
|
||||
network.nodes[r].storeBlock(self.ID, blockID, self.size/self.ECK)
|
||||
return 0
|
||||
elif len(contracted) > self.ECN:
|
||||
print("Warning %i contracted, needed %i." % (len(contracted), self.ECN))
|
||||
else:
|
||||
print("Warning %i contracted, needed %i." % (len(contracted), self.ECN))
|
||||
#print("Contract could not be fulfilled, only %i nodes with enough space, %i needed." % (len(contracted), self.ECN))
|
||||
if self.ECN == network.config.defaultECN/4: # Maximum ECN reduction reached
|
||||
network.rejectedFiles = network.rejectedFiles + 1
|
||||
return 0
|
||||
elif self.ECN > network.config.defaultECN/4: # ECN should be reduced
|
||||
return 1
|
||||
else:
|
||||
print("Warning, the ECN should have not reached this low")
|
||||
return 0
|
||||
|
||||
def reduceECN(self):
|
||||
if (self.ECN == network.config.defaultECN) or (self.ECN == network.config.defaultECN/2):
|
||||
self.ECK = self.ECK/2
|
||||
self.ECM = self.ECM/2
|
||||
self.ECN = self.ECN/2
|
||||
|
||||
class Network():
|
||||
def __init__(self, conf):
|
||||
self.nodes = []
|
||||
self.files = []
|
||||
self.nbNodes = 0
|
||||
self.nbFiles = 0
|
||||
self.rejectedFiles = 0
|
||||
self.nbBlocks = 0
|
||||
self.storageProvided = 0
|
||||
self.storageUsed = 0
|
||||
self.storageAvailable = 0
|
||||
self.config = conf
|
||||
now = datetime.now()
|
||||
self.name = "Codex-"+now.strftime("%y-%m-%d-%H-%M-%S")
|
||||
|
||||
def addNode(self):
|
||||
node = Node(self.nbNodes, self.config.maxStoragePerNode)
|
||||
self.nodes.append(node)
|
||||
self.nbNodes = self.nbNodes + 1
|
||||
self.storageProvided = self.storageProvided + node.storage
|
||||
self.storageAvailable = self.storageAvailable + node.storage
|
||||
|
||||
def addFile(self):
|
||||
file = File(self.nbFiles, self)
|
||||
while (file.disperse(self)):
|
||||
file.reduceECN
|
||||
self.files.append(file)
|
||||
self.nbFiles = self.nbFiles + 1
|
||||
self.nbBlocks = self.nbBlocks + file.ECN
|
||||
self.storageUsed = self.storageUsed + ((file.size/file.ECK)*file.ECN)
|
||||
self.storageAvailable = self.storageAvailable - ((file.size/file.ECK)*file.ECN)
|
||||
|
||||
def trackStats(self, sec, tracker):
|
||||
os.system("clear")
|
||||
print("|==================================================================================|")
|
||||
print("| Day | Hour | Nodes | Files | Rejected | Prov.(TBs) | Used(TBs) | Avai.(TBs) |")
|
||||
print("|==================================================================================|")
|
||||
print("| %04i | %02i:00 | %05i | %08i | %06i | %010.4f | %09.3f | %010.3f |" % (sec/(3600*24), (sec/3600)%24, self.nbNodes, self.nbFiles, self.rejectedFiles, self.storageProvided/(1024*1024), self.storageUsed/(1024*1024), self.storageAvailable/(1024*1024)))
|
||||
print("|==================================================================================|")
|
||||
tracker.storageUsed.append(self.storageUsed/(1024*1024))
|
||||
tracker.storageAvailable.append(self.storageAvailable/(1024*1024))
|
||||
tracker.nbNodes.append(self.nbNodes)
|
||||
tracker.nbFiles.append(self.nbFiles/1000000)
|
||||
|
||||
def snapshot(self):
|
||||
print("Creating a snapshot...")
|
||||
nodesDict = []
|
||||
for node in self.nodes:
|
||||
blocksDict = []
|
||||
#for block in node.blocks:
|
||||
# blocksDict.append({"fileID" : block.fileID, "blockID" : block.blockID, "size" : block.size})
|
||||
nodesDict.append({"nodeID" : node.ID, "storage" : node.storage, "used" : node.used, "available" : node.available, "latency": node.latency, "blocks": blocksDict})
|
||||
|
||||
network = {"Name" : "Codex", "Number of nodes" : self.nbNodes, "nodes" : nodesDict}
|
||||
with open(self.name+"/Codex.json", "w") as outfile:
|
||||
json.dump(network, outfile)
|
||||
print("Snapshot Created.")
|
||||
|
||||
def makeReport(self):
|
||||
output = "<html><head><title>Report "+self.name+"</title></head><body>\n"
|
||||
output = output+"<h1>"+self.name+"</h1><br/>\n"
|
||||
output = output+"<img src='nbNodes.png'><br/>\n"
|
||||
output = output+"<img src='nbFiles.png'><br/>\n"
|
||||
output = output+"<img src='storage.png'><br/>\n"
|
||||
output = output+"<img src='storageNodes.png'><br/>\n"
|
||||
output = output+"<img src='fileECN.png'><br/>\n"
|
||||
output = output+"<a href='Codex.json'> JSON snapshot </a><br/>\n"
|
||||
output = output+"</body></html>"
|
||||
with open(self.name+"/index.html", "w") as outfile:
|
||||
outfile.write(output)
|
||||
outfile.close()
|
||||
print("Report html writen.")
|
||||
|
||||
|
||||
def plotData(self, tracker):
|
||||
dir = os.path.join(os.getcwd(), self.name)
|
||||
if not os.path.exists(dir):
|
||||
os.mkdir(dir)
|
||||
used = []
|
||||
avai = []
|
||||
print("Plotting historic data...")
|
||||
self.nodes.sort(key=lambda x: x.used, reverse=True)
|
||||
for node in self.nodes:
|
||||
used.append(node.used/(1024*1024))
|
||||
avai.append(node.available/(1024*1024))
|
||||
print("Network is using %f TBs and has %f TBs available" % (self.storageUsed/(1024*1024), self.storageAvailable/(1024*1024)))
|
||||
print("Network is using %f TBs and has %f TBs available" % (sum(used), sum(avai)))
|
||||
|
||||
ECNstats = [0] * self.config.defaultECN
|
||||
for file in self.files:
|
||||
ECNstats[file.ECN-1] = ECNstats[file.ECN-1] + 1
|
||||
dECN = []
|
||||
lECN = []
|
||||
for i in range(len(ECNstats)):
|
||||
if ECNstats[i] > 0:
|
||||
dECN.append(ECNstats[i])
|
||||
lECN.append(i+1)
|
||||
print("Network has %i files, %i were treated" % (self.nbFiles, sum(dECN)))
|
||||
|
||||
plotHistoricData(tracker.nbNodes, "Hours", "Nb. of nodes", self.name+"/nbNodes.png")
|
||||
plotHistoricData(tracker.nbFiles, "Hours", "Nb. of files (Millions)", self.name+"/nbFiles.png")
|
||||
plotStackedBars(2, [tracker.storageUsed, tracker.storageAvailable], ["Used", "Available"], "Hours", "Storage (TBs)", self.name+"/storage.png")
|
||||
plotStackedBars(2, [used, avai], ["Used", "Available"], "Nodes", "Storage (TBs)", self.name+"/storageNodes.png")
|
||||
plotPieChart(dECN, lECN, self.name+"/fileECN.png")
|
||||
|
||||
self.makeReport()
|
||||
self.snapshot()
|
||||
|
||||
|
||||
def run():
|
||||
|
||||
# Initialization
|
||||
config = Config()
|
||||
network = Network(config)
|
||||
tracker = Tracker()
|
||||
|
||||
# Main Loop
|
||||
print("Starting simulation %s..." % network.name)
|
||||
for sec in range(60*60*24*config.simulationLength): # X days in seconds
|
||||
# Adding storage nodes to the network
|
||||
if (network.nbNodes < (network.config.minNbNodes)): # Bootstrap a number of nodes
|
||||
network.addNode()
|
||||
else:
|
||||
if ((network.storageUsed * 100 / network.storageProvided) > config.meanUsage): # Only add more storage if X% in use already
|
||||
dice = random.random() * 100
|
||||
if (dice < (network.storageUsed * 100 / network.storageProvided)): # Less space avilable more likelihood of incoming nodes
|
||||
network.addNode()
|
||||
|
||||
# Uploading files to the network
|
||||
if (network.storageProvided - network.storageUsed > network.config.maxFileSize) and (network.nbNodes >= network.config.minNbNodes): # if enough space and enough storage providers
|
||||
network.addFile()
|
||||
|
||||
if (sec % 3600 == 0): # Every hour
|
||||
network.trackStats(sec, tracker)
|
||||
print("Simulation %s finished." % network.name)
|
||||
|
||||
# Plot Data
|
||||
# network.plotData(tracker)
|
||||
print("Done.")
|
||||
|
||||
run()
|
||||
Loading…
x
Reference in New Issue
Block a user