diff --git a/simulator/codex.py b/simulator/codex.py new file mode 100644 index 0000000..5522a30 --- /dev/null +++ b/simulator/codex.py @@ -0,0 +1,287 @@ +#!/bin/python3 + + +import os, time, json, hashlib +import random as random +from datetime import datetime +import matplotlib.pyplot as plt + + +def plotHistoricData(data, xLabel, yLabel, filename): + plt.clf() + plt.plot(data) + plt.grid(True) + plt.ylabel(yLabel) + plt.xlabel(xLabel) + plt.savefig(filename) + print("Figure %s created." % filename) + + +def plotStackedBars(nbBars, data, legend, xLabel, yLabel, filename): + plt.clf() + for i in range(nbBars): + if i == 0: + plt.bar(range(len(data[0])), data[0], label=legend[i]) + else: + plt.bar(range(len(data[i])), data[i], label=legend[i], bottom=data[i-1]) + plt.legend(loc="upper left") + plt.grid(True) + plt.ylabel(yLabel) + plt.xlabel(xLabel) + plt.savefig(filename) + print("Figure %s created." % filename) + + +def plotPieChart(data, legend, filename): + plt.clf() + plt.pie(data, labels=legend, autopct='%1.1f%%', shadow=True, startangle=90) + plt.savefig(filename) + plt.axis("equal") + print("Figure %s created." % filename) + + +class Config: + def __init__(self): + self.maxStoragePerNode = 9 # in Terabytes + self.maxFileSize = 111028 # in Megabytes + self.minFileSize = 1 # in Megabytes + self.defaultECN = 128 # in number of blocks + self.minNbNodes = 32 # in nodes + self.simulationLength = 9 # in days + self.meanUsage = 99 # in % + + +class Tracker: + def __init__(self): + self.storageUsed = [] + self.storageAvailable = [] + self.nbNodes = [] + self.nbFiles = [] + + +class Block(): + def __init__(self, fileID, blockID, size): + self.fileID = fileID + self.blockID = blockID + self.size = size + + +class Node(): + def __init__(self, ID, MaxStorageNode): + self.storage = random.random() * MaxStorageNode * 1024 * 1024 # Storage in MBs + self.available = self.storage + self.ID = ID + self.used = 0 + self.latency = random.random() * 100 # Latency score + self.blocks = [] + + def storeBlock(self, fileID, blockID, size): + self.used = self.used + size + self.available = self.storage - self.used + #block = Block(fileID, blockID, size) + #self.blocks.append(block) # Too much memory + + +class File: + def __init__(self, ID, network): + self.ID = ID #hashlib.sha256(str(ID).encode('utf-8')).hexdigest() + self.size = random.random() * network.config.maxFileSize + ECN = network.config.defaultECN + if network.nbNodes < ECN or self.size < 1: + self.ECK = int(ECN/8) + self.ECM = int(ECN/8) + self.ECN = int(ECN/4) + elif network.nbNodes < ECN*2: + self.ECK = int(ECN/4) + self.ECM = int(ECN/4) + self.ECN = int(ECN/2) + else: + self.ECK = int(ECN/2) + self.ECM = int(ECN/2) + self.ECN = int(ECN) + + def disperse(self, network): + contracted = [] + full = [] + random.seed(time.time()) + for blockID in range(self.ECN): + stored = 0 + while(not stored): + r = random.randint(0, network.nbNodes-1) + if (r not in contracted): + if (network.nodes[r].available > self.size/self.ECK): + contracted.append(r) + stored = 1 + else: + if (r not in full): + full.append(r) + #print("Not enough space on %i: %f available for %f" % (r, network.nodes[r].available, self.size/self.ECK)) + else: + #print("Node %i already used for this file %s || contracted => %i full => %i" % (r, str(self.ID), len(contracted), len(full))) + if (len(contracted) + len(full) >= network.nbNodes): + break + if len(contracted) == self.ECN: + for r in contracted: + network.nodes[r].storeBlock(self.ID, blockID, self.size/self.ECK) + return 0 + elif len(contracted) > self.ECN: + print("Warning %i contracted, needed %i." % (len(contracted), self.ECN)) + else: + print("Warning %i contracted, needed %i." % (len(contracted), self.ECN)) + #print("Contract could not be fulfilled, only %i nodes with enough space, %i needed." % (len(contracted), self.ECN)) + if self.ECN == network.config.defaultECN/4: # Maximum ECN reduction reached + network.rejectedFiles = network.rejectedFiles + 1 + return 0 + elif self.ECN > network.config.defaultECN/4: # ECN should be reduced + return 1 + else: + print("Warning, the ECN should have not reached this low") + return 0 + + def reduceECN(self): + if (self.ECN == network.config.defaultECN) or (self.ECN == network.config.defaultECN/2): + self.ECK = self.ECK/2 + self.ECM = self.ECM/2 + self.ECN = self.ECN/2 + +class Network(): + def __init__(self, conf): + self.nodes = [] + self.files = [] + self.nbNodes = 0 + self.nbFiles = 0 + self.rejectedFiles = 0 + self.nbBlocks = 0 + self.storageProvided = 0 + self.storageUsed = 0 + self.storageAvailable = 0 + self.config = conf + now = datetime.now() + self.name = "Codex-"+now.strftime("%y-%m-%d-%H-%M-%S") + + def addNode(self): + node = Node(self.nbNodes, self.config.maxStoragePerNode) + self.nodes.append(node) + self.nbNodes = self.nbNodes + 1 + self.storageProvided = self.storageProvided + node.storage + self.storageAvailable = self.storageAvailable + node.storage + + def addFile(self): + file = File(self.nbFiles, self) + while (file.disperse(self)): + file.reduceECN + self.files.append(file) + self.nbFiles = self.nbFiles + 1 + self.nbBlocks = self.nbBlocks + file.ECN + self.storageUsed = self.storageUsed + ((file.size/file.ECK)*file.ECN) + self.storageAvailable = self.storageAvailable - ((file.size/file.ECK)*file.ECN) + + def trackStats(self, sec, tracker): + os.system("clear") + print("|==================================================================================|") + print("| Day | Hour | Nodes | Files | Rejected | Prov.(TBs) | Used(TBs) | Avai.(TBs) |") + print("|==================================================================================|") + print("| %04i | %02i:00 | %05i | %08i | %06i | %010.4f | %09.3f | %010.3f |" % (sec/(3600*24), (sec/3600)%24, self.nbNodes, self.nbFiles, self.rejectedFiles, self.storageProvided/(1024*1024), self.storageUsed/(1024*1024), self.storageAvailable/(1024*1024))) + print("|==================================================================================|") + tracker.storageUsed.append(self.storageUsed/(1024*1024)) + tracker.storageAvailable.append(self.storageAvailable/(1024*1024)) + tracker.nbNodes.append(self.nbNodes) + tracker.nbFiles.append(self.nbFiles/1000000) + + def snapshot(self): + print("Creating a snapshot...") + nodesDict = [] + for node in self.nodes: + blocksDict = [] + #for block in node.blocks: + # blocksDict.append({"fileID" : block.fileID, "blockID" : block.blockID, "size" : block.size}) + nodesDict.append({"nodeID" : node.ID, "storage" : node.storage, "used" : node.used, "available" : node.available, "latency": node.latency, "blocks": blocksDict}) + + network = {"Name" : "Codex", "Number of nodes" : self.nbNodes, "nodes" : nodesDict} + with open(self.name+"/Codex.json", "w") as outfile: + json.dump(network, outfile) + print("Snapshot Created.") + + def makeReport(self): + output = "Report "+self.name+"\n" + output = output+"

"+self.name+"


\n" + output = output+"
\n" + output = output+"
\n" + output = output+"
\n" + output = output+"
\n" + output = output+"
\n" + output = output+" JSON snapshot
\n" + output = output+"" + with open(self.name+"/index.html", "w") as outfile: + outfile.write(output) + outfile.close() + print("Report html writen.") + + + def plotData(self, tracker): + dir = os.path.join(os.getcwd(), self.name) + if not os.path.exists(dir): + os.mkdir(dir) + used = [] + avai = [] + print("Plotting historic data...") + self.nodes.sort(key=lambda x: x.used, reverse=True) + for node in self.nodes: + used.append(node.used/(1024*1024)) + avai.append(node.available/(1024*1024)) + print("Network is using %f TBs and has %f TBs available" % (self.storageUsed/(1024*1024), self.storageAvailable/(1024*1024))) + print("Network is using %f TBs and has %f TBs available" % (sum(used), sum(avai))) + + ECNstats = [0] * self.config.defaultECN + for file in self.files: + ECNstats[file.ECN-1] = ECNstats[file.ECN-1] + 1 + dECN = [] + lECN = [] + for i in range(len(ECNstats)): + if ECNstats[i] > 0: + dECN.append(ECNstats[i]) + lECN.append(i+1) + print("Network has %i files, %i were treated" % (self.nbFiles, sum(dECN))) + + plotHistoricData(tracker.nbNodes, "Hours", "Nb. of nodes", self.name+"/nbNodes.png") + plotHistoricData(tracker.nbFiles, "Hours", "Nb. of files (Millions)", self.name+"/nbFiles.png") + plotStackedBars(2, [tracker.storageUsed, tracker.storageAvailable], ["Used", "Available"], "Hours", "Storage (TBs)", self.name+"/storage.png") + plotStackedBars(2, [used, avai], ["Used", "Available"], "Nodes", "Storage (TBs)", self.name+"/storageNodes.png") + plotPieChart(dECN, lECN, self.name+"/fileECN.png") + + self.makeReport() + self.snapshot() + + +def run(): + + # Initialization + config = Config() + network = Network(config) + tracker = Tracker() + + # Main Loop + print("Starting simulation %s..." % network.name) + for sec in range(60*60*24*config.simulationLength): # X days in seconds + # Adding storage nodes to the network + if (network.nbNodes < (network.config.minNbNodes)): # Bootstrap a number of nodes + network.addNode() + else: + if ((network.storageUsed * 100 / network.storageProvided) > config.meanUsage): # Only add more storage if X% in use already + dice = random.random() * 100 + if (dice < (network.storageUsed * 100 / network.storageProvided)): # Less space avilable more likelihood of incoming nodes + network.addNode() + + # Uploading files to the network + if (network.storageProvided - network.storageUsed > network.config.maxFileSize) and (network.nbNodes >= network.config.minNbNodes): # if enough space and enough storage providers + network.addFile() + + if (sec % 3600 == 0): # Every hour + network.trackStats(sec, tracker) + print("Simulation %s finished." % network.name) + + # Plot Data + # network.plotData(tracker) + print("Done.") + +run()