Merge branch 'visual'

New visualizer module with updated plots, and plotting after each run.

DAS/simulator.py

@@ -266,7 +266,7 @@ class Simulator:
             trafficStatsVector.append(trafficStats)
 
             missingSamples, sampleProgress, nodeProgress, validatorAllProgress, validatorProgress = self.glob.getProgress(self.validators)
-            self.logger.debug("step %d, arrived %0.02f %%, ready %0.02f %%, validatedall %0.02f %%, , validated %0.02f %%"
+            self.logger.info("step %d, arrived %0.02f %%, ready %0.02f %%, validatedall %0.02f %%, , validated %0.02f %%"
                               % (steps, sampleProgress*100, nodeProgress*100, validatorAllProgress*100, validatorProgress*100), extra=self.format)
 
             cnS = "samples received"

DAS/visualizer.py

@@ -4,6 +4,7 @@ import time
 import xml.etree.ElementTree as ET
 import matplotlib.pyplot as plt
 import numpy as np
+import pandas as pd
 import seaborn as sns
 from itertools import combinations
 from mplfinance.original_flavor import candlestick_ohlc
@@ -197,10 +198,9 @@ class Visualizer:
                 ylabels = np.sort(np.unique(data[key][labels[1]]))
                 if len(xlabels) < self.minimumDataPoints or len(ylabels) < self.minimumDataPoints:
                     continue
-                hist, xedges, yedges = np.histogram2d(data[key][labels[0]], data[key][labels[1]], bins=(len(xlabels), len(ylabels)), weights=data[key]['ttas'])
+                df = pd.DataFrame.from_dict(data[key]).pivot(columns=labels[0], index=labels[1], values='ttas')
-                hist = hist.T
                 fig, ax = plt.subplots(figsize=(10, 6))
-                sns.heatmap(hist, xticklabels=xlabels, yticklabels=ylabels, cmap='hot_r', cbar_kws={'label': 'Time to block availability (ms)'}, linecolor='black', linewidths=0.3, annot=True, fmt=".2f", ax=ax, vmin=vmin, vmax=vmax)
+                sns.heatmap(df, cmap='hot_r', cbar_kws={'label': 'Time to block availability (ms)'}, linecolor='black', linewidths=0.3, annot=True, fmt=".2f", ax=ax) #vmin=vmin, vmax=vmax
                 plt.xlabel(self.formatLabel(labels[0]))
                 plt.ylabel(self.formatLabel(labels[1]))
                 filename = ""

DAS/visualizor.py

@@ -43,17 +43,46 @@ class Visualizor:
             d[text[i]] = text[i + 1]
         return d
 
+    def plotHeatmaps(self, x, y):
+        """Plot the heatmap using the parameters given as x axis and y axis"""
+        print("Plotting heatmap "+x+" vs "+y)
+        #Find the location of x in shape
+        #Find the location of y in shape
+        #Find the location od r in shape
+
+        #Loop over all results
+            #Add unique values foir every parameter
+
+        #Find number of runs from r
+        #If number of values for x and y > 3 then plot heatmap, otherwise finish
+
+        #Create a 2D grid with the dimensions of the number of values for x and y
+        #For all values of x
+            #For all values of y
+                # For all values in r
+                    #Fixing all other values to 1 (in the mean time)
+                    #Add/sum TTA into 2D grid
+                    #if last r divide by number of runs
+
+        #Plot 2D grid
+
+
+
+
     def plotAll(self):
         """Plot all the important elements of each result"""
         for result in self.results:
-            self.plotMissingSamples(result)
+            plotPath = "results/"+self.execID+"/plots/"+str(result.shape)
-            self.plotProgress(result)
+            os.makedirs(plotPath, exist_ok=True)
-            self.plotSentData(result)
+            self.plotMissingSamples(result, plotPath)
-            self.plotRecvData(result)
+            self.plotProgress(result, plotPath)
-            self.plotDupData(result)
+            self.plotSentData(result, plotPath)
-            self.plotRowCol(result)
+            self.plotRecvData(result, plotPath)
+            self.plotDupData(result, plotPath)
+            if self.config.saveRCdist:
+                self.plotRowCol(result, plotPath)
 
-    def plotMissingSamples(self, result):
+    def plotMissingSamples(self, result, plotPath):
         """Plots the missing samples in the network"""
         conf = {}
         attrbs = self.__get_attrbs__(result)
@@ -69,7 +98,7 @@ class Visualizor:
         conf["ylabel"] = "Number of Missing Samples"
         conf["data"] = [result.missingVector]
         conf["xdots"] = [x*self.config.stepDuration for x in range(len(result.missingVector))]
-        conf["path"] = "results/"+self.execID+"/plots/missingSamples-"+str(result.shape)+".png"
+        conf["path"] = plotPath+"/missingSamples.png"
         maxi = 0
         for v in conf["data"]:
             if max(v) > maxi:
@@ -78,7 +107,7 @@ class Visualizor:
         plotData(conf)
         print("Plot %s created." % conf["path"])
 
-    def plotProgress(self, result):
+    def plotProgress(self, result, plotPath):
         """Plots the percentage of nodes ready in the network"""
         vector1 = result.metrics["progress"]["nodes ready"]
         vector2 = result.metrics["progress"]["validators ready"]
@@ -97,7 +126,7 @@ class Visualizor:
         conf["ylabel"] = "Percentage (%)"
         conf["data"] = [vector1, vector2, vector3]
         conf["xdots"] = [x*self.config.stepDuration for x in range(len(vector1))]
-        conf["path"] = "results/"+self.execID+"/plots/nodesReady-"+str(result.shape)+".png"
+        conf["path"] = plotPath+"/nodesReady.png"
         maxi = 0
         for v in conf["data"]:
             if max(v) > maxi:
@@ -106,7 +135,7 @@ class Visualizor:
         plotData(conf)
         print("Plot %s created." % conf["path"])
 
-    def plotSentData(self, result):
+    def plotSentData(self, result, plotPath):
         """Plots the percentage of nodes ready in the network"""
         vector1 = result.metrics["progress"]["TX builder mean"]
         vector2 = result.metrics["progress"]["TX class1 mean"]
@@ -129,7 +158,7 @@ class Visualizor:
         conf["ylabel"] = "Bandwidth (MBits/s)"
         conf["data"] = [vector1, vector2, vector3]
         conf["xdots"] = [x*self.config.stepDuration for x in range(len(vector1))]
-        conf["path"] = "results/"+self.execID+"/plots/sentData-"+str(result.shape)+".png"
+        conf["path"] = plotPath+"/sentData.png"
         maxi = 0
         for v in conf["data"]:
             if max(v) > maxi:
@@ -138,7 +167,7 @@ class Visualizor:
         plotData(conf)
         print("Plot %s created." % conf["path"])
 
-    def plotRecvData(self, result):
+    def plotRecvData(self, result, plotPath):
         """Plots the percentage of nodes ready in the network"""
         vector1 = result.metrics["progress"]["RX class1 mean"]
         vector2 = result.metrics["progress"]["RX class2 mean"]
@@ -159,7 +188,7 @@ class Visualizor:
         conf["ylabel"] = "Bandwidth (MBits/s)"
         conf["data"] = [vector1, vector2]
         conf["xdots"] = [x*self.config.stepDuration for x in range(len(vector1))]
-        conf["path"] = "results/"+self.execID+"/plots/recvData-"+str(result.shape)+".png"
+        conf["path"] = plotPath+"/recvData.png"
         maxi = 0
         for v in conf["data"]:
             if max(v) > maxi:
@@ -168,7 +197,7 @@ class Visualizor:
         plotData(conf)
         print("Plot %s created." % conf["path"])
 
-    def plotDupData(self, result):
+    def plotDupData(self, result, plotPath):
         """Plots the percentage of nodes ready in the network"""
         vector1 = result.metrics["progress"]["Dup class1 mean"]
         vector2 = result.metrics["progress"]["Dup class2 mean"]
@@ -189,7 +218,7 @@ class Visualizor:
         conf["ylabel"] = "Bandwidth (MBits/s)"
         conf["data"] = [vector1, vector2]
         conf["xdots"] = [x*self.config.stepDuration for x in range(len(vector1))]
-        conf["path"] = "results/"+self.execID+"/plots/dupData-"+str(result.shape)+".png"
+        conf["path"] = plotPath+"/dupData.png"
         maxi = 0
         for v in conf["data"]:
             if max(v) > maxi:
@@ -198,7 +227,7 @@ class Visualizor:
         plotData(conf)
         print("Plot %s created." % conf["path"])
 
-    def plotRowCol(self, result):
+    def plotRowCol(self, result, plotPath):
         """Plots the percentage of nodes ready in the network"""
         vector1 = result.metrics["rowDist"]
         vector2 = result.metrics["columnDist"]
@@ -220,7 +249,7 @@ class Visualizor:
         conf["ylabel"] = "Validators subscribed"
         conf["data"] = [vector1, vector2]
         conf["xdots"] = range(len(vector1))
-        conf["path"] = "results/"+self.execID+"/plots/RowColDist-"+str(result.shape)+".png"
+        conf["path"] = plotPath+"/RowColDist.png"
         maxi = 0
         for v in conf["data"]:
             if np.nanmax(v) > maxi:

study.py

@@ -39,6 +39,10 @@ def runOnce(config, shape, execID):
     if config.dumpXML:
         result.dump()
 
+    if config.visualization:
+        visual = Visualizor(execID, config, [result])
+        visual.plotAll()
+
     return result
 
 def study():
@@ -88,7 +92,7 @@ def study():
         vis.plotHeatmaps()
 
         visual = Visualizor(execID, config, results)
-        visual.plotAll()
+        visual.plotHeatmaps("nn", "fr")
 
 if __name__ == "__main__":
     study()