WIP: Added versions of plotters

src/plotting/csvs.py

@@ -0,0 +1,71 @@
+import os
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+from matplotlib import ticker
+
+sns.set_theme()
+
+language = ["Nim", "Rust", "Go"]
+
+data_to_plot = ["Rx", "Tx", "Rp", "Tp", "Rpd", "Tpd"]
+y_label = ["KBytes/s", "KBytes/s", "KPackets/s", "KPackets/s", "Packets/s", "Packets/s"]
+scale = [True, True, False, False, False, False]
+
+folders_grouped = [("../gossipsubdata_2nd/csv/rx/", "../gossipsubdatarust/csv/rx/", "../gossipsubdatago/csv/rx/"),
+                   ("../gossipsubdata_2nd/csv/tx/", "../gossipsubdatarust/csv/tx/", "../gossipsubdatago/csv/tx/"),
+                   ("../gossipsubdata_2nd/csv/rp/", "../gossipsubdatarust/csv/rp/", "../gossipsubdatago/csv/rp/"),
+                   ("../gossipsubdata_2nd/csv/tp/", "../gossipsubdatarust/csv/tp/", "../gossipsubdatago/csv/tp/"),
+                   ("../gossipsubdata_2nd/csv/rpd/", "../gossipsubdatarust/csv/rpd/", "../gossipsubdatago/csv/rpd/"),
+                   ("../gossipsubdata_2nd/csv/tpd/", "../gossipsubdatarust/csv/tpd/", "../gossipsubdatago/csv/tpd/"), ]
+
+# file_data = ["Rx-500B-1.csv", "Rx-1KB-1.csv", "Rx-2.5KB-1.csv", "Rx-10KB-1.csv", "Rx-20KB-1.csv"]
+# file_data = ["Tx-500B-1.csv", "Tx-1KB-1.csv", "Tx-2.5KB-1.csv", "Tx-10KB-1.csv", "Tx-20KB-1.csv"]
+
+fig, axs = plt.subplots(nrows=3, ncols=2, figsize=(14, 16), sharex=True, sharey='row')
+
+for j, group in enumerate(folders_grouped):
+    final_df = pd.DataFrame()
+    for i, folder_path in enumerate(group):
+        if not os.path.exists(folder_path): continue
+        files = [f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))]
+        files = sorted(files, key=lambda x: float(x.split("-")[1].split("KB")[0]))
+        folder_df = pd.DataFrame()
+
+        for file in files:
+            df = pd.read_csv(folder_path + file, parse_dates=['Time'], index_col='Time')
+
+            column_name = file.split("-")[1]
+
+            df_avg = df.mean()
+
+            df_avg_mean = df_avg.median()
+            vertical_offset = df_avg.median() * 0.05  # offset from median for display
+
+            folder_df = pd.concat([folder_df, df_avg.rename(column_name)], axis=1)
+        folder_df["node"] = language[i]
+        final_df = pd.concat([final_df, folder_df])
+
+    final_df = pd.melt(final_df, id_vars=["node"])
+
+    box_plot = sns.boxplot(data=final_df, x="variable", y="value", hue="node", ax=axs[j // 2, j % 2])
+    box_plot.set_title(f'{data_to_plot[j]} (N=300)')
+
+    box_plot.set(xlabel='Payload size (KB)', ylabel=f"{y_label[j]}")
+    box_plot.tick_params(labelbottom=True)
+    # plt.ylabel(f"{y_label[j]}")
+    # plt.xlabel('Payload size (KB)')
+
+    # sns.move_legend(box_plot, "upper left", bbox_to_anchor=(1, 1))
+    # plt.tight_layout()
+
+    if scale[j]:
+        # Create a custom formatter to divide x-axis ticks by 1000
+        formatter = ticker.FuncFormatter(lambda x, pos: '{:.0f}'.format(x / 1000))
+        # Apply the custom formatter to the x-axis ticks
+        box_plot.yaxis.set_major_formatter(formatter)
+
+plt.tight_layout()
+plt.savefig(f"all.png")
+plt.show()
+# box_plot.figure.savefig(f"{data_to_plot[j]}-melted.png")

src/plotting/loads.py

@@ -0,0 +1,37 @@
+import os
+
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+sns.set_theme()
+
+language = ["Nim", "Rust", "Go"]
+
+
+folders_grouped = [("../gossipsubdata_2nd/load/109/", "../gossipsubdatarust/load/109/", "../gossipsubdatago/load/109/"),
+                   ("../gossipsubdata_2nd/load/198/", "../gossipsubdatarust/load/198/", "../gossipsubdatago/load/198/")]
+
+
+for j, group in enumerate(folders_grouped):
+    for i, folder_path in enumerate(group):
+        if not os.path.exists(folder_path): continue
+        files = [f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))]
+        files = sorted(files, key=lambda x: float(x.split("-")[1].split("KB")[0]))
+
+        for file in files:
+            df = pd.read_csv(folder_path + file)
+
+            # get column as list
+            df = df['1m load average'].tolist()
+
+            sns.lineplot(df, label=file.split("-")[1], legend="full")
+
+        plt.xlabel('Time (1min step)')
+        plt.ylabel('uptime')
+        plt.title(f'Nim Loads ({server[j]})')
+        plt.legend(title='Nodes', bbox_to_anchor=(1, 1), loc='upper left')
+        plt.tight_layout()
+
+        plt.savefig(f"{language[i]}-{server[j]}.png")
+        plt.show()

src/plotting/logs.py

@@ -0,0 +1,66 @@
+import os
+import re
+import seaborn as sns
+import matplotlib.pyplot as plt
+import pandas as pd
+
+sns.set_theme()
+
+#data_folders = ["logs-0.5KB-1/", "logs-1KB-1/", "logs-2.5KB-1/", "logs-5KB-1/", "logs-10KB-1/",
+#           "logs-20KB-1/", "logs-40KB-1/"]
+#node_folders = ["../gossipsubdata_2nd/logs/", "../gossipsubdatarust/logs/", "../gossipsubdatago/logs/"]
+#language = ["Nim", "Rust", "Go"]
+data_folders = ["logs-10KB-1/"]
+node_folders = ["../gossipsubdata_2nd/logs/", "../test/logs/"]
+language = ["Before", "After"]
+
+pattern = r'nds:\s[0-9]+'
+
+final_df = pd.DataFrame()
+
+for j, node_folder in enumerate(node_folders):
+    folder_df = pd.DataFrame()
+    for i, data_folder in enumerate(data_folders):
+        file_df = pd.DataFrame()
+        column_name = data_folder.split("-")[1][:-1]
+        files = os.listdir(node_folder + data_folder)
+
+        for log_file in files:
+            values = []
+            with open(node_folder + data_folder + log_file, 'r') as file:
+                for line in file:
+                    match = re.search(pattern, line)
+                    if match:
+                        value = int(match.group().split(":")[1].strip())
+                        values.append(value)
+            file_df = pd.concat([file_df, pd.Series(values)], ignore_index=True)
+
+        file_df = file_df.rename({file_df.columns[0]: column_name}, axis=1)
+        folder_df = pd.concat([folder_df, file_df], axis=1)
+
+    folder_df["node"] = language[j]
+    final_df = pd.concat([final_df, folder_df])
+
+final_df = pd.melt(final_df, id_vars=["node"])
+final_df = final_df.dropna()
+
+box_plot = sns.boxplot(data=final_df, x="variable", y="value", hue="node")
+
+plt.ylabel('Arrival Time (ms)')
+plt.xlabel('Payload size (KB)')
+plt.title('Times (N=300)')
+plt.show()
+
+box_plot.figure.savefig("test.png")
+
+# remove outliers
+final_df = final_df[final_df["value"] < 1000]
+
+box_plot = sns.boxplot(data=final_df, x="variable", y="value", hue="node")
+
+plt.ylabel('Arrival Time (ms)')
+plt.xlabel('Payload size (KB)')
+plt.title('Times (N=300)')
+plt.show()
+
+box_plot.figure.savefig("test_noo.png")

src/plotting/messages.py

@@ -0,0 +1,27 @@
+import os
+import re
+pattern = r'[0-9]+\s[A-Za-z]+:\s[0-9]+'
+
+folder_dir = "../gossipsubdata_2nd/logs/logs-20KB-1/"
+
+
+def parse_file(data, folder):
+    log_files = os.listdir(folder)
+    for log_file in log_files:
+        with open(folder + log_file, 'r') as file:
+            for line in file:
+                match = re.search(pattern, line)
+                if match:
+                    value = int(match.group().split(" ")[0])
+                    if value not in data:
+                        data[value] = [[log_file], 1]
+                    else:
+                        data[value][0].append(log_file)
+                        data[value][1] += 1
+
+
+all_data = {}
+parse_file(all_data, folder_dir)
+
+for key, value in sorted(all_data.items()):
+    print(f"{key}: {value[1]}")