status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-02-26 23:10:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

writes csv

Browse Source
This commit is contained in:
benbierens 2024-10-17 09:37:15 +02:00
parent 02ca9db001
commit b72b4a850b
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
6 changed files with 179 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
Tools/TranscriptAnalysis/CsvWriter.cs Normal file
View File

@ -0,0 +1,125 @@
namespace TranscriptAnalysis
{
public class CsvWriter
{
public ICsv CreateNew()
{
return new Csv();
}
public void Write(ICsv csv, string filename)
{
var c = (Csv)csv;
using var file = File.OpenWrite(filename);
using var writer = new StreamWriter(file);
c.CreateLines(writer.WriteLine);
}
}
public interface ICsv
{
ICsvColumn GetColumn(string title, float defaultValue);
ICsvColumn GetColumn(string title, string defaultValue);
void AddRow(params CsvCell[] cells);
}
public class Csv : ICsv
{
private readonly string Sep = ",";
private readonly List<CsvColumn> columns = new List<CsvColumn>();
private readonly List<CsvRow> rows = new List<CsvRow>();
public ICsvColumn GetColumn(string title, float defaultValue)
{
return GetColumn(title, defaultValue.ToString());
}
public ICsvColumn GetColumn(string title, string defaultValue)
{
var column = columns.SingleOrDefault(c => c.Title == title);
if (column == null)
{
column = new CsvColumn(title, defaultValue);
columns.Add(column);
}
return column;
}
public void AddRow(params CsvCell[] cells)
{
rows.Add(new CsvRow(cells));
}
public void CreateLines(Action<string> onLine)
{
CreateHeaderLine(onLine);
foreach (var row in rows)
{
CreateRowLine(row, onLine);
}
}
private void CreateHeaderLine(Action<string> onLine)
{
onLine(string.Join(Sep, columns.Select(c => c.Title).ToArray()));
}
private void CreateRowLine(CsvRow row, Action<string> onLine)
{
onLine(string.Join(Sep, columns.Select(c => GetRowCellValue(row, c)).ToArray()));
}
private string GetRowCellValue(CsvRow row, CsvColumn column)
{
var cell = row.Cells.SingleOrDefault(c => c.Column == column);
if (cell == null) return column.DefaultValue;
return cell.Value;
}
}
public class CsvCell
{
public CsvCell(ICsvColumn column, float value)
: this(column, value.ToString())
{
}
public CsvCell(ICsvColumn column, string value)
{
Column = column;
Value = value;
}
public ICsvColumn Column { get; }
public string Value { get; }
}
public interface ICsvColumn
{
string Title { get; }
string DefaultValue { get; }
}
public class CsvColumn : ICsvColumn
{
public CsvColumn(string title, string defaultValue)
{
Title = title;
DefaultValue = defaultValue;
}
public string Title { get; }
public string DefaultValue { get; }
}
public class CsvRow
{
public CsvRow(CsvCell[] cells)
{
Cells = cells;
}
public CsvCell[] Cells { get; }
}
}

85
Tools/TranscriptAnalysis/Program.cs
View File

@ -2,7 +2,6 @@
using Logging;
using OverwatchTranscript;
using TranscriptAnalysis;
using TranscriptAnalysis.Receivers;
public static class Program
{
@ -11,77 +10,28 @@ public static class Program
public static void Main(string[] args)
{
Log("Transcript Analysis");
if (!args.Any())
{
Log("Please pass a .owts file");
Console.ReadLine();
return;
}
var path1 = "d:\\Projects\\cs-codex-dist-tests\\Tests\\CodexTests\\bin\\Debug\\net8.0\\CodexTestLogs\\2024-10\\11\\08-31-52Z_SwarmTests\\";
var path2 = "d:\\Projects\\cs-codex-dist-tests\\Tests\\CodexTests\\bin\\Debug\\net8.0\\CodexTestLogs\\2024-10\\11\\09-28-29Z_SwarmTests\\";
var files1 = new[]
if (!File.Exists(args[0]))
{
(1, 3, "DetectBlockRetransmits[1,3]_swarm_retransmit.owts"),
(1, 5, "DetectBlockRetransmits[1,5]_swarm_retransmit.owts"),
(1, 10, "DetectBlockRetransmits[1,10]_swarm_retransmit.owts"),
(1, 20, "DetectBlockRetransmits[1,20]_swarm_retransmit.owts"),
(5, 3, "DetectBlockRetransmits[5,3]_swarm_retransmit.owts"),
(5, 5, "DetectBlockRetransmits[5,5]_swarm_retransmit.owts"),
(5, 10, "DetectBlockRetransmits[5,10]_swarm_retransmit.owts"),
(5, 20, "DetectBlockRetransmits[5,20]_swarm_retransmit.owts"),
(10, 5, "DetectBlockRetransmits[10,5]_swarm_retransmit.owts"),
(10, 10, "DetectBlockRetransmits[10,10]_swarm_retransmit.owts")
};
var files2 = new[]
Log("File doesn't exist: " + args[0]);
Console.ReadLine();
return;
}
var reader = OpenReader(args[0]);
AppDomain.CurrentDomain.ProcessExit += (e, s) =>
{
(10, 20, "DetectBlockRetransmits[10,20]_swarm_retransmit.owts"),
(20, 3, "DetectBlockRetransmits[20,3]_swarm_retransmit.owts"),
(20, 5, "DetectBlockRetransmits[20,5]_swarm_retransmit.owts"),
(20, 10, "DetectBlockRetransmits[20,10]_swarm_retransmit.owts"),
(20, 20, "DetectBlockRetransmits[20,20]_swarm_retransmit.owts")
CloseReader(reader);
};
var countLines = new List<int[]>();
foreach (var file in files1)
{
var path = Path.Combine(path1, file.Item3);
DuplicateBlocksReceived.Counts.Clear();
Run(path);
countLines.Add(new[] { file.Item1, file.Item2 }.Concat(DuplicateBlocksReceived.Counts).ToArray());
}
foreach (var file in files2)
{
var path = Path.Combine(path2, file.Item3);
DuplicateBlocksReceived.Counts.Clear();
Run(path);
countLines.Add(new[] { file.Item1, file.Item2 }.Concat(DuplicateBlocksReceived.Counts).ToArray());
}
var numColumns = countLines.Max(l => l.Length);
var header = new List<string>() { "filesize", "numNodes" };
for (var i = 0; i < numColumns - 2; i++) header.Add("recv" + (i + 1) + "x");
var lines = new List<string>() { string.Join(",", header.ToArray()) };
foreach (var count in countLines)
{
var tokens = new List<int>();
for (var i = 0; i < numColumns; i++)
{
if (i < count.Length) tokens.Add(count[i]);
else tokens.Add(0);
}
lines.Add(string.Join(",", tokens.Select(t => t.ToString()).ToArray()));
}
File.WriteAllLines("C:\\Users\\Ben\\Desktop\\blockretransmit.csv", lines.ToArray());
Log("Done.");
Console.ReadLine();
}
private static void Run(string file)
{
var reader = OpenReader(file);
var header = reader.GetHeader<OverwatchCodexHeader>("cdx_h");
var receivers = new ReceiverSet(log, reader, header);
var receivers = new ReceiverSet(args[0], log, reader, header);
receivers.InitAll();
var processor = new Processor(log, reader);
@ -90,7 +40,8 @@ public static class Program
receivers.FinishAll();
CloseReader(reader);
Log("Done.");
Console.ReadLine();
}
private static ITranscriptReader OpenReader(string filepath)

8
Tools/TranscriptAnalysis/ReceiverSet.cs
View File

@ -7,7 +7,7 @@ namespace TranscriptAnalysis
{
public interface IEventReceiver
{
void Init(ILog log, OverwatchCodexHeader header);
void Init(string sourceFilename, ILog log, OverwatchCodexHeader header);
void Finish();
}
@ -18,13 +18,15 @@ namespace TranscriptAnalysis
public class ReceiverSet
{
private readonly string sourceFilename;
private readonly ILog log;
private readonly ITranscriptReader reader;
private readonly OverwatchCodexHeader header;
private readonly List<IEventReceiver> receivers = new List<IEventReceiver>();
public ReceiverSet(ILog log, ITranscriptReader reader, OverwatchCodexHeader header)
public ReceiverSet(string sourceFilename, ILog log, ITranscriptReader reader, OverwatchCodexHeader header)
{
this.sourceFilename = sourceFilename;
this.log = log;
this.reader = reader;
this.header = header;
@ -53,7 +55,7 @@ namespace TranscriptAnalysis
mux.Add(receiver);
receivers.Add(receiver);
receiver.Init(log, header);
receiver.Init(sourceFilename, log, header);
}
// We use a mux here because, for each time we call reader.AddEventHandler,

5
Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs
View File

@ -8,15 +8,18 @@ namespace TranscriptAnalysis.Receivers
{
protected ILog log { get; private set; } = new NullLog();
protected OverwatchCodexHeader Header { get; private set; } = null!;
protected CsvWriter CsvWriter { get; private set; } = new CsvWriter();
protected string SourceFilename { get; private set; } = string.Empty;
public abstract string Name { get; }
public abstract void Receive(ActivateEvent<T> @event);
public abstract void Finish();
public void Init(ILog log, OverwatchCodexHeader header)
public void Init(string sourceFilename, ILog log, OverwatchCodexHeader header)
{
this.log = new LogPrefixer(log, $"({Name}) ");
Header = header;
SourceFilename = sourceFilename;
}
protected string? GetPeerId(int nodeIndex)

16
Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs
View File

@ -6,6 +6,7 @@ namespace TranscriptAnalysis.Receivers
public class DuplicateBlocksReceived : BaseReceiver<OverwatchCodexEvent>
{
public static List<int> Counts = new List<int>();
private long uploadSize;
public override string Name => "BlocksReceived";
@ -15,11 +16,17 @@ namespace TranscriptAnalysis.Receivers
{
Handle(@event.Payload, @event.Payload.BlockReceived);
}
if (@event.Payload.FileUploaded != null)
{
var uploadEvent = @event.Payload.FileUploaded;
uploadSize = uploadEvent.ByteSize;
}
}
public override void Finish()
{
Log("Number of BlockReceived events seen: " + seen);
var csv = CsvWriter.CreateNew();
var totalReceived = peerIdBlockAddrCount.Sum(a => a.Value.Sum(p => p.Value));
var maxRepeats = peerIdBlockAddrCount.Max(a => a.Value.Max(p => p.Value));
@ -36,14 +43,23 @@ namespace TranscriptAnalysis.Receivers
if (Counts.Any()) throw new Exception("Should be empty");
float t = totalReceived;
csv.GetColumn("numNodes", Header.Nodes.Length);
csv.GetColumn("filesize", uploadSize.ToString());
var receiveCountColumn = csv.GetColumn("receiveCount", 0.0f);
var occuranceColumn = csv.GetColumn("occurance", 0.0f);
occurances.PrintContinous((i, count) =>
{
float n = count;
float p = 100.0f * (n / t);
Log($"Block received {i} times = {count}x ({p}%)");
Counts.Add(count);
csv.AddRow(
new CsvCell(receiveCountColumn, i),
new CsvCell(occuranceColumn, count)
);
});
CsvWriter.Write(csv, SourceFilename + "_blockduplicates.csv");
}
private int seen = 0;

11
Tools/TranscriptAnalysis/Receivers/NodesDegree.cs
View File

@ -61,6 +61,8 @@ namespace TranscriptAnalysis.Receivers
public override void Finish()
{
var csv = CsvWriter.CreateNew();
var numNodes = dialingNodes.Count;
var redialOccurances = new OccuranceMap();
foreach (var dial in dials.Values)
@ -81,12 +83,21 @@ namespace TranscriptAnalysis.Receivers
});
float tot = numNodes;
csv.GetColumn("numNodes", Header.Nodes.Length);
var degreeColumn = csv.GetColumn("degree", 0.0f);
var occuranceColumn = csv.GetColumn("occurance", 0.0f);
degreeOccurances.Print((i, count) =>
{
float n = count;
float p = 100.0f * (n / tot);
Log($"Degree: {i} = {count}x ({p}%)");
csv.AddRow(
new CsvCell(degreeColumn, i),
new CsvCell(occuranceColumn, n)
);
});
CsvWriter.Write(csv, SourceFilename + "_nodeDegrees.csv");
}
private void AddDial(string peerId, string targetPeerId)