diff --git a/Tools/TranscriptAnalysis/CsvWriter.cs b/Tools/TranscriptAnalysis/CsvWriter.cs new file mode 100644 index 00000000..ad1d2050 --- /dev/null +++ b/Tools/TranscriptAnalysis/CsvWriter.cs @@ -0,0 +1,125 @@ +namespace TranscriptAnalysis +{ + public class CsvWriter + { + public ICsv CreateNew() + { + return new Csv(); + } + + public void Write(ICsv csv, string filename) + { + var c = (Csv)csv; + + using var file = File.OpenWrite(filename); + using var writer = new StreamWriter(file); + c.CreateLines(writer.WriteLine); + } + } + + public interface ICsv + { + ICsvColumn GetColumn(string title, float defaultValue); + ICsvColumn GetColumn(string title, string defaultValue); + void AddRow(params CsvCell[] cells); + } + + public class Csv : ICsv + { + private readonly string Sep = ","; + private readonly List columns = new List(); + private readonly List rows = new List(); + + public ICsvColumn GetColumn(string title, float defaultValue) + { + return GetColumn(title, defaultValue.ToString()); + } + + public ICsvColumn GetColumn(string title, string defaultValue) + { + var column = columns.SingleOrDefault(c => c.Title == title); + if (column == null) + { + column = new CsvColumn(title, defaultValue); + columns.Add(column); + } + return column; + } + + public void AddRow(params CsvCell[] cells) + { + rows.Add(new CsvRow(cells)); + } + + public void CreateLines(Action onLine) + { + CreateHeaderLine(onLine); + foreach (var row in rows) + { + CreateRowLine(row, onLine); + } + } + + private void CreateHeaderLine(Action onLine) + { + onLine(string.Join(Sep, columns.Select(c => c.Title).ToArray())); + } + + private void CreateRowLine(CsvRow row, Action onLine) + { + onLine(string.Join(Sep, columns.Select(c => GetRowCellValue(row, c)).ToArray())); + } + + private string GetRowCellValue(CsvRow row, CsvColumn column) + { + var cell = row.Cells.SingleOrDefault(c => c.Column == column); + if (cell == null) return column.DefaultValue; + return cell.Value; + } + } + + public class CsvCell + { + public CsvCell(ICsvColumn column, float value) + : this(column, value.ToString()) + { + } + + public CsvCell(ICsvColumn column, string value) + { + Column = column; + Value = value; + } + + public ICsvColumn Column { get; } + public string Value { get; } + } + + public interface ICsvColumn + { + string Title { get; } + string DefaultValue { get; } + } + + public class CsvColumn : ICsvColumn + { + public CsvColumn(string title, string defaultValue) + { + Title = title; + DefaultValue = defaultValue; + } + + public string Title { get; } + public string DefaultValue { get; } + } + + public class CsvRow + { + public CsvRow(CsvCell[] cells) + { + Cells = cells; + } + + public CsvCell[] Cells { get; } + } +} diff --git a/Tools/TranscriptAnalysis/Program.cs b/Tools/TranscriptAnalysis/Program.cs index 58cb2a4f..47868c41 100644 --- a/Tools/TranscriptAnalysis/Program.cs +++ b/Tools/TranscriptAnalysis/Program.cs @@ -2,7 +2,6 @@ using Logging; using OverwatchTranscript; using TranscriptAnalysis; -using TranscriptAnalysis.Receivers; public static class Program { @@ -11,77 +10,28 @@ public static class Program public static void Main(string[] args) { Log("Transcript Analysis"); + if (!args.Any()) + { + Log("Please pass a .owts file"); + Console.ReadLine(); + return; + } - var path1 = "d:\\Projects\\cs-codex-dist-tests\\Tests\\CodexTests\\bin\\Debug\\net8.0\\CodexTestLogs\\2024-10\\11\\08-31-52Z_SwarmTests\\"; - var path2 = "d:\\Projects\\cs-codex-dist-tests\\Tests\\CodexTests\\bin\\Debug\\net8.0\\CodexTestLogs\\2024-10\\11\\09-28-29Z_SwarmTests\\"; - var files1 = new[] + if (!File.Exists(args[0])) { - (1, 3, "DetectBlockRetransmits[1,3]_swarm_retransmit.owts"), - (1, 5, "DetectBlockRetransmits[1,5]_swarm_retransmit.owts"), - (1, 10, "DetectBlockRetransmits[1,10]_swarm_retransmit.owts"), - (1, 20, "DetectBlockRetransmits[1,20]_swarm_retransmit.owts"), - (5, 3, "DetectBlockRetransmits[5,3]_swarm_retransmit.owts"), - (5, 5, "DetectBlockRetransmits[5,5]_swarm_retransmit.owts"), - (5, 10, "DetectBlockRetransmits[5,10]_swarm_retransmit.owts"), - (5, 20, "DetectBlockRetransmits[5,20]_swarm_retransmit.owts"), - (10, 5, "DetectBlockRetransmits[10,5]_swarm_retransmit.owts"), - (10, 10, "DetectBlockRetransmits[10,10]_swarm_retransmit.owts") - }; - var files2 = new[] + Log("File doesn't exist: " + args[0]); + Console.ReadLine(); + return; + } + + var reader = OpenReader(args[0]); + AppDomain.CurrentDomain.ProcessExit += (e, s) => { - (10, 20, "DetectBlockRetransmits[10,20]_swarm_retransmit.owts"), - (20, 3, "DetectBlockRetransmits[20,3]_swarm_retransmit.owts"), - (20, 5, "DetectBlockRetransmits[20,5]_swarm_retransmit.owts"), - (20, 10, "DetectBlockRetransmits[20,10]_swarm_retransmit.owts"), - (20, 20, "DetectBlockRetransmits[20,20]_swarm_retransmit.owts") + CloseReader(reader); }; - var countLines = new List(); - - foreach (var file in files1) - { - var path = Path.Combine(path1, file.Item3); - DuplicateBlocksReceived.Counts.Clear(); - Run(path); - - countLines.Add(new[] { file.Item1, file.Item2 }.Concat(DuplicateBlocksReceived.Counts).ToArray()); - } - foreach (var file in files2) - { - var path = Path.Combine(path2, file.Item3); - DuplicateBlocksReceived.Counts.Clear(); - Run(path); - - countLines.Add(new[] { file.Item1, file.Item2 }.Concat(DuplicateBlocksReceived.Counts).ToArray()); - } - - var numColumns = countLines.Max(l => l.Length); - var header = new List() { "filesize", "numNodes" }; - for (var i = 0; i < numColumns - 2; i++) header.Add("recv" + (i + 1) + "x"); - - var lines = new List() { string.Join(",", header.ToArray()) }; - foreach (var count in countLines) - { - var tokens = new List(); - for (var i = 0; i < numColumns; i++) - { - if (i < count.Length) tokens.Add(count[i]); - else tokens.Add(0); - } - lines.Add(string.Join(",", tokens.Select(t => t.ToString()).ToArray())); - } - - File.WriteAllLines("C:\\Users\\Ben\\Desktop\\blockretransmit.csv", lines.ToArray()); - - Log("Done."); - Console.ReadLine(); - } - - private static void Run(string file) - { - var reader = OpenReader(file); var header = reader.GetHeader("cdx_h"); - var receivers = new ReceiverSet(log, reader, header); + var receivers = new ReceiverSet(args[0], log, reader, header); receivers.InitAll(); var processor = new Processor(log, reader); @@ -90,7 +40,8 @@ public static class Program receivers.FinishAll(); CloseReader(reader); - + Log("Done."); + Console.ReadLine(); } private static ITranscriptReader OpenReader(string filepath) diff --git a/Tools/TranscriptAnalysis/ReceiverSet.cs b/Tools/TranscriptAnalysis/ReceiverSet.cs index bdc6c3f6..35c2a700 100644 --- a/Tools/TranscriptAnalysis/ReceiverSet.cs +++ b/Tools/TranscriptAnalysis/ReceiverSet.cs @@ -7,7 +7,7 @@ namespace TranscriptAnalysis { public interface IEventReceiver { - void Init(ILog log, OverwatchCodexHeader header); + void Init(string sourceFilename, ILog log, OverwatchCodexHeader header); void Finish(); } @@ -18,13 +18,15 @@ namespace TranscriptAnalysis public class ReceiverSet { + private readonly string sourceFilename; private readonly ILog log; private readonly ITranscriptReader reader; private readonly OverwatchCodexHeader header; private readonly List receivers = new List(); - public ReceiverSet(ILog log, ITranscriptReader reader, OverwatchCodexHeader header) + public ReceiverSet(string sourceFilename, ILog log, ITranscriptReader reader, OverwatchCodexHeader header) { + this.sourceFilename = sourceFilename; this.log = log; this.reader = reader; this.header = header; @@ -53,7 +55,7 @@ namespace TranscriptAnalysis mux.Add(receiver); receivers.Add(receiver); - receiver.Init(log, header); + receiver.Init(sourceFilename, log, header); } // We use a mux here because, for each time we call reader.AddEventHandler, diff --git a/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs b/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs index 8ab6a617..09251a65 100644 --- a/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs +++ b/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs @@ -8,15 +8,18 @@ namespace TranscriptAnalysis.Receivers { protected ILog log { get; private set; } = new NullLog(); protected OverwatchCodexHeader Header { get; private set; } = null!; + protected CsvWriter CsvWriter { get; private set; } = new CsvWriter(); + protected string SourceFilename { get; private set; } = string.Empty; public abstract string Name { get; } public abstract void Receive(ActivateEvent @event); public abstract void Finish(); - public void Init(ILog log, OverwatchCodexHeader header) + public void Init(string sourceFilename, ILog log, OverwatchCodexHeader header) { this.log = new LogPrefixer(log, $"({Name}) "); Header = header; + SourceFilename = sourceFilename; } protected string? GetPeerId(int nodeIndex) diff --git a/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs b/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs index ed1ef9cf..cbdd71cc 100644 --- a/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs +++ b/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs @@ -6,6 +6,7 @@ namespace TranscriptAnalysis.Receivers public class DuplicateBlocksReceived : BaseReceiver { public static List Counts = new List(); + private long uploadSize; public override string Name => "BlocksReceived"; @@ -15,11 +16,17 @@ namespace TranscriptAnalysis.Receivers { Handle(@event.Payload, @event.Payload.BlockReceived); } + if (@event.Payload.FileUploaded != null) + { + var uploadEvent = @event.Payload.FileUploaded; + uploadSize = uploadEvent.ByteSize; + } } public override void Finish() { Log("Number of BlockReceived events seen: " + seen); + var csv = CsvWriter.CreateNew(); var totalReceived = peerIdBlockAddrCount.Sum(a => a.Value.Sum(p => p.Value)); var maxRepeats = peerIdBlockAddrCount.Max(a => a.Value.Max(p => p.Value)); @@ -36,14 +43,23 @@ namespace TranscriptAnalysis.Receivers if (Counts.Any()) throw new Exception("Should be empty"); float t = totalReceived; + csv.GetColumn("numNodes", Header.Nodes.Length); + csv.GetColumn("filesize", uploadSize.ToString()); + var receiveCountColumn = csv.GetColumn("receiveCount", 0.0f); + var occuranceColumn = csv.GetColumn("occurance", 0.0f); occurances.PrintContinous((i, count) => { float n = count; float p = 100.0f * (n / t); Log($"Block received {i} times = {count}x ({p}%)"); Counts.Add(count); + csv.AddRow( + new CsvCell(receiveCountColumn, i), + new CsvCell(occuranceColumn, count) + ); }); + CsvWriter.Write(csv, SourceFilename + "_blockduplicates.csv"); } private int seen = 0; diff --git a/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs b/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs index 298d8d1a..fb937965 100644 --- a/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs +++ b/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs @@ -61,6 +61,8 @@ namespace TranscriptAnalysis.Receivers public override void Finish() { + var csv = CsvWriter.CreateNew(); + var numNodes = dialingNodes.Count; var redialOccurances = new OccuranceMap(); foreach (var dial in dials.Values) @@ -81,12 +83,21 @@ namespace TranscriptAnalysis.Receivers }); float tot = numNodes; + csv.GetColumn("numNodes", Header.Nodes.Length); + var degreeColumn = csv.GetColumn("degree", 0.0f); + var occuranceColumn = csv.GetColumn("occurance", 0.0f); degreeOccurances.Print((i, count) => { float n = count; float p = 100.0f * (n / tot); Log($"Degree: {i} = {count}x ({p}%)"); + csv.AddRow( + new CsvCell(degreeColumn, i), + new CsvCell(occuranceColumn, n) + ); }); + + CsvWriter.Write(csv, SourceFilename + "_nodeDegrees.csv"); } private void AddDial(string peerId, string targetPeerId)