simple analyser console tool

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -22,6 +22,24 @@ namespace CodexTests.BasicTests
             testFile.AssertIsEqual(downloadedFile);
         }
 
+        [Test]
+        [CreateTranscript(nameof(SwarmTest))]
+        public void SwarmTest()
+        {
+            var uploader = StartCodex(s => s.WithName("uploader"));
+            var downloaders = StartCodex(5, s => s.WithName("downloader"));
+
+            var file = GenerateTestFile(100.MB());
+            var cid = uploader.UploadFile(file);
+
+            var result = Parallel.ForEach(downloaders, d =>
+            {
+                d.DownloadContent(cid);
+            });
+
+            Assert.That(result.IsCompleted);
+        }
+
         [Test]
         public void DownloadingUnknownCidDoesNotCauseCrash()
         {

Tools/TranscriptAnalysis/DuplicateBlocksReceived.cs

@@ -0,0 +1,80 @@
+using CodexPlugin.OverwatchSupport;
+using Logging;
+using OverwatchTranscript;
+
+namespace TranscriptAnalysis
+{
+    public class DuplicateBlocksReceived
+    {
+        private readonly ILog log;
+
+        public DuplicateBlocksReceived(ILog log, ITranscriptReader reader)
+        {
+            this.log = new LogPrefixer(log, "(DuplicateBlocks) ");
+
+            reader.AddEventHandler<OverwatchCodexEvent>(Handle);
+        }
+
+        public void Finish()
+        {
+            Log("Number of BlockReceived events seen: " + seen);
+
+            var totalReceived = peerIdBlockAddrCount.Sum(a => a.Value.Sum(p => p.Value));
+            var maxRepeats = peerIdBlockAddrCount.Max(a => a.Value.Max(p => p.Value));
+            var occurances = new int[maxRepeats + 1];
+
+            foreach (var peerPair in peerIdBlockAddrCount)
+            {
+                foreach (var pair in peerPair.Value)
+                {
+                    occurances[pair.Value]++;
+                }   
+            }
+
+            float t = totalReceived;
+            for (var i = 1; i < occurances.Length; i++)
+            {
+                float n = occurances[i];
+                float p = 100.0f * (n / t);
+                Log($"Block received {i} times = {occurances[i]}x ({p}%)");
+            }
+        }
+
+        private void Handle(ActivateEvent<OverwatchCodexEvent> obj)
+        {
+            if (obj.Payload.BlockReceived != null)
+            {
+                Handle(obj.Payload, obj.Payload.BlockReceived);
+            }
+        }
+
+        private int seen = 0;
+        private readonly Dictionary<string, Dictionary<string, int>> peerIdBlockAddrCount = new Dictionary<string, Dictionary<string, int>>();
+
+        private void Handle(OverwatchCodexEvent payload, BlockReceivedEvent blockReceived)
+        {
+            var receiverPeerId = payload.PeerId;
+            var blockAddress = blockReceived.BlockAddress;
+            seen++;
+
+            if (!peerIdBlockAddrCount.ContainsKey(receiverPeerId))
+            {
+                peerIdBlockAddrCount.Add(receiverPeerId, new Dictionary<string, int>());
+            }
+            var blockAddCount = peerIdBlockAddrCount[receiverPeerId];
+            if (!blockAddCount.ContainsKey(blockAddress))
+            {
+                blockAddCount.Add(blockAddress, 1);
+            }
+            else
+            {
+                blockAddCount[blockAddress]++;
+            }
+        }
+
+        private void Log(string v)
+        {
+            log.Log(v);
+        }
+    }
+}

Tools/TranscriptAnalysis/Processor.cs

@@ -0,0 +1,50 @@
+using Logging;
+using OverwatchTranscript;
+
+namespace TranscriptAnalysis
+{
+    public class Processor
+    {
+        private readonly ILog log;
+        private readonly ITranscriptReader reader;
+
+        public Processor(ILog log, ITranscriptReader reader)
+        {
+            this.log = new LogPrefixer(log, "(Processor) ");
+            this.reader = reader;
+        }
+
+        public void RunAll()
+        {
+            log.Log("Events: " + reader.Header.NumberOfEvents);
+            log.Log("Moments: " + reader.Header.NumberOfMoments);
+
+            log.Log("Processing...");
+            var count = 0;
+            var tenth = reader.Header.NumberOfMoments / 10;
+            var miss = 0;
+            while (true)
+            {
+                if (!reader.Next())
+                {
+                    miss++;
+                    if (miss > 1000)
+                    {
+                        log.Log("Done");
+                        return;
+                    }
+                    Thread.Sleep(1);
+                }
+                else
+                {
+                    miss = 0;
+                    count++;
+                    if (count % tenth == 0)
+                    {
+                        log.Log($"{count} / {reader.Header.NumberOfMoments}...");
+                    }
+                }
+            }
+        }
+    }
+}

Tools/TranscriptAnalysis/Program.cs

@@ -0,0 +1,80 @@
+using Logging;
+using OverwatchTranscript;
+using TranscriptAnalysis;
+
+public static class Program
+{
+    private static ILog log;
+
+    public static void Main(string[] args)
+    {
+        log = new ConsoleLog();
+        args = new[] { "D:\\Projects\\cs-codex-dist-tests\\Tests\\CodexTests\\bin\\Debug\\net7.0\\CodexTestLogs\\2024-08\\06\\08-24-45Z_ThreeClientTest\\SwarmTest_SwarmTest.owts" };
+
+        Log("Transcript Analysis");
+        if (!args.Any())
+        {
+            Log("Please pass a .owts file");
+            Console.ReadLine();
+            return;
+        }
+
+        if (!File.Exists(args[0]))
+        {
+            Log("File doesn't exist: " + args[0]);
+            Console.ReadLine();
+            return;
+        }
+
+        var reader = OpenReader(args[0]);
+        AppDomain.CurrentDomain.ProcessExit += (e, s) =>
+        {
+            CloseReader(reader);
+        };
+
+        var duplicatesReceived = new DuplicateBlocksReceived(log, reader);
+
+        var processor = new Processor(log, reader);
+        processor.RunAll();
+
+        duplicatesReceived.Finish();
+
+        CloseReader(reader);
+        Log("Done.");
+        Console.ReadLine();
+    }
+
+    private static ITranscriptReader OpenReader(string filepath)
+    {
+        try
+        {
+            Log($"Opening: '{filepath}'");
+            return Transcript.NewReader(filepath);
+        }
+        catch (Exception ex)
+        {
+            Log("Failed to open file for reading: " + ex);
+            Console.ReadLine();
+            Environment.Exit(1);
+            return null;
+        }
+    }
+
+    private static void CloseReader(ITranscriptReader reader)
+    {
+        try
+        {
+            Log("Closing...");
+            reader.Close();
+        }
+        catch (Exception ex)
+        {
+            Log("Failed to close reader: " + ex);
+        }
+    }
+
+    private static void Log(string msg)
+    {
+        log.Log(msg);
+    }
+}

Tools/TranscriptAnalysis/TranscriptAnalysis.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

cs-codex-dist-testing.sln

@@ -72,6 +72,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KeyMaker", "Tools\KeyMaker\
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OverwatchTranscript", "Framework\OverwatchTranscript\OverwatchTranscript.csproj", "{870DDFBE-D7ED-4196-9681-13CA947BDEA6}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools\TranscriptAnalysis\TranscriptAnalysis.csproj", "{C0EEBD32-23CB-45EC-A863-79FB948508C8}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -190,6 +192,10 @@ Global
 		{870DDFBE-D7ED-4196-9681-13CA947BDEA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{870DDFBE-D7ED-4196-9681-13CA947BDEA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{870DDFBE-D7ED-4196-9681-13CA947BDEA6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C0EEBD32-23CB-45EC-A863-79FB948508C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C0EEBD32-23CB-45EC-A863-79FB948508C8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C0EEBD32-23CB-45EC-A863-79FB948508C8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C0EEBD32-23CB-45EC-A863-79FB948508C8}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -223,6 +229,7 @@ Global
 		{73599F9C-98BB-4C6A-9D7D-7C50FBF2993B} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{B57A4789-D8EF-42E0-8D20-581C4057FFD3} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{870DDFBE-D7ED-4196-9681-13CA947BDEA6} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
+		{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}