codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

simple analyser console tool

This commit is contained in:
benbierens 2024-08-06 15:56:42 +02:00
parent a72866c725
commit 745b7e43b4
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
6 changed files with 250 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Tests/CodexTests/BasicTests/ThreeClientTest.cs
View File

@ -22,6 +22,24 @@ namespace CodexTests.BasicTests
testFile.AssertIsEqual(downloadedFile);
}
[Test]
[CreateTranscript(nameof(SwarmTest))]
public void SwarmTest()
{
var uploader = StartCodex(s => s.WithName("uploader"));
var downloaders = StartCodex(5, s => s.WithName("downloader"));
var file = GenerateTestFile(100.MB());
var cid = uploader.UploadFile(file);
var result = Parallel.ForEach(downloaders, d =>
{
d.DownloadContent(cid);
});
Assert.That(result.IsCompleted);
}
[Test]
public void DownloadingUnknownCidDoesNotCauseCrash()
{

80
Tools/TranscriptAnalysis/DuplicateBlocksReceived.cs Normal file
View File

@ -0,0 +1,80 @@
using CodexPlugin.OverwatchSupport;
using Logging;
using OverwatchTranscript;
namespace TranscriptAnalysis
{
public class DuplicateBlocksReceived
{
private readonly ILog log;
public DuplicateBlocksReceived(ILog log, ITranscriptReader reader)
{
this.log = new LogPrefixer(log, "(DuplicateBlocks) ");
reader.AddEventHandler<OverwatchCodexEvent>(Handle);
}
public void Finish()
{
Log("Number of BlockReceived events seen: " + seen);
var totalReceived = peerIdBlockAddrCount.Sum(a => a.Value.Sum(p => p.Value));
var maxRepeats = peerIdBlockAddrCount.Max(a => a.Value.Max(p => p.Value));
var occurances = new int[maxRepeats + 1];
foreach (var peerPair in peerIdBlockAddrCount)
{
foreach (var pair in peerPair.Value)
{
occurances[pair.Value]++;
}
}
float t = totalReceived;
for (var i = 1; i < occurances.Length; i++)
{
float n = occurances[i];
float p = 100.0f * (n / t);
Log($"Block received {i} times = {occurances[i]}x ({p}%)");
}
}
private void Handle(ActivateEvent<OverwatchCodexEvent> obj)
{
if (obj.Payload.BlockReceived != null)
{
Handle(obj.Payload, obj.Payload.BlockReceived);
}
}
private int seen = 0;
private readonly Dictionary<string, Dictionary<string, int>> peerIdBlockAddrCount = new Dictionary<string, Dictionary<string, int>>();
private void Handle(OverwatchCodexEvent payload, BlockReceivedEvent blockReceived)
{
var receiverPeerId = payload.PeerId;
var blockAddress = blockReceived.BlockAddress;
seen++;
if (!peerIdBlockAddrCount.ContainsKey(receiverPeerId))
{
peerIdBlockAddrCount.Add(receiverPeerId, new Dictionary<string, int>());
}
var blockAddCount = peerIdBlockAddrCount[receiverPeerId];
if (!blockAddCount.ContainsKey(blockAddress))
{
blockAddCount.Add(blockAddress, 1);
}
else
{
blockAddCount[blockAddress]++;
}
}
private void Log(string v)
{
log.Log(v);
}
}
}

50
Tools/TranscriptAnalysis/Processor.cs Normal file
View File

@ -0,0 +1,50 @@
using Logging;
using OverwatchTranscript;
namespace TranscriptAnalysis
{
public class Processor
{
private readonly ILog log;
private readonly ITranscriptReader reader;
public Processor(ILog log, ITranscriptReader reader)
{
this.log = new LogPrefixer(log, "(Processor) ");
this.reader = reader;
}
public void RunAll()
{
log.Log("Events: " + reader.Header.NumberOfEvents);
log.Log("Moments: " + reader.Header.NumberOfMoments);
log.Log("Processing...");
var count = 0;
var tenth = reader.Header.NumberOfMoments / 10;
var miss = 0;
while (true)
{
if (!reader.Next())
{
miss++;
if (miss > 1000)
{
log.Log("Done");
return;
}
Thread.Sleep(1);
}
else
{
miss = 0;
count++;
if (count % tenth == 0)
{
log.Log($"{count} / {reader.Header.NumberOfMoments}...");
}
}
}
}
}
}

80
Tools/TranscriptAnalysis/Program.cs Normal file
View File

@ -0,0 +1,80 @@
using Logging;
using OverwatchTranscript;
using TranscriptAnalysis;
public static class Program
{
private static ILog log;
public static void Main(string[] args)
{
log = new ConsoleLog();
args = new[] { "D:\\Projects\\cs-codex-dist-tests\\Tests\\CodexTests\\bin\\Debug\\net7.0\\CodexTestLogs\\2024-08\\06\\08-24-45Z_ThreeClientTest\\SwarmTest_SwarmTest.owts" };
Log("Transcript Analysis");
if (!args.Any())
{
Log("Please pass a .owts file");
Console.ReadLine();
return;
}
if (!File.Exists(args[0]))
{
Log("File doesn't exist: " + args[0]);
Console.ReadLine();
return;
}
var reader = OpenReader(args[0]);
AppDomain.CurrentDomain.ProcessExit += (e, s) =>
{
CloseReader(reader);
};
var duplicatesReceived = new DuplicateBlocksReceived(log, reader);
var processor = new Processor(log, reader);
processor.RunAll();
duplicatesReceived.Finish();
CloseReader(reader);
Log("Done.");
Console.ReadLine();
}
private static ITranscriptReader OpenReader(string filepath)
{
try
{
Log($"Opening: '{filepath}'");
return Transcript.NewReader(filepath);
}
catch (Exception ex)
{
Log("Failed to open file for reading: " + ex);
Console.ReadLine();
Environment.Exit(1);
return null;
}
}
private static void CloseReader(ITranscriptReader reader)
{
try
{
Log("Closing...");
reader.Close();
}
catch (Exception ex)
{
Log("Failed to close reader: " + ex);
}
}
private static void Log(string msg)
{
log.Log(msg);
}
}

15
Tools/TranscriptAnalysis/TranscriptAnalysis.csproj Normal file
View File

@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
<ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
</ItemGroup>
</Project>

7
cs-codex-dist-testing.sln
View File

@ -72,6 +72,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KeyMaker", "Tools\KeyMaker\
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OverwatchTranscript", "Framework\OverwatchTranscript\OverwatchTranscript.csproj", "{870DDFBE-D7ED-4196-9681-13CA947BDEA6}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools\TranscriptAnalysis\TranscriptAnalysis.csproj", "{C0EEBD32-23CB-45EC-A863-79FB948508C8}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -190,6 +192,10 @@ Global
{870DDFBE-D7ED-4196-9681-13CA947BDEA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{870DDFBE-D7ED-4196-9681-13CA947BDEA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{870DDFBE-D7ED-4196-9681-13CA947BDEA6}.Release|Any CPU.Build.0 = Release|Any CPU
{C0EEBD32-23CB-45EC-A863-79FB948508C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C0EEBD32-23CB-45EC-A863-79FB948508C8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C0EEBD32-23CB-45EC-A863-79FB948508C8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C0EEBD32-23CB-45EC-A863-79FB948508C8}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -223,6 +229,7 @@ Global
{73599F9C-98BB-4C6A-9D7D-7C50FBF2993B} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
{B57A4789-D8EF-42E0-8D20-581C4057FFD3} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
{870DDFBE-D7ED-4196-9681-13CA947BDEA6} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}