diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs index 5443e11d..c1df2d67 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -10,7 +10,7 @@ namespace CodexPlugin public class ApiChecker { // - private const string OpenApiYamlHash = "AC-19-7F-3A-88-07-CB-43-53-60-4F-21-3D-A6-B1-53-47-65-07-3B-91-C6-88-B9-76-B2-7E-33-6A-1C-69-F4"; + private const string OpenApiYamlHash = "2E-7C-A2-F3-67-D9-F2-A6-4E-D5-FF-A2-EC-65-ED-59-CE-89-A8-92-57-5E-CF-40-9A-83-49-0B-49-42-5D-EC"; private const string OpenApiFilePath = "/codex/openapi.yaml"; private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK"; diff --git a/ProjectPlugins/CodexPlugin/openapi.yaml b/ProjectPlugins/CodexPlugin/openapi.yaml index bfe60466..c57bddfa 100644 --- a/ProjectPlugins/CodexPlugin/openapi.yaml +++ b/ProjectPlugins/CodexPlugin/openapi.yaml @@ -726,7 +726,7 @@ paths: "503": description: Persistence is not enabled - "/node/spr": + "/spr": get: summary: "Get Node's SPR" operationId: getSPR @@ -744,7 +744,7 @@ paths: "503": description: Node SPR not ready, try again later - "/node/peerid": + "/peerid": get: summary: "Get Node's PeerID" operationId: getPeerId diff --git a/Tools/CsvCombiner/CsvCombiner.csproj b/Tools/CsvCombiner/CsvCombiner.csproj new file mode 100644 index 00000000..b88f5505 --- /dev/null +++ b/Tools/CsvCombiner/CsvCombiner.csproj @@ -0,0 +1,14 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + diff --git a/Tools/CsvCombiner/Program.cs b/Tools/CsvCombiner/Program.cs new file mode 100644 index 00000000..6e71cd07 --- /dev/null +++ b/Tools/CsvCombiner/Program.cs @@ -0,0 +1,71 @@ +using Logging; + +public class Program +{ + public static void Main(string[] args) + { + args = ["d:\\CodexTestLogs\\BlockExchange\\experiment2-fetchbatched"]; + var p = new Program(args[0]); + p.Run(); + } + + private static readonly ILog log = new ConsoleLog(); + private string path; + + private readonly Dictionary> combine = new Dictionary>(); + + public Program(string path) + { + this.path = path; + } + + private void Run() + { + Log("Starting in " + path); + + var files = Directory.GetFiles(path) + .Where(f => f.ToLowerInvariant().EndsWith(".csv")).ToArray(); + + foreach (var file in files) + { + AddToMap(file); + } + + var i = 0; + foreach (var pair in combine) + { + var list = pair.Value; + list.Insert(0, pair.Key); + + File.WriteAllLines(Path.Combine(path, "combine_" + i + ".csv"), list.ToArray()); + i++; + } + + Log("done"); + } + + private void AddToMap(string file) + { + var lines = File.ReadAllLines(file); + if (lines.Length > 1) + { + var header = lines[0]; + var list = GetList(header); + list.AddRange(lines.Skip(1)); + } + } + + private List GetList(string header) + { + if (!combine.ContainsKey(header)) + { + combine.Add(header, new List()); + } + return combine[header]; + } + + private void Log(string msg) + { + log.Log(msg); + } +} \ No newline at end of file diff --git a/Tools/TranscriptAnalysis/CsvWriter.cs b/Tools/TranscriptAnalysis/CsvWriter.cs index ad1d2050..2925f0f3 100644 --- a/Tools/TranscriptAnalysis/CsvWriter.cs +++ b/Tools/TranscriptAnalysis/CsvWriter.cs @@ -1,7 +1,16 @@ -namespace TranscriptAnalysis +using Logging; + +namespace TranscriptAnalysis { public class CsvWriter { + private readonly ILog log; + + public CsvWriter(ILog log) + { + this.log = log; + } + public ICsv CreateNew() { return new Csv(); @@ -14,6 +23,8 @@ using var file = File.OpenWrite(filename); using var writer = new StreamWriter(file); c.CreateLines(writer.WriteLine); + + log.Log($"CSV written to: '{filename}'"); } } diff --git a/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs b/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs index 09251a65..c8ed20db 100644 --- a/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs +++ b/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs @@ -8,13 +8,18 @@ namespace TranscriptAnalysis.Receivers { protected ILog log { get; private set; } = new NullLog(); protected OverwatchCodexHeader Header { get; private set; } = null!; - protected CsvWriter CsvWriter { get; private set; } = new CsvWriter(); + protected CsvWriter CsvWriter { get; private set; } protected string SourceFilename { get; private set; } = string.Empty; public abstract string Name { get; } public abstract void Receive(ActivateEvent @event); public abstract void Finish(); + protected BaseReceiver() + { + CsvWriter = new CsvWriter(log); + } + public void Init(string sourceFilename, ILog log, OverwatchCodexHeader header) { this.log = new LogPrefixer(log, $"({Name}) "); diff --git a/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs b/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs index fb937965..d3ab71a0 100644 --- a/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs +++ b/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs @@ -46,6 +46,7 @@ namespace TranscriptAnalysis.Receivers private readonly Dictionary dialingNodes = new Dictionary(); private readonly Dictionary dials = new Dictionary(); + private long uploadSize; public override string Name => "NodesDegree"; @@ -57,6 +58,11 @@ namespace TranscriptAnalysis.Receivers if (peerId == null) return; AddDial(peerId, @event.Payload.DialSuccessful.TargetPeerId); } + if (@event.Payload.FileUploaded != null) + { + var uploadEvent = @event.Payload.FileUploaded; + uploadSize = uploadEvent.ByteSize; + } } public override void Finish() @@ -84,6 +90,7 @@ namespace TranscriptAnalysis.Receivers float tot = numNodes; csv.GetColumn("numNodes", Header.Nodes.Length); + csv.GetColumn("filesize", uploadSize.ToString()); var degreeColumn = csv.GetColumn("degree", 0.0f); var occuranceColumn = csv.GetColumn("occurance", 0.0f); degreeOccurances.Print((i, count) => diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 6041a7c1..67f63100 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -76,6 +76,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CsvCombiner", "Tools\CsvCombiner\CsvCombiner.csproj", "{6230347F-5045-4E25-8E7A-13D7221B7444}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -202,6 +204,10 @@ Global {004614DF-1C65-45E3-882D-59AE44282573}.Debug|Any CPU.Build.0 = Debug|Any CPU {004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.ActiveCfg = Release|Any CPU {004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.Build.0 = Release|Any CPU + {6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -237,6 +243,7 @@ Global {870DDFBE-D7ED-4196-9681-13CA947BDEA6} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} {C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} {004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} + {6230347F-5045-4E25-8E7A-13D7221B7444} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}