diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs index 5e4e5d2..19a100d 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs @@ -8,66 +8,20 @@ namespace CodexPlugin.OverwatchSupport public class CodexLogConverter { private readonly ITranscriptWriter writer; - private readonly NameIdMap nameIdMap; + private readonly IdentityMap identityMap; - public CodexLogConverter(ITranscriptWriter writer, NameIdMap nameIdMap) + public CodexLogConverter(ITranscriptWriter writer, IdentityMap identityMap) { this.writer = writer; - this.nameIdMap = nameIdMap; + this.identityMap = identityMap; } public void ProcessLog(IDownloadedLog log) - { - var peerId = GetIdentity(log); - var runner = new ConversionRunner(writer, nameIdMap, log.ContainerName, peerId); - runner.Run(log); - } - - private CodexNodeIdentity GetIdentity(IDownloadedLog log) { var name = DetermineName(log); - - // We have to use a look-up map to match the node name to its peerId and nodeId, - // because the Codex logging never prints the id in full. - // After we find it, we confirm it be looking for the shortened version. - var peerId = DeterminPeerId(name, log); - var nodeId = DeterminNodeId(name, log); - - return new CodexNodeIdentity - { - PeerId = peerId, - NodeId = nodeId - }; - } - - private string DeterminNodeId(string name, IDownloadedLog log) - { - var nodeId = nameIdMap.GetId(name).NodeId; - var shortNodeId = CodexUtils.ToNodeIdShortId(nodeId); - - // Look for "Starting discovery node" line to confirm nodeId. - var startedLine = log.FindLinesThatContain("Starting discovery node").Single(); - var started = CodexLogLine.Parse(startedLine)!; - var foundId = started.Attributes["node"]; - - if (foundId != shortNodeId) throw new Exception("NodeId from name-lookup did not match NodeId found in codex-started log line."); - - return nodeId; - } - - private string DeterminPeerId(string name, IDownloadedLog log) - { - var peerId = nameIdMap.GetId(name).PeerId; - var shortPeerId = CodexUtils.ToShortId(peerId); - - // Look for "Started codex node" line to confirm peerId. - var startedLine = log.FindLinesThatContain("Started codex node").Single(); - var started = CodexLogLine.Parse(startedLine)!; - var foundId = started.Attributes["id"]; - - if (foundId != shortPeerId) throw new Exception("PeerId from name-lookup did not match PeerId found in codex-started log line."); - - return peerId; + var identityIndex = identityMap.GetIndex(name); + var runner = new ConversionRunner(writer, identityMap, log.ContainerName, identityIndex); + runner.Run(log); } private string DetermineName(IDownloadedLog log) @@ -82,9 +36,9 @@ namespace CodexPlugin.OverwatchSupport public class ConversionRunner { private readonly ITranscriptWriter writer; - private readonly NameIdMap nameIdMap; + private readonly IdentityMap nameIdMap; private readonly string name; - private readonly CodexNodeIdentity nodeIdentity; + private readonly int nodeIdentityIndex; private readonly ILineConverter[] converters = new ILineConverter[] { new BlockReceivedLineConverter(), @@ -93,10 +47,10 @@ namespace CodexPlugin.OverwatchSupport new PeerDroppedLineConverter() }; - public ConversionRunner(ITranscriptWriter writer, NameIdMap nameIdMap, string name, CodexNodeIdentity nodeIdentity) + public ConversionRunner(ITranscriptWriter writer, IdentityMap nameIdMap, string name, int nodeIdentityIndex) { this.name = name; - this.nodeIdentity = nodeIdentity; + this.nodeIdentityIndex = nodeIdentityIndex; this.writer = writer; this.nameIdMap = nameIdMap; } @@ -116,8 +70,7 @@ namespace CodexPlugin.OverwatchSupport { var e = new OverwatchCodexEvent { - Name = name, - Identity = nodeIdentity, + NodeIdentity = nodeIdentityIndex, }; action(e); diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs index 7d456a6..971cbdc 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs @@ -7,15 +7,15 @@ namespace CodexPlugin.OverwatchSupport public class CodexNodeTranscriptWriter : ICodexNodeHooks { private readonly ITranscriptWriter writer; - private readonly NameIdMap nameIdMap; + private readonly IdentityMap identityMap; private readonly string name; - private CodexNodeIdentity identity = new CodexNodeIdentity(); + private int identityIndex = -1; private readonly List<(DateTime, OverwatchCodexEvent)> pendingEvents = new List<(DateTime, OverwatchCodexEvent)>(); - public CodexNodeTranscriptWriter(ITranscriptWriter writer, NameIdMap nameIdMap, string name) + public CodexNodeTranscriptWriter(ITranscriptWriter writer, IdentityMap identityMap, string name) { this.writer = writer; - this.nameIdMap = nameIdMap; + this.identityMap = identityMap; this.name = name; } @@ -32,19 +32,13 @@ namespace CodexPlugin.OverwatchSupport public void OnNodeStarted(string peerId, string nodeId) { - identity.PeerId = peerId; - identity.NodeId = nodeId; - if (string.IsNullOrEmpty(peerId) || string.IsNullOrEmpty(nodeId)) { throw new Exception("Node started - peerId and/or nodeId unknown."); } - nameIdMap.Add(name, new CodexNodeIdentity - { - PeerId = peerId, - NodeId = nodeId - }); + identityMap.Add(name, peerId, nodeId); + identityIndex = identityMap.GetIndex(name); WriteCodexEvent(e => { @@ -121,22 +115,21 @@ namespace CodexPlugin.OverwatchSupport { var e = new OverwatchCodexEvent { - Name = name, - Identity = identity + NodeIdentity = identityIndex }; action(e); - if (string.IsNullOrEmpty(identity.PeerId)) + if (identityIndex < 0) { - // If we don't know our peerId, don't write the events yet. + // If we don't know our id, don't write the events yet. AddToCache(utc, e); } else { e.Write(utc, writer); - // Write any events that we cached when we didn't have our peerId yet. + // Write any events that we cached when we didn't have our id yet. WriteAndClearCache(); } } @@ -152,7 +145,7 @@ namespace CodexPlugin.OverwatchSupport { foreach (var pair in pendingEvents) { - pair.Item2.Identity = identity; + pair.Item2.NodeIdentity = identityIndex; pair.Item2.Write(pair.Item1, writer); } pendingEvents.Clear(); diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs index 4645c71..8fe2549 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs @@ -12,13 +12,13 @@ namespace CodexPlugin.OverwatchSupport private readonly ILog log; private readonly ITranscriptWriter writer; private readonly CodexLogConverter converter; - private readonly NameIdMap nameIdMap = new NameIdMap(); + private readonly IdentityMap identityMap = new IdentityMap(); public CodexTranscriptWriter(ILog log, ITranscriptWriter transcriptWriter) { this.log = log; writer = transcriptWriter; - converter = new CodexLogConverter(writer, nameIdMap); + converter = new CodexLogConverter(writer, identityMap); } public void Finalize(string outputFilepath) @@ -34,7 +34,7 @@ namespace CodexPlugin.OverwatchSupport public ICodexNodeHooks CreateHooks(string nodeName) { nodeName = Str.Between(nodeName, "'", "'"); - return new CodexNodeTranscriptWriter(writer, nameIdMap, nodeName); + return new CodexNodeTranscriptWriter(writer, identityMap, nodeName); } public void IncludeFile(string filepath) @@ -63,8 +63,7 @@ namespace CodexPlugin.OverwatchSupport { writer.Add(DateTime.UtcNow, new OverwatchCodexEvent { - Name = string.Empty, - Identity = new CodexNodeIdentity(), + NodeIdentity = -1, ScenarioFinished = new ScenarioFinishedEvent { Success = success, @@ -77,7 +76,7 @@ namespace CodexPlugin.OverwatchSupport { return new OverwatchCodexHeader { - TotalNumberOfNodes = nameIdMap.Size + Nodes = identityMap.Get() }; } diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/IdentityMap.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/IdentityMap.cs new file mode 100644 index 0000000..d6ca184 --- /dev/null +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/IdentityMap.cs @@ -0,0 +1,63 @@ +namespace CodexPlugin.OverwatchSupport +{ + public class IdentityMap + { + private readonly List nodes = new List(); + private readonly Dictionary nameIndexMap = new Dictionary(); + private readonly Dictionary shortToLong = new Dictionary(); + + public void Add(string name, string peerId, string nodeId) + { + Add(new CodexNodeIdentity + { + Name = name, + PeerId = peerId, + NodeId = nodeId + }); + + nameIndexMap.Add(name, nameIndexMap.Count); + } + + public void Add(CodexNodeIdentity identity) + { + if (string.IsNullOrWhiteSpace(identity.Name)) throw new Exception("Name required"); + if (string.IsNullOrWhiteSpace(identity.PeerId) || identity.PeerId.Length < 11) throw new Exception("PeerId invalid"); + if (string.IsNullOrWhiteSpace(identity.NodeId) || identity.NodeId.Length < 11) throw new Exception("NodeId invalid"); + + nodes.Add(identity); + + shortToLong.Add(CodexUtils.ToShortId(identity.PeerId), identity.PeerId); + shortToLong.Add(CodexUtils.ToShortId(identity.NodeId), identity.NodeId); + } + + public CodexNodeIdentity[] Get() + { + return nodes.ToArray(); + } + + public int GetIndex(string name) + { + return nameIndexMap[name]; + } + + public CodexNodeIdentity GetId(string name) + { + return nodes.Single(n => n.Name == name); + } + + public string ReplaceShortIds(string value) + { + var result = value; + foreach (var pair in shortToLong) + { + result = result.Replace(pair.Key, pair.Value); + } + return result; + } + + public int Size + { + get { return nodes.Count; } + } + } +} diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs index 1bf4fbf..9503cb9 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs @@ -5,14 +5,13 @@ namespace CodexPlugin.OverwatchSupport [Serializable] public class OverwatchCodexHeader { - public int TotalNumberOfNodes { get; set; } + public CodexNodeIdentity[] Nodes { get; set; } = Array.Empty(); } [Serializable] public class OverwatchCodexEvent { - public string Name { get; set; } = string.Empty; - public CodexNodeIdentity Identity { get; set; } = new(); + public int NodeIdentity { get; set; } = -1; public ScenarioFinishedEvent? ScenarioFinished { get; set; } public NodeStartingEvent? NodeStarting { get; set; } public NodeStartedEvent? NodeStarted { get; set; } @@ -28,9 +27,10 @@ namespace CodexPlugin.OverwatchSupport public void Write(DateTime utc, ITranscriptWriter writer) { - if (string.IsNullOrWhiteSpace(Name)) throw new Exception("Name required"); - if (string.IsNullOrWhiteSpace(Identity.PeerId) || Identity.PeerId.Length < 11) throw new Exception("PeerId invalid"); - if (string.IsNullOrWhiteSpace(Identity.NodeId) || Identity.NodeId.Length < 11) throw new Exception("NodeId invalid"); + if (NodeIdentity == -1 && ScenarioFinished == null) + { + throw new Exception("NodeIdentity not set, and event is not ScenarioFinished."); + } if (AllNull()) throw new Exception("No event data was set"); writer.Add(utc, this); @@ -49,6 +49,7 @@ namespace CodexPlugin.OverwatchSupport [Serializable] public class CodexNodeIdentity { + public string Name { get; set; } = string.Empty; public string PeerId { get; set; } = string.Empty; public string NodeId { get; set; } = string.Empty; } diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs deleted file mode 100644 index 55e3320..0000000 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs +++ /dev/null @@ -1,36 +0,0 @@ -namespace CodexPlugin.OverwatchSupport -{ - public class NameIdMap - { - private readonly Dictionary map = new Dictionary(); - private readonly Dictionary shortToLong = new Dictionary(); - - public void Add(string name, CodexNodeIdentity identity) - { - map.Add(name, identity); - - shortToLong.Add(CodexUtils.ToShortId(identity.PeerId), identity.PeerId); - shortToLong.Add(CodexUtils.ToShortId(identity.NodeId), identity.NodeId); - } - - public CodexNodeIdentity GetId(string name) - { - return map[name]; - } - - public string ReplaceShortIds(string value) - { - var result = value; - foreach (var pair in shortToLong) - { - result = result.Replace(pair.Key, pair.Value); - } - return result; - } - - public int Size - { - get { return map.Count; } - } - } -} diff --git a/Tests/DistTestCore/Configuration.cs b/Tests/DistTestCore/Configuration.cs index 3fa34b5..e7e5082 100644 --- a/Tests/DistTestCore/Configuration.cs +++ b/Tests/DistTestCore/Configuration.cs @@ -14,7 +14,7 @@ namespace DistTestCore kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null); logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs"); dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); - AlwaysDownloadContainerLogs = !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", "")); + AlwaysDownloadContainerLogs = true; // !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", "")); } public Configuration(string? kubeConfigFile, string logPath, string dataFilesPath)