wip: rework identity map

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs

@@ -8,66 +8,20 @@ namespace CodexPlugin.OverwatchSupport
     public class CodexLogConverter
     {
         private readonly ITranscriptWriter writer;
-        private readonly NameIdMap nameIdMap;
+        private readonly IdentityMap identityMap;
 
-        public CodexLogConverter(ITranscriptWriter writer, NameIdMap nameIdMap)
+        public CodexLogConverter(ITranscriptWriter writer, IdentityMap identityMap)
         {
             this.writer = writer;
-            this.nameIdMap = nameIdMap;
+            this.identityMap = identityMap;
         }
 
         public void ProcessLog(IDownloadedLog log)
-        {
-            var peerId = GetIdentity(log);
-            var runner = new ConversionRunner(writer, nameIdMap, log.ContainerName, peerId);
-            runner.Run(log);
-        }
-
-        private CodexNodeIdentity GetIdentity(IDownloadedLog log)
         {
             var name = DetermineName(log);
-
+            var identityIndex = identityMap.GetIndex(name);
-            // We have to use a look-up map to match the node name to its peerId and nodeId,
+            var runner = new ConversionRunner(writer, identityMap, log.ContainerName, identityIndex);
-            // because the Codex logging never prints the id in full.
+            runner.Run(log);
-            // After we find it, we confirm it be looking for the shortened version.
-            var peerId = DeterminPeerId(name, log);
-            var nodeId = DeterminNodeId(name, log);
-
-            return new CodexNodeIdentity
-            {
-                PeerId = peerId,
-                NodeId = nodeId
-            };
-        }
-
-        private string DeterminNodeId(string name, IDownloadedLog log)
-        {
-            var nodeId = nameIdMap.GetId(name).NodeId;
-            var shortNodeId = CodexUtils.ToNodeIdShortId(nodeId);
-
-            // Look for "Starting discovery node" line to confirm nodeId.
-            var startedLine = log.FindLinesThatContain("Starting discovery node").Single();
-            var started = CodexLogLine.Parse(startedLine)!;
-            var foundId = started.Attributes["node"];
-
-            if (foundId != shortNodeId) throw new Exception("NodeId from name-lookup did not match NodeId found in codex-started log line.");
-
-            return nodeId;
-        }
-
-        private string DeterminPeerId(string name, IDownloadedLog log)
-        {
-            var peerId = nameIdMap.GetId(name).PeerId;
-            var shortPeerId = CodexUtils.ToShortId(peerId);
-
-            // Look for "Started codex node" line to confirm peerId.
-            var startedLine = log.FindLinesThatContain("Started codex node").Single();
-            var started = CodexLogLine.Parse(startedLine)!;
-            var foundId = started.Attributes["id"];
-
-            if (foundId != shortPeerId) throw new Exception("PeerId from name-lookup did not match PeerId found in codex-started log line.");
-
-            return peerId;
         }
 
         private string DetermineName(IDownloadedLog log)
@@ -82,9 +36,9 @@ namespace CodexPlugin.OverwatchSupport
     public class ConversionRunner
     {
         private readonly ITranscriptWriter writer;
-        private readonly NameIdMap nameIdMap;
+        private readonly IdentityMap nameIdMap;
         private readonly string name;
-        private readonly CodexNodeIdentity nodeIdentity;
+        private readonly int nodeIdentityIndex;
         private readonly ILineConverter[] converters = new ILineConverter[]
         {
             new BlockReceivedLineConverter(),
@@ -93,10 +47,10 @@ namespace CodexPlugin.OverwatchSupport
             new PeerDroppedLineConverter()
         };
 
-        public ConversionRunner(ITranscriptWriter writer, NameIdMap nameIdMap, string name, CodexNodeIdentity nodeIdentity)
+        public ConversionRunner(ITranscriptWriter writer, IdentityMap nameIdMap, string name, int nodeIdentityIndex)
         {
             this.name = name;
-            this.nodeIdentity = nodeIdentity;
+            this.nodeIdentityIndex = nodeIdentityIndex;
             this.writer = writer;
             this.nameIdMap = nameIdMap;
         }
@@ -116,8 +70,7 @@ namespace CodexPlugin.OverwatchSupport
         {
             var e = new OverwatchCodexEvent
             {
-                Name = name,
+                NodeIdentity = nodeIdentityIndex,
-                Identity = nodeIdentity,
             };
             action(e);
 

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs

@@ -7,15 +7,15 @@ namespace CodexPlugin.OverwatchSupport
     public class CodexNodeTranscriptWriter : ICodexNodeHooks
     {
         private readonly ITranscriptWriter writer;
-        private readonly NameIdMap nameIdMap;
+        private readonly IdentityMap identityMap;
         private readonly string name;
-        private CodexNodeIdentity identity = new CodexNodeIdentity();
+        private int identityIndex = -1;
         private readonly List<(DateTime, OverwatchCodexEvent)> pendingEvents = new List<(DateTime, OverwatchCodexEvent)>();
 
-        public CodexNodeTranscriptWriter(ITranscriptWriter writer, NameIdMap nameIdMap, string name)
+        public CodexNodeTranscriptWriter(ITranscriptWriter writer, IdentityMap identityMap, string name)
         {
             this.writer = writer;
-            this.nameIdMap = nameIdMap;
+            this.identityMap = identityMap;
             this.name = name;
         }
 
@@ -32,19 +32,13 @@ namespace CodexPlugin.OverwatchSupport
 
         public void OnNodeStarted(string peerId, string nodeId)
         {
-            identity.PeerId = peerId;
-            identity.NodeId = nodeId;
-
             if (string.IsNullOrEmpty(peerId) || string.IsNullOrEmpty(nodeId))
             {
                 throw new Exception("Node started - peerId and/or nodeId unknown.");
             }
 
-            nameIdMap.Add(name, new CodexNodeIdentity
+            identityMap.Add(name, peerId, nodeId);
-            {
+            identityIndex = identityMap.GetIndex(name);
-                PeerId = peerId,
-                NodeId = nodeId
-            });
 
             WriteCodexEvent(e =>
             {
@@ -121,22 +115,21 @@ namespace CodexPlugin.OverwatchSupport
         {
             var e = new OverwatchCodexEvent
             {
-                Name = name,
+                NodeIdentity = identityIndex
-                Identity = identity
             };
 
             action(e);
 
-            if (string.IsNullOrEmpty(identity.PeerId))
+            if (identityIndex < 0)
             {
-                // If we don't know our peerId, don't write the events yet.
+                // If we don't know our id, don't write the events yet.
                 AddToCache(utc, e);
             }
             else
             {
                 e.Write(utc, writer);
 
-                // Write any events that we cached when we didn't have our peerId yet.
+                // Write any events that we cached when we didn't have our id yet.
                 WriteAndClearCache();
             }
         }
@@ -152,7 +145,7 @@ namespace CodexPlugin.OverwatchSupport
             {
                 foreach (var pair in pendingEvents)
                 {
-                    pair.Item2.Identity = identity;
+                    pair.Item2.NodeIdentity = identityIndex;
                     pair.Item2.Write(pair.Item1, writer);
                 }
                 pendingEvents.Clear();

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs

@@ -12,13 +12,13 @@ namespace CodexPlugin.OverwatchSupport
         private readonly ILog log;
         private readonly ITranscriptWriter writer;
         private readonly CodexLogConverter converter;
-        private readonly NameIdMap nameIdMap = new NameIdMap();
+        private readonly IdentityMap identityMap = new IdentityMap();
 
         public CodexTranscriptWriter(ILog log, ITranscriptWriter transcriptWriter)
         {
             this.log = log;
             writer = transcriptWriter;
-            converter = new CodexLogConverter(writer, nameIdMap);
+            converter = new CodexLogConverter(writer, identityMap);
         }
 
         public void Finalize(string outputFilepath)
@@ -34,7 +34,7 @@ namespace CodexPlugin.OverwatchSupport
         public ICodexNodeHooks CreateHooks(string nodeName)
         {
             nodeName = Str.Between(nodeName, "'", "'");
-            return new CodexNodeTranscriptWriter(writer, nameIdMap, nodeName);
+            return new CodexNodeTranscriptWriter(writer, identityMap, nodeName);
         }
 
         public void IncludeFile(string filepath)
@@ -63,8 +63,7 @@ namespace CodexPlugin.OverwatchSupport
         {
             writer.Add(DateTime.UtcNow, new OverwatchCodexEvent
             {
-                Name = string.Empty,
+                NodeIdentity = -1,
-                Identity = new CodexNodeIdentity(),
                 ScenarioFinished = new ScenarioFinishedEvent
                 {
                     Success = success,
@@ -77,7 +76,7 @@ namespace CodexPlugin.OverwatchSupport
         {
             return new OverwatchCodexHeader
             {
-                TotalNumberOfNodes = nameIdMap.Size
+                Nodes = identityMap.Get()
             };
         }
 

ProjectPlugins/CodexPlugin/OverwatchSupport/IdentityMap.cs

@@ -0,0 +1,63 @@
+namespace CodexPlugin.OverwatchSupport
+{
+    public class IdentityMap
+    {
+        private readonly List<CodexNodeIdentity> nodes = new List<CodexNodeIdentity>();
+        private readonly Dictionary<string, int> nameIndexMap = new Dictionary<string, int>();
+        private readonly Dictionary<string, string> shortToLong = new Dictionary<string, string>();
+
+        public void Add(string name, string peerId, string nodeId)
+        {
+            Add(new CodexNodeIdentity
+            {
+                Name = name,
+                PeerId = peerId,
+                NodeId = nodeId
+            });
+
+            nameIndexMap.Add(name, nameIndexMap.Count);
+        }
+
+        public void Add(CodexNodeIdentity identity)
+        {
+            if (string.IsNullOrWhiteSpace(identity.Name)) throw new Exception("Name required");
+            if (string.IsNullOrWhiteSpace(identity.PeerId) || identity.PeerId.Length < 11) throw new Exception("PeerId invalid");
+            if (string.IsNullOrWhiteSpace(identity.NodeId) || identity.NodeId.Length < 11) throw new Exception("NodeId invalid");
+
+            nodes.Add(identity);
+
+            shortToLong.Add(CodexUtils.ToShortId(identity.PeerId), identity.PeerId);
+            shortToLong.Add(CodexUtils.ToShortId(identity.NodeId), identity.NodeId);
+        }
+
+        public CodexNodeIdentity[] Get()
+        {
+            return nodes.ToArray();
+        }
+
+        public int GetIndex(string name)
+        {
+            return nameIndexMap[name];
+        }
+
+        public CodexNodeIdentity GetId(string name)
+        {
+            return nodes.Single(n => n.Name == name);
+        }
+
+        public string ReplaceShortIds(string value)
+        {
+            var result = value;
+            foreach (var pair in shortToLong)
+            {
+                result = result.Replace(pair.Key, pair.Value);
+            }
+            return result;
+        }
+
+        public int Size
+        {
+            get { return nodes.Count; }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs

@@ -5,14 +5,13 @@ namespace CodexPlugin.OverwatchSupport
     [Serializable]
     public class OverwatchCodexHeader
     {
-        public int TotalNumberOfNodes { get; set; }
+        public CodexNodeIdentity[] Nodes { get; set; } = Array.Empty<CodexNodeIdentity>();
     }
 
     [Serializable]
     public class OverwatchCodexEvent
     {
-        public string Name { get; set; } = string.Empty;
+        public int NodeIdentity { get; set; } = -1;
-        public CodexNodeIdentity Identity { get; set; } = new();
         public ScenarioFinishedEvent? ScenarioFinished { get; set; }
         public NodeStartingEvent? NodeStarting { get; set; }
         public NodeStartedEvent? NodeStarted { get; set; }
@@ -28,9 +27,10 @@ namespace CodexPlugin.OverwatchSupport
 
         public void Write(DateTime utc, ITranscriptWriter writer)
         {
-            if (string.IsNullOrWhiteSpace(Name)) throw new Exception("Name required");
+            if (NodeIdentity == -1 && ScenarioFinished == null)
-            if (string.IsNullOrWhiteSpace(Identity.PeerId) || Identity.PeerId.Length < 11) throw new Exception("PeerId invalid");
+            {
-            if (string.IsNullOrWhiteSpace(Identity.NodeId) || Identity.NodeId.Length < 11) throw new Exception("NodeId invalid");
+                throw new Exception("NodeIdentity not set, and event is not ScenarioFinished.");
+            }
             if (AllNull()) throw new Exception("No event data was set");
 
             writer.Add(utc, this);
@@ -49,6 +49,7 @@ namespace CodexPlugin.OverwatchSupport
     [Serializable]
     public class CodexNodeIdentity
     {
+        public string Name { get; set; } = string.Empty;
         public string PeerId { get; set; } = string.Empty;
         public string NodeId { get; set; } = string.Empty;
     }

ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs

@@ -1,36 +0,0 @@
-namespace CodexPlugin.OverwatchSupport
-{
-    public class NameIdMap
-    {
-        private readonly Dictionary<string, CodexNodeIdentity> map = new Dictionary<string, CodexNodeIdentity>();
-        private readonly Dictionary<string, string> shortToLong = new Dictionary<string, string>();
-
-        public void Add(string name, CodexNodeIdentity identity)
-        {
-            map.Add(name, identity);
-
-            shortToLong.Add(CodexUtils.ToShortId(identity.PeerId), identity.PeerId);
-            shortToLong.Add(CodexUtils.ToShortId(identity.NodeId), identity.NodeId);
-        }
-
-        public CodexNodeIdentity GetId(string name)
-        {
-            return map[name];
-        }
-
-        public string ReplaceShortIds(string value)
-        {
-            var result = value;
-            foreach (var pair in shortToLong)
-            {
-                result = result.Replace(pair.Key, pair.Value);
-            }
-            return result;
-        }
-
-        public int Size
-        {
-            get { return map.Count; }
-        }
-    }
-}

Tests/DistTestCore/Configuration.cs

@@ -14,7 +14,7 @@ namespace DistTestCore
             kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
             logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
             dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
-            AlwaysDownloadContainerLogs = !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", ""));
+            AlwaysDownloadContainerLogs = true; // !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", ""));
         }
 
         public Configuration(string? kubeConfigFile, string logPath, string dataFilesPath)