status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip: rework identity map

This commit is contained in:
benbierens 2024-08-13 14:21:15 +02:00
parent 8136bf1c92
commit 38b987144e
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
7 changed files with 98 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs
View File

@ -8,66 +8,20 @@ namespace CodexPlugin.OverwatchSupport
public class CodexLogConverter
{
private readonly ITranscriptWriter writer;
private readonly NameIdMap nameIdMap;
private readonly IdentityMap identityMap;
public CodexLogConverter(ITranscriptWriter writer, NameIdMap nameIdMap)
public CodexLogConverter(ITranscriptWriter writer, IdentityMap identityMap)
{
this.writer = writer;
this.nameIdMap = nameIdMap;
this.identityMap = identityMap;
}
public void ProcessLog(IDownloadedLog log)
{
var peerId = GetIdentity(log);
var runner = new ConversionRunner(writer, nameIdMap, log.ContainerName, peerId);
runner.Run(log);
}
private CodexNodeIdentity GetIdentity(IDownloadedLog log)
{
var name = DetermineName(log);
// We have to use a look-up map to match the node name to its peerId and nodeId,
// because the Codex logging never prints the id in full.
// After we find it, we confirm it be looking for the shortened version.
var peerId = DeterminPeerId(name, log);
var nodeId = DeterminNodeId(name, log);
return new CodexNodeIdentity
{
PeerId = peerId,
NodeId = nodeId
};
}
private string DeterminNodeId(string name, IDownloadedLog log)
{
var nodeId = nameIdMap.GetId(name).NodeId;
var shortNodeId = CodexUtils.ToNodeIdShortId(nodeId);
// Look for "Starting discovery node" line to confirm nodeId.
var startedLine = log.FindLinesThatContain("Starting discovery node").Single();
var started = CodexLogLine.Parse(startedLine)!;
var foundId = started.Attributes["node"];
if (foundId != shortNodeId) throw new Exception("NodeId from name-lookup did not match NodeId found in codex-started log line.");
return nodeId;
}
private string DeterminPeerId(string name, IDownloadedLog log)
{
var peerId = nameIdMap.GetId(name).PeerId;
var shortPeerId = CodexUtils.ToShortId(peerId);
// Look for "Started codex node" line to confirm peerId.
var startedLine = log.FindLinesThatContain("Started codex node").Single();
var started = CodexLogLine.Parse(startedLine)!;
var foundId = started.Attributes["id"];
if (foundId != shortPeerId) throw new Exception("PeerId from name-lookup did not match PeerId found in codex-started log line.");
return peerId;
var identityIndex = identityMap.GetIndex(name);
var runner = new ConversionRunner(writer, identityMap, log.ContainerName, identityIndex);
runner.Run(log);
}
private string DetermineName(IDownloadedLog log)
@ -82,9 +36,9 @@ namespace CodexPlugin.OverwatchSupport
public class ConversionRunner
{
private readonly ITranscriptWriter writer;
private readonly NameIdMap nameIdMap;
private readonly IdentityMap nameIdMap;
private readonly string name;
private readonly CodexNodeIdentity nodeIdentity;
private readonly int nodeIdentityIndex;
private readonly ILineConverter[] converters = new ILineConverter[]
{
new BlockReceivedLineConverter(),
@ -93,10 +47,10 @@ namespace CodexPlugin.OverwatchSupport
new PeerDroppedLineConverter()
};
public ConversionRunner(ITranscriptWriter writer, NameIdMap nameIdMap, string name, CodexNodeIdentity nodeIdentity)
public ConversionRunner(ITranscriptWriter writer, IdentityMap nameIdMap, string name, int nodeIdentityIndex)
{
this.name = name;
this.nodeIdentity = nodeIdentity;
this.nodeIdentityIndex = nodeIdentityIndex;
this.writer = writer;
this.nameIdMap = nameIdMap;
}
@ -116,8 +70,7 @@ namespace CodexPlugin.OverwatchSupport
{
var e = new OverwatchCodexEvent
{
Name = name,
Identity = nodeIdentity,
NodeIdentity = nodeIdentityIndex,
};
action(e);

29
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs
View File

@ -7,15 +7,15 @@ namespace CodexPlugin.OverwatchSupport
public class CodexNodeTranscriptWriter : ICodexNodeHooks
{
private readonly ITranscriptWriter writer;
private readonly NameIdMap nameIdMap;
private readonly IdentityMap identityMap;
private readonly string name;
private CodexNodeIdentity identity = new CodexNodeIdentity();
private int identityIndex = -1;
private readonly List<(DateTime, OverwatchCodexEvent)> pendingEvents = new List<(DateTime, OverwatchCodexEvent)>();
public CodexNodeTranscriptWriter(ITranscriptWriter writer, NameIdMap nameIdMap, string name)
public CodexNodeTranscriptWriter(ITranscriptWriter writer, IdentityMap identityMap, string name)
{
this.writer = writer;
this.nameIdMap = nameIdMap;
this.identityMap = identityMap;
this.name = name;
}
@ -32,19 +32,13 @@ namespace CodexPlugin.OverwatchSupport
public void OnNodeStarted(string peerId, string nodeId)
{
identity.PeerId = peerId;
identity.NodeId = nodeId;
if (string.IsNullOrEmpty(peerId) || string.IsNullOrEmpty(nodeId))
{
throw new Exception("Node started - peerId and/or nodeId unknown.");
}
nameIdMap.Add(name, new CodexNodeIdentity
{
PeerId = peerId,
NodeId = nodeId
});
identityMap.Add(name, peerId, nodeId);
identityIndex = identityMap.GetIndex(name);
WriteCodexEvent(e =>
{
@ -121,22 +115,21 @@ namespace CodexPlugin.OverwatchSupport
{
var e = new OverwatchCodexEvent
{
Name = name,
Identity = identity
NodeIdentity = identityIndex
};
action(e);
if (string.IsNullOrEmpty(identity.PeerId))
if (identityIndex < 0)
{
// If we don't know our peerId, don't write the events yet.
// If we don't know our id, don't write the events yet.
AddToCache(utc, e);
}
else
{
e.Write(utc, writer);
// Write any events that we cached when we didn't have our peerId yet.
// Write any events that we cached when we didn't have our id yet.
WriteAndClearCache();
}
}
@ -152,7 +145,7 @@ namespace CodexPlugin.OverwatchSupport
{
foreach (var pair in pendingEvents)
{
pair.Item2.Identity = identity;
pair.Item2.NodeIdentity = identityIndex;
pair.Item2.Write(pair.Item1, writer);
}
pendingEvents.Clear();

11
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs
View File

@ -12,13 +12,13 @@ namespace CodexPlugin.OverwatchSupport
private readonly ILog log;
private readonly ITranscriptWriter writer;
private readonly CodexLogConverter converter;
private readonly NameIdMap nameIdMap = new NameIdMap();
private readonly IdentityMap identityMap = new IdentityMap();
public CodexTranscriptWriter(ILog log, ITranscriptWriter transcriptWriter)
{
this.log = log;
writer = transcriptWriter;
converter = new CodexLogConverter(writer, nameIdMap);
converter = new CodexLogConverter(writer, identityMap);
}
public void Finalize(string outputFilepath)
@ -34,7 +34,7 @@ namespace CodexPlugin.OverwatchSupport
public ICodexNodeHooks CreateHooks(string nodeName)
{
nodeName = Str.Between(nodeName, "'", "'");
return new CodexNodeTranscriptWriter(writer, nameIdMap, nodeName);
return new CodexNodeTranscriptWriter(writer, identityMap, nodeName);
}
public void IncludeFile(string filepath)
@ -63,8 +63,7 @@ namespace CodexPlugin.OverwatchSupport
{
writer.Add(DateTime.UtcNow, new OverwatchCodexEvent
{
Name = string.Empty,
Identity = new CodexNodeIdentity(),
NodeIdentity = -1,
ScenarioFinished = new ScenarioFinishedEvent
{
Success = success,
@ -77,7 +76,7 @@ namespace CodexPlugin.OverwatchSupport
{
return new OverwatchCodexHeader
{
TotalNumberOfNodes = nameIdMap.Size
Nodes = identityMap.Get()
};
}

63
ProjectPlugins/CodexPlugin/OverwatchSupport/IdentityMap.cs Normal file
View File

@ -0,0 +1,63 @@
namespace CodexPlugin.OverwatchSupport
{
public class IdentityMap
{
private readonly List<CodexNodeIdentity> nodes = new List<CodexNodeIdentity>();
private readonly Dictionary<string, int> nameIndexMap = new Dictionary<string, int>();
private readonly Dictionary<string, string> shortToLong = new Dictionary<string, string>();
public void Add(string name, string peerId, string nodeId)
{
Add(new CodexNodeIdentity
{
Name = name,
PeerId = peerId,
NodeId = nodeId
});
nameIndexMap.Add(name, nameIndexMap.Count);
}
public void Add(CodexNodeIdentity identity)
{
if (string.IsNullOrWhiteSpace(identity.Name)) throw new Exception("Name required");
if (string.IsNullOrWhiteSpace(identity.PeerId) || identity.PeerId.Length < 11) throw new Exception("PeerId invalid");
if (string.IsNullOrWhiteSpace(identity.NodeId) || identity.NodeId.Length < 11) throw new Exception("NodeId invalid");
nodes.Add(identity);
shortToLong.Add(CodexUtils.ToShortId(identity.PeerId), identity.PeerId);
shortToLong.Add(CodexUtils.ToShortId(identity.NodeId), identity.NodeId);
}
public CodexNodeIdentity[] Get()
{
return nodes.ToArray();
}
public int GetIndex(string name)
{
return nameIndexMap[name];
}
public CodexNodeIdentity GetId(string name)
{
return nodes.Single(n => n.Name == name);
}
public string ReplaceShortIds(string value)
{
var result = value;
foreach (var pair in shortToLong)
{
result = result.Replace(pair.Key, pair.Value);
}
return result;
}
public int Size
{
get { return nodes.Count; }
}
}
}

13
ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs
View File

@ -5,14 +5,13 @@ namespace CodexPlugin.OverwatchSupport
[Serializable]
public class OverwatchCodexHeader
{
public int TotalNumberOfNodes { get; set; }
public CodexNodeIdentity[] Nodes { get; set; } = Array.Empty<CodexNodeIdentity>();
}
[Serializable]
public class OverwatchCodexEvent
{
public string Name { get; set; } = string.Empty;
public CodexNodeIdentity Identity { get; set; } = new();
public int NodeIdentity { get; set; } = -1;
public ScenarioFinishedEvent? ScenarioFinished { get; set; }
public NodeStartingEvent? NodeStarting { get; set; }
public NodeStartedEvent? NodeStarted { get; set; }
@ -28,9 +27,10 @@ namespace CodexPlugin.OverwatchSupport
public void Write(DateTime utc, ITranscriptWriter writer)
{
if (string.IsNullOrWhiteSpace(Name)) throw new Exception("Name required");
if (string.IsNullOrWhiteSpace(Identity.PeerId) || Identity.PeerId.Length < 11) throw new Exception("PeerId invalid");
if (string.IsNullOrWhiteSpace(Identity.NodeId) || Identity.NodeId.Length < 11) throw new Exception("NodeId invalid");
if (NodeIdentity == -1 && ScenarioFinished == null)
{
throw new Exception("NodeIdentity not set, and event is not ScenarioFinished.");
}
if (AllNull()) throw new Exception("No event data was set");
writer.Add(utc, this);
@ -49,6 +49,7 @@ namespace CodexPlugin.OverwatchSupport
[Serializable]
public class CodexNodeIdentity
{
public string Name { get; set; } = string.Empty;
public string PeerId { get; set; } = string.Empty;
public string NodeId { get; set; } = string.Empty;
}

36
ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs
View File

@ -1,36 +0,0 @@
namespace CodexPlugin.OverwatchSupport
{
public class NameIdMap
{
private readonly Dictionary<string, CodexNodeIdentity> map = new Dictionary<string, CodexNodeIdentity>();
private readonly Dictionary<string, string> shortToLong = new Dictionary<string, string>();
public void Add(string name, CodexNodeIdentity identity)
{
map.Add(name, identity);
shortToLong.Add(CodexUtils.ToShortId(identity.PeerId), identity.PeerId);
shortToLong.Add(CodexUtils.ToShortId(identity.NodeId), identity.NodeId);
}
public CodexNodeIdentity GetId(string name)
{
return map[name];
}
public string ReplaceShortIds(string value)
{
var result = value;
foreach (var pair in shortToLong)
{
result = result.Replace(pair.Key, pair.Value);
}
return result;
}
public int Size
{
get { return map.Count; }
}
}
}

2
Tests/DistTestCore/Configuration.cs
View File

@ -14,7 +14,7 @@ namespace DistTestCore
kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
AlwaysDownloadContainerLogs = !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", ""));
AlwaysDownloadContainerLogs = true; // !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", ""));
}
public Configuration(string? kubeConfigFile, string logPath, string dataFilesPath)