From 8136bf1c928e3cc37174b6bb9cf3ab5bf8ad6a12 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 13 Aug 2024 13:48:54 +0200 Subject: [PATCH] Includes nodeId as part of node identity in transcript --- ProjectPlugins/CodexPlugin/CodexNode.cs | 4 +- .../CodexPlugin/CodexStartupConfig.cs | 2 +- ProjectPlugins/CodexPlugin/CodexUtils.cs | 10 ++++ .../CodexPlugin/Hooks/CodexHooksFactory.cs | 2 +- .../CodexPlugin/Hooks/CodexNodeHooks.cs | 2 +- .../OverwatchSupport/CodexLogConverter.cs | 57 ++++++++++++++----- .../CodexNodeTranscriptWriter.cs | 26 ++++++--- .../OverwatchSupport/CodexTranscriptWriter.cs | 2 +- .../OverwatchSupport/ModelExtensions.cs | 11 +++- .../CodexPlugin/OverwatchSupport/NameIdMap.cs | 11 ++-- .../Receivers/DuplicateBlocksReceived.cs | 2 +- .../Receivers/LogReplaceReceiver.cs | 8 +-- .../Receivers/NodesDegree.cs | 2 +- 13 files changed, 102 insertions(+), 37 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index 98b3653..0f8bcfd 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -47,6 +47,7 @@ namespace CodexPlugin private readonly EthAccount? ethAccount; private readonly TransferSpeeds transferSpeeds; private string peerId = string.Empty; + private string nodeId = string.Empty; public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks, EthAccount? ethAccount) { @@ -69,7 +70,7 @@ namespace CodexPlugin public void Initialize() { - hooks.OnNodeStarted(peerId); + hooks.OnNodeStarted(peerId, nodeId); } public RunningPod Pod { get { return CodexAccess.Container; } } @@ -228,6 +229,7 @@ namespace CodexPlugin { var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online"); peerId = debugInfo.Id; + nodeId = debugInfo.Table.LocalNode.NodeId; var nodeName = CodexAccess.Container.Name; if (!debugInfo.Version.IsValid()) diff --git a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs index d6ab162..6390309 100644 --- a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs +++ b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs @@ -8,7 +8,7 @@ namespace CodexPlugin public string? NameOverride { get; set; } public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation; public CodexLogLevel LogLevel { get; set; } - public CodexLogCustomTopics? CustomTopics { get; set; } = new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn); + public CodexLogCustomTopics? CustomTopics { get; set; } = new CodexLogCustomTopics(CodexLogLevel.Info, CodexLogLevel.Warn); public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } diff --git a/ProjectPlugins/CodexPlugin/CodexUtils.cs b/ProjectPlugins/CodexPlugin/CodexUtils.cs index 2b6a507..e4cf96e 100644 --- a/ProjectPlugins/CodexPlugin/CodexUtils.cs +++ b/ProjectPlugins/CodexPlugin/CodexUtils.cs @@ -10,5 +10,15 @@ } return id; } + + // after update of codex-dht, shortID should be consistent! + public static string ToNodeIdShortId(string id) + { + if (id.Length > 10) + { + return $"{id[..2]}*{id[^6..]}"; + } + return id; + } } } diff --git a/ProjectPlugins/CodexPlugin/Hooks/CodexHooksFactory.cs b/ProjectPlugins/CodexPlugin/Hooks/CodexHooksFactory.cs index f668057..4c6b3f9 100644 --- a/ProjectPlugins/CodexPlugin/Hooks/CodexHooksFactory.cs +++ b/ProjectPlugins/CodexPlugin/Hooks/CodexHooksFactory.cs @@ -43,7 +43,7 @@ namespace CodexPlugin.Hooks { } - public void OnNodeStarted(string peerId) + public void OnNodeStarted(string peerId, string nodeId) { } diff --git a/ProjectPlugins/CodexPlugin/Hooks/CodexNodeHooks.cs b/ProjectPlugins/CodexPlugin/Hooks/CodexNodeHooks.cs index 9e8e354..b18dd5d 100644 --- a/ProjectPlugins/CodexPlugin/Hooks/CodexNodeHooks.cs +++ b/ProjectPlugins/CodexPlugin/Hooks/CodexNodeHooks.cs @@ -5,7 +5,7 @@ namespace CodexPlugin.Hooks public interface ICodexNodeHooks { void OnNodeStarting(DateTime startUtc, string image); - void OnNodeStarted(string peerId); + void OnNodeStarted(string peerId, string nodeId); void OnNodeStopping(); void OnFileUploading(string uid, ByteSize size); void OnFileUploaded(string uid, ByteSize size, ContentId cid); diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs index 8f87eff..5e4e5d2 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs @@ -18,23 +18,46 @@ namespace CodexPlugin.OverwatchSupport public void ProcessLog(IDownloadedLog log) { - var peerId = DeterminPeerId(log); + var peerId = GetIdentity(log); var runner = new ConversionRunner(writer, nameIdMap, log.ContainerName, peerId); runner.Run(log); } - private string DeterminPeerId(IDownloadedLog log) + private CodexNodeIdentity GetIdentity(IDownloadedLog log) { - // We have to use a look-up map to match the node name to its peerId, - // because the Codex logging never prints the peerId in full. + var name = DetermineName(log); + + // We have to use a look-up map to match the node name to its peerId and nodeId, + // because the Codex logging never prints the id in full. // After we find it, we confirm it be looking for the shortened version. + var peerId = DeterminPeerId(name, log); + var nodeId = DeterminNodeId(name, log); - // Expected string: - // Downloading container log for '' - var nameLine = log.FindLinesThatContain("Downloading container log for").First(); - var name = Str.Between(nameLine, "'", "'"); + return new CodexNodeIdentity + { + PeerId = peerId, + NodeId = nodeId + }; + } - var peerId = nameIdMap.GetPeerId(name); + private string DeterminNodeId(string name, IDownloadedLog log) + { + var nodeId = nameIdMap.GetId(name).NodeId; + var shortNodeId = CodexUtils.ToNodeIdShortId(nodeId); + + // Look for "Starting discovery node" line to confirm nodeId. + var startedLine = log.FindLinesThatContain("Starting discovery node").Single(); + var started = CodexLogLine.Parse(startedLine)!; + var foundId = started.Attributes["node"]; + + if (foundId != shortNodeId) throw new Exception("NodeId from name-lookup did not match NodeId found in codex-started log line."); + + return nodeId; + } + + private string DeterminPeerId(string name, IDownloadedLog log) + { + var peerId = nameIdMap.GetId(name).PeerId; var shortPeerId = CodexUtils.ToShortId(peerId); // Look for "Started codex node" line to confirm peerId. @@ -46,6 +69,14 @@ namespace CodexPlugin.OverwatchSupport return peerId; } + + private string DetermineName(IDownloadedLog log) + { + // Expected string: + // Downloading container log for '' + var nameLine = log.FindLinesThatContain("Downloading container log for").First(); + return Str.Between(nameLine, "'", "'"); + } } public class ConversionRunner @@ -53,7 +84,7 @@ namespace CodexPlugin.OverwatchSupport private readonly ITranscriptWriter writer; private readonly NameIdMap nameIdMap; private readonly string name; - private readonly string peerId; + private readonly CodexNodeIdentity nodeIdentity; private readonly ILineConverter[] converters = new ILineConverter[] { new BlockReceivedLineConverter(), @@ -62,12 +93,12 @@ namespace CodexPlugin.OverwatchSupport new PeerDroppedLineConverter() }; - public ConversionRunner(ITranscriptWriter writer, NameIdMap nameIdMap, string name, string peerId) + public ConversionRunner(ITranscriptWriter writer, NameIdMap nameIdMap, string name, CodexNodeIdentity nodeIdentity) { this.name = name; + this.nodeIdentity = nodeIdentity; this.writer = writer; this.nameIdMap = nameIdMap; - this.peerId = peerId; } public void Run(IDownloadedLog log) @@ -86,7 +117,7 @@ namespace CodexPlugin.OverwatchSupport var e = new OverwatchCodexEvent { Name = name, - PeerId = peerId, + Identity = nodeIdentity, }; action(e); diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs index f0e5de3..7d456a6 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs @@ -9,7 +9,7 @@ namespace CodexPlugin.OverwatchSupport private readonly ITranscriptWriter writer; private readonly NameIdMap nameIdMap; private readonly string name; - private string peerId = string.Empty; + private CodexNodeIdentity identity = new CodexNodeIdentity(); private readonly List<(DateTime, OverwatchCodexEvent)> pendingEvents = new List<(DateTime, OverwatchCodexEvent)>(); public CodexNodeTranscriptWriter(ITranscriptWriter writer, NameIdMap nameIdMap, string name) @@ -30,10 +30,22 @@ namespace CodexPlugin.OverwatchSupport }); } - public void OnNodeStarted(string peerId) + public void OnNodeStarted(string peerId, string nodeId) { - this.peerId = peerId; - nameIdMap.Add(name, peerId); + identity.PeerId = peerId; + identity.NodeId = nodeId; + + if (string.IsNullOrEmpty(peerId) || string.IsNullOrEmpty(nodeId)) + { + throw new Exception("Node started - peerId and/or nodeId unknown."); + } + + nameIdMap.Add(name, new CodexNodeIdentity + { + PeerId = peerId, + NodeId = nodeId + }); + WriteCodexEvent(e => { e.NodeStarted = new NodeStartedEvent @@ -110,12 +122,12 @@ namespace CodexPlugin.OverwatchSupport var e = new OverwatchCodexEvent { Name = name, - PeerId = peerId + Identity = identity }; action(e); - if (string.IsNullOrEmpty(peerId)) + if (string.IsNullOrEmpty(identity.PeerId)) { // If we don't know our peerId, don't write the events yet. AddToCache(utc, e); @@ -140,7 +152,7 @@ namespace CodexPlugin.OverwatchSupport { foreach (var pair in pendingEvents) { - pair.Item2.PeerId = peerId; + pair.Item2.Identity = identity; pair.Item2.Write(pair.Item1, writer); } pendingEvents.Clear(); diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs index 436f1f7..4645c71 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs @@ -64,7 +64,7 @@ namespace CodexPlugin.OverwatchSupport writer.Add(DateTime.UtcNow, new OverwatchCodexEvent { Name = string.Empty, - PeerId = string.Empty, + Identity = new CodexNodeIdentity(), ScenarioFinished = new ScenarioFinishedEvent { Success = success, diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs index 5f06a8d..1bf4fbf 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs @@ -12,7 +12,7 @@ namespace CodexPlugin.OverwatchSupport public class OverwatchCodexEvent { public string Name { get; set; } = string.Empty; - public string PeerId { get; set; } = string.Empty; + public CodexNodeIdentity Identity { get; set; } = new(); public ScenarioFinishedEvent? ScenarioFinished { get; set; } public NodeStartingEvent? NodeStarting { get; set; } public NodeStartedEvent? NodeStarted { get; set; } @@ -29,6 +29,8 @@ namespace CodexPlugin.OverwatchSupport public void Write(DateTime utc, ITranscriptWriter writer) { if (string.IsNullOrWhiteSpace(Name)) throw new Exception("Name required"); + if (string.IsNullOrWhiteSpace(Identity.PeerId) || Identity.PeerId.Length < 11) throw new Exception("PeerId invalid"); + if (string.IsNullOrWhiteSpace(Identity.NodeId) || Identity.NodeId.Length < 11) throw new Exception("NodeId invalid"); if (AllNull()) throw new Exception("No event data was set"); writer.Add(utc, this); @@ -44,6 +46,13 @@ namespace CodexPlugin.OverwatchSupport } } + [Serializable] + public class CodexNodeIdentity + { + public string PeerId { get; set; } = string.Empty; + public string NodeId { get; set; } = string.Empty; + } + #region Scenario Generated Events [Serializable] diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs index 091c8d8..55e3320 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs @@ -2,17 +2,18 @@ { public class NameIdMap { - private readonly Dictionary map = new Dictionary(); + private readonly Dictionary map = new Dictionary(); private readonly Dictionary shortToLong = new Dictionary(); - public void Add(string name, string peerId) + public void Add(string name, CodexNodeIdentity identity) { - map.Add(name, peerId); + map.Add(name, identity); - shortToLong.Add(CodexUtils.ToShortId(peerId), peerId); + shortToLong.Add(CodexUtils.ToShortId(identity.PeerId), identity.PeerId); + shortToLong.Add(CodexUtils.ToShortId(identity.NodeId), identity.NodeId); } - public string GetPeerId(string name) + public CodexNodeIdentity GetId(string name) { return map[name]; } diff --git a/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs b/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs index d0f8f36..a8229e5 100644 --- a/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs +++ b/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs @@ -45,7 +45,7 @@ namespace TranscriptAnalysis.Receivers private void Handle(OverwatchCodexEvent payload, BlockReceivedEvent blockReceived) { - var receiverPeerId = payload.PeerId; + var receiverPeerId = payload.Identity.PeerId; var blockAddress = blockReceived.BlockAddress; seen++; diff --git a/Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs b/Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs index 7a659c6..8f0a6bd 100644 --- a/Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs +++ b/Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs @@ -12,12 +12,12 @@ namespace TranscriptAnalysis.Receivers public override void Receive(ActivateEvent @event) { - if (!seen.Contains(@event.Payload.PeerId)) + if (!seen.Contains(@event.Payload.Identity.PeerId)) { - seen.Add(@event.Payload.PeerId); + seen.Add(@event.Payload.Identity.PeerId); - log.AddStringReplace(@event.Payload.PeerId, @event.Payload.Name); - log.AddStringReplace(CodexUtils.ToShortId(@event.Payload.PeerId), @event.Payload.Name); + log.AddStringReplace(@event.Payload.Identity.PeerId, @event.Payload.Name); + log.AddStringReplace(CodexUtils.ToShortId(@event.Payload.Identity.PeerId), @event.Payload.Name); } } diff --git a/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs b/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs index 8957235..0f89eb5 100644 --- a/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs +++ b/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs @@ -53,7 +53,7 @@ namespace TranscriptAnalysis.Receivers { if (@event.Payload.DialSuccessful != null) { - AddDial(@event.Payload.PeerId, @event.Payload.DialSuccessful.TargetPeerId); + AddDial(@event.Payload.Identity.PeerId, @event.Payload.DialSuccessful.TargetPeerId); } }