status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Includes nodeId as part of node identity in transcript

This commit is contained in:
benbierens 2024-08-13 13:48:54 +02:00
parent 5ddec99114
commit 8136bf1c92
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
13 changed files with 102 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ProjectPlugins/CodexPlugin/CodexNode.cs
View File

@ -47,6 +47,7 @@ namespace CodexPlugin
private readonly EthAccount? ethAccount;
private readonly TransferSpeeds transferSpeeds;
private string peerId = string.Empty;
private string nodeId = string.Empty;
public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks, EthAccount? ethAccount)
{
@ -69,7 +70,7 @@ namespace CodexPlugin
public void Initialize()
{
hooks.OnNodeStarted(peerId);
hooks.OnNodeStarted(peerId, nodeId);
}
public RunningPod Pod { get { return CodexAccess.Container; } }
@ -228,6 +229,7 @@ namespace CodexPlugin
{
var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
peerId = debugInfo.Id;
nodeId = debugInfo.Table.LocalNode.NodeId;
var nodeName = CodexAccess.Container.Name;
if (!debugInfo.Version.IsValid())

2
ProjectPlugins/CodexPlugin/CodexStartupConfig.cs
View File

@ -8,7 +8,7 @@ namespace CodexPlugin
public string? NameOverride { get; set; }
public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation;
public CodexLogLevel LogLevel { get; set; }
public CodexLogCustomTopics? CustomTopics { get; set; } = new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn);
public CodexLogCustomTopics? CustomTopics { get; set; } = new CodexLogCustomTopics(CodexLogLevel.Info, CodexLogLevel.Warn);
public ByteSize? StorageQuota { get; set; }
public bool MetricsEnabled { get; set; }
public MarketplaceInitialConfig? MarketplaceConfig { get; set; }

10
ProjectPlugins/CodexPlugin/CodexUtils.cs
View File

@ -10,5 +10,15 @@
}
return id;
}
// after update of codex-dht, shortID should be consistent!
public static string ToNodeIdShortId(string id)
{
if (id.Length > 10)
{
return $"{id[..2]}*{id[^6..]}";
}
return id;
}
}
}

2
ProjectPlugins/CodexPlugin/Hooks/CodexHooksFactory.cs
View File

@ -43,7 +43,7 @@ namespace CodexPlugin.Hooks
{
}
public void OnNodeStarted(string peerId)
public void OnNodeStarted(string peerId, string nodeId)
{
}

2
ProjectPlugins/CodexPlugin/Hooks/CodexNodeHooks.cs
View File

@ -5,7 +5,7 @@ namespace CodexPlugin.Hooks
public interface ICodexNodeHooks
{
void OnNodeStarting(DateTime startUtc, string image);
void OnNodeStarted(string peerId);
void OnNodeStarted(string peerId, string nodeId);
void OnNodeStopping();
void OnFileUploading(string uid, ByteSize size);
void OnFileUploaded(string uid, ByteSize size, ContentId cid);

57
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs
View File

@ -18,23 +18,46 @@ namespace CodexPlugin.OverwatchSupport
public void ProcessLog(IDownloadedLog log)
{
var peerId = DeterminPeerId(log);
var peerId = GetIdentity(log);
var runner = new ConversionRunner(writer, nameIdMap, log.ContainerName, peerId);
runner.Run(log);
}
private string DeterminPeerId(IDownloadedLog log)
private CodexNodeIdentity GetIdentity(IDownloadedLog log)
{
// We have to use a look-up map to match the node name to its peerId,
// because the Codex logging never prints the peerId in full.
var name = DetermineName(log);
// We have to use a look-up map to match the node name to its peerId and nodeId,
// because the Codex logging never prints the id in full.
// After we find it, we confirm it be looking for the shortened version.
var peerId = DeterminPeerId(name, log);
var nodeId = DeterminNodeId(name, log);
// Expected string:
// Downloading container log for '<Downloader1>'
var nameLine = log.FindLinesThatContain("Downloading container log for").First();
var name = Str.Between(nameLine, "'", "'");
return new CodexNodeIdentity
{
PeerId = peerId,
NodeId = nodeId
};
}
var peerId = nameIdMap.GetPeerId(name);
private string DeterminNodeId(string name, IDownloadedLog log)
{
var nodeId = nameIdMap.GetId(name).NodeId;
var shortNodeId = CodexUtils.ToNodeIdShortId(nodeId);
// Look for "Starting discovery node" line to confirm nodeId.
var startedLine = log.FindLinesThatContain("Starting discovery node").Single();
var started = CodexLogLine.Parse(startedLine)!;
var foundId = started.Attributes["node"];
if (foundId != shortNodeId) throw new Exception("NodeId from name-lookup did not match NodeId found in codex-started log line.");
return nodeId;
}
private string DeterminPeerId(string name, IDownloadedLog log)
{
var peerId = nameIdMap.GetId(name).PeerId;
var shortPeerId = CodexUtils.ToShortId(peerId);
// Look for "Started codex node" line to confirm peerId.
@ -46,6 +69,14 @@ namespace CodexPlugin.OverwatchSupport
return peerId;
}
private string DetermineName(IDownloadedLog log)
{
// Expected string:
// Downloading container log for '<Downloader1>'
var nameLine = log.FindLinesThatContain("Downloading container log for").First();
return Str.Between(nameLine, "'", "'");
}
}
public class ConversionRunner
@ -53,7 +84,7 @@ namespace CodexPlugin.OverwatchSupport
private readonly ITranscriptWriter writer;
private readonly NameIdMap nameIdMap;
private readonly string name;
private readonly string peerId;
private readonly CodexNodeIdentity nodeIdentity;
private readonly ILineConverter[] converters = new ILineConverter[]
{
new BlockReceivedLineConverter(),
@ -62,12 +93,12 @@ namespace CodexPlugin.OverwatchSupport
new PeerDroppedLineConverter()
};
public ConversionRunner(ITranscriptWriter writer, NameIdMap nameIdMap, string name, string peerId)
public ConversionRunner(ITranscriptWriter writer, NameIdMap nameIdMap, string name, CodexNodeIdentity nodeIdentity)
{
this.name = name;
this.nodeIdentity = nodeIdentity;
this.writer = writer;
this.nameIdMap = nameIdMap;
this.peerId = peerId;
}
public void Run(IDownloadedLog log)
@ -86,7 +117,7 @@ namespace CodexPlugin.OverwatchSupport
var e = new OverwatchCodexEvent
{
Name = name,
PeerId = peerId,
Identity = nodeIdentity,
};
action(e);

26
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs
View File

@ -9,7 +9,7 @@ namespace CodexPlugin.OverwatchSupport
private readonly ITranscriptWriter writer;
private readonly NameIdMap nameIdMap;
private readonly string name;
private string peerId = string.Empty;
private CodexNodeIdentity identity = new CodexNodeIdentity();
private readonly List<(DateTime, OverwatchCodexEvent)> pendingEvents = new List<(DateTime, OverwatchCodexEvent)>();
public CodexNodeTranscriptWriter(ITranscriptWriter writer, NameIdMap nameIdMap, string name)
@ -30,10 +30,22 @@ namespace CodexPlugin.OverwatchSupport
});
}
public void OnNodeStarted(string peerId)
public void OnNodeStarted(string peerId, string nodeId)
{
this.peerId = peerId;
nameIdMap.Add(name, peerId);
identity.PeerId = peerId;
identity.NodeId = nodeId;
if (string.IsNullOrEmpty(peerId) || string.IsNullOrEmpty(nodeId))
{
throw new Exception("Node started - peerId and/or nodeId unknown.");
}
nameIdMap.Add(name, new CodexNodeIdentity
{
PeerId = peerId,
NodeId = nodeId
});
WriteCodexEvent(e =>
{
e.NodeStarted = new NodeStartedEvent
@ -110,12 +122,12 @@ namespace CodexPlugin.OverwatchSupport
var e = new OverwatchCodexEvent
{
Name = name,
PeerId = peerId
Identity = identity
};
action(e);
if (string.IsNullOrEmpty(peerId))
if (string.IsNullOrEmpty(identity.PeerId))
{
// If we don't know our peerId, don't write the events yet.
AddToCache(utc, e);
@ -140,7 +152,7 @@ namespace CodexPlugin.OverwatchSupport
{
foreach (var pair in pendingEvents)
{
pair.Item2.PeerId = peerId;
pair.Item2.Identity = identity;
pair.Item2.Write(pair.Item1, writer);
}
pendingEvents.Clear();

2
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs
View File

@ -64,7 +64,7 @@ namespace CodexPlugin.OverwatchSupport
writer.Add(DateTime.UtcNow, new OverwatchCodexEvent
{
Name = string.Empty,
PeerId = string.Empty,
Identity = new CodexNodeIdentity(),
ScenarioFinished = new ScenarioFinishedEvent
{
Success = success,

11
ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs
View File

@ -12,7 +12,7 @@ namespace CodexPlugin.OverwatchSupport
public class OverwatchCodexEvent
{
public string Name { get; set; } = string.Empty;
public string PeerId { get; set; } = string.Empty;
public CodexNodeIdentity Identity { get; set; } = new();
public ScenarioFinishedEvent? ScenarioFinished { get; set; }
public NodeStartingEvent? NodeStarting { get; set; }
public NodeStartedEvent? NodeStarted { get; set; }
@ -29,6 +29,8 @@ namespace CodexPlugin.OverwatchSupport
public void Write(DateTime utc, ITranscriptWriter writer)
{
if (string.IsNullOrWhiteSpace(Name)) throw new Exception("Name required");
if (string.IsNullOrWhiteSpace(Identity.PeerId) || Identity.PeerId.Length < 11) throw new Exception("PeerId invalid");
if (string.IsNullOrWhiteSpace(Identity.NodeId) || Identity.NodeId.Length < 11) throw new Exception("NodeId invalid");
if (AllNull()) throw new Exception("No event data was set");
writer.Add(utc, this);
@ -44,6 +46,13 @@ namespace CodexPlugin.OverwatchSupport
}
}
[Serializable]
public class CodexNodeIdentity
{
public string PeerId { get; set; } = string.Empty;
public string NodeId { get; set; } = string.Empty;
}
#region Scenario Generated Events
[Serializable]

11
ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs
View File

@ -2,17 +2,18 @@
{
public class NameIdMap
{
private readonly Dictionary<string, string> map = new Dictionary<string, string>();
private readonly Dictionary<string, CodexNodeIdentity> map = new Dictionary<string, CodexNodeIdentity>();
private readonly Dictionary<string, string> shortToLong = new Dictionary<string, string>();
public void Add(string name, string peerId)
public void Add(string name, CodexNodeIdentity identity)
{
map.Add(name, peerId);
map.Add(name, identity);
shortToLong.Add(CodexUtils.ToShortId(peerId), peerId);
shortToLong.Add(CodexUtils.ToShortId(identity.PeerId), identity.PeerId);
shortToLong.Add(CodexUtils.ToShortId(identity.NodeId), identity.NodeId);
}
public string GetPeerId(string name)
public CodexNodeIdentity GetId(string name)
{
return map[name];
}

2
Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs
View File

@ -45,7 +45,7 @@ namespace TranscriptAnalysis.Receivers
private void Handle(OverwatchCodexEvent payload, BlockReceivedEvent blockReceived)
{
var receiverPeerId = payload.PeerId;
var receiverPeerId = payload.Identity.PeerId;
var blockAddress = blockReceived.BlockAddress;
seen++;

8
Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs
View File

@ -12,12 +12,12 @@ namespace TranscriptAnalysis.Receivers
public override void Receive(ActivateEvent<OverwatchCodexEvent> @event)
{
if (!seen.Contains(@event.Payload.PeerId))
if (!seen.Contains(@event.Payload.Identity.PeerId))
{
seen.Add(@event.Payload.PeerId);
seen.Add(@event.Payload.Identity.PeerId);
log.AddStringReplace(@event.Payload.PeerId, @event.Payload.Name);
log.AddStringReplace(CodexUtils.ToShortId(@event.Payload.PeerId), @event.Payload.Name);
log.AddStringReplace(@event.Payload.Identity.PeerId, @event.Payload.Name);
log.AddStringReplace(CodexUtils.ToShortId(@event.Payload.Identity.PeerId), @event.Payload.Name);
}
}

2
Tools/TranscriptAnalysis/Receivers/NodesDegree.cs
View File

@ -53,7 +53,7 @@ namespace TranscriptAnalysis.Receivers
{
if (@event.Payload.DialSuccessful != null)
{
AddDial(@event.Payload.PeerId, @event.Payload.DialSuccessful.TargetPeerId);
AddDial(@event.Payload.Identity.PeerId, @event.Payload.DialSuccessful.TargetPeerId);
}
}