codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Restores debug info

This commit is contained in:
Ben 2024-03-26 10:03:52 +01:00
parent a3253b42d8
commit 05d28d1d0a
No known key found for this signature in database
GPG Key ID: 541B9D8C9F1426A1
8 changed files with 131 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Framework/Core/PluginTools.cs
View File

@ -21,9 +21,9 @@ namespace Core
public interface IHttpFactoryTool
{
IHttp CreateHttp(Action<HttpClient> onClientCreated, string? logAlias = null);
IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet, string? logAlias = null);
IHttp CreateHttp(string? logAlias = null);
IHttp CreateHttp(Action<HttpClient> onClientCreated);
IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
IHttp CreateHttp();
}
public interface IFileTool
@ -51,19 +51,19 @@ namespace Core
log = new LogPrefixer(log, prefix);
}
public IHttp CreateHttp(Action<HttpClient> onClientCreated, string? logAlias = null)
public IHttp CreateHttp(Action<HttpClient> onClientCreated)
{
return CreateHttp(onClientCreated, timeSet, logAlias);
return CreateHttp(onClientCreated, timeSet);
}
public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts, string? logAlias = null)
public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts)
{
return new Http(log, ts, onClientCreated, logAlias);
return new Http(log, ts, onClientCreated);
}
public IHttp CreateHttp(string? logAlias = null)
public IHttp CreateHttp()
{
return new Http(log, timeSet, logAlias);
return new Http(log, timeSet);
}
public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)

23
ProjectPlugins/CodexPlugin/CodexNode.cs
View File

@ -3,7 +3,6 @@ using FileUtils;
using GethPlugin;
using KubernetesWorkflow;
using KubernetesWorkflow.Types;
using Logging;
using MetricsPlugin;
using Utils;
@ -18,7 +17,7 @@ namespace CodexPlugin
TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
LocalDataset[] LocalFiles();
void ConnectToPeer(ICodexNode node);
DebugVersion Version { get; }
DebugInfoVersion Version { get; }
IMarketplaceAccess Marketplace { get; }
CrashWatcher CrashWatcher { get; }
PodInfo GetPodInfo();
@ -41,7 +40,7 @@ namespace CodexPlugin
CodexAccess = codexAccess;
Group = group;
Marketplace = marketplaceAccess;
Version = new DebugVersion();
Version = new DebugInfoVersion();
transferSpeeds = new TransferSpeeds();
}
@ -50,8 +49,9 @@ namespace CodexPlugin
public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
public CodexNodeGroup Group { get; }
public IMarketplaceAccess Marketplace { get; }
public DebugVersion Version { get; private set; }
public DebugInfoVersion Version { get; private set; }
public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
public IMetricsScrapeTarget MetricsScrapeTarget
{
get
@ -59,6 +59,7 @@ namespace CodexPlugin
return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag);
}
}
public EthAddress EthAddress
{
get
@ -76,8 +77,8 @@ namespace CodexPlugin
public DebugInfo GetDebugInfo()
{
var debugInfo = CodexAccess.GetDebugInfo();
var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
Log($"Got DebugInfo with id: '{debugInfo.Id}'. This node knows: {known}");
return debugInfo;
}
@ -156,18 +157,18 @@ namespace CodexPlugin
public void EnsureOnlineGetVersionResponse()
{
var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
var nodePeerId = debugInfo.id;
var nodePeerId = debugInfo.Id;
var nodeName = CodexAccess.Container.Name;
if (!debugInfo.codex.IsValid())
if (!debugInfo.Version.IsValid())
{
throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
}
var log = tools.GetLog();
log.AddStringReplace(nodePeerId, nodeName);
log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
Version = debugInfo.codex;
log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
Version = debugInfo.Version;
}
private string GetPeerMultiAddress(CodexNode peer, DebugInfo peerInfo)

4
ProjectPlugins/CodexPlugin/CodexNodeGroup.cs
View File

@ -20,7 +20,7 @@ namespace CodexPlugin
this.starter = starter;
Containers = containers;
Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
Version = new DebugVersion();
Version = new DebugInfoVersion();
}
public ICodexNode this[int index]
@ -41,7 +41,7 @@ namespace CodexPlugin
public RunningContainers[] Containers { get; private set; }
public CodexNode[] Nodes { get; private set; }
public DebugVersion Version { get; private set; }
public DebugInfoVersion Version { get; private set; }
public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
public IEnumerator<ICodexNode> GetEnumerator()

2
ProjectPlugins/CodexPlugin/CodexStarter.cs
View File

@ -9,7 +9,7 @@ namespace CodexPlugin
{
private readonly IPluginTools pluginTools;
private readonly CodexContainerRecipe recipe = new CodexContainerRecipe();
private DebugVersion? versionResponse;
private DebugInfoVersion? versionResponse;
public CodexStarter(IPluginTools pluginTools)
{

43
ProjectPlugins/CodexPlugin/CodexTypes.cs
View File

@ -1,4 +1,6 @@
namespace CodexPlugin
using Newtonsoft.Json;
namespace CodexPlugin
{
public class DebugInfo
{
@ -6,6 +8,39 @@
public string Spr { get; set; } = string.Empty;
public string Id { get; set; } = string.Empty;
public string[] AnnounceAddresses { get; set; } = Array.Empty<string>();
public DebugInfoVersion Version { get; set; } = new();
public DebugInfoTable Table { get; set; } = new();
}
public class DebugInfoVersion
{
public string Version { get; set; } = string.Empty;
public string Revision { get; set; } = string.Empty;
public bool IsValid()
{
return !string.IsNullOrEmpty(Version) && !string.IsNullOrEmpty(Revision);
}
public override string ToString()
{
return JsonConvert.SerializeObject(this);
}
}
public class DebugInfoTable
{
public DebugInfoTableNode LocalNode { get; set; } = new();
public DebugInfoTableNode[] Nodes { get; set; } = Array.Empty<DebugInfoTableNode>();
}
public class DebugInfoTableNode
{
public string NodeId { get; set; } = string.Empty;
public string PeerId { get; set; } = string.Empty;
public string Record { get; set; } = string.Empty;
public string Address { get; set; } = string.Empty;
public bool Seen { get; set; }
}
public class DebugPeer
@ -19,12 +54,6 @@
public ContentId Cid { get; set; } = new ContentId();
}
public class DebugVersion
{
public string Version { get; internal set; } = string.Empty;
public string Revision { get; internal set; } = string.Empty;
}
public class ContentId
{
public ContentId()

66
ProjectPlugins/CodexPlugin/Mapper.cs
View File

@ -11,8 +11,72 @@ namespace CodexPlugin
Id = debugInfo.Id,
Spr = debugInfo.Spr,
Addrs = debugInfo.Addrs.ToArray(),
AnnounceAddresses = ((JArray) debugInfo.AdditionalProperties["announceAddresses"]).Select(x => x.ToString()).ToArray(),
AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")),
Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table"))
};
}
private DebugInfoVersion MapDebugInfoVersion(JObject obj)
{
return new DebugInfoVersion
{
Version = StringOrEmpty(obj, "version"),
Revision = StringOrEmpty(obj, "revision")
};
}
private DebugInfoTable MapDebugInfoTable(JObject obj)
{
return new DebugInfoTable
{
LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
Nodes = new DebugInfoTableNode[0]
};
}
private DebugInfoTableNode MapDebugInfoTableNode(JToken? token)
{
var obj = token as JObject;
if (obj == null) return new DebugInfoTableNode();
return new DebugInfoTableNode
{
Address = StringOrEmpty(obj, "address"),
NodeId = StringOrEmpty(obj, "nodeId"),
PeerId = StringOrEmpty(obj, "peerId"),
Record = StringOrEmpty(obj, "record"),
Seen = Bool(obj, "seen")
};
}
private JArray JArray(IDictionary<string, object> map, string name)
{
return (JArray)map[name];
}
private JObject JObject(IDictionary<string, object> map, string name)
{
return (JObject)map[name];
}
private string StringOrEmpty(JObject obj, string name)
{
if (obj.TryGetValue(name, out var token))
{
var str = (string?)token;
if (!string.IsNullOrEmpty(str)) return str;
}
return string.Empty;
}
private bool Bool(JObject obj, string name)
{
if (obj.TryGetValue(name, out var token))
{
return (bool)token;
}
return false;
}
}
}

6
Tests/CodexTests/BasicTests/ContinuousSubstitute.cs
View File

@ -103,7 +103,7 @@ namespace CodexTests.BasicTests
private void CheckRoutingTables(IEnumerable<ICodexNode> nodes)
{
var all = nodes.ToArray();
var allIds = all.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray();
var allIds = all.Select(n => n.GetDebugInfo().Table.LocalNode.NodeId).ToArray();
var errors = all.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
@ -116,8 +116,8 @@ namespace CodexTests.BasicTests
private string AreAllPresent(ICodexNode n, string[] allIds)
{
var info = n.GetDebugInfo();
var known = info.table.nodes.Select(n => n.nodeId).ToArray();
var expected = allIds.Where(i => i != info.table.localNode.nodeId).ToArray();
var known = info.Table.Nodes.Select(n => n.NodeId).ToArray();
var expected = allIds.Where(i => i != info.Table.LocalNode.NodeId).ToArray();
if (!expected.All(ex => known.Contains(ex)))
{

6
Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -72,13 +72,13 @@ namespace CodexTests.PeerDiscoveryTests
private string AreAllPresent(DebugInfo info, DebugInfo[] allResponses)
{
var knownIds = info.table.nodes.Select(n => n.nodeId).ToArray();
var knownIds = info.Table.Nodes.Select(n => n.NodeId).ToArray();
var allOthers = GetAllOtherResponses(info, allResponses);
var expectedIds = allOthers.Select(i => i.table.localNode.nodeId).ToArray();
var expectedIds = allOthers.Select(i => i.Table.LocalNode.NodeId).ToArray();
if (!expectedIds.All(ex => knownIds.Contains(ex)))
{
return $"Node {info.id}: Not all of '{string.Join(",", expectedIds)}' were present in routing table: '{string.Join(",", knownIds)}'";
return $"Node {info.Id}: Not all of '{string.Join(",", expectedIds)}' were present in routing table: '{string.Join(",", knownIds)}'";
}
return string.Empty;