diff --git a/DistTestCore/Codex/CodexApiTypes.cs b/DistTestCore/Codex/CodexApiTypes.cs index cde05fd..32bf353 100644 --- a/DistTestCore/Codex/CodexApiTypes.cs +++ b/DistTestCore/Codex/CodexApiTypes.cs @@ -1,5 +1,6 @@ using KubernetesWorkflow; using Logging; +using Newtonsoft.Json; using Utils; namespace DistTestCore.Codex @@ -60,6 +61,16 @@ namespace DistTestCore.Codex { public string version { get; set; } = string.Empty; public string revision { get; set; } = string.Empty; + + public bool IsValid() + { + return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision); + } + + public override string ToString() + { + return JsonConvert.SerializeObject(this); + } } public class CodexDebugPeerResponse diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 4039d54..29bdc9e 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -5,7 +5,7 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-3e80de3"; + private const string DefaultDockerImage = "codexstorage/nim-codex:latest"; public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port"; diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 41dab1f..19c4c8c 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -20,6 +20,7 @@ namespace DistTestCore Setup = setup; Containers = containers; Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray(); + Version = new CodexDebugVersionResponse(); } public IOnlineCodexNode this[int index] @@ -46,6 +47,7 @@ namespace DistTestCore public CodexSetup Setup { get; private set; } public RunningContainers Containers { get; private set; } public OnlineCodexNode[] Nodes { get; private set; } + public CodexDebugVersionResponse Version { get; private set; } public IEnumerator GetEnumerator() { @@ -64,14 +66,17 @@ namespace DistTestCore public void EnsureOnline() { - foreach (var node in Nodes) + foreach (var node in Nodes) node.EnsureOnlineGetVersionResponse(); + var versionResponses = Nodes.Select(n => n.Version); + + var first = versionResponses.First(); + if (!versionResponses.All(v => v.version == first.version && v.revision == first.revision)) { - var debugInfo = node.CodexAccess.GetDebugInfo(); - var nodePeerId = debugInfo.id; - var nodeName = node.CodexAccess.Container.Name; - lifecycle.Log.AddStringReplace(nodePeerId, nodeName); - lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); + throw new Exception("Inconsistent version information received from one or more Codex nodes: " + + string.Join(",", versionResponses.Select(v => v.ToString()))); } + + Version = first; } private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 9f86836..23b60b1 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -22,6 +22,7 @@ namespace DistTestCore var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); var startupConfig = CreateStartupConfig(gethStartResult, codexSetup); + var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location); var metricAccessFactory = CollectMetrics(codexSetup, containers); @@ -29,9 +30,16 @@ namespace DistTestCore var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); + lifecycle.SetCodexVersion(group.Version); + var podInfo = group.Containers.RunningPod.PodInfo; - LogEnd($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers.First().Recipe.Image}' at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}"); + LogEnd($"Started {codexSetup.NumberOfNodes} nodes " + + $"of image '{containers.Containers.First().Recipe.Image}' " + + $"and version '{group.Version}' " + + $"at location '{podInfo.K8SNodeName}'={podInfo.Ip}. " + + $"They are: {group.Describe()}"); LogSeparator(); + return group; } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 252a2f4..6768618 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -28,7 +28,7 @@ namespace DistTestCore var logConfig = configuration.GetLogConfig(); var startTime = DateTime.UtcNow; fixtureLog = new FixtureLog(logConfig, startTime); - statusLog = new StatusLog(logConfig, startTime, new CodexContainerRecipe().Image); + statusLog = new StatusLog(logConfig, startTime); PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this); PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this); @@ -206,7 +206,7 @@ namespace DistTestCore var testResult = GetTestResult(); var testDuration = lifecycle.GetTestDuration(); fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})"); - statusLog.ConcludeTest(testResult, testDuration); + statusLog.ConcludeTest(testResult, testDuration, GetCodexId(lifecycle)); Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => { lifecycle.Log.EndTest(); @@ -216,6 +216,14 @@ namespace DistTestCore }); } + private static string GetCodexId(TestLifecycle lifecycle) + { + var v = lifecycle.CodexVersion; + if (v == null) return new CodexContainerRecipe().Image; + if (v.version != "untagged build") return v.version; + return v.revision; + } + private ITimeSet GetTimeSet() { if (ShouldUseLongTimeouts()) return new LongTimeSet(); diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 60716e7..5a7658c 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -18,6 +18,7 @@ namespace DistTestCore IDownloadedLog DownloadLog(); IMetricsAccess Metrics { get; } IMarketplaceAccess Marketplace { get; } + CodexDebugVersionResponse Version { get; } ICodexSetup BringOffline(); } @@ -34,12 +35,14 @@ namespace DistTestCore Group = group; Metrics = metricsAccess; Marketplace = marketplaceAccess; + Version = new CodexDebugVersionResponse(); } public CodexAccess CodexAccess { get; } public CodexNodeGroup Group { get; } public IMetricsAccess Metrics { get; } public IMarketplaceAccess Marketplace { get; } + public CodexDebugVersionResponse Version { get; private set; } public string GetName() { @@ -111,6 +114,22 @@ namespace DistTestCore return Group.BringOffline(); } + public void EnsureOnlineGetVersionResponse() + { + var debugInfo = CodexAccess.GetDebugInfo(); + var nodePeerId = debugInfo.id; + var nodeName = CodexAccess.Container.Name; + + if (!debugInfo.codex.IsValid()) + { + throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}"); + } + + lifecycle.Log.AddStringReplace(nodePeerId, nodeName); + lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); + Version = debugInfo.codex; + } + private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) { var multiAddress = peerInfo.addrs.First(); diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 50b665a..f33b238 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,4 +1,5 @@ -using DistTestCore.Logs; +using DistTestCore.Codex; +using DistTestCore.Logs; using KubernetesWorkflow; using Logging; using Utils; @@ -7,7 +8,7 @@ namespace DistTestCore { public class TestLifecycle { - private DateTime testStart = DateTime.MinValue; + private readonly DateTime testStart; public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet) : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet))) @@ -25,6 +26,7 @@ namespace DistTestCore PrometheusStarter = new PrometheusStarter(this, workflowCreator); GethStarter = new GethStarter(this, workflowCreator); testStart = DateTime.UtcNow; + CodexVersion = null; Log.WriteLogTag(); } @@ -36,6 +38,7 @@ namespace DistTestCore public CodexStarter CodexStarter { get; } public PrometheusStarter PrometheusStarter { get; } public GethStarter GethStarter { get; } + public CodexDebugVersionResponse? CodexVersion { get; private set; } public void DeleteAllResources() { @@ -60,5 +63,10 @@ namespace DistTestCore var testDuration = DateTime.UtcNow - testStart; return Time.FormatDuration(testDuration); } + + public void SetCodexVersion(CodexDebugVersionResponse version) + { + if (CodexVersion == null) CodexVersion = version; + } } } diff --git a/Logging/StatusLog.cs b/Logging/StatusLog.cs index 740a564..f0a366b 100644 --- a/Logging/StatusLog.cs +++ b/Logging/StatusLog.cs @@ -7,16 +7,14 @@ namespace Logging private readonly object fileLock = new object(); private readonly string fullName; private readonly string fixtureName; - private readonly string codexId; - public StatusLog(LogConfig config, DateTime start, string codexId, string name = "") + public StatusLog(LogConfig config, DateTime start, string name = "") { fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log"; fixtureName = NameUtils.GetRawFixtureName(); - this.codexId = codexId; } - public void ConcludeTest(string resultStatus, string testDuration) + public void ConcludeTest(string resultStatus, string testDuration, string codexId) { Write(new StatusLogJson {