From f23926636c1023024e5ad794933dcf0fb90ec030 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Jul 2023 10:18:37 +0200 Subject: [PATCH 1/4] Setting up way to run different codex images --- DistTestCore/AutoBootstrapDistTest.cs | 5 ++-- DistTestCore/Codex/CodexAccess.cs | 4 +-- DistTestCore/Codex/CodexContainerRecipe.cs | 16 +++++++--- DistTestCore/CodexStarter.cs | 2 +- .../Helpers/PeerConnectionTestHelpers.cs | 23 ++++++++++---- DistTestCore/Http.cs | 14 +++++---- DistTestCore/Timing.cs | 2 +- Logging/TestLog.cs | 7 ++++- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 1 - .../PeerDiscoveryTests/VariableImageTests.cs | 30 +++++++++++++++++++ 10 files changed, 80 insertions(+), 24 deletions(-) create mode 100644 Tests/PeerDiscoveryTests/VariableImageTests.cs diff --git a/DistTestCore/AutoBootstrapDistTest.cs b/DistTestCore/AutoBootstrapDistTest.cs index edf182f..25bb155 100644 --- a/DistTestCore/AutoBootstrapDistTest.cs +++ b/DistTestCore/AutoBootstrapDistTest.cs @@ -6,7 +6,7 @@ namespace DistTestCore { public override IOnlineCodexNode SetupCodexBootstrapNode(Action setup) { - throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " + + throw new Exception("AutoBootstrapDistTest creates and attaches a single bootstrap node for you. " + "If you want to control the bootstrap node from your test, please use DistTest instead."); } @@ -21,7 +21,8 @@ namespace DistTestCore [SetUp] public void SetUpBootstrapNode() { - BootstrapNode = BringOnline(CreateCodexSetup(1))[0]; + var setup = CreateCodexSetup(1).WithName("BOOTSTRAP"); + BootstrapNode = BringOnline(setup)[0]; } protected IOnlineCodexNode BootstrapNode { get; private set; } = null!; diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index fde9997..b6ae9fe 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -22,12 +22,12 @@ namespace DistTestCore.Codex public CodexDebugResponse GetDebugInfo() { - return Http(TimeSpan.FromSeconds(2)).HttpGetJson("debug/info"); + return Http(TimeSpan.FromSeconds(60)).HttpGetJson("debug/info"); } public CodexDebugPeerResponse GetDebugPeer(string peerId) { - return GetDebugPeer(peerId, TimeSpan.FromSeconds(2)); + return GetDebugPeer(peerId, TimeSpan.FromSeconds(10)); } public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index b94e5ba..0bbfc87 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -1,6 +1,5 @@ using DistTestCore.Marketplace; using KubernetesWorkflow; -using System.Net.Sockets; namespace DistTestCore.Codex { @@ -9,8 +8,8 @@ namespace DistTestCore.Codex #if Arm64 public const string DockerImage = "codexstorage/nim-codex:sha-6dd7e55"; #else - //public const string DockerImage = "thatbenbierens/nim-codex:dhting"; - public const string DockerImage = "codexstorage/nim-codex:sha-6dd7e55"; + public const string DockerImage = "thatbenbierens/nim-codex:loopingyeah"; + //public const string DockerImage = "codexstorage/nim-codex:sha-6dd7e55"; #endif public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port"; @@ -19,7 +18,16 @@ namespace DistTestCore.Codex public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0); public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0); - protected override string Image => DockerImage; + public static string DockerImageOverride = string.Empty; + + protected override string Image + { + get + { + if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride; + return DockerImage; + } + } protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 144d55f..9f86836 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -30,7 +30,7 @@ namespace DistTestCore var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); var podInfo = group.Containers.RunningPod.PodInfo; - LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}"); + LogEnd($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers.First().Recipe.Image}' at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}"); LogSeparator(); return group; } diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index 5eb0cdf..4320aa0 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -52,12 +52,12 @@ namespace DistTestCore.Helpers private static void RetryWhilePairs(List pairs, Action action) { - var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(10); + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30); while (pairs.Any() && timeout > DateTime.UtcNow) { action(); - if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(5)); + if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2)); } } @@ -140,6 +140,12 @@ namespace DistTestCore.Helpers } } + public override string ToString() + { + if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN"; + return Response.id; + } + private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node) { var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId); @@ -188,10 +194,15 @@ namespace DistTestCore.Helpers return GetResultMessage() + GetTimePostfix(); } + public override string ToString() + { + return $"[{GetMessage()}]"; + } + private string GetResultMessage() { - var aName = A.Response.id; - var bName = B.Response.id; + var aName = A.ToString(); + var bName = B.ToString(); if (Success) { @@ -203,8 +214,8 @@ namespace DistTestCore.Helpers private string GetTimePostfix() { - var aName = A.Response.id; - var bName = B.Response.id; + var aName = A.ToString(); + var bName = B.ToString(); return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)"; } diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index d4b716a..f20c315 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -127,16 +127,18 @@ namespace DistTestCore private T Retry(Func operation, string description) { - return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay(), description); + return Time.Retry(operation, GetTimeout(), timeSet.HttpCallRetryDelay(), description); } private HttpClient GetClient() { - if (timeoutOverride.HasValue) - { - return GetClient(timeoutOverride.Value); - } - return GetClient(timeSet.HttpCallTimeout()); + return GetClient(GetTimeout()); + } + + private TimeSpan GetTimeout() + { + if (timeoutOverride.HasValue) return timeoutOverride.Value; + return timeSet.HttpCallTimeout(); } private HttpClient GetClient(TimeSpan timeout) diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index c395d07..a653ce2 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -41,7 +41,7 @@ namespace DistTestCore public TimeSpan K8sOperationTimeout() { - return TimeSpan.FromMinutes(5); + return TimeSpan.FromMinutes(1); } public TimeSpan WaitForMetricTimeout() diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index ed2cdfd..1bf5d41 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -43,7 +43,7 @@ namespace Logging if (!string.IsNullOrEmpty(name)) return name; var test = TestContext.CurrentContext.Test; var args = FormatArguments(test); - return $"{test.MethodName}{args}"; + return ReplaceInvalidCharacters($"{test.MethodName}{args}"); } private static string FormatArguments(TestContext.TestAdapter test) @@ -51,5 +51,10 @@ namespace Logging if (test.Arguments == null || !test.Arguments.Any()) return ""; return $"[{string.Join(',', test.Arguments)}]"; } + + private static string ReplaceInvalidCharacters(string name) + { + return name.Replace(":", "_"); + } } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 57b79cf..002fe79 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,7 +1,6 @@ using DistTestCore; using DistTestCore.Helpers; using NUnit.Framework; -using Utils; namespace Tests.PeerDiscoveryTests { diff --git a/Tests/PeerDiscoveryTests/VariableImageTests.cs b/Tests/PeerDiscoveryTests/VariableImageTests.cs new file mode 100644 index 0000000..54af262 --- /dev/null +++ b/Tests/PeerDiscoveryTests/VariableImageTests.cs @@ -0,0 +1,30 @@ +using DistTestCore; +using DistTestCore.Codex; +using NUnit.Framework; + +namespace Tests.PeerDiscoveryTests +{ + [TestFixture] + public class VariableImageTests : DistTest + { + [TestCase("nim-codex:sha-a899384")] + [TestCase("nim-codex:sha-3879ec8")] + [TestCase("nim-codex:sha-6dd7e55")] + [TestCase("nim-codex:sha-3f2b417")] + [TestCase("nim-codex:sha-00f6554")] + [TestCase("nim-codex:sha-f053135")] + public void ThreeNodes(string dockerImage) + { + var img = "codexstorage/" + dockerImage; + Log("Image override: " + img); + CodexContainerRecipe.DockerImageOverride = img; + + var boot = SetupCodexBootstrapNode(); + SetupCodexNode(c => c.WithBootstrapNode(boot)); + SetupCodexNode(c => c.WithBootstrapNode(boot)); + + PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); + } + } +} + From b295314bde164c399df1e49308c40e0f6ef8ec52 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Jul 2023 10:45:26 +0200 Subject: [PATCH 2/4] fixes blockTTL envvar --- DistTestCore/Codex/CodexContainerRecipe.cs | 2 +- Tests/PeerDiscoveryTests/VariableImageTests.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 0bbfc87..67e59ef 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -56,7 +56,7 @@ namespace DistTestCore.Codex } if (config.BlockTTL != null) { - AddEnvVar("BLOCK_TTL", config.BlockTTL.ToString()!); + AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!); } if (config.MetricsEnabled) { diff --git a/Tests/PeerDiscoveryTests/VariableImageTests.cs b/Tests/PeerDiscoveryTests/VariableImageTests.cs index 54af262..95a8dee 100644 --- a/Tests/PeerDiscoveryTests/VariableImageTests.cs +++ b/Tests/PeerDiscoveryTests/VariableImageTests.cs @@ -13,6 +13,7 @@ namespace Tests.PeerDiscoveryTests [TestCase("nim-codex:sha-3f2b417")] [TestCase("nim-codex:sha-00f6554")] [TestCase("nim-codex:sha-f053135")] + [TestCase("nim-codex:sha-711e5e0")] public void ThreeNodes(string dockerImage) { var img = "codexstorage/" + dockerImage; From adeba216653c9eb4e729948e9868159407d1601e Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Jul 2023 08:54:07 +0200 Subject: [PATCH 3/4] Disables debug output by default --- DistTestCore/Configuration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index fceb4e0..03514cd 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -17,7 +17,7 @@ namespace DistTestCore { kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null); logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs"); - logDebug = GetEnvVarOrDefault("LOGDEBUG", "true").ToLowerInvariant() == "true"; + logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true"; dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); From 4815194db007280167288296755ac8aa9294ebd3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Jul 2023 08:56:18 +0200 Subject: [PATCH 4/4] Removes variable image tests. --- .../PeerDiscoveryTests/VariableImageTests.cs | 31 ------------------- 1 file changed, 31 deletions(-) delete mode 100644 Tests/PeerDiscoveryTests/VariableImageTests.cs diff --git a/Tests/PeerDiscoveryTests/VariableImageTests.cs b/Tests/PeerDiscoveryTests/VariableImageTests.cs deleted file mode 100644 index 95a8dee..0000000 --- a/Tests/PeerDiscoveryTests/VariableImageTests.cs +++ /dev/null @@ -1,31 +0,0 @@ -using DistTestCore; -using DistTestCore.Codex; -using NUnit.Framework; - -namespace Tests.PeerDiscoveryTests -{ - [TestFixture] - public class VariableImageTests : DistTest - { - [TestCase("nim-codex:sha-a899384")] - [TestCase("nim-codex:sha-3879ec8")] - [TestCase("nim-codex:sha-6dd7e55")] - [TestCase("nim-codex:sha-3f2b417")] - [TestCase("nim-codex:sha-00f6554")] - [TestCase("nim-codex:sha-f053135")] - [TestCase("nim-codex:sha-711e5e0")] - public void ThreeNodes(string dockerImage) - { - var img = "codexstorage/" + dockerImage; - Log("Image override: " + img); - CodexContainerRecipe.DockerImageOverride = img; - - var boot = SetupCodexBootstrapNode(); - SetupCodexNode(c => c.WithBootstrapNode(boot)); - SetupCodexNode(c => c.WithBootstrapNode(boot)); - - PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); - } - } -} -