Merge branch 'feature/continuous-peer-test'

CodexNetDeployer/CodexNodeStarter.cs

@@ -28,6 +28,7 @@ namespace CodexNetDeployer
             var workflowStartup = new StartupConfig();
             workflowStartup.Add(gethResult);
             workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
+            workflowStartup.NameOverride = GetCodexContainerName(i);
 
             var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
 
@@ -75,6 +76,12 @@ namespace CodexNetDeployer
             return null;
         }
 
+        private string GetCodexContainerName(int i)
+        {
+            if (i == 0) return "BOOTSTRAP";
+            return "CODEX" + i;
+        }
+
         private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
         {
             var codexStart = new CodexStartupConfig(config.CodexLogLevel);

ContinuousTests/Configuration.cs

@@ -22,8 +22,8 @@ namespace ContinuousTests
         [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
         public string KubeConfigFile { get; set; } = "null";
 
-        [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
+        [Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")]
-        public bool StopOnFailure { get; set; } = false;
+        public int StopOnFailure { get; set; } = 0;
 
         [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
         public bool DownloadContainerLogs { get; set; } = false;

ContinuousTests/ContinuousTestRunner.cs

@@ -30,7 +30,7 @@ namespace ContinuousTests
 
             ClearAllCustomNamespaces(allTests, overviewLog);
 
-            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
+            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
 
             foreach (var testLoop in testLoops)
             {

ContinuousTests/SingleTestRun.cs

@@ -23,8 +23,9 @@ namespace ContinuousTests
         private readonly FixtureLog fixtureLog;
         private readonly string testName;
         private readonly string dataFolder;
+        private static int failureCount = 0;
 
-        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken)
+        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
         {
             this.taskFactory = taskFactory;
             this.config = config;
@@ -33,8 +34,9 @@ namespace ContinuousTests
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
             fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
+            ApplyLogReplacements(fixtureLog, startupChecker);
 
-            nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
+            nodes = CreateRandomNodes();
             dataFolder = config.DataPath + "-" + Guid.NewGuid();
             fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
         }
@@ -71,16 +73,26 @@ namespace ContinuousTests
                 fixtureLog.Error("Test run failed with exception: " + ex);
                 fixtureLog.MarkAsFailed();
 
-                if (config.StopOnFailure)
+                failureCount++;
+                if (config.StopOnFailure > 0)
                 {
-                    OverviewLog("Configured to stop on first failure. Downloading cluster logs...");
+                    OverviewLog($"Failures: {failureCount} / {config.StopOnFailure}");
-                    DownloadClusterLogs();
+                    if (failureCount >= config.StopOnFailure)
-                    OverviewLog("Log download finished. Cancelling test runner...");
+                    {
-                    Cancellation.Cts.Cancel();
+                        OverviewLog($"Configured to stop after {config.StopOnFailure} failures. Downloading cluster logs...");
+                        DownloadClusterLogs();
+                        OverviewLog("Log download finished. Cancelling test runner...");
+                        Cancellation.Cts.Cancel();
+                    }
                 }
             }
         }
 
+        private void ApplyLogReplacements(FixtureLog fixtureLog, StartupChecker startupChecker)
+        {
+            foreach (var replacement in startupChecker.LogReplacements) fixtureLog.AddStringReplace(replacement.From, replacement.To);
+        }
+
         private void RunTestMoments()
         {
             var earliestMoment = handle.GetEarliestMoment();
@@ -112,7 +124,7 @@ namespace ContinuousTests
                     {
                         ThrowFailTest();
                     }
-                    OverviewLog(" > Test passed. " + FuturesInfo());
+                    OverviewLog(" > Test passed.");
                     return;
                 }
             }
@@ -120,25 +132,19 @@ namespace ContinuousTests
 
         private void ThrowFailTest()
         {
-            var ex = UnpackException(exceptions.First());
+            var exs = UnpackExceptions(exceptions);
-            Log(ex.ToString());
+            var exceptionsMessage = GetCombinedExceptionsMessage(exs);
-            OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message);
+            Log(exceptionsMessage);
-            throw ex;
+            OverviewLog($" > Test failed: " + exceptionsMessage);
-        }
+            throw new Exception(exceptionsMessage);
-
-        private string FuturesInfo()
-        {
-            var containers = config.CodexDeployment.CodexContainers;
-            var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
-            var f = nodes.Select(n => n.GetDebugFutures().ToString());
-            var msg = $"(Futures: [{string.Join(", ", f)}])";
-            return msg;
         }
 
         private void DownloadClusterLogs()
         {
             var k8sFactory = new K8sFactory();
-            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
+            var log = new NullLog();
+            log.FullFilename = Path.Combine(config.LogPath, "NODE");
+            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), log);
 
             foreach (var container in config.CodexDeployment.CodexContainers)
             {
@@ -146,6 +152,16 @@ namespace ContinuousTests
             }
         }
 
+        private string GetCombinedExceptionsMessage(Exception[] exceptions)
+        {
+            return string.Join(Environment.NewLine, exceptions.Select(ex => ex.ToString()));
+        }
+
+        private Exception[] UnpackExceptions(List<Exception> exceptions)
+        {
+            return exceptions.Select(UnpackException).ToArray();
+        }
+
         private Exception UnpackException(Exception exception)
         {
             if (exception is AggregateException a)
@@ -196,19 +212,26 @@ namespace ContinuousTests
         private void OverviewLog(string msg)
         {
             Log(msg);
-            var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
+            var containerNames = GetContainerNames();
             overviewLog.Log($"{containerNames} {testName}: {msg}");
         }
 
-        private CodexAccess[] CreateRandomNodes(int number)
+        private string GetContainerNames()
         {
-            var containers = SelectRandomContainers(number);
+            if (handle.Test.RequiredNumberOfNodes == -1) return "(All Nodes)";
+            return $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
+        }
+
+        private CodexAccess[] CreateRandomNodes()
+        {
+            var containers = SelectRandomContainers();
             fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
             return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
         }
 
-        private RunningContainer[] SelectRandomContainers(int number)
+        private RunningContainer[] SelectRandomContainers()
         {
+            var number = handle.Test.RequiredNumberOfNodes;
             if (number == -1) return config.CodexDeployment.CodexContainers;
 
             var containers = config.CodexDeployment.CodexContainers.ToList();

ContinuousTests/StartupChecker.cs

@@ -15,6 +15,7 @@ namespace ContinuousTests
         {
             this.config = config;
             this.cancelToken = cancelToken;
+            LogReplacements = new List<BaseLogStringReplacement>();
         }
 
         public void Check()
@@ -28,6 +29,8 @@ namespace ContinuousTests
             log.Log("All OK.");
         }
 
+        public List<BaseLogStringReplacement> LogReplacements { get; }
+
         private void PreflightCheck(Configuration config)
         {
             var tests = testFactory.CreateTests();
@@ -90,6 +93,7 @@ namespace ContinuousTests
                 if (info == null || string.IsNullOrEmpty(info.id)) return false;
 
                 log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'");
+                LogReplacements.Add(new BaseLogStringReplacement(info.id, n.GetName()));
             }
             catch
             {

ContinuousTests/TestLoop.cs

@@ -9,16 +9,18 @@ namespace ContinuousTests
         private readonly BaseLog overviewLog;
         private readonly Type testType;
         private readonly TimeSpan runsEvery;
+        private readonly StartupChecker startupChecker;
         private readonly CancellationToken cancelToken;
         private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
 
-        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken)
+        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
         {
             this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
             this.testType = testType;
             this.runsEvery = runsEvery;
+            this.startupChecker = startupChecker;
             this.cancelToken = cancelToken;
             Name = testType.Name;
         }
@@ -58,7 +60,7 @@ namespace ContinuousTests
         {
             var test = (ContinuousTest)Activator.CreateInstance(testType)!;
             var handle = new TestHandle(test);
-            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken);
+            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, startupChecker, cancelToken);
 
             runFinishedHandle.Reset();
             run.Run(runFinishedHandle);

ContinuousTests/Tests/HoldMyBeerTest.cs

@@ -15,17 +15,7 @@ namespace ContinuousTests.Tests
         [TestMoment(t: Zero)]
         public void UploadTestFile()
         {
-            var metadata = Configuration.CodexDeployment.Metadata;
+            var filesize = 80.MB();
-            var maxQuotaUseMb = metadata.StorageQuotaMB / 2;
-            var safeTTL = Math.Max(metadata.BlockTTL, metadata.BlockMI) + 30;
-            var runsPerTtl = Convert.ToInt32(safeTTL / RunTestEvery.TotalSeconds);
-            var filesizePerUploadMb = Math.Min(80, maxQuotaUseMb / runsPerTtl);
-            // This filesize should keep the quota below 50% of the node's max.
-
-            var filesize = filesizePerUploadMb.MB();
-            double codexDefaultBlockSize = 31 * 64 * 33;
-            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
-            Assert.That(numberOfBlocks, Is.EqualTo(1282));
 
             file = FileManager.GenerateTestFile(filesize);
 

ContinuousTests/Tests/PeersTest.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Codex;
+using DistTestCore.Helpers;
 using NUnit.Framework;
 
 namespace ContinuousTests.Tests
@@ -10,10 +11,24 @@ namespace ContinuousTests.Tests
         public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
 
         [TestMoment(t: 0)]
+        public void CheckConnectivity()
+        {
+            var checker = new PeerConnectionTestHelpers(Log);
+            checker.AssertFullyConnected(Nodes);
+        }
+
+        [TestMoment(t: 10)]
         public void CheckRoutingTables()
         {
-            var allIds = Nodes.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray();
+            var allInfos = Nodes.Select(n =>
+            {
+                var info = n.GetDebugInfo();
+                Log.Log($"{n.GetName()} = {info.table.localNode.nodeId}");
+                Log.AddStringReplace(info.table.localNode.nodeId, n.GetName());
+                return info;
+            }).ToArray();
 
+            var allIds = allInfos.Select(i => i.table.localNode.nodeId).ToArray();
             var errors = Nodes.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
 
             if (errors.Any())
@@ -30,7 +45,10 @@ namespace ContinuousTests.Tests
 
             if (!expected.All(ex => known.Contains(ex)))
             {
-                return $"Not all of '{string.Join(",", expected)}' were present in routing table: '{string.Join(",", known)}'";
+                var nl = Environment.NewLine;
+                return $"{nl}At node '{info.table.localNode.nodeId}'{nl}" +
+                    $"Not all of{nl}'{string.Join(",", expected)}'{nl}" +
+                    $"were present in routing table:{nl}'{string.Join(",", known)}'";
             }
 
             return string.Empty;

ContinuousTests/run.sh

@@ -2,5 +2,5 @@ dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
     --codex-deployment=codex-deployment.json \
     --keep=1 \
-    --stop=1 \
+    --stop=10 \
     --dl-logs=1

DistTestCore/Codex/CodexAccess.cs

@@ -47,12 +47,6 @@ namespace DistTestCore.Codex
             return result;
         }
 
-        public int GetDebugFutures()
-        {
-            // Some Codex images support debug/futures to count the number of open futures.
-            return 0; // Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
-        }
-
         public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
         {
             return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
@@ -75,7 +69,7 @@ namespace DistTestCore.Codex
 
         public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
         {
-            return Http().HttpPostJson<CodexSalesRequestStorageRequest, string>($"storage/request/{contentId}", request);
+            return Http().HttpPostJson($"storage/request/{contentId}", request);
         }
 
         public CodexStoragePurchase GetPurchaseStatus(string purchaseId)

DistTestCore/Codex/CodexApiTypes.cs

@@ -1,7 +1,4 @@
-using KubernetesWorkflow;
+using Newtonsoft.Json;
-using Logging;
-using Newtonsoft.Json;
-using Utils;
 
 namespace DistTestCore.Codex
 {

DistTestCore/Http.cs

@@ -55,25 +55,20 @@ namespace DistTestCore
 
         public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
         {
-            var response = HttpPostJson(route, body);
+            var response = PostJson(route, body);
             var json = Time.Wait(response.Content.ReadAsStringAsync());
-            if(!response.IsSuccessStatusCode) {
+            if (!response.IsSuccessStatusCode)
+            {
                 throw new HttpRequestException(json);
             }
             Log(GetUrl() + route, json);
             return TryJsonDeserialize<TResponse>(json);
         }
 
-        public HttpResponseMessage HttpPostJson<TRequest>(string route, TRequest body)
+        public string HttpPostJson<TRequest>(string route, TRequest body)
         {
-            return Retry(() =>
+            var response = PostJson<TRequest>(route, body);
-            {
+            return Time.Wait(response.Content.ReadAsStringAsync());
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                using var content = JsonContent.Create(body);
-                Log(url, JsonConvert.SerializeObject(body));
-                return Time.Wait(client.PostAsync(url, content));
-            }, $"HTTP-POST-JSON: {route}");
         }
 
         public string HttpPostString(string route, string body)
@@ -122,7 +117,8 @@ namespace DistTestCore
         public T TryJsonDeserialize<T>(string json)
         {
             var errors = new List<string>();
-            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings(){
+            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
+            {
                 Error = delegate(object? sender, Serialization.ErrorEventArgs args)
                 {
                     if (args.CurrentObject == args.ErrorContext.OriginalObject)
@@ -136,15 +132,29 @@ namespace DistTestCore
                     }
                 }
             });
-            if (errors.Count() > 0) {
+            if (errors.Count() > 0)
+            {
                 throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
             }
-            else if (deserialized == null) {
+            else if (deserialized == null)
+            {
                 throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
             }
             return deserialized;
         }
 
+        private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                using var content = JsonContent.Create(body);
+                Log(url, JsonConvert.SerializeObject(body));
+                return Time.Wait(client.PostAsync(url, content));
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
         private string GetUrl()
         {
             return $"{address.Host}:{address.Port}{baseUrl}";

Logging/BaseLog.cs

@@ -60,6 +60,7 @@ namespace Logging
         public virtual void AddStringReplace(string from, string to)
         {
             if (string.IsNullOrWhiteSpace(from)) return;
+            if (replacements.Any(r => r.From == from)) return;
             replacements.Add(new BaseLogStringReplacement(from, to));
         }
 
@@ -98,20 +99,20 @@ namespace Logging
 
     public class BaseLogStringReplacement
     {
-        private readonly string from;
-        private readonly string to;
-
         public BaseLogStringReplacement(string from, string to)
         {
-            this.from = from;
+            From = from;
-            this.to = to;
+            To = to;
 
             if (string.IsNullOrEmpty(from) || string.IsNullOrEmpty(to) || from == to) throw new ArgumentException();
         }
 
+        public string From { get; }
+        public string To { get; }
+
         public string Apply(string msg)
         {
-            return msg.Replace(from, to);
+            return msg.Replace(From, To);
         }
     }
 }

Logging/NullLog.cs

@@ -6,9 +6,11 @@
         {
         }
 
+        public string FullFilename { get; set; } = "NULL";
+
         protected override string GetFullName()
         {
-            return "NULL";
+            return FullFilename;
         }
 
         public override void Log(string message)

Tests/BasicTests/ContinuousSubstitute.cs

@@ -4,6 +4,7 @@ using Utils;
 
 namespace Tests.BasicTests
 {
+    [Ignore("Used for debugging continuous tests")]
     [TestFixture]
     public class ContinuousSubstitute : AutoBootstrapDistTest
     {
@@ -63,13 +64,13 @@ namespace Tests.BasicTests
             while (DateTime.UtcNow < endTime)
             {
                 CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
+                CheckRoutingTables(GetAllOnlineCodexNodes());
 
-                if (DateTime.UtcNow > checkTime)
+                var node = RandomUtils.PickOneRandom(nodes.ToList());
-                {
+                var file = GenerateTestFile(50.MB());
-                    CheckRoutingTables(GetAllOnlineCodexNodes());
+                node.UploadFile(file);
-                }
 
-                Thread.Sleep(5000);
+                Thread.Sleep(20000);
             }
         }