a

ContinuousTests/ContinuousLogDownloader.cs

@@ -7,14 +7,16 @@ namespace ContinuousTests
     public class ContinuousLogDownloader
     {
         private readonly TestLifecycle lifecycle;
-        private readonly CodexDeployment deployment;
+        private readonly RunningContainer[] containers;
+        //private readonly CodexDeployment deployment;
         private readonly string outputPath;
         private readonly CancellationToken cancelToken;
 
-        public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
+        public ContinuousLogDownloader(TestLifecycle lifecycle, RunningContainer[] containers, string outputPath, CancellationToken cancelToken)
         {
             this.lifecycle = lifecycle;
-            this.deployment = deployment;
+            this.containers = containers;
+            //this.deployment = deployment;
             this.outputPath = outputPath;
             this.cancelToken = cancelToken;
         }
@@ -37,7 +39,7 @@ namespace ContinuousTests
 
         private void UpdateLogs()
         {
-            foreach (var container in deployment.CodexContainers)
+            foreach (var container in containers)
             {
                 UpdateLog(container);
             }

ContinuousTests/ContinuousTestRunner.cs

@@ -72,7 +72,7 @@ namespace ContinuousTests
             if (!Directory.Exists(path)) Directory.CreateDirectory(path);
 
             var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
-            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
+            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment.CodexContainers, path, cancelToken);
 
             taskFactory.Run(downloader.Run);
         }

DistTestCore/DistTest.cs

@@ -179,7 +179,7 @@ namespace DistTestCore
             return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
         }
 
-        private TestLifecycle Get()
+        public TestLifecycle Get()
         {
             lock (lifecycleLock)
             {

DistTestCore/Http.cs

@@ -125,7 +125,14 @@ namespace DistTestCore
 
         private T Retry<T>(Func<T> operation, string description)
         {
-            return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
+            try
+            {
+                return operation();
+            }
+            catch (Exception ex)
+            {
+                throw new Exception(description, ex);
+            }
         }
 
         private HttpClient GetClient()

DistTestCore/OnlineCodexNode.cs

@@ -4,6 +4,7 @@ using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
 using Logging;
 using NUnit.Framework;
+using Utils;
 
 namespace DistTestCore
 {
@@ -66,7 +67,8 @@ namespace DistTestCore
         {
             using var fileStream = File.OpenRead(file.Filename);
 
-            var logMessage = $"Uploading file {file.Describe()}...";
+            var logMessage = $"{CodexAccess.Container.Name} Uploading file {file.Describe()}...";
+            Log(logMessage);
             var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream);
@@ -81,7 +83,8 @@ namespace DistTestCore
 
         public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
-            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
+            var logMessage = $"{CodexAccess.Container.Name} Downloading for contentId: '{contentId.Id}'...";
+            Log(logMessage);
             var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
             Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
@@ -116,7 +119,7 @@ namespace DistTestCore
 
         public void EnsureOnlineGetVersionResponse()
         {
-            var debugInfo = CodexAccess.GetDebugInfo();
+            var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
             var nodePeerId = debugInfo.id;
             var nodeName = CodexAccess.Container.Name;
 

Tests/BasicTests/ContinuousSubstitute.cs

@@ -0,0 +1,134 @@
+using ContinuousTests;
+using DistTestCore;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.BasicTests
+{
+    [TestFixture]
+    public class ContinuousSubstitute : AutoBootstrapDistTest
+    {
+        [Test]
+        public void ContinuousTestSubstitute()
+        {
+            var nodes = new List<OnlineCodexNode>();
+            for (var i = 0; i < 5; i++)
+            {
+                nodes.Add((OnlineCodexNode)SetupCodexNode(o => o
+                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: i < 2)
+                    .WithStorageQuota(2.GB())
+                ));
+            }
+
+            var cts = new CancellationTokenSource();
+            var ct = cts.Token;
+            var dlPath = Path.Combine(new FileInfo(Get().Log.LogFile.FullFilename)!.Directory!.FullName, "continuouslogs");
+            Directory.CreateDirectory(dlPath);
+
+            var containers = nodes.Select(n => n.CodexAccess.Container).ToArray();
+            var cd = new ContinuousLogDownloader(Get(), containers, dlPath, ct);
+
+            var logTask = Task.Run(cd.Run);
+
+            try
+            {
+                foreach (var node in nodes)
+                {
+                    node.Marketplace.MakeStorageAvailable(
+                    size: 1.GB(),
+                    minPricePerBytePerSecond: 1.TestTokens(),
+                    maxCollateral: 1024.TestTokens(),
+                    maxDuration: TimeSpan.FromMinutes(5));
+                }
+
+                var endTime = DateTime.UtcNow + TimeSpan.FromHours(1);
+                while (DateTime.UtcNow < endTime)
+                {
+                    var allNodes = nodes.ToList();
+                    var primary = allNodes.PickOneRandom();
+                    var secondary = allNodes.PickOneRandom();
+
+                    Log("Run Test");
+                    PerformTest(primary, secondary);
+
+                    Thread.Sleep(TimeSpan.FromSeconds(30));
+                }
+            }
+            finally
+            {
+                cts.Cancel();
+                logTask.Wait();
+            }
+        }
+
+        [Test]
+        public void JustWaitFirst()
+        {
+            var nodes = new List<OnlineCodexNode>();
+            for (var i = 0; i < 5; i++)
+            {
+                nodes.Add((OnlineCodexNode)SetupCodexNode(o => o
+                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: i < 2)
+                    .WithStorageQuota(2.GB())
+                ));
+            }
+
+            var cts = new CancellationTokenSource();
+            var ct = cts.Token;
+            var dlPath = Path.Combine(new FileInfo(Get().Log.LogFile.FullFilename)!.Directory!.FullName, "continuouslogsjustwait");
+            Directory.CreateDirectory(dlPath);
+
+            var containers = nodes.Select(n => n.CodexAccess.Container).ToArray();
+            var cd = new ContinuousLogDownloader(Get(), containers, dlPath, ct);
+
+            var logTask = Task.Run(cd.Run);
+
+            try
+            {
+                foreach (var node in nodes)
+                {
+                    node.Marketplace.MakeStorageAvailable(
+                    size: 1.GB(),
+                    minPricePerBytePerSecond: 1.TestTokens(),
+                    maxCollateral: 1024.TestTokens(),
+                    maxDuration: TimeSpan.FromMinutes(5));
+                }
+
+                Log("Waiting 30 minutes...");
+                Thread.Sleep(TimeSpan.FromMinutes(30));
+
+                var endTime = DateTime.UtcNow + TimeSpan.FromHours(1);
+                while (DateTime.UtcNow < endTime)
+                {
+                    var allNodes = nodes.ToList();
+                    var primary = allNodes.PickOneRandom();
+                    var secondary = allNodes.PickOneRandom();
+
+                    Log("Run Test");
+                    PerformTest(primary, secondary);
+
+                    Thread.Sleep(TimeSpan.FromSeconds(30));
+                }
+            }
+            finally
+            {
+                cts.Cancel();
+                logTask.Wait();
+            }
+        }
+
+        private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
+        {
+            ScopedTestFiles(() =>
+            {
+                var testFile = GenerateTestFile(10.MB());
+
+                var contentId = primary.UploadFile(testFile);
+
+                var downloadedFile = secondary.DownloadContent(contentId);
+
+                testFile.AssertIsEqual(downloadedFile);
+            });
+        }
+    }
+}

Tests/Tests.csproj

@@ -13,6 +13,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
   </ItemGroup>