Automatic quota line. Holding test.

CodexNetDeployer/Deployer.cs

@@ -85,7 +85,7 @@ namespace CodexNetDeployer
             if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
 
             Log("Starting dashboard service...");
-            var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer);
+            var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
             return (prometheusContainer, grafanaStartInfo);
         }
 

DistTestCore/CodexStarter.cs

@@ -74,7 +74,7 @@ namespace DistTestCore
 
             if (codexSetup.MetricsMode == MetricsMode.Dashboard)
             {
-                lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First());
+                lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
             }
 
             return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);

DistTestCore/GrafanaStarter.cs

@@ -8,11 +8,14 @@ namespace DistTestCore
 {
     public class GrafanaStarter : BaseStarter
     {
+        private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
+        private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
+
         public GrafanaStarter(TestLifecycle lifecycle)
             : base(lifecycle)
         {
         }
-        public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer)
+        public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
         {
             LogStart($"Starting dashboard server");
 
@@ -25,7 +28,7 @@ namespace DistTestCore
             AddDataSource(http, prometheusContainer);
 
             Log("Uploading dashboard configurations...");
-            var jsons = ReadEachDashboardJsonFile();
+            var jsons = ReadEachDashboardJsonFile(codexSetup);
             var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
 
             LogEnd("Dashboard server started.");
@@ -86,7 +89,7 @@ namespace DistTestCore
             return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
         }
 
-        private static string[] ReadEachDashboardJsonFile()
+        private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
         {
             var assembly = Assembly.GetExecutingAssembly();
             var resourceNames = new[]
@@ -94,15 +97,45 @@ namespace DistTestCore
                 "DistTestCore.Metrics.dashboard.json"
             };
 
-            return resourceNames.Select(r => GetManifestResource(assembly, r)).ToArray();
+            return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
         }
 
-        private static string GetManifestResource(Assembly assembly, string resourceName)
+        private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
         {
             using var stream = assembly.GetManifestResourceStream(resourceName);
             if (stream == null) throw new Exception("Unable to find resource " + resourceName);
             using var reader = new StreamReader(stream);
-            return reader.ReadToEnd();
+            return ApplyReplacements(reader.ReadToEnd(), codexSetup);
+        }
+
+        private static string ApplyReplacements(string input, CodexSetup codexSetup)
+        {
+            var quotaString = GetQuotaString(codexSetup);
+            var softMaxString = GetSoftMaxString(codexSetup);
+
+            return input
+                .Replace(StorageQuotaThresholdReplaceToken, quotaString)
+                .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
+        }
+
+        private static string GetQuotaString(CodexSetup codexSetup)
+        {
+            return GetCodexStorageQuotaInBytes(codexSetup).ToString();
+        }
+
+        private static string GetSoftMaxString(CodexSetup codexSetup)
+        {
+            var quota = GetCodexStorageQuotaInBytes(codexSetup);
+            var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
+            return softMax.ToString();
+        }
+
+        private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
+        {
+            if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
+
+            // Codex default: 8GB
+            return 8.GB().SizeInBytes;
         }
 
         private static string GetDashboardCreateRequest(string dashboardJson)

DistTestCore/Http.cs

@@ -148,14 +148,7 @@ namespace DistTestCore
 
         private T Retry<T>(Func<T> operation, string description)
         {
-            try
+            return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
-            {
-                return operation();
-            }
-            catch (Exception ex)
-            {
-                throw new Exception(description, ex);
-            }
         }
 
         private HttpClient GetClient()

DistTestCore/Metrics/dashboard.json

@@ -431,6 +431,8 @@
             "axisColorMode": "text",
             "axisLabel": "",
             "axisPlacement": "auto",
+            "axisSoftMax": "<CODEX_BYTESUSED_SOFTMAX>",
+            "axisSoftMin": 0,
             "barAlignment": 0,
             "drawStyle": "line",
             "fillOpacity": 0,
@@ -453,7 +455,7 @@
               "mode": "none"
             },
             "thresholdsStyle": {
-              "mode": "off"
+              "mode": "line"
             }
           },
           "mappings": [],
@@ -466,7 +468,7 @@
               },
               {
                 "color": "red",
-                "value": 80
+                "value": "<CODEX_STORAGEQUOTA>"
               }
             ]
           }

Tests/BasicTests/ContinuousSubstitute.cs

@@ -1,5 +1,4 @@
-using ContinuousTests;
+using DistTestCore;
-using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
@@ -12,53 +11,34 @@ namespace Tests.BasicTests
         [UseLongTimeouts]
         public void ContinuousTestSubstitute()
         {
-            var nodes = new List<OnlineCodexNode>();
+            var group = SetupCodexNodes(5, o => o
-            for (var i = 0; i < 5; i++)
+                    .EnableMetrics()
+                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
+                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithStorageQuota(3.GB()));
+
+            var nodes = group.Cast<OnlineCodexNode>().ToArray();
+
+            foreach (var node in nodes)
             {
-                nodes.Add((OnlineCodexNode)SetupCodexNode(o => o
+                node.Marketplace.MakeStorageAvailable(
-                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: i < 2)
+                size: 1.GB(),
-                    .WithStorageQuota(3.GB())
+                minPricePerBytePerSecond: 1.TestTokens(),
-                ));
+                maxCollateral: 1024.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(5));
             }
 
-            var cts = new CancellationTokenSource();
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
-            var ct = cts.Token;
+            while (DateTime.UtcNow < endTime)
-            var dlPath = Path.Combine(new FileInfo(Get().Log.LogFile.FullFilename)!.Directory!.FullName, "continuouslogs");
-            Directory.CreateDirectory(dlPath);
-
-            var containers = nodes.Select(n => n.CodexAccess.Container).ToArray();
-            var cd = new ContinuousLogDownloader(Get(), containers, dlPath, ct);
-
-            var logTask = Task.Run(cd.Run);
-
-            try
             {
-                foreach (var node in nodes)
+                var allNodes = nodes.ToList();
-                {
+                var primary = allNodes.PickOneRandom();
-                    node.Marketplace.MakeStorageAvailable(
+                var secondary = allNodes.PickOneRandom();
-                    size: 1.GB(),
-                    minPricePerBytePerSecond: 1.TestTokens(),
-                    maxCollateral: 1024.TestTokens(),
-                    maxDuration: TimeSpan.FromMinutes(5));
-                }
 
-                var endTime = DateTime.UtcNow + TimeSpan.FromHours(1);
+                Log("Run Test");
-                while (DateTime.UtcNow < endTime)
+                PerformTest(primary, secondary);
-                {
-                    var allNodes = nodes.ToList();
-                    var primary = allNodes.PickOneRandom();
-                    var secondary = allNodes.PickOneRandom();
 
-                    Log("Run Test");
+                Thread.Sleep(TimeSpan.FromSeconds(5));
-                    PerformTest(primary, secondary);
-
-                    Thread.Sleep(TimeSpan.FromSeconds(5));
-                }
-            }
-            finally
-            {
-                cts.Cancel();
-                logTask.Wait();
             }
         }
 
@@ -77,5 +57,42 @@ namespace Tests.BasicTests
                 testFile.AssertIsEqual(downloadedFile);
             });
         }
+
+        [Test]
+        [UseLongTimeouts]
+        public void HoldMyBeerTest()
+        {
+            var group = SetupCodexNodes(5, o => o
+                    .EnableMetrics()
+                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
+                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithStorageQuota(3.GB()));
+
+            var nodes = group.Cast<OnlineCodexNode>().ToArray();
+
+            foreach (var node in nodes)
+            {
+                node.Marketplace.MakeStorageAvailable(
+                size: 1.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 1024.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(5));
+            }
+
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(2);
+            while (DateTime.UtcNow < endTime)
+            {
+                foreach (var node in nodes)
+                {
+                    var file = GenerateTestFile(80.MB());
+                    var cid = node.UploadFile(file);
+
+                    var dl = node.DownloadContent(cid);
+                    file.AssertIsEqual(dl);
+                }
+
+                Thread.Sleep(TimeSpan.FromMinutes(2));
+            }
+        }
     }
 }