Merge branch 'spike/local-continuous-debug'

CodexNetDeployer/CodexNodeStarter.cs

@@ -84,7 +84,7 @@ namespace CodexNetDeployer
             var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
             marketplaceConfig.AccountIndexOverride = i;
             codexStart.MarketplaceConfig = marketplaceConfig;
-            codexStart.MetricsEnabled = config.RecordMetrics;
+            codexStart.MetricsMode = config.Metrics;
 
             if (config.BlockTTL != Configuration.SecondsIn1Day)
             {

CodexNetDeployer/Configuration.cs

@@ -1,5 +1,6 @@
 using ArgsUniform;
 using DistTestCore.Codex;
+using DistTestCore.Metrics;
 
 namespace CodexNetDeployer
 {
@@ -43,8 +44,8 @@ namespace CodexNetDeployer
         [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
         public int BlockTTL { get; set; } = SecondsIn1Day;
 
-        [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
+        [Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")]
-        public bool RecordMetrics { get; set; } = false;
+        public MetricsMode Metrics { get; set; } = MetricsMode.None;
 
         [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
             "set this option to override the label value.")]

CodexNetDeployer/Deployer.cs

@@ -27,7 +27,7 @@ namespace CodexNetDeployer
             // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
             var setup = new CodexSetup(999, config.CodexLogLevel);
             setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
-            setup.MetricsEnabled = config.RecordMetrics;
+            setup.MetricsMode = config.Metrics;
 
             Log("Creating Geth instance and deploying contracts...");
             var gethStarter = new GethStarter(lifecycle);
@@ -52,9 +52,9 @@ namespace CodexNetDeployer
                 if (container != null) codexContainers.Add(container);
             }
 
-            var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
+            var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers);
 
-            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
+            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
         }
 
         private TestLifecycle CreateTestLifecycle()
@@ -74,13 +74,19 @@ namespace CodexNetDeployer
             return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
         }
 
-        private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
+        private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
         {
-            if (!setup.MetricsEnabled) return null;
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
 
             Log("Starting metrics service...");
             var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
-            return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+            var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
+
+            Log("Starting dashboard service...");
+            var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
+            return (prometheusContainer, grafanaStartInfo);
         }
 
         private string? GetKubeConfig(string kubeConfigFile)

CodexNetDeployer/Program.cs

@@ -30,7 +30,8 @@ public class Program
             $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
             $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
             $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
-            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl);
+            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl +
+            $"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl);
 
         if (!args.Any(a => a == "-y"))
         {

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -10,4 +10,4 @@ dotnet run \
     --max-collateral=1024 \
     --max-duration=3600000 \
     --block-ttl=300 \
-    --record-metrics=true
+    --metrics=Dashboard

ContinuousTests/ContinuousLogDownloader.cs

@@ -7,14 +7,14 @@ namespace ContinuousTests
     public class ContinuousLogDownloader
     {
         private readonly TestLifecycle lifecycle;
-        private readonly CodexDeployment deployment;
+        private readonly RunningContainer[] containers;
         private readonly string outputPath;
         private readonly CancellationToken cancelToken;
 
-        public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
+        public ContinuousLogDownloader(TestLifecycle lifecycle, RunningContainer[] containers, string outputPath, CancellationToken cancelToken)
         {
             this.lifecycle = lifecycle;
-            this.deployment = deployment;
+            this.containers = containers;
             this.outputPath = outputPath;
             this.cancelToken = cancelToken;
         }
@@ -37,7 +37,7 @@ namespace ContinuousTests
 
         private void UpdateLogs()
         {
-            foreach (var container in deployment.CodexContainers)
+            foreach (var container in containers)
             {
                 UpdateLog(container);
             }
@@ -53,7 +53,7 @@ namespace ContinuousTests
 
             var appender = new LogAppender(filepath);
 
-            lifecycle.CodexStarter.DownloadLog(container, appender);
+            lifecycle.CodexStarter.DownloadLog(container, appender, null);
         }
 
         private static string GetLogName(RunningContainer container)

ContinuousTests/ContinuousTestRunner.cs

@@ -30,7 +30,8 @@ namespace ContinuousTests
 
             ClearAllCustomNamespaces(allTests, overviewLog);
 
-            StartLogDownloader(taskFactory);
+            // Disabled for now. Still looking for a better way.
+            // StartLogDownloader(taskFactory);
 
             var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
 
@@ -72,7 +73,7 @@ namespace ContinuousTests
             if (!Directory.Exists(path)) Directory.CreateDirectory(path);
 
             var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
-            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
+            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment.CodexContainers, path, cancelToken);
 
             taskFactory.Run(downloader.Run);
         }

ContinuousTests/NodeRunner.cs

@@ -5,6 +5,7 @@ using KubernetesWorkflow;
 using NUnit.Framework;
 using Logging;
 using Utils;
+using DistTestCore.Logs;
 
 namespace ContinuousTests
 {
@@ -38,6 +39,21 @@ namespace ContinuousTests
             RunNode(bootstrapNode, operation, 0.TestTokens());
         }
 
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
+        {
+            var subFile = log.CreateSubfile();
+            var description = container.Name;
+            var handler = new LogDownloadHandler(container, description, subFile);
+
+            log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
+
+            var lifecycle = CreateTestLifecycle();
+            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
+            flow.DownloadContainerLog(container, handler, tailLines);
+
+            return new DownloadedLog(subFile, description);
+        }
+
         public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
         {
             var lifecycle = CreateTestLifecycle();

ContinuousTests/Tests/HoldMyBeerTest.cs

@@ -0,0 +1,103 @@
+using DistTestCore;
+using Logging;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class HoldMyBeerTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private ContentId? cid;
+        private TestFile file = null!;
+
+        [TestMoment(t: Zero)]
+        public void UploadTestFile()
+        {
+            var filesize = 80.MB();
+            double codexDefaultBlockSize = 31 * 64 * 33;
+            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
+            var sizeInBytes = filesize.SizeInBytes;
+            Assert.That(numberOfBlocks, Is.EqualTo(1282));
+
+
+            file = FileManager.GenerateTestFile(filesize);
+
+            cid = UploadFile(Nodes[0], file);
+            Assert.That(cid, Is.Not.Null);
+
+            var cidTag = cid!.Id.Substring(cid.Id.Length - 6);
+            Stopwatch.Measure(Log, "upload-log-asserts", () =>
+            {
+                var uploadLog = NodeRunner.DownloadLog(Nodes[0].Container, 50000);
+
+                var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
+                uploadLog.DeleteFile();
+
+                var storeLine = GetLineForCidTag(storeLines, cidTag);
+                AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
+            });
+
+
+            var dl = DownloadFile(Nodes[0], cid!);
+            file.AssertIsEqual(dl);
+
+            Stopwatch.Measure(Log, "download-log-asserts", () =>
+            {
+                var downloadLog = NodeRunner.DownloadLog(Nodes[0].Container, 50000);
+
+                var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
+                downloadLog.DeleteFile();
+
+                var sentLine = GetLineForCidTag(sentLines, cidTag);
+                AssertSentLineContains(sentLine, sizeInBytes);
+            });
+        }
+
+        private void AssertSentLineContains(string sentLine, long sizeInBytes)
+        {
+            var tag = "bytes=";
+            var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
+            var bytes = Convert.ToInt64(token);
+            Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
+        }
+
+        private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
+        {
+            var tokens = storeLine.Split(" ");
+
+            var blocksToken = GetToken(tokens, "blocks=");
+            var sizeToken = GetToken(tokens, "size=");
+            if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
+            if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
+
+            var blocks = Convert.ToInt64(blocksToken);
+            var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
+
+            var lineLog = $" Line: '{storeLine}'";
+            Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
+            Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
+        }
+
+        private string GetLineForCidTag(string[] lines, string cidTag)
+        {
+            var result = lines.SingleOrDefault(l => l.Contains(cidTag));
+            if (result == null)
+            {
+                Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
+                throw new Exception();
+            }
+
+            return result;
+        }
+
+        private string? GetToken(string[] tokens, string tag)
+        {
+            var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
+            if (token == null) return null;
+            return token.Substring(tag.Length);
+        }
+    }
+}

DistTestCore/ByteSize.cs

@@ -38,6 +38,11 @@ namespace DistTestCore
     {
         private const long Kilo = 1024;
 
+        public static ByteSize Bytes(this long i)
+        {
+            return new ByteSize(i);
+        }
+
         public static ByteSize KB(this long i)
         {
             return new ByteSize(i * Kilo);
@@ -58,6 +63,11 @@ namespace DistTestCore
             return (i * Kilo).GB();
         }
 
+        public static ByteSize Bytes(this int i)
+        {
+            return new ByteSize(i);
+        }
+
         public static ByteSize KB(this int i)
         {
             return Convert.ToInt64(i).KB();

DistTestCore/Codex/CodexAccess.cs

@@ -4,10 +4,11 @@ using Utils;
 
 namespace DistTestCore.Codex
 {
-    public class CodexAccess
+    public class CodexAccess : ILogHandler
     {
         private readonly BaseLog log;
         private readonly ITimeSet timeSet;
+        private bool hasContainerCrashed;
 
         public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
         {
@@ -15,6 +16,9 @@ namespace DistTestCore.Codex
             Container = container;
             this.timeSet = timeSet;
             Address = address;
+            hasContainerCrashed = false;
+
+            if (container.CrashWatcher != null) container.CrashWatcher.Start(this);
         }
 
         public RunningContainer Container { get; }
@@ -86,7 +90,30 @@ namespace DistTestCore.Codex
 
         private Http Http()
         {
+            CheckContainerCrashed();
             return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
         }
+
+        private void CheckContainerCrashed()
+        {
+            if (hasContainerCrashed) throw new Exception("Container has crashed.");
+        }
+
+        public void Log(Stream crashLog)
+        {
+            var file = log.CreateSubfile();
+            log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
+
+            using var reader = new StreamReader(crashLog);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                file.Write(line);
+                line = reader.ReadLine();
+            }
+
+            log.Log("Crash log successfully downloaded.");
+            hasContainerCrashed = true;
+        }
     }
 }

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -51,7 +51,15 @@ namespace DistTestCore.Codex
             {
                 AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
             }
-            if (config.MetricsEnabled)
+            if (config.BlockMaintenanceInterval != null)
+            {
+                AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
+            }
+            if (config.BlockMaintenanceNumber != null)
+            {
+                AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
+            }
+            if (config.MetricsMode != Metrics.MetricsMode.None)
             {
                 AddEnvVar("CODEX_METRICS", "true");
                 AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");

DistTestCore/Codex/CodexDeployment.cs

@@ -5,17 +5,19 @@ namespace DistTestCore.Codex
 {
     public class CodexDeployment
     {
-        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata)
         {
             GethStartResult = gethStartResult;
             CodexContainers = codexContainers;
             PrometheusContainer = prometheusContainer;
+            GrafanaStartInfo = grafanaStartInfo;
             Metadata = metadata;
         }
 
         public GethStartResult GethStartResult { get; }
         public RunningContainer[] CodexContainers { get; }
         public RunningContainer? PrometheusContainer { get; }
+        public GrafanaStartInfo? GrafanaStartInfo { get; }
         public DeploymentMetadata Metadata { get; }
     }
 

DistTestCore/Codex/CodexStartupConfig.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
@@ -14,9 +15,11 @@ namespace DistTestCore.Codex
         public Location Location { get; set; }
         public CodexLogLevel LogLevel { get; }
         public ByteSize? StorageQuota { get; set; }
-        public bool MetricsEnabled { get; set; }
+        public MetricsMode MetricsMode { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
         public string? BootstrapSpr { get; set; }
         public int? BlockTTL { get; set; }
+        public TimeSpan? BlockMaintenanceInterval { get; set; }
+        public int? BlockMaintenanceNumber { get; set; }
     }
 }

DistTestCore/CodexSetup.cs

@@ -11,6 +11,8 @@ namespace DistTestCore
         ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
         ICodexSetup WithBlockTTL(TimeSpan duration);
+        ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
+        ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
         ICodexSetup EnableMetrics();
         ICodexSetup EnableMarketplace(TestToken initialBalance);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
@@ -57,9 +59,21 @@ namespace DistTestCore
             return this;
         }
 
+        public ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration)
+        {
+            BlockMaintenanceInterval = duration;
+            return this;
+        }
+
+        public ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks)
+        {
+            BlockMaintenanceNumber = numberOfBlocks;
+            return this;
+        }
+
         public ICodexSetup EnableMetrics()
         {
-            MetricsEnabled = true;
+            MetricsMode = Metrics.MetricsMode.Record;
             return this;
         }
 

DistTestCore/CodexStarter.cs

@@ -47,7 +47,11 @@ namespace DistTestCore
         {
             LogStart($"Stopping {group.Describe()}...");
             var workflow = CreateWorkflow();
-            foreach (var c in group.Containers) workflow.Stop(c);
+            foreach (var c in group.Containers)
+            {
+                StopCrashWatcher(c);
+                workflow.Stop(c);
+            }
             RunningGroups.Remove(group);
             LogEnd("Stopped.");
         }
@@ -60,17 +64,23 @@ namespace DistTestCore
             RunningGroups.Clear();
         }
 
-        public void DownloadLog(RunningContainer container, ILogHandler logHandler)
+        public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             var workflow = CreateWorkflow();
-            workflow.DownloadContainerLog(container, logHandler);
+            workflow.DownloadContainerLog(container, logHandler, tailLines);
         }
 
         private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
         {
-            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
+            if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
 
             var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
+
+            if (codexSetup.MetricsMode == MetricsMode.Dashboard)
+            {
+                lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
+            }
+
             return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
         }
 
@@ -90,7 +100,9 @@ namespace DistTestCore
             for (var i = 0; i < numberOfNodes; i++)
             {
                 var workflow = CreateWorkflow();
-                result.Add(workflow.Start(1, location, recipe, startupConfig));
+                var rc = workflow.Start(1, location, recipe, startupConfig);
+                CreateCrashWatcher(workflow, rc);
+                result.Add(rc);
             }
             return result.ToArray();
         }
@@ -128,5 +140,19 @@ namespace DistTestCore
         {
             Log("----------------------------------------------------------------------------");
         }
+
+        private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc)
+        {
+            var c = rc.Containers.Single();
+            c.CrashWatcher = workflow.CreateCrashWatcher(c);
+        }
+
+        private void StopCrashWatcher(RunningContainers containers)
+        {
+            foreach (var c in containers.Containers)
+            {
+                c.CrashWatcher?.Stop();
+            }
+        }
     }
 }

DistTestCore/DistTest.cs

@@ -175,6 +175,11 @@ namespace DistTestCore
             GetTestLog().Debug(msg);
         }
 
+        public void Measure(string name, Action action)
+        {
+            Stopwatch.Measure(Get().Log, name, action);
+        }
+
         protected CodexSetup CreateCodexSetup(int numberOfNodes)
         {
             return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());

DistTestCore/DistTestCore.csproj

@@ -10,6 +10,14 @@
   <PropertyGroup Condition="'$(IsArm64)'=='true'">
     <DefineConstants>Arm64</DefineConstants>
   </PropertyGroup>
+  <ItemGroup>
+    <None Remove="Metrics\dashboard.json" />
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Metrics\dashboard.json">
+      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+    </EmbeddedResource>
+  </ItemGroup>
 
   <ItemGroup>
 	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />

DistTestCore/GrafanaStarter.cs

@@ -0,0 +1,192 @@
+using DistTestCore.Metrics;
+using IdentityModel.Client;
+using KubernetesWorkflow;
+using Newtonsoft.Json;
+using System.Reflection;
+
+namespace DistTestCore
+{
+    public class GrafanaStarter : BaseStarter
+    {
+        private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
+        private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
+
+        public GrafanaStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+
+        public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
+        {
+            LogStart($"Starting dashboard server");
+
+            var grafanaContainer = StartGrafanaContainer();
+            var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
+
+            var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
+
+            Log("Connecting datasource...");
+            AddDataSource(http, prometheusContainer);
+
+            Log("Uploading dashboard configurations...");
+            var jsons = ReadEachDashboardJsonFile(codexSetup);
+            var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
+
+            LogEnd("Dashboard server started.");
+
+            return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
+        }
+
+        private RunningContainer StartGrafanaContainer()
+        {
+            var startupConfig = new StartupConfig();
+
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
+            if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
+
+            return grafanaContainers.Containers.First();
+        }
+
+        private void AddBasicAuth(HttpClient client)
+        {
+            client.SetBasicAuthentication(
+                GrafanaContainerRecipe.DefaultAdminUser,
+                GrafanaContainerRecipe.DefaultAdminPassword);
+        }
+
+        private static void AddDataSource(Http http, RunningContainer prometheusContainer)
+        {
+            var prometheusAddress = prometheusContainer.ClusterExternalAddress;
+            var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
+            var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
+            {
+                uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
+                name = "CodexPrometheus",
+                type = "prometheus",
+                url = prometheusUrl,
+                access = "proxy",
+                basicAuth = false,
+                jsonData = new GrafanaDataSourceJsonData
+                {
+                    httpMethod = "POST"
+                }
+            });
+
+            if (response.message != "Datasource added")
+            {
+                throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
+            }
+        }
+
+        public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
+        {
+            var request = GetDashboardCreateRequest(dashboardJson);
+            var response = http.HttpPostString("dashboards/db", request);
+            var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
+            if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
+
+            var grafanaAddress = grafanaContainer.ClusterExternalAddress;
+            return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
+        }
+
+        private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
+        {
+            var assembly = Assembly.GetExecutingAssembly();
+            var resourceNames = new[]
+            {
+                "DistTestCore.Metrics.dashboard.json"
+            };
+
+            return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
+        }
+
+        private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
+        {
+            using var stream = assembly.GetManifestResourceStream(resourceName);
+            if (stream == null) throw new Exception("Unable to find resource " + resourceName);
+            using var reader = new StreamReader(stream);
+            return ApplyReplacements(reader.ReadToEnd(), codexSetup);
+        }
+
+        private static string ApplyReplacements(string input, CodexSetup codexSetup)
+        {
+            var quotaString = GetQuotaString(codexSetup);
+            var softMaxString = GetSoftMaxString(codexSetup);
+
+            return input
+                .Replace(StorageQuotaThresholdReplaceToken, quotaString)
+                .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
+        }
+
+        private static string GetQuotaString(CodexSetup codexSetup)
+        {
+            return GetCodexStorageQuotaInBytes(codexSetup).ToString();
+        }
+
+        private static string GetSoftMaxString(CodexSetup codexSetup)
+        {
+            var quota = GetCodexStorageQuotaInBytes(codexSetup);
+            var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
+            return softMax.ToString();
+        }
+
+        private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
+        {
+            if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
+
+            // Codex default: 8GB
+            return 8.GB().SizeInBytes;
+        }
+
+        private static string GetDashboardCreateRequest(string dashboardJson)
+        {
+            return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
+        }
+    }
+
+    public class GrafanaStartInfo
+    {
+        public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
+        {
+            DashboardUrls = dashboardUrls;
+            Container = container;
+        }
+
+        public string[] DashboardUrls { get; }
+        public RunningContainer Container { get; }
+    }
+
+    public class GrafanaDataSourceRequest
+    {
+        public string uid { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+        public string type { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public string access { get; set; } = string.Empty;
+        public bool basicAuth { get; set; }
+        public GrafanaDataSourceJsonData jsonData { get; set; } = new();
+    }
+
+    public class GrafanaDataSourceResponse
+    {
+        public int id { get; set; }
+        public string message { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+    }
+
+    public class GrafanaDataSourceJsonData
+    {
+        public string httpMethod { get; set; } = string.Empty;
+    }
+
+    public class GrafanaPostDashboardResponse
+    {
+        public int id { get; set; }
+        public string slug { get; set; } = string.Empty;
+        public string status { get; set; } = string.Empty;
+        public string uid { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public int version { get; set; }
+    }
+}

DistTestCore/Http.cs

@@ -12,14 +12,21 @@ namespace DistTestCore
         private readonly ITimeSet timeSet;
         private readonly Address address;
         private readonly string baseUrl;
+        private readonly Action<HttpClient> onClientCreated;
         private readonly string? logAlias;
 
         public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
+            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
+        {
+        }
+
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
             this.log = log;
             this.timeSet = timeSet;
             this.address = address;
             this.baseUrl = baseUrl;
+            this.onClientCreated = onClientCreated;
             this.logAlias = logAlias;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
@@ -66,6 +73,22 @@ namespace DistTestCore
             }, $"HTTP-POST-JSON: {route}");
         }
 
+        public string HttpPostString(string route, string body)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                Log(url, body);
+                var content = new StringContent(body);
+                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+                var result = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STRING: {route}");
+        }
+
         public string HttpPostStream(string route, Stream stream)
         {
             return Retry(() =>
@@ -76,7 +99,7 @@ namespace DistTestCore
                 var content = new StreamContent(stream);
                 content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
                 var response = Time.Wait(client.PostAsync(url, content));
-                var str =Time.Wait(response.Content.ReadAsStringAsync());
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
             }, $"HTTP-POST-STREAM: {route}");
@@ -132,7 +155,12 @@ namespace DistTestCore
         {
             var client = new HttpClient();
             client.Timeout = timeSet.HttpCallTimeout();
+            onClientCreated(client);
             return client;
         }
+
+        private static void DoNothing(HttpClient client)
+        {
+        }
     }
 }

DistTestCore/Logs/DownloadedLog.cs

@@ -6,6 +6,8 @@ namespace DistTestCore.Logs
     public interface IDownloadedLog
     {
         void AssertLogContains(string expectedString);
+        string[] FindLinesThatContain(params string[] tags);
+        void DeleteFile();
     }
 
     public class DownloadedLog : IDownloadedLog
@@ -33,5 +35,30 @@ namespace DistTestCore.Logs
 
             Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
         }
+
+        public string[] FindLinesThatContain(params string[] tags)
+        {
+            var result = new List<string>();
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (tags.All(line.Contains))
+                {
+                    result.Add(line);
+                }
+
+                line = streamReader.ReadLine();
+            }
+
+            return result.ToArray();
+        }
+
+        public void DeleteFile()
+        {
+            File.Delete(logFile.FullFilename);
+        }
     }
 }

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -25,7 +25,7 @@ namespace DistTestCore.Marketplace
             WaitUntil(() =>
             {
                 var logHandler = new ContractsReadyLogHandler(Debug);
-                workflow.DownloadContainerLog(container, logHandler);
+                workflow.DownloadContainerLog(container, logHandler, null);
                 return logHandler.Found;
             });
             Log("Contracts deployed. Extracting addresses...");

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -80,7 +80,7 @@ namespace DistTestCore.Marketplace
         private string FetchPubKey()
         {
             var enodeFinder = new PubKeyFinder(s => log.Debug(s));
-            workflow.DownloadContainerLog(container, enodeFinder);
+            workflow.DownloadContainerLog(container, enodeFinder, null);
             return enodeFinder.GetPubKey();
         }
 

DistTestCore/Metrics/GrafanaContainerRecipe.cs

@@ -0,0 +1,25 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Metrics
+{
+    public class GrafanaContainerRecipe : ContainerRecipeFactory
+    {
+        public override string AppName => "grafana";
+        public override string Image => "grafana/grafana-oss:10.0.3";
+
+        public const string DefaultAdminUser = "adminium";
+        public const string DefaultAdminPassword = "passwordium";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            AddExposedPort(3000);
+
+            AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
+
+            AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
+            AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
+        }
+    }
+}

DistTestCore/Metrics/MetricsMode.cs

@@ -0,0 +1,9 @@
+namespace DistTestCore.Metrics
+{
+    public enum MetricsMode
+    {
+        None,
+        Record,
+        Dashboard
+    }
+}

DistTestCore/OnlineCodexNode.cs

@@ -4,6 +4,7 @@ using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
 using Logging;
 using NUnit.Framework;
+using Utils;
 
 namespace DistTestCore
 {
@@ -15,7 +16,7 @@ namespace DistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
-        IDownloadedLog DownloadLog();
+        IDownloadedLog DownloadLog(int? tailLines = null);
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
         CodexDebugVersionResponse Version { get; }
@@ -67,6 +68,7 @@ namespace DistTestCore
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
+            Log(logMessage);
             var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream);
@@ -82,6 +84,7 @@ namespace DistTestCore
         public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
+            Log(logMessage);
             var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
             Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
@@ -100,9 +103,9 @@ namespace DistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public IDownloadedLog DownloadLog()
+        public IDownloadedLog DownloadLog(int? tailLines = null)
         {
-            return lifecycle.DownloadLog(CodexAccess.Container);
+            return lifecycle.DownloadLog(CodexAccess.Container, tailLines);
         }
 
         public ICodexSetup BringOffline()
@@ -116,7 +119,7 @@ namespace DistTestCore
 
         public void EnsureOnlineGetVersionResponse()
         {
-            var debugInfo = CodexAccess.GetDebugInfo();
+            var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
             var nodePeerId = debugInfo.id;
             var nodeName = CodexAccess.Container.Name;
 

DistTestCore/PrometheusStarter.cs

@@ -22,8 +22,6 @@ namespace DistTestCore
             var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
-            LogEnd("Metrics server started.");
-
             return runningContainers;
         }
 
@@ -31,7 +29,7 @@ namespace DistTestCore
         {
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 30s\n";
+            config += "  scrape_interval: 10s\n";
             config += "  scrape_timeout: 10s\n";
             config += "\n";
             config += "scrape_configs:\n";

DistTestCore/TestLifecycle.cs

@@ -24,6 +24,7 @@ namespace DistTestCore
             FileManager = new FileManager(Log, configuration);
             CodexStarter = new CodexStarter(this);
             PrometheusStarter = new PrometheusStarter(this);
+            GrafanaStarter = new GrafanaStarter(this);
             GethStarter = new GethStarter(this);
             testStart = DateTime.UtcNow;
             CodexVersion = null;
@@ -38,6 +39,7 @@ namespace DistTestCore
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }
+        public GrafanaStarter GrafanaStarter { get; }
         public GethStarter GethStarter { get; }
         public CodexDebugVersionResponse? CodexVersion { get; private set; }
 
@@ -47,14 +49,14 @@ namespace DistTestCore
             FileManager.DeleteAllTestFiles();
         }
 
-        public IDownloadedLog DownloadLog(RunningContainer container)
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var subFile = Log.CreateSubfile();
             var description = container.Name;
             var handler = new LogDownloadHandler(container, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(container, handler);
+            CodexStarter.DownloadLog(container, handler, tailLines);
 
             return new DownloadedLog(subFile, description);
         }
@@ -76,7 +78,8 @@ namespace DistTestCore
                 codexId: GetCodexId(),
                 gethId: new GethContainerRecipe().Image,
                 prometheusId: new PrometheusContainerRecipe().Image,
-                codexContractsId: new CodexContractsContainerRecipe().Image
+                codexContractsId: new CodexContractsContainerRecipe().Image,
+                grafanaId: new GrafanaContainerRecipe().Image
             );
         }
 

DistTestCore/Timing.cs

@@ -21,7 +21,7 @@ namespace DistTestCore
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromSeconds(10);
+            return TimeSpan.FromMinutes(5);
         }
 
         public TimeSpan HttpCallRetryTime()
@@ -36,12 +36,12 @@ namespace DistTestCore
 
         public TimeSpan WaitForK8sServiceDelay()
         {
-            return TimeSpan.FromSeconds(1);
+            return TimeSpan.FromSeconds(10);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromMinutes(1);
+            return TimeSpan.FromMinutes(30);
         }
 
         public TimeSpan WaitForMetricTimeout()

KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -40,6 +40,13 @@
             return p;
         }
 
+        protected Port AddExposedPort(int number, string tag = "")
+        {
+            var p = factory.CreatePort(number, tag);
+            exposedPorts.Add(p);
+            return p;
+        }
+
         protected Port AddInternalPort(string tag = "")
         {
             var p = factory.CreatePort(tag);

KubernetesWorkflow/CrashWatcher.cs

@@ -0,0 +1,85 @@
+using k8s;
+using Logging;
+
+namespace KubernetesWorkflow
+{
+    public class CrashWatcher
+    {
+        private readonly BaseLog log;
+        private readonly KubernetesClientConfiguration config;
+        private readonly string k8sNamespace;
+        private readonly RunningContainer container;
+        private ILogHandler? logHandler;
+        private CancellationTokenSource cts;
+        private Task? worker;
+        private Exception? workerException;
+
+        public CrashWatcher(BaseLog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
+        {
+            this.log = log;
+            this.config = config;
+            this.k8sNamespace = k8sNamespace;
+            this.container = container;
+            cts = new CancellationTokenSource();
+        }
+
+        public void Start(ILogHandler logHandler)
+        {
+            if (worker != null) throw new InvalidOperationException();
+
+            this.logHandler = logHandler;
+            cts = new CancellationTokenSource();
+            worker = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            if (worker == null) throw new InvalidOperationException();
+
+            cts.Cancel();
+            worker.Wait();
+            worker = null;
+
+            if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
+        }
+
+        private void Worker()
+        {
+            try
+            {
+                MonitorContainer(cts.Token);
+            }
+            catch (Exception ex)
+            {
+                workerException = ex;
+            }
+        }
+
+        private void MonitorContainer(CancellationToken token)
+        {
+            var client = new Kubernetes(config);
+            while (!token.IsCancellationRequested)
+            {
+                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
+
+                var pod = container.Pod;
+                var recipe = container.Recipe;
+                var podName = pod.PodInfo.Name;
+                var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
+
+                if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0))
+                {
+                    DownloadCrashedContainerLogs(client, podName, recipe);
+                    return;
+                }
+            }
+        }
+
+        private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe)
+        {
+            log.Log("Pod crash detected for " + container.Name);
+            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipe.Name, previous: true);
+            logHandler!.Log(stream);
+        }
+    }
+}

KubernetesWorkflow/K8sController.cs

@@ -53,10 +53,10 @@ namespace KubernetesWorkflow
             WaitUntilPodOffline(pod.PodInfo.Name);
         }
 
-        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
+        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, tailLines: tailLines));
             logHandler.Log(stream);
         }
 
@@ -604,6 +604,11 @@ namespace KubernetesWorkflow
 
         #endregion
 
+        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        {
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sTestNamespace, container);
+        }
+
         private PodInfo FetchNewPod()
         {
             var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;

KubernetesWorkflow/PodLabels.cs

@@ -25,6 +25,7 @@ namespace KubernetesWorkflow
             Add("gethid", applicationIds.GethId);
             Add("prometheusid", applicationIds.PrometheusId);
             Add("codexcontractsid", applicationIds.CodexContractsId);
+            Add("grafanaid", applicationIds.GrafanaId);
         }
 
         public PodLabels GetLabelsForAppName(string appName)

KubernetesWorkflow/RecipeComponentFactory.cs

@@ -7,6 +7,11 @@ namespace KubernetesWorkflow
     {
         private NumberSource portNumberSource = new NumberSource(8080);
 
+        public Port CreatePort(int number, string tag)
+        {
+            return new Port(number, tag);
+        }
+
         public Port CreatePort(string tag)
         {
             return new Port(portNumberSource.GetNextNumber(), tag);

KubernetesWorkflow/RunningContainers.cs

@@ -1,4 +1,5 @@
-using Utils;
+using Newtonsoft.Json;
+using Utils;
 
 namespace KubernetesWorkflow
 {
@@ -39,6 +40,9 @@ namespace KubernetesWorkflow
         public Port[] ServicePorts { get; }
         public Address ClusterExternalAddress { get; }
         public Address ClusterInternalAddress { get; }
+
+        [JsonIgnore]
+        public CrashWatcher? CrashWatcher { get; set; }
     }
 
     public static class RunningContainersExtensions

KubernetesWorkflow/StartupWorkflow.cs

@@ -37,6 +37,11 @@ namespace KubernetesWorkflow
             }, pl);
         }
 
+        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        {
+            return K8s(controller => controller.CreateCrashWatcher(container));
+        }
+
         public void Stop(RunningContainers runningContainers)
         {
             K8s(controller =>
@@ -45,11 +50,11 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler)
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler);
+                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines);
             });
         }
 

Logging/ApplicationIds.cs

@@ -2,17 +2,19 @@
 {
     public class ApplicationIds
     {
-        public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId)
+        public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId)
         {
             CodexId = codexId;
             GethId = gethId;
             PrometheusId = prometheusId;
             CodexContractsId = codexContractsId;
+            GrafanaId = grafanaId;
         }
 
         public string CodexId { get; }
         public string GethId { get; }
         public string PrometheusId { get; }
         public string CodexContractsId { get; }
+        public string GrafanaId { get; }
     }
 }

Logging/StatusLog.cs

@@ -26,6 +26,7 @@ namespace Logging
                 gethid = applicationIds.GethId,
                 prometheusid = applicationIds.PrometheusId,
                 codexcontractsid = applicationIds.CodexContractsId,
+                grafanaid = applicationIds.GrafanaId,
                 category = NameUtils.GetCategoryName(),
                 fixturename = fixtureName,
                 testname = NameUtils.GetTestMethodName(),
@@ -59,6 +60,7 @@ namespace Logging
         public string gethid { get; set; } = string.Empty;
         public string prometheusid { get; set; } = string.Empty;
         public string codexcontractsid { get; set; } = string.Empty;
+        public string grafanaid { get; set; } = string.Empty;
         public string category { get; set; } = string.Empty;
         public string fixturename { get; set; } = string.Empty;
         public string testname { get; set; } = string.Empty;

Tests/BasicTests/ContinuousSubstitute.cs

@@ -0,0 +1,194 @@
+using DistTestCore;
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.BasicTests
+{
+    [TestFixture]
+    public class ContinuousSubstitute : AutoBootstrapDistTest
+    {
+        [Test]
+        [UseLongTimeouts]
+        public void ContinuousTestSubstitute()
+        {
+            var group = SetupCodexNodes(5, o => o
+                    .EnableMetrics()
+                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
+                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithStorageQuota(3.GB()));
+
+            var nodes = group.Cast<OnlineCodexNode>().ToArray();
+
+            foreach (var node in nodes)
+            {
+                node.Marketplace.MakeStorageAvailable(
+                size: 1.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 1024.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(5));
+            }
+
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
+            while (DateTime.UtcNow < endTime)
+            {
+                var allNodes = nodes.ToList();
+                var primary = allNodes.PickOneRandom();
+                var secondary = allNodes.PickOneRandom();
+
+                Log("Run Test");
+                PerformTest(primary, secondary);
+
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+            }
+        }
+
+        private ByteSize fileSize = 80.MB();
+
+        private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
+        {
+            ScopedTestFiles(() =>
+            {
+                var testFile = GenerateTestFile(fileSize);
+
+                var contentId = primary.UploadFile(testFile);
+
+                var downloadedFile = secondary.DownloadContent(contentId);
+
+                testFile.AssertIsEqual(downloadedFile);
+            });
+        }
+        
+        [Test]
+        public void HoldMyBeerTest()
+        {
+            var blockExpirationTime = TimeSpan.FromMinutes(3);
+            var group = SetupCodexNodes(3, o => o
+                    .EnableMetrics()
+                    .WithBlockTTL(blockExpirationTime)
+                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceNumber(10000)
+                    .WithStorageQuota(2000.MB()));
+
+            var nodes = group.Cast<OnlineCodexNode>().ToArray();
+
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(24);
+
+            var filesize = 80.MB();
+            double codexDefaultBlockSize = 31 * 64 * 33;
+            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
+            var sizeInBytes = filesize.SizeInBytes;
+            Assert.That(numberOfBlocks, Is.EqualTo(1282));
+
+            var startTime = DateTime.UtcNow;
+            var successfulUploads = 0;
+            var successfulDownloads = 0;
+
+            while (DateTime.UtcNow < endTime)
+            {
+                foreach (var node in nodes)
+                {
+                    try
+                    {
+                        Thread.Sleep(TimeSpan.FromSeconds(5));
+
+                        ScopedTestFiles(() =>
+                        {
+                            var uploadStartTime = DateTime.UtcNow;
+                            var file = GenerateTestFile(filesize);
+                            var cid = node.UploadFile(file);
+
+                            var cidTag = cid.Id.Substring(cid.Id.Length - 6);
+                            Measure("upload-log-asserts", () =>
+                            {
+                                var uploadLog = node.DownloadLog(tailLines: 50000);
+
+                                var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
+                                uploadLog.DeleteFile();
+
+                                var storeLine = GetLineForCidTag(storeLines, cidTag);
+                                AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
+                            });
+                            successfulUploads++;
+
+                            var uploadTimeTaken = DateTime.UtcNow - uploadStartTime;
+                            if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10)))
+                            {
+                                Assert.Fail("Upload took too long. Blocks already expired.");
+                            }
+
+                            var dl = node.DownloadContent(cid);
+                            file.AssertIsEqual(dl);
+
+                            Measure("download-log-asserts", () =>
+                            {
+                                var downloadLog = node.DownloadLog(tailLines: 50000);
+
+                                var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
+                                downloadLog.DeleteFile();
+
+                                var sentLine = GetLineForCidTag(sentLines, cidTag);
+                                AssertSentLineContains(sentLine, sizeInBytes);
+                            });
+                            successfulDownloads++;
+                        });
+                    }
+                    catch
+                    {
+                        var testDuration = DateTime.UtcNow - startTime;
+                        Log("Test failed. Delaying shut-down by 30 seconds to collect metrics.");
+                        Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads");
+                        Thread.Sleep(TimeSpan.FromSeconds(30));
+                        throw;
+                    }
+                }
+
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+            }
+        }
+
+        private void AssertSentLineContains(string sentLine, long sizeInBytes)
+        {
+            var tag = "bytes=";
+            var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
+            var bytes = Convert.ToInt64(token);
+            Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
+        }
+
+        private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
+        {
+            var tokens = storeLine.Split(" ");
+
+            var blocksToken = GetToken(tokens, "blocks=");
+            var sizeToken = GetToken(tokens, "size=");
+            if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
+            if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
+
+            var blocks = Convert.ToInt64(blocksToken);
+            var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
+
+            var lineLog = $" Line: '{storeLine}'";
+            Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
+            Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
+        }
+
+        private string GetLineForCidTag(string[] lines, string cidTag)
+        {
+            var result = lines.SingleOrDefault(l => l.Contains(cidTag));
+            if (result == null)
+            {
+                Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
+                throw new Exception();
+            }
+
+            return result;
+        }
+
+        private string? GetToken(string[] tokens, string tag)
+        {
+            var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
+            if (token == null) return null;
+            return token.Substring(tag.Length);
+        }
+    }
+}

Tests/Tests.csproj

@@ -13,6 +13,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
   </ItemGroup>