diff --git a/CodexNetDeployer/ArgOrVar.cs b/CodexNetDeployer/ArgOrVar.cs
new file mode 100644
index 0000000..5844b68
--- /dev/null
+++ b/CodexNetDeployer/ArgOrVar.cs
@@ -0,0 +1,93 @@
+namespace CodexNetDeployer
+{
+ public class ArgOrVar
+ {
+ public static readonly ArgVar CodexImage = new ArgVar("codex-image", "CODEXIMAGE", "Docker image of Codex.");
+ public static readonly ArgVar GethImage = new ArgVar("geth-image", "GETHIMAGE", "Docker image of Geth.");
+ public static readonly ArgVar ContractsImage = new ArgVar("contracts-image", "CONTRACTSIMAGE", "Docker image of Codex Contracts deployer.");
+ public static readonly ArgVar KubeConfigFile = new ArgVar("kube-config", "KUBECONFIG", "Path to Kubeconfig file.");
+ public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment.");
+ public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created.");
+ public static readonly ArgVar NumberOfValidatorNodes = new ArgVar("validators", "VALIDATORS", "Number of Codex nodes that will be validating.");
+ public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node.");
+ public static readonly ArgVar LogLevel = new ArgVar("log-level", "LOGLEVEL", "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]");
+
+ private readonly string[] args;
+
+ public ArgOrVar(string[] args)
+ {
+ this.args = args;
+ }
+
+ public string Get(ArgVar key, string defaultValue = "")
+ {
+ var argKey = $"--{key.Arg}=";
+ var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+ if (arg != null)
+ {
+ return arg.Substring(argKey.Length);
+ }
+
+ var env = Environment.GetEnvironmentVariable(key.Var);
+ if (env != null)
+ {
+ return env;
+ }
+
+ return defaultValue;
+ }
+
+ public int? GetInt(ArgVar key)
+ {
+ var str = Get(key);
+ if (string.IsNullOrEmpty(str)) return null;
+ if (int.TryParse(str, out int result))
+ {
+ return result;
+ }
+ return null;
+ }
+
+ public void PrintHelp()
+ {
+ var nl = Environment.NewLine;
+ Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
+ "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
+ "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
+
+ Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
+ "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
+
+ Console.Write("\t[ CLI argument ] or [ Environment variable ]");
+ Console.CursorLeft = 70;
+ Console.Write("(Description)" + nl);
+ var fields = GetType().GetFields();
+ foreach (var field in fields)
+ {
+ var value = (ArgVar)field.GetValue(null)!;
+ value.PrintHelp();
+ }
+ }
+ }
+
+ public class ArgVar
+ {
+ public ArgVar(string arg, string var, string description)
+ {
+ Arg = arg;
+ Var = var;
+ Description = description;
+ }
+
+ public string Arg { get; }
+ public string Var { get; }
+ public string Description { get; }
+
+ public void PrintHelp()
+ {
+ Console.Write($"\t[ --{Arg}=... ] or [ {Var}=... ]");
+ Console.CursorLeft = 70;
+ Console.Write(Description + Environment.NewLine);
+ }
+ }
+}
diff --git a/CodexNetDeployer/CodexNetDeployer.csproj b/CodexNetDeployer/CodexNetDeployer.csproj
new file mode 100644
index 0000000..80f6f4c
--- /dev/null
+++ b/CodexNetDeployer/CodexNetDeployer.csproj
@@ -0,0 +1,14 @@
+
+
+
+ Exe
+ net7.0
+ enable
+ enable
+
+
+
+
+
+
+
diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs
new file mode 100644
index 0000000..ee071db
--- /dev/null
+++ b/CodexNetDeployer/CodexNodeStarter.cs
@@ -0,0 +1,75 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+using Logging;
+
+namespace CodexNetDeployer
+{
+ public class CodexNodeStarter
+ {
+ private readonly Configuration config;
+ private readonly WorkflowCreator workflowCreator;
+ private readonly TestLifecycle lifecycle;
+ private readonly BaseLog log;
+ private readonly ITimeSet timeSet;
+ private readonly GethStartResult gethResult;
+ private string bootstrapSpr = "";
+ private int validatorsLeft;
+
+ public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, BaseLog log, ITimeSet timeSet, GethStartResult gethResult, int numberOfValidators)
+ {
+ this.config = config;
+ this.workflowCreator = workflowCreator;
+ this.lifecycle = lifecycle;
+ this.log = log;
+ this.timeSet = timeSet;
+ this.gethResult = gethResult;
+ this.validatorsLeft = numberOfValidators;
+ }
+
+ public RunningContainer? Start(int i)
+ {
+ Console.Write($" - {i} = ");
+ var workflow = workflowCreator.CreateWorkflow();
+ var workflowStartup = new StartupConfig();
+ workflowStartup.Add(gethResult);
+ workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
+
+ var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
+
+ var container = containers.Containers.First();
+ var address = lifecycle.Configuration.GetAddress(container);
+ var codexNode = new CodexNode(log, timeSet, address);
+ var debugInfo = codexNode.GetDebugInfo();
+
+ if (!string.IsNullOrWhiteSpace(debugInfo.spr))
+ {
+ var pod = container.Pod.PodInfo;
+ Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine);
+
+ if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
+ validatorsLeft--;
+ return container;
+ }
+ else
+ {
+ Console.Write("Unknown failure." + Environment.NewLine);
+ return null;
+ }
+ }
+
+ private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
+ {
+ var codexStart = new CodexStartupConfig(config.CodexLogLevel);
+
+ if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr;
+ codexStart.StorageQuota = config.StorageQuota!.Value.MB();
+ var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
+ marketplaceConfig.AccountIndexOverride = i;
+ codexStart.MarketplaceConfig = marketplaceConfig;
+
+ return codexStart;
+ }
+ }
+}
diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs
new file mode 100644
index 0000000..2383047
--- /dev/null
+++ b/CodexNetDeployer/Configuration.cs
@@ -0,0 +1,101 @@
+using DistTestCore;
+using DistTestCore.Codex;
+
+namespace CodexNetDeployer
+{
+ public class Configuration
+ {
+ public Configuration(
+ string codexImage,
+ string gethImage,
+ string contractsImage,
+ string kubeConfigFile,
+ string kubeNamespace,
+ int? numberOfCodexNodes,
+ int? numberOfValidators,
+ int? storageQuota,
+ CodexLogLevel codexLogLevel,
+ TestRunnerLocation runnerLocation)
+ {
+ CodexImage = codexImage;
+ GethImage = gethImage;
+ ContractsImage = contractsImage;
+ KubeConfigFile = kubeConfigFile;
+ KubeNamespace = kubeNamespace;
+ NumberOfCodexNodes = numberOfCodexNodes;
+ NumberOfValidators = numberOfValidators;
+ StorageQuota = storageQuota;
+ CodexLogLevel = codexLogLevel;
+ RunnerLocation = runnerLocation;
+ }
+
+ public string CodexImage { get; }
+ public string GethImage { get; }
+ public string ContractsImage { get; }
+ public string KubeConfigFile { get; }
+ public string KubeNamespace { get; }
+ public int? NumberOfCodexNodes { get; }
+ public int? NumberOfValidators { get; }
+ public int? StorageQuota { get; }
+ public CodexLogLevel CodexLogLevel { get; }
+ public TestRunnerLocation RunnerLocation { get; }
+
+ public void PrintConfig()
+ {
+ ForEachProperty(onString: Print, onInt: Print);
+ }
+
+ public List Validate()
+ {
+ var errors = new List();
+
+ ForEachProperty(
+ onString: (n, v) => StringIsSet(n, v, errors),
+ onInt: (n, v) => IntIsOverZero(n, v, errors));
+
+ if (NumberOfValidators > NumberOfCodexNodes)
+ {
+ errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes}).");
+ }
+
+ return errors;
+ }
+
+ private void ForEachProperty(Action onString, Action onInt)
+ {
+ var properties = GetType().GetProperties();
+ foreach (var p in properties)
+ {
+ if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!);
+ if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!);
+ }
+ }
+
+ private static void IntIsOverZero(string variable, int? value, List errors)
+ {
+ if (value == null || value.Value < 1)
+ {
+ errors.Add($"{variable} is must be set and must be greater than 0.");
+ }
+ }
+
+ private static void StringIsSet(string variable, string value, List errors)
+ {
+ if (string.IsNullOrWhiteSpace(value))
+ {
+ errors.Add($"{variable} is must be set.");
+ }
+ }
+
+ private static void Print(string variable, string value)
+ {
+ Console.WriteLine($"\t{variable}: '{value}'");
+ }
+
+ private static void Print(string variable, int? value)
+ {
+ if (value != null) Print(variable, value.ToString()!);
+ else Print(variable, "");
+ }
+ }
+}
diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs
new file mode 100644
index 0000000..3e476b4
--- /dev/null
+++ b/CodexNetDeployer/Deployer.cs
@@ -0,0 +1,93 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+
+namespace CodexNetDeployer
+{
+ public class Deployer
+ {
+ private readonly Configuration config;
+ private readonly NullLog log;
+ private readonly DefaultTimeSet timeset;
+
+ public Deployer(Configuration config)
+ {
+ this.config = config;
+ log = new NullLog();
+ timeset = new DefaultTimeSet();
+ }
+
+ public CodexDeployment Deploy()
+ {
+ Log("Initializing...");
+ var (workflowCreator, lifecycle) = CreateFacilities();
+
+ Log("Preparing configuration...");
+ // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
+ var setup = new CodexSetup(999, config.CodexLogLevel);
+ setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
+
+ Log("Creating Geth instance and deploying contracts...");
+ var gethStarter = new GethStarter(lifecycle, workflowCreator);
+ var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
+
+ Log("Geth started. Codex contracts deployed.");
+ Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts.");
+
+ Log("Starting Codex nodes...");
+
+ // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
+ var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, log, timeset, gethResults, config.NumberOfValidators!.Value);
+ var codexContainers = new List();
+ for (var i = 0; i < config.NumberOfCodexNodes; i++)
+ {
+ var container = codexStarter.Start(i);
+ if (container != null) codexContainers.Add(container);
+ }
+
+ return new CodexDeployment(gethResults, codexContainers.ToArray(), CreateMetadata());
+ }
+
+ private (WorkflowCreator, TestLifecycle) CreateFacilities()
+ {
+ var lifecycleConfig = new DistTestCore.Configuration
+ (
+ kubeConfigFile: config.KubeConfigFile,
+ logPath: "null",
+ logDebug: false,
+ dataFilesPath: "notUsed",
+ codexLogLevel: config.CodexLogLevel,
+ runnerLocation: config.RunnerLocation
+ );
+
+ var kubeConfig = new KubernetesWorkflow.Configuration(
+ k8sNamespacePrefix: config.KubeNamespace,
+ kubeConfigFile: config.KubeConfigFile,
+ operationTimeout: timeset.K8sOperationTimeout(),
+ retryDelay: timeset.WaitForK8sServiceDelay());
+
+ var workflowCreator = new WorkflowCreator(log, kubeConfig, testNamespacePostfix: string.Empty);
+ var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
+
+ return (workflowCreator, lifecycle);
+ }
+
+ private DeploymentMetadata CreateMetadata()
+ {
+ return new DeploymentMetadata(
+ codexImage: config.CodexImage,
+ gethImage: config.GethImage,
+ contractsImage: config.ContractsImage,
+ kubeNamespace: config.KubeNamespace,
+ numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
+ numberOfValidators: config.NumberOfValidators!.Value,
+ storageQuotaMB: config.StorageQuota!.Value,
+ codexLogLevel: config.CodexLogLevel);
+ }
+
+ private void Log(string msg)
+ {
+ Console.WriteLine(msg);
+ }
+ }
+}
diff --git a/CodexNetDeployer/NullLog.cs b/CodexNetDeployer/NullLog.cs
new file mode 100644
index 0000000..8417d39
--- /dev/null
+++ b/CodexNetDeployer/NullLog.cs
@@ -0,0 +1,43 @@
+using Logging;
+
+namespace CodexNetDeployer
+{
+ public class NullLog : TestLog
+ {
+ public NullLog() : base("NULL", false, "NULL")
+ {
+ }
+
+ protected override LogFile CreateLogFile()
+ {
+ return null!;
+ }
+
+ public override void Log(string message)
+ {
+ //Console.WriteLine(message);
+ }
+
+ public override void Debug(string message = "", int skipFrames = 0)
+ {
+ //Console.WriteLine(message);
+ }
+
+ public override void Error(string message)
+ {
+ Console.WriteLine("Error: " + message);
+ }
+
+ public override void MarkAsFailed()
+ {
+ }
+
+ public override void AddStringReplace(string from, string to)
+ {
+ }
+
+ public override void Delete()
+ {
+ }
+ }
+}
diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs
new file mode 100644
index 0000000..462667c
--- /dev/null
+++ b/CodexNetDeployer/Program.cs
@@ -0,0 +1,66 @@
+using CodexNetDeployer;
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using Newtonsoft.Json;
+using Utils;
+using Configuration = CodexNetDeployer.Configuration;
+
+public class Program
+{
+ public static void Main(string[] args)
+ {
+ var nl = Environment.NewLine;
+ Console.WriteLine("CodexNetDeployer" + nl + nl);
+
+ var argOrVar = new ArgOrVar(args);
+
+ if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
+ {
+ argOrVar.PrintHelp();
+ return;
+ }
+
+ var location = TestRunnerLocation.InternalToCluster;
+ if (args.Any(a => a == "--external"))
+ {
+ location = TestRunnerLocation.ExternalToCluster;
+ }
+
+ var config = new Configuration(
+ codexImage: argOrVar.Get(ArgOrVar.CodexImage, CodexContainerRecipe.DockerImage),
+ gethImage: argOrVar.Get(ArgOrVar.GethImage, GethContainerRecipe.DockerImage),
+ contractsImage: argOrVar.Get(ArgOrVar.ContractsImage, CodexContractsContainerRecipe.DockerImage),
+ kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile),
+ kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace),
+ numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes),
+ numberOfValidators: argOrVar.GetInt(ArgOrVar.NumberOfValidatorNodes),
+ storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota),
+ codexLogLevel: ParseEnum.Parse(argOrVar.Get(ArgOrVar.LogLevel, nameof(CodexLogLevel.Debug))),
+ runnerLocation: location
+ );
+
+ Console.WriteLine("Using:");
+ config.PrintConfig();
+ Console.WriteLine(nl);
+
+ var errors = config.Validate();
+ if (errors.Any())
+ {
+ Console.WriteLine($"Configuration errors: ({errors.Count})");
+ foreach ( var error in errors ) Console.WriteLine("\t" + error);
+ Console.WriteLine(nl);
+ argOrVar.PrintHelp();
+ return;
+ }
+
+ var deployer = new Deployer(config);
+ var deployment = deployer.Deploy();
+
+ Console.WriteLine("Writing codex-deployment.json...");
+
+ File.WriteAllText("codex-deployment.json", JsonConvert.SerializeObject(deployment, Formatting.Indented));
+
+ Console.WriteLine("Done!");
+ }
+}
diff --git a/ContinuousTests/CodexNodeFactory.cs b/ContinuousTests/CodexNodeFactory.cs
new file mode 100644
index 0000000..43b1fea
--- /dev/null
+++ b/ContinuousTests/CodexNodeFactory.cs
@@ -0,0 +1,19 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+using Logging;
+
+namespace ContinuousTests
+{
+ public class CodexNodeFactory
+ {
+ public CodexNode[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
+ {
+ return containers.Select(container =>
+ {
+ var address = container.ClusterExternalAddress;
+ return new CodexNode(log, timeSet, address);
+ }).ToArray();
+ }
+ }
+}
diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs
new file mode 100644
index 0000000..3fdca13
--- /dev/null
+++ b/ContinuousTests/Configuration.cs
@@ -0,0 +1,98 @@
+using DistTestCore.Codex;
+using Newtonsoft.Json;
+
+namespace ContinuousTests
+{
+ public class Configuration
+ {
+ public string LogPath { get; set; } = string.Empty;
+ public string DataPath { get; set; } = string.Empty;
+ public CodexDeployment CodexDeployment { get; set; } = null!;
+ public bool KeepPassedTestLogs { get; set; }
+ }
+
+ public class ConfigLoader
+ {
+ private const string filename = "config.json";
+
+ public Configuration Load()
+ {
+ var config = Read();
+
+ Validate(config);
+ return config;
+ }
+
+ private Configuration Read()
+ {
+ if (File.Exists(filename))
+ {
+ var lines = File.ReadAllText(filename);
+ try
+ {
+ var result = JsonConvert.DeserializeObject(lines);
+ if (result != null) return result;
+ }
+ catch { }
+ }
+
+ var logPath = Environment.GetEnvironmentVariable("LOGPATH");
+ var dataPath = Environment.GetEnvironmentVariable("DATAPATH");
+ var codexDeploymentJson = Environment.GetEnvironmentVariable("CODEXDEPLOYMENT");
+ var keep = Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS");
+
+ if (!string.IsNullOrEmpty(logPath) &&
+ !string.IsNullOrEmpty(dataPath) &&
+ !string.IsNullOrEmpty(codexDeploymentJson))
+ {
+ try
+ {
+ return new Configuration
+ {
+ LogPath = logPath,
+ DataPath = dataPath,
+ CodexDeployment = ParseCodexDeploymentJson(codexDeploymentJson),
+ KeepPassedTestLogs = keep == "1"
+ };
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine("Exception: " + ex);
+ }
+ }
+
+ var nl = Environment.NewLine;
+ throw new Exception($"Unable to load configuration from '{filename}', and " +
+ "unable to load configuration from environment variables. " + nl +
+ "'LOGPATH' = Path where log files will be saved." + nl +
+ "'DATAPATH' = Path where temporary data files will be saved." + nl +
+ "'CODEXDEPLOYMENT' = Path to codex-deployment JSON file." + nl +
+ nl);
+ }
+
+ private void Validate(Configuration configuration)
+ {
+ if (string.IsNullOrEmpty(configuration.LogPath))
+ {
+ throw new Exception($"Invalid LogPath set: '{configuration.LogPath}'");
+ }
+
+ if (string.IsNullOrEmpty(configuration.DataPath))
+ {
+ throw new Exception($"Invalid DataPath set: '{configuration.DataPath}'");
+ }
+
+ if (configuration.CodexDeployment == null || !configuration.CodexDeployment.CodexContainers.Any())
+ {
+ throw new Exception("No Codex deployment found.");
+ }
+ }
+
+ private CodexDeployment ParseCodexDeploymentJson(string filename)
+ {
+ var d = JsonConvert.DeserializeObject(File.ReadAllText(filename))!;
+ if (d == null) throw new Exception("Unable to parse " + filename);
+ return d;
+ }
+ }
+}
diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs
new file mode 100644
index 0000000..c278aad
--- /dev/null
+++ b/ContinuousTests/ContinuousTest.cs
@@ -0,0 +1,95 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using Logging;
+
+namespace ContinuousTests
+{
+ public abstract class ContinuousTestLongTimeouts : ContinuousTest
+ {
+ public override ITimeSet TimeSet => new LongTimeSet();
+ }
+
+ public abstract class ContinuousTest
+ {
+ protected const int Zero = 0;
+ protected const int MinuteOne = 60;
+ protected const int MinuteFive = MinuteOne * 5;
+ protected const int HourOne = MinuteOne * 60;
+ protected const int DayOne = HourOne * 24;
+ protected const int DayThree = DayOne * 3;
+
+ private const string UploadFailedMessage = "Unable to store block";
+
+ public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager)
+ {
+ Nodes = nodes;
+ Log = log;
+ FileManager = fileManager;
+ }
+
+ public CodexNode[] Nodes { get; private set; } = null!;
+ public BaseLog Log { get; private set; } = null!;
+ public IFileManager FileManager { get; private set; } = null!;
+ public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
+
+ public abstract int RequiredNumberOfNodes { get; }
+ public abstract TimeSpan RunTestEvery { get; }
+ public abstract TestFailMode TestFailMode { get; }
+
+ public string Name
+ {
+ get
+ {
+ return GetType().Name;
+ }
+ }
+
+ public ContentId? UploadFile(CodexNode node, TestFile file)
+ {
+ using var fileStream = File.OpenRead(file.Filename);
+
+ var logMessage = $"Uploading file {file.Describe()}...";
+ var response = Stopwatch.Measure(Log, logMessage, () =>
+ {
+ return node.UploadFile(fileStream);
+ });
+
+ if (response.StartsWith(UploadFailedMessage))
+ {
+ return null;
+ }
+ Log.Log($"Uploaded file. Received contentId: '{response}'.");
+ return new ContentId(response);
+ }
+
+ public TestFile DownloadContent(CodexNode node, ContentId contentId, string fileLabel = "")
+ {
+ var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
+ var file = FileManager.CreateEmptyTestFile(fileLabel);
+ Stopwatch.Measure(Log, logMessage, () => DownloadToFile(node, contentId.Id, file));
+ Log.Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
+ return file;
+ }
+
+ private void DownloadToFile(CodexNode node, string contentId, TestFile file)
+ {
+ using var fileStream = File.OpenWrite(file.Filename);
+ try
+ {
+ using var downloadStream = node.DownloadFile(contentId);
+ downloadStream.CopyTo(fileStream);
+ }
+ catch
+ {
+ Log.Log($"Failed to download file '{contentId}'.");
+ throw;
+ }
+ }
+ }
+
+ public enum TestFailMode
+ {
+ StopAfterFirstFailure,
+ AlwaysRunAllMoments
+ }
+}
diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs
new file mode 100644
index 0000000..a2522aa
--- /dev/null
+++ b/ContinuousTests/ContinuousTestRunner.cs
@@ -0,0 +1,31 @@
+namespace ContinuousTests
+{
+ public class ContinuousTestRunner
+ {
+ private readonly ConfigLoader configLoader = new ConfigLoader();
+ private readonly TestFactory testFactory = new TestFactory();
+ private readonly Configuration config;
+ private readonly StartupChecker startupChecker;
+
+ public ContinuousTestRunner()
+ {
+ config = configLoader.Load();
+ startupChecker = new StartupChecker(config);
+ }
+
+ public void Run()
+ {
+ startupChecker.Check();
+
+ var allTests = testFactory.CreateTests();
+ var testStarters = allTests.Select(t => new TestStarter(config, t.GetType(), t.RunTestEvery)).ToArray();
+
+ foreach (var t in testStarters)
+ {
+ t.Begin();
+ }
+
+ while (true) Thread.Sleep((2 ^ 31) - 1);
+ }
+ }
+}
diff --git a/ContinuousTests/ContinuousTests.csproj b/ContinuousTests/ContinuousTests.csproj
new file mode 100644
index 0000000..593bf86
--- /dev/null
+++ b/ContinuousTests/ContinuousTests.csproj
@@ -0,0 +1,19 @@
+
+
+
+ Exe
+ net7.0
+ enable
+ enable
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/ContinuousTests/Program.cs b/ContinuousTests/Program.cs
new file mode 100644
index 0000000..c265c36
--- /dev/null
+++ b/ContinuousTests/Program.cs
@@ -0,0 +1,12 @@
+using ContinuousTests;
+
+public class Program
+{
+ public static void Main(string[] args)
+ {
+ Console.WriteLine("Codex Continous-Test-Runner.");
+ Console.WriteLine("Running...");
+ var runner = new ContinuousTestRunner();
+ runner.Run();
+ }
+}
diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs
new file mode 100644
index 0000000..8e384d3
--- /dev/null
+++ b/ContinuousTests/SingleTestRun.cs
@@ -0,0 +1,143 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+using Utils;
+using KubernetesWorkflow;
+
+namespace ContinuousTests
+{
+ public class SingleTestRun
+ {
+ private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+ private readonly List exceptions = new List();
+ private readonly Configuration config;
+ private readonly TestHandle handle;
+ private readonly CodexNode[] nodes;
+ private readonly FileManager fileManager;
+ private readonly FixtureLog fixtureLog;
+ private readonly string dataFolder;
+
+ public SingleTestRun(Configuration config, TestHandle handle)
+ {
+ this.config = config;
+ this.handle = handle;
+
+ var testName = handle.Test.GetType().Name;
+ fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName);
+
+ nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
+ dataFolder = config.DataPath + "-" + Guid.NewGuid();
+ fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
+ }
+
+ public void Run()
+ {
+ Task.Run(() =>
+ {
+ try
+ {
+ RunTest();
+
+ if (!config.KeepPassedTestLogs) fixtureLog.Delete();
+ }
+ catch (Exception ex)
+ {
+ fixtureLog.Error("Test run failed with exception: " + ex);
+ fixtureLog.MarkAsFailed();
+ }
+ fileManager.DeleteAllTestFiles();
+ Directory.Delete(dataFolder, true);
+ });
+ }
+
+ private void RunTest()
+ {
+ var earliestMoment = handle.GetEarliestMoment();
+
+ var t = earliestMoment;
+ while (true)
+ {
+ RunMoment(t);
+
+ if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
+ {
+ Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
+ throw exceptions.Single();
+ }
+
+ var nextMoment = handle.GetNextMoment(t);
+ if (nextMoment != null)
+ {
+ Log($" > Next TestMoment in {nextMoment.Value} seconds...");
+ t += nextMoment.Value;
+ Thread.Sleep(nextMoment.Value * 1000);
+ }
+ else
+ {
+ if (exceptions.Any())
+ {
+ Log(" > Completed last test moment. Test failed.");
+ throw exceptions.First();
+ }
+ Log(" > Completed last test moment. Test passed.");
+ return;
+ }
+ }
+ }
+
+ private void RunMoment(int t)
+ {
+ try
+ {
+ handle.InvokeMoment(t, InitializeTest);
+ }
+ catch (Exception ex)
+ {
+ Log($" > TestMoment yielded exception: " + ex);
+ exceptions.Add(ex);
+ }
+
+ DecommissionTest();
+ }
+
+ private void InitializeTest(string name)
+ {
+ Log($" > Running TestMoment '{name}'");
+ handle.Test.Initialize(nodes, fixtureLog, fileManager);
+ }
+
+ private void DecommissionTest()
+ {
+ handle.Test.Initialize(null!, null!, null!);
+ }
+
+ private void Log(string msg)
+ {
+ fixtureLog.Log(msg);
+ }
+
+ private CodexNode[] CreateRandomNodes(int number)
+ {
+ var containers = SelectRandomContainers(number);
+ fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
+ return codexNodeFactory.Create(containers, fixtureLog, handle.Test.TimeSet);
+ }
+
+ private RunningContainer[] SelectRandomContainers(int number)
+ {
+ var containers = config.CodexDeployment.CodexContainers.ToList();
+ var result = new RunningContainer[number];
+ for (var i = 0; i < number; i++)
+ {
+ result[i] = containers.PickOneRandom();
+ }
+ return result;
+ }
+
+ private DistTestCore.Configuration CreateFileManagerConfiguration()
+ {
+ return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
+ CodexLogLevel.Error, TestRunnerLocation.ExternalToCluster);
+ }
+ }
+}
diff --git a/ContinuousTests/StartupChecker.cs b/ContinuousTests/StartupChecker.cs
new file mode 100644
index 0000000..8000b5b
--- /dev/null
+++ b/ContinuousTests/StartupChecker.cs
@@ -0,0 +1,102 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+
+namespace ContinuousTests
+{
+ public class StartupChecker
+ {
+ private readonly TestFactory testFactory = new TestFactory();
+ private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+ private readonly Configuration config;
+
+ public StartupChecker(Configuration config)
+ {
+ this.config = config;
+ }
+
+ public void Check()
+ {
+ var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
+ log.Log("Starting continuous test run...");
+ log.Log("Checking configuration...");
+ PreflightCheck(config);
+ log.Log("Contacting Codex nodes...");
+ CheckCodexNodes(log, config);
+ log.Log("All OK.");
+ }
+
+ private void PreflightCheck(Configuration config)
+ {
+ var tests = testFactory.CreateTests();
+ if (!tests.Any())
+ {
+ throw new Exception("Unable to find any tests.");
+ }
+ foreach (var test in tests)
+ {
+ var handle = new TestHandle(test);
+ handle.GetEarliestMoment();
+ handle.GetLastMoment();
+ }
+
+ var errors = new List();
+ foreach (var test in tests)
+ {
+ if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
+ {
+ errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
+ }
+ }
+
+ if (!Directory.Exists(config.LogPath))
+ {
+ Directory.CreateDirectory(config.LogPath);
+ }
+
+ if (errors.Any())
+ {
+ throw new Exception("Prerun check failed: " + string.Join(", ", errors));
+ }
+ }
+
+ private void CheckCodexNodes(BaseLog log, Configuration config)
+ {
+ var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
+ var pass = true;
+ foreach (var n in nodes)
+ {
+ log.Log($"Checking '{n.Address.Host}'...");
+
+ if (EnsureOnline(n))
+ {
+ log.Log("OK");
+ }
+ else
+ {
+ log.Error($"No response from '{n.Address.Host}'.");
+ pass = false;
+ }
+ }
+ if (!pass)
+ {
+ throw new Exception("Not all codex nodes responded.");
+ }
+ }
+
+ private bool EnsureOnline(CodexNode n)
+ {
+ try
+ {
+ var info = n.GetDebugInfo();
+ if (info == null || string.IsNullOrEmpty(info.id)) return false;
+ }
+ catch
+ {
+ return false;
+ }
+ return true;
+ }
+
+ }
+}
diff --git a/ContinuousTests/TestFactory.cs b/ContinuousTests/TestFactory.cs
new file mode 100644
index 0000000..3a4992a
--- /dev/null
+++ b/ContinuousTests/TestFactory.cs
@@ -0,0 +1,12 @@
+namespace ContinuousTests
+{
+ public class TestFactory
+ {
+ public ContinuousTest[] CreateTests()
+ {
+ var types = GetType().Assembly.GetTypes();
+ var testTypes = types.Where(t => typeof(ContinuousTest).IsAssignableFrom(t) && !t.IsAbstract);
+ return testTypes.Select(t => (ContinuousTest)Activator.CreateInstance(t)!).ToArray();
+ }
+ }
+}
diff --git a/ContinuousTests/TestHandle.cs b/ContinuousTests/TestHandle.cs
new file mode 100644
index 0000000..7071d17
--- /dev/null
+++ b/ContinuousTests/TestHandle.cs
@@ -0,0 +1,86 @@
+using System.Reflection;
+
+namespace ContinuousTests
+{
+ public class TestHandle
+ {
+ private readonly List moments = new List();
+
+ public TestHandle(ContinuousTest test)
+ {
+ Test = test;
+
+ ReflectTestMoments();
+
+ var testName = test.GetType().Name;
+ if (!moments.Any()) throw new Exception($"Test '{testName}' has no moments.");
+ if (moments.Count != moments.Select(m => m.Moment).Distinct().Count()) throw new Exception($"Test '{testName}' has duplicate moments");
+ }
+
+ public ContinuousTest Test { get; }
+
+ public int GetEarliestMoment()
+ {
+ return moments.Min(m => m.Moment);
+ }
+
+ public int GetLastMoment()
+ {
+ return moments.Max(m => m.Moment);
+ }
+
+ public int? GetNextMoment(int currentMoment)
+ {
+ var remainingMoments = moments.Where(m => m.Moment > currentMoment).ToArray();
+ if (!remainingMoments.Any()) return null;
+ return remainingMoments.Min(m => m.Moment);
+ }
+
+ public void InvokeMoment(int currentMoment, Action beforeInvoke)
+ {
+ var moment = moments.SingleOrDefault(m => m.Moment == currentMoment);
+ if (moment == null) return;
+
+ lock (MomentLock.Lock)
+ {
+ beforeInvoke(moment.Method.Name);
+ moment.Method.Invoke(Test, Array.Empty