Sets up codex net downloader for log inspection

CodexNetDownloader/CodexNetDownloader.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
+    <ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+  </ItemGroup>
+
+</Project>

CodexNetDownloader/Configuration.cs

@@ -0,0 +1,22 @@
+using ArgsUniform;
+using DistTestCore;
+using DistTestCore.Codex;
+
+namespace CodexNetDownloader
+{
+    public class Configuration
+    {
+        [Uniform("output-path", "o", "OUTPUT", true, "Path where files will be written.")]
+        public string OutputPath { get; set; } = "output";
+
+        [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
+        public string CodexDeploymentJson { get; set; } = string.Empty;
+
+        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
+        public string KubeConfigFile { get; set; } = "null";
+
+        public CodexDeployment CodexDeployment { get; set; } = null!;
+
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+    }
+}

CodexNetDownloader/Program.cs

@@ -0,0 +1,62 @@
+using ArgsUniform;
+using ContinuousTests;
+using DistTestCore;
+using DistTestCore.Codex;
+using Logging;
+using Newtonsoft.Json;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDownloader" + nl);
+
+        if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
+        {
+            PrintHelp();
+            return;
+        }
+
+        var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(args);
+        var config = uniformArgs.Parse(true);
+
+        if (args.Any(a => a == "--external"))
+        {
+            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
+        }
+
+        config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
+
+        if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
+
+        var k8sFactory = new K8sFactory();
+        var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
+
+        foreach (var container in config.CodexDeployment.CodexContainers)
+        {
+            lifecycle.DownloadLog(container);
+        }
+
+        Console.WriteLine("Done!");
+    }
+
+    private static CodexDeployment ParseCodexDeploymentJson(string filename)
+    {
+        var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
+        if (d == null) throw new Exception("Unable to parse " + filename);
+        return d;
+    }
+
+    private static void PrintHelp()
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
+
+        Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
+            "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
+
+        var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>();
+        uniformArgs.PrintHelp();
+    }
+}

ContinuousTests/ContinuousTestRunner.cs

@@ -58,7 +58,7 @@ namespace ContinuousTests
             if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
 
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
-            var (workflowCreator, _) = k8SFactory.CreateFacilities(config, test.CustomK8sNamespace, new DefaultTimeSet(), log);
+            var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
             workflowCreator.CreateWorkflow().DeleteTestResources();
         }
     }

ContinuousTests/K8sFactory.cs

@@ -7,22 +7,22 @@ namespace ContinuousTests
 {
     public class K8sFactory
     {
-        public (WorkflowCreator, TestLifecycle) CreateFacilities(Configuration config, string customNamespace, ITimeSet timeSet, BaseLog log)
+        public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
         {
-            var kubeConfig = GetKubeConfig(config.KubeConfigFile);
+            var kubeConfig = GetKubeConfig(kubeConfigFile);
             var lifecycleConfig = new DistTestCore.Configuration
             (
                 kubeConfigFile: kubeConfig,
-                logPath: "null",
+                logPath: logPath,
                 logDebug: false,
-                dataFilesPath: config.LogPath,
+                dataFilesPath: dataFilePath,
                 codexLogLevel: CodexLogLevel.Debug,
                 runnerLocation: TestRunnerLocation.ExternalToCluster
             );
 
             var kubeFlowConfig = new KubernetesWorkflow.Configuration(
                 k8sNamespacePrefix: customNamespace,
-            kubeConfigFile: kubeConfig,
+                kubeConfigFile: kubeConfig,
                 operationTimeout: timeSet.K8sOperationTimeout(),
             retryDelay: timeSet.WaitForK8sServiceDelay());
 

ContinuousTests/NodeRunner.cs

@@ -81,7 +81,7 @@ namespace ContinuousTests
 
         private (WorkflowCreator, TestLifecycle) CreateFacilities()
         {
-            return k8SFactory.CreateFacilities(config, customNamespace, timeSet, log);
+            return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
         }
     }
 }

cs-codex-dist-testing.sln

@@ -21,7 +21,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContinuousTests", "Continuo
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDownloader", "CodexNetDownloader\CodexNetDownloader.csproj", "{6CDF35D2-906A-4285-8E1F-4794588B948B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -69,6 +71,10 @@ Global
 		{634324B1-E359-42B4-A269-BDC429936B3C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6CDF35D2-906A-4285-8E1F-4794588B948B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6CDF35D2-906A-4285-8E1F-4794588B948B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE