diff --git a/.github/workflows/docker-keymaker.yml b/.github/workflows/docker-keymaker.yml new file mode 100644 index 00000000..db067c3c --- /dev/null +++ b/.github/workflows/docker-keymaker.yml @@ -0,0 +1,27 @@ +name: Docker - KeyMaker + + +on: + push: + branches: + - master + tags: + - 'v*.*.*' + paths: + - 'Tools/KeyMaker/**' + - 'Framework/**' + - 'ProjectPlugins/**' + - .github/workflows/docker-KeyMaker.yml + - .github/workflows/docker-reusable.yml + workflow_dispatch: + + +jobs: + build-and-push: + name: Build and Push + uses: ./.github/workflows/docker-reusable.yml + with: + docker_file: Tools/KeyMaker/docker/Dockerfile + docker_repo: codexstorage/codex-keymaker + secrets: inherit + diff --git a/Framework/Core/CoreInterface.cs b/Framework/Core/CoreInterface.cs index b43c1c20..81483a8b 100644 --- a/Framework/Core/CoreInterface.cs +++ b/Framework/Core/CoreInterface.cs @@ -30,11 +30,11 @@ namespace Core public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null) { var workflow = entryPoint.Tools.CreateWorkflow(); - var file = entryPoint.Tools.GetLog().CreateSubfile(); - entryPoint.Tools.GetLog().Log($"Downloading container log for '{container.Name}' to file '{file.FullFilename}'..."); - var logHandler = new LogDownloadHandler(container.Name, file); + var msg = $"Downloading container log for '{container.Name}'"; + entryPoint.Tools.GetLog().Log(msg); + var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg); workflow.DownloadContainerLog(container, logHandler, tailLines); - return logHandler.DownloadLog(); + return new DownloadedLog(logHandler); } public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args) diff --git a/Framework/Core/DownloadedLog.cs b/Framework/Core/DownloadedLog.cs index fa4d5597..3979f3ee 100644 --- a/Framework/Core/DownloadedLog.cs +++ b/Framework/Core/DownloadedLog.cs @@ -1,4 +1,5 @@ -using Logging; +using KubernetesWorkflow; +using Logging; namespace Core { @@ -14,9 +15,9 @@ namespace Core { private readonly LogFile logFile; - internal DownloadedLog(LogFile logFile) + internal DownloadedLog(WriteToFileLogHandler logHandler) { - this.logFile = logFile; + logFile = logHandler.LogFile; } public void IterateLines(Action action) diff --git a/Framework/Core/LogDownloadHandler.cs b/Framework/Core/LogDownloadHandler.cs deleted file mode 100644 index e1736ed1..00000000 --- a/Framework/Core/LogDownloadHandler.cs +++ /dev/null @@ -1,28 +0,0 @@ -using KubernetesWorkflow; -using Logging; - -namespace Core -{ - internal class LogDownloadHandler : LogHandler, ILogHandler - { - private readonly LogFile log; - - internal LogDownloadHandler(string description, LogFile log) - { - this.log = log; - - log.Write($"{description} -->> {log.FullFilename}"); - log.WriteRaw(description); - } - - internal IDownloadedLog DownloadLog() - { - return new DownloadedLog(log); - } - - protected override void ProcessLine(string line) - { - log.WriteRaw(line); - } - } -} diff --git a/Framework/KubernetesWorkflow/CrashWatcher.cs b/Framework/KubernetesWorkflow/CrashWatcher.cs index 57e30eb6..7f38cfb5 100644 --- a/Framework/KubernetesWorkflow/CrashWatcher.cs +++ b/Framework/KubernetesWorkflow/CrashWatcher.cs @@ -11,7 +11,6 @@ namespace KubernetesWorkflow private readonly string podName; private readonly string recipeName; private readonly string k8sNamespace; - private ILogHandler? logHandler; private CancellationTokenSource cts; private Task? worker; private Exception? workerException; @@ -27,11 +26,10 @@ namespace KubernetesWorkflow cts = new CancellationTokenSource(); } - public void Start(ILogHandler logHandler) + public void Start() { if (worker != null) throw new InvalidOperationException(); - this.logHandler = logHandler; cts = new CancellationTokenSource(); worker = Task.Run(Worker); } @@ -93,7 +91,8 @@ namespace KubernetesWorkflow private void DownloadCrashedContainerLogs(Kubernetes client) { using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true); - logHandler!.Log(stream); + var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName); + handler.Log(stream); } } } diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 00c7563d..e5fac7c4 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -45,7 +45,7 @@ namespace KubernetesWorkflow public void WaitUntilOnline(RunningContainer container) { - WaitUntilDeploymentOnline(container.Recipe.Name); + WaitUntilDeploymentOnline(container); } public PodInfo GetPodInfo(RunningDeployment deployment) @@ -64,14 +64,14 @@ namespace KubernetesWorkflow if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment); } - public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines) + public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous) { log.Debug(); var podName = GetPodName(container); var recipeName = container.Recipe.Name; - using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines)); + using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous)); logHandler.Log(stream); } @@ -879,15 +879,39 @@ namespace KubernetesWorkflow WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted)); } - private void WaitUntilDeploymentOnline(string deploymentName) + private void WaitUntilDeploymentOnline(RunningContainer container) { WaitUntil(() => { - var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace)); + CheckForCrash(container); + + var deployment = client.Run(c => c.ReadNamespacedDeployment(container.Recipe.Name, K8sNamespace)); return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; }, nameof(WaitUntilDeploymentOnline)); } + private void CheckForCrash(RunningContainer container) + { + var deploymentName = container.Recipe.Name; + var podName = GetPodName(container); + + var podInfo = client.Run(c => c.ReadNamespacedPod(podName, K8sNamespace)); + if (podInfo == null) return; + if (podInfo.Status == null) return; + if (podInfo.Status.ContainerStatuses == null) return; + + var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0); + if (result) + { + var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})"; + log.Error(msg); + + DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true); + + throw new Exception(msg); + } + } + private void WaitUntilDeploymentOffline(string deploymentName) { WaitUntil(() => diff --git a/Framework/KubernetesWorkflow/LogHandler.cs b/Framework/KubernetesWorkflow/LogHandler.cs index 77e57468..f185ba17 100644 --- a/Framework/KubernetesWorkflow/LogHandler.cs +++ b/Framework/KubernetesWorkflow/LogHandler.cs @@ -1,4 +1,6 @@ -namespace KubernetesWorkflow +using Logging; + +namespace KubernetesWorkflow { public interface ILogHandler { @@ -20,4 +22,25 @@ protected abstract void ProcessLine(string line); } + + public class WriteToFileLogHandler : LogHandler, ILogHandler + { + public WriteToFileLogHandler(ILog sourceLog, string description) + { + LogFile = sourceLog.CreateSubfile(); + + var msg = $"{description} -->> {LogFile.FullFilename}"; + sourceLog.Log(msg); + + LogFile.Write(msg); + LogFile.WriteRaw(description); + } + + public LogFile LogFile { get; } + + protected override void ProcessLine(string line) + { + LogFile.WriteRaw(line); + } + } } diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index 3a7326df..6a4d50ca 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -15,7 +15,7 @@ namespace KubernetesWorkflow PodInfo GetPodInfo(RunningPod pod); CrashWatcher CreateCrashWatcher(RunningContainer container); void Stop(RunningPod pod, bool waitTillStopped); - void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null); + void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null); string ExecuteCommand(RunningContainer container, string command, params string[] args); void DeleteNamespace(bool wait); void DeleteNamespacesStartingWith(string namespacePrefix, bool wait); @@ -106,11 +106,11 @@ namespace KubernetesWorkflow }); } - public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null) + public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null) { K8s(controller => { - controller.DownloadPodLog(container, logHandler, tailLines); + controller.DownloadPodLog(container, logHandler, tailLines, previous); }); } diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 80f10611..110a386f 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -8,12 +8,11 @@ using Utils; namespace CodexPlugin { - public class CodexAccess : ILogHandler + public class CodexAccess { private readonly ILog log; private readonly IPluginTools tools; private readonly Mapper mapper = new Mapper(); - private bool hasContainerCrashed; public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher) { @@ -21,9 +20,8 @@ namespace CodexPlugin log = tools.GetLog(); Container = container; CrashWatcher = crashWatcher; - hasContainerCrashed = false; - CrashWatcher.Start(this); + CrashWatcher.Start(); } public RunningPod Container { get; } @@ -209,25 +207,7 @@ namespace CodexPlugin private void CheckContainerCrashed(HttpClient client) { - if (hasContainerCrashed) throw new Exception($"Container {GetName()} has crashed."); - } - - void ILogHandler.Log(Stream crashLog) - { - var file = log.CreateSubfile(); - Log($"Downloading log to '{file.FullFilename}'..."); - file.Write($"Container log for {Container.Name}."); - - using var reader = new StreamReader(crashLog); - var line = reader.ReadLine(); - while (line != null) - { - file.Write(line); - line = reader.ReadLine(); - } - - Log("Container log successfully downloaded."); - hasContainerCrashed = true; + if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed."); } private Retry CreateRetryConfig(string description, Action onFailure) diff --git a/Tests/CodexTests/BasicTests/OneClientTests.cs b/Tests/CodexTests/BasicTests/OneClientTests.cs index 451b1cd0..908fb937 100644 --- a/Tests/CodexTests/BasicTests/OneClientTests.cs +++ b/Tests/CodexTests/BasicTests/OneClientTests.cs @@ -17,18 +17,6 @@ namespace CodexTests.BasicTests LogNodeStatus(primary); } - [Test] - public void RestartTest() - { - var primary = StartCodex(); - - primary.Stop(waitTillStopped: true); - - primary = StartCodex(); - - PerformOneClientTest(primary); - } - private void PerformOneClientTest(ICodexNode primary) { var testFile = GenerateTestFile(1.MB()); diff --git a/Tests/CodexTests/BasicTests/ThreeClientTest.cs b/Tests/CodexTests/BasicTests/ThreeClientTest.cs index 2285aa95..e6f9428e 100644 --- a/Tests/CodexTests/BasicTests/ThreeClientTest.cs +++ b/Tests/CodexTests/BasicTests/ThreeClientTest.cs @@ -1,4 +1,5 @@ -using NUnit.Framework; +using CodexPlugin; +using NUnit.Framework; using Utils; namespace CodexTests.BasicTests @@ -20,5 +21,34 @@ namespace CodexTests.BasicTests testFile.AssertIsEqual(downloadedFile); } + + [Test] + public void DownloadingUnknownCidDoesNotCauseCrash() + { + var node = StartCodex(2).First(); + + var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ"); + + try + { + node.DownloadContent(unknownCid); + } + catch (Exception ex) + { + if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out")) + { + throw; + } + } + + // Check that the node stays alive for at least another 5 minutes. + var start = DateTime.UtcNow; + while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(5)) + { + Thread.Sleep(5000); + var info = node.GetDebugInfo(); + Assert.That(!string.IsNullOrEmpty(info.Id)); + } + } } } diff --git a/Tools/KeyMaker/Controllers/KeyController.cs b/Tools/KeyMaker/Controllers/KeyController.cs new file mode 100644 index 00000000..dd218893 --- /dev/null +++ b/Tools/KeyMaker/Controllers/KeyController.cs @@ -0,0 +1,23 @@ +using GethPlugin; +using Microsoft.AspNetCore.Mvc; + +namespace KeyMaker.Controllers +{ + [Route("api/[controller]")] + [ApiController] + public class KeyController : ControllerBase + { + [HttpGet] + public KeyResponse Get() + { + var account = EthAccount.GenerateNew(); + + return new KeyResponse + { + Public = account.EthAddress.Address, + Private = account.PrivateKey, + Secure = "Not Secure! For demo/development purposes only!" + }; + } + } +} diff --git a/Tools/KeyMaker/KeyMaker.csproj b/Tools/KeyMaker/KeyMaker.csproj new file mode 100644 index 00000000..9ee1ab24 --- /dev/null +++ b/Tools/KeyMaker/KeyMaker.csproj @@ -0,0 +1,18 @@ + + + + net7.0 + enable + enable + + + + + + + + + + + + diff --git a/Tools/KeyMaker/KeyMaker.csproj.user b/Tools/KeyMaker/KeyMaker.csproj.user new file mode 100644 index 00000000..9ff5820a --- /dev/null +++ b/Tools/KeyMaker/KeyMaker.csproj.user @@ -0,0 +1,6 @@ + + + + https + + \ No newline at end of file diff --git a/Tools/KeyMaker/KeyResponse.cs b/Tools/KeyMaker/KeyResponse.cs new file mode 100644 index 00000000..df8fb0b7 --- /dev/null +++ b/Tools/KeyMaker/KeyResponse.cs @@ -0,0 +1,9 @@ +namespace KeyMaker +{ + public class KeyResponse + { + public string Public { get; set; } = string.Empty; + public string Private { get; set; } = string.Empty; + public string Secure { get; set; } = string.Empty; + } +} diff --git a/Tools/KeyMaker/Program.cs b/Tools/KeyMaker/Program.cs new file mode 100644 index 00000000..9c24f8fc --- /dev/null +++ b/Tools/KeyMaker/Program.cs @@ -0,0 +1,31 @@ +var builder = WebApplication.CreateBuilder(args); + +var listenPort = Environment.GetEnvironmentVariable("APIPORT"); +if (string.IsNullOrEmpty(listenPort)) listenPort = "31090"; + +builder.WebHost.ConfigureKestrel((context, options) => +{ + options.ListenAnyIP(Convert.ToInt32(listenPort)); +}); + +builder.Services.AddControllers(); +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddSwaggerGen(); + +var app = builder.Build(); + +if (app.Environment.IsDevelopment()) +{ + app.UseSwagger(); + app.UseSwaggerUI(); +} + +app.UseHttpsRedirection(); + +app.UseAuthorization(); + +app.MapControllers(); + +Console.WriteLine("KeyMaker listening on port " + listenPort); + +app.Run(); diff --git a/Tools/KeyMaker/Properties/launchSettings.json b/Tools/KeyMaker/Properties/launchSettings.json new file mode 100644 index 00000000..7517b23f --- /dev/null +++ b/Tools/KeyMaker/Properties/launchSettings.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:17248", + "sslPort": 44396 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "http://localhost:5069", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "swagger", + "applicationUrl": "https://localhost:7056;http://localhost:5069", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "swagger", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/Tools/KeyMaker/appsettings.Development.json b/Tools/KeyMaker/appsettings.Development.json new file mode 100644 index 00000000..0c208ae9 --- /dev/null +++ b/Tools/KeyMaker/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/Tools/KeyMaker/appsettings.json b/Tools/KeyMaker/appsettings.json new file mode 100644 index 00000000..10f68b8c --- /dev/null +++ b/Tools/KeyMaker/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/Tools/KeyMaker/docker/Dockerfile b/Tools/KeyMaker/docker/Dockerfile new file mode 100644 index 00000000..74d929bb --- /dev/null +++ b/Tools/KeyMaker/docker/Dockerfile @@ -0,0 +1,26 @@ +# Variables +ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0 +ARG IMAGE=${BUILDER} +ARG APP_HOME=/app + + +# Build +FROM ${IMAGE} AS builder +ARG APP_HOME + +WORKDIR ${APP_HOME} +COPY ./Tools/KeyMaker ./Tools/KeyMaker +COPY ./Framework ./Framework +COPY ./ProjectPlugins ./ProjectPlugins +RUN dotnet restore Tools/KeyMaker +RUN dotnet publish Tools/KeyMaker -c Release -o out + + +# Create +FROM ${IMAGE} +ARG APP_HOME +ENV APP_HOME=${APP_HOME} + +WORKDIR ${APP_HOME} +COPY --from=builder ${APP_HOME}/out . +CMD dotnet ${APP_HOME}/KeyMaker.dll diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index ada68933..beec9e6d 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -66,6 +66,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution .editorconfig = .editorconfig EndProjectSection EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KeyMaker", "Tools\KeyMaker\KeyMaker.csproj", "{B57A4789-D8EF-42E0-8D20-581C4057FFD3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -172,6 +174,10 @@ Global {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Debug|Any CPU.Build.0 = Debug|Any CPU {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.ActiveCfg = Release|Any CPU {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.Build.0 = Release|Any CPU + {B57A4789-D8EF-42E0-8D20-581C4057FFD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B57A4789-D8EF-42E0-8D20-581C4057FFD3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B57A4789-D8EF-42E0-8D20-581C4057FFD3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B57A4789-D8EF-42E0-8D20-581C4057FFD3}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -202,6 +208,7 @@ Global {F730DA73-1C92-4107-BCFB-D33759DAB0C3} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} {B07820C4-309F-4454-BCC1-1D4902C9C67B} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} {88C212E9-308A-46A4-BAAD-468E8EBD8EDF} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124} + {B57A4789-D8EF-42E0-8D20-581C4057FFD3} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}