cs-codex-dist-tests/Framework/KubernetesWorkflow/StartupWorkflow.cs

288 lines
11 KiB
C#
Raw Normal View History

using KubernetesWorkflow.Recipe;
using KubernetesWorkflow.Types;
using Logging;
2023-10-23 13:49:14 +00:00
using Newtonsoft.Json;
using Utils;
namespace KubernetesWorkflow
2023-04-12 11:53:55 +00:00
{
2023-09-11 14:57:57 +00:00
public interface IStartupWorkflow
{
2023-09-25 06:47:19 +00:00
IKnownLocations GetAvailableLocations();
2024-04-09 07:30:45 +00:00
FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
PodInfo GetPodInfo(RunningContainer container);
2024-04-13 14:09:17 +00:00
PodInfo GetPodInfo(RunningPod pod);
2023-09-20 10:55:09 +00:00
CrashWatcher CreateCrashWatcher(RunningContainer container);
2024-04-13 14:09:17 +00:00
void Stop(RunningPod pod, bool waitTillStopped);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
2023-09-11 14:57:57 +00:00
string ExecuteCommand(RunningContainer container, string command, params string[] args);
void DeleteNamespace(bool wait);
void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
2023-09-11 14:57:57 +00:00
}
public class StartupWorkflow : IStartupWorkflow
2023-04-12 11:53:55 +00:00
{
2023-09-12 08:31:55 +00:00
private readonly ILog log;
private readonly WorkflowNumberSource numberSource;
private readonly K8sCluster cluster;
2023-09-12 08:31:55 +00:00
private readonly string k8sNamespace;
2023-04-12 11:53:55 +00:00
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
2023-09-25 06:47:19 +00:00
private readonly LocationProvider locationProvider;
2023-04-12 11:53:55 +00:00
2023-10-31 13:48:16 +00:00
internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace)
2023-04-12 11:53:55 +00:00
{
2023-04-25 09:31:15 +00:00
this.log = log;
this.numberSource = numberSource;
this.cluster = cluster;
2023-09-12 08:31:55 +00:00
this.k8sNamespace = k8sNamespace;
2023-09-25 06:47:19 +00:00
locationProvider = new LocationProvider(log, K8s);
}
public IKnownLocations GetAvailableLocations()
{
return locationProvider.GetAvailableLocations();
}
2024-04-09 07:30:45 +00:00
public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
2023-09-25 06:47:19 +00:00
{
return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig);
2023-04-12 11:53:55 +00:00
}
2024-04-09 07:30:45 +00:00
public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
2023-04-12 11:53:55 +00:00
{
return K8s(controller =>
{
componentFactory.Update(controller);
var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
var startResult = controller.BringOnline(recipes, location);
var containers = CreateContainers(startResult, recipes, startupConfig);
2023-04-12 11:53:55 +00:00
var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
cluster.Configuration.Hooks.OnContainersStarted(rc);
if (startResult.ExternalService != null)
{
componentFactory.Update(controller);
}
2024-04-09 07:30:45 +00:00
return new FutureContainers(rc, this);
});
}
2024-04-13 14:09:17 +00:00
public void WaitUntilOnline(RunningPod rc)
2024-04-09 07:30:45 +00:00
{
K8s(controller =>
{
foreach (var c in rc.Containers)
{
controller.WaitUntilOnline(c);
}
2023-09-11 14:57:57 +00:00
});
2023-08-14 13:10:36 +00:00
}
public PodInfo GetPodInfo(RunningContainer container)
{
2024-04-13 14:09:17 +00:00
return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
}
2024-04-13 14:09:17 +00:00
public PodInfo GetPodInfo(RunningPod pod)
{
2024-04-13 14:09:17 +00:00
return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
}
2023-09-20 10:55:09 +00:00
public CrashWatcher CreateCrashWatcher(RunningContainer container)
{
return K8s(c => c.CreateCrashWatcher(container));
}
2024-04-13 14:09:17 +00:00
public void Stop(RunningPod runningPod, bool waitTillStopped)
2023-04-13 09:07:36 +00:00
{
if (runningPod.IsStopped) return;
foreach (var c in runningPod.Containers)
{
c.StopLog = DownloadContainerLog(c);
}
runningPod.IsStopped = true;
2023-04-13 09:07:36 +00:00
K8s(controller =>
{
2024-04-13 14:09:17 +00:00
controller.Stop(runningPod.StartResult, waitTillStopped);
2023-04-13 09:07:36 +00:00
});
cluster.Configuration.Hooks.OnContainersStopped(runningPod);
2023-04-13 09:07:36 +00:00
}
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
2023-04-13 09:30:19 +00:00
{
K8s(controller =>
{
controller.DownloadPodLog(container, logHandler, tailLines, previous);
2023-04-13 09:30:19 +00:00
});
}
public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null)
{
var msg = $"Downloading container log for '{container.Name}'";
log.Log(msg);
var logHandler = new WriteToFileLogHandler(log, msg);
K8s(controller =>
{
controller.DownloadPodLog(container, logHandler, tailLines, previous);
});
return new DownloadedLog(logHandler, container.Name);
}
2023-04-14 07:54:07 +00:00
public string ExecuteCommand(RunningContainer container, string command, params string[] args)
{
return K8s(controller =>
{
return controller.ExecuteCommand(container, command, args);
2023-04-14 07:54:07 +00:00
});
}
public void DeleteNamespace(bool wait)
{
K8s(controller =>
{
controller.DeleteNamespace(wait);
});
2023-04-12 11:53:55 +00:00
}
public void DeleteNamespacesStartingWith(string namespacePrefix, bool wait)
2023-05-03 12:18:37 +00:00
{
K8s(controller =>
{
controller.DeleteAllNamespacesStartingWith(namespacePrefix, wait);
2023-05-03 12:18:37 +00:00
});
}
private RunningContainer[] CreateContainers(StartResult startResult, ContainerRecipe[] recipes, StartupConfig startupConfig)
2023-04-12 11:53:55 +00:00
{
2023-04-25 09:31:15 +00:00
log.Debug();
return recipes.Select(r =>
{
var name = GetContainerName(r, startupConfig);
var addresses = CreateContainerAddresses(startResult, r);
log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);
}).ToArray();
2023-04-12 11:53:55 +00:00
}
private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
{
if (startupConfig == null) return "";
if (!string.IsNullOrEmpty(startupConfig.NameOverride))
{
return $"<{startupConfig.NameOverride}{recipe.Number}>";
}
else
{
return $"<{recipe.Name}>";
}
}
private ContainerAddress[] CreateContainerAddresses(StartResult startResult, ContainerRecipe recipe)
{
var result = new List<ContainerAddress>();
foreach (var exposedPort in recipe.ExposedPorts)
{
result.Add(new ContainerAddress(exposedPort.Tag, GetContainerExternalAddress(startResult, recipe, exposedPort.Tag), false));
result.Add(new ContainerAddress(exposedPort.Tag, GetContainerInternalAddress(startResult, recipe, exposedPort.Tag), true));
}
foreach (var internalPort in recipe.InternalPorts)
{
result.Add(new ContainerAddress(internalPort.Tag, GetContainerInternalAddress(startResult, recipe, internalPort.Tag), true));
}
return result.ToArray();
}
private static Address GetContainerExternalAddress(StartResult startResult, ContainerRecipe recipe, string tag)
{
var port = startResult.GetExternalServicePorts(recipe, tag);
return new Address(
startResult.Cluster.HostAddress,
port.Number);
}
private Address GetContainerInternalAddress(StartResult startResult, ContainerRecipe recipe, string tag)
{
var namespaceName = startResult.Cluster.Configuration.KubernetesNamespace;
var serviceName = startResult.InternalService!.Name;
var port = startResult.GetInternalServicePorts(recipe, tag);
2023-11-07 12:49:28 +00:00
return new Address(
2023-11-07 12:49:28 +00:00
$"http://{serviceName}.{namespaceName}.svc.cluster.local",
port.Number);
}
2023-04-12 11:53:55 +00:00
private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
{
2023-04-25 09:31:15 +00:00
log.Debug();
2023-04-12 11:53:55 +00:00
var result = new List<ContainerRecipe>();
for (var i = 0; i < numberOfContainers; i++)
{
2023-09-13 14:06:05 +00:00
var recipe = recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig);
CheckPorts(recipe);
2023-09-13 14:06:05 +00:00
if (cluster.Configuration.AddAppPodLabel) recipe.PodLabels.Add("app", recipeFactory.AppName);
cluster.Configuration.Hooks.OnContainerRecipeCreated(recipe);
result.Add(recipe);
2023-04-12 11:53:55 +00:00
}
return result.ToArray();
}
private void CheckPorts(ContainerRecipe recipe)
{
var allTags =
recipe.ExposedPorts.Concat(recipe.InternalPorts)
.Select(p => K8sNameUtils.Format(p.Tag)).ToArray();
if (allTags.Length != allTags.Distinct().Count())
{
throw new Exception("Duplicate port tags found in recipe for " + recipe.Name);
}
}
private void K8s(Action<K8sController> action)
{
2023-10-23 13:49:14 +00:00
try
{
2023-10-31 13:48:16 +00:00
var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
2023-10-23 13:49:14 +00:00
action(controller);
controller.Dispose();
}
catch (k8s.Autorest.HttpOperationException ex)
{
2023-11-13 14:11:49 +00:00
log.Error(JsonConvert.SerializeObject(ex.Response));
2023-10-23 13:49:14 +00:00
throw;
}
}
private T K8s<T>(Func<K8sController, T> action)
2023-08-07 13:51:44 +00:00
{
2023-10-23 13:49:14 +00:00
try
{
2023-10-31 13:48:16 +00:00
var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
2023-10-23 13:49:14 +00:00
var result = action(controller);
controller.Dispose();
return result;
}
catch (k8s.Autorest.HttpOperationException ex)
{
2023-11-13 14:11:49 +00:00
log.Error(JsonConvert.SerializeObject(ex.Response));
2023-10-23 13:49:14 +00:00
throw;
}
2023-08-10 09:47:34 +00:00
}
2023-04-13 09:30:19 +00:00
}
2023-04-12 11:53:55 +00:00
}