codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

setup

This commit is contained in:
benbierens 2023-05-03 14:18:37 +02:00
parent 01c8238311
commit 79a40904e4
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
9 changed files with 118 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
DistTestCore/CodexStarter.cs
View File

@ -44,7 +44,7 @@ namespace DistTestCore
public void DeleteAllResources()
{
var workflow = CreateWorkflow();
workflow.DeleteAllResources();
workflow.DeleteTestResources();
RunningGroups.Clear();
}

2
DistTestCore/Configuration.cs
View File

@ -7,7 +7,7 @@ namespace DistTestCore
public KubernetesWorkflow.Configuration GetK8sConfiguration()
{
return new KubernetesWorkflow.Configuration(
k8sNamespace: "codex-test-ns",
k8sNamespacePrefix: "ct-",
kubeConfigFile: null,
operationTimeout: Timing.K8sOperationTimeout(),
retryDelay: Timing.K8sServiceDelay(),

3
DistTestCore/DistTest.cs
View File

@ -153,7 +153,8 @@ namespace DistTestCore
private void CreateNewTestLifecycle()
{
Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () =>
var testName = GetCurrentTestName();
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
{
lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
testStart = DateTime.UtcNow;

36
KubernetesWorkflow/ApplicationLifecycle.cs Normal file
View File

@ -0,0 +1,36 @@
using Utils;
namespace KubernetesWorkflow
{
public class ApplicationLifecycle
{
private static ApplicationLifecycle? instance;
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource namespaceNumberSource = new NumberSource(0);
private ApplicationLifecycle()
{
}
public static ApplicationLifecycle Instance
{
// I know singletons are quite evil. But we need to be sure this object is created only once
// and persists for the entire application lifecycle.
get
{
if (instance == null) instance = new ApplicationLifecycle();
return instance;
}
}
public NumberSource GetServiceNumberSource()
{
return servicePortNumberSource;
}
public string GetTestNamespace()
{
return namespaceNumberSource.GetNextNumber().ToString("D5");
}
}
}

6
KubernetesWorkflow/Configuration.cs
View File

@ -2,16 +2,16 @@
{
public class Configuration
{
public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
{
K8sNamespace = k8sNamespace;
K8sNamespacePrefix = k8sNamespacePrefix;
KubeConfigFile = kubeConfigFile;
OperationTimeout = operationTimeout;
RetryDelay = retryDelay;
LocationMap = locationMap;
}
public string K8sNamespace { get; }
public string K8sNamespacePrefix { get; }
public string? KubeConfigFile { get; }
public TimeSpan OperationTimeout { get; }
public TimeSpan RetryDelay { get; }

87
KubernetesWorkflow/K8sController.cs
View File

@ -11,15 +11,16 @@ namespace KubernetesWorkflow
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownPods;
private readonly WorkflowNumberSource workflowNumberSource;
private readonly string testNamespace;
private readonly Kubernetes client;
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource)
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
{
this.log = log;
this.cluster = cluster;
this.knownPods = knownPods;
this.workflowNumberSource = workflowNumberSource;
this.testNamespace = testNamespace;
client = new Kubernetes(cluster.GetK8sClientConfig());
}
@ -52,14 +53,14 @@ namespace KubernetesWorkflow
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
{
log.Debug();
using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name);
using var stream = client.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name);
logHandler.Log(stream);
}
public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
{
log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
runner.Run();
return runner.GetStdOut();
}
@ -67,11 +68,39 @@ namespace KubernetesWorkflow
public void DeleteAllResources()
{
log.Debug();
DeleteNamespace();
var all = client.ListNamespace().Items;
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix));
foreach (var ns in namespaces)
{
DeleteNamespace(ns);
}
foreach (var ns in namespaces)
{
WaitUntilNamespaceDeleted(ns);
}
}
public void DeleteTestNamespace()
{
log.Debug();
if (IsTestNamespaceOnline())
{
client.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0);
}
WaitUntilNamespaceDeleted();
}
public void DeleteNamespace(string ns)
{
log.Debug();
if (IsNamespaceOnline(ns))
{
client.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0);
}
}
#region Namespace management
private void EnsureTestNamespace()
@ -83,30 +112,27 @@ namespace KubernetesWorkflow
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = K8sNamespace,
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
Name = K8sTestNamespace,
Labels = new Dictionary<string, string> { { "name", K8sTestNamespace } }
}
};
client.CreateNamespace(namespaceSpec);
WaitUntilNamespaceCreated();
}
private void DeleteNamespace()
private string K8sTestNamespace
{
if (IsTestNamespaceOnline())
{
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
}
}
private string K8sNamespace
{
get { return cluster.Configuration.K8sNamespace; }
get { return cluster.Configuration.K8sNamespacePrefix + testNamespace; }
}
private bool IsTestNamespaceOnline()
{
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
return IsNamespaceOnline(K8sTestNamespace);
}
private bool IsNamespaceOnline(string name)
{
return client.ListNamespace().Items.Any(n => n.Metadata.Name == name);
}
#endregion
@ -141,7 +167,7 @@ namespace KubernetesWorkflow
}
};
client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
client.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace);
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
return deploymentSpec.Metadata.Name;
@ -149,7 +175,7 @@ namespace KubernetesWorkflow
private void DeleteDeployment(string deploymentName)
{
client.DeleteNamespacedDeployment(deploymentName, K8sNamespace);
client.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace);
WaitUntilDeploymentOffline(deploymentName);
}
@ -173,7 +199,7 @@ namespace KubernetesWorkflow
return new V1ObjectMeta
{
Name = "deploy-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sNamespace
NamespaceProperty = K8sTestNamespace
};
}
@ -257,14 +283,14 @@ namespace KubernetesWorkflow
}
};
client.CreateNamespacedService(serviceSpec, K8sNamespace);
client.CreateNamespacedService(serviceSpec, K8sTestNamespace);
return (serviceSpec.Metadata.Name, result);
}
private void DeleteService(string serviceName)
{
client.DeleteNamespacedService(serviceName, K8sNamespace);
client.DeleteNamespacedService(serviceName, K8sTestNamespace);
}
private V1ObjectMeta CreateServiceMetadata()
@ -272,7 +298,7 @@ namespace KubernetesWorkflow
return new V1ObjectMeta
{
Name = "service-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sNamespace
NamespaceProperty = K8sTestNamespace
};
}
@ -323,11 +349,16 @@ namespace KubernetesWorkflow
WaitUntil(() => !IsTestNamespaceOnline());
}
private void WaitUntilNamespaceDeleted(string name)
{
WaitUntil(() => !IsNamespaceOnline(name));
}
private void WaitUntilDeploymentOnline(string deploymentName)
{
WaitUntil(() =>
{
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sTestNamespace);
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
});
}
@ -336,7 +367,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var deployments = client.ListNamespacedDeployment(K8sNamespace);
var deployments = client.ListNamespacedDeployment(K8sTestNamespace);
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
return deployment == null || deployment.Status.AvailableReplicas == 0;
});
@ -346,7 +377,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var pods = client.ListNamespacedPod(K8sNamespace).Items;
var pods = client.ListNamespacedPod(K8sTestNamespace).Items;
var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
return pod == null;
});
@ -369,7 +400,7 @@ namespace KubernetesWorkflow
private (string, string) FetchNewPod()
{
var pods = client.ListNamespacedPod(K8sNamespace).Items;
var pods = client.ListNamespacedPod(K8sTestNamespace).Items;
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");

16
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -8,14 +8,16 @@ namespace KubernetesWorkflow
private readonly WorkflowNumberSource numberSource;
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownK8SPods;
private readonly string testNamespace;
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods)
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
{
this.log = log;
this.numberSource = numberSource;
this.cluster = cluster;
this.knownK8SPods = knownK8SPods;
this.testNamespace = testNamespace;
}
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@ -62,6 +64,14 @@ namespace KubernetesWorkflow
});
}
public void DeleteTestResources()
{
K8s(controller =>
{
controller.DeleteTestNamespace();
});
}
private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
{
log.Debug();
@ -82,14 +92,14 @@ namespace KubernetesWorkflow
private void K8s(Action<K8sController> action)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
action(controller);
controller.Dispose();
}
private T K8s<T>(Func<K8sController, T> action)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
var result = action(controller);
controller.Dispose();
return result;

5
KubernetesWorkflow/WorkflowCreator.cs
View File

@ -6,7 +6,6 @@ namespace KubernetesWorkflow
public class WorkflowCreator
{
private readonly NumberSource numberSource = new NumberSource(0);
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource containerNumberSource = new NumberSource(0);
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly K8sCluster cluster;
@ -21,10 +20,10 @@ namespace KubernetesWorkflow
public StartupWorkflow CreateWorkflow()
{
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
servicePortNumberSource,
ApplicationLifecycle.Instance.GetServiceNumberSource(),
containerNumberSource);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, ApplicationLifecycle.Instance.GetTestNamespace());
}
}
}

1
Tests/BasicTests/TwoClientTests.cs
View File

@ -5,6 +5,7 @@ using NUnit.Framework;
namespace Tests.BasicTests
{
[TestFixture]
[Parallelizable(ParallelScope.All)]
public class TwoClientTests : DistTest
{
[Test]