status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip debugging anomalous test failures

This commit is contained in:
benbierens 2023-05-04 08:25:48 +02:00
parent 532eb3d4f9
commit 2ed6993b58
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
10 changed files with 139 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
DistTestCore/Configuration.cs
View File

@ -21,7 +21,7 @@ namespace DistTestCore
public Logging.LogConfig GetLogConfig()
{
return new Logging.LogConfig("CodexTestLogs", debugEnabled: false);
return new Logging.LogConfig("CodexTestLogs", debugEnabled: true);
}
public string GetFileManagerFolder()

1
DistTestCore/DistTest.cs
View File

@ -10,6 +10,7 @@ using System.Reflection;
namespace DistTestCore
{
[SetUpFixture]
[Parallelizable(ParallelScope.All)]
public abstract class DistTest
{
private readonly Configuration configuration = new Configuration();

8
KubernetesWorkflow/CommandRunner.cs
View File

@ -5,7 +5,7 @@ namespace KubernetesWorkflow
{
public class CommandRunner
{
private readonly Kubernetes client;
private readonly K8sClient client;
private readonly string k8sNamespace;
private readonly RunningPod pod;
private readonly string containerName;
@ -13,7 +13,7 @@ namespace KubernetesWorkflow
private readonly string[] arguments;
private readonly List<string> lines = new List<string>();
public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
{
this.client = client;
this.k8sNamespace = k8sNamespace;
@ -27,8 +27,8 @@ namespace KubernetesWorkflow
{
var input = new[] { command }.Concat(arguments).ToArray();
Time.Wait(client.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()));
Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
}
public string GetStdOut()

36
KubernetesWorkflow/K8sClient.cs Normal file
View File

@ -0,0 +1,36 @@
using k8s;
namespace KubernetesWorkflow
{
public class K8sClient
{
private readonly Kubernetes client;
private static readonly object clientLock = new object();
public K8sClient(KubernetesClientConfiguration config)
{
client = new Kubernetes(config);
}
public void Run(Action<Kubernetes> action)
{
lock (clientLock)
{
action(client);
}
}
public T Run<T>(Func<Kubernetes, T> action)
{
lock (clientLock)
{
return action(client);
}
}
public void Dispose()
{
client.Dispose();
}
}
}

44
KubernetesWorkflow/K8sController.cs
View File

@ -11,8 +11,7 @@ namespace KubernetesWorkflow
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownPods;
private readonly WorkflowNumberSource workflowNumberSource;
private readonly string testNamespace;
private readonly Kubernetes client;
private readonly K8sClient client;
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
{
@ -20,8 +19,10 @@ namespace KubernetesWorkflow
this.cluster = cluster;
this.knownPods = knownPods;
this.workflowNumberSource = workflowNumberSource;
this.testNamespace = testNamespace;
client = new Kubernetes(cluster.GetK8sClientConfig());
client = new K8sClient(cluster.GetK8sClientConfig());
K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
log.Debug($"'{K8sTestNamespace}'");
}
public void Dispose()
@ -53,7 +54,7 @@ namespace KubernetesWorkflow
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
{
log.Debug();
using var stream = client.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name);
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
logHandler.Log(stream);
}
@ -69,7 +70,7 @@ namespace KubernetesWorkflow
{
log.Debug();
var all = client.ListNamespace().Items;
var all = client.Run(c => c.ListNamespace().Items);
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix));
foreach (var ns in namespaces)
@ -87,7 +88,7 @@ namespace KubernetesWorkflow
log.Debug();
if (IsTestNamespaceOnline())
{
client.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0);
client.Run(c => c.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0));
}
WaitUntilNamespaceDeleted();
}
@ -97,12 +98,14 @@ namespace KubernetesWorkflow
log.Debug();
if (IsNamespaceOnline(ns))
{
client.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0);
client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
}
}
#region Namespace management
private string K8sTestNamespace { get; }
private void EnsureTestNamespace()
{
if (IsTestNamespaceOnline()) return;
@ -116,15 +119,10 @@ namespace KubernetesWorkflow
Labels = new Dictionary<string, string> { { "name", K8sTestNamespace } }
}
};
client.CreateNamespace(namespaceSpec);
client.Run(c => c.CreateNamespace(namespaceSpec));
WaitUntilNamespaceCreated();
}
private string K8sTestNamespace
{
get { return cluster.Configuration.K8sNamespacePrefix + testNamespace; }
}
private bool IsTestNamespaceOnline()
{
return IsNamespaceOnline(K8sTestNamespace);
@ -132,7 +130,7 @@ namespace KubernetesWorkflow
private bool IsNamespaceOnline(string name)
{
return client.ListNamespace().Items.Any(n => n.Metadata.Name == name);
return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name));
}
#endregion
@ -167,7 +165,7 @@ namespace KubernetesWorkflow
}
};
client.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace);
client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace));
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
return deploymentSpec.Metadata.Name;
@ -175,7 +173,7 @@ namespace KubernetesWorkflow
private void DeleteDeployment(string deploymentName)
{
client.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace);
client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace));
WaitUntilDeploymentOffline(deploymentName);
}
@ -283,14 +281,14 @@ namespace KubernetesWorkflow
}
};
client.CreateNamespacedService(serviceSpec, K8sTestNamespace);
client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
return (serviceSpec.Metadata.Name, result);
}
private void DeleteService(string serviceName)
{
client.DeleteNamespacedService(serviceName, K8sTestNamespace);
client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
}
private V1ObjectMeta CreateServiceMetadata()
@ -358,7 +356,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sTestNamespace);
var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sTestNamespace));
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
});
}
@ -367,7 +365,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var deployments = client.ListNamespacedDeployment(K8sTestNamespace);
var deployments = client.Run(c => c.ListNamespacedDeployment(K8sTestNamespace));
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
return deployment == null || deployment.Status.AvailableReplicas == 0;
});
@ -377,7 +375,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var pods = client.ListNamespacedPod(K8sTestNamespace).Items;
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
return pod == null;
});
@ -400,7 +398,7 @@ namespace KubernetesWorkflow
private (string, string) FetchNewPod()
{
var pods = client.ListNamespacedPod(K8sTestNamespace).Items;
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");

1
Tests/BasicTests/DownloadTests.cs
View File

@ -6,6 +6,7 @@ namespace Tests.ParallelTests
[TestFixture]
public class DownloadTests : DistTest
{
[Ignore("a")]
[TestCase(3, 500)]
[TestCase(5, 100)]
[TestCase(10, 256)]

51
Tests/BasicTests/NetworkIsolationTest.cs Normal file
View File

@ -0,0 +1,51 @@
using DistTestCore;
using NUnit.Framework;
using Utils;
namespace Tests.BasicTests
{
[Ignore("not a real test!")]
[TestFixture]
public class NetworkIsolationTest : DistTest
{
private IOnlineCodexNode? node = null;
// net isolation: only on real cluster?
// parallel upload/download tests?
// operation times.
[Test]
public void SetUpANodeAndWait()
{
node = SetupCodexNode();
while (node != null)
{
Time.Sleep(TimeSpan.FromSeconds(5));
}
}
[Test]
public void ForeignNodeConnects()
{
var myNode = SetupCodexNode();
while (node == null)
{
Time.Sleep(TimeSpan.FromSeconds(1));
}
myNode.ConnectToPeer(node);
var testFile = GenerateTestFile(1.MB());
var contentId = node.UploadFile(testFile);
var downloadedFile = myNode.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
node = null;
}
}
}

22
Tests/BasicTests/TwoClientTests.cs
View File

@ -5,18 +5,25 @@ using NUnit.Framework;
namespace Tests.BasicTests
{
[TestFixture]
[Parallelizable(ParallelScope.All)]
public class TwoClientTests : DistTest
{
[Test]
public void TwoClientsOnePodTest()
[TestCase(1)]
[TestCase(2)]
[TestCase(3)]
[TestCase(4)]
[TestCase(5)]
[TestCase(6)]
[TestCase(7)]
[TestCase(8)]
[TestCase(9)]
public void TwoClientsOnePodTest(int size)
{
var group = SetupCodexNodes(2);
var primary = group[0];
var secondary = group[1];
PerformTwoClientTest(primary, secondary);
PerformTwoClientTest(primary, secondary, size.MB());
}
[Test]
@ -39,10 +46,15 @@ namespace Tests.BasicTests
}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{
PerformTwoClientTest(primary, secondary, 1.MB());
}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary, ByteSize size)
{
primary.ConnectToPeer(secondary);
var testFile = GenerateTestFile(1.MB());
var testFile = GenerateTestFile(size);
var contentId = primary.UploadFile(testFile);

1
Tests/BasicTests/UploadTests.cs
View File

@ -6,6 +6,7 @@ namespace Tests.ParallelTests
[TestFixture]
public class UploadTests : DistTest
{
[Ignore("a")]
[TestCase(3, 50)]
[TestCase(5, 75)]
[TestCase(10, 25)]

6
Tests/Parallelism.cs Normal file
View File

@ -0,0 +1,6 @@
using NUnit.Framework;
[assembly: LevelOfParallelism(30)]
namespace Tests
{
}