status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'less-announce-tests'

This commit is contained in:
Ben 2024-04-25 10:25:29 +02:00
parent 0c700ded9d 50b7e2300d
commit e1c710a093
No known key found for this signature in database
GPG Key ID: 541B9D8C9F1426A1
43 changed files with 583 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Framework/Core/DownloadedLog.cs
View File

@ -4,6 +4,7 @@ namespace Core
{
public interface IDownloadedLog
{
void IterateLines(Action<string> action);
string[] GetLinesContaining(string expectedString);
string[] FindLinesThatContain(params string[] tags);
void DeleteFile();
@ -18,6 +19,19 @@ namespace Core
this.logFile = logFile;
}
public void IterateLines(Action<string> action)
{
using var file = File.OpenRead(logFile.FullFilename);
using var streamReader = new StreamReader(file);
var line = streamReader.ReadLine();
while (line != null)
{
action(line);
line = streamReader.ReadLine();
}
}
public string[] GetLinesContaining(string expectedString)
{
using var file = File.OpenRead(logFile.FullFilename);

10
Framework/Core/TimeSet.cs
View File

@ -5,7 +5,7 @@
TimeSpan HttpCallTimeout();
int HttpMaxNumberOfRetries();
TimeSpan HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationRetryDelay();
TimeSpan K8sOperationTimeout();
}
@ -26,7 +26,7 @@
return TimeSpan.FromSeconds(1);
}
public TimeSpan WaitForK8sServiceDelay()
public TimeSpan K8sOperationRetryDelay()
{
return TimeSpan.FromSeconds(10);
}
@ -54,14 +54,14 @@
return TimeSpan.FromSeconds(2);
}
public TimeSpan WaitForK8sServiceDelay()
public TimeSpan K8sOperationRetryDelay()
{
return TimeSpan.FromSeconds(10);
return TimeSpan.FromSeconds(30);
}
public TimeSpan K8sOperationTimeout()
{
return TimeSpan.FromMinutes(15);
return TimeSpan.FromHours(1);
}
}
}

16
Framework/KubernetesWorkflow/K8sController.cs
View File

@ -705,7 +705,7 @@ namespace KubernetesWorkflow
private string GetPodName(RunningContainer container)
{
return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name;
return GetPodForDeployment(container.RunningPod.StartResult.Deployment).Metadata.Name;
}
private V1Pod GetPodForDeployment(RunningDeployment deployment)
@ -868,7 +868,7 @@ namespace KubernetesWorkflow
private void WaitUntilNamespaceCreated()
{
WaitUntil(() => IsNamespaceOnline(K8sNamespace));
WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
}
private void WaitUntilDeploymentOnline(string deploymentName)
@ -877,7 +877,7 @@ namespace KubernetesWorkflow
{
var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
});
}, nameof(WaitUntilDeploymentOnline));
}
private void WaitUntilDeploymentOffline(string deploymentName)
@ -887,7 +887,7 @@ namespace KubernetesWorkflow
var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
return deployment == null || deployment.Status.AvailableReplicas == 0;
});
}, nameof(WaitUntilDeploymentOffline));
}
private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment)
@ -896,19 +896,19 @@ namespace KubernetesWorkflow
{
var pods = FindPodsByLabel(deployment.PodLabel);
return !pods.Any();
});
}, nameof(WaitUntilPodsForDeploymentAreOffline));
}
private void WaitUntil(Func<bool> predicate)
private void WaitUntil(Func<bool> predicate, string msg)
{
var sw = Stopwatch.Begin(log, true);
try
{
Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay());
Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg);
}
finally
{
sw.End("", 1);
sw.End(msg, 1);
}
}

8
Framework/KubernetesWorkflow/K8sHooks.cs
View File

@ -5,18 +5,18 @@ namespace KubernetesWorkflow
{
public interface IK8sHooks
{
void OnContainersStarted(RunningContainers runningContainers);
void OnContainersStopped(RunningContainers runningContainers);
void OnContainersStarted(RunningPod runningPod);
void OnContainersStopped(RunningPod runningPod);
void OnContainerRecipeCreated(ContainerRecipe recipe);
}
public class DoNothingK8sHooks : IK8sHooks
{
public void OnContainersStarted(RunningContainers runningContainers)
public void OnContainersStarted(RunningPod runningPod)
{
}
public void OnContainersStopped(RunningContainers runningContainers)
public void OnContainersStopped(RunningPod runningPod)
{
}

20
Framework/KubernetesWorkflow/StartupWorkflow.cs
View File

@ -12,9 +12,9 @@ namespace KubernetesWorkflow
FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
PodInfo GetPodInfo(RunningContainer container);
PodInfo GetPodInfo(RunningContainers containers);
PodInfo GetPodInfo(RunningPod pod);
CrashWatcher CreateCrashWatcher(RunningContainer container);
void Stop(RunningContainers containers, bool waitTillStopped);
void Stop(RunningPod pod, bool waitTillStopped);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
string ExecuteCommand(RunningContainer container, string command, params string[] args);
void DeleteNamespace();
@ -60,7 +60,7 @@ namespace KubernetesWorkflow
var startResult = controller.BringOnline(recipes, location);
var containers = CreateContainers(startResult, recipes, startupConfig);
var rc = new RunningContainers(startupConfig, startResult, containers);
var rc = new RunningPod(startupConfig, startResult, containers);
cluster.Configuration.Hooks.OnContainersStarted(rc);
if (startResult.ExternalService != null)
@ -71,7 +71,7 @@ namespace KubernetesWorkflow
});
}
public void WaitUntilOnline(RunningContainers rc)
public void WaitUntilOnline(RunningPod rc)
{
K8s(controller =>
{
@ -84,12 +84,12 @@ namespace KubernetesWorkflow
public PodInfo GetPodInfo(RunningContainer container)
{
return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment));
return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
}
public PodInfo GetPodInfo(RunningContainers containers)
public PodInfo GetPodInfo(RunningPod pod)
{
return K8s(c => c.GetPodInfo(containers.StartResult.Deployment));
return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
}
public CrashWatcher CreateCrashWatcher(RunningContainer container)
@ -97,12 +97,12 @@ namespace KubernetesWorkflow
return K8s(c => c.CreateCrashWatcher(container));
}
public void Stop(RunningContainers runningContainers, bool waitTillStopped)
public void Stop(RunningPod runningPod, bool waitTillStopped)
{
K8s(controller =>
{
controller.Stop(runningContainers.StartResult, waitTillStopped);
cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
controller.Stop(runningPod.StartResult, waitTillStopped);
cluster.Configuration.Hooks.OnContainersStopped(runningPod);
});
}

12
Framework/KubernetesWorkflow/Types/FutureContainers.cs
View File

@ -2,19 +2,19 @@
{
public class FutureContainers
{
private readonly RunningContainers runningContainers;
private readonly RunningPod runningPod;
private readonly StartupWorkflow workflow;
public FutureContainers(RunningContainers runningContainers, StartupWorkflow workflow)
public FutureContainers(RunningPod runningPod, StartupWorkflow workflow)
{
this.runningContainers = runningContainers;
this.runningPod = runningPod;
this.workflow = workflow;
}
public RunningContainers WaitForOnline()
public RunningPod WaitForOnline()
{
workflow.WaitUntilOnline(runningContainers);
return runningContainers;
workflow.WaitUntilOnline(runningPod);
return runningPod;
}
}
}

2
Framework/KubernetesWorkflow/Types/RunningContainer.cs
View File

@ -19,7 +19,7 @@ namespace KubernetesWorkflow.Types
public ContainerAddress[] Addresses { get; }
[JsonIgnore]
public RunningContainers RunningContainers { get; internal set; } = null!;
public RunningPod RunningPod { get; internal set; } = null!;
public Address GetAddress(ILog log, string portTag)
{

13
Framework/KubernetesWorkflow/Types/RunningContainers.cs → Framework/KubernetesWorkflow/Types/RunningPod.cs
View File

@ -2,15 +2,15 @@
namespace KubernetesWorkflow.Types
{
public class RunningContainers
public class RunningPod
{
public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
{
StartupConfig = startupConfig;
StartResult = startResult;
Containers = containers;
foreach (var c in containers) c.RunningContainers = this;
foreach (var c in containers) c.RunningPod = this;
}
public StartupConfig StartupConfig { get; }
@ -31,12 +31,7 @@ namespace KubernetesWorkflow.Types
public static class RunningContainersExtensions
{
public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
{
return runningContainers.SelectMany(c => c.Containers).ToArray();
}
public static string Describe(this RunningContainers[] runningContainers)
public static string Describe(this RunningPod[] runningContainers)
{
return string.Join(",", runningContainers.Select(c => c.Describe()));
}

6
Framework/Utils/Formatter.cs
View File

@ -1,4 +1,6 @@
namespace Utils
using System.Globalization;
namespace Utils
{
public static class Formatter
{
@ -10,7 +12,7 @@
var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1);
return digit.ToString() + sizeSuffixes[sizeOrder];
return digit.ToString(CultureInfo.InvariantCulture) + sizeSuffixes[sizeOrder];
}
}
}

9
Framework/Utils/NumberSource.cs
View File

@ -2,6 +2,7 @@
{
public class NumberSource
{
private readonly object @lock = new object();
private int number;
public NumberSource(int start)
@ -11,8 +12,12 @@
public int GetNextNumber()
{
var n = number;
number++;
var n = -1;
lock (@lock)
{
n = number;
number++;
}
return n;
}
}

13
Framework/Utils/Time.cs
View File

@ -57,24 +57,27 @@
return result;
}
public static void WaitUntil(Func<bool> predicate)
public static void WaitUntil(Func<bool> predicate, string msg)
{
WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1), msg);
}
public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay)
public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay, string msg)
{
var start = DateTime.UtcNow;
var tries = 1;
var state = predicate();
while (!state)
{
if (DateTime.UtcNow - start > timeout)
var duration = DateTime.UtcNow - start;
if (duration > timeout)
{
throw new TimeoutException("Operation timed out.");
throw new TimeoutException($"Operation timed out after {tries} tries over (total) {FormatDuration(duration)}. '{msg}'");
}
Sleep(retryDelay);
state = predicate();
tries++;
}
}

8
ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs
View File

@ -59,7 +59,7 @@ namespace CodexContractsPlugin
var logHandler = new ContractsReadyLogHandler(tools.GetLog());
workflow.DownloadContainerLog(container, logHandler, 100);
return logHandler.Found;
});
}, nameof(DeployContract));
Log("Contracts deployed. Extracting addresses...");
var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
@ -71,7 +71,7 @@ namespace CodexContractsPlugin
Log("Extract completed. Checking sync...");
Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi));
Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
Log("Synced. Codex SmartContracts deployed.");
@ -83,9 +83,9 @@ namespace CodexContractsPlugin
tools.GetLog().Log(msg);
}
private void WaitUntil(Func<bool> predicate)
private void WaitUntil(Func<bool> predicate, string msg)
{
Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2), msg);
}
private StartupConfig CreateStartupConfig(IGethNode gethNode)

8
ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs
View File

@ -29,19 +29,19 @@ namespace CodexDiscordBotPlugin
{
}
public RunningContainers Deploy(DiscordBotStartupConfig config)
public RunningPod Deploy(DiscordBotStartupConfig config)
{
var workflow = tools.CreateWorkflow();
return StartContainer(workflow, config);
}
public RunningContainers DeployRewarder(RewarderBotStartupConfig config)
public RunningPod DeployRewarder(RewarderBotStartupConfig config)
{
var workflow = tools.CreateWorkflow();
return StartRewarderContainer(workflow, config);
}
private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
private RunningPod StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
{
var startupConfig = new StartupConfig();
startupConfig.NameOverride = config.Name;
@ -49,7 +49,7 @@ namespace CodexDiscordBotPlugin
return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
}
private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
{
var startupConfig = new StartupConfig();
startupConfig.Add(config);

4
ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs
View File

@ -5,12 +5,12 @@ namespace CodexDiscordBotPlugin
{
public static class CoreInterfaceExtensions
{
public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
public static RunningPod DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
{
return Plugin(ci).Deploy(config);
}
public static RunningContainers DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
public static RunningPod DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
{
return Plugin(ci).DeployRewarder(config);
}

2
ProjectPlugins/CodexPlugin/ApiChecker.cs
View File

@ -38,7 +38,7 @@ namespace CodexPlugin
if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger.");
}
public void CheckCompatibility(RunningContainers[] containers)
public void CheckCompatibility(RunningPod[] containers)
{
if (checkPassed) return;

6
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -13,7 +13,7 @@ namespace CodexPlugin
private readonly Mapper mapper = new Mapper();
private bool hasContainerCrashed;
public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher)
public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
{
this.tools = tools;
Container = container;
@ -23,7 +23,7 @@ namespace CodexPlugin
CrashWatcher.Start(this);
}
public RunningContainer Container { get; }
public RunningPod Container { get; }
public CrashWatcher CrashWatcher { get; }
public DebugInfo GetDebugInfo()
@ -136,7 +136,7 @@ namespace CodexPlugin
private Address GetAddress()
{
return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
return Container.Containers.Single().GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
}
private void CheckContainerCrashed(HttpClient client)

14
ProjectPlugins/CodexPlugin/CodexDeployment.cs
View File

@ -7,8 +7,8 @@ namespace CodexPlugin
public class CodexDeployment
{
public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
RunningContainers? discordBotContainer, DeploymentMetadata metadata,
CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer,
RunningPod? discordBotContainer, DeploymentMetadata metadata,
String id)
{
Id = id;
@ -24,20 +24,20 @@ namespace CodexPlugin
public CodexInstance[] CodexInstances { get; }
public GethDeployment GethDeployment { get; }
public CodexContractsDeployment CodexContractsDeployment { get; }
public RunningContainers? PrometheusContainer { get; }
public RunningContainers? DiscordBotContainer { get; }
public RunningPod? PrometheusContainer { get; }
public RunningPod? DiscordBotContainer { get; }
public DeploymentMetadata Metadata { get; }
}
public class CodexInstance
{
public CodexInstance(RunningContainers containers, DebugInfo info)
public CodexInstance(RunningPod pod, DebugInfo info)
{
Containers = containers;
Pod = pod;
Info = info;
}
public RunningContainers Containers { get; }
public RunningPod Pod { get; }
public DebugInfo Info { get; }
}

22
ProjectPlugins/CodexPlugin/CodexNode.cs
View File

@ -44,7 +44,9 @@ namespace CodexPlugin
transferSpeeds = new TransferSpeeds();
}
public RunningContainer Container { get { return CodexAccess.Container; } }
public RunningPod Pod { get { return CodexAccess.Container; } }
public RunningContainer Container { get { return Pod.Containers.Single(); } }
public CodexAccess CodexAccess { get; }
public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
public CodexNodeGroup Group { get; }
@ -56,7 +58,7 @@ namespace CodexPlugin
{
get
{
return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag);
return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag);
}
}
@ -71,7 +73,7 @@ namespace CodexPlugin
public string GetName()
{
return CodexAccess.Container.Name;
return Container.Name;
}
public DebugInfo GetDebugInfo()
@ -142,11 +144,13 @@ namespace CodexPlugin
public void Stop(bool waitTillStopped)
{
if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
"individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
"available for codex-nodes in groups of 1.");
Group.BringOffline(waitTillStopped);
CrashWatcher.Stop();
Group.Stop(this, waitTillStopped);
// if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
// "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
// "available for codex-nodes in groups of 1.");
//
// Group.BringOffline(waitTillStopped);
}
public void EnsureOnlineGetVersionResponse()
@ -171,7 +175,7 @@ namespace CodexPlugin
// The peer we want to connect is in a different pod.
// We must replace the default IP with the pod IP in the multiAddress.
var workflow = tools.CreateWorkflow();
var podInfo = workflow.GetPodInfo(peer.Container);
var podInfo = workflow.GetPodInfo(peer.Pod);
return peerInfo.Addrs.Select(a => a
.Replace("0.0.0.0", podInfo.Ip))

2
ProjectPlugins/CodexPlugin/CodexNodeFactory.cs
View File

@ -35,7 +35,7 @@ namespace CodexPlugin
private EthAddress? GetEthAddress(CodexAccess access)
{
var ethAccount = access.Container.Recipe.Additionals.Get<EthAccount>();
var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
if (ethAccount == null) return null;
return ethAccount.EthAddress;
}

17
ProjectPlugins/CodexPlugin/CodexNodeGroup.cs
View File

@ -15,11 +15,11 @@ namespace CodexPlugin
{
private readonly CodexStarter starter;
public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory)
{
this.starter = starter;
Containers = containers;
Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
Version = new DebugInfoVersion();
}
@ -39,7 +39,14 @@ namespace CodexPlugin
Containers = null!;
}
public RunningContainers[] Containers { get; private set; }
public void Stop(CodexNode node, bool waitTillStopped)
{
starter.Stop(node.Pod, waitTillStopped);
Nodes = Nodes.Where(n => n != node).ToArray();
Containers = Containers.Where(c => c != node.Pod).ToArray();
}
public RunningPod[] Containers { get; private set; }
public CodexNode[] Nodes { get; private set; }
public DebugInfoVersion Version { get; private set; }
public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
@ -74,9 +81,9 @@ namespace CodexPlugin
Version = first;
}
private CodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory)
private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
{
var watcher = factory.CreateCrashWatcher(c);
var watcher = factory.CreateCrashWatcher(c.Containers.Single());
var access = new CodexAccess(tools, c, watcher);
return factory.CreateOnlineCodexNode(access, this);
}

4
ProjectPlugins/CodexPlugin/CodexPlugin.cs
View File

@ -32,13 +32,13 @@ namespace CodexPlugin
{
}
public RunningContainers[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
public RunningPod[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
{
var codexSetup = GetSetup(numberOfNodes, setup);
return codexStarter.BringOnline(codexSetup);
}
public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
{
containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
return codexStarter.WrapCodexContainers(coreInterface, containers);

24
ProjectPlugins/CodexPlugin/CodexStarter.cs
View File

@ -19,7 +19,7 @@ namespace CodexPlugin
apiChecker = new ApiChecker(pluginTools);
}
public RunningContainers[] BringOnline(CodexSetup codexSetup)
public RunningPod[] BringOnline(CodexSetup codexSetup)
{
LogSeparator();
Log($"Starting {codexSetup.Describe()}...");
@ -34,14 +34,14 @@ namespace CodexPlugin
{
var podInfo = GetPodInfo(rc);
var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})");
}
LogSeparator();
return containers;
}
public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
{
var codexNodeFactory = new CodexNodeFactory(pluginTools);
@ -65,6 +65,14 @@ namespace CodexPlugin
Log("Stopped.");
}
public void Stop(RunningPod pod, bool waitTillStopped)
{
Log($"Stopping node...");
var workflow = pluginTools.CreateWorkflow();
workflow.Stop(pod, waitTillStopped);
Log("Stopped.");
}
public string GetCodexId()
{
if (versionResponse != null) return versionResponse.Version;
@ -85,7 +93,7 @@ namespace CodexPlugin
return startupConfig;
}
private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
private RunningPod[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
{
var futureContainers = new List<FutureContainers>();
for (var i = 0; i < numberOfNodes; i++)
@ -99,13 +107,13 @@ namespace CodexPlugin
.ToArray();
}
private PodInfo GetPodInfo(RunningContainers rc)
private PodInfo GetPodInfo(RunningPod rc)
{
var workflow = pluginTools.CreateWorkflow();
return workflow.GetPodInfo(rc);
}
private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory)
{
var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);
@ -122,10 +130,10 @@ namespace CodexPlugin
return group;
}
private void CodexNodesNotOnline(CoreInterface coreInterface, RunningContainers[] runningContainers)
private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers)
{
Log("Codex nodes failed to start");
foreach (var container in runningContainers.Containers()) coreInterface.DownloadLog(container);
foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container);
}
private void LogSeparator()

4
ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs
View File

@ -5,12 +5,12 @@ namespace CodexPlugin
{
public static class CoreInterfaceExtensions
{
public static RunningContainers[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
public static RunningPod[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
{
return Plugin(ci).DeployCodexNodes(number, setup);
}
public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers)
public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningPod[] containers)
{
return Plugin(ci).WrapCodexContainers(ci, containers);
}

8
ProjectPlugins/GethPlugin/GethDeployment.cs
View File

@ -7,9 +7,9 @@ namespace GethPlugin
{
public class GethDeployment : IHasContainer
{
public GethDeployment(RunningContainers containers, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
public GethDeployment(RunningPod pod, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
{
Containers = containers;
Pod = pod;
DiscoveryPort = discoveryPort;
HttpPort = httpPort;
WsPort = wsPort;
@ -17,9 +17,9 @@ namespace GethPlugin
PubKey = pubKey;
}
public RunningContainers Containers { get; }
public RunningPod Pod { get; }
[JsonIgnore]
public RunningContainer Container { get { return Containers.Containers.Single(); } }
public RunningContainer Container { get { return Pod.Containers.Single(); } }
public Port DiscoveryPort { get; }
public Port HttpPort { get; }
public Port WsPort { get; }

12
ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs
View File

@ -6,24 +6,24 @@ namespace MetricsPlugin
{
public static class CoreInterfaceExtensions
{
public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
{
return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
}
public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
{
return Plugin(ci).DeployMetricsCollector(scrapeTargets);
}
public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IHasMetricsScrapeTarget scrapeTarget)
public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
{
return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget);
return ci.WrapMetricsCollector(metricsPod, scrapeTarget.MetricsScrapeTarget);
}
public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget)
public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IMetricsScrapeTarget scrapeTarget)
{
return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget);
return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
}
public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)

8
ProjectPlugins/MetricsPlugin/MetricsPlugin.cs
View File

@ -31,15 +31,15 @@ namespace MetricsPlugin
{
}
public RunningContainers DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
{
return starter.CollectMetricsFor(scrapeTargets);
}
public IMetricsAccess WrapMetricsCollectorDeployment(RunningContainers runningContainer, IMetricsScrapeTarget target)
public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)
{
runningContainer = SerializeGate.Gate(runningContainer);
return starter.CreateAccessForTarget(runningContainer, target);
runningPod = SerializeGate.Gate(runningPod);
return starter.CreateAccessForTarget(runningPod, target);
}
public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName)

6
ProjectPlugins/MetricsPlugin/PrometheusStarter.cs
View File

@ -16,7 +16,7 @@ namespace MetricsPlugin
this.tools = tools;
}
public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets)
public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
{
if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
@ -32,9 +32,9 @@ namespace MetricsPlugin
return runningContainers;
}
public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target)
public MetricsAccess CreateAccessForTarget(RunningPod metricsPod, IMetricsScrapeTarget target)
{
var metricsQuery = new MetricsQuery(tools, metricsContainer.Containers.Single());
var metricsQuery = new MetricsQuery(tools, metricsPod.Containers.Single());
return new MetricsAccess(metricsQuery, target);
}

4
Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs
View File

@ -49,8 +49,8 @@ namespace ContinuousTests
var start = startUtc.ToString("o");
var end = endUtc.ToString("o");
var containerName = container.RunningContainers.StartResult.Deployment.Name;
var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
var containerName = container.RunningPod.StartResult.Deployment.Name;
var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
//container_name : codex3-5 - deploymentName as stored in pod
// pod_namespace : codex - continuous - nolimits - tests - 1

10
Tests/CodexContinuousTests/SingleTestRun.cs
View File

@ -125,8 +125,8 @@ namespace ContinuousTests
foreach (var node in nodes)
{
var container = node.Container;
var deploymentName = container.RunningContainers.StartResult.Deployment.Name;
var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
var deploymentName = container.RunningPod.StartResult.Deployment.Name;
var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
var openingLine =
$"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
@ -295,13 +295,13 @@ namespace ContinuousTests
return entryPoint.CreateInterface().WrapCodexContainers(containers).ToArray();
}
private RunningContainers[] SelectRandomContainers()
private RunningPod[] SelectRandomContainers()
{
var number = handle.Test.RequiredNumberOfNodes;
var containers = config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToList();
var containers = config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToList();
if (number == -1) return containers.ToArray();
var result = new RunningContainers[number];
var result = new RunningPod[number];
for (var i = 0; i < number; i++)
{
result[i] = containers.PickOneRandom();

8
Tests/CodexContinuousTests/StartupChecker.cs
View File

@ -43,13 +43,13 @@ namespace ContinuousTests
var workflow = entryPoint.Tools.CreateWorkflow();
foreach (var instance in deployment.CodexInstances)
{
foreach (var container in instance.Containers.Containers)
foreach (var container in instance.Pod.Containers)
{
var podInfo = workflow.GetPodInfo(container);
log.Log($"Codex environment variables for '{container.Name}':");
log.Log(
$"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
$"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}");
$"Namespace: {container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
$"Pod name: {podInfo.Name} - Deployment name: {instance.Pod.StartResult.Deployment.Name}");
var codexVars = container.Recipe.EnvVars;
foreach (var vars in codexVars) log.Log(vars.ToString());
log.Log("");
@ -92,7 +92,7 @@ namespace ContinuousTests
private void CheckCodexNodes(BaseLog log, Configuration config)
{
var nodes = entryPoint.CreateInterface()
.WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
.WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToArray());
var pass = true;
foreach (var n in nodes)
{

4
Tests/CodexTests/BasicTests/ContinuousSubstitute.cs
View File

@ -132,7 +132,7 @@ namespace CodexTests.BasicTests
private const string BytesStoredMetric = "codexRepostoreBytesUsed";
private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningContainers rc)
private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningPod rc)
{
ScopedTestFiles(() =>
{
@ -154,7 +154,7 @@ namespace CodexTests.BasicTests
var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value);
return high > newBytes && newBytes > low;
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2));
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2), nameof(ContinuousSubstitute));
FileUtils.TrackedFile? downloadedFile = null;
LogBytesPerMillisecond(() => downloadedFile = secondary.DownloadContent(contentId));

4
Tests/CodexTests/BasicTests/NetworkIsolationTest.cs
View File

@ -19,7 +19,7 @@ namespace CodexTests.BasicTests
{
node = Ci.StartCodexNode();
Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5));
Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), nameof(SetUpANodeAndWait));
}
[Test]
@ -27,7 +27,7 @@ namespace CodexTests.BasicTests
{
var myNode = Ci.StartCodexNode();
Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5));
Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5), nameof(ForeignNodeConnects));
try
{

83
Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs Normal file
View File

@ -0,0 +1,83 @@
using DistTestCore;
using Logging;
using NUnit.Framework;
using Utils;
namespace CodexTests.ScalabilityTests
{
[TestFixture]
public class ClusterDiscSpeedTests : DistTest
{
private readonly Random random = new Random();
[Test]
[Combinatorial]
public void DiscSpeedTest(
[Values(1, 10, 100, 1024, 1024 * 10, 1024 * 100, 1024 * 1024)] int bufferSizeKb
)
{
long targetSize = (long)(1024 * 1024 * 1024) * 2;
long bufferSizeBytes = ((long)bufferSizeKb) * 1024;
var filename = nameof(DiscSpeedTest);
Thread.Sleep(2000);
if (File.Exists(filename)) File.Delete(filename);
Thread.Sleep(2000);
var writeSpeed = PerformWrite(targetSize, bufferSizeBytes, filename);
Thread.Sleep(2000);
var readSpeed = PerformRead(targetSize, bufferSizeBytes, filename);
Log($"Write speed: {writeSpeed} per second.");
Log($"Read speed: {readSpeed} per second.");
}
private ByteSize PerformWrite(long targetSize, long bufferSizeBytes, string filename)
{
long bytesWritten = 0;
var buffer = new byte[bufferSizeBytes];
random.NextBytes(buffer);
var sw = Stopwatch.Begin(GetTestLog());
using (var stream = File.OpenWrite(filename))
{
while (bytesWritten < targetSize)
{
long remaining = targetSize - bytesWritten;
long toWrite = Math.Min(bufferSizeBytes, remaining);
stream.Write(buffer, 0, Convert.ToInt32(toWrite));
bytesWritten += toWrite;
}
}
var duration = sw.End("WriteTime");
double totalSeconds = duration.TotalSeconds;
double totalBytes = bytesWritten;
double bytesPerSecond = totalBytes / totalSeconds;
return new ByteSize(Convert.ToInt64(bytesPerSecond));
}
private ByteSize PerformRead(long targetSize, long bufferSizeBytes, string filename)
{
long bytesRead = 0;
var buffer = new byte[bufferSizeBytes];
var sw = Stopwatch.Begin(GetTestLog());
using (var stream = File.OpenRead(filename))
{
while (bytesRead < targetSize)
{
long remaining = targetSize - bytesRead;
long toRead = Math.Min(bufferSizeBytes, remaining);
var r = stream.Read(buffer, 0, Convert.ToInt32(toRead));
bytesRead += r;
}
}
var duration = sw.End("ReadTime");
double totalSeconds = duration.TotalSeconds;
double totalBytes = bytesRead;
double bytesPerSecond = totalBytes / totalSeconds;
return new ByteSize(Convert.ToInt64(bytesPerSecond));
}
}
}

120
Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs Normal file
View File

@ -0,0 +1,120 @@
using DistTestCore;
using NUnit.Framework;
using Utils;
namespace CodexTests.ScalabilityTests
{
[TestFixture]
public class MultiPeerDownloadTests : AutoBootstrapDistTest
{
[Test]
[DontDownloadLogs]
[UseLongTimeouts]
[Combinatorial]
public void MultiPeerDownload(
[Values(5, 10, 20)] int numberOfHosts,
[Values(100, 1000)] int fileSize
)
{
var hosts = AddCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
var file = GenerateTestFile(fileSize.MB());
var cid = hosts[0].UploadFile(file);
var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6);
var uploadLog = Ci.DownloadLog(hosts[0]);
var expectedNumberOfBlocks = RoundUp(fileSize.MB().SizeInBytes, 64.KB().SizeInBytes) + 1; // +1 for manifest block.
var blockCids = uploadLog
.FindLinesThatContain("Putting block into network store")
.Select(s =>
{
var start = s.IndexOf("cid=") + 4;
var end = s.IndexOf(" count=");
var len = end - start;
return s.Substring(start, len);
})
.ToArray();
Assert.That(blockCids.Length, Is.EqualTo(expectedNumberOfBlocks));
foreach (var h in hosts) h.DownloadContent(cid);
var client = AddCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
var resultFile = client.DownloadContent(cid);
resultFile!.AssertIsEqual(file);
var downloadLog = Ci.DownloadLog(client);
var host = string.Empty;
var blockCidHostMap = new Dictionary<string, string>();
downloadLog.IterateLines(line =>
{
if (line.Contains("peer=") && line.Contains(" len="))
{
var start = line.IndexOf("peer=") + 5;
var end = line.IndexOf(" len=");
var len = end - start;
host = line.Substring(start, len);
}
else if (!string.IsNullOrEmpty(host) && line.Contains("Storing block with key"))
{
var start = line.IndexOf("cid=") + 4;
var end = line.IndexOf(" count=");
var len = end - start;
var blockCid = line.Substring(start, len);
blockCidHostMap.Add(blockCid, host);
host = string.Empty;
}
});
var totalFetched = blockCidHostMap.Count(p => !string.IsNullOrEmpty(p.Value));
//PrintFullMap(blockCidHostMap);
PrintOverview(blockCidHostMap);
Log("Expected number of blocks: " + expectedNumberOfBlocks);
Log("Total number of block CIDs found in dataset + manifest block: " + blockCids.Length);
Log("Total blocks fetched by hosts: " + totalFetched);
Assert.That(totalFetched, Is.EqualTo(expectedNumberOfBlocks));
}
private void PrintOverview(Dictionary<string, string> blockCidHostMap)
{
var overview = new Dictionary<string, int>();
foreach (var pair in blockCidHostMap)
{
if (!overview.ContainsKey(pair.Value)) overview.Add(pair.Value, 1);
else overview[pair.Value]++;
}
Log("Blocks fetched per host:");
foreach (var pair in overview)
{
Log($"Host: {pair.Key} = {pair.Value}");
}
}
private void PrintFullMap(Dictionary<string, string> blockCidHostMap)
{
Log("Per block, host it was fetched from:");
foreach (var pair in blockCidHostMap)
{
if (string.IsNullOrEmpty(pair.Value))
{
Log($"block: {pair.Key} = Not seen");
}
else
{
Log($"block: {pair.Key} = '{pair.Value}'");
}
}
}
private long RoundUp(long filesize, long blockSize)
{
double f = filesize;
double b = blockSize;
var result = Math.Ceiling(f / b);
return Convert.ToInt64(result);
}
}
}

39
Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs Normal file
View File

@ -0,0 +1,39 @@
using CodexPlugin;
using DistTestCore;
using NUnit.Framework;
using Utils;
namespace CodexTests.ScalabilityTests
{
[TestFixture]
public class OneClientLargeFileTests : CodexDistTest
{
[Test]
[Combinatorial]
[UseLongTimeouts]
public void OneClientLargeFile([Values(
256,
512,
1024, // GB
2048,
4096,
8192,
16384,
32768,
65536,
131072
)] int sizeMb)
{
var testFile = GenerateTestFile(sizeMb.MB());
var node = AddCodex(s => s
.WithLogLevel(CodexLogLevel.Warn)
.WithStorageQuota((sizeMb + 10).MB())
);
var contentId = node.UploadFile(testFile);
var downloadedFile = node.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
}
}
}

129
Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs Normal file
View File

@ -0,0 +1,129 @@
using CodexPlugin;
using DistTestCore;
using FileUtils;
using NUnit.Framework;
using Utils;
namespace CodexTests.ScalabilityTests;
[TestFixture]
public class ScalabilityTests : CodexDistTest
{
private const string PatchedImage = "codexstorage/nim-codex:sha-9aeac06-dist-tests";
private const string MasterImage = "codexstorage/nim-codex:sha-5380912-dist-tests";
/// <summary>
/// We upload a file to node A, then download it with B.
/// Then we stop node A, and download again with node C.
/// </summary>
[Test]
[Combinatorial]
[UseLongTimeouts]
[DontDownloadLogs]
public void ShouldMaintainFileInNetwork(
[Values(10, 40, 80, 100)] int numberOfNodes,
[Values(100, 1000, 5000, 10000)] int fileSizeInMb,
[Values(true, false)] bool usePatchedImage
)
{
CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage;
var logLevel = CodexLogLevel.Info;
var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
var nodes = AddCodex(numberOfNodes - 1, s => s
.WithBootstrapNode(bootstrap)
.WithLogLevel(logLevel)
.WithStorageQuota((fileSizeInMb + 50).MB())
).ToList();
var uploader = nodes.PickOneRandom();
var downloader = nodes.PickOneRandom();
var testFile = GenerateTestFile(fileSizeInMb.MB());
var contentId = uploader.UploadFile(testFile);
var downloadedFile = downloader.DownloadContent(contentId);
downloadedFile!.AssertIsEqual(testFile);
uploader.Stop(true);
var otherDownloader = nodes.PickOneRandom();
downloadedFile = otherDownloader.DownloadContent(contentId);
downloadedFile!.AssertIsEqual(testFile);
}
/// <summary>
/// We upload a file to each node, to put a more wide-spread load on the network.
/// Then we run the same test as ShouldMaintainFileInNetwork.
/// </summary>
[Ignore("Make ShouldMaintainFileInNetwork pass reliably first.")]
[Test]
[Combinatorial]
[UseLongTimeouts]
[DontDownloadLogs]
public void EveryoneGetsAFile(
[Values(10, 40, 80, 100)] int numberOfNodes,
[Values(100, 1000)] int fileSizeInMb,
[Values(true, false)] bool usePatchedImage
)
{
CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage;
var logLevel = CodexLogLevel.Info;
var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
var nodes = AddCodex(numberOfNodes - 1, s => s
.WithBootstrapNode(bootstrap)
.WithLogLevel(logLevel)
.WithStorageQuota((fileSizeInMb + 50).MB())
).ToList();
var pairTasks = nodes.Select(n =>
{
return Task.Run(() =>
{
var file = GenerateTestFile(fileSizeInMb.MB());
var cid = n.UploadFile(file);
return new NodeFilePair(n, file, cid);
});
});
var pairs = pairTasks.Select(t => Time.Wait(t)).ToList();
RunDoubleDownloadTest(
pairs.PickOneRandom(),
pairs.PickOneRandom(),
pairs.PickOneRandom()
);
}
private void RunDoubleDownloadTest(NodeFilePair source, NodeFilePair dl1, NodeFilePair dl2)
{
var expectedFile = source.File;
var cid = source.Cid;
var file1 = dl1.Node.DownloadContent(cid);
file1!.AssertIsEqual(expectedFile);
source.Node.Stop(true);
var file2 = dl2.Node.DownloadContent(cid);
file2!.AssertIsEqual(expectedFile);
}
public class NodeFilePair
{
public NodeFilePair(ICodexNode node, TrackedFile file, ContentId cid)
{
Node = node;
File = file;
Cid = cid;
}
public ICodexNode Node { get; }
public TrackedFile File { get; }
public ContentId Cid { get; }
}
}

5
Tests/DistTestCore/Configuration.cs
View File

@ -24,6 +24,9 @@ namespace DistTestCore
this.dataFilesPath = dataFilesPath;
}
/// <summary>
/// Does not override [DontDownloadLogs] attribute.
/// </summary>
public bool AlwaysDownloadContainerLogs { get; set; }
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace)
@ -36,7 +39,7 @@ namespace DistTestCore
var config = new KubernetesWorkflow.Configuration(
kubeConfigFile: kubeConfigFile,
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay(),
retryDelay: timeSet.K8sOperationRetryDelay(),
kubernetesNamespace: k8sNamespace
);

22
Tests/DistTestCore/DistTest.cs
View File

@ -98,7 +98,7 @@ namespace DistTestCore
}
catch (Exception ex)
{
fixtureLog.Error("Cleanup failed: " + ex.Message);
fixtureLog.Error("Cleanup failed: " + ex);
GlobalTestFailure.HasFailed = true;
}
}
@ -236,9 +236,19 @@ namespace DistTestCore
}
private bool ShouldUseLongTimeouts()
{
return CurrentTestMethodHasAttribute<UseLongTimeoutsAttribute>();
}
private bool HasDontDownloadAttribute()
{
return CurrentTestMethodHasAttribute<DontDownloadLogsAttribute>();
}
private bool CurrentTestMethodHasAttribute<T>() where T : PropertyAttribute
{
// Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
// But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
// But this doesn't work for tests making use of [TestCase] or [Combinatorial]. So instead, we use reflection here to figure out
// if the attribute is present.
var currentTest = TestContext.CurrentContext.Test;
var className = currentTest.ClassName;
@ -247,7 +257,7 @@ namespace DistTestCore
var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
return testMethods.Any(m => m.GetCustomAttribute<T>() != null);
}
private void IncludeLogsOnTestFailure(TestLifecycle lifecycle)
@ -268,9 +278,10 @@ namespace DistTestCore
private bool ShouldDownloadAllLogs(TestStatus testStatus)
{
if (configuration.AlwaysDownloadContainerLogs) return true;
if (!IsDownloadingLogsEnabled()) return false;
if (testStatus == TestStatus.Failed)
{
return IsDownloadingLogsEnabled();
return true;
}
return false;
@ -288,8 +299,7 @@ namespace DistTestCore
private bool IsDownloadingLogsEnabled()
{
var testProperties = TestContext.CurrentContext.Test.Properties;
return !testProperties.ContainsKey(DontDownloadLogsOnFailureAttribute.DontDownloadKey);
return !HasDontDownloadAttribute();
}
}

4
Tests/DistTestCore/DontDownloadLogsOnFailureAttribute.cs → Tests/DistTestCore/DontDownloadLogsAttribute.cs
View File

@ -3,11 +3,11 @@
namespace DistTestCore
{
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class DontDownloadLogsOnFailureAttribute : PropertyAttribute
public class DontDownloadLogsAttribute : PropertyAttribute
{
public const string DontDownloadKey = "DontDownloadLogs";
public DontDownloadLogsOnFailureAttribute()
public DontDownloadLogsAttribute()
: base(DontDownloadKey)
{
}

2
Tests/DistTestCore/Helpers/AssertHelpers.cs
View File

@ -14,7 +14,7 @@ namespace DistTestCore.Helpers
Time.WaitUntil(() => {
var c = constraint.Resolve();
return c.ApplyTo(actual()).IsSuccess;
});
}, "RetryAssert: " + message);
}
catch (TimeoutException)
{

19
Tests/DistTestCore/TestLifecycle.cs
View File

@ -13,7 +13,7 @@ namespace DistTestCore
private const string TestsType = "dist-tests";
private readonly EntryPoint entryPoint;
private readonly Dictionary<string, string> metadata;
private readonly List<RunningContainers> runningContainers = new();
private readonly List<RunningPod> runningContainers = new();
private readonly string deployId;
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId)
@ -65,12 +65,12 @@ namespace DistTestCore
return DateTime.UtcNow - TestStart;
}
public void OnContainersStarted(RunningContainers rc)
public void OnContainersStarted(RunningPod rc)
{
runningContainers.Add(rc);
}
public void OnContainersStopped(RunningContainers rc)
public void OnContainersStopped(RunningPod rc)
{
runningContainers.Remove(rc);
}
@ -93,13 +93,20 @@ namespace DistTestCore
public void DownloadAllLogs()
{
foreach (var rc in runningContainers)
try
{
foreach (var c in rc.Containers)
foreach (var rc in runningContainers)
{
CoreInterface.DownloadLog(c);
foreach (var c in rc.Containers)
{
CoreInterface.DownloadLog(c);
}
}
}
catch (Exception ex)
{
Log.Error("Exception during log download: " + ex);
}
}
}
}

8
Tools/CodexNetDeployer/Deployer.cs
View File

@ -122,7 +122,7 @@ namespace CodexNetDeployer
});
}
private RunningContainers? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
private RunningPod? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
CodexContractsDeployment contractsDeployment)
{
if (!config.DeployDiscordBot) return null;
@ -155,7 +155,7 @@ namespace CodexNetDeployer
return rc;
}
private RunningContainers? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
private RunningPod? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
{
if (!config.MetricsScraper || !startResults.Any()) return null;
@ -180,7 +180,7 @@ namespace CodexNetDeployer
private CodexInstance CreateCodexInstance(ICodexNode node)
{
return new CodexInstance(node.Container.RunningContainers, node.GetDebugInfo());
return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo());
}
private string? GetKubeConfig(string kubeConfigFile)
@ -270,7 +270,7 @@ namespace CodexNetDeployer
return TimeSpan.FromMinutes(10);
}
public TimeSpan WaitForK8sServiceDelay()
public TimeSpan K8sOperationRetryDelay()
{
return TimeSpan.FromSeconds(30);
}

4
Tools/CodexNetDeployer/K8sHook.cs
View File

@ -18,11 +18,11 @@ namespace CodexNetDeployer
this.metadata = metadata;
}
public void OnContainersStarted(RunningContainers rc)
public void OnContainersStarted(RunningPod rc)
{
}
public void OnContainersStopped(RunningContainers rc)
public void OnContainersStopped(RunningPod rc)
{
}