status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-02-23 13:38:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/refactor' into feature/marketplace-contracts

Browse Source
This commit is contained in:
benbierens 2023-04-17 13:01:03 +02:00
commit a12a4fb154
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
97 changed files with 2976 additions and 2358 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
CodexDistTestCore/CodexAPI.cs
View File

@ -1,17 +0,0 @@
namespace CodexDistTestCore
{
public class CodexDebugResponse
{
public string id { get; set; } = string.Empty;
public string[] addrs { get; set; } = new string[0];
public string repo { get; set; } = string.Empty;
public string spr { get; set; } = string.Empty;
public CodexDebugVersionResponse codex { get; set; } = new();
}
public class CodexDebugVersionResponse
{
public string version { get; set; } = string.Empty;
public string revision { get; set; } = string.Empty;
}
}

18
CodexDistTestCore/CodexDistTestCore.csproj
View File

@ -1,18 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<RootNamespace>CodexDistTestCore</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="KubernetesClient" Version="10.1.4" />
<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
<PackageReference Include="nunit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
</ItemGroup>
</Project>

89
CodexDistTestCore/CodexNodeContainer.cs
View File

@ -1,89 +0,0 @@
using CodexDistTestCore.Marketplace;
namespace CodexDistTestCore
{
public class CodexNodeContainer
{
public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort)
{
Name = name;
ServicePort = servicePort;
ServicePortName = servicePortName;
ApiPort = apiPort;
ContainerPortName = containerPortName;
DiscoveryPort = discoveryPort;
ListenPort = listenPort;
DataDir = dataDir;
MetricsPort = metricsPort;
}
public string Name { get; }
public int ServicePort { get; }
public string ServicePortName { get; }
public int ApiPort { get; }
public string ContainerPortName { get; }
public int DiscoveryPort { get; }
public int ListenPort { get; }
public string DataDir { get; }
public int MetricsPort { get; }
public GethCompanionNodeContainer? GethCompanionNodeContainer { get; set; } // :C
}
public class CodexGroupNumberSource
{
private readonly NumberSource codexNodeGroupNumberSource = new NumberSource(0);
private readonly NumberSource groupContainerNameSource = new NumberSource(1);
private readonly NumberSource servicePortSource = new NumberSource(30001);
public int GetNextCodexNodeGroupNumber()
{
return codexNodeGroupNumberSource.GetNextNumber();
}
public string GetNextServicePortName()
{
return $"node{groupContainerNameSource.GetNextNumber()}";
}
public int GetNextServicePort()
{
return servicePortSource.GetNextNumber();
}
}
public class CodexNodeContainerFactory
{
private readonly NumberSource containerNameSource = new NumberSource(1);
private readonly NumberSource codexPortSource = new NumberSource(8080);
private readonly CodexGroupNumberSource numberSource;
public CodexNodeContainerFactory(CodexGroupNumberSource numberSource)
{
this.numberSource = numberSource;
}
public CodexNodeContainer CreateNext(OfflineCodexNodes offline)
{
var n = containerNameSource.GetNextNumber();
return new CodexNodeContainer(
name: $"codex-node{n}",
servicePort: numberSource.GetNextServicePort(),
servicePortName: numberSource.GetNextServicePortName(),
apiPort: codexPortSource.GetNextNumber(),
containerPortName: $"api-{n}",
discoveryPort: codexPortSource.GetNextNumber(),
listenPort: codexPortSource.GetNextNumber(),
dataDir: $"datadir{n}",
metricsPort: GetMetricsPort(offline)
);
}
private int GetMetricsPort(OfflineCodexNodes offline)
{
if (offline.MetricsEnabled) return codexPortSource.GetNextNumber();
return 0;
}
}
}

113
CodexDistTestCore/CodexNodeGroup.cs
View File

@ -1,113 +0,0 @@
using CodexDistTestCore.Config;
using CodexDistTestCore.Marketplace;
using k8s.Models;
using System.Collections;
namespace CodexDistTestCore
{
public interface ICodexNodeGroup : IEnumerable<IOnlineCodexNode>
{
IOfflineCodexNodes BringOffline();
IOnlineCodexNode this[int index] { get; }
}
public class CodexNodeGroup : ICodexNodeGroup
{
private readonly TestLog log;
private readonly IK8sManager k8SManager;
public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
{
this.log = log;
OrderNumber = orderNumber;
Origin = origin;
this.k8SManager = k8SManager;
Nodes = nodes;
foreach (var n in nodes) n.Group = this;
}
public IOnlineCodexNode this[int index]
{
get
{
return Nodes[index];
}
}
public IOfflineCodexNodes BringOffline()
{
return k8SManager.BringOffline(this);
}
public int OrderNumber { get; }
public OfflineCodexNodes Origin { get; }
public OnlineCodexNode[] Nodes { get; }
public V1Deployment? Deployment { get; set; }
public V1Service? Service { get; set; }
public PodInfo? PodInfo { get; set; }
public GethCompanionGroup? GethCompanionGroup { get; set; }
public CodexNodeContainer[] GetContainers()
{
return Nodes.Select(n => n.Container).ToArray();
}
public IEnumerator<IOnlineCodexNode> GetEnumerator()
{
return Nodes.Cast<IOnlineCodexNode>().GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return Nodes.GetEnumerator();
}
public V1ObjectMeta GetServiceMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-entrypoint-" + OrderNumber,
NamespaceProperty = K8sCluster.K8sNamespace
};
}
public V1ObjectMeta GetDeploymentMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-node-" + OrderNumber,
NamespaceProperty = K8sCluster.K8sNamespace
};
}
public CodexNodeLog DownloadLog(IOnlineCodexNode node)
{
var logDownloader = new PodLogDownloader(log, k8SManager);
var n = (OnlineCodexNode)node;
return logDownloader.DownloadLog(n);
}
public Dictionary<string, string> GetSelector()
{
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + OrderNumber } };
}
public string Describe()
{
return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}";
}
}
public class PodInfo
{
public PodInfo(string name, string ip)
{
Name = name;
Ip = ip;
}
public string Name { get; }
public string Ip { get; }
}
}

72
CodexDistTestCore/Config/CodexDockerImage.cs
View File

@ -1,72 +0,0 @@
using k8s.Models;
namespace CodexDistTestCore.Config
{
public class CodexDockerImage
{
public string GetImageTag()
{
return "thatbenbierens/nim-codex:sha-b204837";
}
public string GetExpectedImageRevision()
{
return "b20483";
}
public List<V1EnvVar> CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer container)
{
var formatter = new EnvFormatter();
formatter.Create(node, container);
return formatter.Result;
}
private class EnvFormatter
{
public List<V1EnvVar> Result { get; } = new List<V1EnvVar>();
public void Create(OfflineCodexNodes node, CodexNodeContainer container)
{
AddVar("API_PORT", container.ApiPort.ToString());
AddVar("DATA_DIR", container.DataDir);
AddVar("DISC_PORT", container.DiscoveryPort.ToString());
AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{container.ListenPort}");
if (node.BootstrapNode != null)
{
var debugInfo = node.BootstrapNode.GetDebugInfo();
AddVar("BOOTSTRAP_SPR", debugInfo.spr);
}
if (node.LogLevel != null)
{
AddVar("LOG_LEVEL", node.LogLevel.ToString()!.ToUpperInvariant());
}
if (node.StorageQuota != null)
{
AddVar("STORAGE_QUOTA", node.StorageQuota.SizeInBytes.ToString()!);
}
if (node.MetricsEnabled)
{
AddVar("METRICS_ADDR", "0.0.0.0");
AddVar("METRICS_PORT", container.MetricsPort.ToString());
}
if (node.MarketplaceConfig != null)
{
//ETH_PROVIDER
//ETH_ACCOUNT
//ETH_DEPLOYMENT
AddVar("ETH_ACCOUNT", container.GethCompanionNodeContainer!.Account);
}
}
private void AddVar(string key, string value)
{
Result.Add(new V1EnvVar
{
Name = key,
Value = value
});
}
}
}
}

7
CodexDistTestCore/Config/FileManagerConfig.cs
View File

@ -1,7 +0,0 @@
namespace CodexDistTestCore.Config
{
public class FileManagerConfig
{
public const string Folder = "TestDataFiles";
}
}

40
CodexDistTestCore/Config/K8sCluster.cs
View File

@ -1,40 +0,0 @@
using k8s;
namespace CodexDistTestCore.Config
{
public class K8sCluster
{
public const string K8sNamespace = "codex-test-namespace";
private const string KubeConfigFile = "C:\\kube\\config";
private readonly Dictionary<Location, string> K8sNodeLocationMap = new Dictionary<Location, string>
{
{ Location.BensLaptop, "worker01" },
{ Location.BensOldGamingMachine, "worker02" },
};
private KubernetesClientConfiguration? config;
public KubernetesClientConfiguration GetK8sClientConfig()
{
if (config != null) return config;
//config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
config = KubernetesClientConfiguration.BuildDefaultConfig();
return config;
}
public string GetIp()
{
var c = GetK8sClientConfig();
var host = c.Host.Replace("https://", "");
return host.Substring(0, host.IndexOf(':'));
}
public string GetNodeLabelForLocation(Location location)
{
if (location == Location.Unspecified) return string.Empty;
return K8sNodeLocationMap[location];
}
}
}

7
CodexDistTestCore/Config/LogConfig.cs
View File

@ -1,7 +0,0 @@
namespace CodexDistTestCore.Config
{
public class LogConfig
{
public const string LogRoot = "D:/CodexTestLogs";
}
}

120
CodexDistTestCore/DistTest.cs
View File

@ -1,120 +0,0 @@
using CodexDistTestCore.Config;
using NUnit.Framework;
namespace CodexDistTestCore
{
[SetUpFixture]
public abstract class DistTest
{
private TestLog log = null!;
private FileManager fileManager = null!;
public K8sManager k8sManager = null!;
[OneTimeSetUp]
public void GlobalSetup()
{
// Previous test run may have been interrupted.
// Begin by cleaning everything up.
log = new TestLog();
fileManager = new FileManager(log);
k8sManager = new K8sManager(log, fileManager);
try
{
k8sManager.DeleteAllResources();
fileManager.DeleteAllTestFiles();
}
catch (Exception ex)
{
GlobalTestFailure.HasFailed = true;
log.Error($"Global setup cleanup failed with: {ex}");
throw;
}
log.Log("Global setup cleanup successful");
}
[SetUp]
public void SetUpDistTest()
{
if (GlobalTestFailure.HasFailed)
{
Assert.Inconclusive("Skip test: Previous test failed during clean up.");
}
else
{
var dockerImage = new CodexDockerImage();
log = new TestLog();
log.Log($"Using docker image '{dockerImage.GetImageTag()}'");
fileManager = new FileManager(log);
k8sManager = new K8sManager(log, fileManager);
}
}
[TearDown]
public void TearDownDistTest()
{
try
{
log.EndTest();
IncludeLogsAndMetricsOnTestFailure();
k8sManager.DeleteAllResources();
fileManager.DeleteAllTestFiles();
}
catch (Exception ex)
{
log.Error("Cleanup failed: " + ex.Message);
GlobalTestFailure.HasFailed = true;
}
}
public TestFile GenerateTestFile(ByteSize size)
{
return fileManager.GenerateTestFile(size);
}
public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes)
{
return new OfflineCodexNodes(k8sManager, numberOfNodes);
}
private void IncludeLogsAndMetricsOnTestFailure()
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
if (IsDownloadingLogsAndMetricsEnabled())
{
log.Log("Downloading all CodexNode logs and metrics because of test failure...");
k8sManager.ForEachOnlineGroup(DownloadLogs);
k8sManager.DownloadAllMetrics();
}
else
{
log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
}
}
}
private void DownloadLogs(CodexNodeGroup group)
{
foreach (var node in group)
{
var downloader = new PodLogDownloader(log, k8sManager);
var n = (OnlineCodexNode)node;
downloader.DownloadLog(n);
}
}
private bool IsDownloadingLogsAndMetricsEnabled()
{
var testProperties = TestContext.CurrentContext.Test.Properties;
return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey);
}
}
public static class GlobalTestFailure
{
public static bool HasFailed { get; set; } = false;
}
}

177
CodexDistTestCore/K8sManager.cs
View File

@ -1,177 +0,0 @@
using CodexDistTestCore.Marketplace;
using CodexDistTestCore.Metrics;
namespace CodexDistTestCore
{
public interface IK8sManager
{
ICodexNodeGroup BringOnline(OfflineCodexNodes node);
IOfflineCodexNodes BringOffline(ICodexNodeGroup node);
void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler);
}
public class K8sManager : IK8sManager
{
private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource();
private readonly List<CodexNodeGroup> onlineCodexNodeGroups = new List<CodexNodeGroup>();
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly TestLog log;
private readonly IFileManager fileManager;
private readonly MetricsAggregator metricsAggregator;
private readonly MarketplaceController marketplaceController;
public K8sManager(TestLog log, IFileManager fileManager)
{
this.log = log;
this.fileManager = fileManager;
metricsAggregator = new MetricsAggregator(log, this);
marketplaceController = new MarketplaceController(log, this);
}
public ICodexNodeGroup BringOnline(OfflineCodexNodes offline)
{
var group = CreateOnlineCodexNodes(offline);
if (offline.MarketplaceConfig != null)
{
group.GethCompanionGroup = marketplaceController.BringOnlineMarketplace(offline);
ConnectMarketplace(group);
}
K8s(k => k.BringOnline(group, offline));
if (offline.MetricsEnabled)
{
BringOnlineMetrics(group);
}
log.Log($"{group.Describe()} online.");
return group;
}
public IOfflineCodexNodes BringOffline(ICodexNodeGroup node)
{
var online = GetAndRemoveActiveNodeFor(node);
K8s(k => k.BringOffline(online));
log.Log($"{online.Describe()} offline.");
return online.Origin;
}
public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments)
{
return K8s(k => k.ExecuteCommand(pod, containerName, command, arguments));
}
public void DeleteAllResources()
{
K8s(k => k.DeleteAllResources());
}
public void ForEachOnlineGroup(Action<CodexNodeGroup> action)
{
foreach (var group in onlineCodexNodeGroups) action(group);
}
public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
{
K8s(k => k.FetchPodLog(node, logHandler));
}
public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber)
{
var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config);
return K8s(k => k.BringOnlinePrometheus(spec));
}
public K8sGethBoostrapSpecs CreateGethBootstrapNodeSpec()
{
return new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort());
}
public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec)
{
return K8s(k => k.BringOnlineGethBootstrapNode(spec));
}
public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group)
{
return K8s(k => k.BringOnlineGethCompanionGroup(info, group));
}
public void DownloadAllMetrics()
{
metricsAggregator.DownloadAllMetrics();
}
private void BringOnlineMetrics(CodexNodeGroup group)
{
metricsAggregator.BeginCollectingMetricsFor(DowncastNodes(group));
}
private void ConnectMarketplace(CodexNodeGroup group)
{
for (var i = 0; i < group.Nodes.Length; i++)
{
ConnectMarketplace(group, group.Nodes[i], group.GethCompanionGroup!.Containers[i]);
}
}
private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node, GethCompanionNodeContainer container)
{
node.Container.GethCompanionNodeContainer = container; // :c
var access = new MarketplaceAccess(this, marketplaceController, log, group, container);
access.Initialize();
node.Marketplace = access;
}
private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
{
var containers = CreateContainers(offline);
var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
onlineCodexNodeGroups.Add(result);
return result;
}
private CodexNodeContainer[] CreateContainers(OfflineCodexNodes offline)
{
var factory = new CodexNodeContainerFactory(codexGroupNumberSource);
var containers = new List<CodexNodeContainer>();
for (var i = 0; i < offline.NumberOfNodes; i++) containers.Add(factory.CreateNext(offline));
return containers.ToArray();
}
private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node)
{
var n = (CodexNodeGroup)node;
onlineCodexNodeGroups.Remove(n);
return n;
}
private void K8s(Action<K8sOperations> action)
{
var k8s = new K8sOperations(knownPods);
action(k8s);
k8s.Close();
}
private T K8s<T>(Func<K8sOperations, T> action)
{
var k8s = new K8sOperations(knownPods);
var result = action(k8s);
k8s.Close();
return result;
}
private static OnlineCodexNode[] DowncastNodes(CodexNodeGroup group)
{
return group.Nodes.Cast<OnlineCodexNode>().ToArray();
}
}
}

427
CodexDistTestCore/K8sOperations.cs
View File

@ -1,427 +0,0 @@
using CodexDistTestCore.Config;
using CodexDistTestCore.Marketplace;
using CodexDistTestCore.Metrics;
using k8s;
using k8s.Models;
using NUnit.Framework;
namespace CodexDistTestCore
{
public class K8sOperations
{
private readonly CodexDockerImage dockerImage = new CodexDockerImage();
private readonly K8sCluster k8sCluster = new K8sCluster();
private readonly Kubernetes client;
private readonly KnownK8sPods knownPods;
public K8sOperations(KnownK8sPods knownPods)
{
this.knownPods = knownPods;
client = new Kubernetes(k8sCluster.GetK8sClientConfig());
}
public void Close()
{
client.Dispose();
}
public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline)
{
EnsureTestNamespace();
CreateDeployment(online, offline);
CreateService(online);
WaitUntilOnline(online);
FetchPodInfo(online);
}
public void BringOffline(CodexNodeGroup online)
{
var deploymentName = online.Deployment.Name();
DeleteDeployment(online);
DeleteService(online);
WaitUntilOffline(deploymentName);
}
public void DeleteAllResources()
{
DeleteNamespace();
WaitUntilZeroPods();
WaitUntilNamespaceDeleted();
}
public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
{
var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name);
logHandler.Log(stream);
}
public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments)
{
var runner = new CommandRunner(client, pod, containerName, command, arguments);
runner.Run();
return runner.GetStdOut();
}
public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec)
{
EnsureTestNamespace();
CreatePrometheusDeployment(spec);
CreatePrometheusService(spec);
WaitUntilPrometheusOnline(spec);
return new PrometheusInfo(spec.ServicePort, FetchNewPod());
}
public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec)
{
EnsureTestNamespace();
CreateGethBootstrapDeployment(spec);
CreateGethBootstrapService(spec);
WaitUntilGethBootstrapOnline(spec);
return FetchNewPod();
}
public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group)
{
EnsureTestNamespace();
CreateGethCompanionDeployment(info, group);
WaitUntilGethCompanionGroupOnline(info.Spec, group);
return FetchNewPod();
}
private void FetchPodInfo(CodexNodeGroup online)
{
online.PodInfo = FetchNewPod();
}
private PodInfo FetchNewPod()
{
var pods = client.ListNamespacedPod(K8sNamespace).Items;
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure.");
var newPod = newPods.Single();
var info = new PodInfo(newPod.Name(), newPod.Status.PodIP);
Assert.That(!string.IsNullOrEmpty(info.Name), "Invalid pod name received. Test infra failure.");
Assert.That(!string.IsNullOrEmpty(info.Ip), "Invalid pod IP received. Test infra failure.");
knownPods.Add(newPod.Name());
return info;
}
#region Waiting
private void WaitUntilOnline(CodexNodeGroup online)
{
WaitUntil(() =>
{
online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0;
});
}
private void WaitUntilOffline(string deploymentName)
{
WaitUntil(() =>
{
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
return deployment == null || deployment.Status.AvailableReplicas == 0;
});
}
private void WaitUntilZeroPods()
{
WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any());
}
private void WaitUntilNamespaceDeleted()
{
WaitUntil(() => !IsTestNamespaceOnline());
}
private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec)
{
WaitUntilDeploymentOnline(spec.GetDeploymentName());
}
private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec)
{
WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName());
}
private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group)
{
WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group));
}
private void WaitUntilDeploymentOnline(string deploymentName)
{
WaitUntil(() =>
{
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
});
}
private void WaitUntil(Func<bool> predicate)
{
var start = DateTime.UtcNow;
var state = predicate();
while (!state)
{
if (DateTime.UtcNow - start > Timing.K8sOperationTimeout())
{
Assert.Fail("K8s operation timed out.");
throw new TimeoutException();
}
Timing.WaitForK8sServiceDelay();
state = predicate();
}
}
#endregion
#region Service management
private void CreateService(CodexNodeGroup online)
{
var serviceSpec = new V1Service
{
ApiVersion = "v1",
Metadata = online.GetServiceMetadata(),
Spec = new V1ServiceSpec
{
Type = "NodePort",
Selector = online.GetSelector(),
Ports = CreateServicePorts(online)
}
};
online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
}
private List<V1ServicePort> CreateServicePorts(CodexNodeGroup online)
{
var result = new List<V1ServicePort>();
var containers = online.GetContainers();
foreach (var container in containers)
{
result.Add(new V1ServicePort
{
Name = container.ServicePortName,
Protocol = "TCP",
Port = container.ApiPort,
TargetPort = container.ContainerPortName,
NodePort = container.ServicePort
});
}
return result;
}
private void DeleteService(CodexNodeGroup online)
{
if (online.Service == null) return;
client.DeleteNamespacedService(online.Service.Name(), K8sNamespace);
online.Service = null;
}
private void CreatePrometheusService(K8sPrometheusSpecs spec)
{
client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace);
}
private void CreateGethBootstrapService(K8sGethBoostrapSpecs spec)
{
client.CreateNamespacedService(spec.CreateGethBootstrapService(), K8sNamespace);
}
#endregion
#region Deployment management
private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline)
{
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = online.GetDeploymentMetadata(),
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = online.GetSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = online.GetSelector()
},
Spec = new V1PodSpec
{
NodeSelector = CreateNodeSelector(offline),
Containers = CreateDeploymentContainers(online, offline)
}
}
}
};
online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
}
private IDictionary<string, string> CreateNodeSelector(OfflineCodexNodes offline)
{
if (offline.Location == Location.Unspecified) return new Dictionary<string, string>();
return new Dictionary<string, string>
{
{ "codex-test-location", k8sCluster.GetNodeLabelForLocation(offline.Location) }
};
}
private List<V1Container> CreateDeploymentContainers(CodexNodeGroup group, OfflineCodexNodes offline)
{
var result = new List<V1Container>();
var containers = group.GetContainers();
foreach (var container in containers)
{
result.Add(new V1Container
{
Name = container.Name,
Image = dockerImage.GetImageTag(),
Ports = new List<V1ContainerPort>
{
new V1ContainerPort
{
ContainerPort = container.ApiPort,
Name = container.ContainerPortName
}
},
Env = dockerImage.CreateEnvironmentVariables(offline, container)
});
}
return result;
}
private void DeleteDeployment(CodexNodeGroup group)
{
if (group.Deployment == null) return;
client.DeleteNamespacedDeployment(group.Deployment.Name(), K8sNamespace);
group.Deployment = null;
}
private void CreatePrometheusDeployment(K8sPrometheusSpecs spec)
{
client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace);
}
private void CreateGethBootstrapDeployment(K8sGethBoostrapSpecs spec)
{
client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace);
}
private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group)
{
client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace);
}
#endregion
#region Namespace management
private void EnsureTestNamespace()
{
if (IsTestNamespaceOnline()) return;
var namespaceSpec = new V1Namespace
{
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = K8sNamespace,
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
}
};
client.CreateNamespace(namespaceSpec);
}
private void DeleteNamespace()
{
if (IsTestNamespaceOnline())
{
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
}
}
private string K8sNamespace
{
get { return K8sCluster.K8sNamespace; }
}
#endregion
private bool IsTestNamespaceOnline()
{
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
}
private class CommandRunner
{
private readonly Kubernetes client;
private readonly PodInfo pod;
private readonly string containerName;
private readonly string command;
private readonly string[] arguments;
private readonly List<string> lines = new List<string>();
public CommandRunner(Kubernetes client, PodInfo pod, string containerName, string command, string[] arguments)
{
this.client = client;
this.pod = pod;
this.containerName = containerName;
this.command = command;
this.arguments = arguments;
}
public void Run()
{
var input = new[] { command }.Concat(arguments).ToArray();
Utils.Wait(client.NamespacedPodExecAsync(
pod.Name, K8sCluster.K8sNamespace, containerName, input, false, Callback, new CancellationToken()));
}
public string GetStdOut()
{
return string.Join(Environment.NewLine, lines);
}
private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr)
{
using var streamReader = new StreamReader(stdOut);
var line = streamReader.ReadLine();
while (line != null)
{
lines.Add(line);
line = streamReader.ReadLine();
}
return Task.CompletedTask;
}
}
}
}

35
CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs
View File

@ -1,35 +0,0 @@
namespace CodexDistTestCore.Marketplace
{
public class GethCompanionGroup
{
public GethCompanionGroup(int number, GethCompanionNodeContainer[] containers)
{
Number = number;
Containers = containers;
}
public int Number { get; }
public GethCompanionNodeContainer[] Containers { get; }
public PodInfo? Pod { get; set; }
}
public class GethCompanionNodeContainer
{
public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName, int authRpcPort)
{
Name = name;
ApiPort = apiPort;
AuthRpcPort = authRpcPort;
RpcPort = rpcPort;
ContainerPortName = containerPortName;
}
public string Name { get; }
public int ApiPort { get; }
public int AuthRpcPort { get; }
public int RpcPort { get; }
public string ContainerPortName { get; }
public string Account { get; set; } = string.Empty;
}
}

209
CodexDistTestCore/Marketplace/K8sGethSpecs.cs
View File

@ -1,209 +0,0 @@
using CodexDistTestCore.Config;
using k8s.Models;
namespace CodexDistTestCore.Marketplace
{
public static class GethDockerImage
{
public const string Image = "thatbenbierens/geth-confenv:latest";
public const string AccountFilename = "account_string.txt";
public const string GenesisFilename = "genesis.json";
}
public class K8sGethBoostrapSpecs
{
public const string ContainerName = "dtest-gethb";
private const string portName = "gethb";
private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9";
public K8sGethBoostrapSpecs(int servicePort)
{
ServicePort = servicePort;
}
public int ServicePort { get; }
public string GetBootstrapDeploymentName()
{
return "test-gethb";
}
public string GetCompanionDeploymentName(GethCompanionGroup group)
{
return "test-geth" + group.Number;
}
public V1Deployment CreateGethBootstrapDeployment()
{
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = new V1ObjectMeta
{
Name = GetBootstrapDeploymentName(),
NamespaceProperty = K8sCluster.K8sNamespace
},
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = CreateBootstrapSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = CreateBootstrapSelector()
},
Spec = new V1PodSpec
{
Containers = new List<V1Container>
{
new V1Container
{
Name = ContainerName,
Image = GethDockerImage.Image,
Ports = new List<V1ContainerPort>
{
new V1ContainerPort
{
ContainerPort = 8545,
Name = portName
}
},
Env = new List<V1EnvVar>
{
new V1EnvVar
{
Name = "GETH_ARGS",
Value = ""
},
new V1EnvVar
{
Name = "GENESIS_JSON",
Value = genesisJsonBase64
},
new V1EnvVar
{
Name = "IS_BOOTSTRAP",
Value = "1"
}
}
}
}
}
}
}
};
return deploymentSpec;
}
public V1Service CreateGethBootstrapService()
{
var serviceSpec = new V1Service
{
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = "codex-gethb-service",
NamespaceProperty = K8sCluster.K8sNamespace
},
Spec = new V1ServiceSpec
{
Type = "NodePort",
Selector = CreateBootstrapSelector(),
Ports = new List<V1ServicePort>
{
new V1ServicePort
{
Name = "gethb-service",
Protocol = "TCP",
Port = 8545,
TargetPort = portName,
NodePort = ServicePort
}
}
}
};
return serviceSpec;
}
public V1Deployment CreateGethCompanionDeployment(GethCompanionGroup group, GethBootstrapInfo info)
{
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = new V1ObjectMeta
{
Name = GetCompanionDeploymentName(group),
NamespaceProperty = K8sCluster.K8sNamespace
},
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = CreateCompanionSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = CreateCompanionSelector()
},
Spec = new V1PodSpec
{
Containers = group.Containers.Select(c => CreateContainer(c, info)).ToList()
}
}
}
};
return deploymentSpec;
}
private static V1Container CreateContainer(GethCompanionNodeContainer container, GethBootstrapInfo info)
{
return new V1Container
{
Name = container.Name,
Image = GethDockerImage.Image,
Ports = new List<V1ContainerPort>
{
new V1ContainerPort
{
ContainerPort = container.ApiPort,
Name = container.ContainerPortName
}
},
// todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64
Env = new List<V1EnvVar>
{
new V1EnvVar
{
Name = "GETH_ARGS",
Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.AuthRpcPort} --http.port {container.RpcPort}"
},
new V1EnvVar
{
Name = "GENESIS_JSON",
Value = info.GenesisJsonBase64
}
}
};
}
private Dictionary<string, string> CreateBootstrapSelector()
{
return new Dictionary<string, string> { { "test-gethb", "dtest-gethb" } };
}
private Dictionary<string, string> CreateCompanionSelector()
{
return new Dictionary<string, string> { { "test-gethc", "dtest-gethc" } };
}
}
}

159
CodexDistTestCore/Marketplace/MarketplaceController.cs
View File

@ -1,159 +0,0 @@
using CodexDistTestCore.Config;
using Nethereum.Web3;
using Nethereum.Web3.Accounts;
using Nethereum.Web3.Accounts.Managed;
using NUnit.Framework;
using System.Numerics;
using System.Text;
namespace CodexDistTestCore.Marketplace
{
public class MarketplaceController
{
private readonly TestLog log;
private readonly K8sManager k8sManager;
private readonly NumberSource companionGroupNumberSource = new NumberSource(0);
private List<GethCompanionGroup> companionGroups = new List<GethCompanionGroup>();
private GethBootstrapInfo? bootstrapInfo;
public MarketplaceController(TestLog log, K8sManager k8sManager)
{
this.log = log;
this.k8sManager = k8sManager;
}
public GethCompanionGroup BringOnlineMarketplace(OfflineCodexNodes offline)
{
if (bootstrapInfo == null)
{
BringOnlineBootstrapNode();
}
log.Log($"Initializing companions for {offline.NumberOfNodes} Codex nodes.");
var group = new GethCompanionGroup(companionGroupNumberSource.GetNextNumber(), CreateCompanionContainers(offline));
group.Pod = k8sManager.BringOnlineGethCompanionGroup(bootstrapInfo!, group);
companionGroups.Add(group);
log.Log("Initialized companion nodes.");
return group;
}
private void BringOnlineBootstrapNode()
{
log.Log("Starting Geth bootstrap node...");
var spec = k8sManager.CreateGethBootstrapNodeSpec();
var pod = k8sManager.BringOnlineGethBootstrapNode(spec);
var (account, genesisJson) = ExtractAccountAndGenesisJson(pod);
bootstrapInfo = new GethBootstrapInfo(spec, pod, account, genesisJson);
log.Log($"Geth boothstrap node started.");
}
private GethCompanionNodeContainer[] CreateCompanionContainers(OfflineCodexNodes offline)
{
var numberSource = new NumberSource(8080);
var result = new List<GethCompanionNodeContainer>();
for (var i = 0; i < offline.NumberOfNodes; i++) result.Add(CreateGethNodeContainer(numberSource, i));
return result.ToArray();
}
private GethCompanionNodeContainer CreateGethNodeContainer(NumberSource numberSource, int n)
{
return new GethCompanionNodeContainer(
name: $"geth-node{n}",
apiPort: numberSource.GetNextNumber(),
authRpcPort: numberSource.GetNextNumber(),
rpcPort: numberSource.GetNextNumber(),
containerPortName: $"geth-{n}"
);
}
private readonly K8sCluster k8sCluster = new K8sCluster();
public void AddToBalance(string account, decimal amount)
{
if (amount < 1 || string.IsNullOrEmpty(account)) Assert.Fail("Invalid arguments for AddToBalance");
// call the bootstrap node and convince it to give 'account' 'amount' tokens somehow.
var web3 = CreateWeb3();
//var blockNumber1 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
//Thread.Sleep(TimeSpan.FromSeconds(5));
//var blockNumber2 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
//var bootstrapBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(bootstrapInfo.Account));
var bigint = new BigInteger(amount);
var str = bigint.ToString("X");
var value = new Nethereum.Hex.HexTypes.HexBigInteger(str);
var aaa = Utils.Wait(web3.Eth.TransactionManager.SendTransactionAsync(bootstrapInfo!.Account, account, value));
var receipt = Utils.Wait(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(aaa));
//var receipt = Utils.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(account, amount));
//var targetBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account));
}
public decimal GetBalance(string account)
{
var web3 = CreateWeb3();
var bigInt = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account));
return (decimal)bigInt.Value;
}
private Web3 CreateWeb3()
{
var ip = k8sCluster.GetIp();
var port = bootstrapInfo!.Spec.ServicePort;
//var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^");
return new Web3($"http://{ip}:{port}");
}
private (string, string) ExtractAccountAndGenesisJson(PodInfo pod)
{
var (account, genesisJson) = FetchAccountAndGenesisJson(pod);
if (string.IsNullOrEmpty(account) || string.IsNullOrEmpty(genesisJson))
{
Thread.Sleep(TimeSpan.FromSeconds(15));
(account, genesisJson) = FetchAccountAndGenesisJson(pod);
}
Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure.");
Assert.That(genesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure.");
var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson));
log.Log($"Initialized geth bootstrap node with account '{account}'");
return (account, encoded);
}
private (string, string) FetchAccountAndGenesisJson(PodInfo pod)
{
var bootstrapAccount = ExecuteCommand(pod, "cat", GethDockerImage.AccountFilename);
var bootstrapGenesisJson = ExecuteCommand(pod, "cat", GethDockerImage.GenesisFilename);
return (bootstrapAccount, bootstrapGenesisJson);
}
private string ExecuteCommand(PodInfo pod, string command, params string[] arguments)
{
return k8sManager.ExecuteCommand(pod, K8sGethBoostrapSpecs.ContainerName, command, arguments);
}
}
public class GethBootstrapInfo
{
public GethBootstrapInfo(K8sGethBoostrapSpecs spec, PodInfo pod, string account, string genesisJsonBase64)
{
Spec = spec;
Pod = pod;
Account = account;
GenesisJsonBase64 = genesisJsonBase64;
}
public K8sGethBoostrapSpecs Spec { get; }
public PodInfo Pod { get; }
public string Account { get; }
public string GenesisJsonBase64 { get; }
}
}

122
CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs
View File

@ -1,122 +0,0 @@
using CodexDistTestCore.Config;
using k8s.Models;
namespace CodexDistTestCore.Metrics
{
public class K8sPrometheusSpecs
{
public const string ContainerName = "dtest-prom";
public const string ConfigFilepath = "/etc/prometheus/prometheus.yml";
private const string dockerImage = "thatbenbierens/prometheus-envconf:latest";
private const string portName = "prom-1";
private readonly string config;
public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config)
{
ServicePort = servicePort;
PrometheusNumber = prometheusNumber;
this.config = config;
}
public int ServicePort { get; }
public int PrometheusNumber { get; }
public string GetDeploymentName()
{
return "test-prom" + PrometheusNumber;
}
public V1Deployment CreatePrometheusDeployment()
{
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = new V1ObjectMeta
{
Name = GetDeploymentName(),
NamespaceProperty = K8sCluster.K8sNamespace
},
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = CreateSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = CreateSelector()
},
Spec = new V1PodSpec
{
Containers = new List<V1Container>
{
new V1Container
{
Name = ContainerName,
Image = dockerImage,
Ports = new List<V1ContainerPort>
{
new V1ContainerPort
{
ContainerPort = 9090,
Name = portName
}
},
Env = new List<V1EnvVar>
{
new V1EnvVar
{
Name = "PROM_CONFIG",
Value = config
}
}
}
}
}
}
}
};
return deploymentSpec;
}
public V1Service CreatePrometheusService()
{
var serviceSpec = new V1Service
{
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = "codex-prom-service" + PrometheusNumber,
NamespaceProperty = K8sCluster.K8sNamespace
},
Spec = new V1ServiceSpec
{
Type = "NodePort",
Selector = CreateSelector(),
Ports = new List<V1ServicePort>
{
new V1ServicePort
{
Name = "prom-service" + PrometheusNumber,
Protocol = "TCP",
Port = 9090,
TargetPort = portName,
NodePort = ServicePort
}
}
}
};
return serviceSpec;
}
private Dictionary<string, string> CreateSelector()
{
return new Dictionary<string, string> { { "test-prom", "dtest-prom" } };
}
}
}

78
CodexDistTestCore/Metrics/MetricsAggregator.cs
View File

@ -1,78 +0,0 @@
using NUnit.Framework;
using System.Text;
namespace CodexDistTestCore.Metrics
{
public class MetricsAggregator
{
private readonly NumberSource prometheusNumberSource = new NumberSource(0);
private readonly TestLog log;
private readonly K8sManager k8sManager;
private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses = new Dictionary<MetricsQuery, OnlineCodexNode[]>();
public MetricsAggregator(TestLog log, K8sManager k8sManager)
{
this.log = log;
this.k8sManager = k8sManager;
}
public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes)
{
log.Log($"Starting metrics collecting for {nodes.Length} nodes...");
var config = GeneratePrometheusConfig(nodes);
var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber());
var query = new MetricsQuery(prometheus);
activePrometheuses.Add(query, nodes);
log.Log("Metrics service started.");
foreach (var node in nodes)
{
node.Metrics = new MetricsAccess(query, node);
}
}
public void DownloadAllMetrics()
{
var download = new MetricsDownloader(log, activePrometheuses);
download.DownloadAllMetrics();
}
private string GeneratePrometheusConfig(OnlineCodexNode[] nodes)
{
var config = "";
config += "global:\n";
config += " scrape_interval: 30s\n";
config += " scrape_timeout: 10s\n";
config += "\n";
config += "scrape_configs:\n";
config += " - job_name: services\n";
config += " metrics_path: /metrics\n";
config += " static_configs:\n";
config += " - targets:\n";
foreach (var node in nodes)
{
var ip = node.Group.PodInfo!.Ip;
var port = node.Container.MetricsPort;
config += $" - '{ip}:{port}'\n";
}
var bytes = Encoding.ASCII.GetBytes(config);
return Convert.ToBase64String(bytes);
}
}
public class PrometheusInfo
{
public PrometheusInfo(int servicePort, PodInfo podInfo)
{
ServicePort = servicePort;
PodInfo = podInfo;
}
public int ServicePort { get; }
public PodInfo PodInfo { get; }
}
}

106
CodexDistTestCore/OfflineCodexNodes.cs
View File

@ -1,106 +0,0 @@
using CodexDistTestCore.Marketplace;
namespace CodexDistTestCore
{
public interface IOfflineCodexNodes
{
IOfflineCodexNodes At(Location location);
IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
IOfflineCodexNodes EnableMetrics();
IOfflineCodexNodes EnableMarketplace(int initialBalance);
ICodexNodeGroup BringOnline();
}
public enum CodexLogLevel
{
Trace,
Debug,
Info,
Warn,
Error
}
public enum Location
{
Unspecified,
BensLaptop,
BensOldGamingMachine,
}
public class OfflineCodexNodes : IOfflineCodexNodes
{
private readonly IK8sManager k8SManager;
public int NumberOfNodes { get; }
public Location Location { get; private set; }
public CodexLogLevel? LogLevel { get; private set; }
public IOnlineCodexNode? BootstrapNode { get; private set; }
public ByteSize? StorageQuota { get; private set; }
public bool MetricsEnabled { get; private set; }
public MarketplaceInitialConfig? MarketplaceConfig { get; private set; }
public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes)
{
this.k8SManager = k8SManager;
NumberOfNodes = numberOfNodes;
Location = Location.Unspecified;
MetricsEnabled = false;
}
public ICodexNodeGroup BringOnline()
{
return k8SManager.BringOnline(this);
}
public IOfflineCodexNodes At(Location location)
{
Location = location;
return this;
}
public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node)
{
BootstrapNode = node;
return this;
}
public IOfflineCodexNodes WithLogLevel(CodexLogLevel level)
{
LogLevel = level;
return this;
}
public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota)
{
StorageQuota = storageQuota;
return this;
}
public IOfflineCodexNodes EnableMetrics()
{
MetricsEnabled = true;
return this;
}
public IOfflineCodexNodes EnableMarketplace(int initialBalance)
{
MarketplaceConfig = new MarketplaceInitialConfig(initialBalance);
return this;
}
public string Describe()
{
var args = string.Join(',', DescribeArgs());
return $"{NumberOfNodes} CodexNodes with [{args}]";
}
private IEnumerable<string> DescribeArgs()
{
if (LogLevel != null) yield return ($"LogLevel={LogLevel}");
if (BootstrapNode != null) yield return ("BootstrapNode=set-not-shown-here");
if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}");
}
}
}

73
CodexDistTestCore/PodLogDownloader.cs
View File

@ -1,73 +0,0 @@
using NUnit.Framework;
namespace CodexDistTestCore
{
public interface IPodLogHandler
{
void Log(Stream log);
}
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
{
public DontDownloadLogsAndMetricsOnFailureAttribute()
: base(Timing.UseLongTimeoutsKey)
{
}
}
public class PodLogDownloader
{
public const string DontDownloadLogsOnFailureKey = "DontDownloadLogsOnFailure";
private readonly TestLog log;
private readonly IK8sManager k8SManager;
public PodLogDownloader(TestLog log, IK8sManager k8sManager)
{
this.log = log;
k8SManager = k8sManager;
}
public CodexNodeLog DownloadLog(OnlineCodexNode node)
{
var description = node.Describe();
var subFile = log.CreateSubfile();
log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}");
var handler = new PodLogDownloadHandler(description, subFile);
k8SManager.FetchPodLog(node, handler);
return handler.CreateCodexNodeLog();
}
}
public class PodLogDownloadHandler : IPodLogHandler
{
private readonly string description;
private readonly LogFile log;
public PodLogDownloadHandler(string description, LogFile log)
{
this.description = description;
this.log = log;
}
public CodexNodeLog CreateCodexNodeLog()
{
return new CodexNodeLog(log);
}
public void Log(Stream stream)
{
log.Write($"{description} -->> {log.FilenameWithoutPath}");
log.WriteRaw(description);
var reader = new StreamReader(stream);
var line = reader.ReadLine();
while (line != null)
{
log.WriteRaw(line);
line = reader.ReadLine();
}
}
}
}

144
CodexDistTestCore/TestLog.cs
View File

@ -1,144 +0,0 @@
using CodexDistTestCore.Config;
using NUnit.Framework;
namespace CodexDistTestCore
{
public class TestLog
{
private readonly NumberSource subfileNumberSource = new NumberSource(0);
private readonly LogFile file;
private readonly DateTime now;
public TestLog()
{
now = DateTime.UtcNow;
var name = GetTestName();
file = new LogFile(now, name);
Log($"Begin: {name}");
}
public void Log(string message)
{
file.Write(message);
}
public void Error(string message)
{
Log($"[ERROR] {message}");
}
public void EndTest()
{
var result = TestContext.CurrentContext.Result;
Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
if (!string.IsNullOrEmpty(result.Message))
{
Log(result.Message);
Log($"{result.StackTrace}");
}
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
RenameLogFile();
}
}
private void RenameLogFile()
{
file.ConcatToFilename("_FAILED");
}
public LogFile CreateSubfile(string ext = "log")
{
return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext);
}
private static string GetTestName()
{
var test = TestContext.CurrentContext.Test;
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
var args = FormatArguments(test);
return $"{className}.{test.MethodName}{args}";
}
private static string FormatArguments(TestContext.TestAdapter test)
{
if (test.Arguments == null || !test.Arguments.Any()) return "";
return $"[{string.Join(',', test.Arguments)}]";
}
}
public class LogFile
{
private readonly DateTime now;
private string name;
private readonly string ext;
private readonly string filepath;
public LogFile(DateTime now, string name, string ext = "log")
{
this.now = now;
this.name = name;
this.ext = ext;
filepath = Path.Join(
LogConfig.LogRoot,
$"{now.Year}-{Pad(now.Month)}",
Pad(now.Day));
Directory.CreateDirectory(filepath);
GenerateFilename();
}
public string FullFilename { get; private set; } = string.Empty;
public string FilenameWithoutPath { get; private set; } = string.Empty;
public void Write(string message)
{
WriteRaw($"{GetTimestamp()} {message}");
}
public void WriteRaw(string message)
{
try
{
File.AppendAllLines(FullFilename, new[] { message });
}
catch (Exception ex)
{
Console.WriteLine("Writing to log has failed: " + ex);
}
}
public void ConcatToFilename(string toAdd)
{
var oldFullName = FullFilename;
name += toAdd;
GenerateFilename();
File.Move(oldFullName, FullFilename);
}
private static string Pad(int n)
{
return n.ToString().PadLeft(2, '0');
}
private static string GetTimestamp()
{
return $"[{DateTime.UtcNow.ToString("u")}]";
}
private void GenerateFilename()
{
FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}";
FullFilename = Path.Combine(filepath, FilenameWithoutPath);
}
}
}

101
CodexDistTestCore/TryContract.cs
View File

@ -1,101 +0,0 @@
using Nethereum.Web3;
using Nethereum.ABI.FunctionEncoding.Attributes;
using Nethereum.Contracts.CQS;
using Nethereum.Util;
using Nethereum.Web3.Accounts;
using Nethereum.Hex.HexConvertors.Extensions;
using Nethereum.Contracts;
using Nethereum.Contracts.Extensions;
using System.Numerics;
using NUnit.Framework;
// https://docs.nethereum.com/en/latest/nethereum-smartcontrats-gettingstarted/
namespace CodexDistTestCore
{
public class TryContract
{
[Test]
[Ignore("aaa")]
public void DoThing()
{
var url = "http://testchain.nethereum.com:8545";
var privateKey = "0x7580e7fb49df1c861f0050fae31c2224c6aba908e116b8da44ee8cd927b990b0";
var account = new Account(privateKey);
var web3 = new Web3(account, url);
// Deploy contract:
var deploymentMessage = new StandardTokenDeployment
{
TotalSupply = 100000
};
var deploymentHandler = web3.Eth.GetContractDeploymentHandler<StandardTokenDeployment>();
var transactionReceipt = Utils.Wait(deploymentHandler.SendRequestAndWaitForReceiptAsync(deploymentMessage));
var contractAddress = transactionReceipt.ContractAddress;
// Get balance:
var balanceOfFunctionMessage = new BalanceOfFunction()
{
Owner = account.Address,
};
var balanceHandler = web3.Eth.GetContractQueryHandler<BalanceOfFunction>();
var balance = Utils.Wait(balanceHandler.QueryAsync<BigInteger>(contractAddress, balanceOfFunctionMessage));
long asInt = ((long)balance);
// Transfer:
var receiverAddress = "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe";
var transferHandler = web3.Eth.GetContractTransactionHandler<TransferFunction>();
var transfer = new TransferFunction()
{
To = receiverAddress,
TokenAmount = 100
};
var transferReceipt = Utils.Wait(transferHandler.SendRequestAndWaitForReceiptAsync(contractAddress, transfer));
// Signing:
var signedTransaction = Utils.Wait(transferHandler.SignTransactionAsync(contractAddress, transfer));
}
}
public class StandardTokenDeployment : ContractDeploymentMessage
{
public static string BYTECODE = "0x60606040526040516020806106f5833981016040528080519060200190919050505b80600160005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005081905550806000600050819055505b506106868061006f6000396000f360606040523615610074576000357c010000000000000000000000000000000000000000000000000000000090048063095ea7b31461008157806318160ddd146100b657806323b872dd146100d957806370a0823114610117578063a9059cbb14610143578063dd62ed3e1461017857610074565b61007f5b610002565b565b005b6100a060048080359060200190919080359060200190919050506101ad565b6040518082815260200191505060405180910390f35b6100c36004805050610674565b6040518082815260200191505060405180910390f35b6101016004808035906020019091908035906020019091908035906020019091905050610281565b6040518082815260200191505060405180910390f35b61012d600480803590602001909190505061048d565b6040518082815260200191505060405180910390f35b61016260048080359060200190919080359060200190919050506104cb565b6040518082815260200191505060405180910390f35b610197600480803590602001909190803590602001909190505061060b565b6040518082815260200191505060405180910390f35b600081600260005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060008573ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925846040518082815260200191505060405180910390a36001905061027b565b92915050565b600081600160005060008673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050541015801561031b575081600260005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060003373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000505410155b80156103275750600082115b1561047c5781600160005060008573ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505401925050819055508273ffffffffffffffffffffffffffffffffffffffff168473ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a381600160005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282825054039250508190555081600260005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060003373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505403925050819055506001905061048656610485565b60009050610486565b5b9392505050565b6000600160005060008373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000505490506104c6565b919050565b600081600160005060003373ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050541015801561050c5750600082115b156105fb5781600160005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282825054039250508190555081600160005060008573ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505401925050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a36001905061060556610604565b60009050610605565b5b92915050565b6000600260005060008473ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060008373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005054905061066e565b92915050565b60006000600050549050610683565b9056";
public StandardTokenDeployment() : base(BYTECODE) { }
[Parameter("uint256", "totalSupply")]
public BigInteger TotalSupply { get; set; }
}
[Function("balanceOf", "uint256")]
public class BalanceOfFunction : FunctionMessage
{
[Parameter("address", "_owner", 1)]
public string Owner { get; set; }
}
[Function("transfer", "bool")]
public class TransferFunction : FunctionMessage
{
[Parameter("address", "_to", 1)]
public string To { get; set; }
[Parameter("uint256", "_value", 2)]
public BigInteger TokenAmount { get; set; }
}
[Event("Transfer")]
public class TransferEventDTO : IEventDTO
{
[Parameter("address", "_from", 1, true)]
public string From { get; set; }
[Parameter("address", "_to", 2, true)]
public string To { get; set; }
[Parameter("uint256", "_value", 3, false)]
public BigInteger Value { get; set; }
}
}

16
CodexDistTestCore/Utils.cs
View File

@ -1,16 +0,0 @@
namespace CodexDistTestCore
{
public static class Utils
{
public static void Sleep(TimeSpan span)
{
Thread.Sleep(span);
}
public static T Wait<T>(Task<T> task)
{
task.Wait();
return task.Result;
}
}
}

2
CodexDistTestCore/ByteSize.cs → DistTestCore/ByteSize.cs
View File

@ -1,4 +1,4 @@
namespace CodexDistTestCore
namespace DistTestCore
{
public class ByteSize
{

56
DistTestCore/Codex/CodexAccess.cs Normal file
View File

@ -0,0 +1,56 @@
using KubernetesWorkflow;
namespace DistTestCore.Codex
{
public class CodexAccess
{
public CodexAccess(RunningContainer runningContainer)
{
Container = runningContainer;
}
public RunningContainer Container { get; }
public CodexDebugResponse GetDebugInfo()
{
return Http().HttpGetJson<CodexDebugResponse>("debug/info");
}
public string UploadFile(FileStream fileStream)
{
return Http().HttpPostStream("upload", fileStream);
}
public Stream DownloadFile(string contentId)
{
return Http().HttpGetStream("download/" + contentId);
}
private Http Http()
{
var ip = Container.Pod.Cluster.IP;
var port = Container.ServicePorts[0].Number;
return new Http(ip, port, baseUrl: "/api/codex/v1");
}
public string ConnectToPeer(string peerId, string peerMultiAddress)
{
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
}
}
public class CodexDebugResponse
{
public string id { get; set; } = string.Empty;
public string[] addrs { get; set; } = new string[0];
public string repo { get; set; } = string.Empty;
public string spr { get; set; } = string.Empty;
public CodexDebugVersionResponse codex { get; set; } = new();
}
public class CodexDebugVersionResponse
{
public string version { get; set; } = string.Empty;
public string revision { get; set; } = string.Empty;
}
}

54
DistTestCore/Codex/CodexContainerRecipe.cs Normal file
View File

@ -0,0 +1,54 @@
using DistTestCore.Marketplace;
using KubernetesWorkflow;
namespace DistTestCore.Codex
{
public class CodexContainerRecipe : ContainerRecipeFactory
{
public const string DockerImage = "thatbenbierens/nim-codex:sha-b204837";
public const string MetricsPortTag = "metrics_port";
protected override string Image => DockerImage;
protected override void Initialize(StartupConfig startupConfig)
{
var config = startupConfig.Get<CodexStartupConfig>();
AddExposedPortAndVar("API_PORT");
AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}");
AddInternalPortAndVar("DISC_PORT");
var listenPort = AddInternalPort();
AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
if (config.LogLevel != null)
{
AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
}
if (config.StorageQuota != null)
{
AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
}
if (config.MetricsEnabled)
{
AddEnvVar("METRICS_ADDR", "0.0.0.0");
AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag);
}
if (config.MarketplaceConfig != null)
{
var gethConfig = startupConfig.Get<GethStartResult>();
var companionNode = gethConfig.CompanionNodes[Index];
Additional(companionNode);
// Bootstrap node access from within the cluster:
//var ip = gethConfig.BootstrapNode.RunningContainers.RunningPod.Ip;
//var port = gethConfig.BootstrapNode.RunningContainers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag);
//AddEnvVar("ETH_PROVIDER", "todo");
//AddEnvVar("ETH_ACCOUNT", companionNode.Account);
//AddEnvVar("ETH_DEPLOYMENT", "todo");
}
}
}
}

11
DistTestCore/Codex/CodexLogLevel.cs Normal file
View File

@ -0,0 +1,11 @@
namespace DistTestCore.Codex
{
public enum CodexLogLevel
{
Trace,
Debug,
Info,
Warn,
Error
}
}

16
DistTestCore/Codex/CodexStartupConfig.cs Normal file
View File

@ -0,0 +1,16 @@
using DistTestCore.Marketplace;
using KubernetesWorkflow;
namespace DistTestCore.Codex
{
public class CodexStartupConfig
{
public Location Location { get; set; }
public CodexLogLevel? LogLevel { get; set; }
public ByteSize? StorageQuota { get; set; }
public bool MetricsEnabled { get; set; }
public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
//public IOnlineCodexNode? BootstrapNode { get; private set; }
}
}

32
DistTestCore/CodexNodeFactory.cs Normal file
View File

@ -0,0 +1,32 @@
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
namespace DistTestCore
{
public interface ICodexNodeFactory
{
OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
}
public class CodexNodeFactory : ICodexNodeFactory
{
private readonly TestLifecycle lifecycle;
private readonly IMetricsAccessFactory metricsAccessFactory;
private readonly IMarketplaceAccessFactory marketplaceAccessFactory;
public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory, IMarketplaceAccessFactory marketplaceAccessFactory)
{
this.lifecycle = lifecycle;
this.metricsAccessFactory = metricsAccessFactory;
this.marketplaceAccessFactory = marketplaceAccessFactory;
}
public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
{
var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container);
var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(access);
return new OnlineCodexNode(lifecycle, access, group, metricsAccess, marketplaceAccess);
}
}
}

86
DistTestCore/CodexNodeGroup.cs Normal file
View File

@ -0,0 +1,86 @@
using DistTestCore.Codex;
using KubernetesWorkflow;
using System.Collections;
namespace DistTestCore
{
public interface ICodexNodeGroup : IEnumerable<IOnlineCodexNode>
{
ICodexSetup BringOffline();
IOnlineCodexNode this[int index] { get; }
}
public class CodexNodeGroup : ICodexNodeGroup
{
private readonly TestLifecycle lifecycle;
public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
{
this.lifecycle = lifecycle;
Setup = setup;
Containers = containers;
Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
}
public IOnlineCodexNode this[int index]
{
get
{
return Nodes[index];
}
}
public ICodexSetup BringOffline()
{
lifecycle.CodexStarter.BringOffline(this);
var result = Setup;
// Clear everything. Prevent accidental use.
Setup = null!;
Nodes = Array.Empty<OnlineCodexNode>();
Containers = null!;
return result;
}
public CodexSetup Setup { get; private set; }
public RunningContainers Containers { get; private set; }
public OnlineCodexNode[] Nodes { get; private set; }
public IEnumerator<IOnlineCodexNode> GetEnumerator()
{
return Nodes.Cast<IOnlineCodexNode>().GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return Nodes.GetEnumerator();
}
public string Describe()
{
return $"<CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}>";
}
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
{
var access = new CodexAccess(c);
EnsureOnline(access);
return factory.CreateOnlineCodexNode(access, this);
}
private void EnsureOnline(CodexAccess access)
{
try
{
var debugInfo = access.GetDebugInfo();
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
}
catch (Exception e)
{
lifecycle.Log.Error($"Failed to start codex node: {e}");
throw;
}
}
}
}

84
DistTestCore/CodexSetup.cs Normal file
View File

@ -0,0 +1,84 @@
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using KubernetesWorkflow;
namespace DistTestCore
{
public interface ICodexSetup
{
ICodexSetup At(Location location);
ICodexSetup WithLogLevel(CodexLogLevel level);
//ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node);
ICodexSetup WithStorageQuota(ByteSize storageQuota);
ICodexSetup EnableMetrics();
ICodexSetup EnableMarketplace(int initialBalance);
ICodexNodeGroup BringOnline();
}
public class CodexSetup : CodexStartupConfig, ICodexSetup
{
private readonly CodexStarter starter;
public int NumberOfNodes { get; }
public CodexSetup(CodexStarter starter, int numberOfNodes)
{
this.starter = starter;
NumberOfNodes = numberOfNodes;
}
public ICodexNodeGroup BringOnline()
{
return starter.BringOnline(this);
}
public ICodexSetup At(Location location)
{
Location = location;
return this;
}
//public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node)
//{
// BootstrapNode = node;
// return this;
//}
public ICodexSetup WithLogLevel(CodexLogLevel level)
{
LogLevel = level;
return this;
}
public ICodexSetup WithStorageQuota(ByteSize storageQuota)
{
StorageQuota = storageQuota;
return this;
}
public ICodexSetup EnableMetrics()
{
MetricsEnabled = true;
return this;
}
public ICodexSetup EnableMarketplace(int initialBalance)
{
MarketplaceConfig = new MarketplaceInitialConfig(initialBalance);
return this;
}
public string Describe()
{
var args = string.Join(',', DescribeArgs());
return $"({NumberOfNodes} CodexNodes with [{args}])";
}
private IEnumerable<string> DescribeArgs()
{
if (LogLevel != null) yield return $"LogLevel={LogLevel}";
//if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here";
if (StorageQuota != null) yield return $"StorageQuote={StorageQuota.SizeInBytes}";
}
}
}

85
DistTestCore/CodexStarter.cs Normal file
View File

@ -0,0 +1,85 @@
using DistTestCore.Codex;
using KubernetesWorkflow;
namespace DistTestCore
{
public class CodexStarter
{
private readonly TestLifecycle lifecycle;
private readonly WorkflowCreator workflowCreator;
public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
{
this.lifecycle = lifecycle;
this.workflowCreator = workflowCreator;
}
public List<CodexNodeGroup> RunningGroups { get; } = new List<CodexNodeGroup>();
public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
{
Log($"Starting {codexSetup.Describe()}...");
var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
var startupConfig = new StartupConfig();
startupConfig.Add(codexSetup);
startupConfig.Add(gethStartResult);
var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers);
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
Log($"Started at '{group.Containers.RunningPod.Ip}'");
return group;
}
public void BringOffline(CodexNodeGroup group)
{
Log($"Stopping {group.Describe()}...");
var workflow = CreateWorkflow();
workflow.Stop(group.Containers);
RunningGroups.Remove(group);
Log("Stopped.");
}
public void DeleteAllResources()
{
var workflow = CreateWorkflow();
workflow.DeleteAllResources();
RunningGroups.Clear();
}
public void DownloadLog(RunningContainer container, ILogHandler logHandler)
{
var workflow = CreateWorkflow();
workflow.DownloadContainerLog(container, logHandler);
}
private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
{
var workflow = CreateWorkflow();
return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig);
}
private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
{
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
RunningGroups.Add(group);
return group;
}
private StartupWorkflow CreateWorkflow()
{
return workflowCreator.CreateWorkflow();
}
private void Log(string msg)
{
lifecycle.Log.Log(msg);
}
}
}

32
DistTestCore/Configuration.cs Normal file
View File

@ -0,0 +1,32 @@
using KubernetesWorkflow;
namespace DistTestCore
{
public class Configuration
{
public KubernetesWorkflow.Configuration GetK8sConfiguration()
{
return new KubernetesWorkflow.Configuration(
k8sNamespace: "codex-test-ns",
kubeConfigFile: null,
operationTimeout: Timing.K8sOperationTimeout(),
retryDelay: Timing.K8sServiceDelay(),
locationMap: new[]
{
new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
new ConfigurationLocationEntry(Location.BensLaptop, "worker02"),
}
);
}
public Logging.LogConfig GetLogConfig()
{
return new Logging.LogConfig("D:/CodexTestLogs");
}
public string GetFileManagerFolder()
{
return "TestDataFiles";
}
}
}

194
DistTestCore/DistTest.cs Normal file
View File

@ -0,0 +1,194 @@
using DistTestCore.Codex;
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using KubernetesWorkflow;
using Logging;
using NUnit.Framework;
using Utils;
namespace DistTestCore
{
[SetUpFixture]
public abstract class DistTest
{
private readonly Configuration configuration = new Configuration();
private FixtureLog fixtureLog = null!;
private TestLifecycle lifecycle = null!;
private DateTime testStart = DateTime.MinValue;
[OneTimeSetUp]
public void GlobalSetup()
{
// Previous test run may have been interrupted.
// Begin by cleaning everything up.
fixtureLog = new FixtureLog(configuration.GetLogConfig());
try
{
Stopwatch.Measure(fixtureLog, "Global setup", () =>
{
var wc = new WorkflowCreator(configuration.GetK8sConfiguration());
wc.CreateWorkflow().DeleteAllResources();
});
}
catch (Exception ex)
{
GlobalTestFailure.HasFailed = true;
Error($"Global setup cleanup failed with: {ex}");
throw;
}
fixtureLog.Log("Global setup cleanup successful");
fixtureLog.Log($"Codex image: {CodexContainerRecipe.DockerImage}");
fixtureLog.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}");
fixtureLog.Log($"Geth image: {GethContainerRecipe.DockerImage}");
}
[SetUp]
public void SetUpDistTest()
{
if (GlobalTestFailure.HasFailed)
{
Assert.Inconclusive("Skip test: Previous test failed during clean up.");
}
else
{
CreateNewTestLifecycle();
}
}
[TearDown]
public void TearDownDistTest()
{
try
{
DisposeTestLifecycle();
}
catch (Exception ex)
{
Error("Cleanup failed: " + ex.Message);
GlobalTestFailure.HasFailed = true;
}
}
public TestFile GenerateTestFile(ByteSize size)
{
return lifecycle.FileManager.GenerateTestFile(size);
}
public ICodexSetup SetupCodexNodes(int numberOfNodes)
{
return new CodexSetup(lifecycle.CodexStarter, numberOfNodes);
}
private void IncludeLogsAndMetricsOnTestFailure()
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
fixtureLog.MarkAsFailed();
if (IsDownloadingLogsAndMetricsEnabled())
{
Log("Downloading all CodexNode logs and metrics because of test failure...");
DownloadAllLogs();
DownloadAllMetrics();
}
else
{
Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
}
}
}
private void Log(string msg)
{
lifecycle.Log.Log(msg);
}
private void Error(string msg)
{
lifecycle.Log.Error(msg);
}
private void CreateNewTestLifecycle()
{
Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () =>
{
lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
testStart = DateTime.UtcNow;
});
}
private void DisposeTestLifecycle()
{
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})");
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
IncludeLogsAndMetricsOnTestFailure();
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
private string GetTestDuration()
{
var testDuration = DateTime.UtcNow - testStart;
return Time.FormatDuration(testDuration);
}
private void DownloadAllLogs()
{
OnEachCodexNode(node =>
{
lifecycle.DownloadLog(node);
});
}
private void DownloadAllMetrics()
{
var metricsDownloader = new MetricsDownloader(lifecycle.Log);
OnEachCodexNode(node =>
{
var m = node.Metrics as MetricsAccess;
if (m != null)
{
metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m);
}
});
}
private void OnEachCodexNode(Action<OnlineCodexNode> action)
{
var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
foreach (var node in allNodes)
{
action(node);
}
}
private string GetCurrentTestName()
{
return $"[{TestContext.CurrentContext.Test.Name}]";
}
private string GetTestResult()
{
return TestContext.CurrentContext.Result.Outcome.Status.ToString();
}
private bool IsDownloadingLogsAndMetricsEnabled()
{
var testProperties = TestContext.CurrentContext.Test.Properties;
return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey);
}
}
public static class GlobalTestFailure
{
public static bool HasFailed { get; set; } = false;
}
}

21
DistTestCore/DistTestCore.csproj Normal file
View File

@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<RootNamespace>DistTestCore</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="nunit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
<ProjectReference Include="..\Logging\Logging.csproj" />
<ProjectReference Include="..\Nethereum\NethereumWorkflow.csproj" />
</ItemGroup>
</Project>

28
CodexDistTestCore/FileManager.cs → DistTestCore/FileManager.cs
View File

@ -1,7 +1,7 @@
using CodexDistTestCore.Config;
using Logging;
using NUnit.Framework;
namespace CodexDistTestCore
namespace DistTestCore
{
public interface IFileManager
{
@ -14,20 +14,21 @@ namespace CodexDistTestCore
{
public const int ChunkSize = 1024 * 1024;
private readonly Random random = new Random();
private readonly List<TestFile> activeFiles = new List<TestFile>();
private readonly TestLog log;
private readonly string folder;
public FileManager(TestLog log)
public FileManager(TestLog log, Configuration configuration)
{
if (!Directory.Exists(FileManagerConfig.Folder)) Directory.CreateDirectory(FileManagerConfig.Folder);
folder = configuration.GetFileManagerFolder();
EnsureDirectory();
this.log = log;
}
public TestFile CreateEmptyTestFile()
{
var result = new TestFile(Path.Combine(FileManagerConfig.Folder, Guid.NewGuid().ToString() + "_test.bin"));
var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"));
File.Create(result.Filename).Close();
activeFiles.Add(result);
return result;
}
@ -41,8 +42,7 @@ namespace CodexDistTestCore
public void DeleteAllTestFiles()
{
foreach (var file in activeFiles) File.Delete(file.Filename);
activeFiles.Clear();
DeleteDirectory();
}
private void GenerateFileBytes(TestFile result, ByteSize size)
@ -63,6 +63,16 @@ namespace CodexDistTestCore
using var stream = new FileStream(result.Filename, FileMode.Append);
stream.Write(bytes, 0, bytes.Length);
}
private void EnsureDirectory()
{
if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
}
private void DeleteDirectory()
{
Directory.Delete(folder, true);
}
}
public class TestFile

80
DistTestCore/GethStarter.cs Normal file
View File

@ -0,0 +1,80 @@
using DistTestCore.Marketplace;
using KubernetesWorkflow;
namespace DistTestCore
{
public class GethStarter
{
private readonly GethBootstrapNodeCache bootstrapNodeCache;
private readonly GethCompanionNodeStarter companionNodeStarter;
private readonly TestLifecycle lifecycle;
public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
{
bootstrapNodeCache = new GethBootstrapNodeCache(new GethBootstrapNodeStarter(lifecycle, workflowCreator));
companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator);
this.lifecycle = lifecycle;
}
public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)
{
if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult();
var bootstrapNode = bootstrapNodeCache.Get();
var companionNodes = StartCompanionNodes(codexSetup, bootstrapNode);
TransferInitialBalance(bootstrapNode, codexSetup.MarketplaceConfig.InitialBalance, companionNodes);
return CreateGethStartResult(bootstrapNode, companionNodes);
}
private void TransferInitialBalance(GethBootstrapNodeInfo bootstrapNode, int initialBalance, GethCompanionNodeInfo[] companionNodes)
{
var interaction = bootstrapNode.StartInteraction(lifecycle.Log);
foreach (var node in companionNodes)
{
interaction.TransferTo(node.Account, initialBalance);
}
}
private GethStartResult CreateGethStartResult(GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes)
{
return new GethStartResult(CreateMarketplaceAccessFactory(bootstrapNode), bootstrapNode, companionNodes);
}
private GethStartResult CreateMarketplaceUnavailableResult()
{
return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty<GethCompanionNodeInfo>());
}
private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(GethBootstrapNodeInfo bootstrapNode)
{
return new GethMarketplaceAccessFactory(lifecycle.Log, bootstrapNode!);
}
private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode)
{
return companionNodeStarter.StartCompanionNodesFor(codexSetup, bootstrapNode);
}
}
public class GethBootstrapNodeCache
{
private readonly GethBootstrapNodeStarter bootstrapNodeStarter;
private GethBootstrapNodeInfo? bootstrapNode;
public GethBootstrapNodeCache(GethBootstrapNodeStarter bootstrapNodeStarter)
{
this.bootstrapNodeStarter = bootstrapNodeStarter;
}
public GethBootstrapNodeInfo Get()
{
if (bootstrapNode == null)
{
bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode();
}
return bootstrapNode;
}
}
}

13
CodexDistTestCore/Http.cs → DistTestCore/Http.cs
View File

@ -1,8 +1,9 @@
using Newtonsoft.Json;
using NUnit.Framework;
using System.Net.Http.Headers;
using Utils;
namespace CodexDistTestCore
namespace DistTestCore
{
public class Http
{
@ -26,8 +27,8 @@ namespace CodexDistTestCore
{
using var client = GetClient();
var url = GetUrl() + route;
var result = Utils.Wait(client.GetAsync(url));
return Utils.Wait(result.Content.ReadAsStringAsync());
var result = Time.Wait(client.GetAsync(url));
return Time.Wait(result.Content.ReadAsStringAsync());
});
}
@ -45,9 +46,9 @@ namespace CodexDistTestCore
var content = new StreamContent(stream);
content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
var response = Utils.Wait(client.PostAsync(url, content));
var response = Time.Wait(client.PostAsync(url, content));
return Utils.Wait(response.Content.ReadAsStringAsync());
return Time.Wait(response.Content.ReadAsStringAsync());
});
}
@ -58,7 +59,7 @@ namespace CodexDistTestCore
var client = GetClient();
var url = GetUrl() + route;
return Utils.Wait(client.GetStreamAsync(url));
return Time.Wait(client.GetStreamAsync(url));
});
}

7
CodexDistTestCore/CodexNodeLog.cs → DistTestCore/Logs/CodexNodeLog.cs
View File

@ -1,6 +1,7 @@
using NUnit.Framework;
using Logging;
using NUnit.Framework;
namespace CodexDistTestCore
namespace DistTestCore.Logs
{
public interface ICodexNodeLog
{
@ -28,7 +29,7 @@ namespace CodexDistTestCore
line = streamReader.ReadLine();
}
Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}");
Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
}
}
}

15
DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs Normal file
View File

@ -0,0 +1,15 @@
using NUnit.Framework;
namespace DistTestCore.Logs
{
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
{
public const string DontDownloadKey = "DontDownloadLogsAndMetrics";
public DontDownloadLogsAndMetricsOnFailureAttribute()
: base(DontDownloadKey)
{
}
}
}

28
DistTestCore/Logs/LogDownloadHandler.cs Normal file
View File

@ -0,0 +1,28 @@
using KubernetesWorkflow;
using Logging;
namespace DistTestCore.Logs
{
public class LogDownloadHandler : LogHandler, ILogHandler
{
private readonly LogFile log;
public LogDownloadHandler(string description, LogFile log)
{
this.log = log;
log.Write($"{description} -->> {log.FullFilename}");
log.WriteRaw(description);
}
public CodexNodeLog CreateCodexNodeLog()
{
return new CodexNodeLog(log);
}
protected override void ProcessLine(string line)
{
log.WriteRaw(line);
}
}
}

33
DistTestCore/Marketplace/GethBootstrapNodeInfo.cs Normal file
View File

@ -0,0 +1,33 @@
using KubernetesWorkflow;
using Logging;
using NethereumWorkflow;
namespace DistTestCore.Marketplace
{
public class GethBootstrapNodeInfo
{
public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64, string pubKey, Port discoveryPort)
{
RunningContainers = runningContainers;
Account = account;
GenesisJsonBase64 = genesisJsonBase64;
PubKey = pubKey;
DiscoveryPort = discoveryPort;
}
public RunningContainers RunningContainers { get; }
public string Account { get; }
public string GenesisJsonBase64 { get; }
public string PubKey { get; }
public Port DiscoveryPort { get; }
public NethereumInteraction StartInteraction(TestLog log)
{
var ip = RunningContainers.RunningPod.Cluster.IP;
var port = RunningContainers.Containers[0].ServicePorts[0].Number;
var creator = new NethereumInteractionCreator(log, ip, port, Account);
return creator.CreateWorkflow();
}
}
}

49
DistTestCore/Marketplace/GethBootstrapNodeStarter.cs Normal file
View File

@ -0,0 +1,49 @@
using KubernetesWorkflow;
namespace DistTestCore.Marketplace
{
public class GethBootstrapNodeStarter
{
private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9";
private readonly TestLifecycle lifecycle;
private readonly WorkflowCreator workflowCreator;
public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
{
this.lifecycle = lifecycle;
this.workflowCreator = workflowCreator;
}
public GethBootstrapNodeInfo StartGethBootstrapNode()
{
Log("Starting Geth bootstrap node...");
var startupConfig = CreateBootstrapStartupConfig();
var workflow = workflowCreator.CreateWorkflow();
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
var extractor = new GethInfoExtractor(workflow, containers.Containers[0]);
var account = extractor.ExtractAccount();
var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64();
var pubKey = extractor.ExtractPubKey();
var discoveryPort = containers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
Log($"Geth bootstrap node started with account '{account}'");
return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, discoveryPort);
}
private StartupConfig CreateBootstrapStartupConfig()
{
var config = new StartupConfig();
config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64, null!));
return config;
}
private void Log(string msg)
{
lifecycle.Log.Log(msg);
}
}
}

16
DistTestCore/Marketplace/GethCompanionNodeInfo.cs Normal file
View File

@ -0,0 +1,16 @@
using KubernetesWorkflow;
namespace DistTestCore.Marketplace
{
public class GethCompanionNodeInfo
{
public GethCompanionNodeInfo(RunningContainer runningContainer, string account)
{
RunningContainer = runningContainer;
Account = account;
}
public RunningContainer RunningContainer { get; }
public string Account { get; }
}
}

50
DistTestCore/Marketplace/GethCompanionNodeStarter.cs Normal file
View File

@ -0,0 +1,50 @@
using KubernetesWorkflow;
namespace DistTestCore.Marketplace
{
public class GethCompanionNodeStarter
{
private readonly TestLifecycle lifecycle;
private readonly WorkflowCreator workflowCreator;
public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
{
this.lifecycle = lifecycle;
this.workflowCreator = workflowCreator;
}
public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode)
{
Log($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes.");
var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode);
var workflow = workflowCreator.CreateWorkflow();
var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig);
if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure.");
Log("Initialized companion nodes.");
return containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray();
}
private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container)
{
var extractor = new GethInfoExtractor(workflow, container);
var account = extractor.ExtractAccount();
return new GethCompanionNodeInfo(container, account);
}
private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode)
{
var config = new StartupConfig();
config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64, bootstrapNode));
return config;
}
private void Log(string msg)
{
lifecycle.Log.Log(msg);
}
}
}

49
DistTestCore/Marketplace/GethContainerRecipe.cs Normal file
View File

@ -0,0 +1,49 @@
using KubernetesWorkflow;
namespace DistTestCore.Marketplace
{
public class GethContainerRecipe : ContainerRecipeFactory
{
public const string DockerImage = "thatbenbierens/geth-confenv:latest";
public const string HttpPortTag = "http_port";
public const string DiscoveryPortTag = "disc_port";
public const string AccountFilename = "account_string.txt";
public const string GenesisFilename = "genesis.json";
protected override string Image => DockerImage;
protected override void Initialize(StartupConfig startupConfig)
{
var config = startupConfig.Get<GethStartupConfig>();
var args = CreateArgs(config);
AddEnvVar("GETH_ARGS", args);
AddEnvVar("GENESIS_JSON", config.GenesisJsonBase64);
}
private string CreateArgs(GethStartupConfig config)
{
var discovery = AddInternalPort(tag: DiscoveryPortTag);
if (config.IsBootstrapNode)
{
AddEnvVar("IS_BOOTSTRAP", "1");
var exposedPort = AddExposedPort();
return $"--http.port {exposedPort.Number} --discovery.port {discovery.Number} --nodiscover";
}
var port = AddInternalPort();
var authRpc = AddInternalPort();
var httpPort = AddInternalPort(tag: HttpPortTag);
var bootPubKey = config.BootstrapNode.PubKey;
var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;
var bootPort = config.BootstrapNode.DiscoveryPort.Number;
var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}";
// geth --bootnodes enode://pubkey1@ip1:port1
return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --nodiscover {bootstrapArg}";
}
}
}

110
DistTestCore/Marketplace/GethInfoExtractor.cs Normal file
View File

@ -0,0 +1,110 @@
using KubernetesWorkflow;
using System.Text;
namespace DistTestCore.Marketplace
{
public class GethInfoExtractor
{
private readonly StartupWorkflow workflow;
private readonly RunningContainer container;
public GethInfoExtractor(StartupWorkflow workflow, RunningContainer container)
{
this.workflow = workflow;
this.container = container;
}
public string ExtractAccount()
{
var account = Retry(FetchAccount);
if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
return account;
}
public string ExtractGenesisJsonBase64()
{
var genesisJson = Retry(FetchGenesisJson);
if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure.");
return Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson));
}
public string ExtractPubKey()
{
var pubKey = Retry(FetchPubKey);
if (string.IsNullOrEmpty(pubKey)) throw new InvalidOperationException("Unable to fetch enode from geth node. Test infra failure.");
return pubKey;
}
private string Retry(Func<string> fetch)
{
var result = Catch(fetch);
if (string.IsNullOrEmpty(result))
{
Thread.Sleep(TimeSpan.FromSeconds(5));
result = fetch();
}
return result;
}
private string Catch(Func<string> fetch)
{
try
{
return fetch();
}
catch
{
return string.Empty;
}
}
private string FetchGenesisJson()
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GenesisFilename);
}
private string FetchAccount()
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename);
}
private string FetchPubKey()
{
var enodeFinder = new PubKeyFinder();
workflow.DownloadContainerLog(container, enodeFinder);
return enodeFinder.GetPubKey();
}
}
public class PubKeyFinder : LogHandler, ILogHandler
{
private const string openTag = "self=\"enode://";
private string pubKey = string.Empty;
public string GetPubKey()
{
return pubKey;
}
protected override void ProcessLine(string line)
{
if (line.Contains(openTag))
{
ExtractPubKey(line);
}
}
private void ExtractPubKey(string line)
{
var openIndex = line.IndexOf(openTag) + openTag.Length;
var closeIndex = line.IndexOf("@");
pubKey = line.Substring(
startIndex: openIndex,
length: closeIndex - openIndex);
}
}
}

16
DistTestCore/Marketplace/GethStartResult.cs Normal file
View File

@ -0,0 +1,16 @@
namespace DistTestCore.Marketplace
{
public class GethStartResult
{
public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes)
{
MarketplaceAccessFactory = marketplaceAccessFactory;
BootstrapNode = bootstrapNode;
CompanionNodes = companionNodes;
}
public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
public GethBootstrapNodeInfo BootstrapNode { get; }
public GethCompanionNodeInfo[] CompanionNodes { get; }
}
}

16
DistTestCore/Marketplace/GethStartupConfig.cs Normal file
View File

@ -0,0 +1,16 @@
namespace DistTestCore.Marketplace
{
public class GethStartupConfig
{
public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64, GethBootstrapNodeInfo bootstrapNode)
{
IsBootstrapNode = isBootstrapNode;
GenesisJsonBase64 = genesisJsonBase64;
BootstrapNode = bootstrapNode;
}
public bool IsBootstrapNode { get; }
public string GenesisJsonBase64 { get; }
public GethBootstrapNodeInfo BootstrapNode { get; }
}
}

54
CodexDistTestCore/Marketplace/MarketplaceAccess.cs → DistTestCore/Marketplace/MarketplaceAccess.cs
View File

@ -1,7 +1,8 @@
using NUnit.Framework;
using Logging;
using NUnit.Framework;
using NUnit.Framework.Constraints;
namespace CodexDistTestCore.Marketplace
namespace DistTestCore.Marketplace
{
public interface IMarketplaceAccess
{
@ -13,33 +14,15 @@ namespace CodexDistTestCore.Marketplace
public class MarketplaceAccess : IMarketplaceAccess
{
private readonly K8sManager k8sManager;
private readonly MarketplaceController marketplaceController;
private readonly TestLog log;
private readonly CodexNodeGroup group;
private readonly GethCompanionNodeContainer container;
private readonly GethBootstrapNodeInfo bootstrapNode;
private readonly GethCompanionNodeInfo companionNode;
public MarketplaceAccess(
K8sManager k8sManager,
MarketplaceController marketplaceController,
TestLog log,
CodexNodeGroup group,
GethCompanionNodeContainer container)
public MarketplaceAccess(TestLog log, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo companionNode)
{
this.k8sManager = k8sManager;
this.marketplaceController = marketplaceController;
this.log = log;
this.group = group;
this.container = container;
}
public void Initialize()
{
EnsureAccount();
marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance);
log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}");
this.bootstrapNode = bootstrapNode;
this.companionNode = companionNode;
}
public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
@ -54,28 +37,13 @@ namespace CodexDistTestCore.Marketplace
public void AssertThatBalance(IResolveConstraint constraint, string message = "")
{
throw new NotImplementedException();
Assert.That(GetBalance(), constraint, message);
}
public decimal GetBalance()
{
return marketplaceController.GetBalance(container.Account);
}
private void EnsureAccount()
{
FetchAccount();
if (string.IsNullOrEmpty(container.Account))
{
Thread.Sleep(TimeSpan.FromSeconds(15));
FetchAccount();
}
Assert.That(container.Account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure.");
}
private void FetchAccount()
{
container.Account = k8sManager.ExecuteCommand(group.GethCompanionGroup!.Pod!, container.Name, "cat", GethDockerImage.AccountFilename);
var interaction = bootstrapNode.StartInteraction(log);
return interaction.GetBalance(companionNode.Account);
}
}

42
DistTestCore/Marketplace/MarketplaceAccessFactory.cs Normal file
View File

@ -0,0 +1,42 @@
using DistTestCore.Codex;
using Logging;
namespace DistTestCore.Marketplace
{
public interface IMarketplaceAccessFactory
{
IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access);
}
public class MarketplaceUnavailableAccessFactory : IMarketplaceAccessFactory
{
public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
{
return new MarketplaceUnavailable();
}
}
public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory
{
private readonly TestLog log;
private readonly GethBootstrapNodeInfo bootstrapNode;
public GethMarketplaceAccessFactory(TestLog log, GethBootstrapNodeInfo bootstrapNode)
{
this.log = log;
this.bootstrapNode = bootstrapNode;
}
public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
{
var companionNode = GetGethCompanionNode(access);
return new MarketplaceAccess(log, bootstrapNode, companionNode);
}
private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access)
{
var node = access.Container.Recipe.Additionals.Single(a => a is GethCompanionNodeInfo);
return (GethCompanionNodeInfo)node;
}
}
}

2
CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs → DistTestCore/Marketplace/MarketplaceInitialConfig.cs
View File

@ -1,4 +1,4 @@
namespace CodexDistTestCore.Marketplace
namespace DistTestCore.Marketplace
{
public class MarketplaceInitialConfig
{

25
CodexDistTestCore/Metrics/MetricsAccess.cs → DistTestCore/Metrics/MetricsAccess.cs
View File

@ -1,7 +1,9 @@
using NUnit.Framework;
using KubernetesWorkflow;
using NUnit.Framework;
using NUnit.Framework.Constraints;
using Utils;
namespace CodexDistTestCore.Metrics
namespace DistTestCore.Metrics
{
public interface IMetricsAccess
{
@ -20,9 +22,9 @@ namespace CodexDistTestCore.Metrics
public class MetricsAccess : IMetricsAccess
{
private readonly MetricsQuery query;
private readonly OnlineCodexNode node;
private readonly RunningContainer node;
public MetricsAccess(MetricsQuery query, OnlineCodexNode node)
public MetricsAccess(MetricsQuery query, RunningContainer node)
{
this.query = query;
this.node = node;
@ -30,18 +32,23 @@ namespace CodexDistTestCore.Metrics
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
{
var metricSet = GetMetricWithTimeout(metricName, node);
var metricSet = GetMetricWithTimeout(metricName);
var metricValue = metricSet.Values[0].Value;
Assert.That(metricValue, constraint, message);
}
private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node)
public Metrics? GetAllMetrics()
{
return query.GetAllMetricsForNode(node);
}
private MetricsSet GetMetricWithTimeout(string metricName)
{
var start = DateTime.UtcNow;
while (true)
{
var mostRecent = GetMostRecent(metricName, node);
var mostRecent = GetMostRecent(metricName);
if (mostRecent != null) return mostRecent;
if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
{
@ -49,11 +56,11 @@ namespace CodexDistTestCore.Metrics
throw new TimeoutException();
}
Utils.Sleep(TimeSpan.FromSeconds(2));
Time.Sleep(TimeSpan.FromSeconds(2));
}
}
private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node)
private MetricsSet? GetMostRecent(string metricName)
{
var result = query.GetMostRecent(metricName, node);
if (result == null) return null;

33
DistTestCore/Metrics/MetricsAccessFactory.cs Normal file
View File

@ -0,0 +1,33 @@
using KubernetesWorkflow;
namespace DistTestCore.Metrics
{
public interface IMetricsAccessFactory
{
IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer);
}
public class MetricsUnavailableAccessFactory : IMetricsAccessFactory
{
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
{
return new MetricsUnavailable();
}
}
public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
{
private readonly RunningContainers prometheusContainer;
public CodexNodeMetricsAccessFactory(RunningContainers prometheusContainer)
{
this.prometheusContainer = prometheusContainer;
}
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
{
var query = new MetricsQuery(prometheusContainer);
return new MetricsAccess(query, codexContainer);
}
}
}

33
CodexDistTestCore/Metrics/MetricsDownloader.cs → DistTestCore/Metrics/MetricsDownloader.cs
View File

@ -1,49 +1,32 @@
using System.Globalization;
using Logging;
using System.Globalization;
namespace CodexDistTestCore.Metrics
namespace DistTestCore.Metrics
{
public class MetricsDownloader
{
private readonly TestLog log;
private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses;
public MetricsDownloader(TestLog log, Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses)
public MetricsDownloader(TestLog log)
{
this.log = log;
this.activePrometheuses = activePrometheuses;
}
public void DownloadAllMetrics()
public void DownloadAllMetricsForNode(string nodeName, MetricsAccess access)
{
foreach (var pair in activePrometheuses)
{
DownloadAllMetrics(pair.Key, pair.Value);
}
}
private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes)
{
foreach (var node in nodes)
{
DownloadAllMetricsForNode(query, node);
}
}
private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node)
{
var metrics = query.GetAllMetricsForNode(node);
var metrics = access.GetAllMetrics();
if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return;
var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
var map = CreateValueMap(metrics);
WriteToFile(node.GetName(), headers, map);
WriteToFile(nodeName, headers, map);
}
private void WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
{
var file = log.CreateSubfile("csv");
log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}");
log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}");
file.WriteRaw(string.Join(",", headers));

29
CodexDistTestCore/Metrics/MetricsQuery.cs → DistTestCore/Metrics/MetricsQuery.cs
View File

@ -1,22 +1,26 @@
using CodexDistTestCore.Config;
using DistTestCore.Codex;
using KubernetesWorkflow;
using System.Globalization;
namespace CodexDistTestCore.Metrics
namespace DistTestCore.Metrics
{
public class MetricsQuery
{
private readonly K8sCluster k8sCluster = new K8sCluster();
private readonly Http http;
public MetricsQuery(PrometheusInfo prometheusInfo)
public MetricsQuery(RunningContainers runningContainers)
{
RunningContainers = runningContainers;
http = new Http(
k8sCluster.GetIp(),
prometheusInfo.ServicePort,
runningContainers.RunningPod.Cluster.IP,
runningContainers.Containers[0].ServicePorts[0].Number,
"api/v1");
}
public Metrics? GetMostRecent(string metricName, OnlineCodexNode node)
public RunningContainers RunningContainers { get; }
public Metrics? GetMostRecent(string metricName, RunningContainer node)
{
var response = GetLastOverTime(metricName, GetInstanceStringForNode(node));
if (response == null) return null;
@ -41,7 +45,7 @@ namespace CodexDistTestCore.Metrics
return MapResponseToMetrics(response);
}
public Metrics? GetAllMetricsForNode(OnlineCodexNode node)
public Metrics? GetAllMetricsForNode(RunningContainer node)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}");
if (response.status != "success") return null;
@ -110,13 +114,14 @@ namespace CodexDistTestCore.Metrics
};
}
private string GetInstanceNameForNode(OnlineCodexNode node)
private string GetInstanceNameForNode(RunningContainer node)
{
var pod = node.Group.PodInfo!;
return $"{pod.Ip}:{node.Container.MetricsPort}";
var ip = node.Pod.Ip;
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
return $"{ip}:{port}";
}
private string GetInstanceStringForNode(OnlineCodexNode node)
private string GetInstanceStringForNode(RunningContainer node)
{
return "{instance=\"" + GetInstanceNameForNode(node) + "\"}";
}

19
DistTestCore/Metrics/PrometheusContainerRecipe.cs Normal file
View File

@ -0,0 +1,19 @@
using KubernetesWorkflow;
namespace DistTestCore.Metrics
{
public class PrometheusContainerRecipe : ContainerRecipeFactory
{
public const string DockerImage = "thatbenbierens/prometheus-envconf:latest";
protected override string Image => DockerImage;
protected override void Initialize(StartupConfig startupConfig)
{
var config = startupConfig.Get<PrometheusStartupConfig>();
AddExposedPortAndVar("PROM_PORT");
AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
}
}
}

12
DistTestCore/Metrics/PrometheusStartupConfig.cs Normal file
View File

@ -0,0 +1,12 @@
namespace DistTestCore.Metrics
{
public class PrometheusStartupConfig
{
public PrometheusStartupConfig(string prometheusConfigBase64)
{
PrometheusConfigBase64 = prometheusConfigBase64;
}
public string PrometheusConfigBase64 { get; }
}
}

60
CodexDistTestCore/OnlineCodexNode.cs → DistTestCore/OnlineCodexNode.cs
View File

@ -1,9 +1,10 @@
using CodexDistTestCore.Config;
using CodexDistTestCore.Marketplace;
using CodexDistTestCore.Metrics;
using DistTestCore.Codex;
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using NUnit.Framework;
namespace CodexDistTestCore
namespace DistTestCore
{
public interface IOnlineCodexNode
{
@ -20,31 +21,30 @@ namespace CodexDistTestCore
{
private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
private const string UploadFailedMessage = "Unable to store block";
private readonly TestLifecycle lifecycle;
private readonly K8sCluster k8sCluster = new K8sCluster();
private readonly TestLog log;
private readonly IFileManager fileManager;
public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container)
public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess)
{
this.log = log;
this.fileManager = fileManager;
Container = container;
this.lifecycle = lifecycle;
CodexAccess = codexAccess;
Group = group;
Metrics = metricsAccess;
Marketplace = marketplaceAccess;
}
public CodexNodeContainer Container { get; }
public CodexNodeGroup Group { get; internal set; } = null!;
public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable();
public IMarketplaceAccess Marketplace { set; get; } = new MarketplaceUnavailable();
public CodexAccess CodexAccess { get; }
public CodexNodeGroup Group { get; }
public IMetricsAccess Metrics { get; }
public IMarketplaceAccess Marketplace { get; }
public string GetName()
{
return $"<{Container.Name}>";
return $"<{CodexAccess.Container.Recipe.Name}>";
}
public CodexDebugResponse GetDebugInfo()
{
var response = Http().HttpGetJson<CodexDebugResponse>("debug/info");
var response = CodexAccess.GetDebugInfo();
Log($"Got DebugInfo with id: '{response.id}'.");
return response;
}
@ -53,7 +53,7 @@ namespace CodexDistTestCore
{
Log($"Uploading file of size {file.GetFileSize()}...");
using var fileStream = File.OpenRead(file.Filename);
var response = Http().HttpPostStream("upload", fileStream);
var response = CodexAccess.UploadFile(fileStream);
if (response.StartsWith(UploadFailedMessage))
{
Assert.Fail("Node failed to store block.");
@ -65,7 +65,7 @@ namespace CodexDistTestCore
public TestFile? DownloadContent(ContentId contentId)
{
Log($"Downloading for contentId: '{contentId.Id}'...");
var file = fileManager.CreateEmptyTestFile();
var file = lifecycle.FileManager.CreateEmptyTestFile();
DownloadToFile(contentId.Id, file);
Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'.");
return file;
@ -77,10 +77,7 @@ namespace CodexDistTestCore
Log($"Connecting to peer {peer.GetName()}...");
var peerInfo = node.GetDebugInfo();
var peerId = peerInfo.id;
var peerMultiAddress = GetPeerMultiAddress(peer, peerInfo);
var response = Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
Log($"Successfully connected to peer {peer.GetName()}.");
@ -88,12 +85,12 @@ namespace CodexDistTestCore
public ICodexNodeLog DownloadLog()
{
return Group.DownloadLog(this);
return lifecycle.DownloadLog(this);
}
public string Describe()
{
return $"{Group.Describe()} contains {GetName()}";
return $"({Group.Describe()} contains {GetName()})";
}
private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
@ -108,24 +105,19 @@ namespace CodexDistTestCore
// The peer we want to connect is in a different pod.
// We must replace the default IP with the pod IP in the multiAddress.
return multiAddress.Replace("0.0.0.0", peer.Group.PodInfo!.Ip);
return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip);
}
private void DownloadToFile(string contentId, TestFile file)
{
using var fileStream = File.OpenWrite(file.Filename);
using var downloadStream = Http().HttpGetStream("download/" + contentId);
using var downloadStream = CodexAccess.DownloadFile(contentId);
downloadStream.CopyTo(fileStream);
}
private Http Http()
{
return new Http(ip: k8sCluster.GetIp(), port: Container.ServicePort, baseUrl: "/api/codex/v1");
}
private void Log(string msg)
{
log.Log($"{GetName()}: {msg}");
lifecycle.Log.Log($"{GetName()}: {msg}");
}
}

65
DistTestCore/PrometheusStarter.cs Normal file
View File

@ -0,0 +1,65 @@
using DistTestCore.Codex;
using DistTestCore.Metrics;
using KubernetesWorkflow;
using System.Text;
namespace DistTestCore
{
public class PrometheusStarter
{
private readonly TestLifecycle lifecycle;
private readonly WorkflowCreator workflowCreator;
public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
{
this.lifecycle = lifecycle;
this.workflowCreator = workflowCreator;
}
public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers)
{
if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
Log($"Starting metrics server for {containers.Describe()}");
var startupConfig = new StartupConfig();
startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
var workflow = workflowCreator.CreateWorkflow();
var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
Log("Metrics server started.");
return new CodexNodeMetricsAccessFactory(runningContainers);
}
private string GeneratePrometheusConfig(RunningContainer[] nodes)
{
var config = "";
config += "global:\n";
config += " scrape_interval: 30s\n";
config += " scrape_timeout: 10s\n";
config += "\n";
config += "scrape_configs:\n";
config += " - job_name: services\n";
config += " metrics_path: /metrics\n";
config += " static_configs:\n";
config += " - targets:\n";
foreach (var node in nodes)
{
var ip = node.Pod.Ip;
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
config += $" - '{ip}:{port}'\n";
}
var bytes = Encoding.ASCII.GetBytes(config);
return Convert.ToBase64String(bytes);
}
private void Log(string msg)
{
lifecycle.Log.Log(msg);
}
}
}

19
DistTestCore/Stopwatch.cs Normal file
View File

@ -0,0 +1,19 @@
using Logging;
using Utils;
namespace DistTestCore
{
public class Stopwatch
{
public static void Measure(BaseLog log, string name, Action action)
{
var start = DateTime.UtcNow;
action();
var duration = DateTime.UtcNow - start;
log.Log($"{name} ({Time.FormatDuration(duration)})");
}
}
}

46
DistTestCore/TestLifecycle.cs Normal file
View File

@ -0,0 +1,46 @@
using DistTestCore.Logs;
using KubernetesWorkflow;
using Logging;
namespace DistTestCore
{
public class TestLifecycle
{
private readonly WorkflowCreator workflowCreator;
public TestLifecycle(TestLog log, Configuration configuration)
{
Log = log;
workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration());
FileManager = new FileManager(Log, configuration);
CodexStarter = new CodexStarter(this, workflowCreator);
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
GethStarter = new GethStarter(this, workflowCreator);
}
public TestLog Log { get; }
public FileManager FileManager { get; }
public CodexStarter CodexStarter { get; }
public PrometheusStarter PrometheusStarter { get; }
public GethStarter GethStarter { get; }
public void DeleteAllResources()
{
CodexStarter.DeleteAllResources();
FileManager.DeleteAllTestFiles();
}
public ICodexNodeLog DownloadLog(OnlineCodexNode node)
{
var subFile = Log.CreateSubfile();
var description = node.Describe();
var handler = new LogDownloadHandler(description, subFile);
Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
CodexStarter.DownloadLog(node.CodexAccess.Container, handler);
return new CodexNodeLog(subFile);
}
}
}

9
CodexDistTestCore/Timing.cs → DistTestCore/Timing.cs
View File

@ -1,6 +1,7 @@
using NUnit.Framework;
using Utils;
namespace CodexDistTestCore
namespace DistTestCore
{
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class UseLongTimeoutsAttribute : PropertyAttribute
@ -27,12 +28,12 @@ namespace CodexDistTestCore
public static void HttpCallRetryDelay()
{
Utils.Sleep(GetTimes().HttpCallRetryDelay());
Time.Sleep(GetTimes().HttpCallRetryDelay());
}
public static void WaitForK8sServiceDelay()
public static TimeSpan K8sServiceDelay()
{
Utils.Sleep(GetTimes().WaitForK8sServiceDelay());
return GetTimes().WaitForK8sServiceDelay();
}
public static TimeSpan K8sOperationTimeout()

52
KubernetesWorkflow/CommandRunner.cs Normal file
View File

@ -0,0 +1,52 @@
using k8s;
using Utils;
namespace KubernetesWorkflow
{
public class CommandRunner
{
private readonly Kubernetes client;
private readonly string k8sNamespace;
private readonly RunningPod pod;
private readonly string containerName;
private readonly string command;
private readonly string[] arguments;
private readonly List<string> lines = new List<string>();
public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
{
this.client = client;
this.k8sNamespace = k8sNamespace;
this.pod = pod;
this.containerName = containerName;
this.command = command;
this.arguments = arguments;
}
public void Run()
{
var input = new[] { command }.Concat(arguments).ToArray();
Time.Wait(client.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()));
}
public string GetStdOut()
{
return string.Join(Environment.NewLine, lines);
}
private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr)
{
using var streamReader = new StreamReader(stdOut);
var line = streamReader.ReadLine();
while (line != null)
{
lines.Add(line);
line = streamReader.ReadLine();
}
return Task.CompletedTask;
}
}
}

32
KubernetesWorkflow/Configuration.cs Normal file
View File

@ -0,0 +1,32 @@
namespace KubernetesWorkflow
{
public class Configuration
{
public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
{
K8sNamespace = k8sNamespace;
KubeConfigFile = kubeConfigFile;
OperationTimeout = operationTimeout;
RetryDelay = retryDelay;
LocationMap = locationMap;
}
public string K8sNamespace { get; }
public string? KubeConfigFile { get; }
public TimeSpan OperationTimeout { get; }
public TimeSpan RetryDelay { get; }
public ConfigurationLocationEntry[] LocationMap { get; }
}
public class ConfigurationLocationEntry
{
public ConfigurationLocationEntry(Location location, string workerName)
{
Location = location;
WorkerName = workerName;
}
public Location Location { get; }
public string WorkerName { get; }
}
}

52
KubernetesWorkflow/ContainerRecipe.cs Normal file
View File

@ -0,0 +1,52 @@
namespace KubernetesWorkflow
{
public class ContainerRecipe
{
public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, object[] additionals)
{
Number = number;
Image = image;
ExposedPorts = exposedPorts;
InternalPorts = internalPorts;
EnvVars = envVars;
Additionals = additionals;
}
public string Name { get { return $"ctnr{Number}"; } }
public int Number { get; }
public string Image { get; }
public Port[] ExposedPorts { get; }
public Port[] InternalPorts { get; }
public EnvVar[] EnvVars { get; }
public object[] Additionals { get; }
public Port GetPortByTag(string tag)
{
return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
}
}
public class Port
{
public Port(int number, string tag)
{
Number = number;
Tag = tag;
}
public int Number { get; }
public string Tag { get; }
}
public class EnvVar
{
public EnvVar(string name, string value)
{
Name = name;
Value = value;
}
public string Name { get; }
public string Value { get; }
}
}

74
KubernetesWorkflow/ContainerRecipeFactory.cs Normal file
View File

@ -0,0 +1,74 @@
namespace KubernetesWorkflow
{
public abstract class ContainerRecipeFactory
{
private readonly List<Port> exposedPorts = new List<Port>();
private readonly List<Port> internalPorts = new List<Port>();
private readonly List<EnvVar> envVars = new List<EnvVar>();
private readonly List<object> additionals = new List<object>();
private RecipeComponentFactory factory = null!;
public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config)
{
this.factory = factory;
ContainerNumber = containerNumber;
Index = index;
Initialize(config);
var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), additionals.ToArray());
exposedPorts.Clear();
internalPorts.Clear();
envVars.Clear();
additionals.Clear();
this.factory = null!;
return recipe;
}
protected abstract string Image { get; }
protected int ContainerNumber { get; private set; } = 0;
protected int Index { get; private set; } = 0;
protected abstract void Initialize(StartupConfig config);
protected Port AddExposedPort(string tag = "")
{
var p = factory.CreatePort(tag);
exposedPorts.Add(p);
return p;
}
protected Port AddInternalPort(string tag = "")
{
var p = factory.CreatePort(tag);
internalPorts.Add(p);
return p;
}
protected void AddExposedPortAndVar(string name, string tag = "")
{
AddEnvVar(name, AddExposedPort(tag));
}
protected void AddInternalPortAndVar(string name, string tag = "")
{
AddEnvVar(name, AddInternalPort(tag));
}
protected void AddEnvVar(string name, string value)
{
envVars.Add(factory.CreateEnvVar(name, value));
}
protected void AddEnvVar(string name, Port value)
{
envVars.Add(factory.CreateEnvVar(name, value.Number));
}
protected void Additional(object userData)
{
additionals.Add(userData);
}
}
}

56
KubernetesWorkflow/K8sCluster.cs Normal file
View File

@ -0,0 +1,56 @@
using k8s;
namespace KubernetesWorkflow
{
public class K8sCluster
{
public K8sCluster(Configuration configuration)
{
Configuration = configuration;
}
public Configuration Configuration { get; }
public string IP { get; private set; } = string.Empty;
public KubernetesClientConfiguration GetK8sClientConfig()
{
var config = GetConfig();
UpdateIp(config);
return config;
}
public string GetNodeLabelForLocation(Location location)
{
if (location == Location.Unspecified) return string.Empty;
return Configuration.LocationMap.Single(l => l.Location == location).WorkerName;
}
public TimeSpan K8sOperationTimeout()
{
return Configuration.OperationTimeout;
}
public TimeSpan WaitForK8sServiceDelay()
{
return Configuration.RetryDelay;
}
private KubernetesClientConfiguration GetConfig()
{
if (Configuration.KubeConfigFile != null)
{
return KubernetesClientConfiguration.BuildConfigFromConfigFile(Configuration.KubeConfigFile);
}
else
{
return KubernetesClientConfiguration.BuildDefaultConfig();
}
}
private void UpdateIp(KubernetesClientConfiguration config)
{
var host = config.Host.Replace("https://", "");
IP = host.Substring(0, host.IndexOf(':'));
}
}
}

382
KubernetesWorkflow/K8sController.cs Normal file
View File

@ -0,0 +1,382 @@
using k8s;
using k8s.Models;
namespace KubernetesWorkflow
{
public class K8sController
{
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownPods;
private readonly WorkflowNumberSource workflowNumberSource;
private readonly Kubernetes client;
public K8sController(K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource)
{
this.cluster = cluster;
this.knownPods = knownPods;
this.workflowNumberSource = workflowNumberSource;
client = new Kubernetes(cluster.GetK8sClientConfig());
}
public void Dispose()
{
client.Dispose();
}
public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location)
{
EnsureTestNamespace();
var deploymentName = CreateDeployment(containerRecipes, location);
var (serviceName, servicePortsMap) = CreateService(containerRecipes);
var (podName, podIp) = FetchNewPod();
return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap);
}
public void Stop(RunningPod pod)
{
if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
DeleteDeployment(pod.DeploymentName);
WaitUntilDeploymentOffline(pod.DeploymentName);
WaitUntilPodOffline(pod.Name);
}
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
{
using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name);
logHandler.Log(stream);
}
public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
{
var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
runner.Run();
return runner.GetStdOut();
}
public void DeleteAllResources()
{
DeleteNamespace();
WaitUntilNamespaceDeleted();
}
#region Namespace management
private void EnsureTestNamespace()
{
if (IsTestNamespaceOnline()) return;
var namespaceSpec = new V1Namespace
{
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = K8sNamespace,
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
}
};
client.CreateNamespace(namespaceSpec);
WaitUntilNamespaceCreated();
}
private void DeleteNamespace()
{
if (IsTestNamespaceOnline())
{
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
}
}
private string K8sNamespace
{
get { return cluster.Configuration.K8sNamespace; }
}
private bool IsTestNamespaceOnline()
{
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
}
#endregion
#region Deployment management
private string CreateDeployment(ContainerRecipe[] containerRecipes, Location location)
{
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = CreateDeploymentMetadata(),
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = GetSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = GetSelector()
},
Spec = new V1PodSpec
{
NodeSelector = CreateNodeSelector(location),
Containers = CreateDeploymentContainers(containerRecipes)
}
}
}
};
client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
return deploymentSpec.Metadata.Name;
}
private void DeleteDeployment(string deploymentName)
{
client.DeleteNamespacedDeployment(deploymentName, K8sNamespace);
WaitUntilDeploymentOffline(deploymentName);
}
private IDictionary<string, string> CreateNodeSelector(Location location)
{
if (location == Location.Unspecified) return new Dictionary<string, string>();
return new Dictionary<string, string>
{
{ "codex-test-location", cluster.GetNodeLabelForLocation(location) }
};
}
private IDictionary<string, string> GetSelector()
{
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
}
private V1ObjectMeta CreateDeploymentMetadata()
{
return new V1ObjectMeta
{
Name = "deploy-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sNamespace
};
}
private List<V1Container> CreateDeploymentContainers(ContainerRecipe[] containerRecipes)
{
return containerRecipes.Select(r => CreateDeploymentContainer(r)).ToList();
}
private V1Container CreateDeploymentContainer(ContainerRecipe recipe)
{
return new V1Container
{
Name = recipe.Name,
Image = recipe.Image,
Ports = CreateContainerPorts(recipe),
Env = CreateEnv(recipe)
};
}
private List<V1EnvVar> CreateEnv(ContainerRecipe recipe)
{
return recipe.EnvVars.Select(CreateEnvVar).ToList();
}
private V1EnvVar CreateEnvVar(EnvVar envVar)
{
return new V1EnvVar
{
Name = envVar.Name,
Value = envVar.Value,
};
}
private List<V1ContainerPort> CreateContainerPorts(ContainerRecipe recipe)
{
var exposedPorts = recipe.ExposedPorts.Select(p => CreateContainerPort(recipe, p));
var internalPorts = recipe.InternalPorts.Select(p => CreateContainerPort(recipe, p));
return exposedPorts.Concat(internalPorts).ToList();
}
private V1ContainerPort CreateContainerPort(ContainerRecipe recipe, Port port)
{
return new V1ContainerPort
{
Name = GetNameForPort(recipe, port),
ContainerPort = port.Number
};
}
private string GetNameForPort(ContainerRecipe recipe, Port port)
{
return $"p{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}";
}
#endregion
#region Service management
private (string, Dictionary<ContainerRecipe, Port[]>) CreateService(ContainerRecipe[] containerRecipes)
{
var result = new Dictionary<ContainerRecipe, Port[]>();
var ports = CreateServicePorts(result, containerRecipes);
if (!ports.Any())
{
// None of these container-recipes wish to expose anything via a serice port.
// So, we don't have to create a service.
return (string.Empty, result);
}
var serviceSpec = new V1Service
{
ApiVersion = "v1",
Metadata = CreateServiceMetadata(),
Spec = new V1ServiceSpec
{
Type = "NodePort",
Selector = GetSelector(),
Ports = ports
}
};
client.CreateNamespacedService(serviceSpec, K8sNamespace);
return (serviceSpec.Metadata.Name, result);
}
private void DeleteService(string serviceName)
{
client.DeleteNamespacedService(serviceName, K8sNamespace);
}
private V1ObjectMeta CreateServiceMetadata()
{
return new V1ObjectMeta
{
Name = "service-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sNamespace
};
}
private List<V1ServicePort> CreateServicePorts(Dictionary<ContainerRecipe, Port[]> servicePorts, ContainerRecipe[] recipes)
{
var result = new List<V1ServicePort>();
foreach (var recipe in recipes)
{
result.AddRange(CreateServicePorts(servicePorts, recipe));
}
return result;
}
private List<V1ServicePort> CreateServicePorts(Dictionary<ContainerRecipe, Port[]> servicePorts, ContainerRecipe recipe)
{
var result = new List<V1ServicePort>();
var usedPorts = new List<Port>();
foreach (var port in recipe.ExposedPorts)
{
var servicePort = workflowNumberSource.GetServicePort();
usedPorts.Add(new Port(servicePort, ""));
result.Add(new V1ServicePort
{
Name = GetNameForPort(recipe, port),
Protocol = "TCP",
Port = port.Number,
TargetPort = GetNameForPort(recipe, port),
NodePort = servicePort
});
}
servicePorts.Add(recipe, usedPorts.ToArray());
return result;
}
#endregion
#region Waiting
private void WaitUntilNamespaceCreated()
{
WaitUntil(() => IsTestNamespaceOnline());
}
private void WaitUntilNamespaceDeleted()
{
WaitUntil(() => !IsTestNamespaceOnline());
}
private void WaitUntilDeploymentOnline(string deploymentName)
{
WaitUntil(() =>
{
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
});
}
private void WaitUntilDeploymentOffline(string deploymentName)
{
WaitUntil(() =>
{
var deployments = client.ListNamespacedDeployment(K8sNamespace);
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
return deployment == null || deployment.Status.AvailableReplicas == 0;
});
}
private void WaitUntilPodOffline(string podName)
{
WaitUntil(() =>
{
var pods = client.ListNamespacedPod(K8sNamespace).Items;
var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
return pod == null;
});
}
private void WaitUntil(Func<bool> predicate)
{
var start = DateTime.UtcNow;
var state = predicate();
while (!state)
{
if (DateTime.UtcNow - start > cluster.K8sOperationTimeout())
{
throw new TimeoutException("K8s operation timed out.");
}
cluster.WaitForK8sServiceDelay();
state = predicate();
}
}
#endregion
private (string, string) FetchNewPod()
{
var pods = client.ListNamespacedPod(K8sNamespace).Items;
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");
var newPod = newPods.Single();
var name = newPod.Name();
var ip = newPod.Status.PodIP;
if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
knownPods.Add(name);
return (name, ip);
}
}
}

2
CodexDistTestCore/KnownK8sPods.cs → KubernetesWorkflow/KnownK8sPods.cs
View File

@ -1,4 +1,4 @@
namespace CodexDistTestCore
namespace KubernetesWorkflow
{
public class KnownK8sPods
{

18
KubernetesWorkflow/KubernetesWorkflow.csproj Normal file
View File

@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<RootNamespace>KubernetesWorkflow</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="KubernetesClient" Version="10.1.4" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Utils\Utils.csproj" />
</ItemGroup>
</Project>

9
KubernetesWorkflow/Location.cs Normal file
View File

@ -0,0 +1,9 @@
namespace KubernetesWorkflow
{
public enum Location
{
Unspecified,
BensLaptop,
BensOldGamingMachine
}
}

25
KubernetesWorkflow/RecipeComponentFactory.cs Normal file
View File

@ -0,0 +1,25 @@
using System.Globalization;
using Utils;
namespace KubernetesWorkflow
{
public class RecipeComponentFactory
{
private NumberSource portNumberSource = new NumberSource(8080);
public Port CreatePort(string tag)
{
return new Port(portNumberSource.GetNextNumber(), tag);
}
public EnvVar CreateEnvVar(string name, int value)
{
return CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture));
}
public EnvVar CreateEnvVar(string name, string value)
{
return new EnvVar(name, value);
}
}
}

35
KubernetesWorkflow/RunningContainers.cs Normal file
View File

@ -0,0 +1,35 @@
namespace KubernetesWorkflow
{
public class RunningContainers
{
public RunningContainers(StartupConfig startupConfig, RunningPod runningPod, RunningContainer[] containers)
{
StartupConfig = startupConfig;
RunningPod = runningPod;
Containers = containers;
}
public StartupConfig StartupConfig { get; }
public RunningPod RunningPod { get; }
public RunningContainer[] Containers { get; }
public string Describe()
{
return $"[{RunningPod.Ip}]";
}
}
public class RunningContainer
{
public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts)
{
Pod = pod;
Recipe = recipe;
ServicePorts = servicePorts;
}
public RunningPod Pod { get; }
public ContainerRecipe Recipe { get; }
public Port[] ServicePorts { get; }
}
}

28
KubernetesWorkflow/RunningPod.cs Normal file
View File

@ -0,0 +1,28 @@
namespace KubernetesWorkflow
{
public class RunningPod
{
private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
{
Cluster = cluster;
Name = name;
Ip = ip;
DeploymentName = deploymentName;
ServiceName = serviceName;
this.servicePortMap = servicePortMap;
}
public K8sCluster Cluster { get; }
public string Name { get; }
public string Ip { get; }
internal string DeploymentName { get; }
internal string ServiceName { get; }
public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
{
return servicePortMap[containerRecipe];
}
}
}

18
KubernetesWorkflow/StartupConfig.cs Normal file
View File

@ -0,0 +1,18 @@
namespace KubernetesWorkflow
{
public class StartupConfig
{
private readonly List<object> configs = new List<object>();
public void Add(object config)
{
configs.Add(config);
}
public T Get<T>()
{
var match = configs.Single(c => typeof(T).IsAssignableFrom(c.GetType()));
return (T)match;
}
}
}

115
KubernetesWorkflow/StartupWorkflow.cs Normal file
View File

@ -0,0 +1,115 @@
using System.IO;
namespace KubernetesWorkflow
{
public class StartupWorkflow
{
private readonly WorkflowNumberSource numberSource;
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownK8SPods;
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
internal StartupWorkflow(WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods)
{
this.numberSource = numberSource;
this.cluster = cluster;
this.knownK8SPods = knownK8SPods;
}
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
{
return K8s(controller =>
{
var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
var runningPod = controller.BringOnline(recipes, location);
return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes));
});
}
public void Stop(RunningContainers runningContainers)
{
K8s(controller =>
{
controller.Stop(runningContainers.RunningPod);
});
}
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler)
{
K8s(controller =>
{
controller.DownloadPodLog(container.Pod, container.Recipe, logHandler);
});
}
public string ExecuteCommand(RunningContainer container, string command, params string[] args)
{
return K8s(controller =>
{
return controller.ExecuteCommand(container.Pod, container.Recipe.Name, command, args);
});
}
public void DeleteAllResources()
{
K8s(controller =>
{
controller.DeleteAllResources();
});
}
private static RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes)
{
return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r))).ToArray();
}
private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
{
var result = new List<ContainerRecipe>();
for (var i = 0; i < numberOfContainers; i++)
{
result.Add(recipeFactory.CreateRecipe(i ,numberSource.GetContainerNumber(), componentFactory, startupConfig));
}
return result.ToArray();
}
private void K8s(Action<K8sController> action)
{
var controller = new K8sController(cluster, knownK8SPods, numberSource);
action(controller);
controller.Dispose();
}
private T K8s<T>(Func<K8sController, T> action)
{
var controller = new K8sController(cluster, knownK8SPods, numberSource);
var result = action(controller);
controller.Dispose();
return result;
}
}
public interface ILogHandler
{
void Log(Stream log);
}
public abstract class LogHandler : ILogHandler
{
public void Log(Stream log)
{
using var reader = new StreamReader(log);
var line = reader.ReadLine();
while (line != null)
{
ProcessLine(line);
line = reader.ReadLine();
}
}
protected abstract void ProcessLine(string line);
}
}

27
KubernetesWorkflow/WorkflowCreator.cs Normal file
View File

@ -0,0 +1,27 @@
using Utils;
namespace KubernetesWorkflow
{
public class WorkflowCreator
{
private readonly NumberSource numberSource = new NumberSource(0);
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource containerNumberSource = new NumberSource(0);
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly K8sCluster cluster;
public WorkflowCreator(Configuration configuration)
{
cluster = new K8sCluster(configuration);
}
public StartupWorkflow CreateWorkflow()
{
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
servicePortNumberSource,
containerNumberSource);
return new StartupWorkflow(workflowNumberSource, cluster, knownPods);
}
}
}

29
KubernetesWorkflow/WorkflowNumberSource.cs Normal file
View File

@ -0,0 +1,29 @@
using Utils;
namespace KubernetesWorkflow
{
public class WorkflowNumberSource
{
private readonly NumberSource servicePortNumberSource;
private readonly NumberSource containerNumberSource;
public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource, NumberSource containerNumberSource)
{
WorkflowNumber = workflowNumber;
this.servicePortNumberSource = servicePortNumberSource;
this.containerNumberSource = containerNumberSource;
}
public int WorkflowNumber { get; }
public int GetContainerNumber()
{
return containerNumberSource.GetNextNumber();
}
public int GetServicePort()
{
return servicePortNumberSource.GetNextNumber();
}
}
}

36
Logging/BaseLog.cs Normal file
View File

@ -0,0 +1,36 @@
namespace Logging
{
public abstract class BaseLog
{
private bool hasFailed;
private LogFile? logFile;
protected abstract LogFile CreateLogFile();
protected LogFile LogFile
{
get
{
if (logFile == null) logFile = CreateLogFile();
return logFile;
}
}
public void Log(string message)
{
LogFile.Write(message);
}
public void Error(string message)
{
Log($"[ERROR] {message}");
}
public void MarkAsFailed()
{
if (hasFailed) return;
hasFailed = true;
LogFile.ConcatToFilename("_FAILED");
}
}
}

49
Logging/FixtureLog.cs Normal file
View File

@ -0,0 +1,49 @@
using NUnit.Framework;
namespace Logging
{
public class FixtureLog : BaseLog
{
private readonly DateTime start;
private readonly string fullName;
public FixtureLog(LogConfig config)
{
start = DateTime.UtcNow;
var folder = DetermineFolder(config);
var fixtureName = GetFixtureName();
fullName = Path.Combine(folder, fixtureName);
}
public TestLog CreateTestLog()
{
return new TestLog(fullName);
}
protected override LogFile CreateLogFile()
{
return new LogFile(fullName, "log");
}
private string DetermineFolder(LogConfig config)
{
return Path.Join(
config.LogRoot,
$"{start.Year}-{Pad(start.Month)}",
Pad(start.Day));
}
private string GetFixtureName()
{
var test = TestContext.CurrentContext.Test;
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
}
private static string Pad(int n)
{
return n.ToString().PadLeft(2, '0');
}
}
}

12
Logging/LogConfig.cs Normal file
View File

@ -0,0 +1,12 @@
namespace Logging
{
public class LogConfig
{
public LogConfig(string logRoot)
{
LogRoot = logRoot;
}
public string LogRoot { get; }
}
}

57
Logging/LogFile.cs Normal file
View File

@ -0,0 +1,57 @@
namespace Logging
{
public class LogFile
{
private readonly string extension;
private string filename;
public LogFile(string filename, string extension)
{
this.filename = filename;
this.extension = extension;
FullFilename = filename + "." + extension;
EnsurePathExists(filename);
}
public string FullFilename { get; private set; }
public void Write(string message)
{
WriteRaw($"{GetTimestamp()} {message}");
}
public void WriteRaw(string message)
{
try
{
File.AppendAllLines(FullFilename, new[] { message });
}
catch (Exception ex)
{
Console.WriteLine("Writing to log has failed: " + ex);
}
}
public void ConcatToFilename(string toAdd)
{
var oldFullName = FullFilename;
filename += toAdd;
FullFilename = filename + "." + extension;
File.Move(oldFullName, FullFilename);
}
private static string GetTimestamp()
{
return $"[{DateTime.UtcNow.ToString("u")}]";
}
private void EnsurePathExists(string filename)
{
var path = new FileInfo(filename).Directory!.FullName;
Directory.CreateDirectory(path);
}
}
}

19
Logging/Logging.csproj Normal file
View File

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<RootNamespace>Logging</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="nunit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Utils\Utils.csproj" />
</ItemGroup>
</Project>

64
Logging/TestLog.cs Normal file
View File

@ -0,0 +1,64 @@
using NUnit.Framework;
using Utils;
namespace Logging
{
public class TestLog : BaseLog
{
private readonly NumberSource subfileNumberSource = new NumberSource(0);
private readonly string methodName;
private readonly string fullName;
public TestLog(string folder)
{
methodName = GetMethodName();
fullName = Path.Combine(folder, methodName);
Log($"Begin: {methodName}");
}
public LogFile CreateSubfile(string ext = "log")
{
return new LogFile($"{fullName}_{GetSubfileNumber()}", ext);
}
public void EndTest()
{
var result = TestContext.CurrentContext.Result;
Log($"Finished: {methodName} = {result.Outcome.Status}");
if (!string.IsNullOrEmpty(result.Message))
{
Log(result.Message);
Log($"{result.StackTrace}");
}
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
MarkAsFailed();
}
}
protected override LogFile CreateLogFile()
{
return new LogFile(fullName, "log");
}
private string GetMethodName()
{
var test = TestContext.CurrentContext.Test;
var args = FormatArguments(test);
return $"{test.MethodName}{args}";
}
private string GetSubfileNumber()
{
return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0');
}
private static string FormatArguments(TestContext.TestAdapter test)
{
if (test.Arguments == null || !test.Arguments.Any()) return "";
return $"[{string.Join(',', test.Arguments)}]";
}
}
}

5
LongTests/BasicTests/LargeFileTests.cs
View File

@ -1,7 +1,8 @@
using CodexDistTestCore;
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace LongTests.BasicTests
namespace TestsLong.BasicTests
{
[TestFixture]
public class LargeFileTests : DistTest

5
LongTests/BasicTests/TestInfraTests.cs
View File

@ -1,7 +1,8 @@
using CodexDistTestCore;
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace LongTests.BasicTests
namespace TestsLong.BasicTests
{
public class TestInfraTests : DistTest
{

2
LongTests/LongTests.csproj → LongTests/TestsLong.csproj
View File

@ -13,7 +13,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\CodexDistTestCore\CodexDistTestCore.csproj" />
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
</ItemGroup>
</Project>

58
Nethereum/NethereumInteraction.cs Normal file
View File

@ -0,0 +1,58 @@
using Logging;
using Nethereum.Hex.HexTypes;
using Nethereum.Web3;
using System.Numerics;
using Utils;
namespace NethereumWorkflow
{
public class NethereumInteraction
{
private readonly TestLog log;
private readonly Web3 web3;
private readonly string rootAccount;
internal NethereumInteraction(TestLog log, Web3 web3, string rootAccount)
{
this.log = log;
this.web3 = web3;
this.rootAccount = rootAccount;
}
public void TransferTo(string account, decimal amount)
{
if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance");
var value = ToHexBig(amount);
var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value));
Time.Wait(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId));
Log($"Transferred {amount} to {account}");
}
public decimal GetBalance(string account)
{
var bigInt = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(account));
var result = ToDecimal(bigInt);
Log($"Balance of {account} is {result}");
return result;
}
private HexBigInteger ToHexBig(decimal amount)
{
var bigint = new BigInteger(amount);
var str = bigint.ToString("X");
return new HexBigInteger(str);
}
private decimal ToDecimal(HexBigInteger hexBigInteger)
{
return (decimal)hexBigInteger.Value;
}
private void Log(string msg)
{
log.Log(msg);
}
}
}

32
Nethereum/NethereumInteractionCreator.cs Normal file
View File

@ -0,0 +1,32 @@
using Logging;
using Nethereum.Web3;
namespace NethereumWorkflow
{
public class NethereumInteractionCreator
{
private readonly TestLog log;
private readonly string ip;
private readonly int port;
private readonly string rootAccount;
public NethereumInteractionCreator(TestLog log, string ip, int port, string rootAccount)
{
this.log = log;
this.ip = ip;
this.port = port;
this.rootAccount = rootAccount;
}
public NethereumInteraction CreateWorkflow()
{
return new NethereumInteraction(log, CreateWeb3(), rootAccount);
}
private Web3 CreateWeb3()
{
//var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^");
return new Web3($"http://{ip}:{port}");
}
}
}

19
Nethereum/NethereumWorkflow.csproj Normal file
View File

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<RootNamespace>NethereumWorkflow</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Logging\Logging.csproj" />
<ProjectReference Include="..\Utils\Utils.csproj" />
</ItemGroup>
</Project>

84
Tests/BasicTests/SimpleTests.cs → Tests/BasicTests/ExampleTests.cs
View File

@ -1,11 +1,26 @@
using CodexDistTestCore;
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace Tests.BasicTests
{
[TestFixture]
public class SimpleTests : DistTest
public class ExampleTests : DistTest
{
[Test]
public void CodexLogExample()
{
var primary = SetupCodexNodes(1)
.WithLogLevel(CodexLogLevel.Trace)
.BringOnline()[0];
primary.UploadFile(GenerateTestFile(5.MB()));
var log = primary.DownloadLog();
log.AssertLogContains("Uploaded file");
}
[Test]
public void TwoMetricsExample()
{
@ -34,7 +49,7 @@ namespace Tests.BasicTests
[Test]
public void MarketplaceExample()
{
var group = SetupCodexNodes(4)
var group = SetupCodexNodes(2)
.WithStorageQuota(10.GB())
.EnableMarketplace(initialBalance: 20)
.BringOnline();
@ -67,68 +82,5 @@ namespace Tests.BasicTests
//secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage.");
//primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage.");
}
[Test]
public void OneClientTest()
{
var primary = SetupCodexNodes(1).BringOnline()[0];
var testFile = GenerateTestFile(1.MB());
var contentId = primary.UploadFile(testFile);
var downloadedFile = primary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
}
[Test]
public void TwoClientsOnePodTest()
{
var group = SetupCodexNodes(2).BringOnline();
var primary = group[0];
var secondary = group[1];
PerformTwoClientTest(primary, secondary);
}
[Test]
public void TwoClientsTwoPodsTest()
{
var primary = SetupCodexNodes(1).BringOnline()[0];
var secondary = SetupCodexNodes(1).BringOnline()[0];
PerformTwoClientTest(primary, secondary);
}
[Test]
[Ignore("Requires Location map to be configured for k8s cluster.")]
public void TwoClientsTwoLocationsTest()
{
var primary = SetupCodexNodes(1)
.At(Location.BensLaptop)
.BringOnline()[0];
var secondary = SetupCodexNodes(1)
.At(Location.BensOldGamingMachine)
.BringOnline()[0];
PerformTwoClientTest(primary, secondary);
}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{
primary.ConnectToPeer(secondary);
var testFile = GenerateTestFile(1.MB());
var contentId = primary.UploadFile(testFile);
var downloadedFile = secondary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
}
}
}

40
Tests/BasicTests/OneClientTests.cs Normal file
View File

@ -0,0 +1,40 @@
using DistTestCore;
using NUnit.Framework;
namespace Tests.BasicTests
{
[TestFixture]
public class OneClientTests : DistTest
{
[Test]
public void OneClientTest()
{
var primary = SetupCodexNodes(1).BringOnline()[0];
PerformOneClientTest(primary);
}
[Test]
public void RestartTest()
{
var group = SetupCodexNodes(1).BringOnline();
var setup = group.BringOffline();
var primary = setup.BringOnline()[0];
PerformOneClientTest(primary);
}
private void PerformOneClientTest(IOnlineCodexNode primary)
{
var testFile = GenerateTestFile(1.MB());
var contentId = primary.UploadFile(testFile);
var downloadedFile = primary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
}
}
}

59
Tests/BasicTests/TwoClientTests.cs Normal file
View File

@ -0,0 +1,59 @@
using DistTestCore;
using KubernetesWorkflow;
using NUnit.Framework;
namespace Tests.BasicTests
{
[TestFixture]
public class TwoClientTests : DistTest
{
[Test]
public void TwoClientsOnePodTest()
{
var group = SetupCodexNodes(2).BringOnline();
var primary = group[0];
var secondary = group[1];
PerformTwoClientTest(primary, secondary);
}
[Test]
public void TwoClientsTwoPodsTest()
{
var primary = SetupCodexNodes(1).BringOnline()[0];
var secondary = SetupCodexNodes(1).BringOnline()[0];
PerformTwoClientTest(primary, secondary);
}
[Test]
[Ignore("Requires Location map to be configured for k8s cluster.")]
public void TwoClientsTwoLocationsTest()
{
var primary = SetupCodexNodes(1)
.At(Location.BensLaptop)
.BringOnline()[0];
var secondary = SetupCodexNodes(1)
.At(Location.BensOldGamingMachine)
.BringOnline()[0];
PerformTwoClientTest(primary, secondary);
}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{
primary.ConnectToPeer(secondary);
var testFile = GenerateTestFile(1.MB());
var contentId = primary.UploadFile(testFile);
var downloadedFile = secondary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
}
}
}

2
Tests/Tests.csproj
View File

@ -13,7 +13,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\CodexDistTestCore\CodexDistTestCore.csproj" />
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
</ItemGroup>
</Project>

2
CodexDistTestCore/NumberSource.cs → Utils/NumberSource.cs
View File

@ -1,4 +1,4 @@
namespace CodexDistTestCore
namespace Utils
{
public class NumberSource
{

26
Utils/Time.cs Normal file
View File

@ -0,0 +1,26 @@
namespace Utils
{
public static class Time
{
public static void Sleep(TimeSpan span)
{
Thread.Sleep(span);
}
public static T Wait<T>(Task<T> task)
{
task.Wait();
return task.Result;
}
public static string FormatDuration(TimeSpan d)
{
var result = "";
if (d.Days > 0) result += $"{d.Days} days, ";
if (d.Hours > 0) result += $"{d.Hours} hours, ";
if (d.Minutes > 0) result += $"{d.Minutes} mins, ";
result += $"{d.Seconds} secs";
return result;
}
}
}

10
Utils/Utils.csproj Normal file
View File

@ -0,0 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<RootNamespace>Utils</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

38
cs-codex-dist-testing.sln
View File

@ -3,11 +3,19 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.4.33213.308
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Tests\Tests.csproj", "{57F57B85-A537-4D3A-B7AE-B72C66B74AAB}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj", "{57F57B85-A537-4D3A-B7AE-B72C66B74AAB}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LongTests", "LongTests\LongTests.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestsLong", "LongTests\TestsLong.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDistTestCore", "CodexDistTestCore\CodexDistTestCore.csproj", "{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Utils", "Utils\Utils.csproj", "{957DE3B8-9571-450A-8609-B267DCA8727C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -23,10 +31,26 @@ Global
{AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Release|Any CPU.Build.0 = Release|Any CPU
{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.Build.0 = Release|Any CPU
{47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.Build.0 = Release|Any CPU
{359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Release|Any CPU.Build.0 = Release|Any CPU
{957DE3B8-9571-450A-8609-B267DCA8727C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{957DE3B8-9571-450A-8609-B267DCA8727C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.Build.0 = Release|Any CPU
{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Release|Any CPU.Build.0 = Release|Any CPU
{D6C3555E-D52D-4993-A87B-71AB650398FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D6C3555E-D52D-4993-A87B-71AB650398FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE