status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

splits geth nodes into their own pod

This commit is contained in:
benbierens 2023-04-11 12:06:33 +02:00
parent c977e37ab8
commit d33eb53003
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
7 changed files with 180 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CodexDistTestCore/CodexNodeGroup.cs
View File

@ -46,7 +46,7 @@ namespace CodexDistTestCore
public V1Deployment? Deployment { get; set; }
public V1Service? Service { get; set; }
public PodInfo? PodInfo { get; set; }
public GethInfo? GethInfo { get; set; }
public GethCompanionGroup? GethCompanionGroup { get; set; }
public CodexNodeContainer[] GetContainers()
{

22
CodexDistTestCore/K8sManager.cs
View File

@ -34,7 +34,7 @@ namespace CodexDistTestCore
if (offline.MarketplaceConfig != null)
{
group.GethInfo = marketplaceController.BringOnlineMarketplace(offline);
group.GethCompanionGroup = marketplaceController.BringOnlineMarketplace(offline);
}
K8s(k => k.BringOnline(group, offline));
@ -91,13 +91,21 @@ namespace CodexDistTestCore
return K8s(k => k.BringOnlinePrometheus(spec));
}
public GethInfo BringOnlineGethBootstrapNode()
public K8sGethBoostrapSpecs CreateGethBootstrapNodeSpec()
{
var spec = new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort());
return new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort());
}
public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec)
{
return K8s(k => k.BringOnlineGethBootstrapNode(spec));
}
public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group)
{
return K8s(k => k.BringOnlineGethCompanionGroup(info, group));
}
public void DownloadAllMetrics()
{
metricsAggregator.DownloadAllMetrics();
@ -110,15 +118,15 @@ namespace CodexDistTestCore
private void ConnectMarketplace(CodexNodeGroup group)
{
foreach (var node in DowncastNodes(group))
for (var i = 0; i < group.Nodes.Length; i++)
{
ConnectMarketplace(group, node);
ConnectMarketplace(group, group.Nodes[i], group.GethCompanionGroup!.Containers[i]);
}
}
private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node)
private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node, GethCompanionNodeContainer container)
{
var access = new MarketplaceAccess(this, marketplaceController, log, group, node.Container.GethCompanionNodeContainer!);
var access = new MarketplaceAccess(this, marketplaceController, log, group, container);
access.Initialize();
node.Marketplace = access;
}

37
CodexDistTestCore/K8sOperations.cs
View File

@ -4,7 +4,6 @@ using CodexDistTestCore.Metrics;
using k8s;
using k8s.Models;
using NUnit.Framework;
using System.Numerics;
namespace CodexDistTestCore
{
@ -78,7 +77,7 @@ namespace CodexDistTestCore
return new PrometheusInfo(spec.ServicePort, FetchNewPod());
}
public GethInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec)
public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec)
{
EnsureTestNamespace();
@ -86,7 +85,17 @@ namespace CodexDistTestCore
CreateGethBootstrapService(spec);
WaitUntilGethBootstrapOnline(spec);
return new GethInfo(spec, FetchNewPod());
return FetchNewPod();
}
public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group)
{
EnsureTestNamespace();
CreateGethCompanionDeployment(info, group);
WaitUntilGethCompanionGroupOnline(info.Spec, group);
return FetchNewPod();
}
private void FetchPodInfo(CodexNodeGroup online)
@ -148,7 +157,12 @@ namespace CodexDistTestCore
private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec)
{
WaitUntilDeploymentOnline(spec.GetDeploymentName());
WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName());
}
private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group)
{
WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group));
}
private void WaitUntilDeploymentOnline(string deploymentName)
@ -212,11 +226,6 @@ namespace CodexDistTestCore
TargetPort = container.ContainerPortName,
NodePort = container.ServicePort
});
if (container.GethCompanionNodeContainer != null)
{
result.Add(container.GethCompanionNodeContainer.CreateServicePort());
}
}
return result;
}
@ -303,11 +312,6 @@ namespace CodexDistTestCore
},
Env = dockerImage.CreateEnvironmentVariables(offline, container)
});
if (container.GethCompanionNodeContainer != null)
{
result.Add(container.GethCompanionNodeContainer.CreateDeploymentContainer(online.GethInfo!));
}
}
return result;
@ -330,6 +334,11 @@ namespace CodexDistTestCore
client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace);
}
private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group)
{
client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace);
}
#endregion
#region Namespace management

20
CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs
View File

@ -1,7 +1,18 @@
using k8s.Models;
namespace CodexDistTestCore.Marketplace
namespace CodexDistTestCore.Marketplace
{
public class GethCompanionGroup
{
public GethCompanionGroup(int number, GethCompanionNodeContainer[] containers)
{
Number = number;
Containers = containers;
}
public int Number { get; }
public GethCompanionNodeContainer[] Containers { get; }
public PodInfo? Pod { get; set; }
}
public class GethCompanionNodeContainer
{
public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName)
@ -16,6 +27,7 @@ namespace CodexDistTestCore.Marketplace
public int ApiPort { get; }
public int RpcPort { get; }
public string ContainerPortName { get; }
public string Account { get; set; } = string.Empty;
}
}

69
CodexDistTestCore/Marketplace/K8sGethSpecs.cs
View File

@ -1,6 +1,5 @@
using CodexDistTestCore.Config;
using k8s.Models;
using System.Xml.Linq;
namespace CodexDistTestCore.Marketplace
{
@ -24,11 +23,16 @@ namespace CodexDistTestCore.Marketplace
public int ServicePort { get; }
public string GetDeploymentName()
public string GetBootstrapDeploymentName()
{
return "test-gethb";
}
public string GetCompanionDeploymentName(GethCompanionGroup group)
{
return "test-geth" + group.Number;
}
public V1Deployment CreateGethBootstrapDeployment()
{
var deploymentSpec = new V1Deployment
@ -36,7 +40,7 @@ namespace CodexDistTestCore.Marketplace
ApiVersion = "apps/v1",
Metadata = new V1ObjectMeta
{
Name = GetDeploymentName(),
Name = GetBootstrapDeploymentName(),
NamespaceProperty = K8sCluster.K8sNamespace
},
Spec = new V1DeploymentSpec
@ -44,13 +48,13 @@ namespace CodexDistTestCore.Marketplace
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = CreateSelector()
MatchLabels = CreateBootstrapSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = CreateSelector()
Labels = CreateBootstrapSelector()
},
Spec = new V1PodSpec
{
@ -109,7 +113,7 @@ namespace CodexDistTestCore.Marketplace
Spec = new V1ServiceSpec
{
Type = "NodePort",
Selector = CreateSelector(),
Selector = CreateBootstrapSelector(),
Ports = new List<V1ServicePort>
{
new V1ServicePort
@ -127,18 +131,52 @@ namespace CodexDistTestCore.Marketplace
return serviceSpec;
}
public V1Deployment CreateGethCompanionDeployment(GethInfo gethInfo)
public V1Deployment CreateGethCompanionDeployment(GethCompanionGroup group, GethBootstrapInfo info)
{
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = new V1ObjectMeta
{
Name = GetCompanionDeploymentName(group),
NamespaceProperty = K8sCluster.K8sNamespace
},
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = CreateCompanionSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = CreateCompanionSelector()
},
Spec = new V1PodSpec
{
Containers = group.Containers.Select(c => CreateContainer(c, info)).ToList()
}
}
}
};
return deploymentSpec;
}
private static V1Container CreateContainer(GethCompanionNodeContainer container, GethBootstrapInfo info)
{
return new V1Container
{
Name = Name,
Name = container.Name,
Image = GethDockerImage.Image,
Ports = new List<V1ContainerPort>
{
new V1ContainerPort
{
ContainerPort = ApiPort,
Name = ContainerPortName
ContainerPort = container.ApiPort,
Name = container.ContainerPortName
}
},
// todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64
@ -147,20 +185,25 @@ namespace CodexDistTestCore.Marketplace
new V1EnvVar
{
Name = "GETH_ARGS",
Value = $"--port {ApiPort} --discovery.port {ApiPort} --authrpc.port {RpcPort}"
Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.RpcPort}"
},
new V1EnvVar
{
Name = "GENESIS_JSON",
Value = gethInfo.GenesisJsonBase64
Value = info.GenesisJsonBase64
}
}
};
}
private Dictionary<string, string> CreateSelector()
private Dictionary<string, string> CreateBootstrapSelector()
{
return new Dictionary<string, string> { { "test-gethb", "dtest-gethb" } };
}
private Dictionary<string, string> CreateCompanionSelector()
{
return new Dictionary<string, string> { { "test-gethc", "dtest-gethc" } };
}
}
}

27
CodexDistTestCore/Marketplace/MarketplaceAccess.cs
View File

@ -17,38 +17,37 @@ namespace CodexDistTestCore.Marketplace
private readonly MarketplaceController marketplaceController;
private readonly TestLog log;
private readonly CodexNodeGroup group;
private readonly GethCompanionNodeContainer gethCompanionNodeContainer;
private string account = string.Empty;
private readonly GethCompanionNodeContainer container;
public MarketplaceAccess(
K8sManager k8sManager,
MarketplaceController marketplaceController,
TestLog log,
CodexNodeGroup group,
GethCompanionNodeContainer gethCompanionNodeContainer)
CodexNodeGroup group,
GethCompanionNodeContainer container)
{
this.k8sManager = k8sManager;
this.marketplaceController = marketplaceController;
this.log = log;
this.group = group;
this.gethCompanionNodeContainer = gethCompanionNodeContainer;
this.container = container;
}
public void Initialize()
{
EnsureAccount();
marketplaceController.AddToBalance(account, group.Origin.MarketplaceConfig!.InitialBalance);
marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance);
log.Log($"Initialized Geth companion node with account '{account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}");
log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}");
}
public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates)
public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
{
throw new NotImplementedException();
}
public void MakeStorageAvailable(ByteSize size, float pricePerMBPerSecond, float collateral)
public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral)
{
throw new NotImplementedException();
}
@ -66,28 +65,28 @@ namespace CodexDistTestCore.Marketplace
private void EnsureAccount()
{
FetchAccount();
if (string.IsNullOrEmpty(account))
if (string.IsNullOrEmpty(container.Account))
{
Thread.Sleep(TimeSpan.FromSeconds(15));
FetchAccount();
}
Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure.");
Assert.That(container.Account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure.");
}
private void FetchAccount()
{
account = k8sManager.ExecuteCommand(group.PodInfo!, gethCompanionNodeContainer.Name, "cat", GethDockerImage.AccountFilename);
container.Account = k8sManager.ExecuteCommand(group.PodInfo!, container.Name, "cat", GethDockerImage.AccountFilename);
}
}
public class MarketplaceUnavailable : IMarketplaceAccess
{
public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates)
public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
{
Unavailable();
}
public void MakeStorageAvailable(ByteSize size, float pricePerMBPerSecond, float collateral)
public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral)
{
Unavailable();
}

92
CodexDistTestCore/Marketplace/MarketplaceController.cs
View File

@ -7,9 +7,9 @@ namespace CodexDistTestCore.Marketplace
{
private readonly TestLog log;
private readonly K8sManager k8sManager;
private GethInfo? gethBootstrapNode;
private string bootstrapAccount = string.Empty;
private string bootstrapGenesisJson = string.Empty;
private readonly NumberSource companionGroupNumberSource = new NumberSource(0);
private List<GethCompanionGroup> companionGroups = new List<GethCompanionGroup>();
private GethBootstrapInfo? bootstrapInfo;
public MarketplaceController(TestLog log, K8sManager k8sManager)
{
@ -17,30 +17,47 @@ namespace CodexDistTestCore.Marketplace
this.k8sManager = k8sManager;
}
public GethInfo BringOnlineMarketplace(OfflineCodexNodes offline)
public GethCompanionGroup BringOnlineMarketplace(OfflineCodexNodes offline)
{
if (gethBootstrapNode != null) return gethBootstrapNode;
if (bootstrapInfo == null)
{
BringOnlineBootstrapNode();
}
log.Log("Starting Geth bootstrap node...");
gethBootstrapNode = k8sManager.BringOnlineGethBootstrapNode();
ExtractAccountAndGenesisJson();
log.Log($"Geth boothstrap node started. Initializing companions for {offline.NumberOfNodes} Codex nodes.");
log.Log($"Initializing companions for {offline.NumberOfNodes} Codex nodes.");
var group = new GethCompanionGroup(companionGroupNumberSource.GetNextNumber(), CreateCompanionContainers(offline));
group.Pod = k8sManager.BringOnlineGethCompanionGroup(bootstrapInfo!, group);
companionGroups.Add(group);
return gethBootstrapNode;
log.Log("Initialized companion nodes.");
return group;
}
private void BringOnlineBootstrapNode()
{
log.Log("Starting Geth bootstrap node...");
var spec = k8sManager.CreateGethBootstrapNodeSpec();
var pod = k8sManager.BringOnlineGethBootstrapNode(spec);
var (account, genesisJson) = ExtractAccountAndGenesisJson();
bootstrapInfo = new GethBootstrapInfo(spec, pod, account, genesisJson);
log.Log($"Geth boothstrap node started.");
}
private GethCompanionNodeContainer? CreateGethNodeContainer(OfflineCodexNodes offline, int n)
private GethCompanionNodeContainer[] CreateCompanionContainers(OfflineCodexNodes offline)
{
var numberSource = new NumberSource(8080);
var result = new List<GethCompanionNodeContainer>();
for (var i = 0; i < offline.NumberOfNodes; i++) result.Add(CreateGethNodeContainer(numberSource, i));
return result.ToArray();
}
private GethCompanionNodeContainer CreateGethNodeContainer(NumberSource numberSource, int n)
{
return new GethCompanionNodeContainer(
name: $"geth-node{n}",
servicePort: numberSource.GetNextServicePort(),
servicePortName: numberSource.GetNextServicePortName(),
apiPort: codexPortSource.GetNextNumber(),
rpcPort: codexPortSource.GetNextNumber(),
apiPort: numberSource.GetNextNumber(),
rpcPort: numberSource.GetNextNumber(),
containerPortName: $"geth-{n}"
);
}
@ -53,47 +70,50 @@ namespace CodexDistTestCore.Marketplace
throw new NotImplementedException();
}
private void ExtractAccountAndGenesisJson()
private (string, string) ExtractAccountAndGenesisJson()
{
FetchAccountAndGenesisJson();
if (string.IsNullOrEmpty(bootstrapAccount) || string.IsNullOrEmpty(bootstrapGenesisJson))
var (account, genesisJson) = FetchAccountAndGenesisJson();
if (string.IsNullOrEmpty(account) || string.IsNullOrEmpty(genesisJson))
{
Thread.Sleep(TimeSpan.FromSeconds(15));
FetchAccountAndGenesisJson();
(account, genesisJson) = FetchAccountAndGenesisJson();
}
Assert.That(bootstrapAccount, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure.");
Assert.That(bootstrapGenesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure.");
Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure.");
Assert.That(genesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure.");
gethBootstrapNode!.GenesisJsonBase64 = Convert.ToBase64String(Encoding.ASCII.GetBytes(bootstrapGenesisJson));
var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson));
log.Log($"Initialized geth bootstrap node with account '{bootstrapAccount}'");
log.Log($"Initialized geth bootstrap node with account '{account}'");
return (account, encoded);
}
private void FetchAccountAndGenesisJson()
private (string, string) FetchAccountAndGenesisJson()
{
bootstrapAccount = ExecuteCommand("cat", GethDockerImage.AccountFilename);
bootstrapGenesisJson = ExecuteCommand("cat", GethDockerImage.GenesisFilename);
var bootstrapAccount = ExecuteCommand("cat", GethDockerImage.AccountFilename);
var bootstrapGenesisJson = ExecuteCommand("cat", GethDockerImage.GenesisFilename);
return (bootstrapAccount, bootstrapGenesisJson);
}
private string ExecuteCommand(string command, params string[] arguments)
{
return k8sManager.ExecuteCommand(gethBootstrapNode!.BootstrapPod, K8sGethBoostrapSpecs.ContainerName, command, arguments);
return k8sManager.ExecuteCommand(bootstrapInfo!.Pod, K8sGethBoostrapSpecs.ContainerName, command, arguments);
}
}
public class GethInfo
public class GethBootstrapInfo
{
public GethInfo(K8sGethBoostrapSpecs spec, PodInfo bootstrapPod, PodInfo companionPod)
public GethBootstrapInfo(K8sGethBoostrapSpecs spec, PodInfo pod, string account, string genesisJsonBase64)
{
Spec = spec;
BootstrapPod = bootstrapPod;
CompanionPod = companionPod;
Pod = pod;
Account = account;
GenesisJsonBase64 = genesisJsonBase64;
}
public K8sGethBoostrapSpecs Spec { get; }
public PodInfo BootstrapPod { get; }
public PodInfo CompanionPod { get; }
public string GenesisJsonBase64 { get; set; } = string.Empty;
public PodInfo Pod { get; }
public string Account { get; }
public string GenesisJsonBase64 { get; }
}
}