status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

splits geth nodes into their own pod

This commit is contained in:
benbierens 2023-04-11 12:06:33 +02:00
parent c977e37ab8
commit d33eb53003
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
7 changed files with 180 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CodexDistTestCore/CodexNodeGroup.cs
View File

@ -46,7 +46,7 @@ namespace CodexDistTestCore
public V1Deployment? Deployment { get; set; } public V1Deployment? Deployment { get; set; }
public V1Service? Service { get; set; } public V1Service? Service { get; set; }
public PodInfo? PodInfo { get; set; } public PodInfo? PodInfo { get; set; }
public GethInfo? GethInfo { get; set; } public GethCompanionGroup? GethCompanionGroup { get; set; }
public CodexNodeContainer[] GetContainers() public CodexNodeContainer[] GetContainers()
{ {

22
CodexDistTestCore/K8sManager.cs
View File

@ -34,7 +34,7 @@ namespace CodexDistTestCore
if (offline.MarketplaceConfig != null) if (offline.MarketplaceConfig != null)
{ {
group.GethInfo = marketplaceController.BringOnlineMarketplace(offline); group.GethCompanionGroup = marketplaceController.BringOnlineMarketplace(offline);
} }
K8s(k => k.BringOnline(group, offline)); K8s(k => k.BringOnline(group, offline));
@ -91,13 +91,21 @@ namespace CodexDistTestCore
return K8s(k => k.BringOnlinePrometheus(spec)); return K8s(k => k.BringOnlinePrometheus(spec));
} }
public GethInfo BringOnlineGethBootstrapNode() public K8sGethBoostrapSpecs CreateGethBootstrapNodeSpec()
{ {
var spec = new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); return new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort());
}
public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec)
{
return K8s(k => k.BringOnlineGethBootstrapNode(spec)); return K8s(k => k.BringOnlineGethBootstrapNode(spec));
} }
public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group)
{
return K8s(k => k.BringOnlineGethCompanionGroup(info, group));
}
public void DownloadAllMetrics() public void DownloadAllMetrics()
{ {
metricsAggregator.DownloadAllMetrics(); metricsAggregator.DownloadAllMetrics();
@ -110,15 +118,15 @@ namespace CodexDistTestCore
private void ConnectMarketplace(CodexNodeGroup group) private void ConnectMarketplace(CodexNodeGroup group)
{ {
foreach (var node in DowncastNodes(group)) for (var i = 0; i < group.Nodes.Length; i++)
{ {
ConnectMarketplace(group, node); ConnectMarketplace(group, group.Nodes[i], group.GethCompanionGroup!.Containers[i]);
} }
} }
private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node) private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node, GethCompanionNodeContainer container)
{ {
var access = new MarketplaceAccess(this, marketplaceController, log, group, node.Container.GethCompanionNodeContainer!); var access = new MarketplaceAccess(this, marketplaceController, log, group, container);
access.Initialize(); access.Initialize();
node.Marketplace = access; node.Marketplace = access;
} }

37
CodexDistTestCore/K8sOperations.cs
View File

@ -4,7 +4,6 @@ using CodexDistTestCore.Metrics;
using k8s; using k8s;
using k8s.Models; using k8s.Models;
using NUnit.Framework; using NUnit.Framework;
using System.Numerics;
namespace CodexDistTestCore namespace CodexDistTestCore
{ {
@ -78,7 +77,7 @@ namespace CodexDistTestCore
return new PrometheusInfo(spec.ServicePort, FetchNewPod()); return new PrometheusInfo(spec.ServicePort, FetchNewPod());
} }
public GethInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec)
{ {
EnsureTestNamespace(); EnsureTestNamespace();
@ -86,7 +85,17 @@ namespace CodexDistTestCore
CreateGethBootstrapService(spec); CreateGethBootstrapService(spec);
WaitUntilGethBootstrapOnline(spec); WaitUntilGethBootstrapOnline(spec);
return new GethInfo(spec, FetchNewPod()); return FetchNewPod();
}
public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group)
{
EnsureTestNamespace();
CreateGethCompanionDeployment(info, group);
WaitUntilGethCompanionGroupOnline(info.Spec, group);
return FetchNewPod();
} }
private void FetchPodInfo(CodexNodeGroup online) private void FetchPodInfo(CodexNodeGroup online)
@ -148,7 +157,12 @@ namespace CodexDistTestCore
private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec)
{ {
WaitUntilDeploymentOnline(spec.GetDeploymentName()); WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName());
}
private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group)
{
WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group));
} }
private void WaitUntilDeploymentOnline(string deploymentName) private void WaitUntilDeploymentOnline(string deploymentName)
@ -212,11 +226,6 @@ namespace CodexDistTestCore
TargetPort = container.ContainerPortName, TargetPort = container.ContainerPortName,
NodePort = container.ServicePort NodePort = container.ServicePort
}); });
if (container.GethCompanionNodeContainer != null)
{
result.Add(container.GethCompanionNodeContainer.CreateServicePort());
}
} }
return result; return result;
} }
@ -303,11 +312,6 @@ namespace CodexDistTestCore
}, },
Env = dockerImage.CreateEnvironmentVariables(offline, container) Env = dockerImage.CreateEnvironmentVariables(offline, container)
}); });
if (container.GethCompanionNodeContainer != null)
{
result.Add(container.GethCompanionNodeContainer.CreateDeploymentContainer(online.GethInfo!));
}
} }
return result; return result;
@ -330,6 +334,11 @@ namespace CodexDistTestCore
client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace);
} }
private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group)
{
client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace);
}
#endregion #endregion
#region Namespace management #region Namespace management

18
CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs
View File

@ -1,7 +1,18 @@
using k8s.Models; namespace CodexDistTestCore.Marketplace
namespace CodexDistTestCore.Marketplace
{ {
public class GethCompanionGroup
{
public GethCompanionGroup(int number, GethCompanionNodeContainer[] containers)
{
Number = number;
Containers = containers;
}
public int Number { get; }
public GethCompanionNodeContainer[] Containers { get; }
public PodInfo? Pod { get; set; }
}
public class GethCompanionNodeContainer public class GethCompanionNodeContainer
{ {
public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName) public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName)
@ -17,5 +28,6 @@ namespace CodexDistTestCore.Marketplace
public int RpcPort { get; } public int RpcPort { get; }
public string ContainerPortName { get; } public string ContainerPortName { get; }
public string Account { get; set; } = string.Empty;
} }
} }

69
CodexDistTestCore/Marketplace/K8sGethSpecs.cs
View File

@ -1,6 +1,5 @@
using CodexDistTestCore.Config; using CodexDistTestCore.Config;
using k8s.Models; using k8s.Models;
using System.Xml.Linq;
namespace CodexDistTestCore.Marketplace namespace CodexDistTestCore.Marketplace
{ {
@ -24,11 +23,16 @@ namespace CodexDistTestCore.Marketplace
public int ServicePort { get; } public int ServicePort { get; }
public string GetDeploymentName() public string GetBootstrapDeploymentName()
{ {
return "test-gethb"; return "test-gethb";
} }
public string GetCompanionDeploymentName(GethCompanionGroup group)
{
return "test-geth" + group.Number;
}
public V1Deployment CreateGethBootstrapDeployment() public V1Deployment CreateGethBootstrapDeployment()
{ {
var deploymentSpec = new V1Deployment var deploymentSpec = new V1Deployment
@ -36,7 +40,7 @@ namespace CodexDistTestCore.Marketplace
ApiVersion = "apps/v1", ApiVersion = "apps/v1",
Metadata = new V1ObjectMeta Metadata = new V1ObjectMeta
{ {
Name = GetDeploymentName(), Name = GetBootstrapDeploymentName(),
NamespaceProperty = K8sCluster.K8sNamespace NamespaceProperty = K8sCluster.K8sNamespace
}, },
Spec = new V1DeploymentSpec Spec = new V1DeploymentSpec
@ -44,13 +48,13 @@ namespace CodexDistTestCore.Marketplace
Replicas = 1, Replicas = 1,
Selector = new V1LabelSelector Selector = new V1LabelSelector
{ {
MatchLabels = CreateSelector() MatchLabels = CreateBootstrapSelector()
}, },
Template = new V1PodTemplateSpec Template = new V1PodTemplateSpec
{ {
Metadata = new V1ObjectMeta Metadata = new V1ObjectMeta
{ {
Labels = CreateSelector() Labels = CreateBootstrapSelector()
}, },
Spec = new V1PodSpec Spec = new V1PodSpec
{ {
@ -109,7 +113,7 @@ namespace CodexDistTestCore.Marketplace
Spec = new V1ServiceSpec Spec = new V1ServiceSpec
{ {
Type = "NodePort", Type = "NodePort",
Selector = CreateSelector(), Selector = CreateBootstrapSelector(),
Ports = new List<V1ServicePort> Ports = new List<V1ServicePort>
{ {
new V1ServicePort new V1ServicePort
@ -127,18 +131,52 @@ namespace CodexDistTestCore.Marketplace
return serviceSpec; return serviceSpec;
} }
public V1Deployment CreateGethCompanionDeployment(GethInfo gethInfo) public V1Deployment CreateGethCompanionDeployment(GethCompanionGroup group, GethBootstrapInfo info)
{
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = new V1ObjectMeta
{
Name = GetCompanionDeploymentName(group),
NamespaceProperty = K8sCluster.K8sNamespace
},
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = CreateCompanionSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = CreateCompanionSelector()
},
Spec = new V1PodSpec
{
Containers = group.Containers.Select(c => CreateContainer(c, info)).ToList()
}
}
}
};
return deploymentSpec;
}
private static V1Container CreateContainer(GethCompanionNodeContainer container, GethBootstrapInfo info)
{ {
return new V1Container return new V1Container
{ {
Name = Name, Name = container.Name,
Image = GethDockerImage.Image, Image = GethDockerImage.Image,
Ports = new List<V1ContainerPort> Ports = new List<V1ContainerPort>
{ {
new V1ContainerPort new V1ContainerPort
{ {
ContainerPort = ApiPort, ContainerPort = container.ApiPort,
Name = ContainerPortName Name = container.ContainerPortName
} }
}, },
// todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64
@ -147,20 +185,25 @@ namespace CodexDistTestCore.Marketplace
new V1EnvVar new V1EnvVar
{ {
Name = "GETH_ARGS", Name = "GETH_ARGS",
Value = $"--port {ApiPort} --discovery.port {ApiPort} --authrpc.port {RpcPort}" Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.RpcPort}"
}, },
new V1EnvVar new V1EnvVar
{ {
Name = "GENESIS_JSON", Name = "GENESIS_JSON",
Value = gethInfo.GenesisJsonBase64 Value = info.GenesisJsonBase64
} }
} }
}; };
} }
private Dictionary<string, string> CreateSelector() private Dictionary<string, string> CreateBootstrapSelector()
{ {
return new Dictionary<string, string> { { "test-gethb", "dtest-gethb" } }; return new Dictionary<string, string> { { "test-gethb", "dtest-gethb" } };
} }
private Dictionary<string, string> CreateCompanionSelector()
{
return new Dictionary<string, string> { { "test-gethc", "dtest-gethc" } };
}
} }
} }

25
CodexDistTestCore/Marketplace/MarketplaceAccess.cs
View File

@ -17,38 +17,37 @@ namespace CodexDistTestCore.Marketplace
private readonly MarketplaceController marketplaceController; private readonly MarketplaceController marketplaceController;
private readonly TestLog log; private readonly TestLog log;
private readonly CodexNodeGroup group; private readonly CodexNodeGroup group;
private readonly GethCompanionNodeContainer gethCompanionNodeContainer; private readonly GethCompanionNodeContainer container;
private string account = string.Empty;
public MarketplaceAccess( public MarketplaceAccess(
K8sManager k8sManager, K8sManager k8sManager,
MarketplaceController marketplaceController, MarketplaceController marketplaceController,
TestLog log, TestLog log,
CodexNodeGroup group, CodexNodeGroup group,
GethCompanionNodeContainer gethCompanionNodeContainer) GethCompanionNodeContainer container)
{ {
this.k8sManager = k8sManager; this.k8sManager = k8sManager;
this.marketplaceController = marketplaceController; this.marketplaceController = marketplaceController;
this.log = log; this.log = log;
this.group = group; this.group = group;
this.gethCompanionNodeContainer = gethCompanionNodeContainer; this.container = container;
} }
public void Initialize() public void Initialize()
{ {
EnsureAccount(); EnsureAccount();
marketplaceController.AddToBalance(account, group.Origin.MarketplaceConfig!.InitialBalance); marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance);
log.Log($"Initialized Geth companion node with account '{account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}");
} }
public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates) public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
public void MakeStorageAvailable(ByteSize size, float pricePerMBPerSecond, float collateral) public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
@ -66,28 +65,28 @@ namespace CodexDistTestCore.Marketplace
private void EnsureAccount() private void EnsureAccount()
{ {
FetchAccount(); FetchAccount();
if (string.IsNullOrEmpty(account)) if (string.IsNullOrEmpty(container.Account))
{ {
Thread.Sleep(TimeSpan.FromSeconds(15)); Thread.Sleep(TimeSpan.FromSeconds(15));
FetchAccount(); FetchAccount();
} }
Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); Assert.That(container.Account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure.");
} }
private void FetchAccount() private void FetchAccount()
{ {
account = k8sManager.ExecuteCommand(group.PodInfo!, gethCompanionNodeContainer.Name, "cat", GethDockerImage.AccountFilename); container.Account = k8sManager.ExecuteCommand(group.PodInfo!, container.Name, "cat", GethDockerImage.AccountFilename);
} }
} }
public class MarketplaceUnavailable : IMarketplaceAccess public class MarketplaceUnavailable : IMarketplaceAccess
{ {
public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates) public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
{ {
Unavailable(); Unavailable();
} }
public void MakeStorageAvailable(ByteSize size, float pricePerMBPerSecond, float collateral) public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral)
{ {
Unavailable(); Unavailable();
} }

98
CodexDistTestCore/Marketplace/MarketplaceController.cs
View File

@ -7,9 +7,9 @@ namespace CodexDistTestCore.Marketplace
{ {
private readonly TestLog log; private readonly TestLog log;
private readonly K8sManager k8sManager; private readonly K8sManager k8sManager;
private GethInfo? gethBootstrapNode; private readonly NumberSource companionGroupNumberSource = new NumberSource(0);
private string bootstrapAccount = string.Empty; private List<GethCompanionGroup> companionGroups = new List<GethCompanionGroup>();
private string bootstrapGenesisJson = string.Empty; private GethBootstrapInfo? bootstrapInfo;
public MarketplaceController(TestLog log, K8sManager k8sManager) public MarketplaceController(TestLog log, K8sManager k8sManager)
{ {
@ -17,30 +17,47 @@ namespace CodexDistTestCore.Marketplace
this.k8sManager = k8sManager; this.k8sManager = k8sManager;
} }
public GethInfo BringOnlineMarketplace(OfflineCodexNodes offline) public GethCompanionGroup BringOnlineMarketplace(OfflineCodexNodes offline)
{ {
if (gethBootstrapNode != null) return gethBootstrapNode; if (bootstrapInfo == null)
{
log.Log("Starting Geth bootstrap node..."); BringOnlineBootstrapNode();
gethBootstrapNode = k8sManager.BringOnlineGethBootstrapNode();
ExtractAccountAndGenesisJson();
log.Log($"Geth boothstrap node started. Initializing companions for {offline.NumberOfNodes} Codex nodes.");
return gethBootstrapNode;
} }
log.Log($"Initializing companions for {offline.NumberOfNodes} Codex nodes.");
private GethCompanionNodeContainer? CreateGethNodeContainer(OfflineCodexNodes offline, int n) var group = new GethCompanionGroup(companionGroupNumberSource.GetNextNumber(), CreateCompanionContainers(offline));
group.Pod = k8sManager.BringOnlineGethCompanionGroup(bootstrapInfo!, group);
companionGroups.Add(group);
log.Log("Initialized companion nodes.");
return group;
}
private void BringOnlineBootstrapNode()
{
log.Log("Starting Geth bootstrap node...");
var spec = k8sManager.CreateGethBootstrapNodeSpec();
var pod = k8sManager.BringOnlineGethBootstrapNode(spec);
var (account, genesisJson) = ExtractAccountAndGenesisJson();
bootstrapInfo = new GethBootstrapInfo(spec, pod, account, genesisJson);
log.Log($"Geth boothstrap node started.");
}
private GethCompanionNodeContainer[] CreateCompanionContainers(OfflineCodexNodes offline)
{
var numberSource = new NumberSource(8080);
var result = new List<GethCompanionNodeContainer>();
for (var i = 0; i < offline.NumberOfNodes; i++) result.Add(CreateGethNodeContainer(numberSource, i));
return result.ToArray();
}
private GethCompanionNodeContainer CreateGethNodeContainer(NumberSource numberSource, int n)
{ {
return new GethCompanionNodeContainer( return new GethCompanionNodeContainer(
name: $"geth-node{n}", name: $"geth-node{n}",
servicePort: numberSource.GetNextServicePort(), apiPort: numberSource.GetNextNumber(),
servicePortName: numberSource.GetNextServicePortName(), rpcPort: numberSource.GetNextNumber(),
apiPort: codexPortSource.GetNextNumber(),
rpcPort: codexPortSource.GetNextNumber(),
containerPortName: $"geth-{n}" containerPortName: $"geth-{n}"
); );
} }
@ -53,47 +70,50 @@ namespace CodexDistTestCore.Marketplace
throw new NotImplementedException(); throw new NotImplementedException();
} }
private void ExtractAccountAndGenesisJson() private (string, string) ExtractAccountAndGenesisJson()
{ {
FetchAccountAndGenesisJson(); var (account, genesisJson) = FetchAccountAndGenesisJson();
if (string.IsNullOrEmpty(bootstrapAccount) || string.IsNullOrEmpty(bootstrapGenesisJson)) if (string.IsNullOrEmpty(account) || string.IsNullOrEmpty(genesisJson))
{ {
Thread.Sleep(TimeSpan.FromSeconds(15)); Thread.Sleep(TimeSpan.FromSeconds(15));
FetchAccountAndGenesisJson(); (account, genesisJson) = FetchAccountAndGenesisJson();
} }
Assert.That(bootstrapAccount, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure."); Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure.");
Assert.That(bootstrapGenesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure."); Assert.That(genesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure.");
gethBootstrapNode!.GenesisJsonBase64 = Convert.ToBase64String(Encoding.ASCII.GetBytes(bootstrapGenesisJson)); var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson));
log.Log($"Initialized geth bootstrap node with account '{bootstrapAccount}'"); log.Log($"Initialized geth bootstrap node with account '{account}'");
return (account, encoded);
} }
private void FetchAccountAndGenesisJson() private (string, string) FetchAccountAndGenesisJson()
{ {
bootstrapAccount = ExecuteCommand("cat", GethDockerImage.AccountFilename); var bootstrapAccount = ExecuteCommand("cat", GethDockerImage.AccountFilename);
bootstrapGenesisJson = ExecuteCommand("cat", GethDockerImage.GenesisFilename); var bootstrapGenesisJson = ExecuteCommand("cat", GethDockerImage.GenesisFilename);
return (bootstrapAccount, bootstrapGenesisJson);
} }
private string ExecuteCommand(string command, params string[] arguments) private string ExecuteCommand(string command, params string[] arguments)
{ {
return k8sManager.ExecuteCommand(gethBootstrapNode!.BootstrapPod, K8sGethBoostrapSpecs.ContainerName, command, arguments); return k8sManager.ExecuteCommand(bootstrapInfo!.Pod, K8sGethBoostrapSpecs.ContainerName, command, arguments);
} }
} }
public class GethInfo public class GethBootstrapInfo
{ {
public GethInfo(K8sGethBoostrapSpecs spec, PodInfo bootstrapPod, PodInfo companionPod) public GethBootstrapInfo(K8sGethBoostrapSpecs spec, PodInfo pod, string account, string genesisJsonBase64)
{ {
Spec = spec; Spec = spec;
BootstrapPod = bootstrapPod; Pod = pod;
CompanionPod = companionPod; Account = account;
GenesisJsonBase64 = genesisJsonBase64;
} }
public K8sGethBoostrapSpecs Spec { get; } public K8sGethBoostrapSpecs Spec { get; }
public PodInfo BootstrapPod { get; } public PodInfo Pod { get; }
public PodInfo CompanionPod { get; } public string Account { get; }
public string GenesisJsonBase64 { get; set; } = string.Empty; public string GenesisJsonBase64 { get; }
} }
} }