2023-09-20 10:02:32 +00:00
|
|
|
|
using CodexContractsPlugin;
|
2023-10-24 08:29:07 +00:00
|
|
|
|
using CodexDiscordBotPlugin;
|
2023-09-20 10:02:32 +00:00
|
|
|
|
using CodexPlugin;
|
|
|
|
|
using Core;
|
|
|
|
|
using GethPlugin;
|
2023-11-12 09:07:23 +00:00
|
|
|
|
using KubernetesWorkflow.Types;
|
2023-06-26 13:37:16 +00:00
|
|
|
|
using Logging;
|
2023-09-20 10:02:32 +00:00
|
|
|
|
using MetricsPlugin;
|
2023-06-22 08:17:12 +00:00
|
|
|
|
|
|
|
|
|
namespace CodexNetDeployer
|
|
|
|
|
{
|
|
|
|
|
public class Deployer
|
|
|
|
|
{
|
|
|
|
|
private readonly Configuration config;
|
2023-08-24 09:32:32 +00:00
|
|
|
|
private readonly PeerConnectivityChecker peerConnectivityChecker;
|
2023-09-20 10:02:32 +00:00
|
|
|
|
private readonly EntryPoint entryPoint;
|
2023-09-28 09:31:09 +00:00
|
|
|
|
private readonly LocalCodexBuilder localCodexBuilder;
|
2023-06-22 08:17:12 +00:00
|
|
|
|
|
|
|
|
|
public Deployer(Configuration config)
|
|
|
|
|
{
|
|
|
|
|
this.config = config;
|
2023-08-24 09:32:32 +00:00
|
|
|
|
peerConnectivityChecker = new PeerConnectivityChecker();
|
2023-09-28 09:31:09 +00:00
|
|
|
|
localCodexBuilder = new LocalCodexBuilder(new ConsoleLog(), config.CodexLocalRepoPath);
|
2023-09-20 11:56:01 +00:00
|
|
|
|
|
|
|
|
|
ProjectPlugin.Load<CodexPlugin.CodexPlugin>();
|
|
|
|
|
ProjectPlugin.Load<CodexContractsPlugin.CodexContractsPlugin>();
|
|
|
|
|
ProjectPlugin.Load<GethPlugin.GethPlugin>();
|
|
|
|
|
ProjectPlugin.Load<MetricsPlugin.MetricsPlugin>();
|
2023-10-24 08:29:07 +00:00
|
|
|
|
ProjectPlugin.Load<CodexDiscordBotPlugin.CodexDiscordBotPlugin>();
|
2023-09-20 10:02:32 +00:00
|
|
|
|
entryPoint = CreateEntryPoint(new NullLog());
|
2023-06-22 08:17:12 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
public void AnnouncePlugins()
|
2023-06-22 08:33:21 +00:00
|
|
|
|
{
|
2023-09-20 10:02:32 +00:00
|
|
|
|
var ep = CreateEntryPoint(new ConsoleLog());
|
2023-06-22 08:33:21 +00:00
|
|
|
|
|
2023-09-28 09:31:09 +00:00
|
|
|
|
localCodexBuilder.Intialize();
|
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
Log("Using plugins:" + Environment.NewLine);
|
|
|
|
|
var metadata = ep.GetPluginMetadata();
|
2023-09-21 06:49:09 +00:00
|
|
|
|
var longestKey = metadata.Keys.Max(k => k.Length);
|
2023-09-20 10:02:32 +00:00
|
|
|
|
foreach (var entry in metadata)
|
|
|
|
|
{
|
2023-09-21 06:49:09 +00:00
|
|
|
|
Console.Write(entry.Key);
|
|
|
|
|
Console.CursorLeft = longestKey + 5;
|
|
|
|
|
Console.WriteLine($"= {entry.Value}");
|
2023-09-20 10:02:32 +00:00
|
|
|
|
}
|
2024-02-22 13:41:07 +00:00
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
Log("");
|
|
|
|
|
}
|
2023-06-22 08:33:21 +00:00
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
public CodexDeployment Deploy()
|
|
|
|
|
{
|
2023-09-28 09:31:09 +00:00
|
|
|
|
localCodexBuilder.Build();
|
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
Log("Initializing...");
|
2023-10-16 09:19:57 +00:00
|
|
|
|
var startUtc = DateTime.UtcNow;
|
2023-09-20 10:02:32 +00:00
|
|
|
|
var ci = entryPoint.CreateInterface();
|
2023-06-22 08:33:21 +00:00
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
Log("Deploying Geth instance...");
|
2023-10-23 11:08:49 +00:00
|
|
|
|
var gethDeployment = DeployGeth(ci);
|
2023-09-20 10:02:32 +00:00
|
|
|
|
var gethNode = ci.WrapGethDeployment(gethDeployment);
|
2023-06-22 12:37:37 +00:00
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
Log("Geth started. Deploying Codex contracts...");
|
|
|
|
|
var contractsDeployment = ci.DeployCodexContracts(gethNode);
|
2023-10-30 12:30:14 +00:00
|
|
|
|
var contracts = ci.WrapCodexContractsDeployment(gethNode, contractsDeployment);
|
2023-09-20 10:02:32 +00:00
|
|
|
|
Log("Codex contracts deployed.");
|
2023-06-30 07:54:13 +00:00
|
|
|
|
|
2023-06-22 12:37:37 +00:00
|
|
|
|
Log("Starting Codex nodes...");
|
2023-09-20 10:02:32 +00:00
|
|
|
|
var codexStarter = new CodexNodeStarter(config, ci, gethNode, contracts, config.NumberOfValidators!.Value);
|
2023-08-24 09:32:32 +00:00
|
|
|
|
var startResults = new List<CodexNodeStartResult>();
|
2023-06-22 12:37:37 +00:00
|
|
|
|
for (var i = 0; i < config.NumberOfCodexNodes; i++)
|
2023-06-22 08:33:21 +00:00
|
|
|
|
{
|
2023-08-24 09:32:32 +00:00
|
|
|
|
var result = codexStarter.Start(i);
|
|
|
|
|
if (result != null) startResults.Add(result);
|
2023-06-22 08:33:21 +00:00
|
|
|
|
}
|
2023-06-23 06:44:27 +00:00
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
Log("Codex nodes started.");
|
|
|
|
|
var metricsService = StartMetricsService(ci, startResults);
|
2023-07-11 08:59:41 +00:00
|
|
|
|
|
2023-08-24 09:32:32 +00:00
|
|
|
|
CheckPeerConnectivity(startResults);
|
2023-08-28 09:53:59 +00:00
|
|
|
|
CheckContainerRestarts(startResults);
|
2023-08-24 09:32:32 +00:00
|
|
|
|
|
2023-10-25 07:14:35 +00:00
|
|
|
|
var codexInstances = CreateCodexInstances(startResults);
|
2023-10-24 08:29:07 +00:00
|
|
|
|
|
2023-12-11 10:09:29 +00:00
|
|
|
|
var discordBotContainer = DeployDiscordBot(ci, gethDeployment, contractsDeployment);
|
2023-10-24 08:29:07 +00:00
|
|
|
|
|
2024-02-22 13:41:07 +00:00
|
|
|
|
return new CodexDeployment(codexInstances, gethDeployment, contractsDeployment, metricsService,
|
|
|
|
|
discordBotContainer, CreateMetadata(startUtc), config.DeployId);
|
2023-06-22 08:33:21 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
private EntryPoint CreateEntryPoint(ILog log)
|
2023-06-22 08:17:12 +00:00
|
|
|
|
{
|
2023-06-27 06:29:39 +00:00
|
|
|
|
var kubeConfig = GetKubeConfig(config.KubeConfigFile);
|
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
var configuration = new KubernetesWorkflow.Configuration(
|
|
|
|
|
kubeConfig,
|
2023-10-16 09:19:57 +00:00
|
|
|
|
operationTimeout: TimeSpan.FromMinutes(10),
|
2023-10-07 05:40:20 +00:00
|
|
|
|
retryDelay: TimeSpan.FromSeconds(10),
|
2023-09-20 10:02:32 +00:00
|
|
|
|
kubernetesNamespace: config.KubeNamespace);
|
|
|
|
|
|
2023-10-16 09:19:57 +00:00
|
|
|
|
var result = new EntryPoint(log, configuration, string.Empty, new FastHttpTimeSet());
|
2024-02-22 13:41:07 +00:00
|
|
|
|
configuration.Hooks = new K8sHook(config.TestsTypePodLabel, config.DeployId, result.GetPluginMetadata());
|
2023-09-22 09:10:22 +00:00
|
|
|
|
|
2023-09-26 12:32:28 +00:00
|
|
|
|
return result;
|
2023-06-22 08:33:21 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-10-23 11:08:49 +00:00
|
|
|
|
private GethDeployment DeployGeth(CoreInterface ci)
|
|
|
|
|
{
|
|
|
|
|
return ci.DeployGeth(s =>
|
|
|
|
|
{
|
|
|
|
|
s.IsMiner();
|
|
|
|
|
s.WithName("geth");
|
2023-10-23 11:19:57 +00:00
|
|
|
|
|
|
|
|
|
if (config.IsPublicTestNet)
|
|
|
|
|
{
|
|
|
|
|
s.AsPublicTestNet(new GethTestNetConfig(
|
2023-10-27 06:33:07 +00:00
|
|
|
|
discoveryPort: config.PublicGethDiscPort,
|
|
|
|
|
listenPort: config.PublicGethListenPort
|
2023-10-23 11:19:57 +00:00
|
|
|
|
));
|
|
|
|
|
}
|
2023-10-23 11:08:49 +00:00
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
private RunningPod? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
|
2024-02-22 13:41:07 +00:00
|
|
|
|
CodexContractsDeployment contractsDeployment)
|
2023-10-25 07:14:35 +00:00
|
|
|
|
{
|
|
|
|
|
if (!config.DeployDiscordBot) return null;
|
|
|
|
|
Log("Deploying Discord bot...");
|
|
|
|
|
|
2023-12-11 10:09:29 +00:00
|
|
|
|
var addr = gethDeployment.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag);
|
|
|
|
|
var info = new DiscordBotGethInfo(
|
|
|
|
|
host: addr.Host,
|
|
|
|
|
port: addr.Port,
|
|
|
|
|
privKey: gethDeployment.Account.PrivateKey,
|
|
|
|
|
marketplaceAddress: contractsDeployment.MarketplaceAddress,
|
|
|
|
|
tokenAddress: contractsDeployment.TokenAddress,
|
|
|
|
|
abi: contractsDeployment.Abi
|
|
|
|
|
);
|
|
|
|
|
|
2023-10-25 07:14:35 +00:00
|
|
|
|
var rc = ci.DeployCodexDiscordBot(new DiscordBotStartupConfig(
|
|
|
|
|
name: "discordbot-" + config.DeploymentName,
|
|
|
|
|
token: config.DiscordBotToken,
|
|
|
|
|
serverName: config.DiscordBotServerName,
|
2023-10-25 08:38:21 +00:00
|
|
|
|
adminRoleName: config.DiscordBotAdminRoleName,
|
2023-11-08 10:49:21 +00:00
|
|
|
|
adminChannelName: config.DiscordBotAdminChannelName,
|
2023-12-11 10:09:29 +00:00
|
|
|
|
kubeNamespace: config.KubeNamespace,
|
2024-02-19 08:11:36 +00:00
|
|
|
|
gethInfo: info,
|
|
|
|
|
rewardChannelName: config.DiscordBotRewardChannelName)
|
2023-10-25 09:53:33 +00:00
|
|
|
|
{
|
|
|
|
|
DataPath = config.DiscordBotDataPath
|
|
|
|
|
});
|
2023-10-25 07:14:35 +00:00
|
|
|
|
|
|
|
|
|
Log("Discord bot deployed.");
|
|
|
|
|
return rc;
|
|
|
|
|
}
|
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
private RunningPod? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
|
2023-07-11 08:59:41 +00:00
|
|
|
|
{
|
2023-11-21 08:03:52 +00:00
|
|
|
|
if (!config.MetricsScraper || !startResults.Any()) return null;
|
2023-07-11 10:21:48 +00:00
|
|
|
|
|
2023-07-11 08:59:41 +00:00
|
|
|
|
Log("Starting metrics service...");
|
2023-08-13 07:07:23 +00:00
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
var runningContainer = ci.DeployMetricsCollector(startResults.Select(r => r.CodexNode).ToArray());
|
|
|
|
|
|
|
|
|
|
Log("Metrics service started.");
|
2023-08-13 07:07:23 +00:00
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
return runningContainer;
|
2023-07-11 08:59:41 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-10-25 07:14:35 +00:00
|
|
|
|
private CodexInstance[] CreateCodexInstances(List<CodexNodeStartResult> startResults)
|
|
|
|
|
{
|
2023-10-27 08:56:30 +00:00
|
|
|
|
// When freshly started, the Codex nodes are announcing themselves by an incorrect IP address.
|
|
|
|
|
// Only after fully initialized do they update to the provided NAT address.
|
|
|
|
|
// Therefore, we wait:
|
|
|
|
|
Thread.Sleep(TimeSpan.FromSeconds(5));
|
|
|
|
|
|
2023-10-25 07:14:35 +00:00
|
|
|
|
return startResults.Select(r => CreateCodexInstance(r.CodexNode)).ToArray();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private CodexInstance CreateCodexInstance(ICodexNode node)
|
|
|
|
|
{
|
2024-04-13 14:09:17 +00:00
|
|
|
|
return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo());
|
2023-10-25 07:14:35 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-06-27 06:29:39 +00:00
|
|
|
|
private string? GetKubeConfig(string kubeConfigFile)
|
|
|
|
|
{
|
|
|
|
|
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
|
|
|
|
|
return kubeConfigFile;
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-24 09:32:32 +00:00
|
|
|
|
private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
|
|
|
|
|
{
|
2023-11-21 08:03:52 +00:00
|
|
|
|
if (!config.CheckPeerConnection || !codexContainers.Any()) return;
|
2023-08-24 09:32:32 +00:00
|
|
|
|
|
2023-09-20 10:02:32 +00:00
|
|
|
|
Log("Starting peer connectivity check for deployed nodes...");
|
2023-08-24 09:32:32 +00:00
|
|
|
|
peerConnectivityChecker.CheckConnectivity(codexContainers);
|
|
|
|
|
Log("Check passed.");
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-28 09:53:59 +00:00
|
|
|
|
private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
|
|
|
|
|
{
|
|
|
|
|
var crashes = new List<RunningContainer>();
|
2023-09-20 10:02:32 +00:00
|
|
|
|
Log("Starting container crash check...");
|
2023-08-28 09:53:59 +00:00
|
|
|
|
foreach (var startResult in startResults)
|
|
|
|
|
{
|
2023-09-20 10:55:09 +00:00
|
|
|
|
var watcher = startResult.CodexNode.CrashWatcher;
|
2024-02-22 13:41:07 +00:00
|
|
|
|
if (watcher == null)
|
|
|
|
|
throw new Exception("Expected each CodexNode container to be created with a crash-watcher.");
|
2023-09-20 10:02:32 +00:00
|
|
|
|
if (watcher.HasContainerCrashed()) crashes.Add(startResult.CodexNode.Container);
|
2023-08-28 09:53:59 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!crashes.Any())
|
|
|
|
|
{
|
2023-09-20 10:02:32 +00:00
|
|
|
|
Log("Check passed.");
|
2023-08-28 09:53:59 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
2024-02-22 13:41:07 +00:00
|
|
|
|
Log(
|
|
|
|
|
$"Check failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
|
2023-08-28 09:53:59 +00:00
|
|
|
|
throw new Exception("Deployment failed: One or more containers crashed.");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-10-16 09:19:57 +00:00
|
|
|
|
private DeploymentMetadata CreateMetadata(DateTime startUtc)
|
2023-06-23 07:08:18 +00:00
|
|
|
|
{
|
|
|
|
|
return new DeploymentMetadata(
|
2023-10-23 08:19:52 +00:00
|
|
|
|
name: config.DeploymentName,
|
2023-10-16 09:19:57 +00:00
|
|
|
|
startUtc: startUtc,
|
|
|
|
|
finishedUtc: DateTime.UtcNow,
|
2023-06-23 07:08:18 +00:00
|
|
|
|
kubeNamespace: config.KubeNamespace,
|
|
|
|
|
numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
|
|
|
|
|
numberOfValidators: config.NumberOfValidators!.Value,
|
|
|
|
|
storageQuotaMB: config.StorageQuota!.Value,
|
2023-06-28 06:48:46 +00:00
|
|
|
|
codexLogLevel: config.CodexLogLevel,
|
|
|
|
|
initialTestTokens: config.InitialTestTokens,
|
|
|
|
|
minPrice: config.MinPrice,
|
|
|
|
|
maxCollateral: config.MaxCollateral,
|
2023-08-23 06:29:16 +00:00
|
|
|
|
maxDuration: config.MaxDuration,
|
|
|
|
|
blockTTL: config.BlockTTL,
|
|
|
|
|
blockMI: config.BlockMI,
|
|
|
|
|
blockMN: config.BlockMN);
|
2023-06-23 07:08:18 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-06-22 08:33:21 +00:00
|
|
|
|
private void Log(string msg)
|
|
|
|
|
{
|
|
|
|
|
Console.WriteLine(msg);
|
2023-06-22 08:17:12 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
2023-10-16 09:19:57 +00:00
|
|
|
|
|
|
|
|
|
public class FastHttpTimeSet : ITimeSet
|
|
|
|
|
{
|
|
|
|
|
public TimeSpan HttpCallRetryDelay()
|
|
|
|
|
{
|
|
|
|
|
return TimeSpan.FromSeconds(2);
|
|
|
|
|
}
|
|
|
|
|
|
2023-10-23 08:11:02 +00:00
|
|
|
|
public int HttpMaxNumberOfRetries()
|
2023-10-16 09:19:57 +00:00
|
|
|
|
{
|
2023-10-23 08:11:02 +00:00
|
|
|
|
return 2;
|
2023-10-16 09:19:57 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public TimeSpan HttpCallTimeout()
|
|
|
|
|
{
|
|
|
|
|
return TimeSpan.FromSeconds(10);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public TimeSpan K8sOperationTimeout()
|
|
|
|
|
{
|
|
|
|
|
return TimeSpan.FromMinutes(10);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public TimeSpan WaitForK8sServiceDelay()
|
|
|
|
|
{
|
|
|
|
|
return TimeSpan.FromSeconds(30);
|
|
|
|
|
}
|
|
|
|
|
}
|
2024-02-22 13:41:07 +00:00
|
|
|
|
}
|