cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

175 lines
6.9 KiB
C#
Raw Normal View History

using GethPlugin;
using KubernetesWorkflow;
using KubernetesWorkflow.Recipe;
2023-09-08 07:39:56 +00:00
using Utils;
2023-04-12 11:53:55 +00:00
namespace CodexPlugin
2023-04-12 11:53:55 +00:00
{
2023-09-13 14:06:05 +00:00
public class CodexContainerRecipe : ContainerRecipeFactory
2023-04-12 11:53:55 +00:00
{
2024-04-01 09:00:52 +00:00
private const string DefaultDockerImage = "codexstorage/nim-codex:sha-f2f1dd5-dist-tests";
2023-10-19 09:12:08 +00:00
public const string ApiPortTag = "codex_api_port";
public const string ListenPortTag = "codex_listen_port";
public const string MetricsPortTag = "codex_metrics_port";
public const string DiscoveryPortTag = "codex_discovery_port";
2023-04-13 12:36:17 +00:00
// Used by tests for time-constraint assertions.
public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
2023-08-07 13:51:44 +00:00
public override string AppName => "codex";
2023-09-28 09:31:09 +00:00
public override string Image => GetDockerImage();
2023-09-14 05:17:30 +00:00
2023-09-28 09:31:09 +00:00
public static string DockerImageOverride { get; set; } = string.Empty;
2023-04-12 11:53:55 +00:00
2023-09-13 14:06:05 +00:00
protected override void Initialize(StartupConfig startupConfig)
2023-04-12 11:53:55 +00:00
{
SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
2023-11-07 08:25:51 +00:00
//SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
SetSchedulingAffinity(notIn: "false");
SetSystemCriticalPriority();
2023-04-12 11:53:55 +00:00
var config = startupConfig.Get<CodexStartupConfig>();
var apiPort = CreateApiPort(config, ApiPortTag);
AddEnvVar("CODEX_API_PORT", apiPort);
2023-07-04 14:04:18 +00:00
AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
var dataDir = $"datadir{ContainerNumber}";
AddEnvVar("CODEX_DATA_DIR", dataDir);
AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
var discPort = CreateDiscoveryPort(config);
AddEnvVar("CODEX_DISC_PORT", discPort);
AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
2023-07-04 14:04:18 +00:00
if (config.PublicTestNet != null)
{
// This makes the node announce itself to its public IP address.
2023-10-27 09:21:43 +00:00
AddEnvVar("NAT_IP_AUTO", "false");
2023-12-11 07:38:31 +00:00
AddEnvVar("NAT_PUBLIC_IP_AUTO", PublicIpService.Address);
}
else
{
// This makes the node announce itself to its local (pod) IP address.
AddEnvVar("NAT_IP_AUTO", "true");
}
2023-04-12 11:53:55 +00:00
var listenPort = CreateListenPort(config);
2023-07-04 14:04:18 +00:00
AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
2023-04-12 11:53:55 +00:00
if (!string.IsNullOrEmpty(config.BootstrapSpr))
{
2023-07-04 14:04:18 +00:00
AddEnvVar("CODEX_BOOTSTRAP_NODE", config.BootstrapSpr);
}
2023-04-12 11:53:55 +00:00
if (config.StorageQuota != null)
{
2023-07-04 14:04:18 +00:00
AddEnvVar("CODEX_STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
2023-04-12 11:53:55 +00:00
}
if (config.BlockTTL != null)
{
2023-07-14 08:45:26 +00:00
AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
}
if (config.BlockMaintenanceInterval != null)
{
AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
}
if (config.BlockMaintenanceNumber != null)
{
AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
}
2023-09-13 09:59:21 +00:00
if (config.MetricsEnabled)
{
var metricsPort = CreateApiPort(config, MetricsPortTag);
2023-09-13 09:59:21 +00:00
AddEnvVar("CODEX_METRICS", "true");
AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
AddEnvVar("CODEX_METRICS_PORT", metricsPort);
AddPodAnnotation("prometheus.io/scrape", "true");
AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString());
}
2023-09-14 05:17:30 +00:00
if (config.SimulateProofFailures != null)
{
AddEnvVar("CODEX_SIMULATE_PROOF_FAILURES", config.SimulateProofFailures.ToString()!);
}
2023-09-14 05:11:04 +00:00
2023-09-19 08:24:43 +00:00
if (config.MarketplaceConfig != null)
{
var mconfig = config.MarketplaceConfig;
2023-09-19 09:51:59 +00:00
var gethStart = mconfig.GethNode.StartResult;
2023-11-06 15:10:19 +00:00
var wsAddress = gethStart.Container.GetInternalAddress(GethContainerRecipe.WsPortTag);
var marketplaceAddress = mconfig.CodexContracts.Deployment.MarketplaceAddress;
2023-09-19 08:24:43 +00:00
2023-11-06 15:10:19 +00:00
AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}");
2023-09-19 08:24:43 +00:00
AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress);
var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup;
2023-09-19 08:24:43 +00:00
// Custom scripting in the Codex test image will write this variable to a private-key file,
// and pass the correct filename to Codex.
AddEnvVar("PRIV_KEY", marketplaceSetup.EthAccount.PrivateKey);
Additional(marketplaceSetup.EthAccount);
2023-09-19 08:24:43 +00:00
2024-03-13 09:18:10 +00:00
SetCommandOverride(marketplaceSetup);
if (marketplaceSetup.IsValidator)
2023-09-19 08:24:43 +00:00
{
AddEnvVar("CODEX_VALIDATOR", "true");
2023-09-19 08:24:43 +00:00
}
}
2023-09-13 14:06:05 +00:00
if(!string.IsNullOrEmpty(config.NameOverride))
{
AddEnvVar("CODEX_NODENAME", config.NameOverride);
2023-04-14 08:51:35 +00:00
}
2023-04-12 11:53:55 +00:00
}
2023-06-22 12:37:37 +00:00
2024-03-13 09:18:10 +00:00
private void SetCommandOverride(MarketplaceSetup ms)
{
2024-03-13 09:29:26 +00:00
if (ms.IsStorageNode)
2024-03-13 09:18:10 +00:00
{
2024-03-15 11:48:25 +00:00
OverrideCommand("bash", "/docker-entrypoint.sh", "codex", "persistence", "prover");
2024-03-13 09:18:10 +00:00
}
2024-03-13 09:29:26 +00:00
else
2024-03-13 09:18:10 +00:00
{
2024-03-15 11:48:25 +00:00
OverrideCommand("bash", "/docker-entrypoint.sh", "codex", "persistence");
2024-03-13 09:18:10 +00:00
}
}
private Port CreateApiPort(CodexStartupConfig config, string tag)
{
if (config.PublicTestNet == null) return AddExposedPort(tag);
return AddInternalPort(tag);
}
private Port CreateListenPort(CodexStartupConfig config)
{
if (config.PublicTestNet == null) return AddInternalPort(ListenPortTag);
return AddExposedPort(config.PublicTestNet.PublicListenPort, ListenPortTag);
}
private Port CreateDiscoveryPort(CodexStartupConfig config)
{
if (config.PublicTestNet == null) return AddInternalPort(DiscoveryPortTag, PortProtocol.UDP);
return AddExposedPort(config.PublicTestNet.PublicDiscoveryPort, DiscoveryPortTag, PortProtocol.UDP);
}
2023-09-08 07:39:56 +00:00
private ByteSize GetVolumeCapacity(CodexStartupConfig config)
{
2023-09-08 07:39:56 +00:00
if (config.StorageQuota != null) return config.StorageQuota;
// Default Codex quota: 8 Gb, using +20% to be safe.
return 8.GB().Multiply(1.2);
}
private string GetDockerImage()
{
var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
if (!string.IsNullOrEmpty(image)) return image;
2023-09-28 09:31:09 +00:00
if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride;
return DefaultDockerImage;
}
2023-04-12 11:53:55 +00:00
}
}