2023-09-15 13:52:02 +00:00
|
|
|
|
using KubernetesWorkflow;
|
2023-09-08 07:39:56 +00:00
|
|
|
|
using Utils;
|
2023-04-12 11:53:55 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
namespace CodexPlugin
|
2023-04-12 11:53:55 +00:00
|
|
|
|
{
|
2023-09-13 14:06:05 +00:00
|
|
|
|
public class CodexContainerRecipe : ContainerRecipeFactory
|
2023-04-12 11:53:55 +00:00
|
|
|
|
{
|
2023-09-19 08:24:43 +00:00
|
|
|
|
private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter();
|
|
|
|
|
|
2023-08-17 09:06:45 +00:00
|
|
|
|
private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
|
2023-04-13 12:36:17 +00:00
|
|
|
|
public const string MetricsPortTag = "metrics_port";
|
2023-05-29 06:35:46 +00:00
|
|
|
|
public const string DiscoveryPortTag = "discovery-port";
|
2023-04-13 12:36:17 +00:00
|
|
|
|
|
2023-09-04 07:08:34 +00:00
|
|
|
|
// Used by tests for time-constraint assertions.
|
2023-06-07 06:30:10 +00:00
|
|
|
|
public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
|
|
|
|
|
public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
|
|
|
|
|
|
2023-08-07 13:51:44 +00:00
|
|
|
|
public override string AppName => "codex";
|
2023-09-28 09:31:09 +00:00
|
|
|
|
public override string Image => GetDockerImage();
|
2023-09-14 05:17:30 +00:00
|
|
|
|
|
2023-09-28 09:31:09 +00:00
|
|
|
|
public static string DockerImageOverride { get; set; } = string.Empty;
|
2023-04-12 11:53:55 +00:00
|
|
|
|
|
2023-09-13 14:06:05 +00:00
|
|
|
|
protected override void Initialize(StartupConfig startupConfig)
|
2023-04-12 11:53:55 +00:00
|
|
|
|
{
|
2023-09-21 09:07:27 +00:00
|
|
|
|
SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
|
|
|
|
|
SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
|
2023-09-18 13:45:21 +00:00
|
|
|
|
|
2023-04-12 11:53:55 +00:00
|
|
|
|
var config = startupConfig.Get<CodexStartupConfig>();
|
|
|
|
|
|
2023-07-04 14:04:18 +00:00
|
|
|
|
AddExposedPortAndVar("CODEX_API_PORT");
|
|
|
|
|
AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
|
|
|
|
|
|
2023-09-07 06:19:19 +00:00
|
|
|
|
var dataDir = $"datadir{ContainerNumber}";
|
|
|
|
|
AddEnvVar("CODEX_DATA_DIR", dataDir);
|
2023-09-07 08:37:52 +00:00
|
|
|
|
AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
|
2023-09-07 06:19:19 +00:00
|
|
|
|
|
2023-07-04 14:04:18 +00:00
|
|
|
|
AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
|
2023-09-14 05:04:37 +00:00
|
|
|
|
AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
|
2023-07-04 14:04:18 +00:00
|
|
|
|
|
|
|
|
|
// This makes the node announce itself to its local (pod) IP address.
|
2023-07-13 08:05:40 +00:00
|
|
|
|
AddEnvVar("NAT_IP_AUTO", "true");
|
2023-04-12 11:53:55 +00:00
|
|
|
|
|
|
|
|
|
var listenPort = AddInternalPort();
|
2023-07-04 14:04:18 +00:00
|
|
|
|
AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
|
2023-04-12 11:53:55 +00:00
|
|
|
|
|
2023-04-24 12:09:23 +00:00
|
|
|
|
if (!string.IsNullOrEmpty(config.BootstrapSpr))
|
|
|
|
|
{
|
2023-07-04 14:04:18 +00:00
|
|
|
|
AddEnvVar("CODEX_BOOTSTRAP_NODE", config.BootstrapSpr);
|
2023-04-24 12:09:23 +00:00
|
|
|
|
}
|
2023-04-12 11:53:55 +00:00
|
|
|
|
if (config.StorageQuota != null)
|
|
|
|
|
{
|
2023-07-04 14:04:18 +00:00
|
|
|
|
AddEnvVar("CODEX_STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
|
2023-04-12 11:53:55 +00:00
|
|
|
|
}
|
2023-06-29 14:03:45 +00:00
|
|
|
|
if (config.BlockTTL != null)
|
|
|
|
|
{
|
2023-07-14 08:45:26 +00:00
|
|
|
|
AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
|
2023-06-29 14:03:45 +00:00
|
|
|
|
}
|
2023-08-14 13:51:03 +00:00
|
|
|
|
if (config.BlockMaintenanceInterval != null)
|
|
|
|
|
{
|
|
|
|
|
AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
|
|
|
|
|
}
|
2023-08-14 14:37:31 +00:00
|
|
|
|
if (config.BlockMaintenanceNumber != null)
|
|
|
|
|
{
|
|
|
|
|
AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
|
|
|
|
|
}
|
2023-09-13 09:59:21 +00:00
|
|
|
|
if (config.MetricsEnabled)
|
|
|
|
|
{
|
|
|
|
|
var metricsPort = AddInternalPort(MetricsPortTag);
|
|
|
|
|
AddEnvVar("CODEX_METRICS", "true");
|
|
|
|
|
AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
|
|
|
|
|
AddEnvVar("CODEX_METRICS_PORT", metricsPort);
|
|
|
|
|
AddPodAnnotation("prometheus.io/scrape", "true");
|
|
|
|
|
AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString());
|
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-14 05:17:30 +00:00
|
|
|
|
if (config.SimulateProofFailures != null)
|
2023-09-14 05:02:53 +00:00
|
|
|
|
{
|
|
|
|
|
AddEnvVar("CODEX_SIMULATE_PROOF_FAILURES", config.SimulateProofFailures.ToString()!);
|
|
|
|
|
}
|
2023-09-14 05:11:04 +00:00
|
|
|
|
|
2023-09-19 08:24:43 +00:00
|
|
|
|
if (config.MarketplaceConfig != null)
|
|
|
|
|
{
|
|
|
|
|
var mconfig = config.MarketplaceConfig;
|
2023-09-19 09:51:59 +00:00
|
|
|
|
var gethStart = mconfig.GethNode.StartResult;
|
2023-09-21 08:56:48 +00:00
|
|
|
|
var ip = gethStart.Container.Pod.PodInfo.Ip;
|
2023-09-19 09:51:59 +00:00
|
|
|
|
var port = gethStart.WsPort.Number;
|
2023-09-20 07:16:57 +00:00
|
|
|
|
var marketplaceAddress = mconfig.CodexContracts.Deployment.MarketplaceAddress;
|
2023-09-19 08:24:43 +00:00
|
|
|
|
|
|
|
|
|
AddEnvVar("CODEX_ETH_PROVIDER", $"ws://{ip}:{port}");
|
|
|
|
|
AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress);
|
|
|
|
|
AddEnvVar("CODEX_PERSISTENCE", "true");
|
|
|
|
|
|
|
|
|
|
// Custom scripting in the Codex test image will write this variable to a private-key file,
|
|
|
|
|
// and pass the correct filename to Codex.
|
|
|
|
|
var mStart = marketplaceStarter.Start();
|
|
|
|
|
AddEnvVar("PRIV_KEY", mStart.PrivateKey);
|
|
|
|
|
Additional(mStart);
|
|
|
|
|
|
|
|
|
|
if (config.MarketplaceConfig.IsValidator)
|
|
|
|
|
{
|
2023-09-14 05:04:37 +00:00
|
|
|
|
AddEnvVar("CODEX_VALIDATOR", "true");
|
2023-09-19 08:24:43 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
2023-09-13 14:06:05 +00:00
|
|
|
|
|
2023-09-21 09:07:27 +00:00
|
|
|
|
if(!string.IsNullOrEmpty(config.NameOverride))
|
|
|
|
|
{
|
2023-09-14 05:02:53 +00:00
|
|
|
|
AddEnvVar("CODEX_NODENAME", config.NameOverride);
|
2023-04-14 08:51:35 +00:00
|
|
|
|
}
|
2023-04-12 11:53:55 +00:00
|
|
|
|
}
|
2023-06-22 12:37:37 +00:00
|
|
|
|
|
2023-09-08 07:39:56 +00:00
|
|
|
|
private ByteSize GetVolumeCapacity(CodexStartupConfig config)
|
2023-09-07 08:37:52 +00:00
|
|
|
|
{
|
2023-09-08 07:39:56 +00:00
|
|
|
|
if (config.StorageQuota != null) return config.StorageQuota;
|
|
|
|
|
// Default Codex quota: 8 Gb, using +20% to be safe.
|
|
|
|
|
return 8.GB().Multiply(1.2);
|
2023-09-07 08:37:52 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-07-17 09:12:14 +00:00
|
|
|
|
private string GetDockerImage()
|
|
|
|
|
{
|
|
|
|
|
var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
|
|
|
|
|
if (!string.IsNullOrEmpty(image)) return image;
|
2023-09-28 09:31:09 +00:00
|
|
|
|
if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride;
|
2023-07-21 07:34:37 +00:00
|
|
|
|
return DefaultDockerImage;
|
2023-07-17 09:12:14 +00:00
|
|
|
|
}
|
2023-04-12 11:53:55 +00:00
|
|
|
|
}
|
|
|
|
|
}
|