From 3a8bb760efcca3bfe50ea63094d4b8cb9b07a99f Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 19 Oct 2023 11:08:30 +0200 Subject: [PATCH 1/9] Adding support for multiple exposed container ports --- .../ContainerRecipeFactory.cs | 5 --- .../KubernetesWorkflow/RunnerLocationUtils.cs | 22 ++++++--- .../KubernetesWorkflow/RunningContainers.cs | 33 +++++++++----- .../KubernetesWorkflow/StartupWorkflow.cs | 45 ++++++++++--------- .../CodexPlugin/CodexContainerRecipe.cs | 4 +- 5 files changed, 64 insertions(+), 45 deletions(-) diff --git a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs index 6fb50a5..8d19398 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -132,11 +132,6 @@ namespace KubernetesWorkflow private Port AddExposedPort(Port port) { - if (exposedPorts.Any()) - { - throw new NotImplementedException("Current implementation only support 1 exposed port per container recipe. " + - $"Methods for determining container addresses in {nameof(StartupWorkflow)} currently rely on this constraint."); - } exposedPorts.Add(port); return port; } diff --git a/Framework/KubernetesWorkflow/RunnerLocationUtils.cs b/Framework/KubernetesWorkflow/RunnerLocationUtils.cs index 055aa15..86984f8 100644 --- a/Framework/KubernetesWorkflow/RunnerLocationUtils.cs +++ b/Framework/KubernetesWorkflow/RunnerLocationUtils.cs @@ -16,18 +16,26 @@ namespace KubernetesWorkflow internal static RunnerLocation DetermineRunnerLocation(RunningContainer container) { if (knownLocation != null) return knownLocation.Value; + knownLocation = PingForLocation(container); + return knownLocation.Value; + } + private static RunnerLocation PingForLocation(RunningContainer container) + { if (PingHost(container.Pod.PodInfo.Ip)) { - knownLocation = RunnerLocation.InternalToCluster; - } - else if (PingHost(Format(container.ClusterExternalAddress))) - { - knownLocation = RunnerLocation.ExternalToCluster; + return RunnerLocation.InternalToCluster; } - if (knownLocation == null) throw new Exception("Unable to determine location relative to kubernetes cluster."); - return knownLocation.Value; + foreach (var port in container.ContainerPorts) + { + if (PingHost(Format(port.ExternalAddress))) + { + return RunnerLocation.ExternalToCluster; + } + } + + throw new Exception("Unable to determine location relative to kubernetes cluster."); } private static string Format(Address host) diff --git a/Framework/KubernetesWorkflow/RunningContainers.cs b/Framework/KubernetesWorkflow/RunningContainers.cs index cfcae4d..f515fb9 100644 --- a/Framework/KubernetesWorkflow/RunningContainers.cs +++ b/Framework/KubernetesWorkflow/RunningContainers.cs @@ -24,37 +24,50 @@ namespace KubernetesWorkflow public class RunningContainer { - public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress) + public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, ContainerPort[] containerPorts) { Pod = pod; Recipe = recipe; ServicePorts = servicePorts; Name = name; - ClusterExternalAddress = clusterExternalAddress; - ClusterInternalAddress = clusterInternalAddress; + ContainerPorts = containerPorts; } public string Name { get; } public RunningPod Pod { get; } public ContainerRecipe Recipe { get; } public Port[] ServicePorts { get; } - public Address ClusterExternalAddress { get; } - public Address ClusterInternalAddress { get; } + public ContainerPort[] ContainerPorts { get; } [JsonIgnore] public Address Address { get { - if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster) - { - return ClusterInternalAddress; - } - return ClusterExternalAddress; + throw new Exception("a"); + //if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster) + //{ + // return ClusterInternalAddress; + //} + //return ClusterExternalAddress; } } } + public class ContainerPort + { + public ContainerPort(Port port, Address externalAddress, Address internalAddress) + { + Port = port; + ExternalAddress = externalAddress; + InternalAddress = internalAddress; + } + + public Port Port { get; } + public Address ExternalAddress { get; } + public Address InternalAddress { get; } + } + public static class RunningContainersExtensions { public static RunningContainer[] Containers(this RunningContainers[] runningContainers) diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index bda8a9b..e50e39e 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -118,8 +118,7 @@ namespace KubernetesWorkflow var name = GetContainerName(r, startupConfig); return new RunningContainer(runningPod, r, servicePorts, name, - GetContainerExternalAddress(runningPod, servicePorts), - GetContainerInternalAddress(r)); + CreateContainerPorts(runningPod, r, servicePorts)); }).ToArray(); } @@ -137,35 +136,39 @@ namespace KubernetesWorkflow } } - private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts) + private ContainerPort[] CreateContainerPorts(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts) { - return new Address( - pod.Cluster.HostAddress, - GetServicePort(servicePorts)); + var result = new List(); + foreach (var exposedPort in recipe.ExposedPorts) + { + result.Add(new ContainerPort( + exposedPort, + GetContainerExternalAddress(pod, servicePorts, exposedPort), + GetContainerInternalAddress(exposedPort))); + } + + return result.ToArray(); } - private Address GetContainerInternalAddress(ContainerRecipe recipe) + private static Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts, Port exposedPort) + { + var servicePort = servicePorts.Single(p => p.Tag == exposedPort.Tag); + + return new Address( + pod.Cluster.HostAddress, + servicePort.Number); + } + + private Address GetContainerInternalAddress(Port exposedPort) { var serviceName = "service-" + numberSource.WorkflowNumber; - var port = GetInternalPort(recipe); + var port = exposedPort.Number; return new Address( $"http://{serviceName}.{k8sNamespace}.svc.cluster.local", port); } - - private static int GetServicePort(Port[] servicePorts) - { - if (servicePorts.Any()) return servicePorts.First().Number; - return 0; - } - - private static int GetInternalPort(ContainerRecipe recipe) - { - if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number; - return 0; - } - + private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { log.Debug(); diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index ebef993..905041a 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -34,13 +34,13 @@ namespace CodexPlugin AddEnvVar("CODEX_DATA_DIR", dataDir); AddVolume($"codex/{dataDir}", GetVolumeCapacity(config)); - AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag); + AddExposedPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag); AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics()); // This makes the node announce itself to its local (pod) IP address. AddEnvVar("NAT_IP_AUTO", "true"); - var listenPort = AddInternalPort(); + var listenPort = AddExposedPort(); AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); if (!string.IsNullOrEmpty(config.BootstrapSpr)) From 43fa57dc97ce84c82d9d5ea8e50f99cf851f1ffd Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 19 Oct 2023 11:12:08 +0200 Subject: [PATCH 2/9] Mandatory port tags for exposed ports --- Framework/KubernetesWorkflow/ContainerRecipe.cs | 2 ++ Framework/KubernetesWorkflow/ContainerRecipeFactory.cs | 6 +++--- ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs | 10 ++++++---- .../MetricsPlugin/PrometheusContainerRecipe.cs | 4 +++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Framework/KubernetesWorkflow/ContainerRecipe.cs b/Framework/KubernetesWorkflow/ContainerRecipe.cs index 772ff20..6bac0ff 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipe.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipe.cs @@ -24,6 +24,8 @@ { Name = $"ctnr{Number}"; } + + if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports."); } public string Name { get; } diff --git a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs index 8d19398..6e50a1b 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -50,12 +50,12 @@ namespace KubernetesWorkflow protected int Index { get; private set; } = 0; protected abstract void Initialize(StartupConfig config); - protected Port AddExposedPort(string tag = "") + protected Port AddExposedPort(string tag) { return AddExposedPort(factory.CreatePort(tag)); } - protected Port AddExposedPort(int number, string tag = "") + protected Port AddExposedPort(int number, string tag) { return AddExposedPort(factory.CreatePort(number, tag)); } @@ -67,7 +67,7 @@ namespace KubernetesWorkflow return p; } - protected void AddExposedPortAndVar(string name, string tag = "") + protected void AddExposedPortAndVar(string name, string tag) { AddEnvVar(name, AddExposedPort(tag)); } diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 905041a..867b9bf 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -8,8 +8,10 @@ namespace CodexPlugin private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter(); private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests"; - public const string MetricsPortTag = "metrics_port"; - public const string DiscoveryPortTag = "discovery-port"; + public const string ApiPortTag = "codex_api_port"; + public const string ListenPortTag = "codex_listen_port"; + public const string MetricsPortTag = "codex_metrics_port"; + public const string DiscoveryPortTag = "codex_discovery_port"; // Used by tests for time-constraint assertions. public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0); @@ -27,7 +29,7 @@ namespace CodexPlugin var config = startupConfig.Get(); - AddExposedPortAndVar("CODEX_API_PORT"); + AddExposedPortAndVar("CODEX_API_PORT", ApiPortTag); AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0"); var dataDir = $"datadir{ContainerNumber}"; @@ -40,7 +42,7 @@ namespace CodexPlugin // This makes the node announce itself to its local (pod) IP address. AddEnvVar("NAT_IP_AUTO", "true"); - var listenPort = AddExposedPort(); + var listenPort = AddExposedPort(ListenPortTag); AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); if (!string.IsNullOrEmpty(config.BootstrapSpr)) diff --git a/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs b/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs index 26b9b48..2696915 100644 --- a/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs +++ b/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs @@ -7,11 +7,13 @@ namespace MetricsPlugin public override string AppName => "prometheus"; public override string Image => "codexstorage/dist-tests-prometheus:latest"; + public const string PortTag = "prometheus_port_tag"; + protected override void Initialize(StartupConfig startupConfig) { var config = startupConfig.Get(); - AddExposedPortAndVar("PROM_PORT"); + AddExposedPortAndVar("PROM_PORT", PortTag); AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); } } From 45050c34e48a528d74e385bd07db8a86793613a8 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 19 Oct 2023 11:18:59 +0200 Subject: [PATCH 3/9] Implements GetAddress method for runningContainers. --- Framework/Core/SerializeGate.cs | 6 ++---- Framework/KubernetesWorkflow/RunningContainers.cs | 14 +++++--------- Framework/Utils/Address.cs | 5 +++++ ProjectPlugins/CodexPlugin/CodexAccess.cs | 10 ++++++++-- ProjectPlugins/GethPlugin/GethNode.cs | 2 +- ProjectPlugins/MetricsPlugin/MetricsQuery.cs | 2 +- Tests/CodexContinuousTests/StartupChecker.cs | 5 +++-- 7 files changed, 25 insertions(+), 19 deletions(-) diff --git a/Framework/Core/SerializeGate.cs b/Framework/Core/SerializeGate.cs index aab0de9..5f5d050 100644 --- a/Framework/Core/SerializeGate.cs +++ b/Framework/Core/SerializeGate.cs @@ -5,11 +5,9 @@ namespace Core public static class SerializeGate { /// - /// SerializeGate was added to help ensure deployment objects are serializable - /// and remain viable after deserialization. + /// SerializeGate was added to help ensure deployment objects are serializable and remain viable after deserialization. /// Tools can be built on top of the core interface that rely on deployment objects being serializable. - /// Insert the serialization gate after deployment but before wrapping to ensure any future changes - /// don't break this requirement. + /// Insert the serialization gate after deployment but before wrapping to ensure any future changes don't break this requirement. /// public static T Gate(T anything) { diff --git a/Framework/KubernetesWorkflow/RunningContainers.cs b/Framework/KubernetesWorkflow/RunningContainers.cs index f515fb9..0b9db88 100644 --- a/Framework/KubernetesWorkflow/RunningContainers.cs +++ b/Framework/KubernetesWorkflow/RunningContainers.cs @@ -39,18 +39,14 @@ namespace KubernetesWorkflow public Port[] ServicePorts { get; } public ContainerPort[] ContainerPorts { get; } - [JsonIgnore] - public Address Address + public Address GetAddress(string portTag) { - get + var containerPort = ContainerPorts.Single(c => c.Port.Tag == portTag); + if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster) { - throw new Exception("a"); - //if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster) - //{ - // return ClusterInternalAddress; - //} - //return ClusterExternalAddress; + return containerPort.InternalAddress; } + return containerPort.ExternalAddress; } } diff --git a/Framework/Utils/Address.cs b/Framework/Utils/Address.cs index 510afeb..27ff16a 100644 --- a/Framework/Utils/Address.cs +++ b/Framework/Utils/Address.cs @@ -10,5 +10,10 @@ public string Host { get; } public int Port { get; } + + public override string ToString() + { + return $"{Host}:{Port}"; + } } } diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 48ffb08..123090c 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -1,5 +1,6 @@ using Core; using KubernetesWorkflow; +using Utils; namespace CodexPlugin { @@ -98,12 +99,17 @@ namespace CodexPlugin private IHttp Http() { - return tools.CreateHttp(Container.Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name); + return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name); } private IHttp LongHttp() { - return tools.CreateHttp(Container.Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name); + return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name); + } + + private Address GetAddress() + { + return Container.GetAddress(CodexContainerRecipe.ApiPortTag); } private void CheckContainerCrashed(HttpClient client) diff --git a/ProjectPlugins/GethPlugin/GethNode.cs b/ProjectPlugins/GethPlugin/GethNode.cs index 978a852..8b6a8d8 100644 --- a/ProjectPlugins/GethPlugin/GethNode.cs +++ b/ProjectPlugins/GethPlugin/GethNode.cs @@ -73,7 +73,7 @@ namespace GethPlugin private NethereumInteraction StartInteraction() { - var address = StartResult.Container.Address; + var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag); var account = Account; var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey); diff --git a/ProjectPlugins/MetricsPlugin/MetricsQuery.cs b/ProjectPlugins/MetricsPlugin/MetricsQuery.cs index fbbb827..8a9ad8e 100644 --- a/ProjectPlugins/MetricsPlugin/MetricsQuery.cs +++ b/ProjectPlugins/MetricsPlugin/MetricsQuery.cs @@ -13,7 +13,7 @@ namespace MetricsPlugin public MetricsQuery(IPluginTools tools, RunningContainer runningContainer) { RunningContainer = runningContainer; - http = tools.CreateHttp(RunningContainer.Address, "api/v1"); + http = tools.CreateHttp(RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag), "api/v1"); log = tools.GetLog(); } diff --git a/Tests/CodexContinuousTests/StartupChecker.cs b/Tests/CodexContinuousTests/StartupChecker.cs index aae813f..a6e539d 100644 --- a/Tests/CodexContinuousTests/StartupChecker.cs +++ b/Tests/CodexContinuousTests/StartupChecker.cs @@ -87,7 +87,8 @@ namespace ContinuousTests { cancelToken.ThrowIfCancellationRequested(); - log.Log($"Checking {n.Container.Name} @ '{n.Container.Address.Host}:{n.Container.Address.Port}'..."); + var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag); + log.Log($"Checking {n.Container.Name} @ '{address}'..."); if (EnsureOnline(log, n)) { @@ -95,7 +96,7 @@ namespace ContinuousTests } else { - log.Error($"No response from '{n.Container.Address.Host}'."); + log.Error($"No response from '{address}'."); pass = false; } } From 2fea475237e38030cf8f0c82b6499abf7f7dca82 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 19 Oct 2023 14:03:36 +0200 Subject: [PATCH 4/9] multiple service ports --- Framework/KubernetesWorkflow/ContainerRecipe.cs | 6 ++++++ Framework/KubernetesWorkflow/K8sController.cs | 5 ++--- Framework/KubernetesWorkflow/RunningPod.cs | 10 ++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Framework/KubernetesWorkflow/ContainerRecipe.cs b/Framework/KubernetesWorkflow/ContainerRecipe.cs index 6bac0ff..ed89d78 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipe.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipe.cs @@ -67,6 +67,12 @@ public int Number { get; } public string Tag { get; } + + public override string ToString() + { + if (string.IsNullOrEmpty(Tag)) return $"untagged-port={Number}"; + return $"{Tag}={Number}"; + } } public class EnvVar diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 2cbb358..6a02950 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -572,10 +572,9 @@ namespace KubernetesWorkflow var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace)); foreach (var r in containerRecipes) { - if (r.ExposedPorts.Any()) + foreach (var port in r.ExposedPorts) { - var firstExposedPort = r.ExposedPorts.First(); - var portName = GetNameForPort(r, firstExposedPort); + var portName = GetNameForPort(r, port); var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName); if (matchingServicePorts.Any()) diff --git a/Framework/KubernetesWorkflow/RunningPod.cs b/Framework/KubernetesWorkflow/RunningPod.cs index f4282ba..1e139f3 100644 --- a/Framework/KubernetesWorkflow/RunningPod.cs +++ b/Framework/KubernetesWorkflow/RunningPod.cs @@ -19,12 +19,10 @@ public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) { - if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number)) - { - return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports; - } - - return Array.Empty(); + return PortMapEntries + .Where(p => p.ContainerNumber == containerRecipe.Number) + .SelectMany(p => p.Ports) + .ToArray(); } } From 0fd6a6f06e4f61a68150a13808a507b0151adf3c Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 19 Oct 2023 15:48:49 +0200 Subject: [PATCH 5/9] Fixes port tag mismatch --- Framework/KubernetesWorkflow/K8sController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 6a02950..1cad8ba 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -580,7 +580,7 @@ namespace KubernetesWorkflow if (matchingServicePorts.Any()) { // These service ports belongs to this recipe. - var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName)); + var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, port.Tag)); var ports = optionals.Where(p => p != null).Select(p => p!).ToArray(); result.Add(new ContainerRecipePortMapEntry(r.Number, ports)); From 3b258c9e2ed3cc6ef961c1ae0d7d2d84a971d249 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 20 Oct 2023 08:31:23 +0200 Subject: [PATCH 6/9] Pins contract image to one compatible with current main codex --- .../CodexContractsPlugin/CodexContractsContainerRecipe.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs index 6b6aabe..39ec222 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs @@ -4,7 +4,7 @@ namespace CodexContractsPlugin { public class CodexContractsContainerRecipe : ContainerRecipeFactory { - public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:latest-dist-tests"; + public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests"; public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json"; From bf18fa03a2166364d3d05679e1351cb52505a189 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 22 Oct 2023 11:29:16 +0200 Subject: [PATCH 7/9] adds sleep to the starting of the test screens --- Tests/CodexContinuousTests/deploy-and-run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/CodexContinuousTests/deploy-and-run.sh index 0260f7c..1fa5037 100644 --- a/Tests/CodexContinuousTests/deploy-and-run.sh +++ b/Tests/CodexContinuousTests/deploy-and-run.sh @@ -44,4 +44,6 @@ do --cleanup=1 \ --full-container-logs=1 \ --target-duration=172800 # 48 hours + + sleep 30 done From 45fbd699a9568734c29b0adf60219643d75cb97f Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 23 Oct 2023 09:36:31 +0200 Subject: [PATCH 8/9] Disables calls to custom API endpoints. --- ProjectPlugins/CodexPlugin/CodexNode.cs | 5 ++-- .../Tests/TwoClientTest.cs | 8 +++--- .../BasicTests/BlockExchangeTests.cs | 27 ++++++++++--------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index d2438dd..9175606 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -13,8 +13,9 @@ namespace CodexPlugin string GetName(); CodexDebugResponse GetDebugInfo(); CodexDebugPeerResponse GetDebugPeer(string peerId); - CodexDebugBlockExchangeResponse GetDebugBlockExchange(); - CodexDebugRepoStoreResponse[] GetDebugRepoStore(); + // These debug methods are not available in master-line Codex. Use only for custom builds. + //CodexDebugBlockExchangeResponse GetDebugBlockExchange(); + //CodexDebugRepoStoreResponse[] GetDebugRepoStore(); ContentId UploadFile(TrackedFile file); TrackedFile? DownloadContent(ContentId contentId, string fileLabel = ""); void ConnectToPeer(ICodexNode node); diff --git a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs index 4275202..596ea2b 100644 --- a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs +++ b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs @@ -57,8 +57,8 @@ namespace ContinuousTests.Tests private void LogRepoStore(ICodexNode codexNode) { - var response = codexNode.GetDebugRepoStore(); - Log.Log($"{codexNode.GetName()} has {string.Join(",", response.Select(r => r.cid))}"); + //var response = codexNode.GetDebugRepoStore(); + //Log.Log($"{codexNode.GetName()} has {string.Join(",", response.Select(r => r.cid))}"); } private void LogStoredBytes(ICodexNode node) @@ -90,8 +90,8 @@ namespace ContinuousTests.Tests private void LogBlockExchangeStatus(ICodexNode codexNode, string msg) { - var response = codexNode.GetDebugBlockExchange(); - Log.Log($"{codexNode.GetName()} {msg}: {JsonConvert.SerializeObject(response)}"); + //var response = codexNode.GetDebugBlockExchange(); + //Log.Log($"{codexNode.GetName()} {msg}: {JsonConvert.SerializeObject(response)}"); } } } diff --git a/Tests/CodexTests/BasicTests/BlockExchangeTests.cs b/Tests/CodexTests/BasicTests/BlockExchangeTests.cs index 72d7357..83f67d4 100644 --- a/Tests/CodexTests/BasicTests/BlockExchangeTests.cs +++ b/Tests/CodexTests/BasicTests/BlockExchangeTests.cs @@ -42,21 +42,22 @@ namespace CodexTests.BasicTests { foreach (var node in nodes) { - Time.Retry(() => AssertBlockExchangeIsEmpty(node), nameof(AssertExchangeIsEmpty)); + // API Call not available in master-line Codex image. + //Time.Retry(() => AssertBlockExchangeIsEmpty(node), nameof(AssertExchangeIsEmpty)); } } - private void AssertBlockExchangeIsEmpty(ICodexNode node) - { - var msg = $"BlockExchange for {node.GetName()}: "; - var response = node.GetDebugBlockExchange(); - foreach (var peer in response.peers) - { - var activeWants = peer.wants.Where(w => !w.cancel).ToArray(); - Assert.That(activeWants.Length, Is.EqualTo(0), msg + "thinks a peer has active wants."); - } - Assert.That(response.taskQueue, Is.EqualTo(0), msg + "has tasks in queue."); - Assert.That(response.pendingBlocks, Is.EqualTo(0), msg + "has pending blocks."); - } + //private void AssertBlockExchangeIsEmpty(ICodexNode node) + //{ + // var msg = $"BlockExchange for {node.GetName()}: "; + // var response = node.GetDebugBlockExchange(); + // foreach (var peer in response.peers) + // { + // var activeWants = peer.wants.Where(w => !w.cancel).ToArray(); + // Assert.That(activeWants.Length, Is.EqualTo(0), msg + "thinks a peer has active wants."); + // } + // Assert.That(response.taskQueue, Is.EqualTo(0), msg + "has tasks in queue."); + // Assert.That(response.pendingBlocks, Is.EqualTo(0), msg + "has pending blocks."); + //} } } From 50fbf0ad52b9e5336b89658ce36bdace36eeb4a8 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 23 Oct 2023 10:11:02 +0200 Subject: [PATCH 9/9] Replaces retry-time with maxNumberOfRetries in timesets. --- Framework/Core/Http.cs | 2 +- Framework/Core/TimeSet.cs | 12 ++++++------ Framework/Utils/Time.cs | 30 ++++++++++++++++++------------ Tools/CodexNetDeployer/Deployer.cs | 4 ++-- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/Framework/Core/Http.cs b/Framework/Core/Http.cs index 60e52db..18fc5c7 100644 --- a/Framework/Core/Http.cs +++ b/Framework/Core/Http.cs @@ -196,7 +196,7 @@ namespace Core private T Retry(Func operation, string description) { - return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description); + return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description); } private HttpClient GetClient() diff --git a/Framework/Core/TimeSet.cs b/Framework/Core/TimeSet.cs index 2ca047f..3acd852 100644 --- a/Framework/Core/TimeSet.cs +++ b/Framework/Core/TimeSet.cs @@ -3,7 +3,7 @@ public interface ITimeSet { TimeSpan HttpCallTimeout(); - TimeSpan HttpCallRetryTime(); + int HttpMaxNumberOfRetries(); TimeSpan HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); @@ -13,12 +13,12 @@ { public TimeSpan HttpCallTimeout() { - return TimeSpan.FromMinutes(5); + return TimeSpan.FromMinutes(3); } - public TimeSpan HttpCallRetryTime() + public int HttpMaxNumberOfRetries() { - return TimeSpan.FromMinutes(1); + return 3; } public TimeSpan HttpCallRetryDelay() @@ -44,9 +44,9 @@ return TimeSpan.FromHours(2); } - public TimeSpan HttpCallRetryTime() + public int HttpMaxNumberOfRetries() { - return TimeSpan.FromHours(5); + return 1; } public TimeSpan HttpCallRetryDelay() diff --git a/Framework/Utils/Time.cs b/Framework/Utils/Time.cs index bf0e116..3a53b72 100644 --- a/Framework/Utils/Time.cs +++ b/Framework/Utils/Time.cs @@ -46,33 +46,35 @@ public static void Retry(Action action, string description) { - Retry(action, TimeSpan.FromMinutes(1), description); + Retry(action, 1, description); } public static T Retry(Func action, string description) { - return Retry(action, TimeSpan.FromMinutes(1), description); + return Retry(action, 1, description); } - public static void Retry(Action action, TimeSpan timeout, string description) + public static void Retry(Action action, int maxRetries, string description) { - Retry(action, timeout, TimeSpan.FromSeconds(1), description); + Retry(action, maxRetries, TimeSpan.FromSeconds(1), description); } - public static T Retry(Func action, TimeSpan timeout, string description) + public static T Retry(Func action, int maxRetries, string description) { - return Retry(action, timeout, TimeSpan.FromSeconds(1), description); + return Retry(action, maxRetries, TimeSpan.FromSeconds(1), description); } - public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description) + public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description) { var start = DateTime.UtcNow; + var retries = 0; var exceptions = new List(); while (true) { - if (DateTime.UtcNow - start > timeout) + if (retries > maxRetries) { - throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions)); + var duration = DateTime.UtcNow - start; + throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions)); } try @@ -83,21 +85,24 @@ catch (Exception ex) { exceptions.Add(ex); + retries++; } Sleep(retryTime); } } - public static T Retry(Func action, TimeSpan timeout, TimeSpan retryTime, string description) + public static T Retry(Func action, int maxRetries, TimeSpan retryTime, string description) { var start = DateTime.UtcNow; + var retries = 0; var exceptions = new List(); while (true) { - if (DateTime.UtcNow - start > timeout) + if (retries > maxRetries) { - throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions)); + var duration = DateTime.UtcNow - start; + throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions)); } try @@ -107,6 +112,7 @@ catch (Exception ex) { exceptions.Add(ex); + retries++; } Sleep(retryTime); diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index 796fca8..29f4348 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -180,9 +180,9 @@ namespace CodexNetDeployer return TimeSpan.FromSeconds(2); } - public TimeSpan HttpCallRetryTime() + public int HttpMaxNumberOfRetries() { - return TimeSpan.FromSeconds(2); + return 2; } public TimeSpan HttpCallTimeout()