Merge branch 'master' into app/discord-bot

Framework/Core/Http.cs

@@ -196,7 +196,7 @@ namespace Core
 
         private T Retry<T>(Func<T> operation, string description)
         {
-            return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
+            return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
         }
 
         private HttpClient GetClient()

Framework/Core/SerializeGate.cs

@@ -5,11 +5,9 @@ namespace Core
     public static class SerializeGate
     {
         /// <summary>
-        /// SerializeGate was added to help ensure deployment objects are serializable
+        /// SerializeGate was added to help ensure deployment objects are serializable and remain viable after deserialization.
-        /// and remain viable after deserialization.
         /// Tools can be built on top of the core interface that rely on deployment objects being serializable.
-        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes
+        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes don't break this requirement.
-        /// don't break this requirement.
         /// </summary>
         public static T Gate<T>(T anything)
         {

Framework/Core/TimeSet.cs

@@ -3,7 +3,7 @@
     public interface ITimeSet
     {
         TimeSpan HttpCallTimeout();
-        TimeSpan HttpCallRetryTime();
+        int HttpMaxNumberOfRetries();
         TimeSpan HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
@@ -13,12 +13,12 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromMinutes(5);
+            return TimeSpan.FromMinutes(3);
         }
 
-        public TimeSpan HttpCallRetryTime()
+        public int HttpMaxNumberOfRetries()
         {
-            return TimeSpan.FromMinutes(1);
+            return 3;
         }
 
         public TimeSpan HttpCallRetryDelay()
@@ -44,9 +44,9 @@
             return TimeSpan.FromHours(2);
         }
 
-        public TimeSpan HttpCallRetryTime()
+        public int HttpMaxNumberOfRetries()
         {
-            return TimeSpan.FromHours(5);
+            return 1;
         }
 
         public TimeSpan HttpCallRetryDelay()

Framework/KubernetesWorkflow/ContainerRecipe.cs

@@ -24,6 +24,8 @@
             {
                 Name = $"ctnr{Number}";
             }
+
+            if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
         }
 
         public string Name { get; }
@@ -65,6 +67,12 @@
 
         public int Number { get; }
         public string Tag { get; }
+
+        public override string ToString()
+        {
+            if (string.IsNullOrEmpty(Tag)) return $"untagged-port={Number}";
+            return $"{Tag}={Number}";
+        }
     }
 
     public class EnvVar

Framework/KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -50,12 +50,12 @@ namespace KubernetesWorkflow
         protected int Index { get; private set; } = 0;
         protected abstract void Initialize(StartupConfig config);
 
-        protected Port AddExposedPort(string tag = "")
+        protected Port AddExposedPort(string tag)
         {
             return AddExposedPort(factory.CreatePort(tag));
         }
 
-        protected Port AddExposedPort(int number, string tag = "")
+        protected Port AddExposedPort(int number, string tag)
         {
             return AddExposedPort(factory.CreatePort(number, tag));
         }
@@ -67,7 +67,7 @@ namespace KubernetesWorkflow
             return p;
         }
 
-        protected void AddExposedPortAndVar(string name, string tag = "")
+        protected void AddExposedPortAndVar(string name, string tag)
         {
             AddEnvVar(name, AddExposedPort(tag));
         }
@@ -132,11 +132,6 @@ namespace KubernetesWorkflow
 
         private Port AddExposedPort(Port port)
         {
-            if (exposedPorts.Any())
-            {
-                throw new NotImplementedException("Current implementation only support 1 exposed port per container recipe. " +
-                    $"Methods for determining container addresses in {nameof(StartupWorkflow)} currently rely on this constraint.");
-            }
             exposedPorts.Add(port);
             return port;
         }

Framework/KubernetesWorkflow/K8sController.cs

@@ -572,16 +572,15 @@ namespace KubernetesWorkflow
             var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
             foreach (var r in containerRecipes)
             {
-                if (r.ExposedPorts.Any())
+                foreach (var port in r.ExposedPorts)
                 {
-                    var firstExposedPort = r.ExposedPorts.First();
+                    var portName = GetNameForPort(r, port);
-                    var portName = GetNameForPort(r, firstExposedPort);
 
                     var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
                     if (matchingServicePorts.Any())
                     {
                         // These service ports belongs to this recipe.
-                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
+                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, port.Tag));
                         var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
 
                         result.Add(new ContainerRecipePortMapEntry(r.Number, ports));

Framework/KubernetesWorkflow/RunnerLocationUtils.cs

@@ -16,18 +16,26 @@ namespace KubernetesWorkflow
         internal static RunnerLocation DetermineRunnerLocation(RunningContainer container)
         {
             if (knownLocation != null) return knownLocation.Value;
+            knownLocation = PingForLocation(container);
+            return knownLocation.Value;
+        }
 
+        private static RunnerLocation PingForLocation(RunningContainer container)
+        {
             if (PingHost(container.Pod.PodInfo.Ip))
             {
-                knownLocation = RunnerLocation.InternalToCluster;
+                return RunnerLocation.InternalToCluster;
-            }
-            else if (PingHost(Format(container.ClusterExternalAddress)))
-            {
-                knownLocation = RunnerLocation.ExternalToCluster;
             }
 
-            if (knownLocation == null) throw new Exception("Unable to determine location relative to kubernetes cluster.");
+            foreach (var port in container.ContainerPorts)
-            return knownLocation.Value;
+            {
+                if (PingHost(Format(port.ExternalAddress)))
+                {
+                    return RunnerLocation.ExternalToCluster;
+                }
+            }
+
+            throw new Exception("Unable to determine location relative to kubernetes cluster.");
         }
 
         private static string Format(Address host)

Framework/KubernetesWorkflow/RunningContainers.cs

@@ -24,35 +24,44 @@ namespace KubernetesWorkflow
 
     public class RunningContainer
     {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress)
+        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, ContainerPort[] containerPorts)
         {
             Pod = pod;
             Recipe = recipe;
             ServicePorts = servicePorts;
             Name = name;
-            ClusterExternalAddress = clusterExternalAddress;
+            ContainerPorts = containerPorts;
-            ClusterInternalAddress = clusterInternalAddress;
         }
 
         public string Name { get; }
         public RunningPod Pod { get; }
         public ContainerRecipe Recipe { get; }
         public Port[] ServicePorts { get; }
-        public Address ClusterExternalAddress { get; }
+        public ContainerPort[] ContainerPorts { get; }
-        public Address ClusterInternalAddress { get; }
 
-        [JsonIgnore]
+        public Address GetAddress(string portTag)
-        public Address Address
-        {
-            get
         {
+            var containerPort = ContainerPorts.Single(c => c.Port.Tag == portTag);
             if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
             {
-                    return ClusterInternalAddress;
+                return containerPort.InternalAddress;
             }
-                return ClusterExternalAddress;
+            return containerPort.ExternalAddress;
         }
     }
+
+    public class ContainerPort
+    {
+        public ContainerPort(Port port, Address externalAddress, Address internalAddress)
+        {
+            Port = port;
+            ExternalAddress = externalAddress;
+            InternalAddress = internalAddress;
+        }
+
+        public Port Port { get; }
+        public Address ExternalAddress { get; }
+        public Address InternalAddress { get; }
     }
 
     public static class RunningContainersExtensions

Framework/KubernetesWorkflow/RunningPod.cs

@@ -19,12 +19,10 @@
 
         public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
         {
-            if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
+            return PortMapEntries
-            {
+                .Where(p => p.ContainerNumber == containerRecipe.Number)
-                return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
+                .SelectMany(p => p.Ports)
-            }
+                .ToArray();
-
-            return Array.Empty<Port>();
         }
     }
 

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -118,8 +118,7 @@ namespace KubernetesWorkflow
                 var name = GetContainerName(r, startupConfig);
 
                 return new RunningContainer(runningPod, r, servicePorts, name,
-                    GetContainerExternalAddress(runningPod, servicePorts),
+                    CreateContainerPorts(runningPod, r, servicePorts));
-                    GetContainerInternalAddress(r));
 
             }).ToArray();
         }
@@ -137,35 +136,39 @@ namespace KubernetesWorkflow
             }
         }
 
-        private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        private ContainerPort[] CreateContainerPorts(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts)
         {
-            return new Address(
+            var result = new List<ContainerPort>();
-                pod.Cluster.HostAddress,
+            foreach (var exposedPort in recipe.ExposedPorts)
-                GetServicePort(servicePorts));
+            {
+                result.Add(new ContainerPort(
+                    exposedPort,
+                    GetContainerExternalAddress(pod, servicePorts, exposedPort),
+                    GetContainerInternalAddress(exposedPort)));
             }
 
-        private Address GetContainerInternalAddress(ContainerRecipe recipe)
+            return result.ToArray();
+        }
+
+        private static Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts, Port exposedPort)
+        {
+            var servicePort = servicePorts.Single(p => p.Tag == exposedPort.Tag);
+
+            return new Address(
+                pod.Cluster.HostAddress,
+                servicePort.Number);
+        }
+
+        private Address GetContainerInternalAddress(Port exposedPort)
         {
             var serviceName = "service-" + numberSource.WorkflowNumber;
-            var port = GetInternalPort(recipe);
+            var port = exposedPort.Number;
 
             return new Address(
                 $"http://{serviceName}.{k8sNamespace}.svc.cluster.local",
                 port);
         }
         
-        private static int GetServicePort(Port[] servicePorts)
-        {
-            if (servicePorts.Any()) return servicePorts.First().Number;
-            return 0;
-        }
-
-        private static int GetInternalPort(ContainerRecipe recipe)
-        {
-            if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number;
-            return 0;
-        }
-
         private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             log.Debug();

Framework/Utils/Address.cs

@@ -10,5 +10,10 @@
 
         public string Host { get; }
         public int Port { get; }
+
+        public override string ToString()
+        {
+            return $"{Host}:{Port}";
+        }
     }
 }

Framework/Utils/Time.cs

@@ -46,33 +46,35 @@
 
         public static void Retry(Action action, string description)
         {
-            Retry(action, TimeSpan.FromMinutes(1), description);
+            Retry(action, 1, description);
         }
 
         public static T Retry<T>(Func<T> action, string description)
         {
-            return Retry(action, TimeSpan.FromMinutes(1), description);
+            return Retry(action, 1, description);
         }
 
-        public static void Retry(Action action, TimeSpan timeout, string description)
+        public static void Retry(Action action, int maxRetries, string description)
         {
-            Retry(action, timeout, TimeSpan.FromSeconds(1), description);
+            Retry(action, maxRetries, TimeSpan.FromSeconds(1), description);
         }
 
-        public static T Retry<T>(Func<T> action, TimeSpan timeout, string description)
+        public static T Retry<T>(Func<T> action, int maxRetries, string description)
         {
-            return Retry(action, timeout, TimeSpan.FromSeconds(1), description);
+            return Retry(action, maxRetries, TimeSpan.FromSeconds(1), description);
         }
 
-        public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description)
+        public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
+            var retries = 0;
             var exceptions = new List<Exception>();
             while (true)
             {
-                if (DateTime.UtcNow - start > timeout)
+                if (retries > maxRetries)
                 {
-                    throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
+                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
                 try
@@ -83,21 +85,24 @@
                 catch (Exception ex)
                 {
                     exceptions.Add(ex);
+                    retries++;
                 }
 
                 Sleep(retryTime);
             }
         }
 
-        public static T Retry<T>(Func<T> action, TimeSpan timeout, TimeSpan retryTime, string description)
+        public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
+            var retries = 0;
             var exceptions = new List<Exception>();
             while (true)
             {
-                if (DateTime.UtcNow - start > timeout)
+                if (retries > maxRetries)
                 {
-                    throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
+                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
                 try
@@ -107,6 +112,7 @@
                 catch (Exception ex)
                 {
                     exceptions.Add(ex);
+                    retries++;
                 }
 
                 Sleep(retryTime);

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -4,7 +4,7 @@ namespace CodexContractsPlugin
 {
     public class CodexContractsContainerRecipe : ContainerRecipeFactory
     {
-        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:latest-dist-tests";
+        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests";
 
         public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using Utils;
 
 namespace CodexPlugin
 {
@@ -98,12 +99,17 @@ namespace CodexPlugin
 
         private IHttp Http()
         {
-            return tools.CreateHttp(Container.Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
         }
 
         private IHttp LongHttp()
         {
-            return tools.CreateHttp(Container.Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
+            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
+        }
+
+        private Address GetAddress()
+        {
+            return Container.GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -8,8 +8,10 @@ namespace CodexPlugin
         private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter();
 
         private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
-        public const string MetricsPortTag = "metrics_port";
+        public const string ApiPortTag = "codex_api_port";
-        public const string DiscoveryPortTag = "discovery-port";
+        public const string ListenPortTag = "codex_listen_port";
+        public const string MetricsPortTag = "codex_metrics_port";
+        public const string DiscoveryPortTag = "codex_discovery_port";
 
         // Used by tests for time-constraint assertions.
         public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
@@ -27,20 +29,20 @@ namespace CodexPlugin
 
             var config = startupConfig.Get<CodexStartupConfig>();
 
-            AddExposedPortAndVar("CODEX_API_PORT");
+            AddExposedPortAndVar("CODEX_API_PORT", ApiPortTag);
             AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
 
             var dataDir = $"datadir{ContainerNumber}";
             AddEnvVar("CODEX_DATA_DIR", dataDir);
             AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
 
-            AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
+            AddExposedPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
             AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
 
             // This makes the node announce itself to its local (pod) IP address.
             AddEnvVar("NAT_IP_AUTO", "true");
 
-            var listenPort = AddInternalPort();
+            var listenPort = AddExposedPort(ListenPortTag);
             AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
 
             if (!string.IsNullOrEmpty(config.BootstrapSpr))

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -13,8 +13,9 @@ namespace CodexPlugin
         string GetName();
         CodexDebugResponse GetDebugInfo();
         CodexDebugPeerResponse GetDebugPeer(string peerId);
-        CodexDebugBlockExchangeResponse GetDebugBlockExchange();
+        // These debug methods are not available in master-line Codex. Use only for custom builds.
-        CodexDebugRepoStoreResponse[] GetDebugRepoStore();
+        //CodexDebugBlockExchangeResponse GetDebugBlockExchange();
+        //CodexDebugRepoStoreResponse[] GetDebugRepoStore();
         ContentId UploadFile(TrackedFile file);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(ICodexNode node);

ProjectPlugins/GethPlugin/GethNode.cs

@@ -73,7 +73,7 @@ namespace GethPlugin
 
         private NethereumInteraction StartInteraction()
         {
-            var address = StartResult.Container.Address;
+            var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
             var account = Account;
 
             var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -13,7 +13,7 @@ namespace MetricsPlugin
         public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
         {
             RunningContainer = runningContainer;
-            http = tools.CreateHttp(RunningContainer.Address, "api/v1");
+            http = tools.CreateHttp(RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag), "api/v1");
             log = tools.GetLog();
         }
 

ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs

@@ -7,11 +7,13 @@ namespace MetricsPlugin
         public override string AppName => "prometheus";
         public override string Image => "codexstorage/dist-tests-prometheus:latest";
 
+        public const string PortTag = "prometheus_port_tag";
+
         protected override void Initialize(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<PrometheusStartupConfig>();
 
-            AddExposedPortAndVar("PROM_PORT");
+            AddExposedPortAndVar("PROM_PORT", PortTag);
             AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
         }
     }

Tests/CodexContinuousTests/StartupChecker.cs

@@ -87,7 +87,8 @@ namespace ContinuousTests
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                log.Log($"Checking {n.Container.Name} @ '{n.Container.Address.Host}:{n.Container.Address.Port}'...");
+                var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
+                log.Log($"Checking {n.Container.Name} @ '{address}'...");
 
                 if (EnsureOnline(log, n))
                 {
@@ -95,7 +96,7 @@ namespace ContinuousTests
                 }
                 else
                 {
-                    log.Error($"No response from '{n.Container.Address.Host}'.");
+                    log.Error($"No response from '{address}'.");
                     pass = false;
                 }
             }

Tests/CodexContinuousTests/Tests/TwoClientTest.cs

@@ -57,8 +57,8 @@ namespace ContinuousTests.Tests
 
         private void LogRepoStore(ICodexNode codexNode)
         {
-            var response = codexNode.GetDebugRepoStore();
+            //var response = codexNode.GetDebugRepoStore();
-            Log.Log($"{codexNode.GetName()} has {string.Join(",", response.Select(r => r.cid))}");
+            //Log.Log($"{codexNode.GetName()} has {string.Join(",", response.Select(r => r.cid))}");
         }
 
         private void LogStoredBytes(ICodexNode node)
@@ -90,8 +90,8 @@ namespace ContinuousTests.Tests
 
         private void LogBlockExchangeStatus(ICodexNode codexNode, string msg)
         {
-            var response = codexNode.GetDebugBlockExchange();
+            //var response = codexNode.GetDebugBlockExchange();
-            Log.Log($"{codexNode.GetName()} {msg}: {JsonConvert.SerializeObject(response)}");
+            //Log.Log($"{codexNode.GetName()} {msg}: {JsonConvert.SerializeObject(response)}");
         }
     }
 }

Tests/CodexContinuousTests/deploy-and-run.sh

@@ -44,4 +44,6 @@ do
     --cleanup=1 \
     --full-container-logs=1 \
     --target-duration=172800 # 48 hours
+
+    sleep 30
 done

Tests/CodexTests/BasicTests/BlockExchangeTests.cs

@@ -42,21 +42,22 @@ namespace CodexTests.BasicTests
         {
             foreach (var node in nodes)
             {
-                Time.Retry(() => AssertBlockExchangeIsEmpty(node), nameof(AssertExchangeIsEmpty));
+                // API Call not available in master-line Codex image.
+                //Time.Retry(() => AssertBlockExchangeIsEmpty(node), nameof(AssertExchangeIsEmpty));
             }
         }
 
-        private void AssertBlockExchangeIsEmpty(ICodexNode node)
+        //private void AssertBlockExchangeIsEmpty(ICodexNode node)
-        {
+        //{
-            var msg = $"BlockExchange for {node.GetName()}: ";
+        //    var msg = $"BlockExchange for {node.GetName()}: ";
-            var response = node.GetDebugBlockExchange();
+        //    var response = node.GetDebugBlockExchange();
-            foreach (var peer in response.peers)
+        //    foreach (var peer in response.peers)
-            {
+        //    {
-                var activeWants = peer.wants.Where(w => !w.cancel).ToArray();
+        //        var activeWants = peer.wants.Where(w => !w.cancel).ToArray();
-                Assert.That(activeWants.Length, Is.EqualTo(0), msg + "thinks a peer has active wants.");
+        //        Assert.That(activeWants.Length, Is.EqualTo(0), msg + "thinks a peer has active wants.");
-            }
+        //    }
-            Assert.That(response.taskQueue, Is.EqualTo(0), msg + "has tasks in queue.");
+        //    Assert.That(response.taskQueue, Is.EqualTo(0), msg + "has tasks in queue.");
-            Assert.That(response.pendingBlocks, Is.EqualTo(0), msg + "has pending blocks.");
+        //    Assert.That(response.pendingBlocks, Is.EqualTo(0), msg + "has pending blocks.");
-        }
+        //}
     }
 }

Tools/CodexNetDeployer/Deployer.cs

@@ -180,9 +180,9 @@ namespace CodexNetDeployer
             return TimeSpan.FromSeconds(2);
         }
 
-        public TimeSpan HttpCallRetryTime()
+        public int HttpMaxNumberOfRetries()
         {
-            return TimeSpan.FromSeconds(2);
+            return 2;
         }
 
         public TimeSpan HttpCallTimeout()