Fixes port tag mismatch

Framework/Core/PluginTools.cs

@@ -23,6 +23,7 @@ namespace Core
     public interface IHttpFactoryTool
     {
         IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
+        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet timeSet, string? logAlias = null);
         IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null);
     }
 
@@ -53,7 +54,12 @@ namespace Core
 
         public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
-            return new Http(log, timeSet, address, baseUrl, onClientCreated, logAlias);
+            return CreateHttp(address, baseUrl, onClientCreated, timeSet, logAlias);
+        }
+
+        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet ts, string? logAlias = null)
+        {
+            return new Http(log, ts, address, baseUrl, onClientCreated, logAlias);
         }
 
         public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null)

Framework/Core/SerializeGate.cs

@@ -5,11 +5,9 @@ namespace Core
     public static class SerializeGate
     {
         /// <summary>
-        /// SerializeGate was added to help ensure deployment objects are serializable
-        /// and remain viable after deserialization.
+        /// SerializeGate was added to help ensure deployment objects are serializable and remain viable after deserialization.
         /// Tools can be built on top of the core interface that rely on deployment objects being serializable.
-        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes
-        /// don't break this requirement.
+        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes don't break this requirement.
         /// </summary>
         public static T Gate<T>(T anything)
         {

Framework/Core/TimeSet.cs

@@ -36,4 +36,32 @@
             return TimeSpan.FromMinutes(30);
         }
     }
+
+    public class LongTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromHours(2);
+        }
+
+        public TimeSpan HttpCallRetryTime()
+        {
+            return TimeSpan.FromHours(5);
+        }
+
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromSeconds(2);
+        }
+
+        public TimeSpan WaitForK8sServiceDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(15);
+        }
+    }
 }

Framework/KubernetesWorkflow/ContainerRecipe.cs

@@ -24,6 +24,8 @@
             {
                 Name = $"ctnr{Number}";
             }
+
+            if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
         }
 
         public string Name { get; }
@@ -65,6 +67,12 @@
 
         public int Number { get; }
         public string Tag { get; }
+
+        public override string ToString()
+        {
+            if (string.IsNullOrEmpty(Tag)) return $"untagged-port={Number}";
+            return $"{Tag}={Number}";
+        }
     }
 
     public class EnvVar

Framework/KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -50,12 +50,12 @@ namespace KubernetesWorkflow
         protected int Index { get; private set; } = 0;
         protected abstract void Initialize(StartupConfig config);
 
-        protected Port AddExposedPort(string tag = "")
+        protected Port AddExposedPort(string tag)
         {
             return AddExposedPort(factory.CreatePort(tag));
         }
 
-        protected Port AddExposedPort(int number, string tag = "")
+        protected Port AddExposedPort(int number, string tag)
         {
             return AddExposedPort(factory.CreatePort(number, tag));
         }
@@ -67,7 +67,7 @@ namespace KubernetesWorkflow
             return p;
         }
 
-        protected void AddExposedPortAndVar(string name, string tag = "")
+        protected void AddExposedPortAndVar(string name, string tag)
         {
             AddEnvVar(name, AddExposedPort(tag));
         }
@@ -132,11 +132,6 @@ namespace KubernetesWorkflow
 
         private Port AddExposedPort(Port port)
         {
-            if (exposedPorts.Any())
-            {
-                throw new NotImplementedException("Current implementation only support 1 exposed port per container recipe. " +
-                    $"Methods for determining container addresses in {nameof(StartupWorkflow)} currently rely on this constraint.");
-            }
             exposedPorts.Add(port);
             return port;
         }

Framework/KubernetesWorkflow/K8sController.cs

@@ -572,16 +572,15 @@ namespace KubernetesWorkflow
             var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
             foreach (var r in containerRecipes)
             {
-                if (r.ExposedPorts.Any())
+                foreach (var port in r.ExposedPorts)
                 {
-                    var firstExposedPort = r.ExposedPorts.First();
-                    var portName = GetNameForPort(r, firstExposedPort);
+                    var portName = GetNameForPort(r, port);
 
                     var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
                     if (matchingServicePorts.Any())
                     {
                         // These service ports belongs to this recipe.
-                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
+                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, port.Tag));
                         var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
 
                         result.Add(new ContainerRecipePortMapEntry(r.Number, ports));

Framework/KubernetesWorkflow/RunnerLocationUtils.cs

@@ -16,18 +16,26 @@ namespace KubernetesWorkflow
         internal static RunnerLocation DetermineRunnerLocation(RunningContainer container)
         {
             if (knownLocation != null) return knownLocation.Value;
+            knownLocation = PingForLocation(container);
+            return knownLocation.Value;
+        }
 
+        private static RunnerLocation PingForLocation(RunningContainer container)
+        {
             if (PingHost(container.Pod.PodInfo.Ip))
             {
-                knownLocation = RunnerLocation.InternalToCluster;
-            }
-            else if (PingHost(Format(container.ClusterExternalAddress)))
-            {
-                knownLocation = RunnerLocation.ExternalToCluster;
+                return RunnerLocation.InternalToCluster;
             }
 
-            if (knownLocation == null) throw new Exception("Unable to determine location relative to kubernetes cluster.");
-            return knownLocation.Value;
+            foreach (var port in container.ContainerPorts)
+            {
+                if (PingHost(Format(port.ExternalAddress)))
+                {
+                    return RunnerLocation.ExternalToCluster;
+                }
+            }
+
+            throw new Exception("Unable to determine location relative to kubernetes cluster.");
         }
 
         private static string Format(Address host)

Framework/KubernetesWorkflow/RunningContainers.cs

@@ -24,37 +24,46 @@ namespace KubernetesWorkflow
 
     public class RunningContainer
     {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress)
+        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, ContainerPort[] containerPorts)
         {
             Pod = pod;
             Recipe = recipe;
             ServicePorts = servicePorts;
             Name = name;
-            ClusterExternalAddress = clusterExternalAddress;
-            ClusterInternalAddress = clusterInternalAddress;
+            ContainerPorts = containerPorts;
         }
 
         public string Name { get; }
         public RunningPod Pod { get; }
         public ContainerRecipe Recipe { get; }
         public Port[] ServicePorts { get; }
-        public Address ClusterExternalAddress { get; }
-        public Address ClusterInternalAddress { get; }
+        public ContainerPort[] ContainerPorts { get; }
 
-        [JsonIgnore]
-        public Address Address
+        public Address GetAddress(string portTag)
         {
-            get
+            var containerPort = ContainerPorts.Single(c => c.Port.Tag == portTag);
+            if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
             {
-                if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
-                {
-                    return ClusterInternalAddress;
-                }
-                return ClusterExternalAddress;
+                return containerPort.InternalAddress;
             }
+            return containerPort.ExternalAddress;
         }
     }
 
+    public class ContainerPort
+    {
+        public ContainerPort(Port port, Address externalAddress, Address internalAddress)
+        {
+            Port = port;
+            ExternalAddress = externalAddress;
+            InternalAddress = internalAddress;
+        }
+
+        public Port Port { get; }
+        public Address ExternalAddress { get; }
+        public Address InternalAddress { get; }
+    }
+
     public static class RunningContainersExtensions
     {
         public static RunningContainer[] Containers(this RunningContainers[] runningContainers)

Framework/KubernetesWorkflow/RunningPod.cs

@@ -19,12 +19,10 @@
 
         public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
         {
-            if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
-            {
-                return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
-            }
-
-            return Array.Empty<Port>();
+            return PortMapEntries
+                .Where(p => p.ContainerNumber == containerRecipe.Number)
+                .SelectMany(p => p.Ports)
+                .ToArray();
         }
     }
 

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -118,8 +118,7 @@ namespace KubernetesWorkflow
                 var name = GetContainerName(r, startupConfig);
 
                 return new RunningContainer(runningPod, r, servicePorts, name,
-                    GetContainerExternalAddress(runningPod, servicePorts),
-                    GetContainerInternalAddress(r));
+                    CreateContainerPorts(runningPod, r, servicePorts));
 
             }).ToArray();
         }
@@ -137,35 +136,39 @@ namespace KubernetesWorkflow
             }
         }
 
-        private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        private ContainerPort[] CreateContainerPorts(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts)
         {
-            return new Address(
-                pod.Cluster.HostAddress,
-                GetServicePort(servicePorts));
+            var result = new List<ContainerPort>();
+            foreach (var exposedPort in recipe.ExposedPorts)
+            {
+                result.Add(new ContainerPort(
+                    exposedPort,
+                    GetContainerExternalAddress(pod, servicePorts, exposedPort),
+                    GetContainerInternalAddress(exposedPort)));
+            }
+
+            return result.ToArray();
         }
 
-        private Address GetContainerInternalAddress(ContainerRecipe recipe)
+        private static Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts, Port exposedPort)
+        {
+            var servicePort = servicePorts.Single(p => p.Tag == exposedPort.Tag);
+
+            return new Address(
+                pod.Cluster.HostAddress,
+                servicePort.Number);
+        }
+
+        private Address GetContainerInternalAddress(Port exposedPort)
         {
             var serviceName = "service-" + numberSource.WorkflowNumber;
-            var port = GetInternalPort(recipe);
+            var port = exposedPort.Number;
 
             return new Address(
                 $"http://{serviceName}.{k8sNamespace}.svc.cluster.local",
                 port);
         }
-
-        private static int GetServicePort(Port[] servicePorts)
-        {
-            if (servicePorts.Any()) return servicePorts.First().Number;
-            return 0;
-        }
-
-        private static int GetInternalPort(ContainerRecipe recipe)
-        {
-            if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number;
-            return 0;
-        }
-
+        
         private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             log.Debug();

Framework/Utils/Address.cs

@@ -10,5 +10,10 @@
 
         public string Host { get; }
         public int Port { get; }
+
+        public override string ToString()
+        {
+            return $"{Host}:{Port}";
+        }
     }
 }

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using Utils;
 
 namespace CodexPlugin
 {
@@ -49,6 +50,11 @@ namespace CodexPlugin
             return Http().HttpGetJson<CodexDebugBlockExchangeResponse>("debug/blockexchange");
         }
 
+        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
+        {
+            return LongHttp().HttpGetJson<CodexDebugRepoStoreResponse[]>("debug/repostore");
+        }
+
         public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
         {
             return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
@@ -93,7 +99,17 @@ namespace CodexPlugin
 
         private IHttp Http()
         {
-            return tools.CreateHttp(Container.Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+        }
+
+        private IHttp LongHttp()
+        {
+            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
+        }
+
+        private Address GetAddress()
+        {
+            return Container.GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)
@@ -106,6 +122,7 @@ namespace CodexPlugin
             var log = tools.GetLog();
             var file = log.CreateSubfile();
             log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
+            file.Write($"Container Crash Log for {Container.Name}.");
 
             using var reader = new StreamReader(crashLog);
             var line = reader.ReadLine();

ProjectPlugins/CodexPlugin/CodexApiTypes.cs

@@ -165,4 +165,9 @@ namespace CodexPlugin
         public string wantType { get; set; } = string.Empty;
         public bool sendDontHave { get; set; }
     }
+
+    public class CodexDebugRepoStoreResponse
+    {
+        public string cid { get; set; } = string.Empty; 
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -8,8 +8,10 @@ namespace CodexPlugin
         private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter();
 
         private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
-        public const string MetricsPortTag = "metrics_port";
-        public const string DiscoveryPortTag = "discovery-port";
+        public const string ApiPortTag = "codex_api_port";
+        public const string ListenPortTag = "codex_listen_port";
+        public const string MetricsPortTag = "codex_metrics_port";
+        public const string DiscoveryPortTag = "codex_discovery_port";
 
         // Used by tests for time-constraint assertions.
         public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
@@ -27,20 +29,20 @@ namespace CodexPlugin
 
             var config = startupConfig.Get<CodexStartupConfig>();
 
-            AddExposedPortAndVar("CODEX_API_PORT");
+            AddExposedPortAndVar("CODEX_API_PORT", ApiPortTag);
             AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
 
             var dataDir = $"datadir{ContainerNumber}";
             AddEnvVar("CODEX_DATA_DIR", dataDir);
             AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
 
-            AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
+            AddExposedPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
             AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
 
             // This makes the node announce itself to its local (pod) IP address.
             AddEnvVar("NAT_IP_AUTO", "true");
 
-            var listenPort = AddInternalPort();
+            var listenPort = AddExposedPort(ListenPortTag);
             AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
 
             if (!string.IsNullOrEmpty(config.BootstrapSpr))

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -21,9 +21,10 @@ namespace CodexPlugin
 
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
+        public DeploymentMetadata(DateTime startUtc, DateTime finishedUtc, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
         {
-            DeployDateTimeUtc = DateTime.UtcNow;
+            StartUtc = startUtc;
+            FinishedUtc = finishedUtc;
             KubeNamespace = kubeNamespace;
             NumberOfCodexNodes = numberOfCodexNodes;
             NumberOfValidators = numberOfValidators;
@@ -38,7 +39,8 @@ namespace CodexPlugin
             BlockMN = blockMN;
         }
 
-        public DateTime DeployDateTimeUtc { get; }
+        public DateTime StartUtc { get; }
+        public DateTime FinishedUtc { get; }
         public string KubeNamespace { get; }
         public int NumberOfCodexNodes { get; }
         public int NumberOfValidators { get; }

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -14,6 +14,7 @@ namespace CodexPlugin
         CodexDebugResponse GetDebugInfo();
         CodexDebugPeerResponse GetDebugPeer(string peerId);
         CodexDebugBlockExchangeResponse GetDebugBlockExchange();
+        CodexDebugRepoStoreResponse[] GetDebugRepoStore();
         ContentId UploadFile(TrackedFile file);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(ICodexNode node);
@@ -87,6 +88,11 @@ namespace CodexPlugin
             return CodexAccess.GetDebugBlockExchange();
         }
 
+        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
+        {
+            return CodexAccess.GetDebugRepoStore();
+        }
+
         public ContentId UploadFile(TrackedFile file)
         {
             using var fileStream = File.OpenRead(file.Filename);

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -8,7 +8,7 @@ namespace CodexPlugin
         public string? NameOverride { get; set; }
         public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation;
         public CodexLogLevel LogLevel { get; set; }
-        public CodexLogCustomTopics? CustomTopics { get; set; }
+        public CodexLogCustomTopics? CustomTopics { get; set; } = new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn);
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -205,7 +205,7 @@ namespace CodexPlugin
 
                 if (DateTime.UtcNow - waitStart > timeout)
                 {
-                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within timeout. {statusJson}");
+                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
                 }
             }
             log.Log($"Contract '{desiredState}'.");

ProjectPlugins/GethPlugin/GethNode.cs

@@ -73,7 +73,7 @@ namespace GethPlugin
 
         private NethereumInteraction StartInteraction()
         {
-            var address = StartResult.Container.Address;
+            var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
             var account = Account;
 
             var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -13,7 +13,7 @@ namespace MetricsPlugin
         public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
         {
             RunningContainer = runningContainer;
-            http = tools.CreateHttp(RunningContainer.Address, "api/v1");
+            http = tools.CreateHttp(RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag), "api/v1");
             log = tools.GetLog();
         }
 

ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs

@@ -7,11 +7,13 @@ namespace MetricsPlugin
         public override string AppName => "prometheus";
         public override string Image => "codexstorage/dist-tests-prometheus:latest";
 
+        public const string PortTag = "prometheus_port_tag";
+
         protected override void Initialize(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<PrometheusStartupConfig>();
 
-            AddExposedPortAndVar("PROM_PORT");
+            AddExposedPortAndVar("PROM_PORT", PortTag);
             AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
         }
     }

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -106,7 +106,7 @@ namespace ContinuousTests
             var effectiveStart = testStart.Subtract(TimeSpan.FromSeconds(30));
             if (config.FullContainerLogs)
             {
-                effectiveStart = config.CodexDeployment.Metadata.DeployDateTimeUtc.Subtract(TimeSpan.FromSeconds(30));
+                effectiveStart = config.CodexDeployment.Metadata.StartUtc.Subtract(TimeSpan.FromSeconds(30));
             }
             var effectiveEnd = DateTime.UtcNow;
             var elasticSearchLogDownloader = new ElasticSearchLogDownloader(entryPoint.Tools, fixtureLog);

Tests/CodexContinuousTests/StartupChecker.cs

@@ -87,7 +87,8 @@ namespace ContinuousTests
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                log.Log($"Checking {n.Container.Name} @ '{n.Container.Address.Host}:{n.Container.Address.Port}'...");
+                var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
+                log.Log($"Checking {n.Container.Name} @ '{address}'...");
 
                 if (EnsureOnline(log, n))
                 {
@@ -95,7 +96,7 @@ namespace ContinuousTests
                 }
                 else
                 {
-                    log.Error($"No response from '{n.Container.Address.Host}'.");
+                    log.Error($"No response from '{address}'.");
                     pass = false;
                 }
             }

Tests/CodexContinuousTests/Tests/TwoClientTest.cs

@@ -1,6 +1,7 @@
 using CodexPlugin;
 using FileUtils;
 using Logging;
+using Newtonsoft.Json;
 using NUnit.Framework;
 using Utils;
 
@@ -21,6 +22,9 @@ namespace ContinuousTests.Tests
         [TestMoment(t: Zero)]
         public void UploadTestFile()
         {
+            LogBlockExchangeStatus(Nodes[0], "Before upload");
+            LogBlockExchangeStatus(Nodes[1], "Before upload");
+
             file = FileManager.GenerateFile(size);
 
             LogStoredBytes(Nodes[0]);
@@ -34,9 +38,27 @@ namespace ContinuousTests.Tests
         {
             TrackedFile? dl = null;
 
-            LogBytesPerMillisecond(() => dl = Nodes[1].DownloadContent(cid!));
+            try
+            {
+                LogBytesPerMillisecond(() => dl = Nodes[1].DownloadContent(cid!));
 
-            file.AssertIsEqual(dl);
+                file.AssertIsEqual(dl);
+            }
+            catch
+            {
+                LogRepoStore(Nodes[0]);
+                LogRepoStore(Nodes[1]);
+                throw;
+            }
+
+            LogBlockExchangeStatus(Nodes[0], "After download");
+            LogBlockExchangeStatus(Nodes[1], "After download");
+        }
+
+        private void LogRepoStore(ICodexNode codexNode)
+        {
+            var response = codexNode.GetDebugRepoStore();
+            Log.Log($"{codexNode.GetName()} has {string.Join(",", response.Select(r => r.cid))}");
         }
 
         private void LogStoredBytes(ICodexNode node)
@@ -65,5 +87,11 @@ namespace ContinuousTests.Tests
             var bytesPerMs = totalBytes / totalMs;
             Log.Log($"Bytes per millisecond: {bytesPerMs}");
         }
+
+        private void LogBlockExchangeStatus(ICodexNode codexNode, string msg)
+        {
+            var response = codexNode.GetDebugBlockExchange();
+            Log.Log($"{codexNode.GetName()} {msg}: {JsonConvert.SerializeObject(response)}");
+        }
     }
 }

Tests/CodexContinuousTests/deploy-and-run.sh

@@ -1,6 +1,8 @@
 set -e
 
 replication=5
+name=testnamehere
+filter=TwoClient
 
 echo "Deploying..."
 cd ../../Tools/CodexNetDeployer
@@ -8,8 +10,8 @@ for i in $( seq 0 $replication)
 do
     dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
-    --kube-namespace=codex-continuous-tests-$i \
-    --deploy-file=codex-deployment-$i.json \
+    --kube-namespace=codex-continuous-$name-tests-$i \
+    --deploy-file=codex-deployment-$name-$i.json \
     --nodes=5 \
     --validators=3 \
     --log-level=Trace \
@@ -25,7 +27,7 @@ do
     --check-connect=1 \
     -y
 
-    cp codex-deployment-$i.json ../../Tests/CodexContinuousTests
+    cp codex-deployment-$name-$i.json ../../Tests/CodexContinuousTests
 done
 echo "Starting tests..."
 cd ../../Tests/CodexContinuousTests
@@ -33,12 +35,12 @@ for i in $( seq 0 $replication)
 do
     screen -d -m dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
-    --codex-deployment=codex-deployment-$i.json \
-    --log-path=logs-$i \
-    --data-path=data-$i \
+    --codex-deployment=codex-deployment-$name-$i.json \
+    --log-path=logs-$name-$i \
+    --data-path=data-$name-$i \
     --keep=1 \
     --stop=1 \
-    --filter=TwoClient \
+    --filter=$filter \
     --cleanup=1 \
     --full-container-logs=1 \
     --target-duration=172800 # 48 hours

Tests/CodexLongTests/BasicTests/DownloadTests.cs

@@ -1,7 +1,7 @@
+using CodexTests;
 using DistTestCore;
 using FileUtils;
 using NUnit.Framework;
-using Tests;
 using Utils;
 
 namespace CodexLongTests.BasicTests

Tests/CodexLongTests/BasicTests/LargeFileTests.cs

@@ -1,8 +1,8 @@
 using CodexPlugin;
+using CodexTests;
 using DistTestCore;
 using NUnit.Framework;
 using NUnit.Framework.Interfaces;
-using Tests;
 using Utils;
 
 namespace CodexLongTests.BasicTests

Tests/CodexLongTests/BasicTests/TestInfraTests.cs

@@ -1,6 +1,6 @@
-using DistTestCore;
+using CodexTests;
+using DistTestCore;
 using NUnit.Framework;
-using Tests;
 
 namespace CodexLongTests.BasicTests
 {

Tests/CodexLongTests/BasicTests/UploadTests.cs

@@ -1,8 +1,8 @@
 using CodexPlugin;
+using CodexTests;
 using DistTestCore;
 using FileUtils;
 using NUnit.Framework;
-using Tests;
 using Utils;
 
 namespace CodexLongTests.BasicTests

Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -1,6 +1,6 @@
-using DistTestCore;
+using CodexTests;
+using DistTestCore;
 using NUnit.Framework;
-using Tests;
 using Utils;
 
 namespace CodexLongTests.DownloadConnectivityTests

Tests/CodexTests/AutoBootstrapDistTest.cs

@@ -1,7 +1,7 @@
 using CodexPlugin;
 using NUnit.Framework;
 
-namespace Tests
+namespace CodexTests
 {
     public class AutoBootstrapDistTest : CodexDistTest
     {

Tests/CodexTests/BasicTests/BlockExchangeTests.cs

@@ -0,0 +1,62 @@
+using CodexPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.BasicTests
+{
+    [TestFixture]
+    public class BlockExchangeTests : CodexDistTest
+    {
+        [Test]
+        public void EmptyAfterExchange()
+        {
+            var bootstrap = AddCodex(s => s.WithName("bootstrap"));
+            var node = AddCodex(s => s.WithName("node").WithBootstrapNode(bootstrap));
+
+            AssertExchangeIsEmpty(bootstrap, node);
+
+            var file = GenerateTestFile(1.MB());
+            var cid = bootstrap.UploadFile(file);
+            node.DownloadContent(cid);
+
+            AssertExchangeIsEmpty(bootstrap, node);
+        }
+
+        [Test]
+        public void EmptyAfterExchangeWithBystander()
+        {
+            var bootstrap = AddCodex(s => s.WithName("bootstrap"));
+            var node = AddCodex(s => s.WithName("node").WithBootstrapNode(bootstrap));
+            var bystander = AddCodex(s => s.WithName("bystander").WithBootstrapNode(bootstrap));
+
+            AssertExchangeIsEmpty(bootstrap, node, bystander);
+
+            var file = GenerateTestFile(1.MB());
+            var cid = bootstrap.UploadFile(file);
+            node.DownloadContent(cid);
+
+            AssertExchangeIsEmpty(bootstrap, node, bystander);
+        }
+
+        private void AssertExchangeIsEmpty(params ICodexNode[] nodes)
+        {
+            foreach (var node in nodes)
+            {
+                Time.Retry(() => AssertBlockExchangeIsEmpty(node), nameof(AssertExchangeIsEmpty));
+            }
+        }
+
+        private void AssertBlockExchangeIsEmpty(ICodexNode node)
+        {
+            var msg = $"BlockExchange for {node.GetName()}: ";
+            var response = node.GetDebugBlockExchange();
+            foreach (var peer in response.peers)
+            {
+                var activeWants = peer.wants.Where(w => !w.cancel).ToArray();
+                Assert.That(activeWants.Length, Is.EqualTo(0), msg + "thinks a peer has active wants.");
+            }
+            Assert.That(response.taskQueue, Is.EqualTo(0), msg + "has tasks in queue.");
+            Assert.That(response.pendingBlocks, Is.EqualTo(0), msg + "has pending blocks.");
+        }
+    }
+}

Tests/CodexTests/BasicTests/ContinuousSubstitute.cs

@@ -7,7 +7,7 @@ using MetricsPlugin;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     [Ignore("Used for debugging continuous tests")]
     [TestFixture]
@@ -87,7 +87,7 @@ namespace Tests.BasicTests
                 //CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
                 //CheckRoutingTables(GetAllOnlineCodexNodes());
 
-                var node = RandomUtils.PickOneRandom(nodes.ToList());
+                var node = nodes.ToList().PickOneRandom();
                 var file = GenerateTestFile(50.MB());
                 node.UploadFile(file);
 

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -6,7 +6,7 @@ using MetricsPlugin;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     [TestFixture]
     public class ExampleTests : CodexDistTest
@@ -59,7 +59,7 @@ namespace Tests.BasicTests
                 .WithStorageQuota(11.GB())
                 .EnableMarketplace(geth, contracts, initialEth: 10.Eth(), initialTokens: sellerInitialBalance, isValidator: true)
                 .WithSimulateProofFailures(failEveryNProofs: 3));
-            
+
             AssertBalance(geth, contracts, seller, Is.EqualTo(sellerInitialBalance));
             seller.Marketplace.MakeStorageAvailable(
                 size: 10.GB(),
@@ -72,7 +72,7 @@ namespace Tests.BasicTests
             var buyer = AddCodex(s => s
                             .WithBootstrapNode(seller)
                             .EnableMarketplace(geth, contracts, initialEth: 10.Eth(), initialTokens: buyerInitialBalance));
-            
+
             AssertBalance(geth, contracts, buyer, Is.EqualTo(buyerInitialBalance));
 
             var contentId = buyer.UploadFile(testFile);

Tests/CodexTests/BasicTests/NetworkIsolationTest.cs

@@ -3,7 +3,7 @@ using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     // Warning!
     // This is a test to check network-isolation in the test-infrastructure.

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -3,7 +3,7 @@ using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     [TestFixture]
     public class OneClientTests : DistTest

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -1,8 +1,7 @@
-using DistTestCore;
-using NUnit.Framework;
+using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     [TestFixture]
     public class ThreeClientTest : AutoBootstrapDistTest

Tests/CodexTests/BasicTests/TwoClientTests.cs

@@ -3,7 +3,7 @@ using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     [TestFixture]
     public class TwoClientTests : DistTest

Tests/CodexTests/CodexDistTest.cs

@@ -9,7 +9,7 @@ using GethPlugin;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 
-namespace Tests
+namespace CodexTests
 {
     public class CodexDistTest : DistTest
     {

Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexTests;
 using GethPlugin;
 using NUnit.Framework;
 using Utils;

Tests/CodexTests/MetricsAccessExtensions.cs

@@ -3,7 +3,7 @@ using Logging;
 using MetricsPlugin;
 using NUnit.Framework.Constraints;
 
-namespace Tests
+namespace CodexTests
 {
     public static class MetricsAccessExtensions
     {

Tests/CodexTests/Parallelism.cs

@@ -1,6 +1,6 @@
 using NUnit.Framework;
 
 [assembly: LevelOfParallelism(1)]
-namespace Tests
+namespace CodexTests
 {
 }

Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -1,4 +1,5 @@
 using CodexPlugin;
+using CodexTests;
 using NUnit.Framework;
 
 namespace Tests.PeerDiscoveryTests

Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexTests;
 using GethPlugin;
 using NUnit.Framework;
 

Tests/DistTestCore/LongTimeSet.cs

@@ -1,32 +0,0 @@
-using Core;
-
-namespace DistTestCore
-{
-    public class LongTimeSet : ITimeSet
-    {
-        public TimeSpan HttpCallTimeout()
-        {
-            return TimeSpan.FromHours(2);
-        }
-
-        public TimeSpan HttpCallRetryTime()
-        {
-            return TimeSpan.FromHours(5);
-        }
-
-        public TimeSpan HttpCallRetryDelay()
-        {
-            return TimeSpan.FromSeconds(2);
-        }
-
-        public TimeSpan WaitForK8sServiceDelay()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(15);
-        }
-    }
-}

Tools/CodexNetDeployer/Deployer.cs

@@ -51,6 +51,7 @@ namespace CodexNetDeployer
             localCodexBuilder.Build();
 
             Log("Initializing...");
+            var startUtc = DateTime.UtcNow;
             var ci = entryPoint.CreateInterface();
 
             Log("Deploying Geth instance...");
@@ -78,7 +79,7 @@ namespace CodexNetDeployer
             CheckContainerRestarts(startResults);
 
             var codexContainers = startResults.Select(s => s.CodexNode.Container).ToArray();
-            return new CodexDeployment(codexContainers, gethDeployment, metricsService, CreateMetadata());
+            return new CodexDeployment(codexContainers, gethDeployment, metricsService, CreateMetadata(startUtc));
         }
 
         private EntryPoint CreateEntryPoint(ILog log)
@@ -87,11 +88,11 @@ namespace CodexNetDeployer
 
             var configuration = new KubernetesWorkflow.Configuration(
                 kubeConfig,
-                operationTimeout: TimeSpan.FromSeconds(300),
+                operationTimeout: TimeSpan.FromMinutes(10),
                 retryDelay: TimeSpan.FromSeconds(10),
                 kubernetesNamespace: config.KubeNamespace);
 
-            var result = new EntryPoint(log, configuration, string.Empty);
+            var result = new EntryPoint(log, configuration, string.Empty, new FastHttpTimeSet());
             configuration.Hooks = new K8sHook(config.TestsTypePodLabel, result.GetPluginMetadata());
 
             return result;
@@ -147,9 +148,11 @@ namespace CodexNetDeployer
             }
         }
 
-        private DeploymentMetadata CreateMetadata()
+        private DeploymentMetadata CreateMetadata(DateTime startUtc)
         {
             return new DeploymentMetadata(
+                startUtc: startUtc,
+                finishedUtc: DateTime.UtcNow,
                 kubeNamespace: config.KubeNamespace,
                 numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
                 numberOfValidators: config.NumberOfValidators!.Value,
@@ -169,4 +172,32 @@ namespace CodexNetDeployer
             Console.WriteLine(msg);
         }
     }
+
+    public class FastHttpTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromSeconds(2);
+        }
+
+        public TimeSpan HttpCallRetryTime()
+        {
+            return TimeSpan.FromSeconds(2);
+        }
+
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(10);
+        }
+
+        public TimeSpan WaitForK8sServiceDelay()
+        {
+            return TimeSpan.FromSeconds(30);
+        }
+    }
 }