Merge branch 'feature/public-testnet-deploying' into automated-teststarter

Framework/Core/Http.cs

@@ -13,15 +13,15 @@ namespace Core
         T HttpGetJson<T>(string route);
         TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
         string HttpPostJson<TRequest>(string route, TRequest body);
-        string HttpPostString(string route, string body);
         TResponse HttpPostString<TResponse>(string route, string body);
         string HttpPostStream(string route, Stream stream);
         Stream HttpGetStream(string route);
-        T TryJsonDeserialize<T>(string json);
+        T Deserialize<T>(string json);
     }
 
     internal class Http : IHttp
     {
+        private static readonly object httpLock = new object();
         private readonly ILog log;
         private readonly ITimeSet timeSet;
         private readonly Address address;
@@ -48,70 +48,60 @@ namespace Core
 
         public string HttpGetString(string route)
         {
-            return Retry(() =>
+            return LockRetry(() =>
             {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "");
-                var result = Time.Wait(client.GetAsync(url));
-                var str = Time.Wait(result.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str; ;
+                return GetString(route);
             }, $"HTTP-GET:{route}");
         }
 
         public T HttpGetJson<T>(string route)
         {
-            var json = HttpGetString(route);
-            return TryJsonDeserialize<T>(json);
+            return LockRetry(() =>
+            {
+                var json = GetString(route);
+                return Deserialize<T>(json);
+            }, $"HTTP-GET:{route}");
         }
 
         public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
         {
-            var response = PostJson(route, body);
-            var json = Time.Wait(response.Content.ReadAsStringAsync());
-            if (!response.IsSuccessStatusCode)
+            return LockRetry(() =>
             {
-                throw new HttpRequestException(json);
-            }
-            Log(GetUrl() + route, json);
-            return TryJsonDeserialize<TResponse>(json);
+                var response = PostJson(route, body);
+                var json = Time.Wait(response.Content.ReadAsStringAsync());
+                if (!response.IsSuccessStatusCode)
+                {
+                    throw new HttpRequestException(json);
+                }
+                Log(GetUrl() + route, json);
+                return Deserialize<TResponse>(json);
+            }, $"HTTP-POST-JSON: {route}");
         }
 
         public string HttpPostJson<TRequest>(string route, TRequest body)
         {
-            var response = PostJson(route, body);
-            return Time.Wait(response.Content.ReadAsStringAsync());
-        }
-
-        public string HttpPostString(string route, string body)
-        {
-            return Retry(() =>
+            return LockRetry(() =>
             {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, body);
-                var content = new StringContent(body);
-                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
-                var result = Time.Wait(client.PostAsync(url, content));
-                var str = Time.Wait(result.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str;
-            }, $"HTTP-POST-STRING: {route}");
+                var response = PostJson(route, body);
+                return Time.Wait(response.Content.ReadAsStringAsync());
+            }, $"HTTP-POST-JSON: {route}");
         }
 
         public TResponse HttpPostString<TResponse>(string route, string body)
         {
-            var response = HttpPostString(route, body);
-            if (response == null) throw new Exception("Received no response.");
-            var result = JsonConvert.DeserializeObject<TResponse>(response);
-            if (result == null) throw new Exception("Failed to deserialize response");
-            return result;
+            return LockRetry(() =>
+            {
+                var response = PostJsonString(route, body);
+                if (response == null) throw new Exception("Received no response.");
+                var result = Deserialize<TResponse>(response);
+                if (result == null) throw new Exception("Failed to deserialize response");
+                return result;
+            }, $"HTTO-POST-JSON: {route}");
         }
 
         public string HttpPostStream(string route, Stream stream)
         {
-            return Retry(() =>
+            return LockRetry(() =>
             {
                 using var client = GetClient();
                 var url = GetUrl() + route;
@@ -127,7 +117,7 @@ namespace Core
 
         public Stream HttpGetStream(string route)
         {
-            return Retry(() =>
+            return LockRetry(() =>
             {
                 var client = GetClient();
                 var url = GetUrl() + route;
@@ -136,7 +126,7 @@ namespace Core
             }, $"HTTP-GET-STREAM: {route}");
         }
 
-        public T TryJsonDeserialize<T>(string json)
+        public T Deserialize<T>(string json)
         {
             var errors = new List<string>();
             var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
@@ -154,7 +144,7 @@ namespace Core
                     }
                 }
             });
-            if (errors.Count() > 0)
+            if (errors.Count > 0)
             {
                 throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
             }
@@ -165,16 +155,37 @@ namespace Core
             return deserialized;
         }
 
+        private string GetString(string route)
+        {
+            using var client = GetClient();
+            var url = GetUrl() + route;
+            Log(url, "");
+            var result = Time.Wait(client.GetAsync(url));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
         private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
         {
-            return Retry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                using var content = JsonContent.Create(body);
-                Log(url, JsonConvert.SerializeObject(body));
-                return Time.Wait(client.PostAsync(url, content));
-            }, $"HTTP-POST-JSON: {route}");
+            using var client = GetClient();
+            var url = GetUrl() + route;
+            using var content = JsonContent.Create(body);
+            Log(url, JsonConvert.SerializeObject(body));
+            return Time.Wait(client.PostAsync(url, content));
+        }
+
+        private string PostJsonString(string route, string body)
+        {
+            using var client = GetClient();
+            var url = GetUrl() + route;
+            Log(url, body);
+            var content = new StringContent(body);
+            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+            var result = Time.Wait(client.PostAsync(url, content));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
         }
 
         private string GetUrl()
@@ -194,9 +205,12 @@ namespace Core
             }
         }
 
-        private T Retry<T>(Func<T> operation, string description)
+        private T LockRetry<T>(Func<T> operation, string description)
         {
-            return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
+            lock (httpLock)
+            {
+                return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
+            }
         }
 
         private HttpClient GetClient()

Framework/Core/PluginTools.cs

@@ -23,6 +23,7 @@ namespace Core
     public interface IHttpFactoryTool
     {
         IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
+        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet timeSet, string? logAlias = null);
         IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null);
     }
 
@@ -53,7 +54,12 @@ namespace Core
 
         public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
-            return new Http(log, timeSet, address, baseUrl, onClientCreated, logAlias);
+            return CreateHttp(address, baseUrl, onClientCreated, timeSet, logAlias);
+        }
+
+        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet ts, string? logAlias = null)
+        {
+            return new Http(log, ts, address, baseUrl, onClientCreated, logAlias);
         }
 
         public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null)

Framework/Core/SerializeGate.cs

@@ -5,11 +5,9 @@ namespace Core
     public static class SerializeGate
     {
         /// <summary>
-        /// SerializeGate was added to help ensure deployment objects are serializable
-        /// and remain viable after deserialization.
+        /// SerializeGate was added to help ensure deployment objects are serializable and remain viable after deserialization.
         /// Tools can be built on top of the core interface that rely on deployment objects being serializable.
-        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes
-        /// don't break this requirement.
+        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes don't break this requirement.
         /// </summary>
         public static T Gate<T>(T anything)
         {

Framework/Core/TimeSet.cs

@@ -3,7 +3,7 @@
     public interface ITimeSet
     {
         TimeSpan HttpCallTimeout();
-        TimeSpan HttpCallRetryTime();
+        int HttpMaxNumberOfRetries();
         TimeSpan HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
@@ -13,12 +13,12 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromMinutes(5);
+            return TimeSpan.FromMinutes(3);
         }
 
-        public TimeSpan HttpCallRetryTime()
+        public int HttpMaxNumberOfRetries()
         {
-            return TimeSpan.FromMinutes(1);
+            return 3;
         }
 
         public TimeSpan HttpCallRetryDelay()
@@ -36,4 +36,32 @@
             return TimeSpan.FromMinutes(30);
         }
     }
+
+    public class LongTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromHours(2);
+        }
+
+        public int HttpMaxNumberOfRetries()
+        {
+            return 1;
+        }
+
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromSeconds(2);
+        }
+
+        public TimeSpan WaitForK8sServiceDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(15);
+        }
+    }
 }

Framework/KubernetesWorkflow/ContainerRecipe.cs

@@ -24,6 +24,8 @@
             {
                 Name = $"ctnr{Number}";
             }
+
+            if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
         }
 
         public string Name { get; }
@@ -57,14 +59,38 @@
 
     public class Port
     {
-        public Port(int number, string tag)
+        public Port(int number, string tag, PortProtocol protocol)
         {
             Number = number;
             Tag = tag;
+            Protocol = protocol;
         }
 
         public int Number { get; }
         public string Tag { get; }
+        public PortProtocol Protocol { get; }
+
+        public bool IsTcp()
+        {
+            return Protocol == PortProtocol.TCP;
+        }
+
+        public bool IsUdp()
+        {
+            return Protocol == PortProtocol.UDP;
+        }
+
+        public override string ToString()
+        {
+            if (string.IsNullOrEmpty(Tag)) return $"untagged-port={Number}/{Protocol}";
+            return $"{Tag}={Number}/{Protocol}";
+        }
+    }
+
+    public enum PortProtocol
+    {
+        TCP,
+        UDP
     }
 
     public class EnvVar
@@ -86,15 +112,21 @@
 
     public class VolumeMount
     {
-        public VolumeMount(string volumeName, string mountPath, string resourceQuantity)
+        public VolumeMount(string volumeName, string mountPath, string? subPath = null, string? resourceQuantity = null, string? secret = null, string? hostPath = null)
         {
             VolumeName = volumeName;
             MountPath = mountPath;
+            SubPath = subPath;
             ResourceQuantity = resourceQuantity;
+            Secret = secret;
+            HostPath = hostPath;
         }
 
         public string VolumeName { get; }
         public string MountPath { get; }
-        public string ResourceQuantity { get; }
+        public string? SubPath { get; }
+        public string? ResourceQuantity { get; }
+        public string? Secret { get; }
+        public string? HostPath { get; }
     }
 }

Framework/KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -50,31 +50,31 @@ namespace KubernetesWorkflow
         protected int Index { get; private set; } = 0;
         protected abstract void Initialize(StartupConfig config);
 
-        protected Port AddExposedPort(string tag = "")
+        protected Port AddExposedPort(string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(tag));
+            return AddExposedPort(factory.CreatePort(tag, protocol));
         }
 
-        protected Port AddExposedPort(int number, string tag = "")
+        protected Port AddExposedPort(int number, string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(number, tag));
+            return AddExposedPort(factory.CreatePort(number, tag, protocol));
         }
 
-        protected Port AddInternalPort(string tag = "")
+        protected Port AddInternalPort(string tag = "", PortProtocol protocol = PortProtocol.TCP)
         {
-            var p = factory.CreatePort(tag);
+            var p = factory.CreatePort(tag, protocol);
             internalPorts.Add(p);
             return p;
         }
 
-        protected void AddExposedPortAndVar(string name, string tag = "")
+        protected void AddExposedPortAndVar(string name, string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            AddEnvVar(name, AddExposedPort(tag));
+            AddEnvVar(name, AddExposedPort(tag, protocol));
         }
 
-        protected void AddInternalPortAndVar(string name, string tag = "")
+        protected void AddInternalPortAndVar(string name, string tag = "", PortProtocol protocol = PortProtocol.TCP)
         {
-            AddEnvVar(name, AddInternalPort(tag));
+            AddEnvVar(name, AddInternalPort(tag, protocol));
         }
 
         protected void AddEnvVar(string name, string value)
@@ -97,12 +97,18 @@ namespace KubernetesWorkflow
             podAnnotations.Add(name, value);
         }
 
+        protected void AddVolume(string name, string mountPath, string? subPath = null, string? secret = null, string? hostPath = null)
+        {
+            var size = 10.MB().ToSuffixNotation();
+            volumeMounts.Add(new VolumeMount(name, mountPath, subPath, size, secret, hostPath));
+        }
+
         protected void AddVolume(string mountPath, ByteSize volumeSize)
         {
             volumeMounts.Add(new VolumeMount(
                 $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
                 mountPath,
-                volumeSize.ToSuffixNotation()));
+                resourceQuantity: volumeSize.ToSuffixNotation()));
         }
 
         protected void Additional(object userData)
@@ -132,11 +138,6 @@ namespace KubernetesWorkflow
 
         private Port AddExposedPort(Port port)
         {
-            if (exposedPorts.Any())
-            {
-                throw new NotImplementedException("Current implementation only support 1 exposed port per container recipe. " +
-                    $"Methods for determining container addresses in {nameof(StartupWorkflow)} currently rely on this constraint.");
-            }
             exposedPorts.Add(port);
             return port;
         }

Framework/KubernetesWorkflow/K8sController.cs

@@ -9,15 +9,14 @@ namespace KubernetesWorkflow
     {
         private readonly ILog log;
         private readonly K8sCluster cluster;
-        private readonly KnownK8sPods knownPods;
         private readonly WorkflowNumberSource workflowNumberSource;
         private readonly K8sClient client;
+        private const string podLabelKey = "pod-uuid";
 
-        public K8sController(ILog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
+        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
         {
             this.log = log;
             this.cluster = cluster;
-            this.knownPods = knownPods;
             this.workflowNumberSource = workflowNumberSource;
             client = new K8sClient(cluster.GetK8sClientConfig());
 
@@ -34,13 +33,30 @@ namespace KubernetesWorkflow
             log.Debug();
             EnsureNamespace();
 
-            var deploymentName = CreateDeployment(containerRecipes, location);
+            var podLabel = K8sNameUtils.Format(Guid.NewGuid().ToString());
+            var deploymentName = CreateDeployment(containerRecipes, location, podLabel);
             var (serviceName, servicePortsMap) = CreateService(containerRecipes);
-            var podInfo = FetchNewPod();
+
+            var pod = FindPodByLabel(podLabel);
+            var podInfo = CreatePodInfo(pod);
 
             return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray());
         }
 
+        private V1Pod FindPodByLabel(string podLabel)
+        {
+            var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace));
+            foreach (var pod in pods.Items)
+            {
+                var label = pod.GetLabel(podLabelKey);
+                if (label == podLabel)
+                {
+                    return pod;
+                }
+            }
+            throw new Exception("Unable to find pod by label.");
+        }
+
         public void Stop(RunningPod pod)
         {
             log.Debug();
@@ -81,10 +97,6 @@ namespace KubernetesWorkflow
             {
                 DeleteNamespace(ns);
             }
-            foreach (var ns in namespaces)
-            {
-                WaitUntilNamespaceDeleted(ns);
-            }
         }
 
         public void DeleteNamespace()
@@ -94,7 +106,6 @@ namespace KubernetesWorkflow
             {
                 client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
             }
-            WaitUntilNamespaceDeleted();
         }
 
         public void DeleteNamespace(string ns)
@@ -304,7 +315,7 @@ namespace KubernetesWorkflow
 
         #region Deployment management
 
-        private string CreateDeployment(ContainerRecipe[] containerRecipes, ILocation location)
+        private string CreateDeployment(ContainerRecipe[] containerRecipes, ILocation location, string podLabel)
         {
             var deploymentSpec = new V1Deployment
             {
@@ -321,7 +332,7 @@ namespace KubernetesWorkflow
                     {
                         Metadata = new V1ObjectMeta
                         {
-                            Labels = GetSelector(containerRecipes),
+                            Labels = GetSelector(containerRecipes, podLabel),
                             Annotations = GetAnnotations(containerRecipes)
                         },
                         Spec = new V1PodSpec
@@ -368,6 +379,13 @@ namespace KubernetesWorkflow
             return containerRecipes.First().PodLabels.GetLabels();
         }
 
+        private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes, string podLabel)
+        {
+            var labels = containerRecipes.First().PodLabels.Clone();
+            labels.Add(podLabelKey, podLabel);
+            return labels.GetLabels();
+        }
+
         private IDictionary<string, string> GetRunnerNamespaceSelector()
         {
             return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "default" } };
@@ -446,7 +464,8 @@ namespace KubernetesWorkflow
             return new V1VolumeMount
             {
                 Name = v.VolumeName,
-                MountPath = v.MountPath
+                MountPath = v.MountPath,
+                SubPath = v.SubPath,
             };
         }
 
@@ -462,28 +481,28 @@ namespace KubernetesWorkflow
 
         private V1Volume CreateVolume(VolumeMount v)
         {
-            client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim
+            CreatePersistentVolumeClaimIfNeeded(v);
+
+            if (!string.IsNullOrEmpty(v.HostPath))
             {
-                ApiVersion = "v1",
-                Metadata = new V1ObjectMeta
+                return new V1Volume
                 {
-                    Name = v.VolumeName
-                },
-                Spec = new V1PersistentVolumeClaimSpec
-                {
-                    AccessModes = new List<string>
+                    Name = v.VolumeName,
+                    HostPath = new V1HostPathVolumeSource
                     {
-                        "ReadWriteOnce"
-                    },
-                    Resources = new V1ResourceRequirements
-                    {
-                        Requests = new Dictionary<string, ResourceQuantity>
-                        {
-                            {"storage", new ResourceQuantity(v.ResourceQuantity) }
-                        }
+                        Path = v.HostPath
                     }
-                }
-            }, K8sNamespace));
+                };
+            }
+
+            if (!string.IsNullOrEmpty(v.Secret))
+            {
+                return new V1Volume
+                {
+                    Name = v.VolumeName,
+                    Secret = CreateVolumeSecret(v)
+                };
+            }
 
             return new V1Volume
             {
@@ -495,6 +514,50 @@ namespace KubernetesWorkflow
             };
         }
 
+        private void CreatePersistentVolumeClaimIfNeeded(VolumeMount v)
+        {
+            var pvcs = client.Run(c => c.ListNamespacedPersistentVolumeClaim(K8sNamespace));
+            if (pvcs != null && pvcs.Items.Any(i => i.Name() == v.VolumeName)) return;
+
+            client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim
+            {
+                ApiVersion = "v1",
+                Metadata = new V1ObjectMeta
+                {
+                    Name = v.VolumeName,
+                },
+                Spec = new V1PersistentVolumeClaimSpec
+                {
+                    AccessModes = new List<string>
+                    {
+                        "ReadWriteOnce"
+                    },
+                    Resources = CreateVolumeResourceRequirements(v),
+                },
+            }, K8sNamespace));
+        }
+
+        private V1SecretVolumeSource CreateVolumeSecret(VolumeMount v)
+        {
+            if (string.IsNullOrWhiteSpace(v.Secret)) return null!;
+            return new V1SecretVolumeSource
+            {
+                SecretName = v.Secret
+            };
+        }
+
+        private V1ResourceRequirements CreateVolumeResourceRequirements(VolumeMount v)
+        {
+            if (v.ResourceQuantity == null) return null!;
+            return new V1ResourceRequirements
+            {
+                Requests = new Dictionary<string, ResourceQuantity>()
+                {
+                    {"storage", new ResourceQuantity(v.ResourceQuantity) }
+                }
+            };
+        }
+
         private List<V1EnvVar> CreateEnv(ContainerRecipe recipe)
         {
             return recipe.EnvVars.Select(CreateEnvVar).ToList();
@@ -511,23 +574,42 @@ namespace KubernetesWorkflow
 
         private List<V1ContainerPort> CreateContainerPorts(ContainerRecipe recipe)
         {
-            var exposedPorts = recipe.ExposedPorts.Select(p => CreateContainerPort(recipe, p));
-            var internalPorts = recipe.InternalPorts.Select(p => CreateContainerPort(recipe, p));
+            var exposedPorts = recipe.ExposedPorts.SelectMany(p => CreateContainerPort(recipe, p));
+            var internalPorts = recipe.InternalPorts.SelectMany(p => CreateContainerPort(recipe, p));
             return exposedPorts.Concat(internalPorts).ToList();
         }
 
-        private V1ContainerPort CreateContainerPort(ContainerRecipe recipe, Port port)
+        private List<V1ContainerPort> CreateContainerPort(ContainerRecipe recipe, Port port)
+        {
+            var result = new List<V1ContainerPort>();
+            if (port.IsTcp()) CreateTcpContainerPort(result, recipe, port);
+            if (port.IsUdp()) CreateUdpContainerPort(result, recipe, port);
+            return result;
+        }
+
+        private void CreateUdpContainerPort(List<V1ContainerPort> result, ContainerRecipe recipe, Port port)
+        {
+            result.Add(CreateContainerPort(recipe, port, "UDP"));
+        }
+
+        private void CreateTcpContainerPort(List<V1ContainerPort> result, ContainerRecipe recipe, Port port)
+        {
+            result.Add(CreateContainerPort(recipe, port, "TCP"));
+        }
+
+        private V1ContainerPort CreateContainerPort(ContainerRecipe recipe, Port port, string protocol)
         {
             return new V1ContainerPort
             {
                 Name = GetNameForPort(recipe, port),
-                ContainerPort = port.Number
+                ContainerPort = port.Number,
+                Protocol = protocol
             };
         }
 
         private string GetNameForPort(ContainerRecipe recipe, Port port)
         {
-            return $"p{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}";
+            return $"p{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}-{port.Protocol.ToString().ToLowerInvariant()}";
         }
 
         #endregion
@@ -572,16 +654,15 @@ namespace KubernetesWorkflow
             var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
             foreach (var r in containerRecipes)
             {
-                if (r.ExposedPorts.Any())
+                foreach (var port in r.ExposedPorts)
                 {
-                    var firstExposedPort = r.ExposedPorts.First();
-                    var portName = GetNameForPort(r, firstExposedPort);
+                    var portName = GetNameForPort(r, port);
 
                     var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
                     if (matchingServicePorts.Any())
                     {
                         // These service ports belongs to this recipe.
-                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
+                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, port.Tag, port.Protocol));
                         var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
 
                         result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
@@ -590,10 +671,10 @@ namespace KubernetesWorkflow
             }
         }
 
-        private Port? MapNodePortIfAble(V1ServicePort p, string tag)
+        private Port? MapNodePortIfAble(V1ServicePort p, string tag, PortProtocol protocol)
         {
             if (p.NodePort == null) return null;
-            return new Port(p.NodePort.Value, tag);
+            return new Port(p.NodePort.Value, tag, protocol);
         }
 
         private void DeleteService(string serviceName)
@@ -625,18 +706,23 @@ namespace KubernetesWorkflow
             var result = new List<V1ServicePort>();
             foreach (var port in recipe.ExposedPorts)
             {
-                result.Add(new V1ServicePort
-                {
-                    Name = GetNameForPort(recipe, port),
-                    Protocol = "TCP",
-                    Port = port.Number,
-                    TargetPort = GetNameForPort(recipe, port),
-                });                
+                if (port.IsTcp()) CreateServicePort(result, recipe, port, "TCP");
+                if (port.IsUdp()) CreateServicePort(result, recipe, port, "UDP");
             }
-
             return result;
         }
 
+        private void CreateServicePort(List<V1ServicePort> result, ContainerRecipe recipe, Port port, string protocol)
+        {
+            result.Add(new V1ServicePort
+            {
+                Name = GetNameForPort(recipe, port),
+                Protocol = protocol,
+                Port = port.Number,
+                TargetPort = GetNameForPort(recipe, port),
+            });
+        }
+
         #endregion
 
         #region Waiting
@@ -646,16 +732,6 @@ namespace KubernetesWorkflow
             WaitUntil(() => IsNamespaceOnline(K8sNamespace));
         }
 
-        private void WaitUntilNamespaceDeleted()
-        {
-            WaitUntil(() => !IsNamespaceOnline(K8sNamespace));
-        }
-
-        private void WaitUntilNamespaceDeleted(string name)
-        {
-            WaitUntil(() => !IsNamespaceOnline(name));
-        }
-
         private void WaitUntilDeploymentOnline(string deploymentName)
         {
             WaitUntil(() =>
@@ -705,22 +781,15 @@ namespace KubernetesWorkflow
             return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sNamespace, container);
         }
 
-        private PodInfo FetchNewPod()
+        private PodInfo CreatePodInfo(V1Pod pod)
         {
-            var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items;
-
-            var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
-            if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");
-
-            var newPod = newPods.Single();
-            var name = newPod.Name();
-            var ip = newPod.Status.PodIP;
-            var k8sNodeName = newPod.Spec.NodeName;
+            var name = pod.Name();
+            var ip = pod.Status.PodIP;
+            var k8sNodeName = pod.Spec.NodeName;
 
             if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
             if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
 
-            knownPods.Add(name);
             return new PodInfo(name, ip, k8sNodeName);
         }
     }

Framework/KubernetesWorkflow/KnownK8sPods.cs

@@ -1,17 +0,0 @@
-namespace KubernetesWorkflow
-{
-    public class KnownK8sPods
-    {
-        private readonly List<string> knownActivePodNames = new List<string>();
-
-        public bool Contains(string name)
-        {
-            return knownActivePodNames.Contains(name);
-        }
-
-        public void Add(string name)
-        {
-            knownActivePodNames.Add(name);
-        }
-    }
-}

Framework/KubernetesWorkflow/RecipeComponentFactory.cs

@@ -7,14 +7,14 @@ namespace KubernetesWorkflow
     {
         private NumberSource portNumberSource = new NumberSource(8080);
 
-        public Port CreatePort(int number, string tag)
+        public Port CreatePort(int number, string tag, PortProtocol protocol)
         {
-            return new Port(number, tag);
+            return new Port(number, tag, protocol);
         }
 
-        public Port CreatePort(string tag)
+        public Port CreatePort(string tag, PortProtocol protocol)
         {
-            return new Port(portNumberSource.GetNextNumber(), tag);
+            return new Port(portNumberSource.GetNextNumber(), tag, protocol);
         }
 
         public EnvVar CreateEnvVar(string name, int value)

Framework/KubernetesWorkflow/RunnerLocationUtils.cs

@@ -16,18 +16,26 @@ namespace KubernetesWorkflow
         internal static RunnerLocation DetermineRunnerLocation(RunningContainer container)
         {
             if (knownLocation != null) return knownLocation.Value;
+            knownLocation = PingForLocation(container);
+            return knownLocation.Value;
+        }
 
+        private static RunnerLocation PingForLocation(RunningContainer container)
+        {
             if (PingHost(container.Pod.PodInfo.Ip))
             {
-                knownLocation = RunnerLocation.InternalToCluster;
-            }
-            else if (PingHost(Format(container.ClusterExternalAddress)))
-            {
-                knownLocation = RunnerLocation.ExternalToCluster;
+                return RunnerLocation.InternalToCluster;
             }
 
-            if (knownLocation == null) throw new Exception("Unable to determine location relative to kubernetes cluster.");
-            return knownLocation.Value;
+            foreach (var port in container.ContainerPorts)
+            {
+                if (port.ExternalAddress.IsValid() && PingHost(Format(port.ExternalAddress)))
+                {
+                    return RunnerLocation.ExternalToCluster;
+                }
+            }
+
+            throw new Exception("Unable to determine location relative to kubernetes cluster.");
         }
 
         private static string Format(Address host)

Framework/KubernetesWorkflow/RunningContainers.cs

@@ -1,5 +1,4 @@
-using Newtonsoft.Json;
-using Utils;
+using Utils;
 
 namespace KubernetesWorkflow
 {
@@ -24,35 +23,50 @@ namespace KubernetesWorkflow
 
     public class RunningContainer
     {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress)
+        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, ContainerPort[] containerPorts)
         {
             Pod = pod;
             Recipe = recipe;
             ServicePorts = servicePorts;
             Name = name;
-            ClusterExternalAddress = clusterExternalAddress;
-            ClusterInternalAddress = clusterInternalAddress;
+            ContainerPorts = containerPorts;
         }
 
         public string Name { get; }
         public RunningPod Pod { get; }
         public ContainerRecipe Recipe { get; }
         public Port[] ServicePorts { get; }
-        public Address ClusterExternalAddress { get; }
-        public Address ClusterInternalAddress { get; }
+        public ContainerPort[] ContainerPorts { get; }
 
-        [JsonIgnore]
-        public Address Address
+        public ContainerPort GetContainerPort(string portTag)
         {
-            get
-            {
-                if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
-                {
-                    return ClusterInternalAddress;
-                }
-                return ClusterExternalAddress;
-            }
+            return ContainerPorts.Single(c => c.Port.Tag == portTag);
         }
+
+        public Address GetAddress(string portTag)
+        {
+            var containerPort = GetContainerPort(portTag);
+            if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
+            {
+                return containerPort.InternalAddress;
+            }
+            if (!containerPort.ExternalAddress.IsValid()) throw new Exception($"Getting address by tag {portTag} resulted in an invalid address.");
+            return containerPort.ExternalAddress;
+        }
+    }
+
+    public class ContainerPort
+    {
+        public ContainerPort(Port port, Address externalAddress, Address internalAddress)
+        {
+            Port = port;
+            ExternalAddress = externalAddress;
+            InternalAddress = internalAddress;
+        }
+
+        public Port Port { get; }
+        public Address ExternalAddress { get; }
+        public Address InternalAddress { get; }
     }
 
     public static class RunningContainersExtensions

Framework/KubernetesWorkflow/RunningPod.cs

@@ -19,12 +19,10 @@
 
         public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
         {
-            if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
-            {
-                return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
-            }
-
-            return Array.Empty<Port>();
+            return PortMapEntries
+                .Where(p => p.ContainerNumber == containerRecipe.Number)
+                .SelectMany(p => p.Ports)
+                .ToArray();
         }
     }
 

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -1,4 +1,5 @@
 using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace KubernetesWorkflow
@@ -21,17 +22,15 @@ namespace KubernetesWorkflow
         private readonly ILog log;
         private readonly WorkflowNumberSource numberSource;
         private readonly K8sCluster cluster;
-        private readonly KnownK8sPods knownK8SPods;
         private readonly string k8sNamespace;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
         private readonly LocationProvider locationProvider;
 
-        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string k8sNamespace)
+        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
-            this.knownK8SPods = knownK8SPods;
             this.k8sNamespace = k8sNamespace;
 
             locationProvider = new LocationProvider(log, K8s);
@@ -118,8 +117,7 @@ namespace KubernetesWorkflow
                 var name = GetContainerName(r, startupConfig);
 
                 return new RunningContainer(runningPod, r, servicePorts, name,
-                    GetContainerExternalAddress(runningPod, servicePorts),
-                    GetContainerInternalAddress(r));
+                    CreateContainerPorts(runningPod, r, servicePorts));
 
             }).ToArray();
         }
@@ -137,35 +135,46 @@ namespace KubernetesWorkflow
             }
         }
 
-        private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        private ContainerPort[] CreateContainerPorts(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts)
         {
+            var result = new List<ContainerPort>();
+            foreach (var exposedPort in recipe.ExposedPorts)
+            {
+                result.Add(new ContainerPort(
+                    exposedPort,
+                    GetContainerExternalAddress(pod, servicePorts, exposedPort),
+                    GetContainerInternalAddress(pod, exposedPort)));
+            }
+            foreach (var internalPort in recipe.InternalPorts)
+            {
+                if (!string.IsNullOrEmpty(internalPort.Tag))
+                {
+                    result.Add(new ContainerPort(
+                        internalPort,
+                        new Address(string.Empty, 0),
+                        GetContainerInternalAddress(pod, internalPort)));
+                }
+            }
+
+            return result.ToArray();
+        }
+
+        private static Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts, Port exposedPort)
+        {
+            var servicePort = servicePorts.Single(p => p.Tag == exposedPort.Tag);
+
             return new Address(
                 pod.Cluster.HostAddress,
-                GetServicePort(servicePorts));
+                servicePort.Number);
         }
 
-        private Address GetContainerInternalAddress(ContainerRecipe recipe)
+        private Address GetContainerInternalAddress(RunningPod pod, Port port)
         {
-            var serviceName = "service-" + numberSource.WorkflowNumber;
-            var port = GetInternalPort(recipe);
-
             return new Address(
-                $"http://{serviceName}.{k8sNamespace}.svc.cluster.local",
-                port);
+                $"http://{pod.PodInfo.Ip}",
+                port.Number);
         }
-
-        private static int GetServicePort(Port[] servicePorts)
-        {
-            if (servicePorts.Any()) return servicePorts.First().Number;
-            return 0;
-        }
-
-        private static int GetInternalPort(ContainerRecipe recipe)
-        {
-            if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number;
-            return 0;
-        }
-
+        
         private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             log.Debug();
@@ -183,17 +192,33 @@ namespace KubernetesWorkflow
 
         private void K8s(Action<K8sController> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace);
-            action(controller);
-            controller.Dispose();
+            try
+            {
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
+                action(controller);
+                controller.Dispose();
+            }
+            catch (k8s.Autorest.HttpOperationException ex)
+            {
+                log.Error(JsonConvert.SerializeObject(ex));
+                throw;
+            }
         }
 
         private T K8s<T>(Func<K8sController, T> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace);
-            var result = action(controller);
-            controller.Dispose();
-            return result;
+            try
+            {
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
+                var result = action(controller);
+                controller.Dispose();
+                return result;
+            }
+            catch (k8s.Autorest.HttpOperationException ex)
+            {
+                log.Error(JsonConvert.SerializeObject(ex));
+                throw;
+            }
         }
     }
 

Framework/KubernetesWorkflow/WorkflowCreator.cs

@@ -7,7 +7,6 @@ namespace KubernetesWorkflow
     {
         private readonly NumberSource numberSource = new NumberSource(0);
         private readonly NumberSource containerNumberSource = new NumberSource(0);
-        private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly K8sCluster cluster;
         private readonly ILog log;
         private readonly Configuration configuration;
@@ -26,7 +25,7 @@ namespace KubernetesWorkflow
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, GetNamespace(namespaceOverride));
+            return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride));
         }
 
         private string GetNamespace(string? namespaceOverride)

Framework/Logging/BaseLog.cs

@@ -39,13 +39,12 @@ namespace Logging
             LogFile.Write(ApplyReplacements(message));
         }
 
-        public virtual void Debug(string message = "", int skipFrames = 0)
+        public void Debug(string message = "", int skipFrames = 0)
         {
             if (debug)
             {
                 var callerName = DebugStack.GetCallerName(skipFrames);
-                // We don't use Log because in the debug output we should not have any replacements.
-                LogFile.Write($"(debug)({callerName}) {message}");
+                Log($"(debug)({callerName}) {message}");
             }
         }
 
@@ -73,6 +72,7 @@ namespace Logging
 
         private string ApplyReplacements(string str)
         {
+            if (debug) return str;
             foreach (var replacement in replacements)
             {
                 str = replacement.Apply(str);

Framework/Logging/NullLog.cs

@@ -17,10 +17,6 @@
         {
         }
 
-        public override void Debug(string message = "", int skipFrames = 0)
-        {
-        }
-
         public override void Error(string message)
         {
             Console.WriteLine("Error: " + message);

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -19,29 +19,34 @@ namespace NethereumWorkflow
 
         public void SendEth(string toAddress, decimal ethAmount)
         {
+            log.Debug();
             var receipt = Time.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(toAddress, ethAmount));
             if (!receipt.Succeeded()) throw new Exception("Unable to send Eth");
         }
 
         public decimal GetEthBalance()
         {
+            log.Debug();
             return GetEthBalance(web3.TransactionManager.Account.Address);
         }
 
         public decimal GetEthBalance(string address)
         {
+            log.Debug();
             var balance = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(address));
             return Web3.Convert.FromWei(balance.Value);
         }
 
         public TResult Call<TFunction, TResult>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
+            log.Debug(typeof(TFunction).ToString());
             var handler = web3.Eth.GetContractQueryHandler<TFunction>();
             return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function));
         }
 
         public void SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
+            log.Debug();
             var handler = web3.Eth.GetContractTransactionHandler<TFunction>();
             var receipt = Time.Wait(handler.SendRequestAndWaitForReceiptAsync(contractAddress, function));
             if (!receipt.Succeeded()) throw new Exception("Unable to perform contract transaction.");

Framework/NethereumWorkflow/NethereumInteractionCreator.cs

@@ -20,6 +20,7 @@ namespace NethereumWorkflow
 
         public NethereumInteraction CreateWorkflow()
         {
+            log.Debug("Starting interaction to " + ip + ":" + port);
             return new NethereumInteraction(log, CreateWeb3());
         }
 

Framework/Utils/Address.cs

@@ -10,5 +10,15 @@
 
         public string Host { get; }
         public int Port { get; }
+
+        public override string ToString()
+        {
+            return $"{Host}:{Port}";
+        }
+
+        public bool IsValid()
+        {
+            return !string.IsNullOrEmpty(Host) && Port > 0;
+        }
     }
 }

Framework/Utils/Time.cs

@@ -46,33 +46,35 @@
 
         public static void Retry(Action action, string description)
         {
-            Retry(action, TimeSpan.FromMinutes(1), description);
+            Retry(action, 1, description);
         }
 
         public static T Retry<T>(Func<T> action, string description)
         {
-            return Retry(action, TimeSpan.FromMinutes(1), description);
+            return Retry(action, 1, description);
         }
 
-        public static void Retry(Action action, TimeSpan timeout, string description)
+        public static void Retry(Action action, int maxRetries, string description)
         {
-            Retry(action, timeout, TimeSpan.FromSeconds(1), description);
+            Retry(action, maxRetries, TimeSpan.FromSeconds(1), description);
         }
 
-        public static T Retry<T>(Func<T> action, TimeSpan timeout, string description)
+        public static T Retry<T>(Func<T> action, int maxRetries, string description)
         {
-            return Retry(action, timeout, TimeSpan.FromSeconds(1), description);
+            return Retry(action, maxRetries, TimeSpan.FromSeconds(1), description);
         }
 
-        public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description)
+        public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
+            var retries = 0;
             var exceptions = new List<Exception>();
             while (true)
             {
-                if (DateTime.UtcNow - start > timeout)
+                if (retries > maxRetries)
                 {
-                    throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
+                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
                 try
@@ -83,21 +85,24 @@
                 catch (Exception ex)
                 {
                     exceptions.Add(ex);
+                    retries++;
                 }
 
                 Sleep(retryTime);
             }
         }
 
-        public static T Retry<T>(Func<T> action, TimeSpan timeout, TimeSpan retryTime, string description)
+        public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
+            var retries = 0;
             var exceptions = new List<Exception>();
             while (true)
             {
-                if (DateTime.UtcNow - start > timeout)
+                if (retries > maxRetries)
                 {
-                    throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
+                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
                 try
@@ -107,6 +112,7 @@
                 catch (Exception ex)
                 {
                     exceptions.Add(ex);
+                    retries++;
                 }
 
                 Sleep(retryTime);

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -7,41 +7,43 @@ namespace CodexContractsPlugin
     {
         CodexContractsDeployment Deployment { get; }
 
-        void MintTestTokens(IGethNode gethNode, IHasEthAddress owner, TestToken testTokens);
-        void MintTestTokens(IGethNode gethNode, EthAddress ethAddress, TestToken testTokens);
-        TestToken GetTestTokenBalance(IGethNode gethNode, IHasEthAddress owner);
-        TestToken GetTestTokenBalance(IGethNode gethNode, EthAddress ethAddress);
+        void MintTestTokens(IHasEthAddress owner, TestToken testTokens);
+        void MintTestTokens(EthAddress ethAddress, TestToken testTokens);
+        TestToken GetTestTokenBalance(IHasEthAddress owner);
+        TestToken GetTestTokenBalance(EthAddress ethAddress);
     }
 
     public class CodexContractsAccess : ICodexContracts
     {
         private readonly ILog log;
+        private readonly IGethNode gethNode;
 
-        public CodexContractsAccess(ILog log, CodexContractsDeployment deployment)
+        public CodexContractsAccess(ILog log, IGethNode gethNode, CodexContractsDeployment deployment)
         {
             this.log = log;
+            this.gethNode = gethNode;
             Deployment = deployment;
         }
 
         public CodexContractsDeployment Deployment { get; }
 
-        public void MintTestTokens(IGethNode gethNode, IHasEthAddress owner, TestToken testTokens)
+        public void MintTestTokens(IHasEthAddress owner, TestToken testTokens)
         {
-            MintTestTokens(gethNode, owner.EthAddress, testTokens);
+            MintTestTokens(owner.EthAddress, testTokens);
         }
 
-        public void MintTestTokens(IGethNode gethNode, EthAddress ethAddress, TestToken testTokens)
+        public void MintTestTokens(EthAddress ethAddress, TestToken testTokens)
         {
             var interaction = new ContractInteractions(log, gethNode);
             interaction.MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
         }
 
-        public TestToken GetTestTokenBalance(IGethNode gethNode, IHasEthAddress owner)
+        public TestToken GetTestTokenBalance(IHasEthAddress owner)
         {
-            return GetTestTokenBalance(gethNode, owner.EthAddress);
+            return GetTestTokenBalance(owner.EthAddress);
         }
 
-        public TestToken GetTestTokenBalance(IGethNode gethNode, EthAddress ethAddress)
+        public TestToken GetTestTokenBalance(EthAddress ethAddress)
         {
             var interaction = new ContractInteractions(log, gethNode);
             var balance = interaction.GetBalance(Deployment.TokenAddress, ethAddress.Address);

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -1,10 +1,11 @@
-using KubernetesWorkflow;
+using GethPlugin;
+using KubernetesWorkflow;
 
 namespace CodexContractsPlugin
 {
     public class CodexContractsContainerRecipe : ContainerRecipeFactory
     {
-        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:latest-dist-tests";
+        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests";
 
         public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
@@ -16,10 +17,10 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var ip = config.GethNode.StartResult.Container.Pod.PodInfo.Ip;
-            var port = config.GethNode.StartResult.HttpPort.Number;
+            var containerPort = config.GethNode.StartResult.Container.GetContainerPort(GethContainerRecipe.HttpPortTag);
+            var address = containerPort.InternalAddress;
 
-            AddEnvVar("DISTTEST_NETWORK_URL", $"http://{ip}:{port}");
+            AddEnvVar("DISTTEST_NETWORK_URL", address.ToString());
             AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");
             AddEnvVar("KEEP_ALIVE", "1");
         }

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs

@@ -35,10 +35,10 @@ namespace CodexContractsPlugin
             return starter.Deploy(ci, gethNode);
         }
 
-        public ICodexContracts WrapDeploy(CodexContractsDeployment deployment)
+        public ICodexContracts WrapDeploy(IGethNode gethNode, CodexContractsDeployment deployment)
         {
             deployment = SerializeGate.Gate(deployment);
-            return starter.Wrap(deployment);
+            return starter.Wrap(gethNode, deployment);
         }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -31,20 +31,23 @@ namespace CodexContractsPlugin
 
             try
             {
-                return DeployContract(container, workflow, gethNode);
+                var result = DeployContract(container, workflow, gethNode);
+                workflow.Stop(containers);
+                Log("Container stopped.");
+                return result;
             }
-            catch
+            catch (Exception ex)
             {
-                Log("Failed to deploy contract.");
+                Log("Failed to deploy contract: " + ex);
                 Log("Downloading Codex SmartContracts container log...");
                 ci.DownloadLog(container);
                 throw;
             }
         }
 
-        public ICodexContracts Wrap(CodexContractsDeployment deployment)
+        public ICodexContracts Wrap(IGethNode gethNode, CodexContractsDeployment deployment)
         {
-            return new CodexContractsAccess(tools.GetLog(), deployment);
+            return new CodexContractsAccess(tools.GetLog(), gethNode, deployment);
         }
 
         private CodexContractsDeployment DeployContract(RunningContainer container, IStartupWorkflow workflow, IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -28,6 +28,7 @@ namespace CodexContractsPlugin
 
         public void MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
         {
+            log.Debug($"{amount} -> {address} (token: {tokenAddress})");
             MintTokens(address.Address, amount, tokenAddress);
         }
 
@@ -44,6 +45,7 @@ namespace CodexContractsPlugin
 
         public bool IsSynced(string marketplaceAddress, string marketplaceAbi)
         {
+            log.Debug();
             try
             {
                 return IsBlockNumberOK() && IsContractAvailable(marketplaceAddress, marketplaceAbi);

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -25,6 +25,7 @@ namespace CodexContractsPlugin
             var marketplaceAddress = Retry(FetchMarketplaceAddress);
             if (string.IsNullOrEmpty(marketplaceAddress)) throw new InvalidOperationException("Unable to fetch marketplace account from codex-contracts node. Test infra failure.");
 
+            log.Debug("Got MarketplaceAddress: " + marketplaceAddress);
             return marketplaceAddress;
         }
 
@@ -34,6 +35,7 @@ namespace CodexContractsPlugin
             var marketplaceAbi = Retry(FetchMarketplaceAbi);
             if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
 
+            log.Debug("Got Marketplace ABI: " + marketplaceAbi);
             return marketplaceAbi;
         }
 

ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs

@@ -10,15 +10,15 @@ namespace CodexContractsPlugin
             return Plugin(ci).DeployContracts(ci, gethNode);
         }
 
-        public static ICodexContracts WrapCodexContractsDeployment(this CoreInterface ci, CodexContractsDeployment deployment)
+        public static ICodexContracts WrapCodexContractsDeployment(this CoreInterface ci, IGethNode gethNode, CodexContractsDeployment deployment)
         {
-            return Plugin(ci).WrapDeploy(deployment);
+            return Plugin(ci).WrapDeploy(gethNode, deployment);
         }
 
         public static ICodexContracts StartCodexContracts(this CoreInterface ci, IGethNode gethNode)
         {
             var deployment = DeployCodexContracts(ci, gethNode);
-            return WrapCodexContractsDeployment(ci, deployment);
+            return WrapCodexContractsDeployment(ci, gethNode, deployment);
         }
 
         private static CodexContractsPlugin Plugin(CoreInterface ci)

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -0,0 +1,46 @@
+using Core;
+using KubernetesWorkflow;
+
+namespace CodexDiscordBotPlugin
+{
+    public class CodexDiscordBotPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
+    {
+        private readonly IPluginTools tools;
+
+        public CodexDiscordBotPlugin(IPluginTools tools)
+        {
+            this.tools = tools;
+        }
+
+        public string LogPrefix => "(DiscordBot) ";
+
+        public void Announce()
+        {
+            tools.GetLog().Log($"Codex DiscordBot (BiblioTech) loaded.");
+        }
+
+        public void AddMetadata(IAddMetadata metadata)
+        {
+            metadata.Add("codexdiscordbotid", new DiscordBotContainerRecipe().Image);
+        }
+
+        public void Decommission()
+        {
+        }
+
+        public RunningContainer Deploy(DiscordBotStartupConfig config)
+        {
+            var workflow = tools.CreateWorkflow();
+            return StartContainer(workflow, config);
+        }
+
+        private RunningContainer StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
+        {
+            var startupConfig = new StartupConfig();
+            startupConfig.NameOverride = config.Name;
+            startupConfig.Add(config);
+            var rc = workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
+            return rc.Containers.Single();
+        }
+    }
+}

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
+    <ProjectReference Include="..\CodexPlugin\CodexPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs

@@ -0,0 +1,18 @@
+using Core;
+using KubernetesWorkflow;
+
+namespace CodexDiscordBotPlugin
+{
+    public static class CoreInterfaceExtensions
+    {
+        public static RunningContainer DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
+        {
+            return Plugin(ci).Deploy(config);
+        }
+
+        private static CodexDiscordBotPlugin Plugin(CoreInterface ci)
+        {
+            return ci.GetPlugin<CodexDiscordBotPlugin>();
+        }
+    }
+}

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -0,0 +1,27 @@
+using KubernetesWorkflow;
+using Utils;
+
+namespace CodexDiscordBotPlugin
+{
+    public class DiscordBotContainerRecipe : ContainerRecipeFactory
+    {
+        public override string AppName => "discordbot-bibliotech";
+        public override string Image => "thatbenbierens/codex-discordbot:initial";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<DiscordBotStartupConfig>();
+
+            AddEnvVar("TOKEN", config.Token);
+            AddEnvVar("SERVERNAME", config.ServerName);
+            AddEnvVar("ADMINROLE", config.AdminRoleName);
+            AddEnvVar("ADMINCHANNELNAME", config.AdminChannelName);
+
+            if (!string.IsNullOrEmpty(config.DataPath))
+            {
+                AddEnvVar("DATAPATH", config.DataPath);
+                AddVolume(config.DataPath, 1.GB());
+            }
+        }
+    }
+}

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs

@@ -0,0 +1,21 @@
+namespace CodexDiscordBotPlugin
+{
+    public class DiscordBotStartupConfig
+    {
+        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName)
+        {
+            Name = name;
+            Token = token;
+            ServerName = serverName;
+            AdminRoleName = adminRoleName;
+            AdminChannelName = adminChannelName;
+        }
+
+        public string Name { get; }
+        public string Token { get; }
+        public string ServerName { get; }
+        public string AdminRoleName { get; }
+        public string AdminChannelName { get; }
+        public string? DataPath { get; set; }
+    }
+}

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using Utils;
 
 namespace CodexPlugin
 {
@@ -39,7 +40,7 @@ namespace CodexPlugin
                 };
             }
 
-            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
+            var result = http.Deserialize<CodexDebugPeerResponse>(str);
             result.IsPeerFound = true;
             return result;
         }
@@ -49,6 +50,11 @@ namespace CodexPlugin
             return Http().HttpGetJson<CodexDebugBlockExchangeResponse>("debug/blockexchange");
         }
 
+        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
+        {
+            return LongHttp().HttpGetJson<CodexDebugRepoStoreResponse[]>("debug/repostore");
+        }
+
         public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
         {
             return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
@@ -93,7 +99,17 @@ namespace CodexPlugin
 
         private IHttp Http()
         {
-            return tools.CreateHttp(Container.Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+        }
+
+        private IHttp LongHttp()
+        {
+            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
+        }
+
+        private Address GetAddress()
+        {
+            return Container.GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)
@@ -106,6 +122,7 @@ namespace CodexPlugin
             var log = tools.GetLog();
             var file = log.CreateSubfile();
             log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
+            file.Write($"Container Crash Log for {Container.Name}.");
 
             using var reader = new StreamReader(crashLog);
             var line = reader.ReadLine();

ProjectPlugins/CodexPlugin/CodexApiTypes.cs

@@ -165,4 +165,9 @@ namespace CodexPlugin
         public string wantType { get; set; } = string.Empty;
         public bool sendDontHave { get; set; }
     }
+
+    public class CodexDebugRepoStoreResponse
+    {
+        public string cid { get; set; } = string.Empty; 
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -8,8 +8,10 @@ namespace CodexPlugin
         private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter();
 
         private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
-        public const string MetricsPortTag = "metrics_port";
-        public const string DiscoveryPortTag = "discovery-port";
+        public const string ApiPortTag = "codex_api_port";
+        public const string ListenPortTag = "codex_listen_port";
+        public const string MetricsPortTag = "codex_metrics_port";
+        public const string DiscoveryPortTag = "codex_discovery_port";
 
         // Used by tests for time-constraint assertions.
         public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
@@ -27,20 +29,30 @@ namespace CodexPlugin
 
             var config = startupConfig.Get<CodexStartupConfig>();
 
-            AddExposedPortAndVar("CODEX_API_PORT");
+            var apiPort = CreateApiPort(config, ApiPortTag);
+            AddEnvVar("CODEX_API_PORT", apiPort);
             AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
 
             var dataDir = $"datadir{ContainerNumber}";
             AddEnvVar("CODEX_DATA_DIR", dataDir);
             AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
 
-            AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
+            var discPort = CreateDiscoveryPort(config);
+            AddEnvVar("CODEX_DISC_PORT", discPort);
             AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
 
-            // This makes the node announce itself to its local (pod) IP address.
-            AddEnvVar("NAT_IP_AUTO", "true");
+            if (config.PublicTestNet != null)
+            {
+                AddEnvVar("CODEX_NAT", config.PublicTestNet.PublicNatIP);
+                AddEnvVar("NAT_IP_AUTO", "false");
+            }
+            else
+            {
+                // This makes the node announce itself to its local (pod) IP address.
+                AddEnvVar("NAT_IP_AUTO", "true");
+            }
 
-            var listenPort = AddInternalPort();
+            var listenPort = CreateListenPort(config);
             AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
 
             if (!string.IsNullOrEmpty(config.BootstrapSpr))
@@ -65,7 +77,7 @@ namespace CodexPlugin
             }
             if (config.MetricsEnabled)
             {
-                var metricsPort = AddInternalPort(MetricsPortTag);
+                var metricsPort = CreateApiPort(config, MetricsPortTag);
                 AddEnvVar("CODEX_METRICS", "true");
                 AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
                 AddEnvVar("CODEX_METRICS_PORT", metricsPort);
@@ -108,6 +120,26 @@ namespace CodexPlugin
             }
         }
 
+        private Port CreateApiPort(CodexStartupConfig config, string tag)
+        {
+            if (config.PublicTestNet == null) return AddExposedPort(tag);
+            return AddInternalPort(tag);
+        }
+
+        private Port CreateListenPort(CodexStartupConfig config)
+        {
+            if (config.PublicTestNet == null) return AddInternalPort(ListenPortTag);
+
+            return AddExposedPort(config.PublicTestNet.PublicListenPort, ListenPortTag);
+        }
+
+        private Port CreateDiscoveryPort(CodexStartupConfig config)
+        {
+            if (config.PublicTestNet == null) return AddInternalPort(DiscoveryPortTag, PortProtocol.UDP);
+
+            return AddExposedPort(config.PublicTestNet.PublicDiscoveryPort, DiscoveryPortTag, PortProtocol.UDP);
+        }
+
         private ByteSize GetVolumeCapacity(CodexStartupConfig config)
         {
             if (config.StorageQuota != null) return config.StorageQuota;

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -1,29 +1,48 @@
-using GethPlugin;
+using CodexContractsPlugin;
+using GethPlugin;
 using KubernetesWorkflow;
 
 namespace CodexPlugin
 {
     public class CodexDeployment
     {
-        public CodexDeployment(RunningContainer[] codexContainers, GethDeployment gethDeployment, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
+        public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment, CodexContractsDeployment codexContractsDeployment, RunningContainer? prometheusContainer, RunningContainer? discordBotContainer, DeploymentMetadata metadata)
         {
-            CodexContainers = codexContainers;
+            CodexInstances = codexInstances;
             GethDeployment = gethDeployment;
+            CodexContractsDeployment = codexContractsDeployment;
             PrometheusContainer = prometheusContainer;
+            DiscordBotContainer = discordBotContainer;
             Metadata = metadata;
         }
 
-        public RunningContainer[] CodexContainers { get; }
+        public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
+        public CodexContractsDeployment CodexContractsDeployment { get; }
         public RunningContainer? PrometheusContainer { get; }
+        public RunningContainer? DiscordBotContainer { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
+    public class CodexInstance
+    {
+        public CodexInstance(RunningContainer container, CodexDebugResponse info)
+        {
+            Container = container;
+            Info = info;
+        }
+
+        public RunningContainer Container { get; }
+        public CodexDebugResponse Info { get; }
+    }
+
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
+        public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
         {
-            DeployDateTimeUtc = DateTime.UtcNow;
+            Name = name;
+            StartUtc = startUtc;
+            FinishedUtc = finishedUtc;
             KubeNamespace = kubeNamespace;
             NumberOfCodexNodes = numberOfCodexNodes;
             NumberOfValidators = numberOfValidators;
@@ -38,7 +57,9 @@ namespace CodexPlugin
             BlockMN = blockMN;
         }
 
-        public DateTime DeployDateTimeUtc { get; }
+        public string Name { get; }
+        public DateTime StartUtc { get; }
+        public DateTime FinishedUtc { get; }
         public string KubeNamespace { get; }
         public int NumberOfCodexNodes { get; }
         public int NumberOfValidators { get; }

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -13,7 +13,9 @@ namespace CodexPlugin
         string GetName();
         CodexDebugResponse GetDebugInfo();
         CodexDebugPeerResponse GetDebugPeer(string peerId);
-        CodexDebugBlockExchangeResponse GetDebugBlockExchange();
+        // These debug methods are not available in master-line Codex. Use only for custom builds.
+        //CodexDebugBlockExchangeResponse GetDebugBlockExchange();
+        //CodexDebugRepoStoreResponse[] GetDebugRepoStore();
         ContentId UploadFile(TrackedFile file);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(ICodexNode node);
@@ -87,6 +89,11 @@ namespace CodexPlugin
             return CodexAccess.GetDebugBlockExchange();
         }
 
+        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
+        {
+            return CodexAccess.GetDebugRepoStore();
+        }
+
         public ContentId UploadFile(TrackedFile file)
         {
             using var fileStream = File.OpenRead(file.Filename);

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -52,7 +52,7 @@ namespace CodexPlugin
             foreach (var node in result)
             {
                 mconfig.GethNode.SendEth(node, mconfig.InitialEth);
-                mconfig.CodexContracts.MintTestTokens(mconfig.GethNode, node, mconfig.InitialTokens);
+                mconfig.CodexContracts.MintTestTokens(node, mconfig.InitialTokens);
             }
         }
 

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -22,6 +22,7 @@ namespace CodexPlugin
         /// Provides an invalid proof every N proofs
         /// </summary>
         ICodexSetup WithSimulateProofFailures(uint failEveryNProofs);
+        ICodexSetup AsPublicTestNet(CodexTestNetConfig testNetConfig);
     }
 
     public class CodexLogCustomTopics
@@ -118,6 +119,12 @@ namespace CodexPlugin
             return this;
         }
 
+        public ICodexSetup AsPublicTestNet(CodexTestNetConfig testNetConfig)
+        {
+            PublicTestNet = testNetConfig;
+            return this;
+        }
+
         public string Describe()
         {
             var args = string.Join(',', DescribeArgs());
@@ -126,6 +133,7 @@ namespace CodexPlugin
 
         private IEnumerable<string> DescribeArgs()
         {
+            if (PublicTestNet != null) yield return $"<!>Public TestNet at {PublicTestNet.PublicNatIP}:{PublicTestNet.PublicListenPort}<!>";
             yield return $"LogLevel={LogLevelWithTopics()}";
             if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
             if (StorageQuota != null) yield return $"StorageQuota={StorageQuota}";

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -8,7 +8,7 @@ namespace CodexPlugin
         public string? NameOverride { get; set; }
         public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation;
         public CodexLogLevel LogLevel { get; set; }
-        public CodexLogCustomTopics? CustomTopics { get; set; }
+        public CodexLogCustomTopics? CustomTopics { get; set; } = new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn);
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
@@ -18,6 +18,7 @@ namespace CodexPlugin
         public bool? EnableValidator { get; set; }
         public TimeSpan? BlockMaintenanceInterval { get; set; }
         public int? BlockMaintenanceNumber { get; set; }
+        public CodexTestNetConfig? PublicTestNet { get; set; }
 
         public string LogLevelWithTopics()
         {
@@ -61,4 +62,11 @@ namespace CodexPlugin
             return level;
         }
     }
+
+    public class CodexTestNetConfig
+    {
+        public string PublicNatIP { get; set; } = string.Empty;
+        public int PublicDiscoveryPort { get; set; }
+        public int PublicListenPort { get; set; }
+    }
 }

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -205,7 +205,7 @@ namespace CodexPlugin
 
                 if (DateTime.UtcNow - waitStart > timeout)
                 {
-                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within timeout. {statusJson}");
+                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
                 }
             }
             log.Log($"Contract '{desiredState}'.");

ProjectPlugins/DeployAndRunPlugin/CoreInterfaceExtensions.cs

@@ -0,0 +1,13 @@
+using Core;
+using KubernetesWorkflow;
+
+namespace DeployAndRunPlugin
+{
+    public static class CoreInterfaceExtensions
+    {
+        public static RunningContainer DeployAndRunContinuousTests(this CoreInterface ci, RunConfig runConfig)
+        {
+            return ci.GetPlugin<DeployAndRunPlugin>().Run(runConfig);
+        }
+    }
+}

ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs

@@ -0,0 +1,49 @@
+using KubernetesWorkflow;
+
+namespace DeployAndRunPlugin
+{
+    public class DeployAndRunContainerRecipe : ContainerRecipeFactory
+    {
+        public override string AppName => "deploy-and-run";
+        public override string Image => "thatbenbierens/dist-tests-deployandrun:initial";
+
+        protected override void Initialize(StartupConfig config)
+        {
+            var setup = config.Get<RunConfig>();
+
+            if (setup.CodexImageOverride != null)
+            {
+                AddEnvVar("CODEXDOCKERIMAGE", setup.CodexImageOverride);
+            }
+
+            AddEnvVar("DNR_REP", setup.Replications.ToString());
+            AddEnvVar("DNR_NAME", setup.Name);
+            AddEnvVar("DNR_FILTER", setup.Filter);
+            AddEnvVar("DNR_DURATION", setup.Duration.TotalSeconds.ToString());
+
+            AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml");
+            AddEnvVar("LOGPATH", "/var/log/codex-continuous-tests");
+
+            AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "codex-dist-tests-app-kubeconfig");
+            AddVolume(name: "logs", mountPath: "/var/log/codex-continuous-tests", hostPath: "/var/log/codex-continuous-tests");
+        }
+    }
+
+    public class RunConfig
+    {
+        public RunConfig(string name, string filter, TimeSpan duration, int replications, string? codexImageOverride = null)
+        {
+            Name = name;
+            Filter = filter;
+            Duration = duration;
+            Replications = replications;
+            CodexImageOverride = codexImageOverride;
+        }
+
+        public string Name { get; }
+        public string Filter { get; }
+        public TimeSpan Duration { get; }
+        public int Replications { get; }
+        public string? CodexImageOverride { get; }
+    }
+}

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs

@@ -0,0 +1,36 @@
+using Core;
+using KubernetesWorkflow;
+
+namespace DeployAndRunPlugin
+{
+    public class DeployAndRunPlugin : IProjectPlugin
+    {
+        private readonly IPluginTools tools;
+
+        public DeployAndRunPlugin(IPluginTools tools)
+        {
+            this.tools = tools;
+        }
+
+        public void Announce()
+        {
+            tools.GetLog().Log("Deploy-and-Run plugin loaded.");
+        }
+
+        public void Decommission()
+        {
+        }
+
+        public RunningContainer Run(RunConfig config)
+        {
+            var workflow = tools.CreateWorkflow();
+            var startupConfig = new StartupConfig();
+            startupConfig.NameOverride = "dnr-" + config.Name;
+            startupConfig.Add(config);
+
+            var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd");
+            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig);
+            return containers.Containers.Single();
+        }
+    }
+}

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
+    <ProjectReference Include="..\..\Framework\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/GethPlugin/EthAddress.cs

@@ -13,5 +13,10 @@
         }
 
         public string Address { get; }
+
+        public override string ToString()
+        {
+            return Address;
+        }
     }
 }

ProjectPlugins/GethPlugin/EthTokenExtensions.cs

@@ -28,7 +28,7 @@
 
         public override string ToString()
         {
-            return $"{Wei} Wei";
+            return $"{Eth} Eth";
         }
     }
 

ProjectPlugins/GethPlugin/GethContainerRecipe.cs

@@ -9,7 +9,9 @@ namespace GethPlugin
 
         public const string HttpPortTag = "http_port";
         public const string DiscoveryPortTag = "disc_port";
+        public const string ListenPortTag = "listen_port";
         public const string WsPortTag = "ws_port";
+        public const string AuthRpcPortTag = "auth_rpc_port";
         public const string AccountsFilename = "accounts.csv";
 
         public override string AppName => "geth";
@@ -26,15 +28,16 @@ namespace GethPlugin
 
         private string CreateArgs(GethStartupConfig config)
         {
-            var discovery = AddInternalPort(tag: DiscoveryPortTag);
-
             if (config.IsMiner) AddEnvVar("ENABLE_MINER", "1");
             UnlockAccounts(0, 1);
-            var httpPort = AddExposedPort(tag: HttpPortTag);
-            var args = $"--http.addr 0.0.0.0 --http.port {httpPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
 
-            var authRpc = AddInternalPort();
-            var wsPort = AddInternalPort(tag: WsPortTag);
+            var httpPort = CreateApiPort(config, tag: HttpPortTag);
+            var discovery = CreateDiscoveryPort(config);
+            var listen = CreateListenPort(config);
+            var authRpc = CreateP2pPort(config, tag: AuthRpcPortTag);
+            var wsPort = CreateP2pPort(config, tag: WsPortTag);
+
+            var args = $"--http.addr 0.0.0.0 --http.port {httpPort.Number} --port {listen.Number} --discovery.port {discovery.Number} {GetTestNetArgs(config)} {defaultArgs}";
 
             if (config.BootstrapNode != null)
             {
@@ -57,5 +60,44 @@ namespace GethPlugin
             AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
             AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
         }
+
+        private string GetTestNetArgs(GethStartupConfig config)
+        {
+            if (config.IsPublicTestNet == null) return string.Empty;
+
+            return $"--nat=extip:{config.IsPublicTestNet.PublicIp}";
+        }
+
+        private Port CreateDiscoveryPort(GethStartupConfig config)
+        {
+            if (config.IsPublicTestNet == null) return AddInternalPort(DiscoveryPortTag);
+
+            return AddExposedPort(config.IsPublicTestNet.DiscoveryPort, DiscoveryPortTag, PortProtocol.UDP);
+        }
+
+        private Port CreateListenPort(GethStartupConfig config)
+        {
+            if (config.IsPublicTestNet == null) return AddInternalPort(ListenPortTag);
+
+            return AddExposedPort(config.IsPublicTestNet.ListenPort, ListenPortTag);
+        }
+
+        private Port CreateP2pPort(GethStartupConfig config, string tag)
+        {
+            if (config.IsPublicTestNet != null)
+            {
+                return AddExposedPort(tag);
+            }
+            return AddInternalPort(tag);
+        }
+
+        private Port CreateApiPort(GethStartupConfig config, string tag)
+        {
+            if (config.IsPublicTestNet != null)
+            {
+                return AddInternalPort(tag);
+            }
+            return AddExposedPort(tag);
+        }
     }
 }

ProjectPlugins/GethPlugin/GethDeployment.cs

@@ -5,13 +5,13 @@ namespace GethPlugin
 {
     public class GethDeployment : IHasContainer
     {
-        public GethDeployment(RunningContainer container, Port discoveryPort, Port httpPort, Port wsPort, AllGethAccounts allAccounts, string pubKey)
+        public GethDeployment(RunningContainer container, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
         {
             Container = container;
             DiscoveryPort = discoveryPort;
             HttpPort = httpPort;
             WsPort = wsPort;
-            AllAccounts = allAccounts;
+            Account = account;
             PubKey = pubKey;
         }
 
@@ -19,7 +19,7 @@ namespace GethPlugin
         public Port DiscoveryPort { get; }
         public Port HttpPort { get; }
         public Port WsPort { get; }
-        public AllGethAccounts AllAccounts { get; }
+        public GethAccount Account { get; }
         public string PubKey { get; }
     }
 }

ProjectPlugins/GethPlugin/GethNode.cs

@@ -29,11 +29,9 @@ namespace GethPlugin
         {
             this.log = log;
             StartResult = startResult;
-            Account = startResult.AllAccounts.Accounts.First();
         }
 
         public GethDeployment StartResult { get; }
-        public GethAccount Account { get; }
         public RunningContainer Container => StartResult.Container;
 
         public Ether GetEthBalance()
@@ -73,8 +71,8 @@ namespace GethPlugin
 
         private NethereumInteraction StartInteraction()
         {
-            var address = StartResult.Container.Address;
-            var account = Account;
+            var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
+            var account = StartResult.Account;
 
             var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);
             return creator.CreateWorkflow();

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -26,7 +26,7 @@ namespace GethPlugin
             var container = containers.Containers[0];
 
             var extractor = new GethContainerInfoExtractor(tools.GetLog(), workflow, container);
-            var accounts = extractor.ExtractAccounts();
+            var account = extractor.ExtractAccounts().Accounts.First();
             var pubKey = extractor.ExtractPubKey();
 
             var discoveryPort = container.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
@@ -38,7 +38,7 @@ namespace GethPlugin
 
             Log($"Geth node started.");
 
-            return new GethDeployment(container, discoveryPort, httpPort, wsPort, accounts, pubKey);
+            return new GethDeployment(container, discoveryPort, httpPort, wsPort, account, pubKey);
         }
 
         public IGethNode WrapGethContainer(GethDeployment startResult)

ProjectPlugins/GethPlugin/GethStartupConfig.cs

@@ -5,6 +5,7 @@
         IGethSetup IsMiner();
         IGethSetup WithBootstrapNode(GethBootstrapNode node);
         IGethSetup WithName(string name);
+        IGethSetup AsPublicTestNet(GethTestNetConfig gethTestNetConfig);
     }
 
     public class GethStartupConfig : IGethSetup
@@ -12,6 +13,7 @@
         public bool IsMiner { get; private set; }
         public GethBootstrapNode? BootstrapNode { get; private set; }
         public string? NameOverride { get; private set; }
+        public GethTestNetConfig? IsPublicTestNet { get; private set; }
 
         public IGethSetup WithBootstrapNode(GethBootstrapNode node)
         {
@@ -30,6 +32,26 @@
             IsMiner = true;
             return this;
         }
+
+        public IGethSetup AsPublicTestNet(GethTestNetConfig gethTestNetConfig)
+        {
+            IsPublicTestNet = gethTestNetConfig;
+            return this;
+        }
+    }
+
+    public class GethTestNetConfig
+    {
+        public GethTestNetConfig(string publicIp, int discoveryPort, int listenPort)
+        {
+            PublicIp = publicIp;
+            DiscoveryPort = discoveryPort;
+            ListenPort = listenPort;
+        }
+
+        public string PublicIp { get; }
+        public int DiscoveryPort { get; }
+        public int ListenPort { get; }
     }
 
     public class GethBootstrapNode

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -13,7 +13,7 @@ namespace MetricsPlugin
         public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
         {
             RunningContainer = runningContainer;
-            http = tools.CreateHttp(RunningContainer.Address, "api/v1");
+            http = tools.CreateHttp(RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag), "api/v1");
             log = tools.GetLog();
         }
 

ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs

@@ -7,11 +7,13 @@ namespace MetricsPlugin
         public override string AppName => "prometheus";
         public override string Image => "codexstorage/dist-tests-prometheus:latest";
 
+        public const string PortTag = "prometheus_port_tag";
+
         protected override void Initialize(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<PrometheusStartupConfig>();
 
-            AddExposedPortAndVar("PROM_PORT");
+            AddExposedPortAndVar("PROM_PORT", PortTag);
             AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
         }
     }

Tests/CodexContinuousTests/ContinuousTestRunner.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Logs;
 using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace ContinuousTests
@@ -40,7 +41,9 @@ namespace ContinuousTests
             startupChecker.Check();
 
             var taskFactory = new TaskFactory();
-            overviewLog.Log("Startup checks passed. Continuous tests starting...");
+            overviewLog.Log("Startup checks passed. Configuration:");
+            overviewLog.Log(JsonConvert.SerializeObject(config, Formatting.Indented));
+            overviewLog.Log("Continuous tests starting...");
             overviewLog.Log("");
             var allTests = testFactory.CreateTests();
 
@@ -50,22 +53,25 @@ namespace ContinuousTests
             if (!filteredTests.Any())
             {
                 overviewLog.Log("No tests selected.");
-                return;
+                Cancellation.Cts.Cancel();
             }
-            var testLoops = filteredTests.Select(t => new TestLoop(entryPointFactory, taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
-
-            foreach (var testLoop in testLoops)
+            else
             {
-                if (cancelToken.IsCancellationRequested) break;
+                var testLoops = filteredTests.Select(t => new TestLoop(entryPointFactory, taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
 
-                overviewLog.Log("Launching test-loop for " + testLoop.Name);
-                testLoop.Begin();
-                Thread.Sleep(TimeSpan.FromSeconds(5));
+                foreach (var testLoop in testLoops)
+                {
+                    if (cancelToken.IsCancellationRequested) break;
+
+                    overviewLog.Log("Launching test-loop for " + testLoop.Name);
+                    testLoop.Begin();
+                    Thread.Sleep(TimeSpan.FromSeconds(5));
+                }
+
+                overviewLog.Log("Finished launching test-loops.");
+                WaitUntilFinished(overviewLog, statusLog, startTime, testLoops);
+                overviewLog.Log("Stopping all test-loops...");
             }
-
-            overviewLog.Log("Finished launching test-loops.");
-            WaitUntilFinished(overviewLog, statusLog, startTime, testLoops);
-            overviewLog.Log("Stopping all test-loops...");
             taskFactory.WaitAll();
             overviewLog.Log("All tasks cancelled.");
 
@@ -92,6 +98,7 @@ namespace ContinuousTests
         {
             var testDuration = Time.FormatDuration(DateTime.UtcNow - startTime);
             var testData = FormatTestRuns(testLoops);
+            overviewLog.Log("Total duration: " + testDuration);
 
             if (config.TargetDurationSeconds > 0)
             {

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -106,7 +106,7 @@ namespace ContinuousTests
             var effectiveStart = testStart.Subtract(TimeSpan.FromSeconds(30));
             if (config.FullContainerLogs)
             {
-                effectiveStart = config.CodexDeployment.Metadata.DeployDateTimeUtc.Subtract(TimeSpan.FromSeconds(30));
+                effectiveStart = config.CodexDeployment.Metadata.StartUtc.Subtract(TimeSpan.FromSeconds(30));
             }
             var effectiveEnd = DateTime.UtcNow;
             var elasticSearchLogDownloader = new ElasticSearchLogDownloader(entryPoint.Tools, fixtureLog);
@@ -247,9 +247,9 @@ namespace ContinuousTests
         private RunningContainer[] SelectRandomContainers()
         {
             var number = handle.Test.RequiredNumberOfNodes;
-            if (number == -1) return config.CodexDeployment.CodexContainers;
+            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Container).ToList();
+            if (number == -1) return containers.ToArray();
 
-            var containers = config.CodexDeployment.CodexContainers.ToList();
             var result = new RunningContainer[number];
             for (var i = 0; i < number; i++)
             {

Tests/CodexContinuousTests/StartupChecker.cs

@@ -39,8 +39,9 @@ namespace ContinuousTests
         {
             log.Log("");
             var deployment = config.CodexDeployment;
-            foreach (var container in deployment.CodexContainers)
+            foreach (var instance in deployment.CodexInstances)
             {
+                var container = instance.Container;
                 log.Log($"Codex environment variables for '{container.Name}':");
                 log.Log($"Pod name: {container.Pod.PodInfo.Name} - Deployment name: {container.Pod.DeploymentName}");
                 var codexVars = container.Recipe.EnvVars;
@@ -81,13 +82,14 @@ namespace ContinuousTests
 
         private void CheckCodexNodes(BaseLog log, Configuration config)
         {
-            var nodes = entryPoint.CreateInterface().WrapCodexContainers(config.CodexDeployment.CodexContainers);
+            var nodes = entryPoint.CreateInterface().WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Container).ToArray());
             var pass = true;
             foreach (var n in nodes)
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                log.Log($"Checking {n.Container.Name} @ '{n.Container.Address.Host}:{n.Container.Address.Port}'...");
+                var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
+                log.Log($"Checking {n.Container.Name} @ '{address}'...");
 
                 if (EnsureOnline(log, n))
                 {
@@ -95,7 +97,7 @@ namespace ContinuousTests
                 }
                 else
                 {
-                    log.Error($"No response from '{n.Container.Address.Host}'.");
+                    log.Error($"No response from '{address}'.");
                     pass = false;
                 }
             }
@@ -165,9 +167,9 @@ namespace ContinuousTests
                     {
                         errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
                     }
-                    else if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
+                    else if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexInstances.Length)
                     {
-                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
+                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexInstances.Length}");
                     }
                 }
             }

Tests/CodexContinuousTests/Tests/TwoClientTest.cs

@@ -1,6 +1,7 @@
 using CodexPlugin;
 using FileUtils;
 using Logging;
+using Newtonsoft.Json;
 using NUnit.Framework;
 using Utils;
 
@@ -21,6 +22,9 @@ namespace ContinuousTests.Tests
         [TestMoment(t: Zero)]
         public void UploadTestFile()
         {
+            LogBlockExchangeStatus(Nodes[0], "Before upload");
+            LogBlockExchangeStatus(Nodes[1], "Before upload");
+
             file = FileManager.GenerateFile(size);
 
             LogStoredBytes(Nodes[0]);
@@ -34,9 +38,27 @@ namespace ContinuousTests.Tests
         {
             TrackedFile? dl = null;
 
-            LogBytesPerMillisecond(() => dl = Nodes[1].DownloadContent(cid!));
+            try
+            {
+                LogBytesPerMillisecond(() => dl = Nodes[1].DownloadContent(cid!));
 
-            file.AssertIsEqual(dl);
+                file.AssertIsEqual(dl);
+            }
+            catch
+            {
+                LogRepoStore(Nodes[0]);
+                LogRepoStore(Nodes[1]);
+                throw;
+            }
+
+            LogBlockExchangeStatus(Nodes[0], "After download");
+            LogBlockExchangeStatus(Nodes[1], "After download");
+        }
+
+        private void LogRepoStore(ICodexNode codexNode)
+        {
+            //var response = codexNode.GetDebugRepoStore();
+            //Log.Log($"{codexNode.GetName()} has {string.Join(",", response.Select(r => r.cid))}");
         }
 
         private void LogStoredBytes(ICodexNode node)
@@ -65,5 +87,11 @@ namespace ContinuousTests.Tests
             var bytesPerMs = totalBytes / totalMs;
             Log.Log($"Bytes per millisecond: {bytesPerMs}");
         }
+
+        private void LogBlockExchangeStatus(ICodexNode codexNode, string msg)
+        {
+            //var response = codexNode.GetDebugBlockExchange();
+            //Log.Log($"{codexNode.GetName()} {msg}: {JsonConvert.SerializeObject(response)}");
+        }
     }
 }

Tests/CodexContinuousTests/deploy-and-run.sh

@@ -1,15 +1,19 @@
 set -e
 
-replication=5
+replication=$DNR_REP
+name=$DNR_NAME
+filter=$DNR_FILTER
+duration=$DNR_DURATION
 
 echo "Deploying..."
 cd ../../Tools/CodexNetDeployer
 for i in $( seq 0 $replication)
 do
     dotnet run \
+    --deploy-name=codex-continuous-$name-$i \
     --kube-config=/opt/kubeconfig.yaml \
-    --kube-namespace=codex-continuous-tests-$i \
-    --deploy-file=codex-deployment-$i.json \
+    --kube-namespace=codex-continuous-$name-tests-$i \
+    --deploy-file=codex-deployment-$name-$i.json \
     --nodes=5 \
     --validators=3 \
     --log-level=Trace \
@@ -21,11 +25,12 @@ do
     --block-ttl=99999999 \
     --block-mi=99999999 \
     --block-mn=100 \
-    --metrics=1 \
+    --metrics-endpoints=1 \
+    --metrics-scraper=1 \
     --check-connect=1 \
     -y
 
-    cp codex-deployment-$i.json ../../Tests/CodexContinuousTests
+    cp codex-deployment-$name-$i.json ../../Tests/CodexContinuousTests
 done
 echo "Starting tests..."
 cd ../../Tests/CodexContinuousTests
@@ -33,13 +38,18 @@ for i in $( seq 0 $replication)
 do
     screen -d -m dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
-    --codex-deployment=codex-deployment-$i.json \
-    --log-path=logs-$i \
-    --data-path=data-$i \
+    --codex-deployment=codex-deployment-$name-$i.json \
+    --log-path=/var/log/codex-continuous-tests/logs-$name-$i \
+    --data-path=data-$name-$i \
     --keep=1 \
     --stop=1 \
-    --filter=TwoClient \
+    --filter=$filter \
     --cleanup=1 \
     --full-container-logs=1 \
-    --target-duration=172800 # 48 hours
+    --target-duration=$duration
+
+    sleep 30
 done
+
+echo "Done! Sleeping indefinitely..."
+while true; do sleep 1d; done

Tests/CodexContinuousTests/run.sh

@@ -1,6 +1,7 @@
 dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
     --codex-deployment=codex-deployment.json \
+    --log-path=/var/log/codex-continuous-tests/logs \
     --keep=1 \
     --stop=10 \
     --target-duration=172800 # 48 hours

Tests/CodexLongTests/BasicTests/DownloadTests.cs

@@ -1,7 +1,7 @@
+using CodexTests;
 using DistTestCore;
 using FileUtils;
 using NUnit.Framework;
-using Tests;
 using Utils;
 
 namespace CodexLongTests.BasicTests

Tests/CodexLongTests/BasicTests/LargeFileTests.cs

@@ -1,8 +1,8 @@
 using CodexPlugin;
+using CodexTests;
 using DistTestCore;
 using NUnit.Framework;
 using NUnit.Framework.Interfaces;
-using Tests;
 using Utils;
 
 namespace CodexLongTests.BasicTests

Tests/CodexLongTests/BasicTests/TestInfraTests.cs

@@ -1,6 +1,6 @@
-using DistTestCore;
+using CodexTests;
+using DistTestCore;
 using NUnit.Framework;
-using Tests;
 
 namespace CodexLongTests.BasicTests
 {

Tests/CodexLongTests/BasicTests/UploadTests.cs

@@ -1,8 +1,8 @@
 using CodexPlugin;
+using CodexTests;
 using DistTestCore;
 using FileUtils;
 using NUnit.Framework;
-using Tests;
 using Utils;
 
 namespace CodexLongTests.BasicTests

Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -1,6 +1,6 @@
-using DistTestCore;
+using CodexTests;
+using DistTestCore;
 using NUnit.Framework;
-using Tests;
 using Utils;
 
 namespace CodexLongTests.DownloadConnectivityTests

Tests/CodexTests/AutoBootstrapDistTest.cs

@@ -1,7 +1,7 @@
 using CodexPlugin;
 using NUnit.Framework;
 
-namespace Tests
+namespace CodexTests
 {
     public class AutoBootstrapDistTest : CodexDistTest
     {

Tests/CodexTests/BasicTests/BlockExchangeTests.cs

@@ -0,0 +1,63 @@
+using CodexPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.BasicTests
+{
+    [TestFixture]
+    public class BlockExchangeTests : CodexDistTest
+    {
+        [Test]
+        public void EmptyAfterExchange()
+        {
+            var bootstrap = AddCodex(s => s.WithName("bootstrap"));
+            var node = AddCodex(s => s.WithName("node").WithBootstrapNode(bootstrap));
+
+            AssertExchangeIsEmpty(bootstrap, node);
+
+            var file = GenerateTestFile(1.MB());
+            var cid = bootstrap.UploadFile(file);
+            node.DownloadContent(cid);
+
+            AssertExchangeIsEmpty(bootstrap, node);
+        }
+
+        [Test]
+        public void EmptyAfterExchangeWithBystander()
+        {
+            var bootstrap = AddCodex(s => s.WithName("bootstrap"));
+            var node = AddCodex(s => s.WithName("node").WithBootstrapNode(bootstrap));
+            var bystander = AddCodex(s => s.WithName("bystander").WithBootstrapNode(bootstrap));
+
+            AssertExchangeIsEmpty(bootstrap, node, bystander);
+
+            var file = GenerateTestFile(1.MB());
+            var cid = bootstrap.UploadFile(file);
+            node.DownloadContent(cid);
+
+            AssertExchangeIsEmpty(bootstrap, node, bystander);
+        }
+
+        private void AssertExchangeIsEmpty(params ICodexNode[] nodes)
+        {
+            foreach (var node in nodes)
+            {
+                // API Call not available in master-line Codex image.
+                //Time.Retry(() => AssertBlockExchangeIsEmpty(node), nameof(AssertExchangeIsEmpty));
+            }
+        }
+
+        //private void AssertBlockExchangeIsEmpty(ICodexNode node)
+        //{
+        //    var msg = $"BlockExchange for {node.GetName()}: ";
+        //    var response = node.GetDebugBlockExchange();
+        //    foreach (var peer in response.peers)
+        //    {
+        //        var activeWants = peer.wants.Where(w => !w.cancel).ToArray();
+        //        Assert.That(activeWants.Length, Is.EqualTo(0), msg + "thinks a peer has active wants.");
+        //    }
+        //    Assert.That(response.taskQueue, Is.EqualTo(0), msg + "has tasks in queue.");
+        //    Assert.That(response.pendingBlocks, Is.EqualTo(0), msg + "has pending blocks.");
+        //}
+    }
+}

Tests/CodexTests/BasicTests/ContinuousSubstitute.cs

@@ -7,7 +7,7 @@ using MetricsPlugin;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     [Ignore("Used for debugging continuous tests")]
     [TestFixture]
@@ -87,7 +87,7 @@ namespace Tests.BasicTests
                 //CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
                 //CheckRoutingTables(GetAllOnlineCodexNodes());
 
-                var node = RandomUtils.PickOneRandom(nodes.ToList());
+                var node = nodes.ToList().PickOneRandom();
                 var file = GenerateTestFile(50.MB());
                 node.UploadFile(file);
 

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -6,7 +6,7 @@ using MetricsPlugin;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     [TestFixture]
     public class ExampleTests : CodexDistTest
@@ -59,8 +59,8 @@ namespace Tests.BasicTests
                 .WithStorageQuota(11.GB())
                 .EnableMarketplace(geth, contracts, initialEth: 10.Eth(), initialTokens: sellerInitialBalance, isValidator: true)
                 .WithSimulateProofFailures(failEveryNProofs: 3));
-            
-            AssertBalance(geth, contracts, seller, Is.EqualTo(sellerInitialBalance));
+
+            AssertBalance(contracts, seller, Is.EqualTo(sellerInitialBalance));
             seller.Marketplace.MakeStorageAvailable(
                 size: 10.GB(),
                 minPriceForTotalSpace: 1.TestTokens(),
@@ -72,8 +72,8 @@ namespace Tests.BasicTests
             var buyer = AddCodex(s => s
                             .WithBootstrapNode(seller)
                             .EnableMarketplace(geth, contracts, initialEth: 10.Eth(), initialTokens: buyerInitialBalance));
-            
-            AssertBalance(geth, contracts, buyer, Is.EqualTo(buyerInitialBalance));
+
+            AssertBalance(contracts, buyer, Is.EqualTo(buyerInitialBalance));
 
             var contentId = buyer.UploadFile(testFile);
             var purchaseContract = buyer.Marketplace.RequestStorage(contentId,
@@ -85,12 +85,12 @@ namespace Tests.BasicTests
 
             purchaseContract.WaitForStorageContractStarted(fileSize);
 
-            AssertBalance(geth, contracts, seller, Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
+            AssertBalance(contracts, seller, Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
 
             purchaseContract.WaitForStorageContractFinished();
 
-            AssertBalance(geth, contracts, seller, Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
-            AssertBalance(geth, contracts, buyer, Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
+            AssertBalance(contracts, seller, Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
+            AssertBalance(contracts, buyer, Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
         }
     }
 }

Tests/CodexTests/BasicTests/NetworkIsolationTest.cs

@@ -3,7 +3,7 @@ using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     // Warning!
     // This is a test to check network-isolation in the test-infrastructure.

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -3,7 +3,7 @@ using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     [TestFixture]
     public class OneClientTests : DistTest

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -1,8 +1,7 @@
-using DistTestCore;
-using NUnit.Framework;
+using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     [TestFixture]
     public class ThreeClientTest : AutoBootstrapDistTest

Tests/CodexTests/BasicTests/TwoClientTests.cs

@@ -3,7 +3,7 @@ using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.BasicTests
+namespace CodexTests.BasicTests
 {
     [TestFixture]
     public class TwoClientTests : DistTest

Tests/CodexTests/CodexDistTest.cs

@@ -9,7 +9,7 @@ using GethPlugin;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 
-namespace Tests
+namespace CodexTests
 {
     public class CodexDistTest : DistTest
     {
@@ -77,9 +77,9 @@ namespace Tests
             return onlineCodexNodes;
         }
 
-        public void AssertBalance(IGethNode gethNode, ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "")
+        public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "")
         {
-            AssertHelpers.RetryAssert(constraint, () => contracts.GetTestTokenBalance(gethNode, codexNode), nameof(AssertBalance) + msg);
+            AssertHelpers.RetryAssert(constraint, () => contracts.GetTestTokenBalance(codexNode), nameof(AssertBalance) + msg);
         }
 
         protected virtual void OnCodexSetup(ICodexSetup setup)

Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexTests;
 using GethPlugin;
 using NUnit.Framework;
 using Utils;

Tests/CodexTests/MetricsAccessExtensions.cs

@@ -3,7 +3,7 @@ using Logging;
 using MetricsPlugin;
 using NUnit.Framework.Constraints;
 
-namespace Tests
+namespace CodexTests
 {
     public static class MetricsAccessExtensions
     {

Tests/CodexTests/Parallelism.cs

@@ -1,6 +1,6 @@
 using NUnit.Framework;
 
 [assembly: LevelOfParallelism(1)]
-namespace Tests
+namespace CodexTests
 {
 }

Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -1,4 +1,5 @@
 using CodexPlugin;
+using CodexTests;
 using NUnit.Framework;
 
 namespace Tests.PeerDiscoveryTests

Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexTests;
 using GethPlugin;
 using NUnit.Framework;
 

Tests/DistTestCore/LongTimeSet.cs

@@ -1,32 +0,0 @@
-using Core;
-
-namespace DistTestCore
-{
-    public class LongTimeSet : ITimeSet
-    {
-        public TimeSpan HttpCallTimeout()
-        {
-            return TimeSpan.FromHours(2);
-        }
-
-        public TimeSpan HttpCallRetryTime()
-        {
-            return TimeSpan.FromHours(5);
-        }
-
-        public TimeSpan HttpCallRetryDelay()
-        {
-            return TimeSpan.FromSeconds(2);
-        }
-
-        public TimeSpan WaitForK8sServiceDelay()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(15);
-        }
-    }
-}

Tools/BiblioTech/AdminChecker.cs

@@ -0,0 +1,40 @@
+using Discord.WebSocket;
+
+namespace BiblioTech
+{
+    public class AdminChecker
+    {
+        private SocketGuild guild = null!;
+        private ulong[] adminIds = Array.Empty<ulong>();
+        private DateTime lastUpdate = DateTime.MinValue;
+
+        public void SetGuild(SocketGuild guild)
+        {
+            this.guild = guild;
+        }
+
+        public bool IsUserAdmin(ulong userId)
+        {
+            if (ShouldUpdate()) UpdateAdminIds();
+
+            return adminIds.Contains(userId);
+        }
+
+        public bool IsAdminChannel(ISocketMessageChannel channel)
+        {
+            return channel.Name == Program.Config.AdminChannelName;
+        }
+
+        private bool ShouldUpdate()
+        {
+            return !adminIds.Any() || (DateTime.UtcNow - lastUpdate) > TimeSpan.FromMinutes(10);
+        }
+
+        private void UpdateAdminIds()
+        {
+            lastUpdate = DateTime.UtcNow;
+            var adminRole = guild.Roles.Single(r => r.Name == Program.Config.AdminRoleName);
+            adminIds = adminRole.Members.Select(m => m.Id).ToArray();
+        }
+    }
+}

Tools/BiblioTech/BaseCodexCommand.cs

@@ -0,0 +1,27 @@
+using BiblioTech.Options;
+using CodexPlugin;
+using Core;
+
+namespace BiblioTech
+{
+    public abstract class BaseCodexCommand : BaseDeploymentCommand
+    {
+        private readonly CoreInterface ci;
+
+        public BaseCodexCommand(CoreInterface ci)
+        {
+            this.ci = ci;
+        }
+
+        protected override async Task ExecuteDeploymentCommand(CommandContext context, CodexDeployment codexDeployment)
+        {
+            var codexContainers = codexDeployment.CodexInstances.Select(c => c.Container).ToArray();
+
+            var group = ci.WrapCodexContainers(codexContainers);
+
+            await Execute(context, group);
+        }
+
+        protected abstract Task Execute(CommandContext context, ICodexNodeGroup codexGroup);
+    }
+}

Tools/BiblioTech/BaseCommand.cs

@@ -0,0 +1,62 @@
+using Discord.WebSocket;
+using BiblioTech.Options;
+using Discord;
+
+namespace BiblioTech
+{
+    public abstract class BaseCommand
+    {
+        public abstract string Name { get; }
+        public abstract string StartingMessage { get; }
+        public abstract string Description { get; }
+        public virtual CommandOption[] Options
+        {
+            get
+            {
+                return Array.Empty<CommandOption>();
+            }
+        }
+
+        public async Task SlashCommandHandler(SocketSlashCommand command)
+        {
+            if (command.CommandName != Name) return;
+
+            try
+            {
+                var context = new CommandContext(command, command.Data.Options);
+                await command.RespondAsync(StartingMessage, ephemeral: IsEphemeral(context));
+                await Invoke(context);
+            }
+            catch (Exception ex)
+            {
+                await command.FollowupAsync("Something failed while trying to do that...", ephemeral: true);
+                Console.WriteLine(ex);
+            }
+        }
+
+        private bool IsEphemeral(CommandContext context)
+        {
+            if (IsSenderAdmin(context.Command) && IsInAdminChannel(context.Command)) return false;
+            return true;
+        }
+
+        protected abstract Task Invoke(CommandContext context);
+
+        protected bool IsSenderAdmin(SocketSlashCommand command)
+        {
+            return Program.AdminChecker.IsUserAdmin(command.User.Id);
+        }
+
+        protected bool IsInAdminChannel(SocketSlashCommand command)
+        {
+            return Program.AdminChecker.IsAdminChannel(command.Channel);
+        }
+
+        protected IUser GetUserFromCommand(UserOption userOption, CommandContext context)
+        {
+            var targetUser = userOption.GetUser(context);
+            if (IsSenderAdmin(context.Command) && targetUser != null) return targetUser;
+            return context.Command.User;
+        }
+    }
+}

Tools/BiblioTech/BaseDeploymentCommand.cs

@@ -0,0 +1,36 @@
+using BiblioTech.Options;
+using CodexPlugin;
+
+namespace BiblioTech
+{
+    public abstract class BaseDeploymentCommand : BaseCommand
+    {
+        protected override async Task Invoke(CommandContext context)
+        {
+            var proceed = await OnInvoke(context);
+            if (!proceed) return;
+
+            var deployments = Program.DeploymentFilesMonitor.GetDeployments();
+            if (deployments.Length == 0)
+            {
+                await context.Followup("No deployments are currently available.");
+                return;
+            }
+            if (deployments.Length > 1)
+            {
+                await context.Followup("Multiple deployments are online. I don't know which one to pick!");
+                return;
+            }
+
+            var codexDeployment = deployments.Single();
+            await ExecuteDeploymentCommand(context, codexDeployment);
+        }
+
+        protected abstract Task ExecuteDeploymentCommand(CommandContext context, CodexDeployment codexDeployment);
+
+        protected virtual Task<bool> OnInvoke(CommandContext context)
+        {
+            return Task.FromResult(true);
+        }
+    }
+}

Tools/BiblioTech/BaseGethCommand.cs

@@ -0,0 +1,31 @@
+using BiblioTech.Options;
+using CodexContractsPlugin;
+using CodexPlugin;
+using Core;
+using GethPlugin;
+
+namespace BiblioTech
+{
+    public abstract class BaseGethCommand : BaseDeploymentCommand
+    {
+        private readonly CoreInterface ci;
+
+        public BaseGethCommand(CoreInterface ci)
+        {
+            this.ci = ci;
+        }
+
+        protected override async Task ExecuteDeploymentCommand(CommandContext context, CodexDeployment codexDeployment)
+        {
+            var gethDeployment = codexDeployment.GethDeployment;
+            var contractsDeployment = codexDeployment.CodexContractsDeployment;
+
+            var gethNode = ci.WrapGethDeployment(gethDeployment);
+            var contracts = ci.WrapCodexContractsDeployment(gethNode, contractsDeployment);
+
+            await Execute(context, gethNode, contracts);
+        }
+
+        protected abstract Task Execute(CommandContext context, IGethNode gethNode, ICodexContracts contracts);
+    }
+}

Tools/BiblioTech/BiblioTech.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Discord.Net" Version="3.12.0" />
+    <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

Tools/BiblioTech/CommandHandler.cs

@@ -0,0 +1,63 @@
+using Discord.Net;
+using Discord.WebSocket;
+using Discord;
+using Newtonsoft.Json;
+
+namespace BiblioTech
+{
+    public class CommandHandler
+    {
+        private readonly DiscordSocketClient client;
+        private readonly BaseCommand[] commands;
+
+        public CommandHandler(DiscordSocketClient client, params BaseCommand[] commands)
+        {
+            this.client = client;
+            this.commands = commands;
+
+            client.Ready += Client_Ready;
+            client.SlashCommandExecuted += SlashCommandHandler;
+        }
+
+        private async Task Client_Ready()
+        {
+            var guild = client.Guilds.Single(g => g.Name == Program.Config.ServerName);
+            Program.AdminChecker.SetGuild(guild);
+
+            var builders = commands.Select(c =>
+            {
+                var builder = new SlashCommandBuilder()
+                    .WithName(c.Name)
+                    .WithDescription(c.Description);
+
+                foreach (var option in c.Options)
+                {
+                    builder.AddOption(option.Build());
+                }
+
+                return builder;
+            });
+
+            try
+            {
+                foreach (var builder in builders)
+                {
+                    await guild.CreateApplicationCommandAsync(builder.Build());
+                }
+            }
+            catch (HttpException exception)
+            {
+                var json = JsonConvert.SerializeObject(exception.Errors, Formatting.Indented);
+                Console.WriteLine(json);
+            }
+        }
+
+        private async Task SlashCommandHandler(SocketSlashCommand command)
+        {
+            foreach (var cmd in commands)
+            {
+                await cmd.SlashCommandHandler(command);
+            }
+        }
+    }
+}

Tools/BiblioTech/Commands/AdminCommand.cs

@@ -0,0 +1,362 @@
+using BiblioTech.Options;
+using CodexPlugin;
+using Core;
+using Newtonsoft.Json;
+
+namespace BiblioTech.Commands
+{
+    public class AdminCommand : BaseCommand
+    {
+        private readonly ClearUserAssociationCommand clearCommand = new ClearUserAssociationCommand();
+        private readonly ReportCommand reportCommand = new ReportCommand();
+        private readonly DeployListCommand deployListCommand = new DeployListCommand();
+        private readonly DeployUploadCommand deployUploadCommand = new DeployUploadCommand();
+        private readonly DeployRemoveCommand deployRemoveCommand = new DeployRemoveCommand();
+        private readonly WhoIsCommand whoIsCommand = new WhoIsCommand();
+        private readonly NetInfoCommand netInfoCommand;
+        private readonly DebugPeerCommand debugPeerCommand;
+
+        public AdminCommand(CoreInterface ci)
+        {
+            netInfoCommand = new NetInfoCommand(ci);
+            debugPeerCommand = new DebugPeerCommand(ci);
+        }
+
+        public override string Name => "admin";
+        public override string StartingMessage => "...";
+        public override string Description => "Admins only.";
+
+        public override CommandOption[] Options => new CommandOption[]
+        {
+            clearCommand,
+            reportCommand,
+            deployListCommand,
+            deployUploadCommand,
+            deployRemoveCommand,
+            whoIsCommand,
+            netInfoCommand,
+            debugPeerCommand
+        };
+
+        protected override async Task Invoke(CommandContext context)
+        {
+            if (!IsSenderAdmin(context.Command))
+            {
+                await context.Followup("You're not an admin.");
+                return;
+            }
+
+            if (!IsInAdminChannel(context.Command))
+            {
+                await context.Followup("Please use admin commands only in the admin channel.");
+                return;
+            }
+
+            await clearCommand.CommandHandler(context);
+            await reportCommand.CommandHandler(context);
+            await deployListCommand.CommandHandler(context);
+            await deployUploadCommand.CommandHandler(context);
+            await deployRemoveCommand.CommandHandler(context);
+            await whoIsCommand.CommandHandler(context);
+            await netInfoCommand.CommandHandler(context);
+            await debugPeerCommand.CommandHandler(context);
+        }
+
+        public class ClearUserAssociationCommand : SubCommandOption
+        {
+            private readonly UserOption userOption = new UserOption("User to clear Eth address for.", true);
+
+            public ClearUserAssociationCommand()
+                : base("clear", "Admin only. Clears current Eth address for a user, allowing them to set a new one.")
+            {
+            }
+
+            public override CommandOption[] Options => new[] { userOption };
+
+            protected override async Task onSubCommand(CommandContext context)
+            {
+                var user = userOption.GetUser(context);
+                if (user == null)
+                {
+                    await context.Followup("Failed to get user ID");
+                    return;
+                }
+
+                Program.UserRepo.ClearUserAssociatedAddress(user);
+                await context.Followup("Done.");
+            }
+        }
+
+        public class ReportCommand : SubCommandOption
+        {
+            private readonly UserOption userOption = new UserOption(
+                description: "User to report history for.",
+                isRequired: true);
+
+            public ReportCommand()
+                : base("report", "Admin only. Reports bot-interaction history for a user.")
+            {
+            }
+
+            public override CommandOption[] Options => new[] { userOption };
+
+            protected override async Task onSubCommand(CommandContext context)
+            {
+                var user = userOption.GetUser(context);
+                if (user == null)
+                {
+                    await context.Followup("Failed to get user ID");
+                    return;
+                }
+
+                var report = string.Join(Environment.NewLine, Program.UserRepo.GetInteractionReport(user));
+                if (report.Length > 1900)
+                {
+                    var filename = $"user-{user.Username}.log";
+                    await context.FollowupWithAttachement(filename, report);
+                }
+                else
+                {
+                    await context.Followup(report);
+                }
+            }
+        }
+
+        public class DeployListCommand : SubCommandOption
+        {
+            public DeployListCommand()
+                : base("list", "Lists current deployments.")
+            {
+            }
+
+            protected override async Task onSubCommand(CommandContext context)
+            {
+                var deployments = Program.DeploymentFilesMonitor.GetDeployments();
+
+                if (!deployments.Any())
+                {
+                    await context.Followup("No deployments available.");
+                    return;
+                }
+
+                var nl = Environment.NewLine;
+                await context.Followup($"Deployments:{nl}{string.Join(nl, deployments.Select(FormatDeployment))}");
+            }
+
+            private string FormatDeployment(CodexDeployment deployment)
+            {
+                var m = deployment.Metadata;
+                return $"'{m.Name}' ({m.StartUtc.ToString("o")})";
+            }
+        }
+
+        public class DeployUploadCommand : SubCommandOption
+        {
+            private readonly FileAttachementOption fileOption = new FileAttachementOption(
+                name: "json",
+                description: "Codex-deployment json to add.",
+                isRequired: true);
+
+            public DeployUploadCommand()
+                : base("add", "Upload a new deployment JSON file.")
+            {
+            }
+
+            public override CommandOption[] Options => new[] { fileOption };
+
+            protected override async Task onSubCommand(CommandContext context)
+            {
+                var file = await fileOption.Parse(context);
+                if (file == null) return;
+
+                var result = await Program.DeploymentFilesMonitor.DownloadDeployment(file);
+                if (result)
+                {
+                    await context.Followup("Success!");
+                }
+                else
+                {
+                    await context.Followup("That didn't work.");
+                }
+            }
+        }
+
+        public class DeployRemoveCommand : SubCommandOption
+        {
+            private readonly StringOption stringOption = new StringOption(
+                name: "name",
+                description: "Name of deployment to remove.",
+                isRequired: true);
+
+            public DeployRemoveCommand()
+                : base("remove", "Removes a deployment file.")
+            {
+            }
+
+            public override CommandOption[] Options => new[] { stringOption };
+
+            protected override async Task onSubCommand(CommandContext context)
+            {
+                var str = await stringOption.Parse(context);
+                if (string.IsNullOrEmpty(str)) return;
+
+                var result = Program.DeploymentFilesMonitor.DeleteDeployment(str);
+                if (result)
+                {
+                    await context.Followup("Success!");
+                }
+                else
+                {
+                    await context.Followup("That didn't work.");
+                }
+            }
+        }
+
+        public class WhoIsCommand : SubCommandOption
+        {
+            private readonly UserOption userOption = new UserOption("User", isRequired: false);
+            private readonly EthAddressOption ethAddressOption = new EthAddressOption(isRequired: false);
+
+            public WhoIsCommand()
+                : base(name: "whois",
+                      description: "Fetches info about a user or ethAddress in the testnet.")
+            {
+            }
+
+            public override CommandOption[] Options => new CommandOption[]
+            {
+                userOption,
+                ethAddressOption
+            };
+
+            protected override async Task onSubCommand(CommandContext context)
+            {
+                var user = userOption.GetUser(context);
+                var ethAddr = await ethAddressOption.Parse(context);
+
+                if (user != null)
+                {
+                    await context.Followup(Program.UserRepo.GetUserReport(user));
+                }
+                if (ethAddr != null)
+                {
+                    await context.Followup(Program.UserRepo.GetUserReport(ethAddr));
+                }
+            }
+        }
+
+        public abstract class AdminDeploymentCommand : SubCommandOption
+        {
+            private readonly CoreInterface ci;
+
+            public AdminDeploymentCommand(CoreInterface ci, string name, string description)
+                : base(name, description)
+            {
+                this.ci = ci;
+            }
+
+            protected async Task OnDeployment(CommandContext context, Func<ICodexNodeGroup, string, Task> action)
+            {
+                var deployment = Program.DeploymentFilesMonitor.GetDeployments().SingleOrDefault();
+                if (deployment == null)
+                {
+                    await context.Followup("No deployment found.");
+                    return;
+                }
+
+                try
+                {
+                    var group = ci.WrapCodexContainers(deployment.CodexInstances.Select(i => i.Container).ToArray());
+                    await action(group, deployment.Metadata.Name);
+                }
+                catch (Exception ex)
+                {
+                    await context.Followup("Failed to wrap nodes with exception: " + ex);
+                }
+            }
+        }
+
+        public class NetInfoCommand : AdminDeploymentCommand
+        {
+            public NetInfoCommand(CoreInterface ci)
+                : base(ci, name: "netinfo",
+                      description: "Fetches info endpoints of codex nodes.")
+            {
+            }
+
+            protected override async Task onSubCommand(CommandContext context)
+            {
+                await OnDeployment(context, async (group, name) =>
+                {
+                    var nl = Environment.NewLine;
+                    var content = new List<string>
+                    {
+                        $"{DateTime.UtcNow.ToString("o")} - {group.Count()} Codex nodes."
+                    };
+
+                    foreach (var node in group)
+                    {
+                        try
+                        {
+                            var info = node.GetDebugInfo();
+                            var json = JsonConvert.SerializeObject(info, Formatting.Indented);
+                            var jsonInsert = $"{nl}```{nl}{json}{nl}```{nl}";
+                            content.Add($"Node '{node.GetName()}' responded with {jsonInsert}");
+                        }
+                        catch (Exception ex)
+                        {
+                            content.Add($"Node '{node.GetName()}' failed to respond with exception: " + ex);
+                        }
+                    }
+
+                    var filename = $"netinfo-{NoWhitespaces(name)}.log";
+                    await context.FollowupWithAttachement(filename, string.Join(nl, content.ToArray()));
+                });
+            }
+        }
+
+        public class DebugPeerCommand : AdminDeploymentCommand
+        {
+            private readonly StringOption peerIdOption = new StringOption("peerid", "id of peer to try and reach.", true);
+
+            public DebugPeerCommand(CoreInterface ci)
+                : base(ci, name: "debugpeer",
+                      description: "Calls debug/peer on each codex node.")
+            {
+            }
+
+            public override CommandOption[] Options => new[] { peerIdOption };
+
+            protected override async Task onSubCommand(CommandContext context)
+            {
+                var peerId = await peerIdOption.Parse(context);
+                if (string.IsNullOrEmpty(peerId)) return;
+
+                await OnDeployment(context, async (group, name) =>
+                {
+                    await context.Followup($"Calling debug/peer for '{peerId}' on {group.Count()} Codex nodes.");
+                    foreach (var node in group)
+                    {
+                        try
+                        {
+                            var info = node.GetDebugPeer(peerId);
+                            var nl = Environment.NewLine;
+                            var json = JsonConvert.SerializeObject(info, Formatting.Indented);
+                            var jsonInsert = $"{nl}```{nl}{json}{nl}```{nl}";
+                            await context.Followup($"Node '{node.GetName()}' responded with {jsonInsert}");
+                        }
+                        catch (Exception ex)
+                        {
+                            await context.Followup($"Node '{node.GetName()}' failed to respond with exception: " + ex);
+                        }
+                    }
+                });
+            }
+        }
+
+        private static string NoWhitespaces(string s)
+        {
+            return s.Replace(" ", "-");
+        }
+    }
+}

Tools/BiblioTech/Commands/GetBalanceCommand.cs

@@ -0,0 +1,42 @@
+using BiblioTech.Options;
+using CodexContractsPlugin;
+using Core;
+using GethPlugin;
+
+namespace BiblioTech.Commands
+{
+    public class GetBalanceCommand : BaseGethCommand
+    {
+        private readonly UserAssociateCommand userAssociateCommand;
+        private readonly UserOption optionalUser = new UserOption(
+            description: "If set, get balance for another user. (Optional, admin-only)",
+            isRequired: false);
+
+        public GetBalanceCommand(CoreInterface ci, UserAssociateCommand userAssociateCommand)
+            : base(ci)
+        {
+            this.userAssociateCommand = userAssociateCommand;
+        }
+
+        public override string Name => "balance";
+        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string Description => "Shows Eth and TestToken balance of an eth address.";
+        public override CommandOption[] Options => new[] { optionalUser };
+
+        protected override async Task Execute(CommandContext context, IGethNode gethNode, ICodexContracts contracts)
+        {
+            var userId = GetUserFromCommand(optionalUser, context);
+            var addr = Program.UserRepo.GetCurrentAddressForUser(userId);
+            if (addr == null)
+            {
+                await context.Followup($"No address has been set for this user. Please use '/{userAssociateCommand.Name}' to set it first.");
+                return;
+            }
+
+            var eth = gethNode.GetEthBalance(addr);
+            var testTokens = contracts.GetTestTokenBalance(addr);
+
+            await context.Followup($"{context.Command.User.Username} has {eth} and {testTokens}.");
+        }
+    }
+}

Tools/BiblioTech/Commands/MintCommand.cs

@@ -0,0 +1,85 @@
+using BiblioTech.Options;
+using CodexContractsPlugin;
+using Core;
+using GethPlugin;
+
+namespace BiblioTech.Commands
+{
+    public class MintCommand : BaseGethCommand
+    {
+        private readonly Ether defaultEthToSend = 10.Eth();
+        private readonly TestToken defaultTestTokensToMint = 1024.TestTokens();
+        private readonly UserOption optionalUser = new UserOption(
+            description: "If set, mint tokens for this user. (Optional, admin-only)",
+            isRequired: false);
+        private readonly UserAssociateCommand userAssociateCommand;
+
+        public MintCommand(CoreInterface ci, UserAssociateCommand userAssociateCommand)
+            : base(ci)
+        {
+            this.userAssociateCommand = userAssociateCommand;
+        }
+
+        public override string Name => "mint";
+        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string Description => "Mint some TestTokens and send some Eth to the user if their balance is low.";
+        public override CommandOption[] Options => new[] { optionalUser };
+
+        protected override async Task Execute(CommandContext context, IGethNode gethNode, ICodexContracts contracts)
+        {
+            var userId = GetUserFromCommand(optionalUser, context);
+            var addr = Program.UserRepo.GetCurrentAddressForUser(userId);
+            if (addr == null)
+            {
+                await context.Followup($"No address has been set for this user. Please use '/{userAssociateCommand.Name}' to set it first.");
+                return;
+            }
+
+            var report = new List<string>();
+
+            var sentEth = ProcessEth(gethNode, addr, report);
+            var mintedTokens = ProcessTokens(contracts, addr, report);
+
+            Program.UserRepo.AddMintEventForUser(userId, addr, sentEth, mintedTokens);
+
+            await context.Followup(string.Join(Environment.NewLine, report));
+        }
+
+        private TestToken ProcessTokens(ICodexContracts contracts, EthAddress addr, List<string> report)
+        {
+            if (ShouldMintTestTokens(contracts, addr))
+            {
+                contracts.MintTestTokens(addr, defaultTestTokensToMint);
+                report.Add($"Minted {defaultTestTokensToMint}.");
+                return defaultTestTokensToMint;
+            }
+            
+            report.Add("TestToken balance over threshold.");
+            return 0.TestTokens();
+        }
+
+        private Ether ProcessEth(IGethNode gethNode, EthAddress addr, List<string> report)
+        {
+            if (ShouldSendEth(gethNode, addr))
+            {
+                gethNode.SendEth(addr, defaultEthToSend);
+                report.Add($"Sent {defaultEthToSend}.");
+                return defaultEthToSend;
+            }
+            report.Add("Eth balance is over threshold.");
+            return 0.Eth();
+        }
+
+        private bool ShouldMintTestTokens(ICodexContracts contracts, EthAddress addr)
+        {
+            var testTokens = contracts.GetTestTokenBalance(addr);
+            return testTokens.Amount < 64m;
+        }
+
+        private bool ShouldSendEth(IGethNode gethNode, EthAddress addr)
+        {
+            var eth = gethNode.GetEthBalance(addr);
+            return eth.Eth < 1.0m;
+        }
+    }
+}

Tools/BiblioTech/Commands/SprCommand.cs

@@ -0,0 +1,61 @@
+using BiblioTech.Options;
+using CodexPlugin;
+using Core;
+
+namespace BiblioTech.Commands
+{
+    public class SprCommand : BaseCodexCommand
+    {
+        private readonly Random random = new Random();
+        private readonly List<string> sprCache = new List<string>();
+        private DateTime lastUpdate = DateTime.MinValue;
+
+        public SprCommand(CoreInterface ci) : base(ci)
+        {
+        }
+
+        public override string Name => "boot";
+        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string Description => "Gets an SPR. (Signed peer record, used for bootstrapping.)";
+
+        protected override async Task<bool> OnInvoke(CommandContext context)
+        {
+            if (ShouldUpdate())
+            {
+                return true;
+            }
+
+            await ReplyWithRandomSpr(context);
+            return false;
+        }
+
+        protected override async Task Execute(CommandContext context, ICodexNodeGroup codexGroup)
+        {
+            lastUpdate = DateTime.UtcNow;
+            sprCache.Clear();
+
+            var infos = codexGroup.Select(c => c.GetDebugInfo()).ToArray();
+            sprCache.AddRange(infos.Select(i => i.spr));
+
+            await ReplyWithRandomSpr(context);
+        }
+
+        private async Task ReplyWithRandomSpr(CommandContext context)
+        {
+            if (!sprCache.Any())
+            {
+                await context.Followup("I'm sorry, no SPRs are available... :c");
+                return;
+            }
+
+            var i = random.Next(0, sprCache.Count);
+            var spr = sprCache[i];
+            await context.Followup($"Your SPR: '{spr}'");
+        }
+
+        private bool ShouldUpdate()
+        {
+            return (DateTime.UtcNow - lastUpdate) > TimeSpan.FromMinutes(10);
+        }
+    }
+}

Tools/BiblioTech/Commands/UserAssociateCommand.cs

@@ -0,0 +1,43 @@
+using BiblioTech.Options;
+
+namespace BiblioTech.Commands
+{
+    public class UserAssociateCommand : BaseCommand
+    {
+        private readonly EthAddressOption ethOption = new EthAddressOption(isRequired: false);
+        private readonly UserOption optionalUser = new UserOption(
+            description: "If set, associates Ethereum address for another user. (Optional, admin-only)",
+            isRequired: false);
+
+        public override string Name => "set";
+        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string Description => "Associates a Discord user with an Ethereum address.";
+        public override CommandOption[] Options => new CommandOption[] { ethOption, optionalUser };
+
+        protected override async Task Invoke(CommandContext context)
+        {
+            var user = GetUserFromCommand(optionalUser, context);
+            var data = await ethOption.Parse(context);
+            if (data == null) return;
+
+            var currentAddress = Program.UserRepo.GetCurrentAddressForUser(user);
+            if (currentAddress != null && !IsSenderAdmin(context.Command))
+            {
+                await context.Followup($"You've already set your Ethereum address to {currentAddress}.");
+                return;
+            }
+
+            // private commands
+
+            var result = Program.UserRepo.AssociateUserWithAddress(user, data);
+            if (result)
+            {
+                await context.Followup("Done! Thank you for joining the test net!");
+            }
+            else
+            {
+                await context.Followup("That didn't work.");
+            }
+        }
+    }
+}

Tools/BiblioTech/Configuration.cs

@@ -0,0 +1,38 @@
+using ArgsUniform;
+
+namespace BiblioTech
+{
+    public class Configuration
+    {
+        [Uniform("token", "t", "TOKEN", true, "Discord Application Token")]
+        public string ApplicationToken { get; set; } = string.Empty;
+
+        [Uniform("server-name", "sn", "SERVERNAME", true, "Name of the Discord server")]
+        public string ServerName { get; set; } = string.Empty;
+
+        [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
+        public string DataPath { get; set; } = "datapath";
+        
+        [Uniform("admin-role", "a", "ADMINROLE", true, "Name of the Discord server admin role")]
+        public string AdminRoleName { get; set; } = string.Empty;
+
+        [Uniform("admin-channel-name", "ac", "ADMINCHANNELNAME", true, "Name of the Discord server channel where admin commands are allowed.")]
+        public string AdminChannelName { get; set; } = "admin-channel";
+
+        public string EndpointsPath
+        {
+            get
+            {
+                return Path.Combine(DataPath, "endpoints");
+            }
+        }
+
+        public string UserDataPath
+        {
+            get
+            {
+                return Path.Combine(DataPath, "users");
+            }
+        }
+    }
+}

Tools/BiblioTech/DeploymentsFilesMonitor.cs

@@ -0,0 +1,89 @@
+using CodexPlugin;
+using Discord;
+using Newtonsoft.Json;
+
+namespace BiblioTech
+{
+    public class DeploymentsFilesMonitor
+    {
+        private readonly List<CodexDeployment> deployments = new List<CodexDeployment>();
+
+        public void Initialize()
+        {
+            LoadDeployments();
+        }
+
+        public CodexDeployment[] GetDeployments()
+        {
+            return deployments.ToArray();
+        }
+
+        public async Task<bool> DownloadDeployment(IAttachment file)
+        {
+            using var http = new HttpClient();
+            var response = await http.GetAsync(file.Url);
+            var str = await response.Content.ReadAsStringAsync();
+            if (string.IsNullOrEmpty(str)) return false;
+
+            try
+            {
+                var deploy = JsonConvert.DeserializeObject<CodexDeployment>(str);
+                if (deploy != null)
+                {
+                    var targetFile = Path.Combine(Program.Config.EndpointsPath, Guid.NewGuid().ToString().ToLowerInvariant() + ".json");
+                    File.WriteAllText(targetFile, str);
+                    deployments.Add(deploy);
+                    return true;
+                }
+            }
+            catch { }
+            return false;
+        }
+
+        public bool DeleteDeployment(string deploymentName)
+        {
+            var path = Program.Config.EndpointsPath;
+            if (!Directory.Exists(path)) return false;
+            var files = Directory.GetFiles(path);
+
+            foreach (var file in files)
+            {
+                var deploy = ProcessFile(file);
+                if (deploy != null && deploy.Metadata.Name == deploymentName)
+                {
+                    File.Delete(file);
+                    deployments.Remove(deploy);
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        private void LoadDeployments()
+        {
+            var path = Program.Config.EndpointsPath;
+            if (!Directory.Exists(path))
+            {
+                Directory.CreateDirectory(path);
+                File.WriteAllText(Path.Combine(path, "readme.txt"), "Place codex-deployment.json here.");
+                return;
+            }
+
+            var files = Directory.GetFiles(path);
+            deployments.AddRange(files.Select(ProcessFile).Where(d => d != null).Cast<CodexDeployment>());
+        }
+
+        private CodexDeployment? ProcessFile(string filename)
+        {
+            try
+            {
+                var lines = string.Join(" ", File.ReadAllLines(filename));
+                return JsonConvert.DeserializeObject<CodexDeployment>(lines);
+            }
+            catch
+            {
+                return null;
+            }
+        }
+    }
+}

Tools/BiblioTech/Options/CommandContext.cs

@@ -0,0 +1,33 @@
+using Discord.WebSocket;
+
+namespace BiblioTech.Options
+{
+    public class CommandContext
+    {
+        public CommandContext(SocketSlashCommand command, IReadOnlyCollection<SocketSlashCommandDataOption> options)
+        {
+            Command = command;
+            Options = options;
+        }
+
+        public SocketSlashCommand Command { get; }
+        public IReadOnlyCollection<SocketSlashCommandDataOption> Options { get; }
+
+        public async Task Followup(string message)
+        {
+            await Command.ModifyOriginalResponseAsync(m =>
+            {
+                m.Content = message;
+            });
+        }
+
+        public async Task FollowupWithAttachement(string filename, string content)
+        {
+            using var fileStream = new MemoryStream();
+            using var streamWriter = new StreamWriter(fileStream);
+            await streamWriter.WriteAsync(content);
+
+            await Command.FollowupWithFileAsync(fileStream, filename);
+        }
+    }
+}

Tools/BiblioTech/Options/CommandOption.cs

@@ -0,0 +1,29 @@
+using Discord;
+
+namespace BiblioTech.Options
+{
+    public abstract class CommandOption
+    {
+        public CommandOption(string name, string description, ApplicationCommandOptionType type, bool isRequired)
+        {
+            Name = name;
+            Description = description;
+            Type = type;
+            IsRequired = isRequired;
+        }
+
+        public string Name { get; }
+        public string Description { get; }
+        public ApplicationCommandOptionType Type { get; }
+        public bool IsRequired { get; }
+
+        public virtual SlashCommandOptionBuilder Build()
+        {
+            return new SlashCommandOptionBuilder()
+                .WithName(Name)
+                .WithDescription(Description)
+                .WithType(Type)
+                .WithRequired(IsRequired);
+        }
+    }
+}

Tools/BiblioTech/Options/EthAddressOption.cs

@@ -0,0 +1,42 @@
+using GethPlugin;
+using Nethereum.Util;
+
+namespace BiblioTech.Options
+{
+    public class EthAddressOption : CommandOption
+    {
+        public EthAddressOption(bool isRequired)
+            : base(name: "ethaddress",
+                  description: "Ethereum address starting with '0x'.",
+                  type: Discord.ApplicationCommandOptionType.String,
+                  isRequired)
+        {
+        }
+
+        public async Task<EthAddress?> Parse(CommandContext context)
+        {
+            var ethOptionData = context.Options.SingleOrDefault(o => o.Name == Name);
+            if (ethOptionData == null)
+            {
+                await context.Followup("EthAddress option not received.");
+                return null;
+            }
+            var ethAddressStr = ethOptionData.Value as string;
+            if (string.IsNullOrEmpty(ethAddressStr))
+            {
+                await context.Followup("EthAddress is null or empty.");
+                return null;
+            }
+
+            if (!AddressUtil.Current.IsValidAddressLength(ethAddressStr) ||
+                !AddressUtil.Current.IsValidEthereumAddressHexFormat(ethAddressStr))
+                // !AddressUtil.Current.IsChecksumAddress(ethAddressStr)) - this might make a good option later, but for now it might just annoy users.
+            {
+                await context.Followup("EthAddress is not valid.");
+                return null;
+            }
+
+            return new EthAddress(ethAddressStr);
+        }
+    }
+}

Tools/BiblioTech/Options/FileAttachementOption.cs

@@ -0,0 +1,30 @@
+using Discord;
+
+namespace BiblioTech.Options
+{
+    public class FileAttachementOption : CommandOption
+    {
+        public FileAttachementOption(string name, string description, bool isRequired)
+            : base(name, description, type: ApplicationCommandOptionType.Attachment, isRequired)
+        {
+        }
+
+        public async Task<IAttachment?> Parse(CommandContext context)
+        {
+            var fileOptionData = context.Options.SingleOrDefault(o => o.Name == Name);
+            if (fileOptionData == null)
+            {
+                await context.Followup("Attachement option not received.");
+                return null;
+            }
+            var attachement = fileOptionData.Value as IAttachment;
+            if (attachement == null)
+            {
+                await context.Followup("Attachement is null or empty.");
+                return null;
+            }
+
+            return attachement;
+        }
+    }
+}

Tools/BiblioTech/Options/StringOption.cs

@@ -0,0 +1,23 @@
+using Discord;
+
+namespace BiblioTech.Options
+{
+    public class StringOption : CommandOption
+    {
+        public StringOption(string name, string description, bool isRequired)
+            : base(name, description, type: ApplicationCommandOptionType.String, isRequired)
+        {
+        }
+
+        public async Task<string?> Parse(CommandContext context)
+        {
+            var strData = context.Options.SingleOrDefault(o => o.Name == Name);
+            if (strData == null)
+            {
+                await context.Followup("String option not received.");
+                return null;
+            }
+            return strData.Value as string;
+        }
+    }
+}

Tools/BiblioTech/Options/SubCommandOption.cs

@@ -0,0 +1,40 @@
+using Discord;
+
+namespace BiblioTech.Options
+{
+    public abstract class SubCommandOption : CommandOption
+    {
+        public SubCommandOption(string name, string description)
+            : base(name, description, type: ApplicationCommandOptionType.SubCommand, isRequired: false)
+        {
+        }
+
+        public override SlashCommandOptionBuilder Build()
+        {
+            var builder = base.Build();
+            foreach (var option in Options)
+            {
+                builder.AddOption(option.Build());
+            }
+            return builder;
+        }
+
+        public async Task CommandHandler(CommandContext context)
+        {
+            var mine = context.Options.SingleOrDefault(o => o.Name == Name);
+            if (mine == null) return;
+
+            await onSubCommand(new CommandContext(context.Command, mine.Options));
+        }
+
+        public virtual CommandOption[] Options
+        {
+            get
+            {
+                return Array.Empty<CommandOption>();
+            }
+        }
+
+        protected abstract Task onSubCommand(CommandContext context);
+    }
+}