cleanup

# Conflicts:
#	Tests/CodexTests/UtilityTests/DiscordBotTests.cs

Framework/ArgsUniform/ArgsUniform.cs

@@ -4,9 +4,8 @@ namespace ArgsUniform
 {
     public class ArgsUniform<T>
     {
+        private readonly Assigner<T> assigner;
         private readonly Action printAppInfo;
-        private readonly object? defaultsProvider;
-        private readonly IEnv.IEnv env;
         private readonly string[] args;
         private const int cliStart = 8;
         private const int shortStart = 38;
@@ -31,9 +30,9 @@ namespace ArgsUniform
         public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
         {
             this.printAppInfo = printAppInfo;
-            this.defaultsProvider = defaultsProvider;
-            this.env = env;
             this.args = args;
+
+            assigner = new Assigner<T>(env, args, defaultsProvider);
         }
 
         public T Parse(bool printResult = false)
@@ -42,7 +41,7 @@ namespace ArgsUniform
             {
                 printAppInfo();
                 PrintHelp();
-                throw new Exception();
+                Environment.Exit(0);
             }
 
             var result = Activator.CreateInstance<T>();
@@ -53,18 +52,16 @@ namespace ArgsUniform
                 var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
                 if (attr != null)
                 {
-                    if (!UniformAssign(result, attr, uniformProperty) && attr.Required)
+                    if (!assigner.UniformAssign(result, attr, uniformProperty) && attr.Required)
                     {
-                        {
-                            missingRequired.Add(uniformProperty);
-                        }
+                        missingRequired.Add(uniformProperty);
                     }
                 }
             }
 
             if (missingRequired.Any())
             {
-                PrintResults(result, uniformProperties);
+                PrintResults(printResult,result, uniformProperties);
                 Print("");
                 foreach (var missing in missingRequired)
                 {
@@ -75,37 +72,39 @@ namespace ArgsUniform
                 }
 
                 PrintHelp();
-                throw new ArgumentException("Unable to assemble all required arguments");
+                Environment.Exit(1);
             }
 
-            if (printResult)
-            {
-                PrintResults(result, uniformProperties);
-            }
+            PrintResults(printResult, result, uniformProperties);
 
             return result;
         }
 
-        private void PrintResults(T result, PropertyInfo[] uniformProperties)
-        {
-            Print("");
-            foreach (var p in uniformProperties)
-            {
-                Print($"\t{p.Name} = {p.GetValue(result)}");
-            }
-            Print("");
-        }
-
         public void PrintHelp()
         {
             Print("");
-            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description");
-            var attrs = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).Select(p => p.GetCustomAttribute<UniformAttribute>()).Where(a => a != null).ToArray();
-            foreach (var attr in attrs)
+            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description", "(default)");
+            var props = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
+            foreach (var prop in props)
             {
-                var a = attr!;
-                var optional = !a.Required ? " *" : "";
-                PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional);
+                var a = prop.GetCustomAttribute<UniformAttribute>();
+                if (a != null)
+                {
+                    var optional = !a.Required ? " (optional)" : "";
+                    var def = assigner.DescribeDefaultFor(prop);
+                    PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional, $"({def})");
+                }
+            }
+            Print("");
+        }
+
+        private void PrintResults(bool printResult, T result, PropertyInfo[] uniformProperties)
+        {
+            if (!printResult) return;
+            Print("");
+            foreach (var p in uniformProperties)
+            {
+                Print($"\t{p.Name} = {p.GetValue(result)}");
             }
             Print("");
         }
@@ -115,7 +114,7 @@ namespace ArgsUniform
             Console.WriteLine(msg);
         }
 
-        private void PrintAligned(string cli, string s, string env, string desc)
+        private void PrintAligned(string cli, string s, string env, string desc, string def)
         {
             Console.CursorLeft = cliStart;
             Console.Write(cli);
@@ -124,132 +123,8 @@ namespace ArgsUniform
             Console.CursorLeft = envStart;
             Console.Write(env);
             Console.CursorLeft = descStart;
-            Console.Write(desc + Environment.NewLine);
-        }
-
-        private object GetDefaultValue(Type t)
-        {
-            if (t.IsValueType) return Activator.CreateInstance(t)!;
-            return null!;
-        }
-
-        private bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
-            return false;
-        }
-
-        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
-        {
-            var currentValue = uniformProperty.GetValue(result);
-            var isEmptryString = (currentValue as string) == string.Empty;
-            if (currentValue != GetDefaultValue(uniformProperty.PropertyType) && !isEmptryString) return true;
-
-            if (defaultsProvider == null) return false;
-
-            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
-            if (defaultProperty == null) return false;
-
-            var value = defaultProperty.GetValue(defaultsProvider);
-            if (value != null)
-            {
-                return Assign(result, uniformProperty, value);
-            }
-            return false;
-        }
-
-        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
-            if (!string.IsNullOrEmpty(e))
-            {
-                return Assign(result, uniformProperty, e);
-            }
-            return false;
-        }
-
-        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var fromArg = GetFromArgs(attr.Arg);
-            if (fromArg != null)
-            {
-                return Assign(result, uniformProperty, fromArg);
-            }
-            var fromShort = GetFromArgs(attr.ArgShort);
-            if (fromShort != null)
-            {
-                return Assign(result, uniformProperty, fromShort);
-            }
-            return false;
-        }
-
-        private bool Assign(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (uniformProperty.PropertyType == value.GetType())
-            {
-                uniformProperty.SetValue(result, value);
-                return true;
-            }
-            else
-            {
-                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
-                {
-                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
-                    return true;
-                }
-                else
-                {
-                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
-                    
-                    throw new NotSupportedException();
-                }
-            }
-        }
-
-        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
-            {
-                uniformProperty.SetValue(result, e);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (int.TryParse(value.ToString(), out int i))
-            {
-                uniformProperty.SetValue(result, i);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
-            {
-                uniformProperty.SetValue(result, true);
-            }
-            return true;
-        }
-
-        private string? GetFromArgs(string key)
-        {
-            var argKey = $"--{key}=";
-            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
-            if (arg != null)
-            {
-                return arg.Substring(argKey.Length);
-            }
-            return null;
+            Console.Write(desc + " ");
+            Console.Write(def + Environment.NewLine);
         }
     }
 }

Framework/ArgsUniform/Assigner.cs

@@ -0,0 +1,186 @@
+using System.Globalization;
+using System.Numerics;
+using System.Reflection;
+
+namespace ArgsUniform
+{
+    public class Assigner<T>
+    {
+        private readonly IEnv.IEnv env;
+        private readonly string[] args;
+        private readonly object? defaultsProvider;
+
+        public Assigner(IEnv.IEnv env, string[] args, object? defaultsProvider)
+        {
+            this.env = env;
+            this.args = args;
+            this.defaultsProvider = defaultsProvider;
+        }
+
+        public bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
+            return false;
+        }
+
+        public string DescribeDefaultFor(PropertyInfo property)
+        {
+            var obj = Activator.CreateInstance<T>();
+            var defaultValue = GetDefaultValue(obj, property);
+            if (defaultValue == null) return "";
+            if (defaultValue is string str)
+            {
+                return "\"" + str + "\"";
+            }
+            return defaultValue.ToString() ?? string.Empty;
+        }
+
+        private object? GetDefaultValue(T result, PropertyInfo uniformProperty)
+        {
+            // Get value from object's static initializer if it's there.
+            var currentValue = uniformProperty.GetValue(result);
+            if (currentValue != null) return currentValue;
+
+            // Get value from defaults-provider object if it's there.
+            if (defaultsProvider == null) return null;
+            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
+            if (defaultProperty == null) return null;
+            return defaultProperty.GetValue(defaultsProvider);
+        }
+
+        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
+        {
+            var defaultValue = GetDefaultValue(result, uniformProperty);
+            var isEmptryString = (defaultValue as string) == string.Empty;
+            if (defaultValue != null && defaultValue != GetDefaultValueForType(uniformProperty.PropertyType) && !isEmptryString)
+            {
+                return Assign(result, uniformProperty, defaultValue);
+            }
+            return false;
+        }
+
+        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
+            if (!string.IsNullOrEmpty(e))
+            {
+                return Assign(result, uniformProperty, e);
+            }
+            return false;
+        }
+
+        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var fromArg = GetFromArgs(attr.Arg);
+            if (fromArg != null)
+            {
+                return Assign(result, uniformProperty, fromArg);
+            }
+            var fromShort = GetFromArgs(attr.ArgShort);
+            if (fromShort != null)
+            {
+                return Assign(result, uniformProperty, fromShort);
+            }
+            return false;
+        }
+
+        private bool Assign(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (uniformProperty.PropertyType == value.GetType())
+            {
+                uniformProperty.SetValue(result, value);
+                return true;
+            }
+            else
+            {
+                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
+                {
+                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
+                    return true;
+                }
+                else
+                {
+                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(ulong)) return AssignUlong(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(BigInteger)) return AssignBigInt(result, uniformProperty, value);
+
+                    throw new NotSupportedException(
+                        $"Unsupported property type '${uniformProperty.PropertyType}' " +
+                        $"for property '${uniformProperty.Name}'.");
+                }
+            }
+        }
+
+        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
+            {
+                uniformProperty.SetValue(result, e);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (int.TryParse(value.ToString(), CultureInfo.InvariantCulture, out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignUlong(T? result, PropertyInfo uniformProperty, object value)
+        {
+            if (ulong.TryParse(value.ToString(), CultureInfo.InvariantCulture, out ulong i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignBigInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (BigInteger.TryParse(value.ToString(), CultureInfo.InvariantCulture, out BigInteger i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
+            {
+                uniformProperty.SetValue(result, true);
+            }
+            return true;
+        }
+
+        private string? GetFromArgs(string key)
+        {
+            var argKey = $"--{key}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+            return null;
+        }
+
+        private static object GetDefaultValueForType(Type t)
+        {
+            if (t.IsValueType) return Activator.CreateInstance(t)!;
+            return null!;
+        }
+    }
+}

Framework/Core/Http.cs

@@ -58,7 +58,7 @@ namespace Core
         {
             lock (httpLock)
             {
-                return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
+                return Time.Retry(operation, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
             }
         }
 

Framework/Core/TimeSet.cs

@@ -2,10 +2,31 @@
 {
     public interface ITimeSet
     {
+        /// <summary>
+        /// Timeout for a single HTTP call.
+        /// </summary>
         TimeSpan HttpCallTimeout();
-        int HttpMaxNumberOfRetries();
+
+        /// <summary>
+        /// Maximum total time to attempt to make a successful HTTP call to a service.
+        /// When HTTP calls time out during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan HttpRetryTimeout();
+
+        /// <summary>
+        /// After a failed HTTP call, wait this long before trying again.
+        /// </summary>
         TimeSpan HttpCallRetryDelay();
+
+        /// <summary>
+        /// After a failed K8s operation, wait this long before trying again.
+        /// </summary>
         TimeSpan K8sOperationRetryDelay();
+
+        /// <summary>
+        /// Maximum total time to attempt to perform a successful k8s operation.
+        /// If k8s operations fail during this timespan, retries will be made.
+        /// </summary>
         TimeSpan K8sOperationTimeout();
     }
 
@@ -16,9 +37,9 @@
             return TimeSpan.FromMinutes(3);
         }
 
-        public int HttpMaxNumberOfRetries()
+        public TimeSpan HttpRetryTimeout()
         {
-            return 3;
+            return TimeSpan.FromMinutes(10);
         }
 
         public TimeSpan HttpCallRetryDelay()
@@ -41,17 +62,17 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromHours(2);
+            return TimeSpan.FromMinutes(30);
         }
 
-        public int HttpMaxNumberOfRetries()
+        public TimeSpan HttpRetryTimeout()
         {
-            return 1;
+            return TimeSpan.FromHours(2.2);
         }
 
         public TimeSpan HttpCallRetryDelay()
         {
-            return TimeSpan.FromSeconds(2);
+            return TimeSpan.FromSeconds(20);
         }
 
         public TimeSpan K8sOperationRetryDelay()

Framework/FileUtils/TrackedFile.cs

@@ -65,7 +65,7 @@ namespace FileUtils
 
                 if (readExpected == 0 && readActual == 0)
                 {
-                    log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
+                    log.Log($"OK: {Describe()} is equal to {actual.Describe()}.");
                     return;
                 }
 

Framework/KubernetesWorkflow/K8sController.cs

@@ -712,7 +712,7 @@ namespace KubernetesWorkflow
         {
             return Time.Retry(() => GetPodForDeplomentInternal(deployment),
                 // We will wait up to 1 minute, k8s might be moving pods around.
-                maxRetries: 6,
+                maxTimeout: TimeSpan.FromMinutes(1),
                 retryTime: TimeSpan.FromSeconds(10),
                 description: "Find pod by label for deployment.");
         }

Framework/KubernetesWorkflow/Types/RunningPod.cs

@@ -20,7 +20,7 @@ namespace KubernetesWorkflow.Types
         [JsonIgnore]
         public string Name
         {
-            get { return $"{Containers.Length}x '{Containers.First().Name}'"; }
+            get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
         }
 
         public string Describe()

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -20,6 +20,7 @@ namespace NethereumWorkflow.BlockUtils
 
         public BlockTimeEntry Get(ulong blockNumber)
         {
+            bounds.Initialize();
             var b = cache.Get(blockNumber);
             if (b != null) return b;
             return GetBlock(blockNumber);
@@ -45,7 +46,7 @@ namespace NethereumWorkflow.BlockUtils
 
         private ulong Log(Func<ulong> operation)
         {
-            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder));
+            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder), true);
             var result = operation();
             sw.End($"(Bounds: [{bounds.Genesis.BlockNumber}-{bounds.Current.BlockNumber}] Cache: {cache.Size})");
 

Framework/Utils/Time.cs

@@ -1,4 +1,6 @@
-namespace Utils
+using System.Diagnostics;
+
+namespace Utils
 {
     public static class Time
     {
@@ -18,6 +20,12 @@
             task.Wait();
         }
 
+        public static string FormatDuration(TimeSpan? d)
+        {
+            if (d == null) return "[NULL]";
+            return FormatDuration(d.Value);
+        }
+
         public static string FormatDuration(TimeSpan d)
         {
             var result = "";
@@ -83,37 +91,40 @@
 
         public static void Retry(Action action, string description)
         {
-            Retry(action, 1, description);
+            Retry(action, TimeSpan.FromSeconds(30), description);
         }
 
         public static T Retry<T>(Func<T> action, string description)
         {
-            return Retry(action, 1, description);
+            return Retry(action, TimeSpan.FromSeconds(30), description);
         }
 
-        public static void Retry(Action action, int maxRetries, string description)
+        public static void Retry(Action action, TimeSpan maxTimeout, string description)
         {
-            Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
+            Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
         }
 
-        public static T Retry<T>(Func<T> action, int maxRetries, string description)
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, string description)
         {
-            return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
+            return Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
         }
 
-        public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
+        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
-            var retries = 0;
-            var exceptions = new List<Exception>();
+            var tries = 1;
+            var tryInfo = new List<(Exception, TimeSpan)>();
+
             while (true)
             {
-                if (retries > maxRetries)
+                var duration = DateTime.UtcNow - start;
+                if (duration > maxTimeout)
                 {
-                    var duration = DateTime.UtcNow - start;
-                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
+                    var info = FormatTryInfos(tryInfo);
+                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.{Environment.NewLine}{info}");
                 }
 
+                var sw = Stopwatch.StartNew();
                 try
                 {
                     action();
@@ -121,25 +132,42 @@
                 }
                 catch (Exception ex)
                 {
-                    exceptions.Add(ex);
-                    retries++;
+                    tryInfo.Add((ex, sw.Elapsed));
+                    tries++;
                 }
 
                 Sleep(retryTime);
             }
         }
 
-        public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
+        private static string FormatTryInfos(List<(Exception, TimeSpan)> tryInfo)
+        {
+            return string.Join(Environment.NewLine, tryInfo.Select(FormatTryInfo).ToArray());
+        }
+
+        private static string FormatTryInfo((Exception, TimeSpan) info, int index)
+        {
+            return $"Attempt {index} took {FormatDuration(info.Item2)} and failed with exception {info.Item1}.";
+        }
+
+        private static Action<int> failedCallback = i => { };
+        public static void SetRetryFailedCallback(Action<int> onRetryFailed)
+        {
+            failedCallback = onRetryFailed;
+        }
+
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
-            var retries = 0;
+            var tries = 1;
             var exceptions = new List<Exception>();
+
             while (true)
             {
-                if (retries > maxRetries)
+                var duration = DateTime.UtcNow - start;
+                if (duration > maxTimeout)
                 {
-                    var duration = DateTime.UtcNow - start;
-                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
+                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
                 try
@@ -149,7 +177,8 @@
                 catch (Exception ex)
                 {
                     exceptions.Add(ex);
-                    retries++;
+                    failedCallback(tries);
+                    tries++;
                 }
 
                 Sleep(retryTime);

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -67,7 +67,7 @@ namespace CodexContractsPlugin
 
         public string MintTestTokens(EthAddress ethAddress, TestToken testTokens)
         {
-            return StartInteraction().MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
+            return StartInteraction().MintTestTokens(ethAddress, testTokens.TstWei, Deployment.TokenAddress);
         }
 
         public TestToken GetTestTokenBalance(IHasEthAddress owner)
@@ -78,7 +78,7 @@ namespace CodexContractsPlugin
         public TestToken GetTestTokenBalance(EthAddress ethAddress)
         {
             var balance = StartInteraction().GetBalance(Deployment.TokenAddress, ethAddress.Address);
-            return balance.TestTokens();
+            return balance.TstWei();
         }
 
         public Request[] GetStorageRequests(BlockInterval blockRange)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -44,7 +44,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        public string MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
+        public string MintTestTokens(EthAddress address, BigInteger amount, string tokenAddress)
         {
             log.Debug($"{amount} -> {address} (token: {tokenAddress})");
             return MintTokens(address.Address, amount, tokenAddress);
@@ -85,7 +85,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        private string MintTokens(string account, decimal amount, string tokenAddress)
+        private string MintTokens(string account, BigInteger amount, string tokenAddress)
         {
             log.Debug($"({tokenAddress}) {amount} --> {account}");
             if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
@@ -93,7 +93,7 @@ namespace CodexContractsPlugin
             var function = new MintTokensFunction
             {
                 Holder = account,
-                Amount = amount.ToBig()
+                Amount = amount
             };
 
             return gethNode.SendTransaction(tokenAddress, function);

ProjectPlugins/CodexContractsPlugin/Marketplace/README.md

@@ -1 +1,14 @@
 This code was generated using the Nethereum code generator, here: http://playground.nethereum.com
+
+1. Go to site -> Abi Code Gen.
+1. Contract name = "Marketplace".
+1. In container, get "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json".
+1. Save only ABI section as new JSON. (top-level is a json array.)
+1. From original JSON get byte code.
+1. Put ABI JSON and byte code into site.
+1. Generate.
+1. From site generated code, copy `public partial class MarketplaceDeployment` and everything after it. (be considerate of namespace brackets!)
+1. In Marketplace/Marketplace.cs, replace content of 'namespace CodexContractsPlugin.Marketplace'.
+
+
+

ProjectPlugins/CodexContractsPlugin/TestTokenExtensions.cs

@@ -1,45 +1,102 @@
-namespace CodexContractsPlugin
+using System.Numerics;
+
+namespace CodexContractsPlugin
 {
     public class TestToken : IComparable<TestToken>
     {
-        public TestToken(decimal amount)
+        public static BigInteger WeiFactor = new BigInteger(1000000000000000000);
+
+        public TestToken(BigInteger tstWei)
         {
-            Amount = amount;
+            TstWei = tstWei;
+            Tst = tstWei / WeiFactor;
         }
 
-        public decimal Amount { get; }
+        public BigInteger TstWei { get; }
+        public BigInteger Tst { get; }
 
         public int CompareTo(TestToken? other)
         {
-            return Amount.CompareTo(other!.Amount);
+            return TstWei.CompareTo(other!.TstWei);
         }
 
         public override bool Equals(object? obj)
         {
-            return obj is TestToken token && Amount == token.Amount;
+            return obj is TestToken token && TstWei == token.TstWei;
         }
 
         public override int GetHashCode()
         {
-            return HashCode.Combine(Amount);
+            return HashCode.Combine(TstWei);
         }
 
         public override string ToString()
         {
-            return $"{Amount} TestTokens";
+            var weiOnly = TstWei % WeiFactor;
+
+            var tokens = new List<string>();
+            if (Tst > 0) tokens.Add($"{Tst} TST");
+            if (weiOnly > 0) tokens.Add($"{weiOnly} TSTWEI");
+
+            return string.Join(" + ", tokens);
+        }
+
+        public static TestToken operator +(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei + b.TstWei);
+        }
+
+        public static bool operator <(TestToken a, TestToken b)
+        {
+            return a.TstWei < b.TstWei;
+        }
+
+        public static bool operator >(TestToken a, TestToken b)
+        {
+            return a.TstWei > b.TstWei;
+        }
+
+        public static bool operator ==(TestToken a, TestToken b)
+        {
+            return a.TstWei == b.TstWei;
+        }
+
+        public static bool operator !=(TestToken a, TestToken b)
+        {
+            return a.TstWei != b.TstWei;
         }
     }
 
-    public static class TokensIntExtensions
+    public static class TestTokensExtensions
     {
-        public static TestToken TestTokens(this int i)
+        public static TestToken TstWei(this int i)
         {
-            return TestTokens(Convert.ToDecimal(i));
+            return TstWei(Convert.ToDecimal(i));
         }
 
-        public static TestToken TestTokens(this decimal i)
+        public static TestToken TstWei(this decimal i)
+        {
+            return new TestToken(new BigInteger(i));
+        }
+
+        public static TestToken TstWei(this BigInteger i)
         {
             return new TestToken(i);
         }
+
+        public static TestToken Tst(this int i)
+        {
+            return Tst(Convert.ToDecimal(i));
+        }
+
+        public static TestToken Tst(this decimal i)
+        {
+            return new TestToken(new BigInteger(i) * TestToken.WeiFactor);
+        }
+
+        public static TestToken Tst(this BigInteger i)
+        {
+            return new TestToken(i * TestToken.WeiFactor);
+        }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -1,11 +1,13 @@
 using Core;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Utils;
 
 namespace CodexDiscordBotPlugin
 {
     public class CodexDiscordBotPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
     {
+        private const string ExpectedStartupMessage = "Debug option is set. Discord connection disabled!";
         private readonly IPluginTools tools;
 
         public CodexDiscordBotPlugin(IPluginTools tools)
@@ -46,14 +48,56 @@ namespace CodexDiscordBotPlugin
             var startupConfig = new StartupConfig();
             startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
+            var pod = workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
+            WaitForStartupMessage(workflow, pod);
+            return pod;
         }
 
         private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
+            startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
             return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
         }
+
+        private void WaitForStartupMessage(IStartupWorkflow workflow, RunningPod pod)
+        {
+            var finder = new LogLineFinder(ExpectedStartupMessage, workflow);
+            Time.WaitUntil(() =>
+            {
+                finder.FindLine(pod);
+                return finder.Found;
+            }, nameof(WaitForStartupMessage));
+        }
+
+        public class LogLineFinder : LogHandler
+        {
+            private readonly string message;
+            private readonly IStartupWorkflow workflow;
+
+            public LogLineFinder(string message, IStartupWorkflow workflow)
+            {
+                this.message = message;
+                this.workflow = workflow;
+            }
+
+            public void FindLine(RunningPod pod)
+            {
+                Found = false;
+                foreach (var c in pod.Containers)
+                {
+                    workflow.DownloadContainerLog(c, this);
+                    if (Found) return;
+                }
+            }
+
+            public bool Found { get; private set; }
+
+            protected override void ProcessLine(string line)
+            {
+                if (!Found && line.Contains(message)) Found = true;
+            }
+        }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin
     public class DiscordBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-bibliotech";
-        public override string Image => "codexstorage/codex-discordbot:sha-8c64352";
+        public override string Image => "codexstorage/codex-discordbot:sha-22cf82b";
 
         public static string RewardsPort = "bot_rewards_port";
 
@@ -33,6 +33,8 @@ namespace CodexDiscordBotPlugin
             AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
             AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
 
+            AddEnvVar("NODISCORD", "1");
+
             AddInternalPortAndVar("REWARDAPIPORT", RewardsPort);
 
             if (!string.IsNullOrEmpty(config.DataPath))

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs

@@ -27,8 +27,9 @@
 
     public class RewarderBotStartupConfig
     {
-        public RewarderBotStartupConfig(string discordBotHost, int discordBotPort, string intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath)
+        public RewarderBotStartupConfig(string name, string discordBotHost, int discordBotPort, int intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath)
         {
+            Name = name;
             DiscordBotHost = discordBotHost;
             DiscordBotPort = discordBotPort;
             IntervalMinutes = intervalMinutes;
@@ -37,9 +38,10 @@
             DataPath = dataPath;
         }
 
+        public string Name { get; }
         public string DiscordBotHost { get; }
         public int DiscordBotPort { get; }
-        public string IntervalMinutes { get; }
+        public int IntervalMinutes { get; }
         public DateTime HistoryStartUtc { get; }
         public DiscordBotGethInfo GethInfo { get; }
         public string? DataPath { get; set; }

ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin
     public class RewarderBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-rewarder";
-        public override string Image => "codexstorage/codex-rewarderbot:sha-2ab84e2";
+        public override string Image => "codexstorage/codex-rewarderbot:sha-12dc7ef";
 
         protected override void Initialize(StartupConfig startupConfig)
         {
@@ -17,7 +17,7 @@ namespace CodexDiscordBotPlugin
 
             AddEnvVar("DISCORDBOTHOST", config.DiscordBotHost);
             AddEnvVar("DISCORDBOTPORT", config.DiscordBotPort.ToString());
-            AddEnvVar("INTERVALMINUTES", config.IntervalMinutes);
+            AddEnvVar("INTERVALMINUTES", config.IntervalMinutes.ToString());
             var offset = new DateTimeOffset(config.HistoryStartUtc);
             AddEnvVar("CHECKHISTORY", offset.ToUnixTimeSeconds().ToString());
 

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -9,7 +9,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "5A-B0-2A-AC-42-B1-A2-49-6F-9D-4E-D8-56-40-10-A6-67-F4-0D-2A-9F-E0-84-5C-EB-B8-2D-4F-D8-56-79-6C";
+        private const string OpenApiYamlHash = "0F-C8-02-1E-2C-2C-15-F6-91-6A-01-31-11-49-95-06-79-26-25-BF-27-3C-A8-2E-5F-7F-34-FD-C0-57-A0-9A";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,8 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-1524803-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-267266a-dist-tests";
+
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -108,8 +109,9 @@ namespace CodexPlugin
 
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
-                AddEnvVar("PRIV_KEY", marketplaceSetup.EthAccount.PrivateKey);
-                Additional(marketplaceSetup.EthAccount);
+                var account = marketplaceSetup.EthAccountSetup.GetNew();
+                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                Additional(account);
 
                 SetCommandOverride(marketplaceSetup);
                 if (marketplaceSetup.IsValidator)

ProjectPlugins/CodexPlugin/CodexLogLine.cs

@@ -0,0 +1,105 @@
+using System.Globalization;
+
+namespace CodexPlugin
+{
+    public class CodexLogLine
+    {
+        public static CodexLogLine? Parse(string line)
+        {
+            try
+            {
+                if (string.IsNullOrEmpty(line) ||
+                    line.Length < 34 ||
+                    line[3] != ' ' ||
+                    line[33] != ' ') return null;
+
+                line = line.Replace(Environment.NewLine, string.Empty);
+
+                var level = line.Substring(0, 3);
+                var dtLine = line.Substring(4, 23);
+
+                var firstEqualSign = line.IndexOf('=');
+                var msgStart = 34;
+                var msgEnd = line.Substring(0, firstEqualSign).LastIndexOf(' ');
+                var msg = line.Substring(msgStart, msgEnd - msgStart).Trim();
+                var attrsLine = line.Substring(msgEnd);
+
+                var attrs = SplitAttrs(attrsLine);
+
+                var format = "yyyy-MM-dd HH:mm:ss.fff";
+                var dt = DateTime.ParseExact(dtLine, format, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToUniversalTime();
+
+                return new CodexLogLine()
+                {
+                    LogLevel = level,
+                    TimestampUtc = dt,
+                    Message = msg,
+                    Attributes = attrs
+                };
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        public string LogLevel { get; set; } = string.Empty;
+        public DateTime TimestampUtc { get; set; }
+        public string Message { get; set; } = string.Empty;
+        public Dictionary<string, string> Attributes { get; private set; } = new Dictionary<string, string>();
+
+        /// <summary>
+        /// After too much time spent cursing at regexes, here's what I got:
+        /// Parses input string into 'key=value' pair, considerate of quoted (") values.
+        /// </summary>
+        private static Dictionary<string, string> SplitAttrs(string input)
+        {
+            input += " ";
+            var result = new Dictionary<string, string>();
+
+            var key = string.Empty;
+            var value = string.Empty;
+            var mode = 1;
+            var inQuote = false;
+
+            foreach (var c in input)
+            {
+                if (mode == 1)
+                {
+                    if (c == '=') mode = 2;
+                    else if (c == ' ')
+                    {
+                        if (string.IsNullOrEmpty(key)) continue;
+                        else
+                        {
+                            result.Add(key, string.Empty);
+                            key = string.Empty;
+                            value = string.Empty;
+                        }
+                    }
+                    else key += c;
+                }
+                else if (mode == 2)
+                {
+                    if (c == ' ' && !inQuote)
+                    {
+                        result.Add(key, value);
+                        key = string.Empty;
+                        value = string.Empty;
+                        mode = 1;
+                    }
+                    else if (c == '\"')
+                    {
+                        inQuote = !inQuote;
+                    }
+                    else
+                    {
+                        value += c;
+                    }
+                }
+            }
+
+            return result;
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -23,6 +23,7 @@ namespace CodexPlugin
         CrashWatcher CrashWatcher { get; }
         PodInfo GetPodInfo();
         ITransferSpeeds TransferSpeeds { get; }
+        EthAccount EthAccount { get; }
         void Stop(bool waitTillStopped);
     }
 
@@ -30,13 +31,13 @@ namespace CodexPlugin
     {
         private const string UploadFailedMessage = "Unable to store block";
         private readonly IPluginTools tools;
-        private readonly EthAddress? ethAddress;
+        private readonly EthAccount? ethAccount;
         private readonly TransferSpeeds transferSpeeds;
 
-        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAddress? ethAddress)
+        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAccount? ethAccount)
         {
             this.tools = tools;
-            this.ethAddress = ethAddress;
+            this.ethAccount = ethAccount;
             CodexAccess = codexAccess;
             Group = group;
             Marketplace = marketplaceAccess;
@@ -66,8 +67,17 @@ namespace CodexPlugin
         {
             get
             {
-                if (ethAddress == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
-                return ethAddress;
+                EnsureMarketplace();
+                return ethAccount!.EthAddress;
+            }
+        }
+
+        public EthAccount EthAccount
+        {
+            get
+            {
+                EnsureMarketplace();
+                return ethAccount!;
             }
         }
 
@@ -197,6 +207,11 @@ namespace CodexPlugin
             }
         }
 
+        private void EnsureMarketplace()
+        {
+            if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
+        }
+
         private void Log(string msg)
         {
             tools.GetLog().Log($"{GetName()}: {msg}");

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -22,22 +22,22 @@ namespace CodexPlugin
 
         public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
         {
-            var ethAddress = GetEthAddress(access);
-            var marketplaceAccess = GetMarketplaceAccess(access, ethAddress);
-            return new CodexNode(tools, access, group, marketplaceAccess, ethAddress);
+            var ethAccount = GetEthAccount(access);
+            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount);
+            return new CodexNode(tools, access, group, marketplaceAccess, ethAccount);
         }
 
-        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAddress? ethAddress)
+        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount)
         {
-            if (ethAddress == null) return new MarketplaceUnavailable();
+            if (ethAccount == null) return new MarketplaceUnavailable();
             return new MarketplaceAccess(tools.GetLog(), codexAccess);
         }
 
-        private EthAddress? GetEthAddress(CodexAccess access)
+        private EthAccount? GetEthAccount(CodexAccess access)
         {
             var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
             if (ethAccount == null) return null;
-            return ethAccount.EthAddress;
+            return ethAccount;
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer c)

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -52,6 +52,7 @@ namespace CodexPlugin
         public CodexLogLevel Libp2p { get; set; }
         public CodexLogLevel ContractClock { get; set; } = CodexLogLevel.Warn;
         public CodexLogLevel? BlockExchange { get; }
+        public CodexLogLevel JsonSerialize { get; set; } = CodexLogLevel.Warn;
     }
 
     public class CodexSetup : CodexStartupConfig, ICodexSetup
@@ -167,8 +168,8 @@ namespace CodexPlugin
         public bool IsStorageNode { get; private set; }
         public bool IsValidator { get; private set; }
         public Ether InitialEth { get; private set; } = 0.Eth();
-        public TestToken InitialTestTokens { get; private set; } = 0.TestTokens();
-        public EthAccount EthAccount { get; private set; } = EthAccount.GenerateNew();
+        public TestToken InitialTestTokens { get; private set; } = 0.Tst();
+        public EthAccountSetup EthAccountSetup { get; private set; } = new EthAccountSetup();
 
         public IMarketplaceSetup AsStorageNode()
         {
@@ -184,7 +185,7 @@ namespace CodexPlugin
 
         public IMarketplaceSetup WithAccount(EthAccount account)
         {
-            EthAccount = account;
+            EthAccountSetup.Pin(account);
             return this;
         }
 
@@ -200,10 +201,41 @@ namespace CodexPlugin
             var result = "[(clientNode)"; // When marketplace is enabled, being a clientNode is implicit.
             result += IsStorageNode ? "(storageNode)" : "()";
             result += IsValidator ? "(validator)" : "() ";
-            result += $"Address: '{EthAccount.EthAddress}' ";
-            result += $"InitialEth/TT({InitialEth.Eth}/{InitialTestTokens.Amount})";
+            result += $"Address: '{EthAccountSetup}' ";
+            result += $"{InitialEth.Eth} / {InitialTestTokens}";
             result += "] ";
             return result;
         }
     }
+
+    public class EthAccountSetup
+    {
+        private readonly List<EthAccount> accounts = new List<EthAccount>();
+        private bool pinned = false;
+
+        public void Pin(EthAccount account)
+        {
+            accounts.Add(account);
+            pinned = true;
+        }
+
+        public EthAccount GetNew()
+        {
+            if (pinned) return accounts.Last();
+
+            var a = EthAccount.GenerateNew();
+            accounts.Add(a);
+            return a;
+        }
+
+        public EthAccount[] GetAll()
+        {
+            return accounts.ToArray();
+        }
+
+        public override string ToString()
+        {
+            return string.Join(",", accounts.Select(a => a.ToString()).ToArray());
+        }
+    }
 }

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -73,11 +73,18 @@ namespace CodexPlugin
                     "contracts",
                     "clock"
                 };
+                var jsonSerializeTopics = new[]
+                {
+                    "serde",
+                    "json",
+                    "serialization"
+                };
 
                 level = $"{level};" +
                     $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
                     $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" +
-                    $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)}";
+                    $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)};" +
+                    $"{CustomTopics.JsonSerialize.ToString().ToLowerInvariant()}:{string.Join(",", jsonSerializeTopics)}";
 
                 if (CustomTopics.BlockExchange != null)
                 {

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -56,7 +56,7 @@ namespace CodexPlugin
                 ProofProbability = ToDecInt(purchase.ProofProbability),
                 Reward = ToDecInt(purchase.PricePerSlotPerSecond),
                 Collateral = ToDecInt(purchase.RequiredCollateral),
-                Expiry = ToDecInt(DateTimeOffset.UtcNow.ToUnixTimeSeconds() + purchase.Expiry.TotalSeconds),
+                Expiry = ToDecInt(purchase.Expiry.TotalSeconds),
                 Nodes = Convert.ToInt32(purchase.MinRequiredNumberOfNodes),
                 Tolerance = Convert.ToInt32(purchase.NodeFailureTolerance)
             };
@@ -74,10 +74,10 @@ namespace CodexPlugin
         public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD read)
         {
             return new StorageAvailability(
-                totalSpace: new Utils.ByteSize(Convert.ToInt64(read.TotalSize)),
+                totalSpace: new ByteSize(Convert.ToInt64(read.TotalSize)),
                 maxDuration: TimeSpan.FromSeconds(Convert.ToDouble(read.Duration)),
-                minPriceForTotalSpace: new TestToken(Convert.ToDecimal(read.MinPrice)),
-                maxCollateral: new TestToken(Convert.ToDecimal(read.MaxCollateral))
+                minPriceForTotalSpace: new TestToken(BigInteger.Parse(read.MinPrice)),
+                maxCollateral: new TestToken(BigInteger.Parse(read.MaxCollateral))
             )
             {
                 Id = read.Id
@@ -165,8 +165,7 @@ namespace CodexPlugin
 
         private string ToDecInt(TestToken t)
         {
-            var i = new BigInteger(t.Amount);
-            return i.ToString("D");
+            return t.TstWei.ToString("D");
         }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -1,5 +1,4 @@
 using Logging;
-using Newtonsoft.Json;
 using Utils;
 
 namespace CodexPlugin
@@ -7,7 +6,7 @@ namespace CodexPlugin
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(StorageAvailability availability);
-        StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
+        IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
     }
 
     public class MarketplaceAccess : IMarketplaceAccess
@@ -21,7 +20,7 @@ namespace CodexPlugin
             this.codexAccess = codexAccess;
         }
 
-        public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
+        public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             purchase.Log(log);
 
@@ -38,7 +37,9 @@ namespace CodexPlugin
 
             Log($"Storage requested successfully. PurchaseId: '{response}'.");
 
-            return new StoragePurchaseContract(log, codexAccess, response, purchase);
+            var contract = new StoragePurchaseContract(log, codexAccess, response, purchase);
+            contract.WaitForStorageContractSubmitted();
+            return contract;
         }
 
         public string MakeStorageAvailable(StorageAvailability availability)
@@ -54,7 +55,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            log.Log($"{codexAccess.Container.Name} {msg}");
+            log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
         }
     }
 
@@ -66,7 +67,7 @@ namespace CodexPlugin
             throw new NotImplementedException();
         }
 
-        public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
+        public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             Unavailable();
             throw new NotImplementedException();
@@ -78,78 +79,4 @@ namespace CodexPlugin
             throw new InvalidOperationException();
         }
     }
-
-    public class StoragePurchaseContract
-    {
-        private readonly ILog log;
-        private readonly CodexAccess codexAccess;
-        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(10);
-        private DateTime? contractStartUtc;
-
-        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase)
-        {
-            this.log = log;
-            this.codexAccess = codexAccess;
-            PurchaseId = purchaseId;
-            Purchase = purchase;
-        }
-
-        public string PurchaseId { get; }
-        public StoragePurchaseRequest Purchase { get; }
-
-        public void WaitForStorageContractStarted()
-        {
-            var timeout = Purchase.Expiry + gracePeriod;
-
-            WaitForStorageContractState(timeout, "started");
-            contractStartUtc = DateTime.UtcNow;
-        }
-
-        public void WaitForStorageContractFinished()
-        {
-            if (!contractStartUtc.HasValue)
-            {
-                WaitForStorageContractStarted();
-            }
-            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
-            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
-            WaitForStorageContractState(timeout, "finished");
-        }
-
-        public StoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return codexAccess.GetPurchaseStatus(purchaseId);
-        }
-
-        private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
-        {
-            var lastState = "";
-            var waitStart = DateTime.UtcNow;
-
-            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'.");
-            while (lastState != desiredState)
-            {
-                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
-                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
-                if (purchaseStatus != null && purchaseStatus.State != lastState)
-                {
-                    lastState = purchaseStatus.State;
-                    log.Debug("Purchase status: " + statusJson);
-                }
-
-                Thread.Sleep(1000);
-
-                if (lastState == "errored")
-                {
-                    FrameworkAssert.Fail("Contract errored: " + statusJson);
-                }
-
-                if (DateTime.UtcNow - waitStart > timeout)
-                {
-                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
-                }
-            }
-            log.Log($"Contract '{desiredState}'.");
-        }
-    }
 }

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -12,8 +12,8 @@ namespace CodexPlugin
         }
 
         public ContentId ContentId { get; set; }
-        public TestToken PricePerSlotPerSecond { get; set; } = 1.TestTokens();
-        public TestToken RequiredCollateral { get; set; } = 1.TestTokens();
+        public TestToken PricePerSlotPerSecond { get; set; } = 1.TstWei();
+        public TestToken RequiredCollateral { get; set; } = 1.TstWei();
         public uint MinRequiredNumberOfNodes { get; set; }
         public uint NodeFailureTolerance { get; set; }
         public int ProofProbability { get; set; }

ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs

@@ -0,0 +1,140 @@
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace CodexPlugin
+{
+    public interface IStoragePurchaseContract
+    {
+        void WaitForStorageContractSubmitted();
+        void WaitForStorageContractStarted();
+        void WaitForStorageContractFinished();
+    }
+
+    public class StoragePurchaseContract : IStoragePurchaseContract
+    {
+        private readonly ILog log;
+        private readonly CodexAccess codexAccess;
+        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
+        private readonly DateTime contractPendingUtc = DateTime.UtcNow;
+        private DateTime? contractSubmittedUtc = DateTime.UtcNow;
+        private DateTime? contractStartedUtc;
+        private DateTime? contractFinishedUtc;
+
+        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase)
+        {
+            this.log = log;
+            this.codexAccess = codexAccess;
+            PurchaseId = purchaseId;
+            Purchase = purchase;
+        }
+
+        public string PurchaseId { get; }
+        public StoragePurchaseRequest Purchase { get; }
+
+        public TimeSpan? PendingToSubmitted => contractSubmittedUtc - contractPendingUtc;
+        public TimeSpan? SubmittedToStarted => contractStartedUtc - contractSubmittedUtc;
+        public TimeSpan? SubmittedToFinished => contractFinishedUtc - contractSubmittedUtc;
+
+        public void WaitForStorageContractSubmitted()
+        {
+            WaitForStorageContractState(gracePeriod, "submitted", sleep: 200);
+            contractSubmittedUtc = DateTime.UtcNow;
+            LogSubmittedDuration();
+            AssertDuration(PendingToSubmitted, gracePeriod, nameof(PendingToSubmitted));
+        }
+
+        public void WaitForStorageContractStarted()
+        {
+            var timeout = Purchase.Expiry + gracePeriod;
+
+            WaitForStorageContractState(timeout, "started");
+            contractStartedUtc = DateTime.UtcNow;
+            LogStartedDuration();
+            AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
+        }
+
+        public void WaitForStorageContractFinished()
+        {
+            if (!contractStartedUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
+            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "finished");
+            contractFinishedUtc = DateTime.UtcNow;
+            LogFinishedDuration();
+            AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
+        }
+
+        public StoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return codexAccess.GetPurchaseStatus(purchaseId);
+        }
+
+        private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)
+        {
+            var lastState = "";
+            var waitStart = DateTime.UtcNow;
+
+            Log($"Waiting for {Time.FormatDuration(timeout)} to reach state '{desiredState}'.");
+            while (lastState != desiredState)
+            {
+                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
+                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
+                if (purchaseStatus != null && purchaseStatus.State != lastState)
+                {
+                    lastState = purchaseStatus.State;
+                    log.Debug("Purchase status: " + statusJson);
+                }
+
+                Thread.Sleep(sleep);
+
+                if (lastState == "errored")
+                {
+                    FrameworkAssert.Fail("Contract errored: " + statusJson);
+                }
+
+                if (DateTime.UtcNow - waitStart > timeout)
+                {
+                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
+                }
+            }
+        }
+
+        private void LogSubmittedDuration()
+        {
+            Log($"Pending to Submitted in {Time.FormatDuration(PendingToSubmitted)} " +
+                $"( < {Time.FormatDuration(gracePeriod)})");
+        }
+
+        private void LogStartedDuration()
+        {
+            Log($"Submitted to Started in {Time.FormatDuration(SubmittedToStarted)} " +
+                $"( < {Time.FormatDuration(Purchase.Expiry + gracePeriod)})");
+        }
+
+        private void LogFinishedDuration()
+        {
+            Log($"Submitted to Finished in {Time.FormatDuration(SubmittedToFinished)} " +
+                $"( < {Time.FormatDuration(Purchase.Duration + gracePeriod)})");
+        }
+
+        private void AssertDuration(TimeSpan? span, TimeSpan max, string message)
+        {
+            if (span == null) throw new ArgumentNullException(nameof(MarketplaceAccess) + ": " + message + " (IsNull)");
+            if (span.Value.TotalDays >= max.TotalSeconds)
+            {
+                throw new Exception(nameof(MarketplaceAccess) +
+                    $": Duration out of range. Max: {Time.FormatDuration(max)} but was: {Time.FormatDuration(span.Value)} " +
+                    message);
+            }
+        }
+
+        private void Log(string msg)
+        {
+            log.Log($"[{PurchaseId}] {msg}");
+        }
+    }
+}

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -213,8 +213,7 @@ components:
           description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots
         expiry:
           type: string
-          description: Number as decimal string that represents expiry time of the request (in unix timestamp)
-
+          description: Number as decimal string that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself.
     StorageAsk:
       type: object
       required:

ProjectPlugins/GethPlugin/EthAccount.cs

@@ -24,5 +24,10 @@ namespace GethPlugin
 
             return new EthAccount(ethAddress, account.PrivateKey);
         }
+
+        public override string ToString()
+        {
+            return EthAddress.ToString();
+        }
     }
 }

Tests/CodexLongTests/BasicTests/DownloadTests.cs

@@ -1,3 +1,4 @@
+using CodexPlugin;
 using CodexTests;
 using DistTestCore;
 using FileUtils;
@@ -7,35 +8,45 @@ using Utils;
 namespace CodexLongTests.BasicTests
 {
     [TestFixture]
-    public class DownloadTests : CodexDistTest
+    public class DownloadTests : AutoBootstrapDistTest
     {
-        [TestCase(3, 500)]
-        [TestCase(5, 100)]
-        [TestCase(10, 256)]
+        [Test]
+        [Combinatorial]
         [UseLongTimeouts]
-        public void ParallelDownload(int numberOfNodes, int filesizeMb)
+        public void ParallelDownload(
+            [Values(1, 3, 5)] int numberOfFiles,
+            [Values(10, 50, 100)] int filesizeMb)
         {
-            var group = AddCodex(numberOfNodes);
-            var host = AddCodex();
+            var host = StartCodex();
+            var client = StartCodex();
 
-            foreach (var node in group)
+            var testfiles = new List<TrackedFile>();
+            var contentIds = new List<ContentId>();
+            var downloadedFiles = new List<TrackedFile?>();
+
+            for (int i = 0; i < numberOfFiles; i++)
             {
-                host.ConnectToPeer(node);
+                testfiles.Add(GenerateTestFile(filesizeMb.MB()));
+                contentIds.Add(new ContentId());
+                downloadedFiles.Add(null);
             }
 
-            var testFile = GenerateTestFile(filesizeMb.MB());
-            var contentId = host.UploadFile(testFile);
-            var list = new List<Task<TrackedFile?>>();
-
-            foreach (var node in group)
+            for (int i = 0; i < numberOfFiles; i++)
             {
-                list.Add(Task.Run(() => { return node.DownloadContent(contentId); }));
+                contentIds[i] = host.UploadFile(testfiles[i]);
             }
 
-            Task.WaitAll(list.ToArray());
-            foreach (var task in list)
+            var downloadTasks = new List<Task>();
+            for (int i = 0; i < numberOfFiles; i++)
             {
-                testFile.AssertIsEqual(task.Result);
+                downloadTasks.Add(Task.Run(() => { downloadedFiles[i] = client.DownloadContent(contentIds[i]); }));
+            }
+
+            Task.WaitAll(downloadTasks.ToArray());
+
+            for (int i = 0; i < numberOfFiles; i++)
+            {
+                testfiles[i].AssertIsEqual(downloadedFiles[i]);
             }
         }
     }

Tests/CodexLongTests/BasicTests/LargeFileTests.cs

@@ -48,7 +48,9 @@ namespace CodexLongTests.BasicTests
 
             var expectedFile = GenerateTestFile(sizeMB);
 
-            var node = AddCodex(s => s.WithStorageQuota((size + 10).MB()));
+            var node = StartCodex(s => s.WithStorageQuota((size + 10).MB()));
+
+            Time.SetRetryFailedCallback(i => OnFailed(i, node));
 
             var uploadStart = DateTime.UtcNow;
             var cid = node.UploadFile(expectedFile);
@@ -60,6 +62,17 @@ namespace CodexLongTests.BasicTests
             AssertTimeConstraint(uploadStart, downloadStart, downloadFinished, size);
         }
 
+        private void OnFailed(int tries, ICodexNode node)
+        {
+            if (tries < 5) return;
+
+            if (tries % 10 == 0)
+            {
+                Log($"After try {tries}, downloading node log.");
+                Ci.DownloadLog(node);
+            }
+        }
+
         private void AssertTimeConstraint(DateTime uploadStart, DateTime downloadStart, DateTime downloadFinished, long size)
         {
             float sizeInMB = size;

Tests/CodexLongTests/BasicTests/TestInfraTests.cs

@@ -6,10 +6,12 @@ namespace CodexLongTests.BasicTests
 {
     public class TestInfraTests : CodexDistTest
     {
-        [Test, UseLongTimeouts]
+        [Test]
+        [UseLongTimeouts]
+        [Ignore("Not supported atm")]
         public void TestInfraShouldHave1000AddressSpacesPerPod()
         {
-            var group = AddCodex(1000, s => s.EnableMetrics());
+            var group = StartCodex(1000, s => s.EnableMetrics());
 
             var nodeIds = group.Select(n => n.GetDebugInfo().Id).ToArray();
 
@@ -17,12 +19,14 @@ namespace CodexLongTests.BasicTests
                 "Not all created nodes provided a unique id.");
         }
 
-        [Test, UseLongTimeouts]
+        [Test]
+        [UseLongTimeouts]
+        [Ignore("Not supported atm")]
         public void TestInfraSupportsManyConcurrentPods()
         {
             for (var i = 0; i < 20; i++)
             {
-                var n = AddCodex();
+                var n = StartCodex();
 
                 Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().Id));
             }

Tests/CodexLongTests/BasicTests/UploadTests.cs

@@ -8,41 +8,39 @@ using Utils;
 namespace CodexLongTests.BasicTests
 {
     [TestFixture]
-    public class UploadTests : CodexDistTest
+    public class UploadTests : AutoBootstrapDistTest
     {
-        [TestCase(3, 50)]
-        [TestCase(5, 75)]
-        [TestCase(10, 25)]
+        [Test]
+        [Combinatorial] 
         [UseLongTimeouts]
-        public void ParallelUpload(int numberOfNodes, int filesizeMb)
+        public void ParallelUpload(
+            [Values(1, 3, 5)] int numberOfFiles,
+            [Values(10, 50, 100)] int filesizeMb)
         {
-            var group = AddCodex(numberOfNodes);
-            var host = AddCodex();
-
-            foreach (var node in group)
-            {
-                host.ConnectToPeer(node);
-            }
+            var host = StartCodex();
+            var client = StartCodex();
 
             var testfiles = new List<TrackedFile>();
-            var contentIds = new List<Task<ContentId>>();
+            var contentIds = new List<ContentId>();
 
-            for (int i = 0; i < group.Count(); i++)
+            for (int i = 0; i < numberOfFiles; i++)
             {
                 testfiles.Add(GenerateTestFile(filesizeMb.MB()));
-                var n = i;
-                contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); }));
+                contentIds.Add(new ContentId());
             }
-            var downloads = new List<Task<TrackedFile?>>();
-            for (int i = 0; i < group.Count(); i++)
+
+            var uploadTasks = new List<Task>();
+            for (int i = 0; i < numberOfFiles; i++)
             {
-                var n = i;
-                downloads.Add(Task.Run(() => { return group[n].DownloadContent(contentIds[n].Result); }));
+                uploadTasks.Add(Task.Run(() => { contentIds[i] = host.UploadFile(testfiles[i]); }));
             }
-            Task.WaitAll(downloads.ToArray());
-            for (int i = 0; i < group.Count(); i++)
+
+            Task.WaitAll(uploadTasks.ToArray());
+
+            for (int i = 0; i < numberOfFiles; i++)
             {
-                testfiles[i].AssertIsEqual(downloads[i].Result);
+                var downloaded = client.DownloadContent(contentIds[i]);
+                testfiles[i].AssertIsEqual(downloaded);
             }
         }
     }

Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -15,7 +15,7 @@ namespace CodexLongTests.DownloadConnectivityTests
             [Values(10, 15, 20)] int numberOfNodes,
             [Values(10, 100)] int sizeMBs)
         {
-            for (var i = 0; i < numberOfNodes; i++) AddCodex();
+            for (var i = 0; i < numberOfNodes; i++) StartCodex();
 
             CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }

Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs

@@ -16,7 +16,7 @@ namespace CodexTests.ScalabilityTests
             [Values(100, 1000)] int fileSize
         )
         {
-            var hosts = AddCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
+            var hosts = StartCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
             var file = GenerateTestFile(fileSize.MB());
             var cid = hosts[0].UploadFile(file);
             var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6);
@@ -38,7 +38,7 @@ namespace CodexTests.ScalabilityTests
 
             foreach (var h in hosts) h.DownloadContent(cid);
 
-            var client = AddCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
+            var client = StartCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
             var resultFile = client.DownloadContent(cid);
             resultFile!.AssertIsEqual(file);
 

Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs

@@ -9,9 +9,6 @@ namespace CodexTests.ScalabilityTests;
 [TestFixture]
 public class ScalabilityTests : CodexDistTest
 {
-    private const string PatchedImage = "codexstorage/nim-codex:sha-9aeac06-dist-tests";
-    private const string MasterImage = "codexstorage/nim-codex:sha-5380912-dist-tests";
-
     /// <summary>
     /// We upload a file to node A, then download it with B.
     /// Then we stop node A, and download again with node C.
@@ -22,16 +19,13 @@ public class ScalabilityTests : CodexDistTest
     [DontDownloadLogs]
     public void ShouldMaintainFileInNetwork(
         [Values(10, 40, 80, 100)] int numberOfNodes,
-        [Values(100, 1000, 5000, 10000)] int fileSizeInMb,
-        [Values(true, false)] bool usePatchedImage
+        [Values(100, 1000, 5000, 10000)] int fileSizeInMb
     )
     {
-        CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage;
-
         var logLevel = CodexLogLevel.Info;
 
-        var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
-        var nodes = AddCodex(numberOfNodes - 1, s => s
+        var bootstrap = StartCodex(s => s.WithLogLevel(logLevel));
+        var nodes = StartCodex(numberOfNodes - 1, s => s
             .WithBootstrapNode(bootstrap)
             .WithLogLevel(logLevel)
             .WithStorageQuota((fileSizeInMb + 50).MB())
@@ -58,23 +52,20 @@ public class ScalabilityTests : CodexDistTest
     /// We upload a file to each node, to put a more wide-spread load on the network.
     /// Then we run the same test as ShouldMaintainFileInNetwork.
     /// </summary>
-    [Ignore("Make ShouldMaintainFileInNetwork pass reliably first.")]
+    [Ignore("Fix ShouldMaintainFileInNetwork for all values first")]
     [Test]
     [Combinatorial]
     [UseLongTimeouts]
     [DontDownloadLogs]
     public void EveryoneGetsAFile(
         [Values(10, 40, 80, 100)] int numberOfNodes,
-        [Values(100, 1000)] int fileSizeInMb,
-        [Values(true, false)] bool usePatchedImage
+        [Values(100, 1000, 5000, 10000)] int fileSizeInMb
     )
     {
-        CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage;
-
         var logLevel = CodexLogLevel.Info;
 
-        var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
-        var nodes = AddCodex(numberOfNodes - 1, s => s
+        var bootstrap = StartCodex(s => s.WithLogLevel(logLevel));
+        var nodes = StartCodex(numberOfNodes - 1, s => s
             .WithBootstrapNode(bootstrap)
             .WithLogLevel(logLevel)
             .WithStorageQuota((fileSizeInMb + 50).MB())

Tests/CodexTests/AutoBootstrapDistTest.cs

@@ -8,7 +8,7 @@ namespace CodexTests
         [SetUp]
         public void SetUpBootstrapNode()
         {
-            BootstrapNode = AddCodex(s => s.WithName("BOOTSTRAP"));
+            BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP"));
         }
 
         [TearDown]

Tests/CodexTests/BasicTests/ContinuousSubstitute.cs

@@ -1,298 +0,0 @@
-using CodexContractsPlugin;
-using CodexPlugin;
-using GethPlugin;
-using KubernetesWorkflow.Types;
-using Logging;
-using MetricsPlugin;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.BasicTests
-{
-    [Ignore("Used for debugging continuous tests")]
-    [TestFixture]
-    public class ContinuousSubstitute : AutoBootstrapDistTest
-    {
-        [Test]
-        public void ContinuousTestSubstitute()
-        {
-            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("geth"));
-            var contract = Ci.StartCodexContracts(geth);
-
-            var group = AddCodex(5, o => o
-                    .EnableMetrics()
-                    .EnableMarketplace(geth, contract, s => s
-                        .WithInitial(10.Eth(), 100000.TestTokens())
-                        .AsStorageNode()
-                        .AsValidator())
-                    .WithBlockTTL(TimeSpan.FromMinutes(5))
-                    .WithBlockMaintenanceInterval(TimeSpan.FromSeconds(10))
-                    .WithBlockMaintenanceNumber(100)
-                    .WithStorageQuota(1.GB()));
-
-            var nodes = group.Cast<CodexNode>().ToArray();
-
-            var rc = Ci.DeployMetricsCollector(nodes);
-
-            var availability = new StorageAvailability(
-                totalSpace: 500.MB(),
-                maxDuration: TimeSpan.FromMinutes(5),
-                minPriceForTotalSpace: 500.TestTokens(),
-                maxCollateral: 1024.TestTokens()
-            );
-
-            foreach (var node in nodes)
-            {
-                node.Marketplace.MakeStorageAvailable(availability);
-            }
-
-            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
-            while (DateTime.UtcNow < endTime)
-            {
-                var allNodes = nodes.ToList();
-                var primary = allNodes.PickOneRandom();
-                var secondary = allNodes.PickOneRandom();
-
-                Log("Run Test");
-                PerformTest(primary, secondary, rc);
-
-                Thread.Sleep(TimeSpan.FromSeconds(5));
-            }
-        }
-
-        private void LogBytesPerMillisecond(Action action)
-        {
-            var sw = Stopwatch.Begin(GetTestLog());
-            action();
-            var duration = sw.End();
-            double totalMs = duration.TotalMilliseconds;
-            double totalBytes = fileSize.SizeInBytes;
-
-            var bytesPerMs = totalBytes / totalMs;
-            Log($"Bytes per millisecond: {bytesPerMs}");
-        }
-
-        [Test]
-        public void PeerTest()
-        {
-            var group = AddCodex(5, o => o
-                    //.EnableMetrics()
-                    //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
-                    .WithBlockTTL(TimeSpan.FromMinutes(2))
-                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
-                    .WithBlockMaintenanceNumber(10000)
-                    .WithBlockTTL(TimeSpan.FromMinutes(2))
-                    .WithStorageQuota(1.GB()));
-
-            var nodes = group.Cast<CodexNode>().ToArray();
-
-            var checkTime = DateTime.UtcNow + TimeSpan.FromMinutes(1);
-            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
-            while (DateTime.UtcNow < endTime)
-            {
-                //CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
-                //CheckRoutingTables(GetAllOnlineCodexNodes());
-
-                var node = nodes.ToList().PickOneRandom();
-                var file = GenerateTestFile(50.MB());
-                node.UploadFile(file);
-
-                Thread.Sleep(20000);
-            }
-        }
-
-        private void CheckRoutingTables(IEnumerable<ICodexNode> nodes)
-        {
-            var all = nodes.ToArray();
-            var allIds = all.Select(n => n.GetDebugInfo().Table.LocalNode.NodeId).ToArray();
-
-            var errors = all.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
-
-            if (errors.Any())
-            {
-                Assert.Fail(string.Join(Environment.NewLine, errors));
-            }
-        }
-
-        private string AreAllPresent(ICodexNode n, string[] allIds)
-        {
-            var info = n.GetDebugInfo();
-            var known = info.Table.Nodes.Select(n => n.NodeId).ToArray();
-            var expected = allIds.Where(i => i != info.Table.LocalNode.NodeId).ToArray();
-
-            if (!expected.All(ex => known.Contains(ex)))
-            {
-                return $"Not all of '{string.Join(",", expected)}' were present in routing table: '{string.Join(",", known)}'";
-            }
-
-            return string.Empty;
-        }
-
-        private ByteSize fileSize = 80.MB();
-
-        private const string BytesStoredMetric = "codexRepostoreBytesUsed";
-
-        private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningPod rc)
-        {
-            ScopedTestFiles(() =>
-            {
-                var testFile = GenerateTestFile(fileSize);
-
-                var metrics = Ci.WrapMetricsCollector(rc, primary);
-                var beforeBytesStored = metrics.GetMetric(BytesStoredMetric);
-
-                ContentId contentId = null!;
-                LogBytesPerMillisecond(() => contentId = primary.UploadFile(testFile));
-
-                var low = fileSize.SizeInBytes;
-                var high = low * 1.2;
-                Log("looking for: " + low + " < " + high);
-
-                Time.WaitUntil(() =>
-                {
-                    var afterBytesStored = metrics.GetMetric(BytesStoredMetric);
-                    var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value);
-
-                    return high > newBytes && newBytes > low;
-                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2), nameof(ContinuousSubstitute));
-
-                FileUtils.TrackedFile? downloadedFile = null;
-                LogBytesPerMillisecond(() => downloadedFile = secondary.DownloadContent(contentId));
-
-                testFile.AssertIsEqual(downloadedFile);
-            });
-        }
-
-        [Test]
-        public void HoldMyBeerTest()
-        {
-            var blockExpirationTime = TimeSpan.FromMinutes(3);
-            var group = AddCodex(3, o => o
-                    .EnableMetrics()
-                    .WithBlockTTL(blockExpirationTime)
-                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
-                    .WithBlockMaintenanceNumber(10000)
-                    .WithStorageQuota(2000.MB()));
-
-            var nodes = group.Cast<CodexNode>().ToArray();
-
-            var endTime = DateTime.UtcNow + TimeSpan.FromHours(24);
-
-            var filesize = 80.MB();
-            double codexDefaultBlockSize = 31 * 64 * 33;
-            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
-            var sizeInBytes = filesize.SizeInBytes;
-            Assert.That(numberOfBlocks, Is.EqualTo(1282));
-
-            var startTime = DateTime.UtcNow;
-            var successfulUploads = 0;
-            var successfulDownloads = 0;
-
-            while (DateTime.UtcNow < endTime)
-            {
-                foreach (var node in nodes)
-                {
-                    try
-                    {
-                        Thread.Sleep(TimeSpan.FromSeconds(5));
-
-                        ScopedTestFiles(() =>
-                        {
-                            var uploadStartTime = DateTime.UtcNow;
-                            var file = GenerateTestFile(filesize);
-                            var cid = node.UploadFile(file);
-
-                            var cidTag = cid.Id.Substring(cid.Id.Length - 6);
-                            Measure("upload-log-asserts", () =>
-                            {
-                                var uploadLog = Ci.DownloadLog(node, tailLines: 50000);
-
-                                var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
-                                uploadLog.DeleteFile();
-
-                                var storeLine = GetLineForCidTag(storeLines, cidTag);
-                                AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
-                            });
-                            successfulUploads++;
-
-                            var uploadTimeTaken = DateTime.UtcNow - uploadStartTime;
-                            if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10)))
-                            {
-                                Assert.Fail("Upload took too long. Blocks already expired.");
-                            }
-
-                            var dl = node.DownloadContent(cid);
-                            file.AssertIsEqual(dl);
-
-                            Measure("download-log-asserts", () =>
-                            {
-                                var downloadLog = Ci.DownloadLog(node, tailLines: 50000);
-
-                                var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
-                                downloadLog.DeleteFile();
-
-                                var sentLine = GetLineForCidTag(sentLines, cidTag);
-                                AssertSentLineContains(sentLine, sizeInBytes);
-                            });
-                            successfulDownloads++;
-                        });
-                    }
-                    catch
-                    {
-                        var testDuration = DateTime.UtcNow - startTime;
-                        Log("Test failed. Delaying shut-down by 30 seconds to collect metrics.");
-                        Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads");
-                        Thread.Sleep(TimeSpan.FromSeconds(30));
-                        throw;
-                    }
-                }
-
-                Thread.Sleep(TimeSpan.FromSeconds(5));
-            }
-        }
-
-        private void AssertSentLineContains(string sentLine, long sizeInBytes)
-        {
-            var tag = "bytes=";
-            var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
-            var bytes = Convert.ToInt64(token);
-            Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
-        }
-
-        private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
-        {
-            var tokens = storeLine.Split(" ");
-
-            var blocksToken = GetToken(tokens, "blocks=");
-            var sizeToken = GetToken(tokens, "size=");
-            if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
-            if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
-
-            var blocks = Convert.ToInt64(blocksToken);
-            var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
-
-            var lineLog = $" Line: '{storeLine}'";
-            Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
-            Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
-        }
-
-        private string GetLineForCidTag(string[] lines, string cidTag)
-        {
-            var result = lines.SingleOrDefault(l => l.Contains(cidTag));
-            if (result == null)
-            {
-                Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
-                throw new Exception();
-            }
-
-            return result;
-        }
-
-        private string? GetToken(string[] tokens, string tag)
-        {
-            var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
-            if (token == null) return null;
-            return token.Substring(tag.Length);
-        }
-    }
-}

Tests/CodexTests/BasicTests/DiscordBotTests.cs

@@ -1,110 +0,0 @@
-using CodexContractsPlugin;
-using CodexDiscordBotPlugin;
-using CodexPlugin;
-using GethPlugin;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.BasicTests
-{
-    [TestFixture]
-    public class DiscordBotTests : AutoBootstrapDistTest
-    {
-        [Test]
-        public void BotRewardTest()
-        {
-            var myAccount = EthAccount.GenerateNew();
-
-            var sellerInitialBalance = 234.TestTokens();
-            var buyerInitialBalance = 100000.TestTokens();
-            var fileSize = 11.MB();
-
-            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
-            var contracts = Ci.StartCodexContracts(geth);
-
-            // start bot and rewarder
-            var gethInfo = new DiscordBotGethInfo(
-                host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host,
-                port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port,
-                privKey: geth.StartResult.Account.PrivateKey,
-                marketplaceAddress: contracts.Deployment.MarketplaceAddress,
-                tokenAddress: contracts.Deployment.TokenAddress,
-                abi: contracts.Deployment.Abi
-            );
-            var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig(
-                name: "bot",
-                token: "aaa",
-                serverName: "ThatBen's server",
-                adminRoleName: "bottest-admins",
-                adminChannelName: "admin-channel",
-                rewardChannelName: "rewards-channel",
-                kubeNamespace: "notneeded",
-                gethInfo: gethInfo
-            ));
-            var botContainer = bot.Containers.Single();
-            Ci.DeployRewarderBot(new RewarderBotStartupConfig(
-                //discordBotHost: "http://" + botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Host,
-                //discordBotPort: botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Port,
-                discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host,
-                discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port,
-                intervalMinutes: "1",
-                historyStartUtc: GetTestRunTimeRange().From - TimeSpan.FromMinutes(3),
-                gethInfo: gethInfo,
-                dataPath: null
-            ));
-
-            var numberOfHosts = 3;
-
-            for (var i = 0; i < numberOfHosts; i++)
-            {
-                var seller = AddCodex(s => s
-                    .WithName("Seller")
-                    .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
-                    {
-                        ContractClock = CodexLogLevel.Trace,
-                    })
-                    .WithStorageQuota(11.GB())
-                    .EnableMarketplace(geth, contracts, m => m
-                        .WithAccount(myAccount)
-                        .WithInitial(10.Eth(), sellerInitialBalance)
-                        .AsStorageNode()
-                        .AsValidator()));
-
-                var availability = new StorageAvailability(
-                    totalSpace: 10.GB(),
-                    maxDuration: TimeSpan.FromMinutes(30),
-                    minPriceForTotalSpace: 1.TestTokens(),
-                    maxCollateral: 20.TestTokens()
-                );
-                seller.Marketplace.MakeStorageAvailable(availability);
-            }
-
-            var testFile = GenerateTestFile(fileSize);
-
-            var buyer = AddCodex(s => s
-                .WithName("Buyer")
-                .EnableMarketplace(geth, contracts, m => m
-                    .WithAccount(myAccount)
-                    .WithInitial(10.Eth(), buyerInitialBalance)));
-
-            var contentId = buyer.UploadFile(testFile);
-
-            var purchase = new StoragePurchaseRequest(contentId)
-            {
-                PricePerSlotPerSecond = 2.TestTokens(),
-                RequiredCollateral = 10.TestTokens(),
-                MinRequiredNumberOfNodes = 5,
-                NodeFailureTolerance = 2,
-                ProofProbability = 5,
-                Duration = TimeSpan.FromMinutes(6),
-                Expiry = TimeSpan.FromMinutes(5)
-            };
-
-            var purchaseContract = buyer.Marketplace.RequestStorage(purchase);
-
-            purchaseContract.WaitForStorageContractStarted();
-
-            purchaseContract.WaitForStorageContractFinished();
-        }
-    }
-}

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -13,7 +13,7 @@ namespace CodexTests.BasicTests
         [Test]
         public void CodexLogExample()
         {
-            var primary = AddCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn)));
+            var primary = StartCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn)));
 
             var cid = primary.UploadFile(GenerateTestFile(5.MB()));
 
@@ -28,8 +28,8 @@ namespace CodexTests.BasicTests
         [Test]
         public void TwoMetricsExample()
         {
-            var group = AddCodex(2, s => s.EnableMetrics());
-            var group2 = AddCodex(2, s => s.EnableMetrics());
+            var group = StartCodex(2, s => s.EnableMetrics());
+            var group2 = StartCodex(2, s => s.EnableMetrics());
 
             var primary = group[0];
             var secondary = group[1];

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -13,15 +13,15 @@ namespace CodexTests.BasicTests
         [Test]
         public void MarketplaceExample()
         {
-            var hostInitialBalance = 234.TestTokens();
-            var clientInitialBalance = 100000.TestTokens();
+            var hostInitialBalance = 234.TstWei();
+            var clientInitialBalance = 100000.TstWei();
             var fileSize = 10.MB();
 
             var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
             var contracts = Ci.StartCodexContracts(geth);
             
             var numberOfHosts = 5;
-            var hosts = AddCodex(numberOfHosts, s => s
+            var hosts = StartCodex(numberOfHosts, s => s
                 .WithName("Host")
                 .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
                 {
@@ -33,7 +33,7 @@ namespace CodexTests.BasicTests
                     .AsStorageNode()
                     .AsValidator()));
 
-            var expectedHostBalance = (numberOfHosts * hostInitialBalance.Amount).TestTokens();
+            var expectedHostBalance = (numberOfHosts * hostInitialBalance.TstWei).TstWei();
             foreach (var host in hosts)
             {
                 AssertBalance(contracts, host, Is.EqualTo(expectedHostBalance));
@@ -41,15 +41,15 @@ namespace CodexTests.BasicTests
                 var availability = new StorageAvailability(
                     totalSpace: 10.GB(),
                     maxDuration: TimeSpan.FromMinutes(30),
-                    minPriceForTotalSpace: 1.TestTokens(),
-                    maxCollateral: 20.TestTokens()
+                    minPriceForTotalSpace: 1.TstWei(),
+                    maxCollateral: 20.TstWei()
                 );
                 host.Marketplace.MakeStorageAvailable(availability);
             }
 
             var testFile = GenerateTestFile(fileSize);
 
-            var client = AddCodex(s => s
+            var client = StartCodex(s => s
                 .WithName("Client")
                 .EnableMarketplace(geth, contracts, m => m
                     .WithInitial(10.Eth(), clientInitialBalance)));
@@ -60,13 +60,13 @@ namespace CodexTests.BasicTests
 
             var purchase = new StoragePurchaseRequest(contentId)
             {
-                PricePerSlotPerSecond = 2.TestTokens(),
-                RequiredCollateral = 10.TestTokens(),
+                PricePerSlotPerSecond = 2.TstWei(),
+                RequiredCollateral = 10.TstWei(),
                 MinRequiredNumberOfNodes = 5,
                 NodeFailureTolerance = 2,
                 ProofProbability = 5,
-                Duration = TimeSpan.FromMinutes(5),
-                Expiry = TimeSpan.FromMinutes(4)
+                Duration = TimeSpan.FromMinutes(6),
+                Expiry = TimeSpan.FromMinutes(5)
             };
 
             var purchaseContract = client.Marketplace.RequestStorage(purchase);
@@ -92,10 +92,10 @@ namespace CodexTests.BasicTests
                 var blockRange = geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange());
                 var slotFilledEvents = contracts.GetSlotFilledEvents(blockRange);
 
-                Log($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}");
+                Debug($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}");
 
                 if (slotFilledEvents.Length != purchase.MinRequiredNumberOfNodes) throw new Exception();
-            }, Convert.ToInt32(purchase.Duration.TotalSeconds / 5) + 10, TimeSpan.FromSeconds(5), "Checking SlotFilled events");
+            }, purchase.Expiry + TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5), "Checking SlotFilled events");
         }
 
         private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer)

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -1,5 +1,4 @@
 using CodexPlugin;
-using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
@@ -11,7 +10,7 @@ namespace CodexTests.BasicTests
         [Test]
         public void OneClientTest()
         {
-            var primary = Ci.StartCodexNode();
+            var primary = StartCodex();
 
             PerformOneClientTest(primary);
         }
@@ -19,11 +18,11 @@ namespace CodexTests.BasicTests
         [Test]
         public void RestartTest()
         {
-            var primary = Ci.StartCodexNode();
+            var primary = StartCodex();
 
             primary.Stop(waitTillStopped: true);
 
-            primary = Ci.StartCodexNode();
+            primary = StartCodex();
 
             PerformOneClientTest(primary);
         }

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -9,8 +9,8 @@ namespace CodexTests.BasicTests
         [Test]
         public void ThreeClient()
         {
-            var primary = AddCodex();
-            var secondary = AddCodex();
+            var primary = StartCodex();
+            var secondary = StartCodex();
 
             var testFile = GenerateTestFile(10.MB());
 

Tests/CodexTests/BasicTests/TwoClientTests.cs

@@ -10,8 +10,8 @@ namespace CodexTests.BasicTests
         [Test]
         public void TwoClientTest()
         {
-            var uploader = AddCodex(s => s.WithName("Uploader"));
-            var downloader = AddCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
+            var uploader = StartCodex(s => s.WithName("Uploader"));
+            var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
 
             PerformTwoClientTest(uploader, downloader);
         }

Tests/CodexTests/CodexDistTest.cs

@@ -39,22 +39,22 @@ namespace CodexTests
             onlineCodexNodes.Remove(lifecycle);
         }
 
-        public ICodexNode AddCodex()
+        public ICodexNode StartCodex()
         {
-            return AddCodex(s => { });
+            return StartCodex(s => { });
         }
 
-        public ICodexNode AddCodex(Action<ICodexSetup> setup)
+        public ICodexNode StartCodex(Action<ICodexSetup> setup)
         {
-            return AddCodex(1, setup)[0];
+            return StartCodex(1, setup)[0];
         }
 
-        public ICodexNodeGroup AddCodex(int numberOfNodes)
+        public ICodexNodeGroup StartCodex(int numberOfNodes)
         {
-            return AddCodex(numberOfNodes, s => { });
+            return StartCodex(numberOfNodes, s => { });
         }
 
-        public ICodexNodeGroup AddCodex(int numberOfNodes, Action<ICodexSetup> setup)
+        public ICodexNodeGroup StartCodex(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var group = Ci.StartCodexNodes(numberOfNodes, s =>
             {

Tests/CodexTests/CodexTests.csproj

@@ -13,11 +13,13 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexDiscordBotPlugin\CodexDiscordBotPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\GethPlugin\GethPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\MetricsPlugin\MetricsPlugin.csproj" />
+    <ProjectReference Include="..\..\Tools\TestNetRewarder\TestNetRewarder.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
   </ItemGroup>
 

Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -11,7 +11,7 @@ namespace CodexTests.DownloadConnectivityTests
         [Test]
         public void MetricsDoesNotInterfereWithPeerDownload()
         {
-            AddCodex(2, s => s.EnableMetrics());
+            StartCodex(2, s => s.EnableMetrics());
 
             AssertAllNodesConnected();
         }
@@ -21,8 +21,8 @@ namespace CodexTests.DownloadConnectivityTests
         {
             var geth = Ci.StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
-            AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
-                .WithInitial(10.Eth(), 1000.TestTokens())));
+            StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
+                .WithInitial(10.Eth(), 1000.TstWei())));
 
             AssertAllNodesConnected();
         }
@@ -33,7 +33,7 @@ namespace CodexTests.DownloadConnectivityTests
             [Values(2, 5)] int numberOfNodes,
             [Values(1, 10)] int sizeMBs)
         {
-            AddCodex(numberOfNodes);
+            StartCodex(numberOfNodes);
 
             AssertAllNodesConnected(sizeMBs);
         }

Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -8,10 +8,10 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void TwoLayersTest()
         {
-            var root = AddCodex();
-            var l1Source = AddCodex(s => s.WithBootstrapNode(root));
-            var l1Node = AddCodex(s => s.WithBootstrapNode(root));
-            var l2Target = AddCodex(s => s.WithBootstrapNode(l1Node));
+            var root = StartCodex();
+            var l1Source = StartCodex(s => s.WithBootstrapNode(root));
+            var l1Node = StartCodex(s => s.WithBootstrapNode(root));
+            var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node));
 
             AssertAllNodesConnected();
         }
@@ -19,11 +19,11 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void ThreeLayersTest()
         {
-            var root = AddCodex();
-            var l1Source = AddCodex(s => s.WithBootstrapNode(root));
-            var l1Node = AddCodex(s => s.WithBootstrapNode(root));
-            var l2Node = AddCodex(s => s.WithBootstrapNode(l1Node));
-            var l3Target = AddCodex(s => s.WithBootstrapNode(l2Node));
+            var root = StartCodex();
+            var l1Source = StartCodex(s => s.WithBootstrapNode(root));
+            var l1Node = StartCodex(s => s.WithBootstrapNode(root));
+            var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node));
+            var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node));
 
             AssertAllNodesConnected();
         }
@@ -33,10 +33,10 @@ namespace CodexTests.PeerDiscoveryTests
         [TestCase(10)]
         public void NodeChainTest(int chainLength)
         {
-            var node = AddCodex();
+            var node = StartCodex();
             for (var i = 1; i < chainLength; i++)
             {
-                node = AddCodex(s => s.WithBootstrapNode(node));
+                node = StartCodex(s => s.WithBootstrapNode(node));
             }
 
             AssertAllNodesConnected();

Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -12,7 +12,7 @@ namespace CodexTests.PeerDiscoveryTests
         public void CanReportUnknownPeerId()
         {
             var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK";
-            var node = AddCodex();
+            var node = StartCodex();
 
             var result = node.GetDebugPeer(unknownId);
             Assert.That(result.IsPeerFound, Is.False);
@@ -21,7 +21,7 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void MetricsDoesNotInterfereWithPeerDiscovery()
         {
-            AddCodex(2, s => s.EnableMetrics());
+            StartCodex(2, s => s.EnableMetrics());
 
             AssertAllNodesConnected();
         }
@@ -31,8 +31,8 @@ namespace CodexTests.PeerDiscoveryTests
         {
             var geth = Ci.StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
-            AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
-                .WithInitial(10.Eth(), 1000.TestTokens())));
+            StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
+                .WithInitial(10.Eth(), 1000.TstWei())));
 
             AssertAllNodesConnected();
         }
@@ -42,7 +42,7 @@ namespace CodexTests.PeerDiscoveryTests
         [TestCase(10)]
         public void VariableNodes(int number)
         {
-            AddCodex(number);
+            StartCodex(number);
 
             AssertAllNodesConnected();
         }

Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs

@@ -1,39 +0,0 @@
-using CodexPlugin;
-using DistTestCore;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.ScalabilityTests
-{
-    [TestFixture]
-    public class OneClientLargeFileTests : CodexDistTest
-    {
-        [Test]
-        [Combinatorial]
-        [UseLongTimeouts]
-        public void OneClientLargeFile([Values(
-            256,
-            512,
-            1024, // GB
-            2048,
-            4096,
-            8192,
-            16384,
-            32768,
-            65536,
-            131072
-        )] int sizeMb)
-        {
-            var testFile = GenerateTestFile(sizeMb.MB());
-
-            var node = AddCodex(s => s
-                .WithLogLevel(CodexLogLevel.Warn)
-                .WithStorageQuota((sizeMb + 10).MB())
-            );
-            var contentId = node.UploadFile(testFile);
-            var downloadedFile = node.DownloadContent(contentId);
-
-            testFile.AssertIsEqual(downloadedFile);
-        }
-    }
-}

Tests/CodexTests/UtilityTests/ClusterSpeedTests.cs

@@ -3,7 +3,7 @@ using Logging;
 using NUnit.Framework;
 using Utils;
 
-namespace CodexTests.ScalabilityTests
+namespace CodexTests.UtilityTests
 {
     [TestFixture]
     public class ClusterDiscSpeedTests : DistTest
@@ -12,12 +12,13 @@ namespace CodexTests.ScalabilityTests
 
         [Test]
         [Combinatorial]
+        [Ignore("Used to measure disc io speeds in cluster.")]
         public void DiscSpeedTest(
             [Values(1, 10, 100, 1024, 1024 * 10, 1024 * 100, 1024 * 1024)] int bufferSizeKb
         )
         {
             long targetSize = (long)(1024 * 1024 * 1024) * 2;
-            long bufferSizeBytes = ((long)bufferSizeKb) * 1024;
+            long bufferSizeBytes = (long)bufferSizeKb * 1024;
 
             var filename = nameof(DiscSpeedTest);
 
@@ -27,7 +28,7 @@ namespace CodexTests.ScalabilityTests
             var writeSpeed = PerformWrite(targetSize, bufferSizeBytes, filename);
             Thread.Sleep(2000);
             var readSpeed = PerformRead(targetSize, bufferSizeBytes, filename);
-            
+
             Log($"Write speed: {writeSpeed} per second.");
             Log($"Read speed: {readSpeed} per second.");
         }

Tests/CodexTests/UtilityTests/DiscordBotTests.cs

@@ -0,0 +1,452 @@
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
+using CodexDiscordBotPlugin;
+using CodexPlugin;
+using Core;
+using DiscordRewards;
+using GethPlugin;
+using KubernetesWorkflow.Types;
+using Logging;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using TestNetRewarder;
+using Utils;
+
+namespace CodexTests.UtilityTests
+{
+    [TestFixture]
+    public class DiscordBotTests : AutoBootstrapDistTest
+    {
+        private readonly RewardRepo repo = new RewardRepo();
+        private readonly TestToken hostInitialBalance = 3000000.TstWei();
+        private readonly TestToken clientInitialBalance = 1000000000.TstWei();
+        private readonly EthAccount clientAccount = EthAccount.GenerateNew();
+        private readonly List<EthAccount> hostAccounts = new List<EthAccount>();
+        private readonly List<ulong> rewardsSeen = new List<ulong>();
+        private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1);
+
+        [Test]
+        public void BotRewardTest()
+        {
+            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
+            var contracts = Ci.StartCodexContracts(geth);
+            var gethInfo = CreateGethInfo(geth, contracts);
+
+            var monitor = new ChainMonitor(contracts, geth, GetTestLog());
+            monitor.Start();
+
+            var botContainer = StartDiscordBot(gethInfo);
+
+            StartHosts(geth, contracts);
+
+            var rewarderContainer = StartRewarderBot(gethInfo, botContainer);
+
+            var client = StartClient(geth, contracts);
+
+            var apiCalls = new RewardApiCalls(Ci, botContainer);
+            apiCalls.Start(OnCommand);
+            var rewarderLog = new RewarderLogMonitor(Ci, rewarderContainer.Containers.Single());
+            rewarderLog.Start(l => Log("Rewarder ChainState: " + l));
+
+            var purchaseContract = ClientPurchasesStorage(client);
+            purchaseContract.WaitForStorageContractFinished();
+
+            rewarderLog.Stop();
+            apiCalls.Stop();
+            monitor.Stop();
+
+            Log("Done!");
+
+            Thread.Sleep(rewarderInterval * 2);
+
+            Log("Seen:");
+            foreach (var seen in rewardsSeen)
+            {
+                Log(seen.ToString());
+            }
+            Log("");
+
+            foreach (var r in repo.Rewards)
+            {
+                var seen = rewardsSeen.Any(s => r.RoleId == s);
+
+                Log($"{r.RoleId} = {seen}");
+            }
+
+            Assert.That(repo.Rewards.All(r => rewardsSeen.Contains(r.RoleId)));
+        }
+
+        private void OnCommand(GiveRewardsCommand call)
+        {
+            if (call.Averages.Any()) Log($"{call.Averages.Length} average.");
+            if (call.EventsOverview.Any()) Log($"{call.EventsOverview.Length} events.");
+            foreach (var r in call.Rewards)
+            {
+                var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId);
+                if (r.UserAddresses.Any()) rewardsSeen.Add(reward.RoleId);
+                foreach (var address in r.UserAddresses)
+                {
+                    var user = IdentifyAccount(address);
+                    Log(user + ": " + reward.Message);
+                }
+            }
+        }
+
+        private IStoragePurchaseContract ClientPurchasesStorage(ICodexNode client)
+        {
+            var testFile = GenerateTestFile(GetMinFileSize());
+            var contentId = client.UploadFile(testFile);
+            var purchase = new StoragePurchaseRequest(contentId)
+            {
+                PricePerSlotPerSecond = 2.TstWei(),
+                RequiredCollateral = 10.TstWei(),
+                MinRequiredNumberOfNodes = GetNumberOfRequiredHosts(),
+                NodeFailureTolerance = 2,
+                ProofProbability = 5,
+                Duration = TimeSpan.FromMinutes(6),
+                Expiry = TimeSpan.FromMinutes(5)
+            };
+
+            return client.Marketplace.RequestStorage(purchase);
+        }
+
+        private ICodexNode StartClient(IGethNode geth, ICodexContracts contracts)
+        {
+            var node = StartCodex(s => s
+                .WithName("Client")
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithAccount(clientAccount)
+                    .WithInitial(10.Eth(), clientInitialBalance)));
+
+            Log($"Client {node.EthAccount.EthAddress}");
+            return node;
+        }
+
+        private RunningPod StartRewarderBot(DiscordBotGethInfo gethInfo, RunningContainer botContainer)
+        {
+            return Ci.DeployRewarderBot(new RewarderBotStartupConfig(
+                name: "rewarder-bot",
+                discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host,
+                discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port,
+                intervalMinutes: Convert.ToInt32(Math.Round(rewarderInterval.TotalMinutes)),
+                historyStartUtc: DateTime.UtcNow,
+                gethInfo: gethInfo,
+                dataPath: null
+            ));
+        }
+
+        private DiscordBotGethInfo CreateGethInfo(IGethNode geth, ICodexContracts contracts)
+        {
+            return new DiscordBotGethInfo(
+                host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host,
+                port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port,
+                privKey: geth.StartResult.Account.PrivateKey,
+                marketplaceAddress: contracts.Deployment.MarketplaceAddress,
+                tokenAddress: contracts.Deployment.TokenAddress,
+                abi: contracts.Deployment.Abi
+            );
+        }
+
+        private RunningContainer StartDiscordBot(DiscordBotGethInfo gethInfo)
+        {
+            var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig(
+                name: "discord-bot",
+                token: "aaa",
+                serverName: "ThatBen's server",
+                adminRoleName: "bottest-admins",
+                adminChannelName: "admin-channel",
+                rewardChannelName: "rewards-channel",
+                kubeNamespace: "notneeded",
+                gethInfo: gethInfo
+            ));
+            return bot.Containers.Single();
+        }
+
+        private void StartHosts(IGethNode geth, ICodexContracts contracts)
+        {
+            var hosts = StartCodex(GetNumberOfLiveHosts(), s => s
+                .WithName("Host")
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
+                {
+                    ContractClock = CodexLogLevel.Trace,
+                })
+                .WithStorageQuota(Mult(GetMinFileSizePlus(50), GetNumberOfLiveHosts()))
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithInitial(10.Eth(), hostInitialBalance)
+                    .AsStorageNode()
+                    .AsValidator()));
+
+            var availability = new StorageAvailability(
+                totalSpace: Mult(GetMinFileSize(), GetNumberOfLiveHosts()),
+                maxDuration: TimeSpan.FromMinutes(30),
+                minPriceForTotalSpace: 1.TstWei(),
+                maxCollateral: hostInitialBalance
+            );
+
+            foreach (var host in hosts)
+            {
+                hostAccounts.Add(host.EthAccount);
+                host.Marketplace.MakeStorageAvailable(availability);
+            }
+        }
+
+        private int GetNumberOfLiveHosts()
+        {
+            return Convert.ToInt32(GetNumberOfRequiredHosts()) + 3;
+        }
+
+        private ByteSize Mult(ByteSize size, int mult)
+        {
+            return new ByteSize(size.SizeInBytes * mult);
+        }
+
+        private ByteSize GetMinFileSizePlus(int plusMb)
+        {
+            return new ByteSize(GetMinFileSize().SizeInBytes + plusMb.MB().SizeInBytes);
+        }
+
+        private ByteSize GetMinFileSize()
+        {
+            ulong minSlotSize = 0;
+            ulong minNumHosts = 0;
+            foreach (var r in repo.Rewards)
+            {
+                var s = Convert.ToUInt64(r.CheckConfig.MinSlotSize.SizeInBytes);
+                var h = r.CheckConfig.MinNumberOfHosts;
+                if (s > minSlotSize) minSlotSize = s;
+                if (h > minNumHosts) minNumHosts = h;
+            }
+
+            var minFileSize = ((minSlotSize + 1024) * minNumHosts);
+            return new ByteSize(Convert.ToInt64(minFileSize));
+        }
+
+        private uint GetNumberOfRequiredHosts()
+        {
+            return Convert.ToUInt32(repo.Rewards.Max(r => r.CheckConfig.MinNumberOfHosts));
+        }
+
+        private string IdentifyAccount(string address)
+        {
+            if (address == clientAccount.EthAddress.Address) return "Client";
+            try
+            {
+                var index = hostAccounts.FindIndex(a => a.EthAddress.Address == address);
+                return "Host" + index;
+            }
+            catch
+            {
+                return "UNKNOWN";
+            }
+        }
+
+        public class RewardApiCalls
+        {
+            private readonly ContainerFileMonitor monitor;
+            private readonly Dictionary<string, GiveRewardsCommand> commands = new Dictionary<string, GiveRewardsCommand>();
+
+            public RewardApiCalls(CoreInterface ci, RunningContainer botContainer)
+            {
+                monitor = new ContainerFileMonitor(ci, botContainer, "/app/datapath/logs/discordbot.log");
+            }
+
+            public void Start(Action<GiveRewardsCommand> onCommand)
+            {
+                monitor.Start(line => ParseLine(line, onCommand));
+            }
+
+            public void Stop()
+            {
+                monitor.Stop();
+            }
+
+            private void ParseLine(string line, Action<GiveRewardsCommand> onCommand)
+            {
+                try
+                {
+                    var timestamp = line.Substring(0, 30);
+                    if (commands.ContainsKey(timestamp)) return;
+                    var json = line.Substring(31);
+
+                    var cmd = JsonConvert.DeserializeObject<GiveRewardsCommand>(json);
+                    if (cmd != null)
+                    {
+                        commands.Add(timestamp, cmd);
+                        onCommand(cmd);
+                    }
+                }
+                catch
+                {
+                }
+            }
+        }
+
+        public class RewarderLogMonitor
+        {
+            private readonly ContainerFileMonitor monitor;
+            private readonly Dictionary<string, GiveRewardsCommand> commands = new Dictionary<string, GiveRewardsCommand>();
+
+            public RewarderLogMonitor(CoreInterface ci, RunningContainer botContainer)
+            {
+                monitor = new ContainerFileMonitor(ci, botContainer, "/app/datapath/logs/testnetrewarder.log");
+            }
+
+            public void Start(Action<string> onCommand)
+            {
+                monitor.Start(l => ProcessLine(l, onCommand));
+            }
+
+            public void Stop()
+            {
+                monitor.Stop();
+            }
+
+            private void ProcessLine(string line, Action<string> log)
+            {
+                //$"ChainState=[{JsonConvert.SerializeObject(this)}]" +
+                //$"HistoricState=[{historicState.EntireString()}]";
+
+                var stateOpenTag = "ChainState=[";
+                var historicOpenTag = "]HistoricState=[";
+
+                if (!line.Contains(stateOpenTag)) return;
+                if (!line.Contains(historicOpenTag)) return;
+
+                var stateStr = Between(line, stateOpenTag, historicOpenTag);
+                var historicStr = Between(line, historicOpenTag, "]");
+
+                var chainState = JsonConvert.DeserializeObject<ChainState>(stateStr);
+                var historicState = JsonConvert.DeserializeObject<TestNetRewarder.StorageRequest[]>(historicStr)!;
+                chainState!.Set(new HistoricState(historicState));
+
+                log(string.Join(",", chainState!.GenerateOverview()));
+            }
+
+            private string Between(string s, string open, string close)
+            {
+                var start = s.IndexOf(open) + open.Length;
+                var end = s.LastIndexOf(close);
+                return s.Substring(start, end - start);
+            }
+        }
+
+        public class ContainerFileMonitor
+        {
+            private readonly CoreInterface ci;
+            private readonly RunningContainer botContainer;
+            private readonly string filePath;
+            private readonly CancellationTokenSource cts = new CancellationTokenSource();
+            private readonly List<string> seenLines = new List<string>();
+            private Task worker = Task.CompletedTask;
+            private Action<string> onNewLine = c => { };
+
+            public ContainerFileMonitor(CoreInterface ci, RunningContainer botContainer, string filePath)
+            {
+                this.ci = ci;
+                this.botContainer = botContainer;
+                this.filePath = filePath;
+            }
+
+            public void Start(Action<string> onNewLine)
+            {
+                this.onNewLine = onNewLine;
+                worker = Task.Run(Worker);
+            }
+
+            public void Stop()
+            {
+                cts.Cancel();
+                worker.Wait();
+            }
+
+            private void Worker()
+            {
+                while (!cts.IsCancellationRequested)
+                {
+                    Update();
+                }
+            }
+
+            private void Update()
+            {
+                Thread.Sleep(TimeSpan.FromSeconds(10));
+                if (cts.IsCancellationRequested) return;
+
+                var botLog = ci.ExecuteContainerCommand(botContainer, "cat", filePath);
+                var lines = botLog.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
+                foreach (var line in lines)
+                {
+                    if (!seenLines.Contains(line))
+                    {
+                        seenLines.Add(line);
+                        onNewLine(line);
+                    }
+                }
+            }
+        }
+
+        public class ChainMonitor
+        {
+            private readonly ICodexContracts contracts;
+            private readonly IGethNode geth;
+            private readonly ILog log;
+            private readonly CancellationTokenSource cts = new CancellationTokenSource();
+            private Task worker = Task.CompletedTask;
+            private DateTime last = DateTime.UtcNow;
+
+            public ChainMonitor(ICodexContracts contracts, IGethNode geth, ILog log)
+            {
+                this.contracts = contracts;
+                this.geth = geth;
+                this.log = log;
+            }
+
+            public void Start()
+            {
+                last = DateTime.UtcNow;
+                worker = Task.Run(Worker);
+            }
+
+            public void Stop()
+            {
+                cts.Cancel();
+                worker.Wait();
+            }
+
+            private void Worker()
+            {
+                while (!cts.IsCancellationRequested)
+                {
+                    Thread.Sleep(TimeSpan.FromSeconds(10));
+                    if (cts.IsCancellationRequested) return;
+
+                    Update();
+
+                }
+            }
+
+            private void Update()
+            {
+                var start = last;
+                var stop = DateTime.UtcNow;
+                last = stop;
+
+                var range = geth.ConvertTimeRangeToBlockRange(new TimeRange(start, stop));
+
+
+                LogEvents(nameof(contracts.GetStorageRequests), contracts.GetStorageRequests, range);
+                LogEvents(nameof(contracts.GetRequestFulfilledEvents), contracts.GetRequestFulfilledEvents, range);
+                LogEvents(nameof(contracts.GetRequestCancelledEvents), contracts.GetRequestCancelledEvents, range);
+                LogEvents(nameof(contracts.GetSlotFilledEvents), contracts.GetSlotFilledEvents, range);
+                LogEvents(nameof(contracts.GetSlotFreedEvents), contracts.GetSlotFreedEvents, range);
+            }
+
+            private void LogEvents(string n, Func<BlockInterval, object> f, BlockInterval r)
+            {
+                var a = (object[])f(r);
+
+                a.ToList().ForEach(request => log.Log(n + " - " + JsonConvert.SerializeObject(request)));
+            }
+        }
+    }
+}

Tests/CodexTests/UtilityTests/LogHelperTests.cs

@@ -0,0 +1,56 @@
+using CodexPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.UtilityTests
+{
+    [TestFixture]
+    public class LogHelperTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [Ignore("Used to find the most common log messages.")]
+        public void FindMostCommonLogMessages()
+        {
+            var uploader = StartCodex(s => s.WithName("uploader").WithLogLevel(CodexLogLevel.Trace));
+            var downloader = StartCodex(s => s.WithName("downloader").WithLogLevel(CodexLogLevel.Trace));
+
+            var cid = uploader.UploadFile(GenerateTestFile(100.MB()));
+
+            Thread.Sleep(1000);
+            var logStartUtc = DateTime.UtcNow;
+            Thread.Sleep(1000);
+
+            downloader.DownloadContent(cid);
+
+            var map = GetLogMap(downloader, logStartUtc).OrderByDescending(p => p.Value);
+            Log("Downloader - Receive");
+            foreach (var entry in map)
+            {
+                if (entry.Value > 9)
+                {
+                    Log($"'{entry.Key}' = {entry.Value}");
+                }
+            }
+        }
+
+        private Dictionary<string, int> GetLogMap(ICodexNode node, DateTime? startUtc = null)
+        {
+            var log = Ci.DownloadLog(node);
+            var map = new Dictionary<string, int>();
+            log.IterateLines(line =>
+            {
+                var log = CodexLogLine.Parse(line);
+                if (log == null) return;
+
+                if (startUtc.HasValue)
+                {
+                    if (log.TimestampUtc < startUtc) return;
+                }
+
+                if (map.ContainsKey(log.Message)) map[log.Message] += 1;
+                else map.Add(log.Message, 1);
+            });
+            return map;
+        }
+    }
+}

Tests/CodexTests/UtilityTests/NetworkIsolationTest.cs

@@ -3,7 +3,7 @@ using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
-namespace CodexTests.BasicTests
+namespace CodexTests.UtilityTests
 {
     // Warning!
     // This is a test to check network-isolation in the test-infrastructure.

Tests/FrameworkTests/CodexContractsPlugin/TestTokenTests.cs

@@ -0,0 +1,46 @@
+using CodexContractsPlugin;
+using NUnit.Framework;
+using System.Numerics;
+
+namespace FrameworkTests.CodexContractsPlugin
+{
+    [TestFixture]
+    public class TestTokenTests
+    {
+        private const decimal factor = 1000000000000000000m;
+
+        [Test]
+        public void RepresentsSmallAmount()
+        {
+            var t = 10.TstWei();
+
+            Assert.That(t.TstWei, Is.EqualTo(new BigInteger(10)));
+            Assert.That(t.Tst, Is.EqualTo(new BigInteger(0)));
+            Assert.That(t.ToString(), Is.EqualTo("10 TSTWEI"));
+        }
+
+        [Test]
+        public void RepresentsLargeAmount()
+        {
+            var t = 10.Tst();
+
+            var expected = new BigInteger(10 * factor);
+            Assert.That(t.TstWei, Is.EqualTo(expected));
+            Assert.That(t.Tst, Is.EqualTo(new BigInteger(10)));
+            Assert.That(t.ToString(), Is.EqualTo("10 TST"));
+        }
+
+        [Test]
+        public void RepresentsLongAmount()
+        {
+            var a = 10.Tst();
+            var b = 20.TstWei();
+            var t = a + b;
+
+            var expected = new BigInteger((10 * factor) + 20);
+            Assert.That(t.TstWei, Is.EqualTo(expected));
+            Assert.That(t.Tst, Is.EqualTo(new BigInteger(10)));
+            Assert.That(t.ToString(), Is.EqualTo("10 TST + 20 TSTWEI"));
+        }
+    }
+}

Tests/FrameworkTests/FrameworkTests.csproj

@@ -16,6 +16,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\NethereumWorkflow\NethereumWorkflow.csproj" />
     <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
   </ItemGroup>
 
 </Project>

Tools/BiblioTech/AdminChecker.cs

@@ -24,7 +24,7 @@ namespace BiblioTech
 
         public bool IsAdminChannel(IChannel channel)
         {
-            return channel.Name == Program.Config.AdminChannelName;
+            return channel.Id == Program.Config.AdminChannelId;
         }
 
         public ISocketMessageChannel GetAdminChannel()
@@ -45,7 +45,7 @@ namespace BiblioTech
         private void UpdateAdminIds()
         {
             lastUpdate = DateTime.UtcNow;
-            var adminRole = guild.Roles.Single(r => r.Name == Program.Config.AdminRoleName);
+            var adminRole = guild.Roles.Single(r => r.Id == Program.Config.AdminRoleId);
             adminIds = adminRole.Members.Select(m => m.Id).ToArray();
         }
     }

Tools/BiblioTech/CommandHandler.cs

@@ -22,7 +22,7 @@ namespace BiblioTech
 
         private async Task Client_Ready()
         {
-            var guild = client.Guilds.Single(g => g.Name == Program.Config.ServerName);
+            var guild = client.Guilds.Single(g => g.Id == Program.Config.ServerId);
             Program.AdminChecker.SetGuild(guild);
             Program.Log.Log($"Initializing for guild: '{guild.Name}'");
 

Tools/BiblioTech/Commands/GetBalanceCommand.cs

@@ -32,7 +32,7 @@ namespace BiblioTech.Commands
             }
 
             var eth = 0.Eth();
-            var testTokens = 0.TestTokens();
+            var testTokens = 0.TstWei();
 
             await Task.Run(() =>
             {

Tools/BiblioTech/Commands/MintCommand.cs

@@ -51,7 +51,7 @@ namespace BiblioTech.Commands
         {
             if (ShouldMintTestTokens(contracts, addr))
             {
-                var tokens = Program.Config.MintTT.TestTokens();
+                var tokens = Program.Config.MintTT.TstWei();
                 var transaction = contracts.MintTestTokens(addr, tokens);
                 report.Add($"Minted {tokens} {FormatTransactionLink(transaction)}");
                 return new Transaction<TestToken>(tokens, transaction);
@@ -77,7 +77,7 @@ namespace BiblioTech.Commands
         private bool ShouldMintTestTokens(ICodexContracts contracts, EthAddress addr)
         {
             var testTokens = contracts.GetTestTokenBalance(addr);
-            return testTokens.Amount < Program.Config.MintTT;
+            return testTokens < Program.Config.MintTT.TstWei();
         }
 
         private bool ShouldSendEth(IGethNode gethNode, EthAddress addr)

Tools/BiblioTech/Configuration.cs

@@ -1,4 +1,5 @@
 using ArgsUniform;
+using System.Numerics;
 
 namespace BiblioTech
 {
@@ -7,55 +8,39 @@ namespace BiblioTech
         [Uniform("token", "t", "TOKEN", true, "Discord Application Token")]
         public string ApplicationToken { get; set; } = string.Empty;
 
-        [Uniform("server-name", "sn", "SERVERNAME", true, "Name of the Discord server")]
-        public string ServerName { get; set; } = string.Empty;
+        [Uniform("server-id", "sn", "SERVERID", true, "ID of the Discord server")] 
+        public ulong ServerId { get; set; }
 
-        [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
+        [Uniform("datapath", "dp", "DATAPATH", true, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
         
-        [Uniform("admin-role", "a", "ADMINROLE", true, "Name of the Discord server admin role")]
-        public string AdminRoleName { get; set; } = string.Empty;
+        [Uniform("admin-role-id", "a", "ADMINROLEID", true, "ID of the Discord server admin role")]
+        public ulong AdminRoleId { get; set; }
 
-        [Uniform("admin-channel-name", "ac", "ADMINCHANNELNAME", true, "Name of the Discord server channel where admin commands are allowed.")]
-        public string AdminChannelName { get; set; } = "admin-channel";
+        [Uniform("admin-channel-id", "ac", "ADMINCHANNELID", true, "ID of the Discord server channel where admin commands are allowed.")]
+        public ulong AdminChannelId{ get; set; }
 
-        [Uniform("rewards-channel-name", "rc", "REWARDSCHANNELNAME", false, "Name of the Discord server channel where participation rewards will be announced.")]
-        public string RewardsChannelName { get; set; } = "";
+        [Uniform("rewards-channel-id", "rc", "REWARDSCHANNELID", false, "ID of the Discord server channel where participation rewards will be announced.")]
+        public ulong RewardsChannelId { get; set; }
 
-        [Uniform("chain-events-channel-name", "cc", "CHAINEVENTSCHANNELNAME", false, "Name of the Discord server channel where chain events will be posted.")]
-        public string ChainEventsChannelName { get; set; } = "";
+        [Uniform("chain-events-channel-id", "cc", "CHAINEVENTSCHANNELID", false, "ID of the Discord server channel where chain events will be posted.")]
+        public ulong ChainEventsChannelId { get; set; }
 
-        [Uniform("reward-api-port", "rp", "REWARDAPIPORT", false, "TCP listen port for the reward API.")]
+        [Uniform("reward-api-port", "rp", "REWARDAPIPORT", true, "TCP listen port for the reward API.")]
         public int RewardApiPort { get; set; } = 31080;
 
-        [Uniform("send-eth", "se", "SENDETH", false, "Amount of Eth send by the mint command. Default: 10.")]
+        [Uniform("send-eth", "se", "SENDETH", true, "Amount of Eth send by the mint command.")]
         public int SendEth { get; set; } = 10;
 
-        [Uniform("mint-tt", "mt", "MINTTT", false, "Amount of TestTokens minted by the mint command. Default: 1073741824")]
-        public int MintTT { get; set; } = 1073741824;
+        [Uniform("mint-tt", "mt", "MINTTT", true, "Amount of TSTWEI minted by the mint command.")]
+        public BigInteger MintTT { get; set; } = 1073741824;
 
-        public string EndpointsPath
-        {
-            get
-            {
-                return Path.Combine(DataPath, "endpoints");
-            }
-        }
+        [Uniform("no-discord", "nd", "NODISCORD", false, "For debugging: Bypasses all Discord API calls.")]
+        public int NoDiscord { get; set; } = 0;
 
-        public string UserDataPath
-        {
-            get
-            {
-                return Path.Combine(DataPath, "users");
-            }
-        }
-
-        public string LogPath
-        {
-            get
-            {
-                return Path.Combine(DataPath, "logs");
-            }
-        }
+        public string EndpointsPath => Path.Combine(DataPath, "endpoints");
+        public string UserDataPath => Path.Combine(DataPath, "users");
+        public string LogPath => Path.Combine(DataPath, "logs");
+        public bool DebugNoDiscord => NoDiscord == 1;
     }
 }

Tools/BiblioTech/LoggingRoleDriver.cs

@@ -0,0 +1,24 @@
+using BiblioTech.Rewards;
+using DiscordRewards;
+using Logging;
+using Newtonsoft.Json;
+
+namespace BiblioTech
+{
+    public class LoggingRoleDriver : IDiscordRoleDriver
+    {
+        private readonly ILog log;
+
+        public LoggingRoleDriver(ILog log)
+        {
+            this.log = log;
+        }
+
+        public async Task GiveRewards(GiveRewardsCommand rewards)
+        {
+            await Task.CompletedTask;
+
+            log.Log(JsonConvert.SerializeObject(rewards, Formatting.None));
+        }
+    }
+}

Tools/BiblioTech/Program.cs

@@ -21,6 +21,8 @@ namespace BiblioTech
 
         public static Task Main(string[] args)
         {
+            Log = new ConsoleLog();
+
             var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
             Config = uniformArgs.Parse();
 
@@ -39,25 +41,15 @@ namespace BiblioTech
         public async Task MainAsync(string[] args)
         {
             Log.Log("Starting Codex Discord Bot...");
-            client = new DiscordSocketClient();
-            client.Log += ClientLog;
-
-            var notifyCommand = new NotifyCommand();
-            var associateCommand = new UserAssociateCommand(notifyCommand);
-            var sprCommand = new SprCommand();
-            var handler = new CommandHandler(client,
-                new GetBalanceCommand(associateCommand), 
-                new MintCommand(associateCommand),
-                sprCommand,
-                associateCommand,
-                notifyCommand,
-                new AdminCommand(sprCommand),
-                new MarketCommand()
-            );
-
-            await client.LoginAsync(TokenType.Bot, Config.ApplicationToken);
-            await client.StartAsync();
-            AdminChecker = new AdminChecker();
+            if (Config.DebugNoDiscord)
+            {
+                Log.Log("Debug option is set. Discord connection disabled!");
+                RoleDriver = new LoggingRoleDriver(Log);
+            }
+            else
+            {
+                await StartDiscordBot();
+            }
 
             var builder = WebApplication.CreateBuilder(args);
             builder.WebHost.ConfigureKestrel((context, options) =>
@@ -73,6 +65,29 @@ namespace BiblioTech
             await Task.Delay(-1);
         }
 
+        private async Task StartDiscordBot()
+        {
+            client = new DiscordSocketClient();
+            client.Log += ClientLog;
+
+            var notifyCommand = new NotifyCommand();
+            var associateCommand = new UserAssociateCommand(notifyCommand);
+            var sprCommand = new SprCommand();
+            var handler = new CommandHandler(client,
+                new GetBalanceCommand(associateCommand),
+                new MintCommand(associateCommand),
+                sprCommand,
+                associateCommand,
+                notifyCommand,
+                new AdminCommand(sprCommand),
+                new MarketCommand()
+            );
+
+            await client.LoginAsync(TokenType.Bot, Config.ApplicationToken);
+            await client.StartAsync();
+            AdminChecker = new AdminChecker();
+        }
+
         private static void PrintHelp()
         {
             Log.Log("BiblioTech - Codex Discord Bot");

Tools/BiblioTech/Rewards/RoleDriver.cs

@@ -16,8 +16,8 @@ namespace BiblioTech.Rewards
         {
             this.client = client;
 
-            rewardsChannel = GetChannel(Program.Config.RewardsChannelName);
-            eventsChannel = GetChannel(Program.Config.ChainEventsChannelName);
+            rewardsChannel = GetChannel(Program.Config.RewardsChannelId);
+            eventsChannel = GetChannel(Program.Config.ChainEventsChannelId);
         }
 
         public async Task GiveRewards(GiveRewardsCommand rewards)
@@ -45,10 +45,10 @@ namespace BiblioTech.Rewards
             await context.ProcessGiveRewardsCommand(LookUpUsers(rewards));
         }
 
-        private SocketTextChannel? GetChannel(string name)
+        private SocketTextChannel? GetChannel(ulong id)
         {
-            if (string.IsNullOrEmpty(name)) return null;
-            return GetGuild().TextChannels.SingleOrDefault(c => c.Name == name);
+            if (id == 0) return null;
+            return GetGuild().TextChannels.SingleOrDefault(c => c.Id == id);
         }
 
         private async Task ProcessChainEvents(string[] eventsOverview)
@@ -147,11 +147,11 @@ namespace BiblioTech.Rewards
 
         private SocketGuild GetGuild()
         {
-            var guild = client.Guilds.SingleOrDefault(g => g.Name == Program.Config.ServerName);
+            var guild = client.Guilds.SingleOrDefault(g => g.Id == Program.Config.ServerId);
             if (guild == null)
             {
-                throw new Exception($"Unable to find guild by name: '{Program.Config.ServerName}'. " +
-                    $"Known guilds: [{string.Join(",", client.Guilds.Select(g => g.Name))}]");
+                throw new Exception($"Unable to find guild by id: '{Program.Config.ServerId}'. " +
+                    $"Known guilds: [{string.Join(",", client.Guilds.Select(g => g.Name + " (" + g.Id + ")"))}]");
             }
             return guild;
         }

Tools/CodexNetDeployer/CodexNodeStarter.cs

@@ -43,7 +43,7 @@ namespace CodexNetDeployer
                     {
                         s.EnableMarketplace(gethNode, contracts, m =>
                         {
-                            m.WithInitial(100.Eth(), config.InitialTestTokens.TestTokens());
+                            m.WithInitial(100.Eth(), config.InitialTestTokens.TstWei());
                             if (validatorsLeft > 0) m.AsValidator();
                             if (config.ShouldMakeStorageAvailable) m.AsStorageNode();
                         });
@@ -71,8 +71,8 @@ namespace CodexNetDeployer
                         var availability = new StorageAvailability(
                             totalSpace: config.StorageSell!.Value.MB(),
                             maxDuration: TimeSpan.FromSeconds(config.MaxDuration),
-                            minPriceForTotalSpace: config.MinPrice.TestTokens(),
-                            maxCollateral: config.MaxCollateral.TestTokens()
+                            minPriceForTotalSpace: config.MinPrice.TstWei(),
+                            maxCollateral: config.MaxCollateral.TstWei()
                         );
 
                         var response = codexNode.Marketplace.MakeStorageAvailable(availability);

Tools/CodexNetDeployer/Deployer.cs

@@ -255,9 +255,9 @@ namespace CodexNetDeployer
             return TimeSpan.FromSeconds(2);
         }
 
-        public int HttpMaxNumberOfRetries()
+        public TimeSpan HttpRetryTimeout()
         {
-            return 2;
+            return TimeSpan.FromSeconds(30);
         }
 
         public TimeSpan HttpCallTimeout()

Tools/TestNetRewarder/ChainState.cs

@@ -8,7 +8,7 @@ namespace TestNetRewarder
 {
     public class ChainState
     {
-        private readonly HistoricState historicState;
+        private HistoricState historicState;
         private readonly string[] colorIcons = new[]
         {
             "🔴",
@@ -50,15 +50,48 @@ namespace TestNetRewarder
             SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange);
         }
 
+        public ChainState(
+            Request[] newRequests,
+            RequestFulfilledEventDTO[] requestFulfilledEvents,
+            RequestCancelledEventDTO[] requestCancelledEvents,
+             SlotFilledEventDTO[] slotFilledEvents,
+             SlotFreedEventDTO[] slotFreedEvents)
+        {
+            NewRequests = newRequests;
+            RequestFulfilledEvents = requestFulfilledEvents;
+            RequestCancelledEvents = requestCancelledEvents;
+            SlotFilledEvents = slotFilledEvents;
+            SlotFreedEvents = slotFreedEvents;
+
+            historicState = new HistoricState();
+            StartedRequests = Array.Empty<StorageRequest>();
+            FinishedRequests = Array.Empty<StorageRequest>();
+        }
+
         public Request[] NewRequests { get; }
+        [JsonIgnore]
         public StorageRequest[] AllRequests => historicState.StorageRequests;
+        [JsonIgnore]
         public StorageRequest[] StartedRequests { get; private set; }
+        [JsonIgnore]
         public StorageRequest[] FinishedRequests { get; private set; }
         public RequestFulfilledEventDTO[] RequestFulfilledEvents { get; }
         public RequestCancelledEventDTO[] RequestCancelledEvents { get; }
         public SlotFilledEventDTO[] SlotFilledEvents { get; }
         public SlotFreedEventDTO[] SlotFreedEvents { get; }
 
+        public string EntireString()
+        {
+            return
+                $"ChainState=[{JsonConvert.SerializeObject(this)}]" +
+                $"HistoricState=[{historicState.EntireString()}]";
+        }
+
+        public void Set(HistoricState h)
+        {
+            historicState = h;
+        }
+
         public string[] GenerateOverview()
         {
             var entries = new List<StringBlockNumberPair>();

Tools/TestNetRewarder/Configuration.cs

@@ -4,16 +4,16 @@ namespace TestNetRewarder
 {
     public class Configuration
     {
-        [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
+        [Uniform("datapath", "dp", "DATAPATH", true, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
 
         [Uniform("discordbot-host", "dh", "DISCORDBOTHOST", true, "http address of the discord bot.")]
         public string DiscordHost { get; set; } = "host";
 
-        [Uniform("discordbot-port", "dp", "DISCORDBOTPORT", true, "port number of the discord bot reward API. (31080 by default)")]
+        [Uniform("discordbot-port", "dp", "DISCORDBOTPORT", true, "port number of the discord bot reward API.")]
         public int DiscordPort { get; set; } = 31080;
 
-        [Uniform("interval-minutes", "im", "INTERVALMINUTES", false, "time in minutes between reward updates. (default 15)")]
+        [Uniform("interval-minutes", "im", "INTERVALMINUTES", true, "time in minutes between reward updates.")]
         public int IntervalMinutes { get; set; } = 15;
 
         [Uniform("check-history", "ch", "CHECKHISTORY", true, "Unix epoc timestamp of a moment in history on which processing begins. Required for hosting rewards. Should be 'launch of the testnet'.")]
@@ -22,7 +22,7 @@ namespace TestNetRewarder
         [Uniform("market-insights", "mi", "MARKETINSIGHTS", false, "Semi-colon separated integers. Each represents a multiple of intervals, for which a market insights average will be generated.")]
         public string MarketInsights { get; set; } = "1;96";
 
-        [Uniform("events-overview", "eo", "EVENTSOVERVIEW", false, "When greater than zero, chain event summary will be generated. (default 1)")]
+        [Uniform("events-overview", "eo", "EVENTSOVERVIEW", false, "When greater than zero, chain event summary will be generated.")]
         public int CreateChainEventsOverview { get; set; } = 1;
 
         public string LogPath

Tools/TestNetRewarder/HistoricState.cs

@@ -29,6 +29,20 @@ namespace TestNetRewarder
                 r.State == RequestState.Failed
             );
         }
+
+        public string EntireString()
+        {
+            return JsonConvert.SerializeObject(StorageRequests);
+        }
+
+        public HistoricState()
+        {
+        }
+
+        public HistoricState(StorageRequest[] requests)
+        {
+            storageRequests.AddRange(requests);
+        }
     }
 
     public class StorageRequest

Tools/TestNetRewarder/Processor.cs

@@ -58,6 +58,8 @@ namespace TestNetRewarder
 
         private async Task ProcessChainState(ChainState chainState)
         {
+            log.Log(chainState.EntireString());
+
             var outgoingRewards = new List<RewardUsersCommand>();
             foreach (var reward in rewardRepo.Rewards)
             {