Removes rest server

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj

@@ -6,6 +6,11 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Nethereum.Generators" Version="4.21.4" />
+    <PackageReference Include="Nethereum.Generators.Net" Version="4.21.4" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
     <ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -1,4 +1,5 @@
-using Core;
+using CodexContractsPlugin.Marketplace;
+using Core;
 using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
@@ -64,7 +65,8 @@ namespace CodexContractsPlugin
 
             var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
             var marketplaceAddress = extractor.ExtractMarketplaceAddress();
-            var abi = extractor.ExtractMarketplaceAbi();
+            var (abi, bytecode) = extractor.ExtractMarketplaceAbiAndByteCode();
+            EnsureCompatbility(abi, bytecode);
 
             var interaction = new ContractInteractions(tools.GetLog(), gethNode);
             var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
@@ -78,6 +80,18 @@ namespace CodexContractsPlugin
             return new CodexContractsDeployment(marketplaceAddress, abi, tokenAddress);
         }
 
+        private void EnsureCompatbility(string abi, string bytecode)
+        {
+            var expectedByteCode = MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant();
+
+            if (bytecode != expectedByteCode)
+            {
+                Log("Deployed contract is incompatible with current build of CodexContracts plugin. Running self-updater...");
+                var selfUpdater = new SelfUpdater();
+                selfUpdater.Update(abi, bytecode);
+            }
+        }
+
         private void Log(string msg)
         {
             tools.GetLog().Log(msg);

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -31,14 +31,14 @@ namespace CodexContractsPlugin
             return marketplaceAddress;
         }
 
-        public string ExtractMarketplaceAbi()
+        public (string, string) ExtractMarketplaceAbiAndByteCode()
         {
             log.Debug();
-            var marketplaceAbi = Retry(FetchMarketplaceAbi);
-            if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
+            var (abi, bytecode) = Retry(FetchMarketplaceAbiAndByteCode);
+            if (string.IsNullOrEmpty(abi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
 
-            log.Debug("Got Marketplace ABI: " + marketplaceAbi);
-            return marketplaceAbi;
+            log.Debug("Got Marketplace ABI: " + abi);
+            return (abi, bytecode);
         }
 
         private string FetchMarketplaceAddress()
@@ -48,7 +48,7 @@ namespace CodexContractsPlugin
             return marketplace!.address;
         }
 
-        private string FetchMarketplaceAbi()
+        private (string, string) FetchMarketplaceAbiAndByteCode()
         {
             var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceArtifactFilename);
 
@@ -56,19 +56,12 @@ namespace CodexContractsPlugin
             var abi = artifact["abi"];
             var byteCode = artifact["bytecode"];
             var abiResult = abi!.ToString(Formatting.None);
-            var byteCodeResult = byteCode!.ToString(Formatting.None);
-
-            if (byteCodeResult
-                .ToLowerInvariant()
-                .Replace("\"", "") != MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant())
-            {
-                throw new Exception("BYTECODE in CodexContractsPlugin does not match BYTECODE deployed by container. Update Marketplace.cs generated code?");
-            }
-
-            return abiResult;
+            var byteCodeResult = byteCode!.ToString(Formatting.None).ToLowerInvariant().Replace("\"", "");
+          
+            return (abiResult, byteCodeResult);
         }
 
-        private static string Retry(Func<string> fetch)
+        private static T Retry<T>(Func<T> fetch)
         {
             return Time.Retry(fetch, nameof(ContractsContainerInfoExtractor));
         }

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -0,0 +1,108 @@
+namespace CodexContractsPlugin
+{
+    public class SelfUpdater
+    {
+        public void Update(string abi, string bytecode)
+        {
+            var filePath = GetMarketplaceFilePath();
+            var content = GenerateContent(abi, bytecode);
+            var contentLines = content.Split("\r\n");
+
+            var beginWith = new string[]
+            {
+                "using Nethereum.ABI.FunctionEncoding.Attributes;",
+                "using Nethereum.Contracts;",
+                "using System.Numerics;",
+                "",
+                "// Generated code, do not modify.",
+                "",
+                "#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.",
+                "namespace CodexContractsPlugin.Marketplace",
+                "{"
+            };
+
+            var endWith = new string[]
+            {
+                "}",
+                "#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable."
+            };
+
+            File.Delete(filePath);
+            File.WriteAllLines(filePath,
+                beginWith.Concat(
+                contentLines.Concat(
+                endWith))
+            );
+
+            throw new Exception("Oh no! CodexContracts were updated. Current build of CodexContractsPlugin is incompatible. " +
+                "But fear not! SelfUpdater.cs has automatically updated the plugin. Just rebuild and rerun and it should work. " +
+                "Just in case, manual update instructions are found here: 'CodexContractsPlugin/Marketplace/README.md'.");
+        }
+
+        private string GetMarketplaceFilePath()
+        {
+            var here = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var path = GetMarketplaceFile(here);
+                if (path != null) return path;
+
+                var parent = Directory.GetParent(here);
+                var up = parent?.FullName;
+                if (up == null || up == here) throw new Exception("Unable to locate ProjectPlugins folder. Unable to update contracts.");
+                here = up;
+            }
+        }
+
+        private string? GetMarketplaceFile(string root)
+        {
+            var path = Path.Combine(root, "ProjectPlugins", "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (File.Exists(path)) return path;
+            return null;
+        }
+
+        private string GenerateContent(string abi, string bytecode)
+        {
+            var deserializer = new Nethereum.Generators.Net.GeneratorModelABIDeserialiser();
+            var abiModel = deserializer.DeserialiseABI(abi);
+            var abiCtor = abiModel.Constructor;
+            var c = new Nethereum.Generators.CQS.ContractDeploymentCQSMessageGenerator(abiCtor, "namespace", bytecode, "Marketplace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+            var lines = "";
+            lines += c.GenerateClass();
+            lines += "\r\n";
+
+            foreach (var eventAbi in abiModel.Events)
+            {
+                var d = new Nethereum.Generators.DTOs.EventDTOGenerator(eventAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += d.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var errorAbi in abiModel.Errors)
+            {
+                var e = new Nethereum.Generators.DTOs.ErrorDTOGenerator(errorAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += e.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var funcAbi in abiModel.Functions)
+            {
+                var f = new Nethereum.Generators.DTOs.FunctionOutputDTOGenerator(funcAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                var ff = new Nethereum.Generators.CQS.FunctionCQSMessageGenerator(funcAbi, "namespace", "funcoutput", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += f.GenerateClass();
+                lines += "\r\n";
+                lines += ff.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var structAbi in abiModel.Structs)
+            {
+                var g = new Nethereum.Generators.DTOs.StructTypeGenerator(structAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += g.GenerateClass();
+                lines += "\r\n";
+            }
+
+            return lines;
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-64b82de-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-1e2ad95-dist-tests";
 
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";

Tools/AutoClient/App.cs

@@ -0,0 +1,36 @@
+using Logging;
+
+namespace AutoClient
+{
+    public class App
+    {
+        public App(Configuration config)
+        {
+            Config = config;
+
+            Log = new LogSplitter(
+                new FileLog(Path.Combine(config.LogPath, "autoclient")),
+                new ConsoleLog()
+            );
+
+            Generator = CreateGenerator();
+            CidRepo = new CidRepo(config);
+        }
+
+        public Configuration Config { get; }
+        public ILog Log { get; }
+        public IFileGenerator Generator { get; }
+        public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
+        public CidRepo CidRepo { get; }
+        public Performance Performance { get; } = new Performance();
+
+        private IFileGenerator CreateGenerator()
+        {
+            if (Config.FileSizeMb > 0)
+            {
+                return new RandomFileGenerator(Config, Log);
+            }
+            return new ImageGenerator(Log);
+        }
+    }
+}

Tools/AutoClient/CidRepo.cs

@@ -0,0 +1,85 @@
+namespace AutoClient
+{
+    public class CidRepo
+    {
+        private readonly Random random = new Random();
+        private readonly object _lock = new object();
+        private readonly List<CidEntry> entries = new List<CidEntry>();
+        private readonly Configuration config;
+
+        public CidRepo(Configuration config)
+        {
+            this.config = config;
+        }
+
+        public void Add(string nodeId, string cid, long knownSize)
+        {
+            lock (_lock)
+            {
+                entries.Add(new CidEntry(nodeId, cid, knownSize));
+            }
+        }
+
+        public void AddEncoded(string originalCid, string encodedCid)
+        {
+            lock (_lock)
+            {
+                var entry = entries.SingleOrDefault(e => e.Cid == originalCid);
+                if (entry == null) return;
+
+                entry.Encoded = encodedCid;
+            }
+        }
+
+        public string? GetForeignCid(string myNodeId)
+        {
+            lock (_lock)
+            {
+                while (true)
+                {
+                    if (!entries.Any()) return null;
+                    var available = entries.Where(e => e.NodeId != myNodeId).ToArray();
+                    if (!available.Any()) return null;
+
+                    var i = random.Next(0, available.Length);
+                    var entry = available[i];
+
+                    if (entry.CreatedUtc < (DateTime.UtcNow + TimeSpan.FromMinutes(config.ContractDurationMinutes)))
+                    {
+                        entries.Remove(entry);
+                    }
+                    else
+                    {
+                        return entry.Cid;
+                    }
+                }
+            }
+        }
+
+        public long? GetSizeForCid(string cid)
+        {
+            lock (_lock)
+            {
+                var entry = entries.SingleOrDefault(e => e.Cid == cid);
+                if (entry == null) return null;
+                return entry.KnownSize;
+            }
+        }
+    }
+
+    public class CidEntry
+    {
+        public CidEntry(string nodeId, string cid, long knownSize)
+        {
+            NodeId = nodeId;
+            Cid = cid;
+            KnownSize = knownSize;
+        }
+
+        public string NodeId { get; }
+        public string Cid { get; }
+        public string Encoded { get; set; } = string.Empty;
+        public long KnownSize { get; }
+        public DateTime CreatedUtc { get; } = DateTime.UtcNow;
+    }
+}

Tools/AutoClient/CodexUser.cs

@@ -0,0 +1,57 @@
+using CodexOpenApi;
+using Logging;
+using Utils;
+
+namespace AutoClient
+{
+    public class CodexUser
+    {
+        private readonly App app;
+        private readonly CodexApi codex;
+        private readonly HttpClient client;
+        private readonly Address address;
+        private readonly List<Purchaser> purchasers = new List<Purchaser>();
+        private Task starterTask = Task.CompletedTask;
+        private readonly string nodeId = Guid.NewGuid().ToString();
+
+        public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
+        {
+            this.app = app;
+            this.codex = codex;
+            this.client = client;
+            this.address = address;
+        }
+
+        public void Start(int index)
+        {
+            for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
+            {
+                purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
+            }
+
+            var delayPerPurchaser =
+                TimeSpan.FromSeconds(10 * index) +
+                TimeSpan.FromMinutes(app.Config.ContractDurationMinutes) / app.Config.NumConcurrentPurchases;
+
+            starterTask = Task.Run(() => StartPurchasers(delayPerPurchaser));
+        }
+
+        private async Task StartPurchasers(TimeSpan delayPerPurchaser)
+        {
+            foreach (var purchaser in purchasers)
+            {
+                purchaser.Start();
+                await Task.Delay(delayPerPurchaser);
+            }
+        }
+
+        public void Stop()
+        {
+            starterTask.Wait();
+            foreach (var purchaser in purchasers)
+            {
+                purchaser.Stop();
+            }
+        }
+    }
+}

Tools/AutoClient/Configuration.cs

@@ -4,11 +4,8 @@ namespace AutoClient
 {
     public class Configuration
     {
-        [Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default 'http://localhost')")]
-        public string CodexHost { get; set; } = "http://localhost";
-
-        [Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")]
-        public int CodexPort { get; set; } = 8080;
+        [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
+        public string CodexEndpoints { get; set; } = "http://localhost:8080";
 
         [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";

Tools/AutoClient/ImageGenerator.cs

@@ -11,16 +11,16 @@ namespace AutoClient
 
     public class ImageGenerator : IFileGenerator
     {
-        private LogSplitter log;
+        private readonly ILog log;
 
-        public ImageGenerator(LogSplitter log)
+        public ImageGenerator(ILog log)
         {
             this.log = log;
         }
 
         public async Task<string> Generate()
         {
-            log.Log("Fetching random image from picsum.photos...");
+            log.Debug("Fetching random image from picsum.photos...");
             var httpClient = new HttpClient();
             var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160");
 

Tools/AutoClient/Performance.cs

@@ -0,0 +1,45 @@
+namespace AutoClient
+{
+    public class Performance
+    {
+        internal void DownloadFailed(Exception ex)
+        {
+            throw new NotImplementedException();
+        }
+
+        internal void DownloadSuccessful(long? size, TimeSpan time)
+        {
+            throw new NotImplementedException();
+        }
+
+        internal void StorageContractCancelled()
+        {
+            throw new NotImplementedException();
+        }
+
+        internal void StorageContractErrored(string error)
+        {
+            throw new NotImplementedException();
+        }
+
+        internal void StorageContractFinished()
+        {
+            throw new NotImplementedException();
+        }
+
+        internal void StorageContractStarted()
+        {
+            throw new NotImplementedException();
+        }
+
+        internal void UploadFailed(Exception exc)
+        {
+            throw new NotImplementedException();
+        }
+
+        internal void UploadSuccessful(long length, TimeSpan time)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

Tools/AutoClient/Program.cs

@@ -1,16 +1,20 @@
 using ArgsUniform;
 using AutoClient;
 using CodexOpenApi;
-using Core;
-using Logging;
 using Utils;
 
-public static class Program
+public class Program
 {
+    private readonly App app;
+
+    public Program(Configuration config)
+    {
+        app = new App(config);
+    }
+
     public static async Task Main(string[] args)
     {
         var cts = new CancellationTokenSource();
-        var cancellationToken = cts.Token;
         Console.CancelKeyPress += (sender, args) => cts.Cancel();
 
         var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
@@ -21,58 +25,70 @@ public static class Program
             throw new Exception("Number of concurrent purchases must be > 0");
         }
 
-        var log = new LogSplitter(
-            new FileLog(Path.Combine(config.LogPath, "autoclient")),
-            new ConsoleLog()
-        );
+        var p = new Program(config);
+        await p.Run();
+    }
+
+    public async Task Run()
+    {
+        var codexUsers = await CreateUsers();
+
+        var i = 0;
+        foreach (var user in codexUsers)
+        {
+            user.Start(i);
+            i++;
+        }
+
+        app.Cts.Token.WaitHandle.WaitOne();
+
+        foreach (var user in codexUsers) user.Stop();
+
+        app.Log.Log("Done");
+    }
+
+    private async Task<CodexUser[]> CreateUsers()
+    {
+        var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
+        var result = new List<CodexUser>();
+
+        foreach (var e in endpointStrs)
+        {
+            result.Add(await CreateUser(e));
+        }
+
+        return result.ToArray();
+    }
+
+    private async Task<CodexUser> CreateUser(string endpoint)
+    {
+        var splitIndex = endpoint.LastIndexOf(':');
+        var host = endpoint.Substring(0, splitIndex);
+        var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
 
         var address = new Address(
-            host: config.CodexHost,
-            port: config.CodexPort
+            host: host,
+            port: port
         );
 
-        log.Log($"Start. Address: {address}");
-
-        var generator = CreateGenerator(config, log);
-
         var client = new HttpClient();
         var codex = new CodexApi(client);
         codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
 
-        await CheckCodex(codex, log);
+        app.Log.Log($"Checking Codex at {address}...");
+        await CheckCodex(codex);
+        app.Log.Log("OK");
 
-        var purchasers = new List<Purchaser>();
-        for (var i = 0; i < config.NumConcurrentPurchases; i++)
-        {
-            purchasers.Add(
-                new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, generator, cancellationToken)
-            );
-        }
-
-        var delayPerPurchaser = TimeSpan.FromMinutes(config.ContractDurationMinutes) / config.NumConcurrentPurchases;
-        foreach (var purchaser in purchasers) 
-        {
-            purchaser.Start();
-            await Task.Delay(delayPerPurchaser);
-        }
-
-        cancellationToken.WaitHandle.WaitOne();
-
-        log.Log("Done.");
+        return new CodexUser(
+            app,
+            codex,
+            client,
+            address
+        );
     }
 
-    private static IFileGenerator CreateGenerator(Configuration config, LogSplitter log)
+    private async Task CheckCodex(CodexApi codex)
     {
-        if (config.FileSizeMb > 0)
-        {
-            return new RandomFileGenerator(config, log);
-        }
-        return new ImageGenerator(log);
-    }
-
-    private static async Task CheckCodex(CodexApi codex, ILog log)
-    {
-        log.Log("Checking Codex...");
         try
         {
             var info = await codex.GetDebugInfoAsync();
@@ -80,7 +96,7 @@ public static class Program
         }
         catch (Exception ex)
         {
-            log.Log($"Codex not OK: {ex}");
+            app.Log.Error($"Codex not OK: {ex}");
             throw;
         }
     }

Tools/AutoClient/Purchaser.cs

@@ -8,36 +8,76 @@ namespace AutoClient
 {
     public class Purchaser
     {
+        private readonly App app;
+        private readonly string nodeId;
         private readonly ILog log;
         private readonly HttpClient client;
         private readonly Address address;
         private readonly CodexApi codex;
-        private readonly Configuration config;
-        private readonly IFileGenerator generator;
-        private readonly CancellationToken ct;
+        private Task workerTask = Task.CompletedTask;
 
-        public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, IFileGenerator generator, CancellationToken ct)
+        public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
         {
+            this.app = app;
+            this.nodeId = nodeId;
             this.log = log;
             this.client = client;
             this.address = address;
             this.codex = codex;
-            this.config = config;
-            this.generator = generator;
-            this.ct = ct;
         }
 
         public void Start()
         {
-            Task.Run(Worker);
+            workerTask = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            workerTask.Wait();
         }
 
         private async Task Worker()
         {
-            while (!ct.IsCancellationRequested)
+            log.Log("Worker started.");
+            while (!app.Cts.Token.IsCancellationRequested)
             {
-                var pid = await StartNewPurchase();
-                await WaitTillFinished(pid);
+                try
+                {
+                    var pid = await StartNewPurchase();
+                    await WaitTillFinished(pid);
+                    await DownloadForeignCid();
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Worker failed with: " + ex);
+                    await Task.Delay(TimeSpan.FromHours(6));
+                }
+            }
+        }
+
+        private async Task DownloadForeignCid()
+        {
+            var cid = app.CidRepo.GetForeignCid(nodeId);
+            if (cid == null) return;
+            var size = app.CidRepo.GetSizeForCid(cid);
+            if (cid == null) return;
+
+            try
+            {
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var filename = Guid.NewGuid().ToString().ToLowerInvariant();
+                {
+                    using var fileStream = File.OpenWrite(filename);
+                    var fileResponse = await codex.DownloadNetworkAsync(cid);
+                    fileResponse.Stream.CopyTo(fileStream);
+                }
+                var time = sw.Elapsed;
+                File.Delete(filename);
+                app.Performance.DownloadSuccessful(size, time);
+            }
+            catch (Exception ex)
+            {
+                app.Performance.DownloadFailed(ex);
             }
         }
 
@@ -50,72 +90,96 @@ namespace AutoClient
 
         private async Task<string> CreateFile()
         {
-            return await generator.Generate();
+            return await app.Generator.Generate();
         }
 
         private async Task<ContentId> UploadFile(string filename)
         {
-            // Copied from CodexNode :/
             using var fileStream = File.OpenRead(filename);
+            try
+            {
+                var info = new FileInfo(filename);
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var cid = await UploadStream(fileStream);
+                var time = sw.Elapsed;
+                app.Performance.UploadSuccessful(info.Length, time);
+                app.CidRepo.Add(nodeId, cid.Id, info.Length);
+                return cid;
+            }
+            catch (Exception exc)
+            {
+                app.Performance.UploadFailed(exc);
+                throw;
+            }
+        }
 
-            log.Log($"Uploading file {filename}...");
-            var response = await codex.UploadAsync(fileStream, ct);
+        private async Task<ContentId> UploadStream(FileStream fileStream)
+        {
+            log.Debug($"Uploading file...");
+            var response = await codex.UploadAsync(fileStream, app.Cts.Token);
 
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
 
-            log.Log($"Uploaded file. Received contentId: '{response}'.");
+            log.Debug($"Uploaded file. Received contentId: '{response}'.");
             return new ContentId(response);
         }
 
         private async Task<string> RequestStorage(ContentId cid)
         {
-            log.Log("Requesting storage for " + cid.Id);
+            log.Debug("Requesting storage for " + cid.Id);
             var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
             {
-                Collateral = config.RequiredCollateral.ToString(),
-                Duration = (config.ContractDurationMinutes * 60).ToString(),
-                Expiry = (config.ContractExpiryMinutes * 60).ToString(),
-                Nodes = config.NumHosts,
-                Reward = config.Price.ToString(),
+                Collateral = app.Config.RequiredCollateral.ToString(),
+                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
+                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
+                Nodes = app.Config.NumHosts,
+                Reward = app.Config.Price.ToString(),
                 ProofProbability = "15",
-                Tolerance = config.HostTolerance
-            }, ct);
+                Tolerance = app.Config.HostTolerance
+            }, app.Cts.Token);
 
-            log.Log("Purchase ID: " + result);
+            log.Debug("Purchase ID: " + result);
+
+            var encoded = await GetEncodedCid(result);
+            app.CidRepo.AddEncoded(cid.Id, encoded);
 
             return result;
         }
 
-        private async Task<string?> GetPurchaseState(string pid)
+        private async Task<string> GetEncodedCid(string pid)
         {
             try
             {
-                // openapi still don't match code.
-                var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
-                if (string.IsNullOrEmpty(str)) return null;
-                var sp = JsonConvert.DeserializeObject<StoragePurchase>(str)!;
-                log.Log($"Purchase {pid} is {sp.State}");
-                if (!string.IsNullOrEmpty(sp.Error)) log.Log($"Purchase {pid} error is {sp.Error}");
-                return sp.State;
+                var sp = await GetStoragePurchase(pid)!;
+                return sp.Request.Content.Cid;
             }
-            catch
+            catch (Exception ex)
             {
-                return null;
+                log.Error(ex.ToString());
+                throw;
             }
         }
 
+        private async Task<StoragePurchase?> GetStoragePurchase(string pid)
+        {
+            // openapi still don't match code.
+            var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
+            if (string.IsNullOrEmpty(str)) return null;
+            return JsonConvert.DeserializeObject<StoragePurchase>(str);
+        }
+
         private async Task WaitTillFinished(string pid)
         {
-            log.Log("Waiting...");
             try
             {
                 var emptyResponseTolerance = 10;
-                while (true)
+                while (!app.Cts.Token.IsCancellationRequested)
                 {
-                    var status = (await GetPurchaseState(pid))?.ToLowerInvariant();
-                    if (string.IsNullOrEmpty(status))
+                    var purchase = await GetStoragePurchase(pid);
+                    if (purchase == null)
                     {
+                        await FixedShortDelay();
                         emptyResponseTolerance--;
                         if (emptyResponseTolerance == 0)
                         {
@@ -123,19 +187,28 @@ namespace AutoClient
                             await ExpiryTimeDelay();
                             return;
                         }
+                        continue;
                     }
-                    else
+                    var status = purchase.State.ToLowerInvariant();
+                    if (status.Contains("cancel"))
                     {
-                        if (status.Contains("cancel") ||
-                            status.Contains("error") ||
-                            status.Contains("finished"))
-                        {
-                            return;
-                        }
-                        if (status.Contains("started"))
-                        {
-                            await FixedDurationDelay();
-                        }
+                        app.Performance.StorageContractCancelled();
+                        return;
+                    }
+                    if (status.Contains("error"))
+                    {
+                        app.Performance.StorageContractErrored(purchase.Error);
+                        return;
+                    }
+                    if (status.Contains("finished"))
+                    {
+                        app.Performance.StorageContractFinished();
+                        return;
+                    }
+                    if (status.Contains("started"))
+                    {
+                        app.Performance.StorageContractStarted();
+                        await FixedDurationDelay();
                     }
 
                     await FixedShortDelay();
@@ -150,17 +223,17 @@ namespace AutoClient
 
         private async Task FixedDurationDelay()
         {
-            await Task.Delay(config.ContractDurationMinutes * 60 * 1000, ct);
+            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
         }
 
         private async Task ExpiryTimeDelay()
         {
-            await Task.Delay(config.ContractExpiryMinutes * 60 * 1000, ct);
+            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
         }
 
         private async Task FixedShortDelay()
         {
-            await Task.Delay(15 * 1000, ct);
+            await Task.Delay(15 * 1000, app.Cts.Token);
         }
     }
 }

Tools/MarketInsights/Dockerfile

@@ -3,8 +3,6 @@
 FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
 USER app
 WORKDIR /app
-EXPOSE 8080
-EXPOSE 8081
 
 FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 ARG BUILD_CONFIGURATION=Release

Tools/MarketInsights/MarketInsights.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <UserSecretsId>ae71e621-bb16-41b2-b6f3-c597d2d21157</UserSecretsId>

Tools/MarketInsights/Program.cs

@@ -1,6 +1,4 @@
 using ArgsUniform;
-using Microsoft.Extensions.Options;
-using Nethereum.Model;
 using System.Reflection;
 
 namespace MarketInsights
@@ -28,6 +26,14 @@ namespace MarketInsights
 
             var builder = WebApplication.CreateBuilder(args);
 
+            var listenPort = Environment.GetEnvironmentVariable("APIPORT");
+            if (string.IsNullOrEmpty(listenPort)) listenPort = "31090";
+
+            builder.WebHost.ConfigureKestrel((context, options) =>
+            {
+                options.ListenAnyIP(Convert.ToInt32(listenPort));
+            });
+
             builder.Services.AddSingleton(appState);
 
             builder.Services.AddControllers();
@@ -54,6 +60,8 @@ namespace MarketInsights
 
             app.MapControllers();
 
+            Console.WriteLine("MarketInsights listening on port " + listenPort);
+
             updater.Run();
             app.Run();
         }

cs-codex-dist-testing.sln

@@ -74,7 +74,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OverwatchTranscript", "Fram
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools\TranscriptAnalysis\TranscriptAnalysis.csproj", "{C0EEBD32-23CB-45EC-A863-79FB948508C8}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution