Merge branch 'feature/better-autoclient'

.github/workflows/docker-autoclientcenter.yml

@@ -0,0 +1,24 @@
+name: Docker - AutoClientCenter
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/AutoClientCenter/**'
+      - '!Tools/AutoClientCenter/Dockerfile'
+      - .github/workflows/docker-autoclientcenter.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/AutoClientCenter/Dockerfile
+      docker_repo: codexstorage/codex-autoclientcenter
+    secrets: inherit
+

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-64b82de-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-1e2ad95-dist-tests";
 
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";

Tools/AutoClientCenter/AutoClientCenter.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <UserSecretsId>80f55fc5-9f8f-4bb4-89bc-314ed7379c5f</UserSecretsId>
+    <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.20.1" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+</Project>

Tools/AutoClientCenter/AutoClientCenter.csproj.user

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <ActiveDebugProfile>IIS Express</ActiveDebugProfile>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <DebuggerFlavor>ProjectDebugger</DebuggerFlavor>
+  </PropertyGroup>
+</Project>

Tools/AutoClientCenter/AutoClientCenter.http

@@ -0,0 +1,6 @@
+@AutoClientCenter_HostAddress = http://localhost:5185
+
+GET {{AutoClientCenter_HostAddress}}/weatherforecast/
+Accept: application/json
+
+###

Tools/AutoClientCenter/CidRepo.cs

@@ -0,0 +1,76 @@
+namespace AutoClientCenter
+{
+    public class CidRepo
+    {
+        private readonly Random random = new Random();
+        private readonly object _lock = new object();
+        private readonly List<CidEntry> entries = new List<CidEntry>();
+
+        public void Add(string cid, long knownSize)
+        {
+            lock (_lock)
+            {
+                entries.Add(new CidEntry(cid, knownSize));
+            }
+        }
+
+        public void AddEncoded(string originalCid, string encodedCid)
+        {
+            lock (_lock)
+            {
+                var entry = entries.SingleOrDefault(e => e.Cid == originalCid);
+                if (entry == null) return;
+
+                entry.Encoded = encodedCid;
+            }
+        }
+
+        public void Assign(AcDownloadStep downloadStep)
+        {
+            lock (_lock)
+            {
+                while (true)
+                {
+                    if (!entries.Any()) return;
+
+                    var i = random.Next(0, entries.Count);
+                    var entry = entries[i];
+
+                    if (entry.CreatedUtc < (DateTime.UtcNow + TimeSpan.FromHours(18)))
+                    {
+                        entries.RemoveAt(i);
+                    }
+                    else
+                    {
+                        downloadStep.Cid = entry.Cid;
+                        return;
+                    }
+                }
+            }
+        }
+
+        public long? GetSizeKbsForCid(string cid)
+        {
+            lock (_lock)
+            {
+                var entry = entries.SingleOrDefault(e => e.Cid == cid);
+                if (entry == null) return null;
+                return entry.KnownSize;
+            }
+        }
+    }
+
+    public class CidEntry
+    {
+        public CidEntry(string cid, long knownSize)
+        {
+            Cid = cid;
+            KnownSize = knownSize;
+        }
+
+        public string Cid { get; }
+        public string Encoded { get; set; } = string.Empty;
+        public long KnownSize { get; }
+        public DateTime CreatedUtc { get; } = DateTime.UtcNow;
+    }
+}

Tools/AutoClientCenter/Controllers/ConfigController.cs

@@ -0,0 +1,28 @@
+using Microsoft.AspNetCore.Mvc;
+
+namespace AutoClientCenter.Controllers
+{
+    [ApiController]
+    [Route("[controller]")]
+    public class ConfigController : ControllerBase
+    {
+        private readonly ITaskService taskService;
+
+        public ConfigController(ITaskService taskService)
+        {
+            this.taskService = taskService;
+        }
+
+        [HttpGet("Stats")]
+        public AcStats Get()
+        {
+            return taskService.GetStats();
+        }
+
+        [HttpPost("Set")]
+        public void Post([FromBody] AcTasks tasks)
+        {
+            taskService.SetConfig(tasks);
+        }
+    }
+}

Tools/AutoClientCenter/Controllers/TasksController.cs

@@ -0,0 +1,35 @@
+using Microsoft.AspNetCore.Mvc;
+
+namespace AutoClientCenter.Controllers
+{
+    [ApiController]
+    [Route("[controller]")]
+    public class TasksController : ControllerBase
+    {
+        private readonly ITaskService taskService;
+        private static readonly object processLock = new object();
+
+        public TasksController(ITaskService taskService)
+        {
+            this.taskService = taskService;
+        }
+
+        [HttpGet]
+        public AcTasks Get()
+        {
+            return taskService.GetTasks();
+        }
+
+        [HttpPost("Results")]
+        public void Post([FromBody] AcTaskStep[] taskSteps)
+        {
+            Task.Run(() =>
+            {
+                lock (processLock)
+                {
+                    taskService.ProcessResults(taskSteps);
+                }
+            });
+        }
+    }
+}

Tools/AutoClientCenter/Dockerfile

@@ -0,0 +1,23 @@
+#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging.
+
+FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
+USER app
+WORKDIR /app
+
+FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
+ARG BUILD_CONFIGURATION=Release
+WORKDIR /src
+COPY ["AutoClientCenter/AutoClientCenter.csproj", "AutoClientCenter/"]
+RUN dotnet restore "./AutoClientCenter/AutoClientCenter.csproj"
+COPY . .
+WORKDIR "/src/AutoClientCenter"
+RUN dotnet build "./AutoClientCenter.csproj" -c $BUILD_CONFIGURATION -o /app/build
+
+FROM build AS publish
+ARG BUILD_CONFIGURATION=Release
+RUN dotnet publish "./AutoClientCenter.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
+
+FROM base AS final
+WORKDIR /app
+COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "AutoClientCenter.dll"]

Tools/AutoClientCenter/Model.cs

@@ -0,0 +1,60 @@
+namespace AutoClientCenter
+{
+    public class AcTasks
+    {
+        public int StartTaskEverySeconds { get; set; }
+        public AcTask[] Tasks { get; set; } = Array.Empty<AcTask>();
+    }
+
+    public class AcTask
+    {
+        public int ChanceWeight { get; set; }
+        public AcTaskStep[] Steps { get; set; } = Array.Empty<AcTaskStep>();
+    }
+
+    public class AcTaskStep
+    {
+        public string Id { get; set; } = string.Empty;
+        public AcUploadStep? UploadStep { get; set; }
+        public AcStoreStep? StoreStep { get; set; }
+        public AcDownloadStep? DownloadStep { get; set; }
+        public string? ResultErrorMsg { get; set; }
+    }
+
+    public class AcUploadStep
+    {
+        public long SizeInBytes { get; set; }
+        public string? ResultCid { get; set; }
+    }
+
+    public class AcStoreStep
+    {
+        public int ContractDurationMinutes { get; set; }
+        public int ContractExpiryMinutes { get; set; }
+        public int NumHosts { get; set; }
+        public int HostTolerance { get; set; }
+        public int Price { get; set; }
+        public int RequiredCollateral { get; set; }
+        public string? ResultPurchaseId { get; set; }
+        public string? ResultOriginalCid { get; set; }
+        public string? ResultEncodedCid { get; set; }
+    }
+
+    public class AcDownloadStep
+    {
+        public string Cid { get; set; } = string.Empty;
+        public long ResultDownloadTimeMilliseconds { get; set; }
+    }
+
+    public class AcStats
+    {
+        public DateTime ServiceStartUtc { get; set; } = DateTime.MinValue;
+        public int TotalUploads { get; set; }
+        public int TotalUploadsFailed { get; set; }
+        public int TotalDownloads { get; set; }
+        public long[] DownloadTimesMillisecondsPerKb { get; set; } = Array.Empty<long>();
+        public int TotalDownloadsFailed { get; set; }
+        public int TotalContractsStarted { get; set; }
+        public int TotalContractStartsFailed { get; set; }
+    }
+}

Tools/AutoClientCenter/Program.cs

@@ -0,0 +1,43 @@
+
+namespace AutoClientCenter
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            var builder = WebApplication.CreateBuilder(args);
+
+            var listenPort = Environment.GetEnvironmentVariable("APIPORT");
+            if (string.IsNullOrEmpty(listenPort)) listenPort = "31090";
+
+            builder.WebHost.ConfigureKestrel((context, options) =>
+            {
+                options.ListenAnyIP(Convert.ToInt32(listenPort));
+            });
+
+            builder.Services.AddSingleton<ITaskService>(new TaskService());
+            builder.Services.AddControllers();
+            builder.Services.AddEndpointsApiExplorer();
+            builder.Services.AddSwaggerGen();
+
+            var app = builder.Build();
+
+            if (app.Environment.IsDevelopment())
+            {
+                app.UseSwagger();
+                app.UseSwaggerUI();
+            }
+
+            app.UseHttpsRedirection();
+
+            app.UseAuthorization();
+
+
+            app.MapControllers();
+
+            Console.WriteLine("AutoClientCenter listening on port " + listenPort);
+
+            app.Run();
+        }
+    }
+}

Tools/AutoClientCenter/Properties/launchSettings.json

@@ -0,0 +1,52 @@
+{
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "http://localhost:5185"
+    },
+    "https": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "https://localhost:7077;http://localhost:5185"
+    },
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "Container (Dockerfile)": {
+      "commandName": "Docker",
+      "launchBrowser": true,
+      "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}/swagger",
+      "environmentVariables": {
+        "ASPNETCORE_HTTPS_PORTS": "8081",
+        "ASPNETCORE_HTTP_PORTS": "8080"
+      },
+      "publishAllPorts": true,
+      "useSSL": true
+    }
+  },
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:50475",
+      "sslPort": 44395
+    }
+  }
+}

Tools/AutoClientCenter/TaskService.cs

@@ -0,0 +1,116 @@
+namespace AutoClientCenter
+{
+    public interface ITaskService
+    {
+        AcStats GetStats();
+        AcTasks GetTasks();
+        void ProcessResults(AcTaskStep[] taskSteps);
+        void SetConfig(AcTasks tasks);
+    }
+
+    public class TaskService : ITaskService
+    {
+        private readonly CidRepo cidRepo = new CidRepo();
+        private readonly List<long> downloadTimes = new List<long>();
+        private readonly AcStats stats = new AcStats
+        {
+            ServiceStartUtc = DateTime.UtcNow,
+        };
+
+        private AcTasks tasks = new AcTasks
+        {
+            StartTaskEverySeconds = Convert.ToInt32(TimeSpan.FromHours(8).TotalSeconds),
+            Tasks = Array.Empty<AcTask>()
+        };
+
+        public AcStats GetStats()
+        {
+            stats.DownloadTimesMillisecondsPerKb = downloadTimes.ToArray();
+            return stats;
+        }
+
+        public AcTasks GetTasks()
+        {
+            foreach (var task in tasks.Tasks)
+            {
+                foreach (var step in task.Steps)
+                {
+                    if (step.DownloadStep != null) cidRepo.Assign(step.DownloadStep);
+                }
+            }
+            return tasks;
+        }
+
+        public void SetConfig(AcTasks newTasks)
+        {
+            if (newTasks.StartTaskEverySeconds < (10 * 60)) return;
+            foreach (var task in newTasks.Tasks)
+            {
+                if (task.ChanceWeight < 1) return;
+                foreach (var step in task.Steps)
+                {
+                    if (string.IsNullOrWhiteSpace(step.Id)) return;
+                    if (step.UploadStep == null && step.StoreStep == null && step.DownloadStep == null) return;
+                }
+            }
+
+            tasks = newTasks;
+        }
+
+        public void ProcessResults(AcTaskStep[] taskSteps)
+        {
+            foreach (var step in taskSteps) ProcessResults(step);
+        }
+
+        private void ProcessResults(AcTaskStep step)
+        {
+            ProcessResult(step.UploadStep);
+            ProcessResult(step.StoreStep);
+            ProcessResult(step.DownloadStep);
+        }
+
+        private void ProcessResult(AcUploadStep? uploadStep)
+        {
+            if (uploadStep == null) return;
+
+            if (string.IsNullOrWhiteSpace(uploadStep.ResultCid))
+            {
+                stats.TotalUploadsFailed++;
+            }
+            else
+            {
+                stats.TotalUploads++;
+                cidRepo.Add(uploadStep.ResultCid, uploadStep.SizeInBytes);
+            }
+        }
+
+        private void ProcessResult(AcStoreStep? storeStep)
+        {
+            if (storeStep == null) return;
+
+            if (string.IsNullOrWhiteSpace(storeStep.ResultOriginalCid) ||
+                string.IsNullOrWhiteSpace(storeStep.ResultEncodedCid) ||
+                string.IsNullOrWhiteSpace(storeStep.ResultPurchaseId))
+            {
+                stats.TotalContractStartsFailed++;
+            }
+            else
+            {
+                stats.TotalContractsStarted++;
+                cidRepo.AddEncoded(storeStep.ResultOriginalCid, storeStep.ResultEncodedCid);
+            }
+        }
+
+        private void ProcessResult(AcDownloadStep? downloadStep)
+        {
+            if (downloadStep == null) return;
+
+            var kbs = cidRepo.GetSizeKbsForCid(downloadStep.Cid);
+            if (kbs == null) return;
+            var milliseconds = downloadStep.ResultDownloadTimeMilliseconds;
+
+            var millisecondsPerKb = milliseconds / kbs.Value;
+            downloadTimes.Add(millisecondsPerKb);
+        }
+    }
+}

Tools/AutoClientCenter/appsettings.Development.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

Tools/AutoClientCenter/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

Tools/MarketInsights/Dockerfile

@@ -3,8 +3,6 @@
 FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
 USER app
 WORKDIR /app
-EXPOSE 8080
-EXPOSE 8081
 
 FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 ARG BUILD_CONFIGURATION=Release

Tools/MarketInsights/Program.cs

@@ -1,6 +1,4 @@
 using ArgsUniform;
-using Microsoft.Extensions.Options;
-using Nethereum.Model;
 using System.Reflection;
 
 namespace MarketInsights
@@ -28,6 +26,14 @@ namespace MarketInsights
 
             var builder = WebApplication.CreateBuilder(args);
 
+            var listenPort = Environment.GetEnvironmentVariable("APIPORT");
+            if (string.IsNullOrEmpty(listenPort)) listenPort = "31090";
+
+            builder.WebHost.ConfigureKestrel((context, options) =>
+            {
+                options.ListenAnyIP(Convert.ToInt32(listenPort));
+            });
+
             builder.Services.AddSingleton(appState);
 
             builder.Services.AddControllers();
@@ -54,6 +60,8 @@ namespace MarketInsights
 
             app.MapControllers();
 
+            Console.WriteLine("MarketInsights listening on port " + listenPort);
+
             updater.Run();
             app.Run();
         }

cs-codex-dist-testing.sln

@@ -74,7 +74,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OverwatchTranscript", "Fram
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools\TranscriptAnalysis\TranscriptAnalysis.csproj", "{C0EEBD32-23CB-45EC-A863-79FB948508C8}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoClientCenter", "AutoClientCenter\AutoClientCenter.csproj", "{023B6C60-406B-47A1-9ED8-12688759D2CC}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -202,6 +204,10 @@ Global
 		{004614DF-1C65-45E3-882D-59AE44282573}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.Build.0 = Release|Any CPU
+		{023B6C60-406B-47A1-9ED8-12688759D2CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{023B6C60-406B-47A1-9ED8-12688759D2CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{023B6C60-406B-47A1-9ED8-12688759D2CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{023B6C60-406B-47A1-9ED8-12688759D2CC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -237,6 +243,7 @@ Global
 		{870DDFBE-D7ED-4196-9681-13CA947BDEA6} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
 		{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
+		{023B6C60-406B-47A1-9ED8-12688759D2CC} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}