diff --git a/.github/workflows/docker-autoclientcenter.yml b/.github/workflows/docker-autoclientcenter.yml deleted file mode 100644 index 4e797cb..0000000 --- a/.github/workflows/docker-autoclientcenter.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Docker - AutoClientCenter - -on: - push: - branches: - - master - tags: - - 'v*.*.*' - paths: - - 'Tools/AutoClientCenter/**' - - '!Tools/AutoClientCenter/Dockerfile' - - .github/workflows/docker-autoclientcenter.yml - - .github/workflows/docker-reusable.yml - workflow_dispatch: - -jobs: - build-and-push: - name: Build and Push - uses: ./.github/workflows/docker-reusable.yml - with: - docker_file: Tools/AutoClientCenter/Dockerfile - docker_repo: codexstorage/codex-autoclientcenter - secrets: inherit - diff --git a/Tools/AutoClient/App.cs b/Tools/AutoClient/App.cs new file mode 100644 index 0000000..d88ac66 --- /dev/null +++ b/Tools/AutoClient/App.cs @@ -0,0 +1,36 @@ +using Logging; + +namespace AutoClient +{ + public class App + { + public App(Configuration config) + { + Config = config; + + Log = new LogSplitter( + new FileLog(Path.Combine(config.LogPath, "autoclient")), + new ConsoleLog() + ); + + Generator = CreateGenerator(); + CidRepo = new CidRepo(config); + } + + public Configuration Config { get; } + public ILog Log { get; } + public IFileGenerator Generator { get; } + public CancellationTokenSource Cts { get; } = new CancellationTokenSource(); + public CidRepo CidRepo { get; } + public Performance Performance { get; } = new Performance(); + + private IFileGenerator CreateGenerator() + { + if (Config.FileSizeMb > 0) + { + return new RandomFileGenerator(Config, Log); + } + return new ImageGenerator(Log); + } + } +} diff --git a/Tools/AutoClientCenter/CidRepo.cs b/Tools/AutoClient/CidRepo.cs similarity index 59% rename from Tools/AutoClientCenter/CidRepo.cs rename to Tools/AutoClient/CidRepo.cs index 542c2d4..e9038e3 100644 --- a/Tools/AutoClientCenter/CidRepo.cs +++ b/Tools/AutoClient/CidRepo.cs @@ -1,16 +1,22 @@ -namespace AutoClientCenter +namespace AutoClient { public class CidRepo { private readonly Random random = new Random(); private readonly object _lock = new object(); private readonly List entries = new List(); + private readonly Configuration config; - public void Add(string cid, long knownSize) + public CidRepo(Configuration config) + { + this.config = config; + } + + public void Add(string nodeId, string cid, long knownSize) { lock (_lock) { - entries.Add(new CidEntry(cid, knownSize)); + entries.Add(new CidEntry(nodeId, cid, knownSize)); } } @@ -25,31 +31,32 @@ } } - public void Assign(AcDownloadStep downloadStep) + public string? GetForeignCid(string myNodeId) { lock (_lock) { while (true) { - if (!entries.Any()) return; + if (!entries.Any()) return null; + var available = entries.Where(e => e.NodeId != myNodeId).ToArray(); + if (!available.Any()) return null; - var i = random.Next(0, entries.Count); - var entry = entries[i]; + var i = random.Next(0, available.Length); + var entry = available[i]; - if (entry.CreatedUtc < (DateTime.UtcNow + TimeSpan.FromHours(18))) + if (entry.CreatedUtc < (DateTime.UtcNow + TimeSpan.FromMinutes(config.ContractDurationMinutes))) { - entries.RemoveAt(i); + entries.Remove(entry); } else { - downloadStep.Cid = entry.Cid; - return; + return entry.Cid; } } } } - public long? GetSizeKbsForCid(string cid) + public long? GetSizeForCid(string cid) { lock (_lock) { @@ -62,12 +69,14 @@ public class CidEntry { - public CidEntry(string cid, long knownSize) + public CidEntry(string nodeId, string cid, long knownSize) { + NodeId = nodeId; Cid = cid; KnownSize = knownSize; } + public string NodeId { get; } public string Cid { get; } public string Encoded { get; set; } = string.Empty; public long KnownSize { get; } diff --git a/Tools/AutoClient/CodexUser.cs b/Tools/AutoClient/CodexUser.cs new file mode 100644 index 0000000..fd3259c --- /dev/null +++ b/Tools/AutoClient/CodexUser.cs @@ -0,0 +1,57 @@ +using CodexOpenApi; +using Logging; +using Utils; + +namespace AutoClient +{ + public class CodexUser + { + private readonly App app; + private readonly CodexApi codex; + private readonly HttpClient client; + private readonly Address address; + private readonly List purchasers = new List(); + private Task starterTask = Task.CompletedTask; + private readonly string nodeId = Guid.NewGuid().ToString(); + + public CodexUser(App app, CodexApi codex, HttpClient client, Address address) + { + this.app = app; + this.codex = codex; + this.client = client; + this.address = address; + } + + public void Start(int index) + { + for (var i = 0; i < app.Config.NumConcurrentPurchases; i++) + { + purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex)); + } + + var delayPerPurchaser = + TimeSpan.FromSeconds(10 * index) + + TimeSpan.FromMinutes(app.Config.ContractDurationMinutes) / app.Config.NumConcurrentPurchases; + + starterTask = Task.Run(() => StartPurchasers(delayPerPurchaser)); + } + + private async Task StartPurchasers(TimeSpan delayPerPurchaser) + { + foreach (var purchaser in purchasers) + { + purchaser.Start(); + await Task.Delay(delayPerPurchaser); + } + } + + public void Stop() + { + starterTask.Wait(); + foreach (var purchaser in purchasers) + { + purchaser.Stop(); + } + } + } +} diff --git a/Tools/AutoClient/Configuration.cs b/Tools/AutoClient/Configuration.cs index d21deb5..c97b397 100644 --- a/Tools/AutoClient/Configuration.cs +++ b/Tools/AutoClient/Configuration.cs @@ -4,11 +4,8 @@ namespace AutoClient { public class Configuration { - [Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default 'http://localhost')")] - public string CodexHost { get; set; } = "http://localhost"; - - [Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")] - public int CodexPort { get; set; } = 8080; + [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")] + public string CodexEndpoints { get; set; } = "http://localhost:8080"; [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")] public string DataPath { get; set; } = "datapath"; diff --git a/Tools/AutoClient/ImageGenerator.cs b/Tools/AutoClient/ImageGenerator.cs index 390381b..bfb95e1 100644 --- a/Tools/AutoClient/ImageGenerator.cs +++ b/Tools/AutoClient/ImageGenerator.cs @@ -11,16 +11,16 @@ namespace AutoClient public class ImageGenerator : IFileGenerator { - private LogSplitter log; + private readonly ILog log; - public ImageGenerator(LogSplitter log) + public ImageGenerator(ILog log) { this.log = log; } public async Task Generate() { - log.Log("Fetching random image from picsum.photos..."); + log.Debug("Fetching random image from picsum.photos..."); var httpClient = new HttpClient(); var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160"); diff --git a/Tools/AutoClient/Performance.cs b/Tools/AutoClient/Performance.cs new file mode 100644 index 0000000..ba3213b --- /dev/null +++ b/Tools/AutoClient/Performance.cs @@ -0,0 +1,45 @@ +namespace AutoClient +{ + public class Performance + { + internal void DownloadFailed(Exception ex) + { + throw new NotImplementedException(); + } + + internal void DownloadSuccessful(long? size, TimeSpan time) + { + throw new NotImplementedException(); + } + + internal void StorageContractCancelled() + { + throw new NotImplementedException(); + } + + internal void StorageContractErrored(string error) + { + throw new NotImplementedException(); + } + + internal void StorageContractFinished() + { + throw new NotImplementedException(); + } + + internal void StorageContractStarted() + { + throw new NotImplementedException(); + } + + internal void UploadFailed(Exception exc) + { + throw new NotImplementedException(); + } + + internal void UploadSuccessful(long length, TimeSpan time) + { + throw new NotImplementedException(); + } + } +} diff --git a/Tools/AutoClient/Program.cs b/Tools/AutoClient/Program.cs index d5c4aa0..6d1c1fd 100644 --- a/Tools/AutoClient/Program.cs +++ b/Tools/AutoClient/Program.cs @@ -1,16 +1,20 @@ using ArgsUniform; using AutoClient; using CodexOpenApi; -using Core; -using Logging; using Utils; -public static class Program +public class Program { + private readonly App app; + + public Program(Configuration config) + { + app = new App(config); + } + public static async Task Main(string[] args) { var cts = new CancellationTokenSource(); - var cancellationToken = cts.Token; Console.CancelKeyPress += (sender, args) => cts.Cancel(); var uniformArgs = new ArgsUniform(PrintHelp, args); @@ -21,58 +25,70 @@ public static class Program throw new Exception("Number of concurrent purchases must be > 0"); } - var log = new LogSplitter( - new FileLog(Path.Combine(config.LogPath, "autoclient")), - new ConsoleLog() - ); + var p = new Program(config); + await p.Run(); + } + + public async Task Run() + { + var codexUsers = await CreateUsers(); + + var i = 0; + foreach (var user in codexUsers) + { + user.Start(i); + i++; + } + + app.Cts.Token.WaitHandle.WaitOne(); + + foreach (var user in codexUsers) user.Stop(); + + app.Log.Log("Done"); + } + + private async Task CreateUsers() + { + var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries); + var result = new List(); + + foreach (var e in endpointStrs) + { + result.Add(await CreateUser(e)); + } + + return result.ToArray(); + } + + private async Task CreateUser(string endpoint) + { + var splitIndex = endpoint.LastIndexOf(':'); + var host = endpoint.Substring(0, splitIndex); + var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1)); var address = new Address( - host: config.CodexHost, - port: config.CodexPort + host: host, + port: port ); - log.Log($"Start. Address: {address}"); - - var generator = CreateGenerator(config, log); - var client = new HttpClient(); var codex = new CodexApi(client); codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; - await CheckCodex(codex, log); + app.Log.Log($"Checking Codex at {address}..."); + await CheckCodex(codex); + app.Log.Log("OK"); - var purchasers = new List(); - for (var i = 0; i < config.NumConcurrentPurchases; i++) - { - purchasers.Add( - new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, generator, cancellationToken) - ); - } - - var delayPerPurchaser = TimeSpan.FromMinutes(config.ContractDurationMinutes) / config.NumConcurrentPurchases; - foreach (var purchaser in purchasers) - { - purchaser.Start(); - await Task.Delay(delayPerPurchaser); - } - - cancellationToken.WaitHandle.WaitOne(); - - log.Log("Done."); + return new CodexUser( + app, + codex, + client, + address + ); } - private static IFileGenerator CreateGenerator(Configuration config, LogSplitter log) + private async Task CheckCodex(CodexApi codex) { - if (config.FileSizeMb > 0) - { - return new RandomFileGenerator(config, log); - } - return new ImageGenerator(log); - } - - private static async Task CheckCodex(CodexApi codex, ILog log) - { - log.Log("Checking Codex..."); try { var info = await codex.GetDebugInfoAsync(); @@ -80,7 +96,7 @@ public static class Program } catch (Exception ex) { - log.Log($"Codex not OK: {ex}"); + app.Log.Error($"Codex not OK: {ex}"); throw; } } diff --git a/Tools/AutoClient/Purchaser.cs b/Tools/AutoClient/Purchaser.cs index d0d6995..cc5966e 100644 --- a/Tools/AutoClient/Purchaser.cs +++ b/Tools/AutoClient/Purchaser.cs @@ -8,36 +8,76 @@ namespace AutoClient { public class Purchaser { + private readonly App app; + private readonly string nodeId; private readonly ILog log; private readonly HttpClient client; private readonly Address address; private readonly CodexApi codex; - private readonly Configuration config; - private readonly IFileGenerator generator; - private readonly CancellationToken ct; + private Task workerTask = Task.CompletedTask; - public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, IFileGenerator generator, CancellationToken ct) + public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex) { + this.app = app; + this.nodeId = nodeId; this.log = log; this.client = client; this.address = address; this.codex = codex; - this.config = config; - this.generator = generator; - this.ct = ct; } public void Start() { - Task.Run(Worker); + workerTask = Task.Run(Worker); + } + + public void Stop() + { + workerTask.Wait(); } private async Task Worker() { - while (!ct.IsCancellationRequested) + log.Log("Worker started."); + while (!app.Cts.Token.IsCancellationRequested) { - var pid = await StartNewPurchase(); - await WaitTillFinished(pid); + try + { + var pid = await StartNewPurchase(); + await WaitTillFinished(pid); + await DownloadForeignCid(); + } + catch (Exception ex) + { + log.Error("Worker failed with: " + ex); + await Task.Delay(TimeSpan.FromHours(6)); + } + } + } + + private async Task DownloadForeignCid() + { + var cid = app.CidRepo.GetForeignCid(nodeId); + if (cid == null) return; + var size = app.CidRepo.GetSizeForCid(cid); + if (cid == null) return; + + try + { + var sw = System.Diagnostics.Stopwatch.StartNew(); + var filename = Guid.NewGuid().ToString().ToLowerInvariant(); + { + using var fileStream = File.OpenWrite(filename); + var fileResponse = await codex.DownloadNetworkAsync(cid); + fileResponse.Stream.CopyTo(fileStream); + } + var time = sw.Elapsed; + File.Delete(filename); + app.Performance.DownloadSuccessful(size, time); + } + catch (Exception ex) + { + app.Performance.DownloadFailed(ex); } } @@ -50,72 +90,96 @@ namespace AutoClient private async Task CreateFile() { - return await generator.Generate(); + return await app.Generator.Generate(); } private async Task UploadFile(string filename) { - // Copied from CodexNode :/ using var fileStream = File.OpenRead(filename); + try + { + var info = new FileInfo(filename); + var sw = System.Diagnostics.Stopwatch.StartNew(); + var cid = await UploadStream(fileStream); + var time = sw.Elapsed; + app.Performance.UploadSuccessful(info.Length, time); + app.CidRepo.Add(nodeId, cid.Id, info.Length); + return cid; + } + catch (Exception exc) + { + app.Performance.UploadFailed(exc); + throw; + } + } - log.Log($"Uploading file {filename}..."); - var response = await codex.UploadAsync(fileStream, ct); + private async Task UploadStream(FileStream fileStream) + { + log.Debug($"Uploading file..."); + var response = await codex.UploadAsync(fileStream, app.Cts.Token); if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response."); if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block."); - log.Log($"Uploaded file. Received contentId: '{response}'."); + log.Debug($"Uploaded file. Received contentId: '{response}'."); return new ContentId(response); } private async Task RequestStorage(ContentId cid) { - log.Log("Requesting storage for " + cid.Id); + log.Debug("Requesting storage for " + cid.Id); var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation() { - Collateral = config.RequiredCollateral.ToString(), - Duration = (config.ContractDurationMinutes * 60).ToString(), - Expiry = (config.ContractExpiryMinutes * 60).ToString(), - Nodes = config.NumHosts, - Reward = config.Price.ToString(), + Collateral = app.Config.RequiredCollateral.ToString(), + Duration = (app.Config.ContractDurationMinutes * 60).ToString(), + Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(), + Nodes = app.Config.NumHosts, + Reward = app.Config.Price.ToString(), ProofProbability = "15", - Tolerance = config.HostTolerance - }, ct); + Tolerance = app.Config.HostTolerance + }, app.Cts.Token); - log.Log("Purchase ID: " + result); + log.Debug("Purchase ID: " + result); + + var encoded = await GetEncodedCid(result); + app.CidRepo.AddEncoded(cid.Id, encoded); return result; } - private async Task GetPurchaseState(string pid) + private async Task GetEncodedCid(string pid) { try { - // openapi still don't match code. - var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}"); - if (string.IsNullOrEmpty(str)) return null; - var sp = JsonConvert.DeserializeObject(str)!; - log.Log($"Purchase {pid} is {sp.State}"); - if (!string.IsNullOrEmpty(sp.Error)) log.Log($"Purchase {pid} error is {sp.Error}"); - return sp.State; + var sp = await GetStoragePurchase(pid)!; + return sp.Request.Content.Cid; } - catch + catch (Exception ex) { - return null; + log.Error(ex.ToString()); + throw; } } + private async Task GetStoragePurchase(string pid) + { + // openapi still don't match code. + var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}"); + if (string.IsNullOrEmpty(str)) return null; + return JsonConvert.DeserializeObject(str); + } + private async Task WaitTillFinished(string pid) { - log.Log("Waiting..."); try { var emptyResponseTolerance = 10; - while (true) + while (!app.Cts.Token.IsCancellationRequested) { - var status = (await GetPurchaseState(pid))?.ToLowerInvariant(); - if (string.IsNullOrEmpty(status)) + var purchase = await GetStoragePurchase(pid); + if (purchase == null) { + await FixedShortDelay(); emptyResponseTolerance--; if (emptyResponseTolerance == 0) { @@ -123,19 +187,28 @@ namespace AutoClient await ExpiryTimeDelay(); return; } + continue; } - else + var status = purchase.State.ToLowerInvariant(); + if (status.Contains("cancel")) { - if (status.Contains("cancel") || - status.Contains("error") || - status.Contains("finished")) - { - return; - } - if (status.Contains("started")) - { - await FixedDurationDelay(); - } + app.Performance.StorageContractCancelled(); + return; + } + if (status.Contains("error")) + { + app.Performance.StorageContractErrored(purchase.Error); + return; + } + if (status.Contains("finished")) + { + app.Performance.StorageContractFinished(); + return; + } + if (status.Contains("started")) + { + app.Performance.StorageContractStarted(); + await FixedDurationDelay(); } await FixedShortDelay(); @@ -150,17 +223,17 @@ namespace AutoClient private async Task FixedDurationDelay() { - await Task.Delay(config.ContractDurationMinutes * 60 * 1000, ct); + await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token); } private async Task ExpiryTimeDelay() { - await Task.Delay(config.ContractExpiryMinutes * 60 * 1000, ct); + await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token); } private async Task FixedShortDelay() { - await Task.Delay(15 * 1000, ct); + await Task.Delay(15 * 1000, app.Cts.Token); } } } diff --git a/Tools/AutoClientCenter/AutoClientCenter.csproj b/Tools/AutoClientCenter/AutoClientCenter.csproj deleted file mode 100644 index d5e2255..0000000 --- a/Tools/AutoClientCenter/AutoClientCenter.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net7.0 - enable - enable - 80f55fc5-9f8f-4bb4-89bc-314ed7379c5f - Linux - - - - - - - - diff --git a/Tools/AutoClientCenter/AutoClientCenter.csproj.user b/Tools/AutoClientCenter/AutoClientCenter.csproj.user deleted file mode 100644 index db3a939..0000000 --- a/Tools/AutoClientCenter/AutoClientCenter.csproj.user +++ /dev/null @@ -1,9 +0,0 @@ - - - - IIS Express - - - ProjectDebugger - - \ No newline at end of file diff --git a/Tools/AutoClientCenter/AutoClientCenter.http b/Tools/AutoClientCenter/AutoClientCenter.http deleted file mode 100644 index 251d650..0000000 --- a/Tools/AutoClientCenter/AutoClientCenter.http +++ /dev/null @@ -1,6 +0,0 @@ -@AutoClientCenter_HostAddress = http://localhost:5185 - -GET {{AutoClientCenter_HostAddress}}/weatherforecast/ -Accept: application/json - -### diff --git a/Tools/AutoClientCenter/Controllers/ConfigController.cs b/Tools/AutoClientCenter/Controllers/ConfigController.cs deleted file mode 100644 index e259ae5..0000000 --- a/Tools/AutoClientCenter/Controllers/ConfigController.cs +++ /dev/null @@ -1,28 +0,0 @@ -using Microsoft.AspNetCore.Mvc; - -namespace AutoClientCenter.Controllers -{ - [ApiController] - [Route("[controller]")] - public class ConfigController : ControllerBase - { - private readonly ITaskService taskService; - - public ConfigController(ITaskService taskService) - { - this.taskService = taskService; - } - - [HttpGet("Stats")] - public AcStats Get() - { - return taskService.GetStats(); - } - - [HttpPost("Set")] - public void Post([FromBody] AcTasks tasks) - { - taskService.SetConfig(tasks); - } - } -} diff --git a/Tools/AutoClientCenter/Controllers/TasksController.cs b/Tools/AutoClientCenter/Controllers/TasksController.cs deleted file mode 100644 index 9b5e184..0000000 --- a/Tools/AutoClientCenter/Controllers/TasksController.cs +++ /dev/null @@ -1,35 +0,0 @@ -using Microsoft.AspNetCore.Mvc; - -namespace AutoClientCenter.Controllers -{ - [ApiController] - [Route("[controller]")] - public class TasksController : ControllerBase - { - private readonly ITaskService taskService; - private static readonly object processLock = new object(); - - public TasksController(ITaskService taskService) - { - this.taskService = taskService; - } - - [HttpGet] - public AcTasks Get() - { - return taskService.GetTasks(); - } - - [HttpPost("Results")] - public void Post([FromBody] AcTaskStep[] taskSteps) - { - Task.Run(() => - { - lock (processLock) - { - taskService.ProcessResults(taskSteps); - } - }); - } - } -} diff --git a/Tools/AutoClientCenter/Dockerfile b/Tools/AutoClientCenter/Dockerfile deleted file mode 100644 index 9fa6897..0000000 --- a/Tools/AutoClientCenter/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. - -FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base -USER app -WORKDIR /app - -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build -ARG BUILD_CONFIGURATION=Release -WORKDIR /src -COPY ["AutoClientCenter/AutoClientCenter.csproj", "AutoClientCenter/"] -RUN dotnet restore "./AutoClientCenter/AutoClientCenter.csproj" -COPY . . -WORKDIR "/src/AutoClientCenter" -RUN dotnet build "./AutoClientCenter.csproj" -c $BUILD_CONFIGURATION -o /app/build - -FROM build AS publish -ARG BUILD_CONFIGURATION=Release -RUN dotnet publish "./AutoClientCenter.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false - -FROM base AS final -WORKDIR /app -COPY --from=publish /app/publish . -ENTRYPOINT ["dotnet", "AutoClientCenter.dll"] \ No newline at end of file diff --git a/Tools/AutoClientCenter/Model.cs b/Tools/AutoClientCenter/Model.cs deleted file mode 100644 index 9f3a8c9..0000000 --- a/Tools/AutoClientCenter/Model.cs +++ /dev/null @@ -1,60 +0,0 @@ -namespace AutoClientCenter -{ - public class AcTasks - { - public int StartTaskEverySeconds { get; set; } - public AcTask[] Tasks { get; set; } = Array.Empty(); - } - - public class AcTask - { - public int ChanceWeight { get; set; } - public AcTaskStep[] Steps { get; set; } = Array.Empty(); - } - - public class AcTaskStep - { - public string Id { get; set; } = string.Empty; - public AcUploadStep? UploadStep { get; set; } - public AcStoreStep? StoreStep { get; set; } - public AcDownloadStep? DownloadStep { get; set; } - public string? ResultErrorMsg { get; set; } - } - - public class AcUploadStep - { - public long SizeInBytes { get; set; } - public string? ResultCid { get; set; } - } - - public class AcStoreStep - { - public int ContractDurationMinutes { get; set; } - public int ContractExpiryMinutes { get; set; } - public int NumHosts { get; set; } - public int HostTolerance { get; set; } - public int Price { get; set; } - public int RequiredCollateral { get; set; } - public string? ResultPurchaseId { get; set; } - public string? ResultOriginalCid { get; set; } - public string? ResultEncodedCid { get; set; } - } - - public class AcDownloadStep - { - public string Cid { get; set; } = string.Empty; - public long ResultDownloadTimeMilliseconds { get; set; } - } - - public class AcStats - { - public DateTime ServiceStartUtc { get; set; } = DateTime.MinValue; - public int TotalUploads { get; set; } - public int TotalUploadsFailed { get; set; } - public int TotalDownloads { get; set; } - public long[] DownloadTimesMillisecondsPerKb { get; set; } = Array.Empty(); - public int TotalDownloadsFailed { get; set; } - public int TotalContractsStarted { get; set; } - public int TotalContractStartsFailed { get; set; } - } -} diff --git a/Tools/AutoClientCenter/Program.cs b/Tools/AutoClientCenter/Program.cs deleted file mode 100644 index 2d1f1fb..0000000 --- a/Tools/AutoClientCenter/Program.cs +++ /dev/null @@ -1,43 +0,0 @@ - -namespace AutoClientCenter -{ - public class Program - { - public static void Main(string[] args) - { - var builder = WebApplication.CreateBuilder(args); - - var listenPort = Environment.GetEnvironmentVariable("APIPORT"); - if (string.IsNullOrEmpty(listenPort)) listenPort = "31090"; - - builder.WebHost.ConfigureKestrel((context, options) => - { - options.ListenAnyIP(Convert.ToInt32(listenPort)); - }); - - builder.Services.AddSingleton(new TaskService()); - builder.Services.AddControllers(); - builder.Services.AddEndpointsApiExplorer(); - builder.Services.AddSwaggerGen(); - - var app = builder.Build(); - - if (app.Environment.IsDevelopment()) - { - app.UseSwagger(); - app.UseSwaggerUI(); - } - - app.UseHttpsRedirection(); - - app.UseAuthorization(); - - - app.MapControllers(); - - Console.WriteLine("AutoClientCenter listening on port " + listenPort); - - app.Run(); - } - } -} diff --git a/Tools/AutoClientCenter/Properties/launchSettings.json b/Tools/AutoClientCenter/Properties/launchSettings.json deleted file mode 100644 index 4f58636..0000000 --- a/Tools/AutoClientCenter/Properties/launchSettings.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "profiles": { - "http": { - "commandName": "Project", - "launchBrowser": true, - "launchUrl": "swagger", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "dotnetRunMessages": true, - "applicationUrl": "http://localhost:5185" - }, - "https": { - "commandName": "Project", - "launchBrowser": true, - "launchUrl": "swagger", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "dotnetRunMessages": true, - "applicationUrl": "https://localhost:7077;http://localhost:5185" - }, - "IIS Express": { - "commandName": "IISExpress", - "launchBrowser": true, - "launchUrl": "swagger", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "Container (Dockerfile)": { - "commandName": "Docker", - "launchBrowser": true, - "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}/swagger", - "environmentVariables": { - "ASPNETCORE_HTTPS_PORTS": "8081", - "ASPNETCORE_HTTP_PORTS": "8080" - }, - "publishAllPorts": true, - "useSSL": true - } - }, - "$schema": "http://json.schemastore.org/launchsettings.json", - "iisSettings": { - "windowsAuthentication": false, - "anonymousAuthentication": true, - "iisExpress": { - "applicationUrl": "http://localhost:50475", - "sslPort": 44395 - } - } -} \ No newline at end of file diff --git a/Tools/AutoClientCenter/TaskService.cs b/Tools/AutoClientCenter/TaskService.cs deleted file mode 100644 index fc27a5b..0000000 --- a/Tools/AutoClientCenter/TaskService.cs +++ /dev/null @@ -1,116 +0,0 @@ -namespace AutoClientCenter -{ - public interface ITaskService - { - AcStats GetStats(); - AcTasks GetTasks(); - void ProcessResults(AcTaskStep[] taskSteps); - void SetConfig(AcTasks tasks); - } - - public class TaskService : ITaskService - { - private readonly CidRepo cidRepo = new CidRepo(); - private readonly List downloadTimes = new List(); - private readonly AcStats stats = new AcStats - { - ServiceStartUtc = DateTime.UtcNow, - }; - - private AcTasks tasks = new AcTasks - { - StartTaskEverySeconds = Convert.ToInt32(TimeSpan.FromHours(8).TotalSeconds), - Tasks = Array.Empty() - }; - - public AcStats GetStats() - { - stats.DownloadTimesMillisecondsPerKb = downloadTimes.ToArray(); - return stats; - } - - public AcTasks GetTasks() - { - foreach (var task in tasks.Tasks) - { - foreach (var step in task.Steps) - { - if (step.DownloadStep != null) cidRepo.Assign(step.DownloadStep); - } - } - return tasks; - } - - public void SetConfig(AcTasks newTasks) - { - if (newTasks.StartTaskEverySeconds < (10 * 60)) return; - foreach (var task in newTasks.Tasks) - { - if (task.ChanceWeight < 1) return; - foreach (var step in task.Steps) - { - if (string.IsNullOrWhiteSpace(step.Id)) return; - if (step.UploadStep == null && step.StoreStep == null && step.DownloadStep == null) return; - } - } - - tasks = newTasks; - } - - public void ProcessResults(AcTaskStep[] taskSteps) - { - foreach (var step in taskSteps) ProcessResults(step); - } - - private void ProcessResults(AcTaskStep step) - { - ProcessResult(step.UploadStep); - ProcessResult(step.StoreStep); - ProcessResult(step.DownloadStep); - } - - private void ProcessResult(AcUploadStep? uploadStep) - { - if (uploadStep == null) return; - - if (string.IsNullOrWhiteSpace(uploadStep.ResultCid)) - { - stats.TotalUploadsFailed++; - } - else - { - stats.TotalUploads++; - cidRepo.Add(uploadStep.ResultCid, uploadStep.SizeInBytes); - } - } - - private void ProcessResult(AcStoreStep? storeStep) - { - if (storeStep == null) return; - - if (string.IsNullOrWhiteSpace(storeStep.ResultOriginalCid) || - string.IsNullOrWhiteSpace(storeStep.ResultEncodedCid) || - string.IsNullOrWhiteSpace(storeStep.ResultPurchaseId)) - { - stats.TotalContractStartsFailed++; - } - else - { - stats.TotalContractsStarted++; - cidRepo.AddEncoded(storeStep.ResultOriginalCid, storeStep.ResultEncodedCid); - } - } - - private void ProcessResult(AcDownloadStep? downloadStep) - { - if (downloadStep == null) return; - - var kbs = cidRepo.GetSizeKbsForCid(downloadStep.Cid); - if (kbs == null) return; - var milliseconds = downloadStep.ResultDownloadTimeMilliseconds; - - var millisecondsPerKb = milliseconds / kbs.Value; - downloadTimes.Add(millisecondsPerKb); - } - } -} diff --git a/Tools/AutoClientCenter/appsettings.Development.json b/Tools/AutoClientCenter/appsettings.Development.json deleted file mode 100644 index 0c208ae..0000000 --- a/Tools/AutoClientCenter/appsettings.Development.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} diff --git a/Tools/AutoClientCenter/appsettings.json b/Tools/AutoClientCenter/appsettings.json deleted file mode 100644 index 10f68b8..0000000 --- a/Tools/AutoClientCenter/appsettings.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "AllowedHosts": "*" -} diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 5da95e4..6041a7c 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -76,8 +76,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoClientCenter", "Tools\AutoClientCenter\AutoClientCenter.csproj", "{9290DA91-F120-4DBC-94B7-CFC3CE41AF4A}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -204,10 +202,6 @@ Global {004614DF-1C65-45E3-882D-59AE44282573}.Debug|Any CPU.Build.0 = Debug|Any CPU {004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.ActiveCfg = Release|Any CPU {004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.Build.0 = Release|Any CPU - {9290DA91-F120-4DBC-94B7-CFC3CE41AF4A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9290DA91-F120-4DBC-94B7-CFC3CE41AF4A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9290DA91-F120-4DBC-94B7-CFC3CE41AF4A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9290DA91-F120-4DBC-94B7-CFC3CE41AF4A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -243,7 +237,6 @@ Global {870DDFBE-D7ED-4196-9681-13CA947BDEA6} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} {C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} {004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} - {9290DA91-F120-4DBC-94B7-CFC3CE41AF4A} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}