status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/better-autoclient'

This commit is contained in:
benbierens 2024-09-12 14:42:42 +02:00
parent a02d9558e5 a41272f160
commit 75fcc68caf
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
22 changed files with 352 additions and 555 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.github/workflows/docker-autoclientcenter.yml vendored
View File

@ -1,24 +0,0 @@
name: Docker - AutoClientCenter
on:
push:
branches:
- master
tags:
- 'v*.*.*'
paths:
- 'Tools/AutoClientCenter/**'
- '!Tools/AutoClientCenter/Dockerfile'
- .github/workflows/docker-autoclientcenter.yml
- .github/workflows/docker-reusable.yml
workflow_dispatch:
jobs:
build-and-push:
name: Build and Push
uses: ./.github/workflows/docker-reusable.yml
with:
docker_file: Tools/AutoClientCenter/Dockerfile
docker_repo: codexstorage/codex-autoclientcenter
secrets: inherit

36
Tools/AutoClient/App.cs Normal file
View File

@ -0,0 +1,36 @@
using Logging;
namespace AutoClient
{
public class App
{
public App(Configuration config)
{
Config = config;
Log = new LogSplitter(
new FileLog(Path.Combine(config.LogPath, "autoclient")),
new ConsoleLog()
);
Generator = CreateGenerator();
CidRepo = new CidRepo(config);
}
public Configuration Config { get; }
public ILog Log { get; }
public IFileGenerator Generator { get; }
public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
public CidRepo CidRepo { get; }
public Performance Performance { get; } = new Performance();
private IFileGenerator CreateGenerator()
{
if (Config.FileSizeMb > 0)
{
return new RandomFileGenerator(Config, Log);
}
return new ImageGenerator(Log);
}
}
}

35
Tools/AutoClientCenter/CidRepo.cs → Tools/AutoClient/CidRepo.cs
View File

@ -1,16 +1,22 @@
namespace AutoClientCenter
namespace AutoClient
{
public class CidRepo
{
private readonly Random random = new Random();
private readonly object _lock = new object();
private readonly List<CidEntry> entries = new List<CidEntry>();
private readonly Configuration config;
public void Add(string cid, long knownSize)
public CidRepo(Configuration config)
{
this.config = config;
}
public void Add(string nodeId, string cid, long knownSize)
{
lock (_lock)
{
entries.Add(new CidEntry(cid, knownSize));
entries.Add(new CidEntry(nodeId, cid, knownSize));
}
}
@ -25,31 +31,32 @@
}
}
public void Assign(AcDownloadStep downloadStep)
public string? GetForeignCid(string myNodeId)
{
lock (_lock)
{
while (true)
{
if (!entries.Any()) return;
if (!entries.Any()) return null;
var available = entries.Where(e => e.NodeId != myNodeId).ToArray();
if (!available.Any()) return null;
var i = random.Next(0, entries.Count);
var entry = entries[i];
var i = random.Next(0, available.Length);
var entry = available[i];
if (entry.CreatedUtc < (DateTime.UtcNow + TimeSpan.FromHours(18)))
if (entry.CreatedUtc < (DateTime.UtcNow + TimeSpan.FromMinutes(config.ContractDurationMinutes)))
{
entries.RemoveAt(i);
entries.Remove(entry);
}
else
{
downloadStep.Cid = entry.Cid;
return;
return entry.Cid;
}
}
}
}
public long? GetSizeKbsForCid(string cid)
public long? GetSizeForCid(string cid)
{
lock (_lock)
{
@ -62,12 +69,14 @@
public class CidEntry
{
public CidEntry(string cid, long knownSize)
public CidEntry(string nodeId, string cid, long knownSize)
{
NodeId = nodeId;
Cid = cid;
KnownSize = knownSize;
}
public string NodeId { get; }
public string Cid { get; }
public string Encoded { get; set; } = string.Empty;
public long KnownSize { get; }

57
Tools/AutoClient/CodexUser.cs Normal file
View File

@ -0,0 +1,57 @@
using CodexOpenApi;
using Logging;
using Utils;
namespace AutoClient
{
public class CodexUser
{
private readonly App app;
private readonly CodexApi codex;
private readonly HttpClient client;
private readonly Address address;
private readonly List<Purchaser> purchasers = new List<Purchaser>();
private Task starterTask = Task.CompletedTask;
private readonly string nodeId = Guid.NewGuid().ToString();
public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
{
this.app = app;
this.codex = codex;
this.client = client;
this.address = address;
}
public void Start(int index)
{
for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
{
purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
}
var delayPerPurchaser =
TimeSpan.FromSeconds(10 * index) +
TimeSpan.FromMinutes(app.Config.ContractDurationMinutes) / app.Config.NumConcurrentPurchases;
starterTask = Task.Run(() => StartPurchasers(delayPerPurchaser));
}
private async Task StartPurchasers(TimeSpan delayPerPurchaser)
{
foreach (var purchaser in purchasers)
{
purchaser.Start();
await Task.Delay(delayPerPurchaser);
}
}
public void Stop()
{
starterTask.Wait();
foreach (var purchaser in purchasers)
{
purchaser.Stop();
}
}
}
}

7
Tools/AutoClient/Configuration.cs
View File

@ -4,11 +4,8 @@ namespace AutoClient
{
public class Configuration
{
[Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default 'http://localhost')")]
public string CodexHost { get; set; } = "http://localhost";
[Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")]
public int CodexPort { get; set; } = 8080;
[Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
public string CodexEndpoints { get; set; } = "http://localhost:8080";
[Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
public string DataPath { get; set; } = "datapath";

6
Tools/AutoClient/ImageGenerator.cs
View File

@ -11,16 +11,16 @@ namespace AutoClient
public class ImageGenerator : IFileGenerator
{
private LogSplitter log;
private readonly ILog log;
public ImageGenerator(LogSplitter log)
public ImageGenerator(ILog log)
{
this.log = log;
}
public async Task<string> Generate()
{
log.Log("Fetching random image from picsum.photos...");
log.Debug("Fetching random image from picsum.photos...");
var httpClient = new HttpClient();
var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160");

45
Tools/AutoClient/Performance.cs Normal file
View File

@ -0,0 +1,45 @@
namespace AutoClient
{
public class Performance
{
internal void DownloadFailed(Exception ex)
{
throw new NotImplementedException();
}
internal void DownloadSuccessful(long? size, TimeSpan time)
{
throw new NotImplementedException();
}
internal void StorageContractCancelled()
{
throw new NotImplementedException();
}
internal void StorageContractErrored(string error)
{
throw new NotImplementedException();
}
internal void StorageContractFinished()
{
throw new NotImplementedException();
}
internal void StorageContractStarted()
{
throw new NotImplementedException();
}
internal void UploadFailed(Exception exc)
{
throw new NotImplementedException();
}
internal void UploadSuccessful(long length, TimeSpan time)
{
throw new NotImplementedException();
}
}
}

106
Tools/AutoClient/Program.cs
View File

@ -1,16 +1,20 @@
using ArgsUniform;
using AutoClient;
using CodexOpenApi;
using Core;
using Logging;
using Utils;
public static class Program
public class Program
{
private readonly App app;
public Program(Configuration config)
{
app = new App(config);
}
public static async Task Main(string[] args)
{
var cts = new CancellationTokenSource();
var cancellationToken = cts.Token;
Console.CancelKeyPress += (sender, args) => cts.Cancel();
var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
@ -21,58 +25,70 @@ public static class Program
throw new Exception("Number of concurrent purchases must be > 0");
}
var log = new LogSplitter(
new FileLog(Path.Combine(config.LogPath, "autoclient")),
new ConsoleLog()
);
var p = new Program(config);
await p.Run();
}
public async Task Run()
{
var codexUsers = await CreateUsers();
var i = 0;
foreach (var user in codexUsers)
{
user.Start(i);
i++;
}
app.Cts.Token.WaitHandle.WaitOne();
foreach (var user in codexUsers) user.Stop();
app.Log.Log("Done");
}
private async Task<CodexUser[]> CreateUsers()
{
var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
var result = new List<CodexUser>();
foreach (var e in endpointStrs)
{
result.Add(await CreateUser(e));
}
return result.ToArray();
}
private async Task<CodexUser> CreateUser(string endpoint)
{
var splitIndex = endpoint.LastIndexOf(':');
var host = endpoint.Substring(0, splitIndex);
var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
var address = new Address(
host: config.CodexHost,
port: config.CodexPort
host: host,
port: port
);
log.Log($"Start. Address: {address}");
var generator = CreateGenerator(config, log);
var client = new HttpClient();
var codex = new CodexApi(client);
codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
await CheckCodex(codex, log);
app.Log.Log($"Checking Codex at {address}...");
await CheckCodex(codex);
app.Log.Log("OK");
var purchasers = new List<Purchaser>();
for (var i = 0; i < config.NumConcurrentPurchases; i++)
{
purchasers.Add(
new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, generator, cancellationToken)
);
}
var delayPerPurchaser = TimeSpan.FromMinutes(config.ContractDurationMinutes) / config.NumConcurrentPurchases;
foreach (var purchaser in purchasers)
{
purchaser.Start();
await Task.Delay(delayPerPurchaser);
}
cancellationToken.WaitHandle.WaitOne();
log.Log("Done.");
return new CodexUser(
app,
codex,
client,
address
);
}
private static IFileGenerator CreateGenerator(Configuration config, LogSplitter log)
private async Task CheckCodex(CodexApi codex)
{
if (config.FileSizeMb > 0)
{
return new RandomFileGenerator(config, log);
}
return new ImageGenerator(log);
}
private static async Task CheckCodex(CodexApi codex, ILog log)
{
log.Log("Checking Codex...");
try
{
var info = await codex.GetDebugInfoAsync();
@ -80,7 +96,7 @@ public static class Program
}
catch (Exception ex)
{
log.Log($"Codex not OK: {ex}");
app.Log.Error($"Codex not OK: {ex}");
throw;
}
}

179
Tools/AutoClient/Purchaser.cs
View File

@ -8,36 +8,76 @@ namespace AutoClient
{
public class Purchaser
{
private readonly App app;
private readonly string nodeId;
private readonly ILog log;
private readonly HttpClient client;
private readonly Address address;
private readonly CodexApi codex;
private readonly Configuration config;
private readonly IFileGenerator generator;
private readonly CancellationToken ct;
private Task workerTask = Task.CompletedTask;
public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, IFileGenerator generator, CancellationToken ct)
public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
{
this.app = app;
this.nodeId = nodeId;
this.log = log;
this.client = client;
this.address = address;
this.codex = codex;
this.config = config;
this.generator = generator;
this.ct = ct;
}
public void Start()
{
Task.Run(Worker);
workerTask = Task.Run(Worker);
}
public void Stop()
{
workerTask.Wait();
}
private async Task Worker()
{
while (!ct.IsCancellationRequested)
log.Log("Worker started.");
while (!app.Cts.Token.IsCancellationRequested)
{
var pid = await StartNewPurchase();
await WaitTillFinished(pid);
try
{
var pid = await StartNewPurchase();
await WaitTillFinished(pid);
await DownloadForeignCid();
}
catch (Exception ex)
{
log.Error("Worker failed with: " + ex);
await Task.Delay(TimeSpan.FromHours(6));
}
}
}
private async Task DownloadForeignCid()
{
var cid = app.CidRepo.GetForeignCid(nodeId);
if (cid == null) return;
var size = app.CidRepo.GetSizeForCid(cid);
if (cid == null) return;
try
{
var sw = System.Diagnostics.Stopwatch.StartNew();
var filename = Guid.NewGuid().ToString().ToLowerInvariant();
{
using var fileStream = File.OpenWrite(filename);
var fileResponse = await codex.DownloadNetworkAsync(cid);
fileResponse.Stream.CopyTo(fileStream);
}
var time = sw.Elapsed;
File.Delete(filename);
app.Performance.DownloadSuccessful(size, time);
}
catch (Exception ex)
{
app.Performance.DownloadFailed(ex);
}
}
@ -50,72 +90,96 @@ namespace AutoClient
private async Task<string> CreateFile()
{
return await generator.Generate();
return await app.Generator.Generate();
}
private async Task<ContentId> UploadFile(string filename)
{
// Copied from CodexNode :/
using var fileStream = File.OpenRead(filename);
try
{
var info = new FileInfo(filename);
var sw = System.Diagnostics.Stopwatch.StartNew();
var cid = await UploadStream(fileStream);
var time = sw.Elapsed;
app.Performance.UploadSuccessful(info.Length, time);
app.CidRepo.Add(nodeId, cid.Id, info.Length);
return cid;
}
catch (Exception exc)
{
app.Performance.UploadFailed(exc);
throw;
}
}
log.Log($"Uploading file {filename}...");
var response = await codex.UploadAsync(fileStream, ct);
private async Task<ContentId> UploadStream(FileStream fileStream)
{
log.Debug($"Uploading file...");
var response = await codex.UploadAsync(fileStream, app.Cts.Token);
if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
log.Log($"Uploaded file. Received contentId: '{response}'.");
log.Debug($"Uploaded file. Received contentId: '{response}'.");
return new ContentId(response);
}
private async Task<string> RequestStorage(ContentId cid)
{
log.Log("Requesting storage for " + cid.Id);
log.Debug("Requesting storage for " + cid.Id);
var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
{
Collateral = config.RequiredCollateral.ToString(),
Duration = (config.ContractDurationMinutes * 60).ToString(),
Expiry = (config.ContractExpiryMinutes * 60).ToString(),
Nodes = config.NumHosts,
Reward = config.Price.ToString(),
Collateral = app.Config.RequiredCollateral.ToString(),
Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
Nodes = app.Config.NumHosts,
Reward = app.Config.Price.ToString(),
ProofProbability = "15",
Tolerance = config.HostTolerance
}, ct);
Tolerance = app.Config.HostTolerance
}, app.Cts.Token);
log.Log("Purchase ID: " + result);
log.Debug("Purchase ID: " + result);
var encoded = await GetEncodedCid(result);
app.CidRepo.AddEncoded(cid.Id, encoded);
return result;
}
private async Task<string?> GetPurchaseState(string pid)
private async Task<string> GetEncodedCid(string pid)
{
try
{
// openapi still don't match code.
var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
if (string.IsNullOrEmpty(str)) return null;
var sp = JsonConvert.DeserializeObject<StoragePurchase>(str)!;
log.Log($"Purchase {pid} is {sp.State}");
if (!string.IsNullOrEmpty(sp.Error)) log.Log($"Purchase {pid} error is {sp.Error}");
return sp.State;
var sp = await GetStoragePurchase(pid)!;
return sp.Request.Content.Cid;
}
catch
catch (Exception ex)
{
return null;
log.Error(ex.ToString());
throw;
}
}
private async Task<StoragePurchase?> GetStoragePurchase(string pid)
{
// openapi still don't match code.
var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
if (string.IsNullOrEmpty(str)) return null;
return JsonConvert.DeserializeObject<StoragePurchase>(str);
}
private async Task WaitTillFinished(string pid)
{
log.Log("Waiting...");
try
{
var emptyResponseTolerance = 10;
while (true)
while (!app.Cts.Token.IsCancellationRequested)
{
var status = (await GetPurchaseState(pid))?.ToLowerInvariant();
if (string.IsNullOrEmpty(status))
var purchase = await GetStoragePurchase(pid);
if (purchase == null)
{
await FixedShortDelay();
emptyResponseTolerance--;
if (emptyResponseTolerance == 0)
{
@ -123,19 +187,28 @@ namespace AutoClient
await ExpiryTimeDelay();
return;
}
continue;
}
else
var status = purchase.State.ToLowerInvariant();
if (status.Contains("cancel"))
{
if (status.Contains("cancel") ||
status.Contains("error") ||
status.Contains("finished"))
{
return;
}
if (status.Contains("started"))
{
await FixedDurationDelay();
}
app.Performance.StorageContractCancelled();
return;
}
if (status.Contains("error"))
{
app.Performance.StorageContractErrored(purchase.Error);
return;
}
if (status.Contains("finished"))
{
app.Performance.StorageContractFinished();
return;
}
if (status.Contains("started"))
{
app.Performance.StorageContractStarted();
await FixedDurationDelay();
}
await FixedShortDelay();
@ -150,17 +223,17 @@ namespace AutoClient
private async Task FixedDurationDelay()
{
await Task.Delay(config.ContractDurationMinutes * 60 * 1000, ct);
await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
}
private async Task ExpiryTimeDelay()
{
await Task.Delay(config.ContractExpiryMinutes * 60 * 1000, ct);
await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
}
private async Task FixedShortDelay()
{
await Task.Delay(15 * 1000, ct);
await Task.Delay(15 * 1000, app.Cts.Token);
}
}
}

16
Tools/AutoClientCenter/AutoClientCenter.csproj
View File

@ -1,16 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<UserSecretsId>80f55fc5-9f8f-4bb4-89bc-314ed7379c5f</UserSecretsId>
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.20.1" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
</ItemGroup>
</Project>

9
Tools/AutoClientCenter/AutoClientCenter.csproj.user
View File

@ -1,9 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<ActiveDebugProfile>IIS Express</ActiveDebugProfile>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
<DebuggerFlavor>ProjectDebugger</DebuggerFlavor>
</PropertyGroup>
</Project>

6
Tools/AutoClientCenter/AutoClientCenter.http
View File

@ -1,6 +0,0 @@
@AutoClientCenter_HostAddress = http://localhost:5185
GET {{AutoClientCenter_HostAddress}}/weatherforecast/
Accept: application/json
###

28
Tools/AutoClientCenter/Controllers/ConfigController.cs
View File

@ -1,28 +0,0 @@
using Microsoft.AspNetCore.Mvc;
namespace AutoClientCenter.Controllers
{
[ApiController]
[Route("[controller]")]
public class ConfigController : ControllerBase
{
private readonly ITaskService taskService;
public ConfigController(ITaskService taskService)
{
this.taskService = taskService;
}
[HttpGet("Stats")]
public AcStats Get()
{
return taskService.GetStats();
}
[HttpPost("Set")]
public void Post([FromBody] AcTasks tasks)
{
taskService.SetConfig(tasks);
}
}
}

35
Tools/AutoClientCenter/Controllers/TasksController.cs
View File

@ -1,35 +0,0 @@
using Microsoft.AspNetCore.Mvc;
namespace AutoClientCenter.Controllers
{
[ApiController]
[Route("[controller]")]
public class TasksController : ControllerBase
{
private readonly ITaskService taskService;
private static readonly object processLock = new object();
public TasksController(ITaskService taskService)
{
this.taskService = taskService;
}
[HttpGet]
public AcTasks Get()
{
return taskService.GetTasks();
}
[HttpPost("Results")]
public void Post([FromBody] AcTaskStep[] taskSteps)
{
Task.Run(() =>
{
lock (processLock)
{
taskService.ProcessResults(taskSteps);
}
});
}
}
}

23
Tools/AutoClientCenter/Dockerfile
View File

@ -1,23 +0,0 @@
#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging.
FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
USER app
WORKDIR /app
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
ARG BUILD_CONFIGURATION=Release
WORKDIR /src
COPY ["AutoClientCenter/AutoClientCenter.csproj", "AutoClientCenter/"]
RUN dotnet restore "./AutoClientCenter/AutoClientCenter.csproj"
COPY . .
WORKDIR "/src/AutoClientCenter"
RUN dotnet build "./AutoClientCenter.csproj" -c $BUILD_CONFIGURATION -o /app/build
FROM build AS publish
ARG BUILD_CONFIGURATION=Release
RUN dotnet publish "./AutoClientCenter.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
FROM base AS final
WORKDIR /app
COPY --from=publish /app/publish .
ENTRYPOINT ["dotnet", "AutoClientCenter.dll"]

60
Tools/AutoClientCenter/Model.cs
View File

@ -1,60 +0,0 @@
namespace AutoClientCenter
{
public class AcTasks
{
public int StartTaskEverySeconds { get; set; }
public AcTask[] Tasks { get; set; } = Array.Empty<AcTask>();
}
public class AcTask
{
public int ChanceWeight { get; set; }
public AcTaskStep[] Steps { get; set; } = Array.Empty<AcTaskStep>();
}
public class AcTaskStep
{
public string Id { get; set; } = string.Empty;
public AcUploadStep? UploadStep { get; set; }
public AcStoreStep? StoreStep { get; set; }
public AcDownloadStep? DownloadStep { get; set; }
public string? ResultErrorMsg { get; set; }
}
public class AcUploadStep
{
public long SizeInBytes { get; set; }
public string? ResultCid { get; set; }
}
public class AcStoreStep
{
public int ContractDurationMinutes { get; set; }
public int ContractExpiryMinutes { get; set; }
public int NumHosts { get; set; }
public int HostTolerance { get; set; }
public int Price { get; set; }
public int RequiredCollateral { get; set; }
public string? ResultPurchaseId { get; set; }
public string? ResultOriginalCid { get; set; }
public string? ResultEncodedCid { get; set; }
}
public class AcDownloadStep
{
public string Cid { get; set; } = string.Empty;
public long ResultDownloadTimeMilliseconds { get; set; }
}
public class AcStats
{
public DateTime ServiceStartUtc { get; set; } = DateTime.MinValue;
public int TotalUploads { get; set; }
public int TotalUploadsFailed { get; set; }
public int TotalDownloads { get; set; }
public long[] DownloadTimesMillisecondsPerKb { get; set; } = Array.Empty<long>();
public int TotalDownloadsFailed { get; set; }
public int TotalContractsStarted { get; set; }
public int TotalContractStartsFailed { get; set; }
}
}

43
Tools/AutoClientCenter/Program.cs
View File

@ -1,43 +0,0 @@
namespace AutoClientCenter
{
public class Program
{
public static void Main(string[] args)
{
var builder = WebApplication.CreateBuilder(args);
var listenPort = Environment.GetEnvironmentVariable("APIPORT");
if (string.IsNullOrEmpty(listenPort)) listenPort = "31090";
builder.WebHost.ConfigureKestrel((context, options) =>
{
options.ListenAnyIP(Convert.ToInt32(listenPort));
});
builder.Services.AddSingleton<ITaskService>(new TaskService());
builder.Services.AddControllers();
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();
var app = builder.Build();
if (app.Environment.IsDevelopment())
{
app.UseSwagger();
app.UseSwaggerUI();
}
app.UseHttpsRedirection();
app.UseAuthorization();
app.MapControllers();
Console.WriteLine("AutoClientCenter listening on port " + listenPort);
app.Run();
}
}
}

52
Tools/AutoClientCenter/Properties/launchSettings.json
View File

@ -1,52 +0,0 @@
{
"profiles": {
"http": {
"commandName": "Project",
"launchBrowser": true,
"launchUrl": "swagger",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
},
"dotnetRunMessages": true,
"applicationUrl": "http://localhost:5185"
},
"https": {
"commandName": "Project",
"launchBrowser": true,
"launchUrl": "swagger",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
},
"dotnetRunMessages": true,
"applicationUrl": "https://localhost:7077;http://localhost:5185"
},
"IIS Express": {
"commandName": "IISExpress",
"launchBrowser": true,
"launchUrl": "swagger",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"Container (Dockerfile)": {
"commandName": "Docker",
"launchBrowser": true,
"launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}/swagger",
"environmentVariables": {
"ASPNETCORE_HTTPS_PORTS": "8081",
"ASPNETCORE_HTTP_PORTS": "8080"
},
"publishAllPorts": true,
"useSSL": true
}
},
"$schema": "http://json.schemastore.org/launchsettings.json",
"iisSettings": {
"windowsAuthentication": false,
"anonymousAuthentication": true,
"iisExpress": {
"applicationUrl": "http://localhost:50475",
"sslPort": 44395
}
}
}

116
Tools/AutoClientCenter/TaskService.cs
View File

@ -1,116 +0,0 @@
namespace AutoClientCenter
{
public interface ITaskService
{
AcStats GetStats();
AcTasks GetTasks();
void ProcessResults(AcTaskStep[] taskSteps);
void SetConfig(AcTasks tasks);
}
public class TaskService : ITaskService
{
private readonly CidRepo cidRepo = new CidRepo();
private readonly List<long> downloadTimes = new List<long>();
private readonly AcStats stats = new AcStats
{
ServiceStartUtc = DateTime.UtcNow,
};
private AcTasks tasks = new AcTasks
{
StartTaskEverySeconds = Convert.ToInt32(TimeSpan.FromHours(8).TotalSeconds),
Tasks = Array.Empty<AcTask>()
};
public AcStats GetStats()
{
stats.DownloadTimesMillisecondsPerKb = downloadTimes.ToArray();
return stats;
}
public AcTasks GetTasks()
{
foreach (var task in tasks.Tasks)
{
foreach (var step in task.Steps)
{
if (step.DownloadStep != null) cidRepo.Assign(step.DownloadStep);
}
}
return tasks;
}
public void SetConfig(AcTasks newTasks)
{
if (newTasks.StartTaskEverySeconds < (10 * 60)) return;
foreach (var task in newTasks.Tasks)
{
if (task.ChanceWeight < 1) return;
foreach (var step in task.Steps)
{
if (string.IsNullOrWhiteSpace(step.Id)) return;
if (step.UploadStep == null && step.StoreStep == null && step.DownloadStep == null) return;
}
}
tasks = newTasks;
}
public void ProcessResults(AcTaskStep[] taskSteps)
{
foreach (var step in taskSteps) ProcessResults(step);
}
private void ProcessResults(AcTaskStep step)
{
ProcessResult(step.UploadStep);
ProcessResult(step.StoreStep);
ProcessResult(step.DownloadStep);
}
private void ProcessResult(AcUploadStep? uploadStep)
{
if (uploadStep == null) return;
if (string.IsNullOrWhiteSpace(uploadStep.ResultCid))
{
stats.TotalUploadsFailed++;
}
else
{
stats.TotalUploads++;
cidRepo.Add(uploadStep.ResultCid, uploadStep.SizeInBytes);
}
}
private void ProcessResult(AcStoreStep? storeStep)
{
if (storeStep == null) return;
if (string.IsNullOrWhiteSpace(storeStep.ResultOriginalCid) ||
string.IsNullOrWhiteSpace(storeStep.ResultEncodedCid) ||
string.IsNullOrWhiteSpace(storeStep.ResultPurchaseId))
{
stats.TotalContractStartsFailed++;
}
else
{
stats.TotalContractsStarted++;
cidRepo.AddEncoded(storeStep.ResultOriginalCid, storeStep.ResultEncodedCid);
}
}
private void ProcessResult(AcDownloadStep? downloadStep)
{
if (downloadStep == null) return;
var kbs = cidRepo.GetSizeKbsForCid(downloadStep.Cid);
if (kbs == null) return;
var milliseconds = downloadStep.ResultDownloadTimeMilliseconds;
var millisecondsPerKb = milliseconds / kbs.Value;
downloadTimes.Add(millisecondsPerKb);
}
}
}

8
Tools/AutoClientCenter/appsettings.Development.json
View File

@ -1,8 +0,0 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
}
}

9
Tools/AutoClientCenter/appsettings.json
View File

@ -1,9 +0,0 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
},
"AllowedHosts": "*"
}

7
cs-codex-dist-testing.sln
View File

@ -76,8 +76,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoClientCenter", "Tools\AutoClientCenter\AutoClientCenter.csproj", "{9290DA91-F120-4DBC-94B7-CFC3CE41AF4A}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -204,10 +202,6 @@ Global
{004614DF-1C65-45E3-882D-59AE44282573}.Debug|Any CPU.Build.0 = Debug|Any CPU
{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.ActiveCfg = Release|Any CPU
{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.Build.0 = Release|Any CPU
{9290DA91-F120-4DBC-94B7-CFC3CE41AF4A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9290DA91-F120-4DBC-94B7-CFC3CE41AF4A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9290DA91-F120-4DBC-94B7-CFC3CE41AF4A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9290DA91-F120-4DBC-94B7-CFC3CE41AF4A}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -243,7 +237,6 @@ Global
{870DDFBE-D7ED-4196-9681-13CA947BDEA6} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
{004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
{9290DA91-F120-4DBC-94B7-CFC3CE41AF4A} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}