diff --git a/.github/workflows/docker-autoclient.yml b/.github/workflows/docker-autoclient.yml new file mode 100644 index 00000000..41d31018 --- /dev/null +++ b/.github/workflows/docker-autoclient.yml @@ -0,0 +1,26 @@ +name: Docker - AutoClient + +on: + push: + branches: + - master + tags: + - 'v*.*.*' + paths: + - 'Tools/AutoClient/**' + - '!Tools/AutoClient/docker/docker-compose.yaml' + - 'Framework/**' + - 'ProjectPlugins/**' + - .github/workflows/docker-autoclient.yml + - .github/workflows/docker-reusable.yml + workflow_dispatch: + +jobs: + build-and-push: + name: Build and Push + uses: ./.github/workflows/docker-reusable.yml + with: + docker_file: Tools/AutoClient/docker/Dockerfile + docker_repo: codexstorage/codex-autoclient + secrets: inherit + diff --git a/.gitignore b/.gitignore index c134feec..488beaf6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vs obj bin -.vscode \ No newline at end of file +.vscode +Tools/AutoClient/datapath diff --git a/Framework/FileUtils/FileManager.cs b/Framework/FileUtils/FileManager.cs index d96d2972..0d0fb694 100644 --- a/Framework/FileUtils/FileManager.cs +++ b/Framework/FileUtils/FileManager.cs @@ -70,16 +70,27 @@ namespace FileUtils public void ScopedFiles(Action action) { PushFileSet(); - action(); - PopFileSet(); + try + { + action(); + } + finally + { + PopFileSet(); + } } public T ScopedFiles(Func action) { PushFileSet(); - var result = action(); - PopFileSet(); - return result; + try + { + return action(); + } + finally + { + PopFileSet(); + } } private void PushFileSet() diff --git a/Tools/AutoClient/AutoClient.csproj b/Tools/AutoClient/AutoClient.csproj new file mode 100644 index 00000000..9340dc97 --- /dev/null +++ b/Tools/AutoClient/AutoClient.csproj @@ -0,0 +1,16 @@ + + + + Exe + net7.0 + enable + enable + + + + + + + + + diff --git a/Tools/AutoClient/Configuration.cs b/Tools/AutoClient/Configuration.cs new file mode 100644 index 00000000..a3808829 --- /dev/null +++ b/Tools/AutoClient/Configuration.cs @@ -0,0 +1,45 @@ +using ArgsUniform; + +namespace AutoClient +{ + public class Configuration + { + [Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default 'http://localhost')")] + public string CodexHost { get; set; } = "http://localhost"; + + [Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")] + public int CodexPort { get; set; } = 8080; + + [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")] + public string DataPath { get; set; } = "datapath"; + + [Uniform("purchases", "np", "PURCHASES", false, "Number of concurrent purchases.")] + public int NumConcurrentPurchases { get; set; } = 10; + + [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 30)")] + public int ContractDurationMinutes { get; set; } = 30; + + [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15)")] + public int ContractExpiryMinutes { get; set; } = 15; + + [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")] + public int NumHosts { get; set; } = 5; + + [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 2)")] + public int HostTolerance { get; set; } = 2; + + [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")] + public int Price { get; set; } = 10; + + [Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")] + public int RequiredCollateral { get; set; } = 1; + + public string LogPath + { + get + { + return Path.Combine(DataPath, "logs"); + } + } + } +} diff --git a/Tools/AutoClient/ImageGenerator.cs b/Tools/AutoClient/ImageGenerator.cs new file mode 100644 index 00000000..eeab0d32 --- /dev/null +++ b/Tools/AutoClient/ImageGenerator.cs @@ -0,0 +1,17 @@ +namespace AutoClient +{ + public class ImageGenerator + { + public async Task GenerateImage() + { + var httpClient = new HttpClient(); + var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160"); + + var filename = $"{Guid.NewGuid().ToString().ToLowerInvariant()}.jpg"; + using var file = File.OpenWrite(filename); + await thing.CopyToAsync(file); + + return filename; + } + } +} diff --git a/Tools/AutoClient/Program.cs b/Tools/AutoClient/Program.cs new file mode 100644 index 00000000..5eed206c --- /dev/null +++ b/Tools/AutoClient/Program.cs @@ -0,0 +1,80 @@ +using ArgsUniform; +using AutoClient; +using CodexOpenApi; +using Core; +using Logging; + +public static class Program +{ + public static async Task Main(string[] args) + { + var cts = new CancellationTokenSource(); + var cancellationToken = cts.Token; + Console.CancelKeyPress += (sender, args) => cts.Cancel(); + + var uniformArgs = new ArgsUniform(PrintHelp, args); + var config = uniformArgs.Parse(true); + + if (config.NumConcurrentPurchases < 1) + { + throw new Exception("Number of concurrent purchases must be > 0"); + } + + var log = new LogSplitter( + new FileLog(Path.Combine(config.LogPath, "autoclient")), + new ConsoleLog() + ); + + var address = new Utils.Address( + host: config.CodexHost, + port: config.CodexPort + ); + + log.Log($"Start. Address: {address}"); + + var imgGenerator = new ImageGenerator(); + + var client = new HttpClient(); + var codex = new CodexApi(client); + codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; + + await CheckCodex(codex, log); + + var purchasers = new List(); + for (var i = 0; i < config.NumConcurrentPurchases; i++) + { + purchasers.Add( + new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, cancellationToken, config, imgGenerator) + ); + } + + var delayPerPurchaser = TimeSpan.FromMinutes(config.ContractDurationMinutes) / config.NumConcurrentPurchases; + foreach (var purchaser in purchasers) + { + purchaser.Start(); + await Task.Delay(delayPerPurchaser); + } + + log.Log("Done."); + } + + private static async Task CheckCodex(CodexApi codex, ILog log) + { + log.Log("Checking Codex..."); + try + { + var info = await codex.GetDebugInfoAsync(); + if (string.IsNullOrEmpty(info.Id)) throw new Exception("Failed to fetch Codex node id"); + } + catch (Exception ex) + { + log.Log($"Codex not OK: {ex}"); + throw; + } + } + + private static void PrintHelp() + { + Console.WriteLine("Generates fake data and creates Codex storage contracts for it."); + } +} \ No newline at end of file diff --git a/Tools/AutoClient/Purchaser.cs b/Tools/AutoClient/Purchaser.cs new file mode 100644 index 00000000..3d03258d --- /dev/null +++ b/Tools/AutoClient/Purchaser.cs @@ -0,0 +1,166 @@ +using CodexOpenApi; +using CodexPlugin; +using Logging; +using Newtonsoft.Json; +using Utils; + +namespace AutoClient +{ + public class Purchaser + { + private readonly ILog log; + private readonly HttpClient client; + private readonly Address address; + private readonly CodexApi codex; + private readonly CancellationToken ct; + private readonly Configuration config; + private readonly ImageGenerator generator; + + public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, CancellationToken ct, Configuration config, ImageGenerator generator) + { + this.log = log; + this.client = client; + this.address = address; + this.codex = codex; + this.ct = ct; + this.config = config; + this.generator = generator; + } + + public void Start() + { + Task.Run(Worker); + } + + private async Task Worker() + { + while (!ct.IsCancellationRequested) + { + var pid = await StartNewPurchase(); + await WaitTillFinished(pid); + } + } + + private async Task StartNewPurchase() + { + var file = await CreateFile(); + var cid = await UploadFile(file); + return await RequestStorage(cid); + } + + private async Task CreateFile() + { + return await generator.GenerateImage(); + } + + private async Task UploadFile(string filename) + { + // Copied from CodexNode :/ + using var fileStream = File.OpenRead(filename); + + log.Log($"Uploading file {filename}..."); + var response = await codex.UploadAsync(fileStream, ct); + + if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response."); + if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block."); + + log.Log($"Uploaded file. Received contentId: '{response}'."); + return new ContentId(response); + } + + private async Task RequestStorage(ContentId cid) + { + log.Log("Requesting storage for " + cid.Id); + var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation() + { + Collateral = config.RequiredCollateral.ToString(), + Duration = (config.ContractDurationMinutes * 60).ToString(), + Expiry = (config.ContractExpiryMinutes * 60).ToString(), + Nodes = config.NumHosts, + Reward = config.Price.ToString(), + ProofProbability = "15", + Tolerance = config.HostTolerance + }, ct); + + log.Log("Purchase ID: " + result); + + return result; + } + + private async Task GetPurchaseState(string pid) + { + try + { + // openapi still don't match code. + var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}"); + if (string.IsNullOrEmpty(str)) return null; + var sp = JsonConvert.DeserializeObject(str)!; + log.Log($"Purchase {pid} is {sp.State}"); + if (!string.IsNullOrEmpty(sp.Error)) log.Log($"Purchase {pid} error is {sp.Error}"); + return sp.State; + } + catch + { + return null; + } + } + + private async Task WaitTillFinished(string pid) + { + log.Log("Waiting..."); + try + { + var emptyResponseTolerance = 10; + while (true) + { + var status = (await GetPurchaseState(pid))?.ToLowerInvariant(); + if (string.IsNullOrEmpty(status)) + { + emptyResponseTolerance--; + if (emptyResponseTolerance == 0) + { + log.Log("Received 10 empty responses. Stop tracking this purchase."); + await ExpiryTimeDelay(); + return; + } + } + else + { + if (status.Contains("cancel") || + status.Contains("error") || + status.Contains("finished")) + { + return; + } + if (status.Contains("started")) + { + await FixedDurationDelay(); + } + } + + await FixedShortDelay(); + } + } + catch (Exception ex) + { + log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time."); + await ExpiryTimeDelay(); + } + } + + private async Task FixedDurationDelay() + { + await Task.Delay(config.ContractDurationMinutes * 60 * 1000, ct); + } + + private async Task ExpiryTimeDelay() + { + await Task.Delay(config.ContractExpiryMinutes * 60 * 1000, ct); + } + + private async Task FixedShortDelay() + { + await Task.Delay(15 * 1000, ct); + } + } +} diff --git a/Tools/AutoClient/docker/Dockerfile b/Tools/AutoClient/docker/Dockerfile new file mode 100644 index 00000000..a1436c24 --- /dev/null +++ b/Tools/AutoClient/docker/Dockerfile @@ -0,0 +1,24 @@ +# Variables +ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0 +ARG IMAGE=${BUILDER} +ARG APP_HOME=/app + +# Build +FROM ${IMAGE} AS builder +ARG APP_HOME + +WORKDIR ${APP_HOME} +COPY ./Tools/AutoClient ./Tools/AutoClient +COPY ./Framework ./Framework +COPY ./ProjectPlugins ./ProjectPlugins +RUN dotnet restore Tools/AutoClient +RUN dotnet publish Tools/AutoClient -c Release -o out + +# Create +FROM ${IMAGE} +ARG APP_HOME +ENV APP_HOME=${APP_HOME} + +WORKDIR ${APP_HOME} +COPY --from=builder ${APP_HOME}/out . +CMD dotnet ${APP_HOME}/AutoClient.dll diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index beec9e6d..8c95ef09 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -66,6 +66,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution .editorconfig = .editorconfig EndProjectSection EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoClient", "Tools\AutoClient\AutoClient.csproj", "{73599F9C-98BB-4C6A-9D7D-7C50FBF2993B}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KeyMaker", "Tools\KeyMaker\KeyMaker.csproj", "{B57A4789-D8EF-42E0-8D20-581C4057FFD3}" EndProject Global @@ -174,6 +176,10 @@ Global {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Debug|Any CPU.Build.0 = Debug|Any CPU {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.ActiveCfg = Release|Any CPU {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.Build.0 = Release|Any CPU + {73599F9C-98BB-4C6A-9D7D-7C50FBF2993B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {73599F9C-98BB-4C6A-9D7D-7C50FBF2993B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {73599F9C-98BB-4C6A-9D7D-7C50FBF2993B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {73599F9C-98BB-4C6A-9D7D-7C50FBF2993B}.Release|Any CPU.Build.0 = Release|Any CPU {B57A4789-D8EF-42E0-8D20-581C4057FFD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B57A4789-D8EF-42E0-8D20-581C4057FFD3}.Debug|Any CPU.Build.0 = Debug|Any CPU {B57A4789-D8EF-42E0-8D20-581C4057FFD3}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -208,6 +214,7 @@ Global {F730DA73-1C92-4107-BCFB-D33759DAB0C3} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} {B07820C4-309F-4454-BCC1-1D4902C9C67B} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} {88C212E9-308A-46A4-BAAD-468E8EBD8EDF} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124} + {73599F9C-98BB-4C6A-9D7D-7C50FBF2993B} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} {B57A4789-D8EF-42E0-8D20-581C4057FFD3} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution