From 4c75cebcd6b0f15364ed8082a5362dcdcdbbb1c5 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Jun 2024 15:38:13 +0200 Subject: [PATCH] Implements autoclient with image generator --- Tools/AutoClient/Codex.cs | 118 ------------------------ Tools/AutoClient/Configuration.cs | 3 - Tools/AutoClient/ImageGenerator.cs | 17 ++++ Tools/AutoClient/Program.cs | 24 ++--- Tools/AutoClient/Runner.cs | 141 +++++++++++++++++++---------- 5 files changed, 121 insertions(+), 182 deletions(-) delete mode 100644 Tools/AutoClient/Codex.cs create mode 100644 Tools/AutoClient/ImageGenerator.cs diff --git a/Tools/AutoClient/Codex.cs b/Tools/AutoClient/Codex.cs deleted file mode 100644 index 8ecdf366..00000000 --- a/Tools/AutoClient/Codex.cs +++ /dev/null @@ -1,118 +0,0 @@ -using CodexOpenApi; -using CodexPlugin; -using Core; -using Utils; -using DebugInfo = CodexPlugin.DebugInfo; - -namespace AutoClient -{ - public class Codex - { - private readonly IPluginTools tools; - private readonly Address address; - private readonly Mapper mapper = new Mapper(); - - /// - /// This class was largely copied from CodexAccess in CodexPlugin. - /// Should really be generalized so CodexPlugin supports talking to custom Codex instances. - /// - public Codex(IPluginTools tools, Address address) - { - this.tools = tools; - this.address = address; - } - - public DebugInfo GetDebugInfo() - { - return mapper.Map(OnCodex(api => api.GetDebugInfoAsync())); - } - - public DebugPeer GetDebugPeer(string peerId) - { - // Cannot use openAPI: debug/peer endpoint is not specified there. - var endpoint = GetEndpoint(); - var str = endpoint.HttpGetString($"debug/peer/{peerId}"); - - if (str.ToLowerInvariant() == "unable to find peer!") - { - return new DebugPeer - { - IsPeerFound = false - }; - } - - var result = endpoint.Deserialize(str); - result.IsPeerFound = true; - return result; - } - - public void ConnectToPeer(string peerId, string[] peerMultiAddresses) - { - OnCodex(api => - { - Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses)); - return Task.FromResult(string.Empty); - }); - } - - public string UploadFile(FileStream fileStream) - { - return OnCodex(api => api.UploadAsync(fileStream)); - } - - public Stream DownloadFile(string contentId) - { - var fileResponse = OnCodex(api => api.DownloadNetworkAsync(contentId)); - if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode); - return fileResponse.Stream; - } - - public LocalDatasetList LocalFiles() - { - return mapper.Map(OnCodex(api => api.ListDataAsync())); - } - - public StorageAvailability SalesAvailability(StorageAvailability request) - { - var body = mapper.Map(request); - var read = OnCodex(api => api.OfferStorageAsync(body)); - return mapper.Map(read); - } - - public string RequestStorage(StoragePurchaseRequest request) - { - var body = mapper.Map(request); - return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); - } - - public StoragePurchase GetPurchaseStatus(string purchaseId) - { - return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId))); - } - - public string GetPurchaseStatusRaw(string purchaseId) - { - var endpoint = GetEndpoint(); - return endpoint.HttpGetString($"storage/purchases/{purchaseId}"); - } - - private T OnCodex(Func> action) - { - var result = tools.CreateHttp() - .OnClient(client => - { - var api = new CodexApi(client); - api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; - return Time.Wait(action(api)); - }); - return result; - } - - private IEndpoint GetEndpoint() - { - return tools - .CreateHttp() - .CreateEndpoint(address, "/api/codex/v1/"); - } - } -} diff --git a/Tools/AutoClient/Configuration.cs b/Tools/AutoClient/Configuration.cs index 8f409b48..75aca2e2 100644 --- a/Tools/AutoClient/Configuration.cs +++ b/Tools/AutoClient/Configuration.cs @@ -19,9 +19,6 @@ namespace AutoClient [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15)")] public int ContractExpiryMinutes { get; set; } = 15; - [Uniform("dataset-size", "ds", "DATASETSIZE", false, "Total dataset size in bytes. (default 10MB).")] - public int DatasetSizeBytes { get; set; } = 10 * 1024 * 1024; - [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")] public int NumHosts { get; set; } = 5; diff --git a/Tools/AutoClient/ImageGenerator.cs b/Tools/AutoClient/ImageGenerator.cs new file mode 100644 index 00000000..eeab0d32 --- /dev/null +++ b/Tools/AutoClient/ImageGenerator.cs @@ -0,0 +1,17 @@ +namespace AutoClient +{ + public class ImageGenerator + { + public async Task GenerateImage() + { + var httpClient = new HttpClient(); + var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160"); + + var filename = $"{Guid.NewGuid().ToString().ToLowerInvariant()}.jpg"; + using var file = File.OpenWrite(filename); + await thing.CopyToAsync(file); + + return filename; + } + } +} diff --git a/Tools/AutoClient/Program.cs b/Tools/AutoClient/Program.cs index ebaefb9c..ff2ad394 100644 --- a/Tools/AutoClient/Program.cs +++ b/Tools/AutoClient/Program.cs @@ -1,14 +1,14 @@ using ArgsUniform; using AutoClient; -using CodexPlugin; +using CodexOpenApi; using Core; using Logging; -using static Org.BouncyCastle.Math.EC.ECCurve; public static class Program { - public static void Main(string[] args) + public static async Task Main(string[] args) { + var cts = new CancellationTokenSource(); var cancellationToken = cts.Token; Console.CancelKeyPress += (sender, args) => cts.Cancel(); @@ -28,24 +28,26 @@ public static class Program log.Log($"Start. Address: {address}"); - var tools = CreateTools(log, config); - var fileManager = tools.GetFileManager(); - var codex = new Codex(tools, address); + var imgGenerator = new ImageGenerator(); - CheckCodex(codex, log); + var client = new HttpClient(); + var codex = new CodexApi(client); + codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; - var runner = new Runner(log, codex, fileManager, cancellationToken, config); - runner.Run(); + await CheckCodex(codex, log); + + var runner = new Runner(log, client, address, codex, cancellationToken, config, imgGenerator); + await runner.Run(); log.Log("Done."); } - private static void CheckCodex(Codex codex, ILog log) + private static async Task CheckCodex(CodexApi codex, ILog log) { log.Log("Checking Codex..."); try { - var info = codex.GetDebugInfo(); + var info = await codex.GetDebugInfoAsync(); if (string.IsNullOrEmpty(info.Id)) throw new Exception("Failed to fetch Codex node id"); } catch (Exception ex) diff --git a/Tools/AutoClient/Runner.cs b/Tools/AutoClient/Runner.cs index 8b236807..c57f95f0 100644 --- a/Tools/AutoClient/Runner.cs +++ b/Tools/AutoClient/Runner.cs @@ -1,7 +1,7 @@ -using CodexContractsPlugin; +using CodexOpenApi; using CodexPlugin; -using FileUtils; using Logging; +using Newtonsoft.Json; using Utils; namespace AutoClient @@ -9,21 +9,25 @@ namespace AutoClient public class Runner { private readonly ILog log; - private readonly Codex codex; - private readonly IFileManager fileManager; + private readonly HttpClient client; + private readonly Address address; + private readonly CodexApi codex; private readonly CancellationToken ct; private readonly Configuration config; + private readonly ImageGenerator generator; - public Runner(ILog log, Codex codex, IFileManager fileManager, CancellationToken ct, Configuration config) + public Runner(ILog log, HttpClient client, Address address, CodexApi codex, CancellationToken ct, Configuration config, ImageGenerator generator) { this.log = log; + this.client = client; + this.address = address; this.codex = codex; - this.fileManager = fileManager; this.ct = ct; this.config = config; + this.generator = generator; } - public void Run() + public async Task Run() { while (!ct.IsCancellationRequested) { @@ -31,10 +35,7 @@ namespace AutoClient try { - fileManager.ScopedFiles(() => - { - DoRun(); - }); + await DoRun(); log.Log("Run succcessful."); } @@ -42,32 +43,32 @@ namespace AutoClient { log.Error("Exception during run: " + ex); } - - FixedShortDelay(); + + await FixedShortDelay(); } } - private void DoRun() + private async Task DoRun() { - var file = CreateFile(); - var cid = UploadFile(file); - var pid = RequestStorage(cid); - WaitUntilStarted(pid); + var file = await CreateFile(); + var cid = await UploadFile(file); + var pid = await RequestStorage(cid); + await WaitUntilStarted(pid); } - private TrackedFile CreateFile() + private async Task CreateFile() { - return fileManager.GenerateFile(new ByteSize(Convert.ToInt64(config.DatasetSizeBytes))); + return await generator.GenerateImage(); } - private ContentId UploadFile(TrackedFile file) + private async Task UploadFile(string filename) { // Copied from CodexNode :/ - using var fileStream = File.OpenRead(file.Filename); + using var fileStream = File.OpenRead(filename); - var logMessage = $"Uploading file {file.Describe()}..."; + var logMessage = $"Uploading file {filename}..."; log.Log(logMessage); - var response = codex.UploadFile(fileStream); + var response = await codex.UploadAsync(fileStream, ct); if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response."); if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block."); @@ -76,22 +77,44 @@ namespace AutoClient return new ContentId(response); } - private string RequestStorage(ContentId cid) + private async Task RequestStorage(ContentId cid) { - var request = new StoragePurchaseRequest(cid) + log.Log("Requesting storage for " + cid.Id); + var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation() { - PricePerSlotPerSecond = config.Price.TestTokens(), - RequiredCollateral = config.RequiredCollateral.TestTokens(), - MinRequiredNumberOfNodes = Convert.ToUInt32(config.NumHosts), - NodeFailureTolerance = Convert.ToUInt32(config.HostTolerance), - Duration = TimeSpan.FromMinutes(config.ContractDurationMinutes), - Expiry = TimeSpan.FromMinutes(config.ContractExpiryMinutes) - }; - request.Log(log); - return codex.RequestStorage(request); + Collateral = config.RequiredCollateral.ToString(), + Duration = (config.ContractDurationMinutes * 60).ToString(), + Expiry = (config.ContractExpiryMinutes * 60).ToString(), + Nodes = config.NumHosts, + Reward = config.Price.ToString(), + ProofProbability = "15", + Tolerance = config.HostTolerance + }, ct); + + log.Log("Response: " + result); + + return result; } - private void WaitUntilStarted(string pid) + private async Task GetPurchaseState(string pid) + { + try + { + // openapi still don't match code. + var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}"); + if (string.IsNullOrEmpty(str)) return null; + var sp = JsonConvert.DeserializeObject(str)!; + log.Log($"Purchase {pid} is {sp.State}"); + if (!string.IsNullOrEmpty(sp.Error)) log.Log($"Purchase {pid} error is {sp.Error}"); + return sp.State; + } + catch (Exception ex) + { + return null; + } + } + + private async Task WaitUntilStarted(string pid) { log.Log("Waiting till contract is started, or expired..."); try @@ -99,30 +122,43 @@ namespace AutoClient var emptyResponseTolerance = 10; while (true) { - FixedShortDelay(); - var status = codex.GetPurchaseStatusRaw(pid).ToLowerInvariant(); - log.Log($"Status response: '{status}'"); + await FixedShortDelay(); + var status = await GetPurchaseState(pid); if (string.IsNullOrEmpty(status)) { emptyResponseTolerance--; if (emptyResponseTolerance == 0) { - log.Log("Received 10 empty responses to '/storage/purchases/'. Applying expiry delay, then carrying on."); - ExpiryTimeDelay(); + log.Log("Received 10 empty responses. Applying expiry delay, then carrying on."); + await ExpiryTimeDelay(); return; } - FixedShortDelay(); + await FixedShortDelay(); } else { if (status.Contains("pending") || status.Contains("submitted")) { - FixedShortDelay(); + await FixedShortDelay(); + } + else if (status.Contains("started")) + { + log.Log("Started."); + await FixedDurationDelay(); + } + else if (status.Contains("finished")) + { + log.Log("Purchase finished."); + return; + } + else if (status.Contains("error")) + { + await FixedShortDelay(); + return; } else { - log.Log("Wait finished."); - return; + await FixedShortDelay(); } } } @@ -130,18 +166,23 @@ namespace AutoClient catch (Exception ex) { log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time."); - ExpiryTimeDelay(); + await ExpiryTimeDelay(); } } - private void ExpiryTimeDelay() + private async Task FixedDurationDelay() { - Thread.Sleep(config.ContractExpiryMinutes * 60 * 1000); + await Task.Delay(config.ContractDurationMinutes * 60 * 1000); } - private void FixedShortDelay() + private async Task ExpiryTimeDelay() { - Thread.Sleep(15 * 1000); + await Task.Delay(config.ContractExpiryMinutes * 60 * 1000); + } + + private async Task FixedShortDelay() + { + await Task.Delay(15 * 1000); } } }