diff --git a/AutoClient/AutoClient.csproj b/AutoClient/AutoClient.csproj new file mode 100644 index 00000000..fbbe4500 --- /dev/null +++ b/AutoClient/AutoClient.csproj @@ -0,0 +1,16 @@ + + + + Exe + net7.0 + enable + enable + + + + + + + + + diff --git a/AutoClient/Codex.cs b/AutoClient/Codex.cs new file mode 100644 index 00000000..f667c276 --- /dev/null +++ b/AutoClient/Codex.cs @@ -0,0 +1,112 @@ +using CodexOpenApi; +using CodexPlugin; +using Core; +using Utils; +using DebugInfo = CodexPlugin.DebugInfo; + +namespace AutoClient +{ + public class Codex + { + private readonly IPluginTools tools; + private readonly Address address; + private readonly Mapper mapper = new Mapper(); + + /// + /// This class was largely copied from CodexAccess in CodexPlugin. + /// Should really be generalized so CodexPlugin supports talking to custom Codex instances. + /// + public Codex(IPluginTools tools, Address address) + { + this.tools = tools; + this.address = address; + } + + public DebugInfo GetDebugInfo() + { + return mapper.Map(OnCodex(api => api.GetDebugInfoAsync())); + } + + public DebugPeer GetDebugPeer(string peerId) + { + // Cannot use openAPI: debug/peer endpoint is not specified there. + var endpoint = GetEndpoint(); + var str = endpoint.HttpGetString($"debug/peer/{peerId}"); + + if (str.ToLowerInvariant() == "unable to find peer!") + { + return new DebugPeer + { + IsPeerFound = false + }; + } + + var result = endpoint.Deserialize(str); + result.IsPeerFound = true; + return result; + } + + public void ConnectToPeer(string peerId, string[] peerMultiAddresses) + { + OnCodex(api => + { + Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses)); + return Task.FromResult(string.Empty); + }); + } + + public string UploadFile(FileStream fileStream) + { + return OnCodex(api => api.UploadAsync(fileStream)); + } + + public Stream DownloadFile(string contentId) + { + var fileResponse = OnCodex(api => api.DownloadNetworkAsync(contentId)); + if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode); + return fileResponse.Stream; + } + + public LocalDatasetList LocalFiles() + { + return mapper.Map(OnCodex(api => api.ListDataAsync())); + } + + public StorageAvailability SalesAvailability(StorageAvailability request) + { + var body = mapper.Map(request); + var read = OnCodex(api => api.OfferStorageAsync(body)); + return mapper.Map(read); + } + + public string RequestStorage(StoragePurchaseRequest request) + { + var body = mapper.Map(request); + return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); + } + + public StoragePurchase GetPurchaseStatus(string purchaseId) + { + return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId))); + } + + private T OnCodex(Func> action) + { + var result = tools.CreateHttp() + .OnClient(client => + { + var api = new CodexApi(client); + api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; + return Time.Wait(action(api)); + }); + return result; + } + + private IEndpoint GetEndpoint() + { + return tools + .CreateHttp() + .CreateEndpoint(address, "/api/codex/v1/"); + } + } +} diff --git a/AutoClient/Configuration.cs b/AutoClient/Configuration.cs new file mode 100644 index 00000000..4be2a268 --- /dev/null +++ b/AutoClient/Configuration.cs @@ -0,0 +1,45 @@ +using ArgsUniform; + +namespace AutoClient +{ + public class Configuration + { + [Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default localhost)")] + public string CodexHost { get; set; } = "localhost"; + + [Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")] + public int CodexPort { get; set; } = 8080; + + [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")] + public string DataPath { get; set; } = "datapath"; + + [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 30)")] + public int ContractDurationMinutes { get; set; } = 30; + + [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15)")] + public int ContractExpiryMinutes { get; set; } = 15; + + [Uniform("dataset-size", "ds", "DATASETSIZE", false, "Total dataset size in bytes. (default 10MB).")] + public int DatasetSizeBytes { get; set; } = 10 * 1024 * 1024; + + [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")] + public int NumHosts { get; set; } = 5; + + [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 2)")] + public int HostTolerance { get; set; } = 2; + + [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")] + public int Price { get; set; } = 10; + + [Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")] + public int RequiredCollateral { get; set; } = 1; + + public string LogPath + { + get + { + return Path.Combine(DataPath, "logs"); + } + } + } +} diff --git a/AutoClient/Program.cs b/AutoClient/Program.cs new file mode 100644 index 00000000..1358586b --- /dev/null +++ b/AutoClient/Program.cs @@ -0,0 +1,57 @@ +using ArgsUniform; +using AutoClient; +using CodexPlugin; +using Core; +using Logging; +using static Org.BouncyCastle.Math.EC.ECCurve; + +public static class Program +{ + public static void Main(string[] args) + { + var cts = new CancellationTokenSource(); + var cancellationToken = cts.Token; + Console.CancelKeyPress += (sender, args) => cts.Cancel(); + + var uniformArgs = new ArgsUniform(PrintHelp, args); + var config = uniformArgs.Parse(true); + + var log = new LogSplitter( + new FileLog(Path.Combine(config.LogPath, "autoclient")), + new ConsoleLog() + ); + + var address = new Utils.Address( + host: config.CodexHost, + port: config.CodexPort + ); + + log.Log($"Start. Address: {address}"); + + var tools = CreateTools(log, config); + var fileManager = tools.GetFileManager(); + var codex = new Codex(tools, address); + + var runner = new Runner(log, codex, fileManager, cancellationToken, config); + runner.Run(); + + log.Log("Done."); + } + + private static void PrintHelp() + { + Console.WriteLine("Generates fake data and creates Codex storage contracts for it."); + } + + private static IPluginTools CreateTools(ILog log, Configuration config) + { + var configuration = new KubernetesWorkflow.Configuration( + null, + operationTimeout: TimeSpan.FromMinutes(10), + retryDelay: TimeSpan.FromSeconds(10), + kubernetesNamespace: "notUsed!#"); + + var result = new EntryPoint(log, configuration, config.DataPath, new DefaultTimeSet()); + return result.Tools; + } +} \ No newline at end of file diff --git a/AutoClient/Runner.cs b/AutoClient/Runner.cs new file mode 100644 index 00000000..f58c683c --- /dev/null +++ b/AutoClient/Runner.cs @@ -0,0 +1,135 @@ +using CodexContractsPlugin; +using CodexPlugin; +using FileUtils; +using Logging; +using Utils; + +namespace AutoClient +{ + public class Runner + { + private readonly ILog log; + private readonly Codex codex; + private readonly IFileManager fileManager; + private readonly CancellationToken ct; + private readonly Configuration config; + + public Runner(ILog log, Codex codex, IFileManager fileManager, CancellationToken ct, Configuration config) + { + this.log = log; + this.codex = codex; + this.fileManager = fileManager; + this.ct = ct; + this.config = config; + } + + public void Run() + { + while (!ct.IsCancellationRequested) + { + log.Log("New run!"); + + try + { + fileManager.ScopedFiles(() => + { + DoRun(); + }); + + log.Log("Run succcessful."); + } + catch (Exception ex) + { + log.Error("Exception during run: " + ex); + } + + FixedShortDelay(); + } + } + + private void DoRun() + { + var file = CreateFile(); + var cid = UploadFile(file); + var pid = RequestStorage(cid); + WaitUntilStarted(pid); + } + + private TrackedFile CreateFile() + { + return fileManager.GenerateFile(new ByteSize(Convert.ToInt64(config.DatasetSizeBytes))); + } + + private ContentId UploadFile(TrackedFile file) + { + // Copied from CodexNode :/ + using var fileStream = File.OpenRead(file.Filename); + + var logMessage = $"Uploading file {file.Describe()}..."; + log.Log(logMessage); + var response = codex.UploadFile(fileStream); + + if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response."); + if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block."); + + log.Log($"Uploaded file. Received contentId: '{response}'."); + return new ContentId(response); + } + + private string RequestStorage(ContentId cid) + { + var request = new StoragePurchaseRequest(cid) + { + PricePerSlotPerSecond = config.Price.TestTokens(), + RequiredCollateral = config.RequiredCollateral.TestTokens(), + MinRequiredNumberOfNodes = Convert.ToUInt32(config.NumHosts), + NodeFailureTolerance = Convert.ToUInt32(config.HostTolerance), + Duration = TimeSpan.FromMinutes(config.ContractDurationMinutes), + Expiry = TimeSpan.FromMinutes(config.ContractExpiryMinutes) + }; + log.Log($"Requesting storage: {request}"); + return codex.RequestStorage(request); + } + + private void WaitUntilStarted(string pid) + { + log.Log("Waiting till contract is started, or expired..."); + try + { + while (true) + { + FixedShortDelay(); + var status = codex.GetPurchaseStatus(pid); + if (status != null) + { + if (!string.IsNullOrEmpty(status.Error)) log.Log("Contract errored: " + status.Error); + var state = status.State.ToLowerInvariant(); + if (state.Contains("pending") || state.Contains("submitted")) + { + FixedShortDelay(); + } + else + { + log.Log("Wait finished with contract status: " + state); + } + } + } + } + catch (Exception ex) + { + log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time."); + ExpiryTimeDelay(); + } + } + + private void ExpiryTimeDelay() + { + Thread.Sleep(config.ContractExpiryMinutes * 60 * 1000); + } + + private void FixedShortDelay() + { + Thread.Sleep(15 * 1000); + } + } +} diff --git a/Framework/FileUtils/FileManager.cs b/Framework/FileUtils/FileManager.cs index 10ecf241..4a283b14 100644 --- a/Framework/FileUtils/FileManager.cs +++ b/Framework/FileUtils/FileManager.cs @@ -57,16 +57,27 @@ namespace FileUtils public void ScopedFiles(Action action) { PushFileSet(); - action(); - PopFileSet(); + try + { + action(); + } + finally + { + PopFileSet(); + } } public T ScopedFiles(Func action) { PushFileSet(); - var result = action(); - PopFileSet(); - return result; + try + { + return action(); + } + finally + { + PopFileSet(); + } } private void PushFileSet() diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index ada68933..809f9713 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -66,6 +66,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution .editorconfig = .editorconfig EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoClient", "AutoClient\AutoClient.csproj", "{8B8BF2B9-5855-4C92-A5DA-D13D778B7934}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -172,6 +174,10 @@ Global {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Debug|Any CPU.Build.0 = Debug|Any CPU {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.ActiveCfg = Release|Any CPU {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.Build.0 = Release|Any CPU + {8B8BF2B9-5855-4C92-A5DA-D13D778B7934}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8B8BF2B9-5855-4C92-A5DA-D13D778B7934}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8B8BF2B9-5855-4C92-A5DA-D13D778B7934}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8B8BF2B9-5855-4C92-A5DA-D13D778B7934}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -202,6 +208,7 @@ Global {F730DA73-1C92-4107-BCFB-D33759DAB0C3} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} {B07820C4-309F-4454-BCC1-1D4902C9C67B} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} {88C212E9-308A-46A4-BAAD-468E8EBD8EDF} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124} + {8B8BF2B9-5855-4C92-A5DA-D13D778B7934} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}