codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Sets up autoclient.

This commit is contained in:
benbierens 2024-04-01 20:40:03 +02:00
parent 4cd22f3719
commit d532d9505a
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
7 changed files with 388 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
AutoClient/AutoClient.csproj Normal file
View File

@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Framework\ArgsUniform\ArgsUniform.csproj" />
<ProjectReference Include="..\Framework\Logging\Logging.csproj" />
<ProjectReference Include="..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
</ItemGroup>
</Project>

112
AutoClient/Codex.cs Normal file
View File

@ -0,0 +1,112 @@
using CodexOpenApi;
using CodexPlugin;
using Core;
using Utils;
using DebugInfo = CodexPlugin.DebugInfo;
namespace AutoClient
{
public class Codex
{
private readonly IPluginTools tools;
private readonly Address address;
private readonly Mapper mapper = new Mapper();
/// <summary>
/// This class was largely copied from CodexAccess in CodexPlugin.
/// Should really be generalized so CodexPlugin supports talking to custom Codex instances.
/// </summary>
public Codex(IPluginTools tools, Address address)
{
this.tools = tools;
this.address = address;
}
public DebugInfo GetDebugInfo()
{
return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
}
public DebugPeer GetDebugPeer(string peerId)
{
// Cannot use openAPI: debug/peer endpoint is not specified there.
var endpoint = GetEndpoint();
var str = endpoint.HttpGetString($"debug/peer/{peerId}");
if (str.ToLowerInvariant() == "unable to find peer!")
{
return new DebugPeer
{
IsPeerFound = false
};
}
var result = endpoint.Deserialize<DebugPeer>(str);
result.IsPeerFound = true;
return result;
}
public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
{
OnCodex(api =>
{
Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses));
return Task.FromResult(string.Empty);
});
}
public string UploadFile(FileStream fileStream)
{
return OnCodex(api => api.UploadAsync(fileStream));
}
public Stream DownloadFile(string contentId)
{
var fileResponse = OnCodex(api => api.DownloadNetworkAsync(contentId));
if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
return fileResponse.Stream;
}
public LocalDatasetList LocalFiles()
{
return mapper.Map(OnCodex(api => api.ListDataAsync()));
}
public StorageAvailability SalesAvailability(StorageAvailability request)
{
var body = mapper.Map(request);
var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
return mapper.Map(read);
}
public string RequestStorage(StoragePurchaseRequest request)
{
var body = mapper.Map(request);
return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
}
public StoragePurchase GetPurchaseStatus(string purchaseId)
{
return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
}
private T OnCodex<T>(Func<CodexApi, Task<T>> action)
{
var result = tools.CreateHttp()
.OnClient(client =>
{
var api = new CodexApi(client);
api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
return Time.Wait(action(api));
});
return result;
}
private IEndpoint GetEndpoint()
{
return tools
.CreateHttp()
.CreateEndpoint(address, "/api/codex/v1/");
}
}
}

45
AutoClient/Configuration.cs Normal file
View File

@ -0,0 +1,45 @@
using ArgsUniform;
namespace AutoClient
{
public class Configuration
{
[Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default localhost)")]
public string CodexHost { get; set; } = "localhost";
[Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")]
public int CodexPort { get; set; } = 8080;
[Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
public string DataPath { get; set; } = "datapath";
[Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 30)")]
public int ContractDurationMinutes { get; set; } = 30;
[Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15)")]
public int ContractExpiryMinutes { get; set; } = 15;
[Uniform("dataset-size", "ds", "DATASETSIZE", false, "Total dataset size in bytes. (default 10MB).")]
public int DatasetSizeBytes { get; set; } = 10 * 1024 * 1024;
[Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")]
public int NumHosts { get; set; } = 5;
[Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 2)")]
public int HostTolerance { get; set; } = 2;
[Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
public int Price { get; set; } = 10;
[Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
public int RequiredCollateral { get; set; } = 1;
public string LogPath
{
get
{
return Path.Combine(DataPath, "logs");
}
}
}
}

57
AutoClient/Program.cs Normal file
View File

@ -0,0 +1,57 @@
using ArgsUniform;
using AutoClient;
using CodexPlugin;
using Core;
using Logging;
using static Org.BouncyCastle.Math.EC.ECCurve;
public static class Program
{
public static void Main(string[] args)
{
var cts = new CancellationTokenSource();
var cancellationToken = cts.Token;
Console.CancelKeyPress += (sender, args) => cts.Cancel();
var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
var config = uniformArgs.Parse(true);
var log = new LogSplitter(
new FileLog(Path.Combine(config.LogPath, "autoclient")),
new ConsoleLog()
);
var address = new Utils.Address(
host: config.CodexHost,
port: config.CodexPort
);
log.Log($"Start. Address: {address}");
var tools = CreateTools(log, config);
var fileManager = tools.GetFileManager();
var codex = new Codex(tools, address);
var runner = new Runner(log, codex, fileManager, cancellationToken, config);
runner.Run();
log.Log("Done.");
}
private static void PrintHelp()
{
Console.WriteLine("Generates fake data and creates Codex storage contracts for it.");
}
private static IPluginTools CreateTools(ILog log, Configuration config)
{
var configuration = new KubernetesWorkflow.Configuration(
null,
operationTimeout: TimeSpan.FromMinutes(10),
retryDelay: TimeSpan.FromSeconds(10),
kubernetesNamespace: "notUsed!#");
var result = new EntryPoint(log, configuration, config.DataPath, new DefaultTimeSet());
return result.Tools;
}
}

135
AutoClient/Runner.cs Normal file
View File

@ -0,0 +1,135 @@
using CodexContractsPlugin;
using CodexPlugin;
using FileUtils;
using Logging;
using Utils;
namespace AutoClient
{
public class Runner
{
private readonly ILog log;
private readonly Codex codex;
private readonly IFileManager fileManager;
private readonly CancellationToken ct;
private readonly Configuration config;
public Runner(ILog log, Codex codex, IFileManager fileManager, CancellationToken ct, Configuration config)
{
this.log = log;
this.codex = codex;
this.fileManager = fileManager;
this.ct = ct;
this.config = config;
}
public void Run()
{
while (!ct.IsCancellationRequested)
{
log.Log("New run!");
try
{
fileManager.ScopedFiles(() =>
{
DoRun();
});
log.Log("Run succcessful.");
}
catch (Exception ex)
{
log.Error("Exception during run: " + ex);
}
FixedShortDelay();
}
}
private void DoRun()
{
var file = CreateFile();
var cid = UploadFile(file);
var pid = RequestStorage(cid);
WaitUntilStarted(pid);
}
private TrackedFile CreateFile()
{
return fileManager.GenerateFile(new ByteSize(Convert.ToInt64(config.DatasetSizeBytes)));
}
private ContentId UploadFile(TrackedFile file)
{
// Copied from CodexNode :/
using var fileStream = File.OpenRead(file.Filename);
var logMessage = $"Uploading file {file.Describe()}...";
log.Log(logMessage);
var response = codex.UploadFile(fileStream);
if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
log.Log($"Uploaded file. Received contentId: '{response}'.");
return new ContentId(response);
}
private string RequestStorage(ContentId cid)
{
var request = new StoragePurchaseRequest(cid)
{
PricePerSlotPerSecond = config.Price.TestTokens(),
RequiredCollateral = config.RequiredCollateral.TestTokens(),
MinRequiredNumberOfNodes = Convert.ToUInt32(config.NumHosts),
NodeFailureTolerance = Convert.ToUInt32(config.HostTolerance),
Duration = TimeSpan.FromMinutes(config.ContractDurationMinutes),
Expiry = TimeSpan.FromMinutes(config.ContractExpiryMinutes)
};
log.Log($"Requesting storage: {request}");
return codex.RequestStorage(request);
}
private void WaitUntilStarted(string pid)
{
log.Log("Waiting till contract is started, or expired...");
try
{
while (true)
{
FixedShortDelay();
var status = codex.GetPurchaseStatus(pid);
if (status != null)
{
if (!string.IsNullOrEmpty(status.Error)) log.Log("Contract errored: " + status.Error);
var state = status.State.ToLowerInvariant();
if (state.Contains("pending") || state.Contains("submitted"))
{
FixedShortDelay();
}
else
{
log.Log("Wait finished with contract status: " + state);
}
}
}
}
catch (Exception ex)
{
log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
ExpiryTimeDelay();
}
}
private void ExpiryTimeDelay()
{
Thread.Sleep(config.ContractExpiryMinutes * 60 * 1000);
}
private void FixedShortDelay()
{
Thread.Sleep(15 * 1000);
}
}
}

21
Framework/FileUtils/FileManager.cs
View File

@ -57,16 +57,27 @@ namespace FileUtils
public void ScopedFiles(Action action)
{
PushFileSet();
action();
PopFileSet();
try
{
action();
}
finally
{
PopFileSet();
}
}
public T ScopedFiles<T>(Func<T> action)
{
PushFileSet();
var result = action();
PopFileSet();
return result;
try
{
return action();
}
finally
{
PopFileSet();
}
}
private void PushFileSet()

7
cs-codex-dist-testing.sln
View File

@ -66,6 +66,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
.editorconfig = .editorconfig
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoClient", "AutoClient\AutoClient.csproj", "{8B8BF2B9-5855-4C92-A5DA-D13D778B7934}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -172,6 +174,10 @@ Global
{88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.Build.0 = Release|Any CPU
{8B8BF2B9-5855-4C92-A5DA-D13D778B7934}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8B8BF2B9-5855-4C92-A5DA-D13D778B7934}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8B8BF2B9-5855-4C92-A5DA-D13D778B7934}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8B8BF2B9-5855-4C92-A5DA-D13D778B7934}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -202,6 +208,7 @@ Global
{F730DA73-1C92-4107-BCFB-D33759DAB0C3} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
{B07820C4-309F-4454-BCC1-1D4902C9C67B} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
{88C212E9-308A-46A4-BAAD-468E8EBD8EDF} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124}
{8B8BF2B9-5855-4C92-A5DA-D13D778B7934} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}