Compare commits

..
Author SHA1 Message Date
Ben 1d9f22c96d Creates new overview after 3 new changes. 2024-12-16 15:03:39 +01:00
Ben ed764bc38c wip 2024-12-02 14:04:17 +01:00
benbierens 4bcdbf3e8c Only check CID when purchase needs to be made 2024-11-29 16:22:49 +01:00
Ben 513853d929 Better safe-duration 2024-11-27 13:43:38 +01:00
benbierens a40d5d77d7 Slow down logging when waiting for requests to start. 2024-11-27 11:35:56 +01:00
benbierens 23622245f0 cleaner log 2024-11-27 11:26:19 +01:00
benbierens f2b84ebfd6 Stores encoded-CID in worker state 2024-11-27 11:09:47 +01:00
benbierens 9853a0b7db Limits number of concurrently starting contracts 2024-11-27 10:44:05 +01:00
benbierens 025c85c1aa fixes file-backed worker issue 2024-11-27 10:30:30 +01:00
benbierens f0cbc0a53a Debugging worker files 2024-11-27 10:25:52 +01:00
benbierens e91a574b2c writes overview CID to file 2024-11-27 10:18:33 +01:00
Ben f18ff24bb6 savestate 2024-11-26 16:23:56 +01:00
Ben 529d48a758 show index 2024-11-26 16:22:43 +01:00
Ben d136345df4 sleep 2024-11-26 16:19:54 +01:00
Ben a3b9e7bf8d Uploads zipfile with archive overview periodically 2024-11-26 16:15:28 +01:00
Ben 6f778ec04f Fixes re-purchase issue 2024-11-26 15:57:34 +01:00
Ben 4b7ceda572 Split to files 2024-11-26 15:14:31 +01:00
Ben db46a0c686 Logs at start of ensure-calls 2024-11-26 12:37:36 +01:00
Ben c2df15436f Merge branch 'master' into feature/folder-saver 2024-11-26 12:27:47 +01:00
Ben 3c7892e4be Switches to latest, updates API 2024-11-21 09:39:28 +01:00
benbierens 833421b2b2 workaround for faulty local-dataset api match 2024-11-12 16:51:32 +01:00
benbierens 55cc0ab0ef sets http client timeout to 1h 2024-11-11 16:21:58 +01:00
benbierens 0129af6fd7 for vm running 2024-11-11 16:12:25 +01:00
benbierens d48caa44d6 stops autoclient after 5 failures 2024-11-11 13:41:02 +01:00
Ben e0755a1101 Updates to 0.1.8. Fixes issue in blockchain monitor 2024-11-04 12:21:39 +01:00
Ben Bierens 1dd17037ba Merge pull request #107 from codex-storage/feat/switch-docker-variable
Use ETH_PRIVATE_KEY variable instead of deprecated PRIV_KEY
2024-11-04 12:02:55 +01:00
Slava 22e6439731 Use ETH_PRIVATE_KEY variable instead of deprecated PRIV_KEY
https://github.com/codex-storage/nim-codex/pull/982
2024-11-04 12:36:25 +02:00
benbierens 365032978b better status logging, faster revisiting 2024-11-01 11:14:07 +01:00
Ben 3e12baaafe debug 2024-10-30 14:32:49 +01:00
Ben 605bb6411f logging 2024-10-30 11:43:57 +01:00
Ben 2554645abc debug 2024-10-30 11:38:30 +01:00
Ben 2dfcf20ecd rigged for local running on eth mainnet history 2024-10-30 11:15:33 +01:00
Ben c35784c90f Implements folder-storing 2024-10-30 11:09:13 +01:00
Ben b54c9ff9a3 Retrofitting autoclient to support folder-uploader mode 2024-10-30 08:56:31 +01:00
37 changed files with 1238 additions and 671 deletions
-9
View File
@@ -1,9 +0,0 @@
<Application x:Class="DevconBoothImages.App"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:local="clr-namespace:DevconBoothImages"
StartupUri="MainWindow.xaml">
<Application.Resources>
</Application.Resources>
</Application>
-14
View File
@@ -1,14 +0,0 @@
using System.Configuration;
using System.Data;
using System.Windows;
namespace DevconBoothImages
{
/// <summary>
/// Interaction logic for App.xaml
/// </summary>
public partial class App : Application
{
}
}
-10
View File
@@ -1,10 +0,0 @@
using System.Windows;
[assembly: ThemeInfo(
ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located
//(used if a resource is not found in the page,
// or application resource dictionaries)
ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located
//(used if a resource is not found in the page,
// app, or any theme specific resource dictionaries)
)]
-77
View File
@@ -1,77 +0,0 @@
using CodexOpenApi;
using IdentityModel.Client;
using System.Net.Http;
using System.Windows;
using Utils;
namespace DevconBoothImages
{
public class Codexes
{
public Codexes(CodexApi local, CodexApi testnet)
{
Local = local;
Testnet = testnet;
}
public CodexApi Local { get; }
public CodexApi Testnet { get; }
}
public class CodexWrapper
{
public async Task<Codexes> GetCodexes()
{
var config = new Configuration();
return new Codexes(
await GetCodexWithPort(config.CodexLocalEndpoint),
await GetCodexWithoutPort(config.CodexPublicEndpoint, config.AuthUser, config.AuthPw)
);
}
private async Task<CodexApi> GetCodexWithPort(string endpoint)
{
var splitIndex = endpoint.LastIndexOf(':');
var host = endpoint.Substring(0, splitIndex);
var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
var address = new Address(
host: host,
port: port
);
var client = new HttpClient();
var codex = new CodexApi(client);
codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
await CheckCodex(codex, endpoint);
return codex;
}
private async Task<CodexApi> GetCodexWithoutPort(string endpoint, string user, string pw)
{
var client = new HttpClient();
client.SetBasicAuthentication(user, pw);
var codex = new CodexApi(client);
codex.BaseUrl = $"{endpoint}/api/codex/v1";
await CheckCodex(codex, endpoint);
return codex;
}
private async Task CheckCodex(CodexApi codex, string endpoint)
{
try
{
var info = await codex.GetDebugInfoAsync();
if (string.IsNullOrEmpty(info.Id)) throw new Exception("Failed to fetch Codex node id");
}
catch (Exception ex)
{
MessageBox.Show($"Failed to connect to codex '{endpoint}': {ex}");
throw;
}
}
}
}
-13
View File
@@ -1,13 +0,0 @@
namespace DevconBoothImages
{
public class Configuration
{
public string CodexLocalEndpoint { get; } = "http://localhost:8080";
public string CodexPublicEndpoint { get; } = "https://api.testnet.codex.storage/storage/node-9";
public string AuthUser { get; } = "";
public string AuthPw { get; } = "";
public string LocalNodeBootstrapInfo { get; } = "";
public string WorkingDir { get; } = "D:\\DevconBoothApp";
}
}
@@ -1,20 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>WinExe</OutputType>
<TargetFramework>net8.0-windows</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<UseWPF>true</UseWPF>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="QRCoder" Version="1.6.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
<ProjectReference Include="..\Tools\AutoClient\AutoClient.csproj" />
</ItemGroup>
</Project>
@@ -1,14 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup />
<ItemGroup>
<ApplicationDefinition Update="App.xaml">
<SubType>Designer</SubType>
</ApplicationDefinition>
</ItemGroup>
<ItemGroup>
<Page Update="MainWindow.xaml">
<SubType>Designer</SubType>
</Page>
</ItemGroup>
</Project>
-38
View File
@@ -1,38 +0,0 @@
<Window x:Class="DevconBoothImages.MainWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:local="clr-namespace:DevconBoothImages"
mc:Ignorable="d"
Title="CodexBoothImages" Height="450" Width="800" WindowState="Maximized">
<Grid>
<Grid.RowDefinitions>
<RowDefinition Height="*"/>
<RowDefinition Height="Auto"/>
</Grid.RowDefinitions>
<Grid Grid.Row="0">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="*"/>
<ColumnDefinition Width="Auto"/>
</Grid.ColumnDefinitions>
<Image Grid.Column="0" Name="Img" />
<StackPanel Grid.Column="1">
<TextBlock Text="Instructions:"/>
<Image Name="ImgInstructions" />
<TextBlock Text="Local CID:"/>
<Image Name="ImgLocalCid" />
<TextBlock Text="TestNet CID:"/>
<Image Name="ImgTestnetCid" />
</StackPanel>
</Grid>
<StackPanel Grid.Row="1">
<TextBlock Name="Txt" HorizontalAlignment="Center" />
<Button Content="Check Codex connections" Click="Button_Click_2" />
<Button Content="Generate image -> Upload to Codex -> Put CID info in clipboard" Padding="10" Click="Button_Click"/>
<Button Content="Put last CID info in clipboard" Padding="10" Click="Button_Click_1"/>
</StackPanel>
</Grid>
</Window>
-160
View File
@@ -1,160 +0,0 @@
using AutoClient;
using CodexOpenApi;
using Logging;
using QRCoder;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Windows;
using System.Windows.Media;
using System.Windows.Media.Imaging;
namespace DevconBoothImages
{
public partial class MainWindow : Window
{
private readonly Configuration config = new Configuration();
private readonly CodexWrapper codexWrapper = new CodexWrapper();
private readonly ImageGenerator imageGenerator = new ImageGenerator(new NullLog());
private string currentLocalCid = string.Empty;
private string currentPublicCid = string.Empty;
public MainWindow()
{
InitializeComponent();
Application.Current.DispatcherUnhandledException += Current_DispatcherUnhandledException;
}
private void Current_DispatcherUnhandledException(object sender, System.Windows.Threading.DispatcherUnhandledExceptionEventArgs e)
{
MessageBox.Show("Unhandled exception: " + e.Exception);
}
private async void Button_Click(object sender, RoutedEventArgs e)
{
// image
Log("Getting image...");
var file = await imageGenerator.Generate();
var filename = Path.Combine(config.WorkingDir, file);
File.Copy(file, filename);
var bmp = new BitmapImage();
bmp.BeginInit();
bmp.UriSource = new Uri(filename);
bmp.EndInit();
Img.Source = bmp;
Log("Uploading...");
// upload
await UploadToCodexes(filename, file);
// clipboard info
InfoToClipboard();
}
private BitmapImage GenerateQr(string text)
{
using (QRCodeGenerator qrGenerator = new QRCodeGenerator())
using (QRCodeData qrCodeData = qrGenerator.CreateQrCode(text, QRCodeGenerator.ECCLevel.Default))
using (PngByteQRCode qrCode = new PngByteQRCode(qrCodeData))
{
byte[] qrCodeImage = qrCode.GetGraphic(7);
using (var ms = new MemoryStream(qrCodeImage))
{
var img = Image.FromStream(ms);
using (var ms2 = new MemoryStream())
{
img.Save(ms2, ImageFormat.Png);
ms.Seek(0, SeekOrigin.Begin);
var bitmapImage = new BitmapImage();
bitmapImage.BeginInit();
bitmapImage.CacheOption = BitmapCacheOption.OnLoad;
bitmapImage.StreamSource = ms2;
bitmapImage.EndInit();
return bitmapImage;
}
}
}
}
private async void Button_Click_1(object sender, RoutedEventArgs e)
{
InfoToClipboard();
}
private async void Button_Click_2(object sender, RoutedEventArgs e)
{
// check codexes
Log("Checking Codex connections...");
await codexWrapper.GetCodexes();
Log("Connections OK");
}
private async Task UploadToCodexes(string filename, string shortName)
{
var codexes = await codexWrapper.GetCodexes();
try
{
currentLocalCid = await UploadFile(filename, shortName, codexes.Local);
currentPublicCid = await UploadFile(filename, shortName, codexes.Testnet);
}
catch (Exception ex)
{
MessageBox.Show("Upload failed: " + ex);
}
Log($"Generated CIDs");
}
private async Task<string> UploadFile(string filename, string shortName, CodexApi codex)
{
using (var fileStream = File.OpenRead(filename))
{
var response = await codex.UploadAsync(
"image/jpeg",
$"attachment; filename=\"{shortName}\"",
fileStream);
if (string.IsNullOrEmpty(response) ||
response.ToLowerInvariant().Contains("unable to store block"))
{
throw new Exception("Unable to upload image. Response empty or error message.");
}
return response;
}
}
private void InfoToClipboard()
{
Clipboard.Clear();
if (string.IsNullOrEmpty(currentLocalCid) || string.IsNullOrEmpty(currentPublicCid))
{
Log("No CIDs were generated! Clipboard cleared.");
return;
}
var nl = Environment.NewLine;
var msg =
$"** Codex@Devcon 💻 Raspberry Pi Challenge **{nl}" +
$"📢 A new image is available. Download it and bring it to the booth!{nl}" +
$"Public Testnet CID: `{currentPublicCid}`{nl}" +
$"Local Devcon network CID: `{currentLocalCid}`{nl}" +
$"Setup instructions: [Here](https://docs.codex.storage){nl}" +
$"Local Devcon network information: [Here](https://github.com/codex-storage/codex-testnet-starter/blob/master/SETUP_DEVCONNET.md)";
Clipboard.SetText(msg);
Log("CID info copied to clipboard. Paste it in Discord plz!");
ImgLocalCid.Source = GenerateQr(currentLocalCid);
ImgTestnetCid.Source = GenerateQr(currentPublicCid);
ImgInstructions.Source = GenerateQr("https://github.com/codex-storage/codex-testnet-starter/blob/master/SETUP_DEVCONNET.md");
}
private void Log(string v)
{
Txt.Text = v;
}
}
}
+2 -1
View File
@@ -15,12 +15,13 @@
To = from;
}
TimeRange = timeRange;
NumberOfBlocks = (To - From) + 1;
}
public ulong From { get; }
public ulong To { get; }
public TimeRange TimeRange { get; }
public ulong NumberOfBlocks => To - From;
public ulong NumberOfBlocks { get; }
public override string ToString()
{
@@ -75,11 +75,12 @@ namespace CodexContractsPlugin.ChainMonitor
throw new Exception(msg);
}
log.Log($"ChainState updating: {events.BlockInterval}");
log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
// Run through each block and apply the events to the state in order.
var span = events.BlockInterval.TimeRange.Duration;
var numBlocks = events.BlockInterval.NumberOfBlocks;
if (numBlocks == 0) return;
var spanPerBlock = span / numBlocks;
var eventUtc = events.BlockInterval.TimeRange.From;
+1 -1
View File
@@ -10,7 +10,7 @@ namespace CodexPlugin
public class ApiChecker
{
// <INSERT-OPENAPI-YAML-HASH>
private const string OpenApiYamlHash = "09-53-C3-A6-31-A5-0C-8B-53-1C-3D-C7-2B-1E-85-C7-17-60-54-43-01-C4-49-4E-D9-68-35-7D-F7-41-13-B5";
private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
private const string OpenApiFilePath = "/codex/openapi.yaml";
private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
+14 -1
View File
@@ -82,7 +82,20 @@ namespace CodexPlugin
public LocalDatasetList LocalFiles()
{
return mapper.Map(OnCodex(api => api.ListDataAsync()));
// API for listData mismatches.
//return mapper.Map(OnCodex(api => api.ListDataAsync()));
return mapper.Map(CrashCheck(() =>
{
var endpoint = GetEndpoint();
return Time.Retry(() =>
{
var str = endpoint.HttpGetString("data");
if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
}, nameof(LocalFiles));
}));
}
public StorageAvailability SalesAvailability(StorageAvailability request)
@@ -109,7 +109,7 @@ namespace CodexPlugin
// Custom scripting in the Codex test image will write this variable to a private-key file,
// and pass the correct filename to Codex.
var account = marketplaceSetup.EthAccountSetup.GetNew();
AddEnvVar("PRIV_KEY", account.PrivateKey);
AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
Additional(account);
SetCommandOverride(marketplaceSetup);
+1 -1
View File
@@ -140,7 +140,7 @@ namespace CodexPlugin
public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
{
return UploadFile(file, "application/x-binary", $"attachment; filename=\"{file.Filename}\"", onFailure);
return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
}
public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
+69 -2
View File
@@ -21,6 +21,14 @@ namespace CodexPlugin
};
}
public LocalDatasetList Map(LocalDatasetListJson json)
{
return new LocalDatasetList
{
Content = json.Content.Select(Map).ToArray()
};
}
public LocalDatasetList Map(CodexOpenApi.DataList dataList)
{
return new LocalDatasetList
@@ -38,6 +46,15 @@ namespace CodexPlugin
};
}
public LocalDataset Map(LocalDatasetListJsonItem item)
{
return new LocalDataset
{
Cid = new ContentId(item.Cid),
Manifest = MapManifest(item.Manifest)
};
}
public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
{
return new CodexOpenApi.SalesAvailabilityCREATE
@@ -182,8 +199,20 @@ namespace CodexPlugin
return new Manifest
{
BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
RootHash = manifest.RootHash,
OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
RootHash = manifest.TreeCid,
Protected = manifest.Protected
};
}
public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
{
return new Manifest
{
// needs update
BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
RootHash = manifest.TreeCid,
Protected = manifest.Protected
};
}
@@ -243,4 +272,42 @@ namespace CodexPlugin
return new ByteSize(Convert.ToInt64(size));
}
}
//"content": [
// {
// "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
// "manifest": {
// "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
// "datasetSize": 5242880,
// "blockSize": 65536,
// "filename": null,
// "mimetype": "application/octet-stream",
// "uploadedAt": 1731426230,
// "protected": false
// }
// }
// ]
public class LocalDatasetListJson
{
public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
}
public class LocalDatasetListJsonItem
{
public string Cid { get; set; } = string.Empty;
public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
}
public class LocalDatasetListJsonItemManifest
{
public string TreeCid { get; set; } = string.Empty;
public int DatasetSize { get; set; }
public int BlockSize { get; set; }
public string? Filename { get; set; } = string.Empty;
public string? MimeType { get; set; } = string.Empty;
public int? UploadedAt { get; set; }
public bool Protected { get; set; }
}
}
@@ -40,6 +40,12 @@ namespace CodexPlugin
public string State { get; set; } = string.Empty;
public string Error { get; set; } = string.Empty;
public StorageRequest Request { get; set; } = null!;
public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
public bool IsError => State.ToLowerInvariant().Contains("error");
public bool IsFinished => State.ToLowerInvariant().Contains("finished");
public bool IsStarted => State.ToLowerInvariant().Contains("started");
public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
}
public class StorageRequest
+6 -3
View File
@@ -344,10 +344,10 @@ components:
ManifestItem:
type: object
properties:
rootHash:
treeCid:
$ref: "#/components/schemas/Cid"
description: "Root hash of the content"
originalBytes:
description: "Unique data identifier"
datasetSize:
type: integer
format: int64
description: "Length of original content in bytes"
@@ -359,15 +359,18 @@ components:
description: "Indicates if content is protected by erasure-coding"
filename:
type: string
nullable: true
description: "The original name of the uploaded content (optional)"
example: codex.png
mimetype:
type: string
nullable: true
description: "The original mimetype of the uploaded content (optional)"
example: image/png
uploadedAt:
type: integer
format: int64
nullable: true
description: "The UTC upload timestamp in seconds"
example: 1729244192
+12 -1
View File
@@ -1,4 +1,5 @@
using Logging;
using AutoClient.Modes.FolderStore;
using Logging;
namespace AutoClient
{
@@ -19,6 +20,15 @@ namespace AutoClient
new FileLog(Path.Combine(config.LogPath, "performance")),
new ConsoleLog()
));
if (!string.IsNullOrEmpty(config.FolderToStore))
{
FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore);
}
else
{
FolderWorkDispatcher = null!;
}
}
public Configuration Config { get; }
@@ -27,6 +37,7 @@ namespace AutoClient
public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
public CidRepo CidRepo { get; }
public Performance Performance { get; }
public FolderWorkDispatcher FolderWorkDispatcher { get; }
private IFileGenerator CreateGenerator()
{
+165
View File
@@ -0,0 +1,165 @@
using CodexOpenApi;
using CodexPlugin;
using Logging;
using Newtonsoft.Json;
using Utils;
namespace AutoClient
{
public class AutomaticPurchaser
{
private readonly ILog log;
private readonly ICodexInstance instance;
private readonly CodexNode codex;
private Task workerTask = Task.CompletedTask;
private App app => instance.App;
public AutomaticPurchaser(ILog log, ICodexInstance instance, CodexNode codex)
{
this.log = log;
this.instance = instance;
this.codex = codex;
}
public void Start()
{
workerTask = Task.Run(Worker);
}
public void Stop()
{
workerTask.Wait();
}
private async Task Worker()
{
log.Log("Worker started.");
while (!app.Cts.Token.IsCancellationRequested)
{
try
{
var pid = await StartNewPurchase();
await WaitTillFinished(pid);
await DownloadForeignCid();
}
catch (Exception ex)
{
log.Error("Worker failed with: " + ex);
await Task.Delay(TimeSpan.FromHours(6));
}
}
}
private async Task DownloadForeignCid()
{
var cid = app.CidRepo.GetForeignCid(instance.NodeId);
if (cid == null) return;
var size = app.CidRepo.GetSizeForCid(cid);
if (size == null) return;
var filename = Guid.NewGuid().ToString().ToLowerInvariant();
await codex.DownloadCid(filename, cid, size);
DeleteFile(filename);
}
private async Task<string> StartNewPurchase()
{
var file = await CreateFile();
try
{
var cid = await codex.UploadFile(file);
var response = await codex.RequestStorage(cid);
return response.PurchaseId;
}
finally
{
DeleteFile(file);
}
}
private async Task<string> CreateFile()
{
return await app.Generator.Generate();
}
private void DeleteFile(string file)
{
try
{
File.Delete(file);
}
catch (Exception exc)
{
app.Log.Error($"Failed to delete file '{file}': {exc}");
}
}
private async Task WaitTillFinished(string pid)
{
try
{
var emptyResponseTolerance = 10;
while (!app.Cts.Token.IsCancellationRequested)
{
var purchase = await codex.GetStoragePurchase(pid);
if (purchase == null)
{
await FixedShortDelay();
emptyResponseTolerance--;
if (emptyResponseTolerance == 0)
{
log.Log("Received 10 empty responses. Stop tracking this purchase.");
await ExpiryTimeDelay();
return;
}
continue;
}
if (purchase.IsCancelled)
{
app.Performance.StorageContractCancelled();
return;
}
if (purchase.IsError)
{
app.Performance.StorageContractErrored(purchase.Error);
return;
}
if (purchase.IsFinished)
{
app.Performance.StorageContractFinished();
return;
}
if (purchase.IsStarted)
{
app.Performance.StorageContractStarted();
await FixedDurationDelay();
}
await FixedShortDelay();
}
}
catch (Exception ex)
{
log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
await ExpiryTimeDelay();
}
}
private async Task FixedDurationDelay()
{
await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
}
private async Task ExpiryTimeDelay()
{
await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
}
private async Task FixedShortDelay()
{
await Task.Delay(15 * 1000, app.Cts.Token);
}
}
}
+1
View File
@@ -17,6 +17,7 @@
lock (_lock)
{
entries.Add(new CidEntry(nodeId, cid, knownSize));
if (entries.Count > 1000) entries.Clear();
}
}
+161
View File
@@ -0,0 +1,161 @@
using CodexOpenApi;
using CodexPlugin;
using Logging;
using Nethereum.Model;
using Newtonsoft.Json;
using Utils;
namespace AutoClient
{
public interface ICodexInstance
{
string NodeId { get; }
App App { get; }
CodexApi Codex { get; }
HttpClient Client { get; }
Address Address { get; }
}
public class CodexInstance : ICodexInstance
{
public CodexInstance(App app, CodexApi codex, HttpClient client, Address address)
{
App = app;
Codex = codex;
Client = client;
Address = address;
NodeId = Guid.NewGuid().ToString();
}
public string NodeId { get; }
public App App { get; }
public CodexApi Codex { get; }
public HttpClient Client { get; }
public Address Address { get; }
}
public class CodexNode
{
private readonly App app;
private readonly ICodexInstance codex;
public CodexNode(App app, ICodexInstance instance)
{
this.app = app;
codex = instance;
}
public async Task DownloadCid(string filename, string cid, long? size)
{
try
{
var sw = System.Diagnostics.Stopwatch.StartNew();
using var fileStream = File.OpenWrite(filename);
var fileResponse = await codex.Codex.DownloadNetworkStreamAsync(cid);
fileResponse.Stream.CopyTo(fileStream);
var time = sw.Elapsed;
app.Performance.DownloadSuccessful(size, time);
}
catch (Exception ex)
{
app.Performance.DownloadFailed(ex);
}
}
public async Task<ContentId> UploadFile(string filename)
{
using var fileStream = File.OpenRead(filename);
try
{
var info = new FileInfo(filename);
var sw = System.Diagnostics.Stopwatch.StartNew();
var cid = await UploadStream(fileStream, filename);
var time = sw.Elapsed;
app.Performance.UploadSuccessful(info.Length, time);
app.CidRepo.Add(codex.NodeId, cid.Id, info.Length);
return cid;
}
catch (Exception exc)
{
app.Performance.UploadFailed(exc);
throw;
}
}
public async Task<RequestStorageResult> RequestStorage(ContentId cid)
{
app.Log.Debug("Requesting storage for " + cid.Id);
var result = await codex.Codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
{
Collateral = app.Config.RequiredCollateral.ToString(),
Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
Nodes = app.Config.NumHosts,
Reward = app.Config.Price.ToString(),
ProofProbability = "15",
Tolerance = app.Config.HostTolerance
}, app.Cts.Token);
app.Log.Debug("Purchase ID: " + result);
var encoded = await GetEncodedCid(result);
app.CidRepo.AddEncoded(cid.Id, encoded);
return new RequestStorageResult(result, new ContentId(encoded));
}
public class RequestStorageResult
{
public RequestStorageResult(string purchaseId, ContentId encodedCid)
{
PurchaseId = purchaseId;
EncodedCid = encodedCid;
}
public string PurchaseId { get; }
public ContentId EncodedCid { get; }
public override string ToString()
{
return $"{PurchaseId} (cid: {EncodedCid})";
}
}
public async Task<StoragePurchase?> GetStoragePurchase(string pid)
{
// openapi still don't match code.
var str = await codex.Client.GetStringAsync($"{codex.Address.Host}:{codex.Address.Port}/api/codex/v1/storage/purchases/{pid}");
if (string.IsNullOrEmpty(str)) return null;
return JsonConvert.DeserializeObject<StoragePurchase>(str);
}
private async Task<ContentId> UploadStream(FileStream fileStream, string filename)
{
app.Log.Debug($"Uploading file...");
var response = await codex.Codex.UploadAsync(
content_type: "application/octet-stream",
content_disposition: $"attachment; filename=\"{filename}\"",
fileStream, app.Cts.Token);
if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
app.Log.Debug($"Uploaded file. Received contentId: '{response}'.");
return new ContentId(response);
}
private async Task<string> GetEncodedCid(string pid)
{
try
{
var sp = (await GetStoragePurchase(pid))!;
return sp.Request.Content.Cid;
}
catch (Exception ex)
{
app.Log.Error(ex.ToString());
throw;
}
}
}
}
+13 -6
View File
@@ -5,7 +5,8 @@ namespace AutoClient
public class Configuration
{
[Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
public string CodexEndpoints { get; set; } = "http://localhost:8080";
public string CodexEndpoints { get; set; } =
"http://localhost:8080";
[Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
public string DataPath { get; set; } = "datapath";
@@ -14,19 +15,22 @@ namespace AutoClient
public int NumConcurrentPurchases { get; set; } = 10;
[Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 6 hours)")]
public int ContractDurationMinutes { get; set; } = 60 * 6;
public int ContractDurationMinutes { get; set; } =
60 * 24 * 6; // 6 days
//60 * 6; 6 hours
// Cluster nodes configured for max 7-day storage.
[Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
public int ContractExpiryMinutes { get; set; } = 15;
public int ContractExpiryMinutes { get; set; } = 60;
[Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")]
public int NumHosts { get; set; } = 10;
public int NumHosts { get; set; } = 5;
[Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 5)")]
public int HostTolerance { get; set; } = 5;
public int HostTolerance { get; set; } = 1;
[Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
public int Price { get; set; } = 10;
public int Price { get; set; } = 1000;
[Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
public int RequiredCollateral { get; set; } = 1;
@@ -34,6 +38,9 @@ namespace AutoClient
[Uniform("filesizemb", "smb", "FILESIZEMB", false, "When greater than zero, size of file generated and uploaded. When zero, random images are used instead.")]
public int FileSizeMb { get; set; } = 0;
[Uniform("folderToStore", "fts", "FOLDERTOSTORE", false, "When set, autoclient will attempt to upload and purchase storage for every non-JSON file in the provided folder.")]
public string FolderToStore { get; set; } = "/data/EthereumMainnetPreMergeEraFiles";
public string LogPath
{
get
@@ -0,0 +1,58 @@
using static AutoClient.Modes.FolderStore.FileWorker;
namespace AutoClient.Modes.FolderStore
{
public class FileStatus : JsonBacked<WorkerStatus>
{
private readonly PurchaseInfo purchaseInfo;
public FileStatus(App app, string folder, string filePath, PurchaseInfo purchaseInfo)
: base(app, folder, filePath + ".json")
{
this.purchaseInfo = purchaseInfo;
}
public bool IsBusy()
{
if (!State.Purchases.Any()) return false;
return State.Purchases.Any(p =>
p.Submitted.HasValue &&
!p.Started.HasValue &&
!p.Expiry.HasValue &&
!p.Finish.HasValue &&
p.Created > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
);
}
public bool IsCurrentlyRunning()
{
if (!State.Purchases.Any()) return false;
return State.Purchases.Any(p =>
p.Submitted.HasValue &&
p.Started.HasValue &&
!p.Expiry.HasValue &&
!p.Finish.HasValue &&
p.Started.Value > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
);
}
public bool IsCurrentlyFailed()
{
if (!State.Purchases.Any()) return false;
var mostRecent = GetMostRecent();
if (mostRecent == null) return false;
return mostRecent.Expiry.HasValue;
}
protected WorkerPurchase? GetMostRecent()
{
if (!State.Purchases.Any()) return null;
var maxCreated = State.Purchases.Max(p => p.Created);
return State.Purchases.SingleOrDefault(p => p.Created == maxCreated);
}
}
}
@@ -0,0 +1,293 @@
using Logging;
namespace AutoClient.Modes.FolderStore
{
public class FileWorker : FileStatus
{
private readonly App app;
private readonly ILog log;
private readonly ICodexInstance instance;
private readonly PurchaseInfo purchaseInfo;
private readonly string sourceFilename;
private readonly Action onFileUploaded;
private readonly Action onNewPurchase;
private readonly CodexNode codex;
public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, FileIndex fileIndex, Action onFileUploaded, Action onNewPurchase)
: base(app, folder, fileIndex.File + ".json", purchaseInfo)
{
this.app = app;
log = new LogPrefixer(app.Log, GetFileTag(fileIndex));
this.instance = instance;
this.purchaseInfo = purchaseInfo;
sourceFilename = fileIndex.File;
if (sourceFilename.ToLowerInvariant().EndsWith(".json")) throw new Exception("Not an era file.");
this.onFileUploaded = onFileUploaded;
this.onNewPurchase = onNewPurchase;
codex = new CodexNode(app, instance);
}
public int FailureCounter => State.FailureCounter;
protected override void OnNewState(WorkerStatus newState)
{
newState.LastUpdate = DateTime.MinValue;
}
public async Task Update()
{
try
{
if (IsCurrentlyRunning() && UpdatedRecently()) return;
Log($"Updating for '{sourceFilename}'...");
await EnsureRecentPurchase();
SaveState();
app.Log.Log("");
}
catch (Exception exc)
{
app.Log.Error("Exception during fileworker update: " + exc);
State.Error = exc.ToString();
SaveState();
throw;
}
}
private bool UpdatedRecently()
{
var now = DateTime.UtcNow;
return State.LastUpdate + TimeSpan.FromMinutes(15) > now;
}
private async Task<string> EnsureCid()
{
Log($"Checking CID...");
if (!string.IsNullOrEmpty(State.EncodedCid) &&
await DoesCidExistInNetwork(State.EncodedCid))
{
Log("Encoded-CID successfully found in the network.");
// TODO: Using the encoded CID currently would result in double-encoding of the dataset.
// See: https://github.com/codex-storage/nim-codex/issues/1005
// Always use the basic CID for now, even though we have to repeat the encoding.
// When using encoded CID works: return State.EncodedCid;
}
if (!string.IsNullOrEmpty(State.Cid) &&
await DoesCidExistInNetwork(State.Cid))
{
Log("Basic-CID successfully found in the network.");
return State.Cid;
}
if (string.IsNullOrEmpty(State.Cid))
{
Log("File was not previously uploaded.");
}
Log($"Uploading...");
var cid = await codex.UploadFile(sourceFilename);
onFileUploaded();
Log("Got Basic-CID: " + cid);
State.Cid = cid.Id;
SaveState();
return State.Cid;
}
private async Task<bool> DoesCidExistInNetwork(string cid)
{
try
{
// This should not take longer than a few seconds. If it does, cancel it.
var cts = new CancellationTokenSource();
var cancelTask = Task.Run(() =>
{
Thread.Sleep(TimeSpan.FromSeconds(15));
cts.Cancel();
});
var manifest = await instance.Codex.DownloadNetworkManifestAsync(cid, cts.Token);
if (manifest == null) return false;
}
catch
{
return false;
}
return true;
}
private async Task EnsureRecentPurchase()
{
Log($"Checking recent purchase...");
var recent = GetMostRecent();
if (recent == null)
{
Log($"No recent purchase.");
await MakeNewPurchase();
return;
}
await UpdatePurchase(recent);
if (recent.Expiry.HasValue)
{
Log($"Purchase has failed or expired.");
await MakeNewPurchase();
State.FailureCounter++;
return;
}
if (recent.Finish.HasValue)
{
Log($"Purchase has finished.");
await MakeNewPurchase();
return;
}
var safeEnd = recent.Created + purchaseInfo.PurchaseDurationSafe;
if (recent.Started.HasValue && DateTime.UtcNow > safeEnd)
{
Log($"Purchase is going to expire soon.");
await MakeNewPurchase();
return;
}
if (!recent.Submitted.HasValue)
{
Log($"Purchase is waiting to be submitted.");
return;
}
if (recent.Submitted.HasValue && !recent.Started.HasValue)
{
Log($"Purchase is submitted and waiting to start.");
return;
}
Log($"Purchase is running.");
}
private async Task UpdatePurchase(WorkerPurchase recent)
{
if (string.IsNullOrEmpty(recent.Pid)) throw new Exception("No purchaseID!");
var now = DateTime.UtcNow;
var purchase = await codex.GetStoragePurchase(recent.Pid);
if (purchase == null)
{
Log($"No purchase information found for PID '{recent.Pid}'. Consider this one expired.");
recent.Expiry = now;
return;
}
if (purchase.IsSubmitted)
{
if (!recent.Submitted.HasValue) recent.Submitted = now;
}
if (purchase.IsStarted)
{
if (!recent.Submitted.HasValue) recent.Submitted = now;
if (!recent.Started.HasValue) recent.Started = now;
}
if (purchase.IsCancelled)
{
if (!recent.Submitted.HasValue) recent.Submitted = now;
if (!recent.Expiry.HasValue) recent.Expiry = now;
}
if (purchase.IsError)
{
if (!recent.Submitted.HasValue) recent.Submitted = now;
if (!recent.Expiry.HasValue) recent.Expiry = now;
}
if (purchase.IsFinished)
{
if (!recent.Submitted.HasValue) recent.Submitted = now;
if (!recent.Started.HasValue) recent.Started = now;
if (!recent.Finish.HasValue) recent.Finish = now;
}
State.LastUpdate = now;
SaveState();
}
private async Task MakeNewPurchase()
{
var cid = await EnsureCid();
if (string.IsNullOrEmpty(cid)) throw new Exception("No cid!");
Log($"Creating new purchase...");
var response = await codex.RequestStorage(new CodexPlugin.ContentId(cid));
var purchaseId = response.PurchaseId;
var encodedCid = response.EncodedCid;
if (string.IsNullOrEmpty(purchaseId) ||
purchaseId == "Unable to encode manifest" ||
purchaseId == "Purchasing not available" ||
purchaseId == "Expiry required" ||
purchaseId == "Expiry needs to be in future" ||
purchaseId == "Expiry has to be before the request's end (now + duration)")
{
throw new InvalidOperationException(purchaseId);
}
var newPurchase = new WorkerPurchase
{
Created = DateTime.UtcNow,
Pid = purchaseId
};
State.Purchases = State.Purchases.Concat([newPurchase]).ToArray();
State.EncodedCid = encodedCid.Id;
SaveState();
onNewPurchase();
Log($"New purchase created. PID: '{purchaseId}'.");
Log("Got Encoded-CID: " + encodedCid);
Log("Waiting for submit...");
Thread.Sleep(500);
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
while (DateTime.UtcNow < timeout)
{
Thread.Sleep(5000);
await UpdatePurchase(newPurchase);
if (newPurchase.Submitted.HasValue)
{
Log("New purchase successfully submitted.");
return;
}
}
Log("New purchase was not submitted within 5-minute timeout. Will check again later...");
}
private void Log(string msg)
{
log.Log(msg);
}
private string GetFileTag(FileIndex filename)
{
return $"({filename.Index.ToString("00000")}) ";
}
[Serializable]
public class WorkerStatus
{
public DateTime LastUpdate { get; set; }
public string Cid { get; set; } = string.Empty;
public string EncodedCid { get; set; } = string.Empty;
public int FailureCounter { get; set; } = 0;
public string Error { get; set; } = string.Empty;
public WorkerPurchase[] Purchases { get; set; } = Array.Empty<WorkerPurchase>();
}
[Serializable]
public class WorkerPurchase
{
public string Pid { get; set; } = string.Empty;
public DateTime Created { get; set; }
public DateTime? Submitted { get; set; }
public DateTime? Started { get; set; }
public DateTime? Expiry { get; set; }
public DateTime? Finish { get; set; }
}
}
}
@@ -0,0 +1,70 @@
using Logging;
namespace AutoClient.Modes.FolderStore
{
public class FolderWorkDispatcher
{
private readonly string[] files = Array.Empty<string>();
private readonly ILog log;
private int index = 0;
private int busyCount = 0;
public FolderWorkDispatcher(ILog log, string folder)
{
var fs = Directory.GetFiles(folder);
var result = new List<string>();
foreach (var f in fs)
{
if (!f.ToLowerInvariant().Contains(".json"))
{
var info = new FileInfo(f);
if (info.Exists && info.Length > 1024 * 1024) // larger than 1MB
{
result.Add(f);
}
}
}
files = result.ToArray();
this.log = log;
}
public FileIndex GetFileToCheck()
{
if (busyCount > 0)
{
log.Log("");
log.Log("Max number of busy workers reached. Waiting until contracts are started before creating any more.");
log.Log("");
ResetIndex();
Thread.Sleep(TimeSpan.FromMinutes(1));
}
var file = new FileIndex(files[index], index);
index = (index + 1) % files.Length;
return file;
}
public void ResetIndex()
{
index = 0;
busyCount = 0;
}
public void WorkerIsBusy()
{
busyCount++;
}
}
public class FileIndex
{
public FileIndex(string file, int index)
{
File = file;
Index = index;
}
public string File { get; }
public int Index { get; }
}
}
@@ -0,0 +1,136 @@
using CodexOpenApi;
using System.IO.Compression;
using static AutoClient.Modes.FolderStore.FolderWorkOverview;
namespace AutoClient.Modes.FolderStore
{
public class FolderWorkOverview : JsonBacked<WorkMonitorStatus>
{
private const string OverviewFilename = "codex_folder_saver_overview.json";
private readonly App app;
private readonly PurchaseInfo purchaseInfo;
public FolderWorkOverview(App app, PurchaseInfo purchaseInfo, string folder)
: base(app, folder, Path.Combine(folder, OverviewFilename))
{
this.app = app;
this.purchaseInfo = purchaseInfo;
}
protected override void OnNewState(WorkMonitorStatus newState)
{
newState.LastOverviewUpdate = DateTime.MinValue;
}
public async Task Update(ICodexInstance instance)
{
var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList();
var total = 0;
var successful = 0;
var failed = 0;
foreach (var file in jsonFiles)
{
try
{
var worker = new FileStatus(app, Folder, file.Substring(0, file.Length - 5), purchaseInfo);
total++;
if (worker.IsCurrentlyRunning()) successful++;
if (worker.IsCurrentlyFailed()) failed++;
}
catch (Exception exc)
{
app.Log.Error("Exception in workoverview update: " + exc);
}
}
State.TotalFiles = total;
State.SuccessfulStored = successful;
State.StoreFailed = failed;
SaveState();
if (State.UncommitedChanges > 3)
{
State.UncommitedChanges = 0;
SaveState();
await CreateNewOverviewZip(jsonFiles, FilePath, instance);
}
}
public void MarkUncommitedChange()
{
State.UncommitedChanges++;
SaveState();
}
private async Task CreateNewOverviewZip(List<string> jsonFiles, string filePath, ICodexInstance instance)
{
Log("");
Log("");
Log("Creating new overview zipfile...");
var zipFilename = CreateZipFile(jsonFiles, filePath);
Log("Uploading to Codex...");
try
{
var codex = new CodexNode(app, instance);
var cid = await codex.UploadFile(zipFilename);
Log($"Upload successful: New overview zipfile CID = '{cid.Id}'");
Log("Requesting storage for it...");
var result = await codex.RequestStorage(cid);
Log("Storage requested. Purchase ID: " + result);
var outFile = Path.Combine(app.Config.DataPath, "OverviewZip.cid");
File.AppendAllLines(outFile, [DateTime.UtcNow.ToString("o") + " - " + result.EncodedCid.Id]);
Log($">>> [{outFile}] has been updated. <<<");
}
catch (Exception exc)
{
Log("Failed to upload new overview zipfile: " + exc);
}
Log("");
Log("");
}
private string CreateZipFile(List<string> jsonFiles, string filePath)
{
var zipFilename = Guid.NewGuid().ToString() + ".zip";
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
archive.CreateEntryFromFile(filePath, "overview.json");
foreach (var file in jsonFiles)
{
archive.CreateEntryFromFile(file, Path.GetFileName(file));
}
}
using (var fileStream = new FileStream(zipFilename, FileMode.Create))
{
memoryStream.Seek(0, SeekOrigin.Begin);
memoryStream.CopyTo(fileStream);
}
}
return zipFilename;
}
private void Log(string msg)
{
app.Log.Log(msg);
}
[Serializable]
public class WorkMonitorStatus
{
public int TotalFiles { get; set; }
public int SuccessfulStored { get; set; }
public int StoreFailed { get; set; }
public DateTime LastOverviewUpdate { get; set; }
public int UncommitedChanges { get; set; }
}
}
}
@@ -0,0 +1,58 @@
using Newtonsoft.Json;
namespace AutoClient.Modes.FolderStore
{
public abstract class JsonBacked<T> where T : new()
{
private readonly App app;
protected JsonBacked(App app, string folder, string filePath)
{
this.app = app;
Folder = folder;
FilePath = filePath;
LoadState();
}
private void LoadState()
{
try
{
if (!File.Exists(FilePath))
{
State = new T();
OnNewState(State);
SaveState();
}
var text = File.ReadAllText(FilePath);
State = JsonConvert.DeserializeObject<T>(text)!;
if (State == null) throw new Exception("Didn't deserialize " + FilePath);
}
catch (Exception exc)
{
app.Log.Error("Failed to load state: " + exc);
}
}
protected string Folder { get; }
protected string FilePath { get; }
protected T State { get; private set; } = default!;
protected virtual void OnNewState(T newState)
{
}
protected void SaveState()
{
try
{
var json = JsonConvert.SerializeObject(State);
File.WriteAllText(FilePath, json);
}
catch (Exception exc)
{
app.Log.Error("Failed to save state: " + exc);
}
}
}
}
@@ -0,0 +1,18 @@
namespace AutoClient.Modes.FolderStore
{
public class PurchaseInfo
{
public PurchaseInfo(TimeSpan purchaseDurationTotal, TimeSpan purchaseDurationSafe)
{
PurchaseDurationTotal = purchaseDurationTotal;
PurchaseDurationSafe = purchaseDurationSafe;
if (PurchaseDurationTotal < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationTotal));
if (PurchaseDurationSafe < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationSafe));
if (PurchaseDurationTotal < PurchaseDurationSafe) throw new Exception("TotalDuration < SafeDuration");
}
public TimeSpan PurchaseDurationTotal { get; }
public TimeSpan PurchaseDurationSafe { get; }
}
}
+86
View File
@@ -0,0 +1,86 @@
using AutoClient.Modes.FolderStore;
namespace AutoClient.Modes
{
public class FolderStoreMode : IMode
{
private readonly App app;
private readonly string folder;
private readonly PurchaseInfo purchaseInfo;
private readonly CancellationTokenSource cts = new CancellationTokenSource();
private Task checkTask = Task.CompletedTask;
public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo)
{
this.app = app;
this.folder = folder;
this.purchaseInfo = purchaseInfo;
}
public void Start(ICodexInstance instance, int index)
{
checkTask = Task.Run(async () =>
{
try
{
await RunChecker(instance);
}
catch (Exception ex)
{
app.Log.Error("Exception in FolderStoreMode worker: " + ex);
Environment.Exit(1);
}
});
}
private async Task RunChecker(ICodexInstance instance)
{
var i = 0;
while (!cts.IsCancellationRequested)
{
Thread.Sleep(2000);
var worker = await ProcessWorkItem(instance);
if (worker.FailureCounter > 5)
{
throw new Exception("Worker has failure count > 5. Stopping AutoClient...");
}
i++;
if (i > 5)
{
i = 0;
var overview = new FolderWorkOverview(app, purchaseInfo, folder);
await overview.Update(instance);
}
}
}
private async Task<FileWorker> ProcessWorkItem(ICodexInstance instance)
{
var file = app.FolderWorkDispatcher.GetFileToCheck();
var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnFileUploaded, OnNewPurchase);
await worker.Update();
if (worker.IsBusy()) app.FolderWorkDispatcher.WorkerIsBusy();
return worker;
}
private void OnFileUploaded()
{
}
private void OnNewPurchase()
{
app.FolderWorkDispatcher.ResetIndex();
var overview = new FolderWorkOverview(app, purchaseInfo, folder);
overview.MarkUncommitedChange();
}
public void Stop()
{
cts.Cancel();
checkTask.Wait();
}
}
}
+14
View File
@@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace AutoClient.Modes
{
public interface IMode
{
void Start(ICodexInstance instance, int index);
void Stop();
}
}
@@ -1,32 +1,23 @@
using CodexOpenApi;
using Logging;
using Utils;
using Logging;
namespace AutoClient
namespace AutoClient.Modes
{
public class CodexUser
public class PurchasingMode : IMode
{
private readonly List<AutomaticPurchaser> purchasers = new List<AutomaticPurchaser>();
private readonly App app;
private readonly CodexApi codex;
private readonly HttpClient client;
private readonly Address address;
private readonly List<Purchaser> purchasers = new List<Purchaser>();
private Task starterTask = Task.CompletedTask;
private readonly string nodeId = Guid.NewGuid().ToString();
public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
public PurchasingMode(App app)
{
this.app = app;
this.codex = codex;
this.client = client;
this.address = address;
}
public void Start(int index)
public void Start(ICodexInstance instance, int index)
{
for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
{
purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
purchasers.Add(new AutomaticPurchaser(new LogPrefixer(app.Log, $"({i}) "), instance, new CodexNode(app, instance)));
}
var delayPerPurchaser =
+4 -2
View File
@@ -16,11 +16,13 @@ namespace AutoClient
Log($"Download failed: {ex}");
}
public void DownloadSuccessful(long size, TimeSpan time)
public void DownloadSuccessful(long? size, TimeSpan time)
{
if (!size.HasValue) return;
long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
if (milliseconds < 1) milliseconds = 1;
long bytesPerSecond = 1000 * (size / milliseconds);
long bytesPerSecond = 1000 * (size.Value / milliseconds);
Log($"Download successful: {bytesPerSecond} bytes per second");
}
+38 -10
View File
@@ -1,11 +1,14 @@
using ArgsUniform;
using AutoClient;
using AutoClient.Modes;
using AutoClient.Modes.FolderStore;
using CodexOpenApi;
using Utils;
public class Program
{
private readonly App app;
private readonly List<IMode> modes = new List<IMode>();
public Program(Configuration config)
{
@@ -31,36 +34,60 @@ public class Program
public async Task Run()
{
var codexUsers = await CreateUsers();
var codexInstances = await CreateCodexInstances();
var i = 0;
foreach (var user in codexUsers)
foreach (var cdx in codexInstances)
{
user.Start(i);
var mode = CreateMode();
modes.Add(mode);
mode.Start(cdx, i);
i++;
}
app.Cts.Token.WaitHandle.WaitOne();
foreach (var user in codexUsers) user.Stop();
foreach (var mode in modes) mode.Stop();
modes.Clear();
app.Log.Log("Done");
}
private async Task<CodexUser[]> CreateUsers()
private IMode CreateMode()
{
if (!string.IsNullOrEmpty(app.Config.FolderToStore))
{
return CreateFolderStoreMode();
}
return new PurchasingMode(app);
}
private IMode CreateFolderStoreMode()
{
if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!");
return new FolderStoreMode(app, app.Config.FolderToStore, new PurchaseInfo(
purchaseDurationTotal: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes),
purchaseDurationSafe: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes - 120)
));
}
private async Task<CodexInstance[]> CreateCodexInstances()
{
var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
var result = new List<CodexUser>();
var result = new List<CodexInstance>();
foreach (var e in endpointStrs)
{
result.Add(await CreateUser(e));
result.Add(await CreateCodexInstance(e));
}
return result.ToArray();
}
private async Task<CodexUser> CreateUser(string endpoint)
private async Task<CodexInstance> CreateCodexInstance(string endpoint)
{
var splitIndex = endpoint.LastIndexOf(':');
var host = endpoint.Substring(0, splitIndex);
@@ -72,6 +99,7 @@ public class Program
);
var client = new HttpClient();
client.Timeout = TimeSpan.FromMinutes(60.0);
var codex = new CodexApi(client);
codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
@@ -79,7 +107,7 @@ public class Program
await CheckCodex(codex);
app.Log.Log("OK");
return new CodexUser(
return new CodexInstance(
app,
codex,
client,
@@ -105,4 +133,4 @@ public class Program
{
Console.WriteLine("Generates fake data and creates Codex storage contracts for it.");
}
}
}
-261
View File
@@ -1,261 +0,0 @@
using CodexOpenApi;
using CodexPlugin;
using Logging;
using Newtonsoft.Json;
using Utils;
namespace AutoClient
{
public class Purchaser
{
private readonly App app;
private readonly string nodeId;
private readonly ILog log;
private readonly HttpClient client;
private readonly Address address;
private readonly CodexApi codex;
private Task workerTask = Task.CompletedTask;
public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
{
this.app = app;
this.nodeId = nodeId;
this.log = log;
this.client = client;
this.address = address;
this.codex = codex;
}
public void Start()
{
workerTask = Task.Run(Worker);
}
public void Stop()
{
workerTask.Wait();
}
private async Task Worker()
{
log.Log("Worker started.");
while (!app.Cts.Token.IsCancellationRequested)
{
try
{
var pid = await StartNewPurchase();
await WaitTillFinished(pid);
await DownloadForeignCid();
}
catch (Exception ex)
{
log.Error("Worker failed with: " + ex);
await Task.Delay(TimeSpan.FromHours(6));
}
}
}
private async Task DownloadForeignCid()
{
var cid = app.CidRepo.GetForeignCid(nodeId);
if (cid == null) return;
var size = app.CidRepo.GetSizeForCid(cid);
if (size == null) return;
try
{
var sw = System.Diagnostics.Stopwatch.StartNew();
var filename = Guid.NewGuid().ToString().ToLowerInvariant();
{
using var fileStream = File.OpenWrite(filename);
var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
fileResponse.Stream.CopyTo(fileStream);
}
var time = sw.Elapsed;
File.Delete(filename);
app.Performance.DownloadSuccessful(size.Value, time);
}
catch (Exception ex)
{
app.Performance.DownloadFailed(ex);
}
}
private async Task<string> StartNewPurchase()
{
var file = await CreateFile();
try
{
var cid = await UploadFile(file);
return await RequestStorage(cid);
}
finally
{
DeleteFile(file);
}
}
private async Task<string> CreateFile()
{
return await app.Generator.Generate();
}
private void DeleteFile(string file)
{
try
{
File.Delete(file);
}
catch (Exception exc)
{
app.Log.Error($"Failed to delete file '{file}': {exc}");
}
}
private async Task<ContentId> UploadFile(string filename)
{
using var fileStream = File.OpenRead(filename);
try
{
var info = new FileInfo(filename);
var sw = System.Diagnostics.Stopwatch.StartNew();
var cid = await UploadStream(fileStream, filename);
var time = sw.Elapsed;
app.Performance.UploadSuccessful(info.Length, time);
app.CidRepo.Add(nodeId, cid.Id, info.Length);
return cid;
}
catch (Exception exc)
{
app.Performance.UploadFailed(exc);
throw;
}
}
private async Task<ContentId> UploadStream(FileStream fileStream, string filename)
{
log.Debug($"Uploading file...");
var response = await codex.UploadAsync(
content_type: "application/x-binary",
content_disposition: $"attachment; filename=\"{filename}\"",
fileStream, app.Cts.Token);
if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
log.Debug($"Uploaded file. Received contentId: '{response}'.");
return new ContentId(response);
}
private async Task<string> RequestStorage(ContentId cid)
{
log.Debug("Requesting storage for " + cid.Id);
var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
{
Collateral = app.Config.RequiredCollateral.ToString(),
Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
Nodes = app.Config.NumHosts,
Reward = app.Config.Price.ToString(),
ProofProbability = "15",
Tolerance = app.Config.HostTolerance
}, app.Cts.Token);
log.Debug("Purchase ID: " + result);
var encoded = await GetEncodedCid(result);
app.CidRepo.AddEncoded(cid.Id, encoded);
return result;
}
private async Task<string> GetEncodedCid(string pid)
{
try
{
var sp = (await GetStoragePurchase(pid))!;
return sp.Request.Content.Cid;
}
catch (Exception ex)
{
log.Error(ex.ToString());
throw;
}
}
private async Task<StoragePurchase?> GetStoragePurchase(string pid)
{
// openapi still don't match code.
var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
if (string.IsNullOrEmpty(str)) return null;
return JsonConvert.DeserializeObject<StoragePurchase>(str);
}
private async Task WaitTillFinished(string pid)
{
try
{
var emptyResponseTolerance = 10;
while (!app.Cts.Token.IsCancellationRequested)
{
var purchase = await GetStoragePurchase(pid);
if (purchase == null)
{
await FixedShortDelay();
emptyResponseTolerance--;
if (emptyResponseTolerance == 0)
{
log.Log("Received 10 empty responses. Stop tracking this purchase.");
await ExpiryTimeDelay();
return;
}
continue;
}
var status = purchase.State.ToLowerInvariant();
if (status.Contains("cancel"))
{
app.Performance.StorageContractCancelled();
return;
}
if (status.Contains("error"))
{
app.Performance.StorageContractErrored(purchase.Error);
return;
}
if (status.Contains("finished"))
{
app.Performance.StorageContractFinished();
return;
}
if (status.Contains("started"))
{
app.Performance.StorageContractStarted();
await FixedDurationDelay();
}
await FixedShortDelay();
}
}
catch (Exception ex)
{
log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
await ExpiryTimeDelay();
}
}
private async Task FixedDurationDelay()
{
await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
}
private async Task ExpiryTimeDelay()
{
await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
}
private async Task FixedShortDelay()
{
await Task.Delay(15 * 1000, app.Cts.Token);
}
}
}
+1 -1
View File
@@ -66,7 +66,7 @@ namespace BiblioTech
success: true,
title: $"Success: '{content.Cid}'",
error: "",
$"size: {content.Manifest.OriginalBytes} bytes",
$"size: {content.Manifest.DatasetSize} bytes",
$"blockSize: {content.Manifest.BlockSize} bytes",
$"protected: {content.Manifest.Protected}"
);
+1 -8
View File
@@ -76,9 +76,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CsvCombiner", "Tools\CsvCombiner\CsvCombiner.csproj", "{6230347F-5045-4E25-8E7A-13D7221B7444}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DevconBoothImages", "DevconBoothImages\DevconBoothImages.csproj", "{92AC64E7-F6B1-474E-B915-30C8EEE2F9D7}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CsvCombiner", "Tools\CsvCombiner\CsvCombiner.csproj", "{6230347F-5045-4E25-8E7A-13D7221B7444}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -210,10 +208,6 @@ Global
{6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.Build.0 = Release|Any CPU
{92AC64E7-F6B1-474E-B915-30C8EEE2F9D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{92AC64E7-F6B1-474E-B915-30C8EEE2F9D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{92AC64E7-F6B1-474E-B915-30C8EEE2F9D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{92AC64E7-F6B1-474E-B915-30C8EEE2F9D7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -250,7 +244,6 @@ Global
{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
{004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
{6230347F-5045-4E25-8E7A-13D7221B7444} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
{92AC64E7-F6B1-474E-B915-30C8EEE2F9D7} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}