logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-06-04 07:29:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/multi-codex-folder-saver

Browse Source
This commit is contained in:
ThatBen 2025-07-30 14:54:20 +02:00
commit 0ffde95838
No known key found for this signature in database
GPG Key ID: E020A7DDCD52E1AB
58 changed files with 843 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Framework/ArgsUniform/Assigner.cs
View File

@ -95,7 +95,10 @@ namespace ArgsUniform
}
else
{
if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
if (
uniformProperty.PropertyType == typeof(string) ||
uniformProperty.PropertyType == typeof(int) ||
uniformProperty.PropertyType == typeof(decimal))
{
uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
return true;

1
Framework/Core/PluginTools.cs
View File

@ -7,7 +7,6 @@ namespace Core
{
public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactory, IFileTool
{
IWebCallTimeSet WebCallTimeSet { get; }
IK8sTimeSet K8STimeSet { get; }
/// <summary>

8
Framework/KubernetesWorkflow/K8sController.cs
View File

@ -724,9 +724,11 @@ namespace KubernetesWorkflow
private V1Pod GetPodForDeployment(RunningDeployment deployment)
{
return Time.Retry(() => GetPodForDeplomentInternal(deployment),
// We will wait up to 1 minute, k8s might be moving pods around.
maxTimeout: TimeSpan.FromMinutes(1),
retryTime: TimeSpan.FromSeconds(10),
// K8s might be moving pods around. If it's scaling the cluster
// to handle the increased load, it might take a while before the new
// VMs are up and ready. So we use a generous timeout.
maxTimeout: TimeSpan.FromMinutes(15.0),
retryTime: TimeSpan.FromSeconds(30.0),
description: "Find pod by label for deployment.");
}

7
Framework/KubernetesWorkflow/StartupWorkflow.cs
View File

@ -61,8 +61,7 @@ namespace KubernetesWorkflow
var startResult = controller.BringOnline(recipes, location);
var containers = CreateContainers(startResult, recipes, startupConfig);
var info = GetPodInfo(startResult.Deployment);
var rc = new RunningPod(Guid.NewGuid().ToString(), info, startupConfig, startResult, containers);
var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
cluster.Configuration.Hooks.OnContainersStarted(rc);
if (startResult.ExternalService != null)
@ -73,7 +72,7 @@ namespace KubernetesWorkflow
});
}
public void WaitUntilOnline(RunningPod rc)
public PodInfo WaitUntilOnline(RunningPod rc)
{
K8s(controller =>
{
@ -82,6 +81,8 @@ namespace KubernetesWorkflow
controller.WaitUntilOnline(c);
}
});
return GetPodInfo(rc.StartResult.Deployment);
}
public PodInfo GetPodInfo(RunningDeployment deployment)

3
Framework/KubernetesWorkflow/Types/FutureContainers.cs
View File

@ -13,7 +13,8 @@
public RunningPod WaitForOnline()
{
workflow.WaitUntilOnline(runningPod);
var podInfo = workflow.WaitUntilOnline(runningPod);
runningPod.Initialize(podInfo);
return runningPod;
}
}

11
Framework/KubernetesWorkflow/Types/RunningPod.cs
View File

@ -4,10 +4,10 @@ namespace KubernetesWorkflow.Types
{
public class RunningPod
{
public RunningPod(string id, PodInfo podInfo, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
{
Id = id;
PodInfo = podInfo;
PodInfo = null!;
StartupConfig = startupConfig;
StartResult = startResult;
Containers = containers;
@ -16,7 +16,7 @@ namespace KubernetesWorkflow.Types
}
public string Id { get; }
public PodInfo PodInfo { get; }
public PodInfo PodInfo { get; private set; }
public StartupConfig StartupConfig { get; }
public StartResult StartResult { get; }
public RunningContainer[] Containers { get; }
@ -30,6 +30,11 @@ namespace KubernetesWorkflow.Types
[JsonIgnore]
public bool IsStopped { get; internal set; }
public void Initialize(PodInfo podInfo)
{
PodInfo = podInfo;
}
public string Describe()
{
return string.Join(",", Containers.Select(c => c.Name));

41
Framework/NethereumWorkflow/NethereumInteraction.cs
View File

@ -1,4 +1,5 @@
using BlockchainUtils;
using System.Numerics;
using BlockchainUtils;
using Logging;
using Nethereum.ABI.FunctionEncoding.Attributes;
using Nethereum.Contracts;
@ -22,25 +23,26 @@ namespace NethereumWorkflow
this.blockCache = blockCache;
}
public string SendEth(string toAddress, decimal ethAmount)
public string SendEth(string toAddress, BigInteger ethAmount)
{
log.Debug();
var receipt = Time.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(toAddress, ethAmount));
var receipt = Time.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(toAddress, ((decimal)ethAmount)));
if (!receipt.Succeeded()) throw new Exception("Unable to send Eth");
return receipt.TransactionHash;
}
public decimal GetEthBalance()
public BigInteger GetEthBalance()
{
log.Debug();
return GetEthBalance(web3.TransactionManager.Account.Address);
}
public decimal GetEthBalance(string address)
public BigInteger GetEthBalance(string address)
{
log.Debug();
var balance = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(address));
return Web3.Convert.FromWei(balance.Value);
return balance.Value;
}
public TResult Call<TFunction, TResult>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
@ -57,6 +59,13 @@ namespace NethereumWorkflow
return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function, new BlockParameter(blockNumber)));
}
public void Call<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
{
log.Debug(typeof(TFunction).ToString());
var handler = web3.Eth.GetContractQueryHandler<TFunction>();
Time.Wait(handler.QueryRawAsync(contractAddress, function));
}
public void Call<TFunction>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new()
{
log.Debug(typeof(TFunction).ToString());
@ -110,15 +119,29 @@ namespace NethereumWorkflow
public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
{
var eventHandler = web3.Eth.GetEvent<TEvent>(address);
var logs = new List<FilterLog>();
var p = web3.Processing.Logs.CreateProcessor(
action: logs.Add,
minimumBlockConfirmations: 1,
criteria: l => l.IsLogForEvent<TEvent>()
);
var from = new BlockParameter(fromBlockNumber);
var to = new BlockParameter(toBlockNumber);
var blockFilter = Time.Wait(eventHandler.CreateFilterBlockRangeAsync(from, to));
return Time.Wait(eventHandler.GetAllChangesAsync(blockFilter));
var ct = new CancellationTokenSource().Token;
Time.Wait(p.ExecuteAsync(toBlockNumber: to.BlockNumber, cancellationToken: ct, startAtBlockNumberIfNotProcessed: from.BlockNumber));
return logs
.Where(l => l.IsLogForEvent<TEvent>())
.Select(l => l.DecodeEvent<TEvent>())
.ToList();
}
public BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange)
{
if (timeRange.To - timeRange.From < TimeSpan.FromSeconds(1.0))
throw new Exception(nameof(ConvertTimeRangeToBlockRange) + ": Time range too small.");
var wrapper = new Web3Wrapper(web3, log);
var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);

26
Framework/Utils/EthTokenExtensions.cs
View File

@ -1,15 +1,17 @@
namespace Utils
using System.Numerics;
namespace Utils
{
public class Ether : IComparable<Ether>
{
public Ether(decimal wei)
public Ether(BigInteger wei)
{
Wei = wei;
Eth = wei / TokensIntExtensions.WeiPerEth;
}
public decimal Wei { get; }
public decimal Eth { get; }
public BigInteger Wei { get; }
public BigInteger Eth { get; }
public int CompareTo(Ether? other)
{
@ -75,7 +77,7 @@
public static class TokensIntExtensions
{
public const decimal WeiPerEth = 1000000000000000000;
public static readonly BigInteger WeiPerEth = new BigInteger(1000000000000000000);
public static Ether Eth(this int i)
{
@ -89,10 +91,22 @@
public static Ether Eth(this decimal i)
{
return new Ether(i * WeiPerEth);
var a = new BigInteger(i);
return new Ether(a * WeiPerEth);
}
public static Ether Wei(this decimal i)
{
var a = new BigInteger(i);
return new Ether(a);
}
public static Ether Eth(this BigInteger i)
{
return new Ether(i * WeiPerEth);
}
public static Ether Wei(this BigInteger i)
{
return new Ether(i);
}

14
Framework/WebUtils/HttpFactory.cs
View File

@ -7,12 +7,14 @@ namespace WebUtils
IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet);
IHttp CreateHttp(string id);
IWebCallTimeSet WebCallTimeSet { get; }
}
public class HttpFactory : IHttpFactory
{
private readonly ILog log;
private readonly IWebCallTimeSet defaultTimeSet;
private readonly IWebCallTimeSet timeSet;
private readonly Action<HttpClient> factoryOnClientCreated;
public HttpFactory(ILog log)
@ -33,13 +35,15 @@ namespace WebUtils
public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet, Action<HttpClient> onClientCreated)
{
this.log = log;
this.defaultTimeSet = defaultTimeSet;
this.factoryOnClientCreated = onClientCreated;
timeSet = defaultTimeSet;
factoryOnClientCreated = onClientCreated;
}
public IWebCallTimeSet WebCallTimeSet => timeSet;
public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
{
return CreateHttp(id, onClientCreated, defaultTimeSet);
return CreateHttp(id, onClientCreated, timeSet);
}
public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet ts)
@ -53,7 +57,7 @@ namespace WebUtils
public IHttp CreateHttp(string id)
{
return new Http(id, log, defaultTimeSet, factoryOnClientCreated);
return new Http(id, log, timeSet, factoryOnClientCreated);
}
private static void DoNothing(HttpClient client)

4
Framework/WebUtils/WebCallTimeSet.cs
View File

@ -23,12 +23,12 @@
{
public TimeSpan HttpCallTimeout()
{
return TimeSpan.FromMinutes(2);
return TimeSpan.FromMinutes(5);
}
public TimeSpan HttpRetryTimeout()
{
return TimeSpan.FromMinutes(5);
return TimeSpan.FromMinutes(20);
}
public TimeSpan HttpCallRetryDelay()

20
ProjectPlugins/CodexClient/CodexAccess.cs
View File

@ -1,4 +1,5 @@
using CodexOpenApi;
using System.Threading;
using CodexOpenApi;
using Logging;
using Newtonsoft.Json;
using Utils;
@ -119,7 +120,7 @@ namespace CodexClient
public Stream DownloadFile(string contentId)
{
var fileResponse = OnCodex(api => api.DownloadNetworkStreamAsync(contentId));
var fileResponse = OnCodexNoRetry(api => api.DownloadNetworkStreamAsync(contentId));
if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
return fileResponse.Stream;
}
@ -212,15 +213,22 @@ namespace CodexClient
processControl.DeleteDataDirFolder();
}
private T OnCodex<T>(Func<CodexApiClient, Task<T>> action)
private T OnCodexNoRetry<T>(Func<CodexApiClient, Task<T>> action)
{
var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action));
var timeSet = httpFactory.WebCallTimeSet;
var noRetry = new Retry(nameof(OnCodexNoRetry),
maxTimeout: TimeSpan.FromSeconds(1.0),
sleepAfterFail: TimeSpan.FromSeconds(2.0),
onFail: f => { },
failFast: true);
var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action), noRetry);
return result;
}
private T OnCodex<T>(Func<CodexApiClient, Task<T>> action, Retry retry)
private T OnCodex<T>(Func<CodexApiClient, Task<T>> action)
{
var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action), retry);
var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action));
return result;
}

26
ProjectPlugins/CodexClient/CodexNode.cs
View File

@ -17,6 +17,7 @@ namespace CodexClient
ContentId UploadFile(TrackedFile file);
ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition);
TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
TrackedFile? DownloadContent(ContentId contentId, TimeSpan timeout, string fileLabel = "");
LocalDataset DownloadStreamless(ContentId cid);
/// <summary>
/// TODO: This will monitor the quota-used of the node until 'size' bytes are added. That's a very bad way
@ -77,7 +78,16 @@ namespace CodexClient
public void Initialize()
{
InitializePeerNodeId();
// This is the moment we first connect to a codex node. Sometimes, Kubernetes takes a while to spin up the
// container. So we'll adding a custom, generous retry here.
var kubeSpinupRetry = new Retry("CodexNode_Initialize",
maxTimeout: TimeSpan.FromMinutes(10.0),
sleepAfterFail: TimeSpan.FromSeconds(10.0),
onFail: f => { },
failFast: false);
kubeSpinupRetry.Run(InitializePeerNodeId);
InitializeLogReplacements();
hooks.OnNodeStarted(this, peerId, nodeId);
@ -185,13 +195,18 @@ namespace CodexClient
}
public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
{
return DownloadContent(contentId, TimeSpan.FromMinutes(10.0), fileLabel);
}
public TrackedFile? DownloadContent(ContentId contentId, TimeSpan timeout, string fileLabel = "")
{
var file = fileManager.CreateEmptyFile(fileLabel);
hooks.OnFileDownloading(contentId);
Log($"Downloading '{contentId}'...");
var logMessage = $"Downloaded '{contentId}' to '{file.Filename}'";
var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file));
var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file, timeout));
var size = file.GetFilesize();
transferSpeeds.AddDownloadSample(size, measurement);
@ -339,13 +354,14 @@ namespace CodexClient
.ToArray();
}
private void DownloadToFile(string contentId, TrackedFile file)
private void DownloadToFile(string contentId, TrackedFile file, TimeSpan timeout)
{
using var fileStream = File.OpenWrite(file.Filename);
var timeout = TimeSpan.FromMinutes(2.0); // todo: make this user-controllable.
try
{
// Type of stream generated by openAPI client does not support timeouts.
// So we use a task and cancellation token to track our timeout manually.
var start = DateTime.UtcNow;
var cts = new CancellationTokenSource();
var downloadTask = Task.Run(() =>
@ -373,7 +389,7 @@ namespace CodexClient
public void WaitUntilQuotaUsedIncreased(CodexSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed)
{
WaitUntilQuotaUsedIncreased(startSpace, expectedIncreaseOfQuotaUsed, TimeSpan.FromMinutes(2));
WaitUntilQuotaUsedIncreased(startSpace, expectedIncreaseOfQuotaUsed, TimeSpan.FromMinutes(30));
}
public void WaitUntilQuotaUsedIncreased(

7
ProjectPlugins/CodexClient/StoragePurchaseContract.cs
View File

@ -12,6 +12,7 @@ namespace CodexClient
ContentId ContentId { get; }
StoragePurchase? GetStatus();
void WaitForStorageContractSubmitted();
void WaitForStorageContractExpired();
void WaitForStorageContractStarted();
void WaitForStorageContractFinished();
void WaitForContractFailed(IMarketplaceConfigInput config);
@ -81,6 +82,12 @@ namespace CodexClient
AssertDuration(PendingToSubmitted, timeout, nameof(PendingToSubmitted));
}
public void WaitForStorageContractExpired()
{
var timeout = Purchase.Expiry + gracePeriod + gracePeriod;
WaitForStorageContractState(timeout, StoragePurchaseState.Cancelled);
}
public void WaitForStorageContractStarted()
{
var timeout = Purchase.Expiry + gracePeriod;

6
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs
View File

@ -7,7 +7,7 @@ namespace CodexContractsPlugin.ChainMonitor
{
private ChainEvents(
BlockInterval blockInterval,
Request[] requests,
StorageRequestedEventDTO[] requests,
RequestFulfilledEventDTO[] fulfilled,
RequestCancelledEventDTO[] cancelled,
RequestFailedEventDTO[] failed,
@ -30,7 +30,7 @@ namespace CodexContractsPlugin.ChainMonitor
}
public BlockInterval BlockInterval { get; }
public Request[] Requests { get; }
public StorageRequestedEventDTO[] Requests { get; }
public RequestFulfilledEventDTO[] Fulfilled { get; }
public RequestCancelledEventDTO[] Cancelled { get; }
public RequestFailedEventDTO[] Failed { get; }
@ -54,7 +54,7 @@ namespace CodexContractsPlugin.ChainMonitor
{
return new ChainEvents(
events.BlockInterval,
events.GetStorageRequests(),
events.GetStorageRequestedEvents(),
events.GetRequestFulfilledEvents(),
events.GetRequestCancelledEvents(),
events.GetRequestFailedEvents(),

29
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs
View File

@ -116,25 +116,22 @@ namespace CodexContractsPlugin.ChainMonitor
ApplyTimeImplicitEvents(blockNumber, eventsUtc);
}
private void ApplyEvent(Request request)
private void ApplyEvent(StorageRequestedEventDTO @event)
{
if (requests.Any(r => Equal(r.Request.RequestId, request.RequestId)))
if (requests.Any(r => Equal(r.RequestId, @event.RequestId)))
{
var r = FindRequest(request);
var r = FindRequest(@event);
if (r == null) throw new Exception("ChainState is inconsistent. Received already-known requestId that's not known.");
if (request.Block.BlockNumber != r.Request.Block.BlockNumber) throw new Exception("Same request found in different blocks.");
if (request.Client != r.Request.Client) throw new Exception("Same request belongs to different clients.");
if (request.Content.Cid.ToHex() != r.Request.Content.Cid.ToHex()) throw new Exception("Same request has different CIDs.");
if (@event.Block.BlockNumber != @event.Block.BlockNumber) throw new Exception("Same request found in different blocks.");
log.Log("Received the same request-creation event multiple times.");
return;
}
var newRequest = new ChainStateRequest(log, request, RequestState.New);
var request = contracts.GetRequest(@event.RequestId);
var newRequest = new ChainStateRequest(log, @event.RequestId, @event.Block, request, RequestState.New);
requests.Add(newRequest);
handler.OnNewRequest(new RequestEvent(request.Block, newRequest));
handler.OnNewRequest(new RequestEvent(@event.Block, newRequest));
}
private void ApplyEvent(RequestFulfilledEventDTO @event)
@ -209,22 +206,22 @@ namespace CodexContractsPlugin.ChainMonitor
}
}
private ChainStateRequest? FindRequest(IHasRequestId request)
private ChainStateRequest? FindRequest(IHasBlockAndRequestId hasBoth)
{
var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
var r = requests.SingleOrDefault(r => Equal(r.RequestId, hasBoth.RequestId));
if (r != null) return r;
try
{
var req = contracts.GetRequest(request.RequestId);
var state = contracts.GetRequestState(req);
var newRequest = new ChainStateRequest(log, req, state);
var req = contracts.GetRequest(hasBoth.RequestId);
var state = contracts.GetRequestState(hasBoth.RequestId);
var newRequest = new ChainStateRequest(log, hasBoth.RequestId, hasBoth.Block, req, state);
requests.Add(newRequest);
return newRequest;
}
catch (Exception ex)
{
var msg = "Failed to get request from chain: " + ex;
var msg = $"Failed to get request with id '{hasBoth.RequestId.ToHex()}' from chain: {ex}";
log.Error(msg);
handler.OnError(msg);
return null;

22
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs
View File

@ -1,11 +1,15 @@
using CodexContractsPlugin.Marketplace;
using BlockchainUtils;
using CodexContractsPlugin.Marketplace;
using Logging;
using Nethereum.Hex.HexConvertors.Extensions;
using Utils;
namespace CodexContractsPlugin.ChainMonitor
{
public interface IChainStateRequest
{
byte[] RequestId { get; }
public BlockTimeEntry Block { get; }
Request Request { get; }
RequestState State { get; }
DateTime ExpiryUtc { get; }
@ -18,21 +22,27 @@ namespace CodexContractsPlugin.ChainMonitor
{
private readonly ILog log;
public ChainStateRequest(ILog log, Request request, RequestState state)
public ChainStateRequest(ILog log, byte[] requestId, BlockTimeEntry block, Request request, RequestState state)
{
if (requestId == null || requestId.Length != 32) throw new ArgumentException(nameof(requestId));
this.log = log;
RequestId = requestId;
Block = block;
Request = request;
State = state;
ExpiryUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Expiry);
FinishedUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration);
ExpiryUtc = Block.Utc + TimeSpan.FromSeconds((double)request.Expiry);
FinishedUtc = Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration);
Log($"[{request.Block.BlockNumber}] Created as {State}.");
Log($"[{Block.BlockNumber}] Created as {State}.");
Client = new EthAddress(request.Client);
Hosts = new RequestHosts();
}
public byte[] RequestId { get; }
public BlockTimeEntry Block { get; }
public Request Request { get; }
public RequestState State { get; private set; }
public DateTime ExpiryUtc { get; }
@ -48,7 +58,7 @@ namespace CodexContractsPlugin.ChainMonitor
public void Log(string msg)
{
log.Log($"Request '{Request.Id}': {msg}");
log.Log($"Request '{RequestId.ToHex()}': {msg}");
}
}

44
ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs
View File

@ -46,4 +46,48 @@ namespace CodexContractsPlugin.ChainMonitor
{
}
}
public class DoNothingThrowingChainEventHandler : IChainStateChangeHandler
{
public void OnNewRequest(RequestEvent requestEvent)
{
}
public void OnRequestCancelled(RequestEvent requestEvent)
{
}
public void OnRequestFailed(RequestEvent requestEvent)
{
}
public void OnRequestFinished(RequestEvent requestEvent)
{
}
public void OnRequestFulfilled(RequestEvent requestEvent)
{
}
public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex, bool isRepair)
{
}
public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
{
}
public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
{
}
public void OnError(string msg)
{
throw new Exception(msg);
}
public void OnProofSubmitted(BlockTimeEntry block, string id)
{
}
}
}

5
ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs
View File

@ -1,4 +1,5 @@
using Logging;
using Nethereum.Hex.HexConvertors.Extensions;
using Utils;
namespace CodexContractsPlugin.ChainMonitor
@ -46,7 +47,7 @@ namespace CodexContractsPlugin.ChainMonitor
{
for (ulong slotIndex = 0; slotIndex < request.Request.Ask.Slots; slotIndex++)
{
var state = contracts.GetProofState(request.Request, slotIndex, lastBlockInPeriod, periodNumber);
var state = contracts.GetProofState(request.RequestId, slotIndex, lastBlockInPeriod, periodNumber);
total++;
if (state.Required)
@ -116,7 +117,7 @@ namespace CodexContractsPlugin.ChainMonitor
var missed = "None";
if (MissedProofs.Length > 0)
{
missed = string.Join("+", MissedProofs.Select(p => $"{p.FormatHost()} missed {p.Request.Request.Id} slot {p.SlotIndex}"));
missed = string.Join("+", MissedProofs.Select(p => $"{p.FormatHost()} missed {p.Request.RequestId.ToHex()} slot {p.SlotIndex}"));
}
return $"Period:{PeriodNumber}=[Slots:{TotalNumSlots},ProofsRequired:{TotalProofsRequired},ProofsMissed:{missed}]";
}

42
ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs
View File

@ -20,15 +20,18 @@ namespace CodexContractsPlugin
string MintTestTokens(EthAddress ethAddress, TestToken testTokens);
TestToken GetTestTokenBalance(IHasEthAddress owner);
TestToken GetTestTokenBalance(EthAddress ethAddress);
void TransferTestTokens(EthAddress to, TestToken amount);
ICodexContractsEvents GetEvents(TimeRange timeRange);
ICodexContractsEvents GetEvents(BlockInterval blockInterval);
EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
RequestState GetRequestState(Request request);
EthAddress? GetSlotHost(byte[] requestId, decimal slotIndex);
RequestState GetRequestState(byte[] requestId);
Request GetRequest(byte[] requestId);
ulong GetPeriodNumber(DateTime utc);
void WaitUntilNextPeriod();
ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period);
ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong blockNumber, ulong period);
ICodexContracts WithDifferentGeth(IGethNode node);
}
public class ProofState
@ -93,6 +96,11 @@ namespace CodexContractsPlugin
return balance.TstWei();
}
public void TransferTestTokens(EthAddress to, TestToken amount)
{
StartInteraction().TransferTestTokens(Deployment.TokenAddress, to.Address, amount.TstWei);
}
public ICodexContractsEvents GetEvents(TimeRange timeRange)
{
return GetEvents(gethNode.ConvertTimeRangeToBlockRange(timeRange));
@ -103,9 +111,9 @@ namespace CodexContractsPlugin
return new CodexContractsEvents(log, gethNode, Deployment, blockInterval);
}
public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
public EthAddress? GetSlotHost(byte[] requestId, decimal slotIndex)
{
var slotId = GetSlotId(storageRequest, slotIndex);
var slotId = GetSlotId(requestId, slotIndex);
var func = new GetHostFunction
{
SlotId = slotId
@ -115,17 +123,22 @@ namespace CodexContractsPlugin
return new EthAddress(address);
}
public RequestState GetRequestState(Request request)
public RequestState GetRequestState(byte[] requestId)
{
if (requestId == null) throw new ArgumentNullException(nameof(requestId));
if (requestId.Length != 32) throw new InvalidDataException(nameof(requestId) + $"{nameof(requestId)} length should be 32 bytes, but was: {requestId.Length}" + requestId.Length);
var func = new RequestStateFunction
{
RequestId = request.RequestId
RequestId = requestId
};
return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
}
public Request GetRequest(byte[] requestId)
{
if (requestId == null) throw new ArgumentNullException(nameof(requestId));
if (requestId.Length != 32) throw new InvalidDataException(nameof(requestId) + $"{nameof(requestId)} length should be 32 bytes, but was: {requestId.Length}" + requestId.Length);
var func = new GetRequestFunction
{
RequestId = requestId
@ -152,9 +165,9 @@ namespace CodexContractsPlugin
Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
}
public ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period)
public ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong blockNumber, ulong period)
{
var slotId = GetSlotId(storageRequest, slotIndex);
var slotId = GetSlotId(requestId, slotIndex);
var required = IsProofRequired(slotId, blockNumber);
if (!required) return new ProofState(false, false);
@ -163,11 +176,16 @@ namespace CodexContractsPlugin
return new ProofState(required, missing);
}
private byte[] GetSlotId(Request request, decimal slotIndex)
public ICodexContracts WithDifferentGeth(IGethNode node)
{
return new CodexContractsAccess(log, node, Deployment);
}
private byte[] GetSlotId(byte[] requestId, decimal slotIndex)
{
var encoder = new ABIEncode();
var encoded = encoder.GetABIEncoded(
new ABIValue("bytes32", request.RequestId),
new ABIValue("bytes32", requestId),
new ABIValue("uint256", slotIndex.ToBig())
);
@ -193,7 +211,7 @@ namespace CodexContractsPlugin
SlotId = slotId,
Period = period
};
gethNode.Call<MarkProofAsMissingFunction>(Deployment.MarketplaceAddress, funcB, blockNumber);
gethNode.Call(Deployment.MarketplaceAddress, funcB, blockNumber);
}
catch (AggregateException exc)
{

2
ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs
View File

@ -7,7 +7,7 @@ namespace CodexContractsPlugin
{
public class CodexContractsContainerRecipe : ContainerRecipeFactory
{
public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
public const string DeployedAddressesFilename = "/hardhat/ignition/deployments/chain-789988/deployed_addresses.json";
public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
private readonly DebugInfoVersion versionInfo;

14
ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs
View File

@ -11,7 +11,7 @@ namespace CodexContractsPlugin
public interface ICodexContractsEvents
{
BlockInterval BlockInterval { get; }
Request[] GetStorageRequests();
StorageRequestedEventDTO[] GetStorageRequestedEvents();
RequestFulfilledEventDTO[] GetRequestFulfilledEvents();
RequestCancelledEventDTO[] GetRequestCancelledEvents();
RequestFailedEventDTO[] GetRequestFailedEvents();
@ -38,18 +38,10 @@ namespace CodexContractsPlugin
public BlockInterval BlockInterval { get; }
public Request[] GetStorageRequests()
public StorageRequestedEventDTO[] GetStorageRequestedEvents()
{
var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
var i = new ContractInteractions(log, gethNode);
return events.Select(e =>
{
var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
var result = requestEvent.ReturnValue1;
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
result.RequestId = e.Event.RequestId;
return result;
}).ToArray();
return events.Select(SetBlockOnEvent).ToArray();
}
public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()

5
ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs
View File

@ -74,11 +74,16 @@ namespace CodexContractsPlugin
var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
var marketplaceAddress = extractor.ExtractMarketplaceAddress();
if (string.IsNullOrEmpty(marketplaceAddress)) throw new Exception("Marketplace address not received.");
var (abi, bytecode) = extractor.ExtractMarketplaceAbiAndByteCode();
if (string.IsNullOrEmpty(abi)) throw new Exception("ABI not received.");
if (string.IsNullOrEmpty(bytecode)) throw new Exception("bytecode not received.");
EnsureCompatbility(abi, bytecode);
var interaction = new ContractInteractions(tools.GetLog(), gethNode);
var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
if (string.IsNullOrEmpty(tokenAddress)) throw new Exception("Token address not received.");
Log("TokenAddress: " + tokenAddress);
Log("Extract completed. Checking sync...");

48
ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs
View File

@ -2,8 +2,6 @@
using CodexContractsPlugin.Marketplace;
using GethPlugin;
using Logging;
using Nethereum.ABI.FunctionEncoding.Attributes;
using Nethereum.Contracts;
using Nethereum.Hex.HexConvertors.Extensions;
using System.Numerics;
using Utils;
@ -33,9 +31,9 @@ namespace CodexContractsPlugin
try
{
log.Debug(tokenAddress);
var function = new GetTokenNameFunction();
var function = new NameFunction();
return gethNode.Call<GetTokenNameFunction, string>(tokenAddress, function);
return gethNode.Call<NameFunction, string>(tokenAddress, function);
}
catch (Exception ex)
{
@ -53,12 +51,24 @@ namespace CodexContractsPlugin
public decimal GetBalance(string tokenAddress, string account)
{
log.Debug($"({tokenAddress}) {account}");
var function = new GetTokenBalanceFunction
var function = new BalanceOfFunction
{
Owner = account
Account = account
};
return gethNode.Call<GetTokenBalanceFunction, BigInteger>(tokenAddress, function).ToDecimal();
return gethNode.Call<BalanceOfFunction, BigInteger>(tokenAddress, function).ToDecimal();
}
public void TransferTestTokens(string tokenAddress, string toAccount, BigInteger amount)
{
log.Debug($"({tokenAddress}) {toAccount} {amount}");
var function = new TransferFunction
{
To = toAccount,
Value = amount
};
gethNode.SendTransaction(tokenAddress, function);
}
public GetRequestOutputDTO GetRequest(string marketplaceAddress, byte[] requestId)
@ -90,7 +100,7 @@ namespace CodexContractsPlugin
log.Debug($"({tokenAddress}) {amount} --> {account}");
if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
var function = new MintTokensFunction
var function = new MintFunction
{
Holder = account,
Amount = amount
@ -110,26 +120,4 @@ namespace CodexContractsPlugin
return gethNode.IsContractAvailable(marketplaceAbi, marketplaceAddress);
}
}
[Function("name", "string")]
public class GetTokenNameFunction : FunctionMessage
{
}
[Function("mint")]
public class MintTokensFunction : FunctionMessage
{
[Parameter("address", "holder", 1)]
public string Holder { get; set; } = string.Empty;
[Parameter("uint256", "amount", 2)]
public BigInteger Amount { get; set; }
}
[Function("balanceOf", "uint256")]
public class GetTokenBalanceFunction : FunctionMessage
{
[Parameter("address", "owner", 1)]
public string Owner { get; set; } = string.Empty;
}
}

15
ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs
View File

@ -27,7 +27,7 @@ namespace CodexContractsPlugin
var marketplaceAddress = Retry(FetchMarketplaceAddress);
if (string.IsNullOrEmpty(marketplaceAddress)) throw new InvalidOperationException("Unable to fetch marketplace account from codex-contracts node. Test infra failure.");
log.Debug("Got MarketplaceAddress: " + marketplaceAddress);
log.Log("MarketplaceAddress: " + marketplaceAddress);
return marketplaceAddress;
}
@ -43,9 +43,10 @@ namespace CodexContractsPlugin
private string FetchMarketplaceAddress()
{
var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceAddressFilename);
var marketplace = JsonConvert.DeserializeObject<MarketplaceJson>(json);
return marketplace!.address;
var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.DeployedAddressesFilename);
json = json.Replace("#", "_");
var addresses = JsonConvert.DeserializeObject<DeployedAddressesJson>(json);
return addresses!.Marketplace_Marketplace;
}
private (string, string) FetchMarketplaceAbiAndByteCode()
@ -67,8 +68,10 @@ namespace CodexContractsPlugin
}
}
public class MarketplaceJson
public class DeployedAddressesJson
{
public string address { get; set; } = string.Empty;
public string Token_TestToken { get; set; } = string.Empty;
public string Verifier_Groth16Verifier { get; set; } = string.Empty;
public string Marketplace_Marketplace { get; set; } = string.Empty;
}
}

39
ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs
View File

@ -17,48 +17,45 @@ namespace CodexContractsPlugin.Marketplace
byte[] RequestId { get; set; }
}
public interface IHasBlockAndRequestId : IHasBlock, IHasRequestId
{
}
public interface IHasSlotIndex
{
ulong SlotIndex { get; set; }
}
public partial class Request : RequestBase, IHasBlock, IHasRequestId
public partial class Request
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
public byte[] RequestId { get; set; }
public EthAddress ClientAddress { get { return new EthAddress(Client); } }
[JsonIgnore]
public string Id
{
get
{
return BitConverter.ToString(RequestId).Replace("-", "").ToLowerInvariant();
}
}
}
public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
public partial class StorageRequestedEventDTO : IHasBlockAndRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
}
public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
public partial class RequestFulfilledEventDTO : IHasBlockAndRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
}
public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
public partial class RequestCancelledEventDTO : IHasBlockAndRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
}
public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId, IHasSlotIndex
public partial class RequestFailedEventDTO : IHasBlockAndRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
}
public partial class SlotFilledEventDTO : IHasBlockAndRequestId, IHasSlotIndex
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
@ -70,13 +67,13 @@ namespace CodexContractsPlugin.Marketplace
}
}
public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId, IHasSlotIndex
public partial class SlotFreedEventDTO : IHasBlockAndRequestId, IHasSlotIndex
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
}
public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId, IHasSlotIndex
public partial class SlotReservationsFullEventDTO : IHasBlockAndRequestId, IHasSlotIndex
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
@ -88,7 +85,7 @@ namespace CodexContractsPlugin.Marketplace
public BlockTimeEntry Block { get; set; }
}
public partial class ReserveSlotFunction : IHasBlock, IHasRequestId, IHasSlotIndex
public partial class ReserveSlotFunction : IHasBlockAndRequestId, IHasSlotIndex
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }

2
ProjectPlugins/CodexContractsPlugin/Marketplace/Marketplace.cs
View File

File diff suppressed because one or more lines are too long

207
ProjectPlugins/CodexContractsPlugin/TestTokenContract.cs Normal file
View File

File diff suppressed because one or more lines are too long

2
ProjectPlugins/CodexPlugin/CodexDockerImage.cs
View File

@ -2,7 +2,7 @@
{
public class CodexDockerImage
{
private const string DefaultDockerImage = "codexstorage/nim-codex:sha-e324ac8-dist-tests";
private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
public static string Override { get; set; } = string.Empty;

11
ProjectPlugins/GethPlugin/GethContainerInfoExtractor.cs
View File

@ -60,7 +60,16 @@ namespace GethPlugin
private static string Retry(Func<string> fetch)
{
return Time.Retry(fetch, nameof(GethContainerInfoExtractor));
// This class is the first moment where we interact with our new geth container.
// K8s might be moving pods and/or setting up new VMs.
// So we apply a generous retry timeout.
var retry = new Retry(nameof(GethContainerInfoExtractor),
maxTimeout: TimeSpan.FromMinutes(15.0),
sleepAfterFail: TimeSpan.FromSeconds(20.0),
onFail: f => { },
failFast: false);
return retry.Run(fetch);
}
}

29
ProjectPlugins/GethPlugin/GethNode.cs
View File

@ -21,6 +21,7 @@ namespace GethPlugin
string SendEth(EthAddress account, Ether eth);
TResult Call<TFunction, TResult>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new();
TResult Call<TFunction, TResult>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new();
void Call<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new();
void Call<TFunction>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new();
string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new();
Transaction GetTransaction(string transactionHash);
@ -32,6 +33,7 @@ namespace GethPlugin
BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
BlockTimeEntry GetBlockForNumber(ulong number);
void IterateFunctionCalls<TFunc>(BlockInterval blockInterval, Action<BlockTimeEntry, TFunc> onCall) where TFunc : FunctionMessage, new();
IGethNode WithDifferentAccount(EthAccount account);
}
public class DeploymentGethNode : BaseGethNode, IGethNode
@ -68,6 +70,21 @@ namespace GethPlugin
var creator = new NethereumInteractionCreator(log, blockCache, address.Host, address.Port, account.PrivateKey);
return creator.CreateWorkflow();
}
public IGethNode WithDifferentAccount(EthAccount account)
{
return new DeploymentGethNode(log, blockCache,
new GethDeployment(
StartResult.Pod,
StartResult.DiscoveryPort,
StartResult.HttpPort,
StartResult.WsPort,
new GethAccount(
account.EthAddress.Address,
account.PrivateKey
),
account.PrivateKey));
}
}
public class CustomGethNode : BaseGethNode, IGethNode
@ -95,6 +112,11 @@ namespace GethPlugin
throw new NotImplementedException();
}
public IGethNode WithDifferentAccount(EthAccount account)
{
return new CustomGethNode(log, blockCache, gethHost, gethPort, account.PrivateKey);
}
protected override NethereumInteraction StartInteraction()
{
var creator = new NethereumInteractionCreator(log, blockCache, gethHost, gethPort, privateKey);
@ -116,7 +138,7 @@ namespace GethPlugin
public Ether GetEthBalance(EthAddress address)
{
return StartInteraction().GetEthBalance(address.Address).Eth();
return StartInteraction().GetEthBalance(address.Address).Wei();
}
public string SendEth(IHasEthAddress owner, Ether eth)
@ -139,6 +161,11 @@ namespace GethPlugin
return StartInteraction().Call<TFunction, TResult>(contractAddress, function, blockNumber);
}
public void Call<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
{
StartInteraction().Call(contractAddress, function);
}
public void Call<TFunction>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new()
{
StartInteraction().Call(contractAddress, function, blockNumber);

82
Tests/CodexReleaseTests/DataTests/DataExpiryTest.cs
View File

@ -1,4 +1,5 @@
using CodexContractsPlugin;
using CodexPlugin;
using CodexTests;
using NUnit.Framework;
using Utils;
@ -8,17 +9,23 @@ namespace CodexReleaseTests.DataTests
[TestFixture]
public class DataExpiryTest : CodexDistTest
{
private readonly TimeSpan blockTtl = TimeSpan.FromMinutes(1.0);
private readonly TimeSpan blockInterval = TimeSpan.FromSeconds(10.0);
private readonly int blockCount = 100000;
private ICodexSetup WithFastBlockExpiry(ICodexSetup setup)
{
return setup
.WithBlockTTL(blockTtl)
.WithBlockMaintenanceInterval(blockInterval)
.WithBlockMaintenanceNumber(blockCount);
}
[Test]
public void DeletesExpiredData()
{
var fileSize = 100.MB();
var blockTtl = TimeSpan.FromMinutes(1.0);
var interval = TimeSpan.FromSeconds(10.0);
var node = StartCodex(s => s
.WithBlockTTL(blockTtl)
.WithBlockMaintenanceInterval(interval)
.WithBlockMaintenanceNumber(100000)
);
var fileSize = 3.MB();
var node = StartCodex(s => WithFastBlockExpiry(s));
var startSpace = node.Space();
Assert.That(startSpace.QuotaUsedBytes, Is.EqualTo(0));
@ -46,18 +53,13 @@ namespace CodexReleaseTests.DataTests
[Test]
public void DeletesExpiredDataUsedByStorageRequests()
{
var fileSize = 100.MB();
var blockTtl = TimeSpan.FromMinutes(1.0);
var interval = TimeSpan.FromSeconds(10.0);
var fileSize = 3.MB();
var bootstrapNode = StartCodex();
var geth = StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth, bootstrapNode.Version);
var node = StartCodex(s => s
var node = StartCodex(s => WithFastBlockExpiry(s)
.EnableMarketplace(geth, contracts, m => m.WithInitial(100.Eth(), 100.Tst()))
.WithBlockTTL(blockTtl)
.WithBlockMaintenanceInterval(interval)
.WithBlockMaintenanceNumber(100000)
);
var startSpace = node.Space();
@ -92,5 +94,55 @@ namespace CodexReleaseTests.DataTests
Assert.That(cleanupSpace.QuotaUsedBytes, Is.EqualTo(startSpace.QuotaUsedBytes));
Assert.That(cleanupSpace.FreeBytes, Is.EqualTo(startSpace.FreeBytes));
}
[Test]
[Ignore("Issue not fixed. Ticket: https://github.com/codex-storage/nim-codex/issues/1291")]
public void StorageRequestsKeepManifests()
{
var bootstrapNode = StartCodex(s => s.WithName("Bootstrap"));
var geth = StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth, bootstrapNode.Version);
var client = StartCodex(s => WithFastBlockExpiry(s)
.WithName("client")
.WithBootstrapNode(bootstrapNode)
.EnableMarketplace(geth, contracts, m => m.WithInitial(100.Eth(), 100.Tst()))
);
var hosts = StartCodex(3, s => WithFastBlockExpiry(s)
.WithName("host")
.WithBootstrapNode(bootstrapNode)
.EnableMarketplace(geth, contracts, m => m.AsStorageNode().WithInitial(100.Eth(), 100.Tst()))
);
foreach (var host in hosts) host.Marketplace.MakeStorageAvailable(new CodexClient.CreateStorageAvailability(
totalSpace: 2.GB(),
maxDuration: TimeSpan.FromDays(2.0),
minPricePerBytePerSecond: 1.TstWei(),
totalCollateral: 10.Tst()));
var uploadCid = client.UploadFile(GenerateTestFile(5.MB()));
var request = client.Marketplace.RequestStorage(new CodexClient.StoragePurchaseRequest(uploadCid)
{
CollateralPerByte = 1.TstWei(),
Duration = TimeSpan.FromDays(1.0),
Expiry = TimeSpan.FromHours(1.0),
MinRequiredNumberOfNodes = 3,
NodeFailureTolerance = 1,
PricePerBytePerSecond = 10.TstWei(),
ProofProbability = 99999
});
request.WaitForStorageContractSubmitted();
request.WaitForStorageContractStarted();
var storeCid = request.ContentId;
var clientManifest = client.DownloadManifestOnly(storeCid);
Assert.That(clientManifest.Manifest.Protected, Is.True);
client.Stop(waitTillStopped: true);
Thread.Sleep(blockTtl * 2.0);
var checker = StartCodex(s => s.WithName("checker").WithBootstrapNode(bootstrapNode));
var manifest = checker.DownloadManifestOnly(storeCid);
Assert.That(manifest.Manifest.Protected, Is.True);
}
}
}

4
Tests/CodexReleaseTests/DataTests/SwarmTest.cs
View File

@ -9,7 +9,7 @@ namespace CodexReleaseTests.DataTests
{
[TestFixture(2, 10)]
[TestFixture(5, 20)]
[TestFixture(10, 50)]
[TestFixture(10, 20)]
public class SwarmTests : AutoBootstrapDistTest
{
private readonly int numberOfNodes;
@ -89,7 +89,7 @@ namespace CodexReleaseTests.DataTests
var file = remaining.PickOneRandom();
try
{
var dl = node.DownloadContent(file.Cid);
var dl = node.DownloadContent(file.Cid, TimeSpan.FromMinutes(30));
lock (file.Lock)
{
file.Downloaded.Add(dl);

2
Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs
View File

@ -19,7 +19,7 @@ namespace CodexReleaseTests.DataTests
try
{
node.DownloadContent(unknownCid);
node.DownloadContent(unknownCid, TimeSpan.FromMinutes(2.0));
}
catch (Exception ex)
{

49
Tests/CodexReleaseTests/MarketTests/ChainMonitor.cs
View File

@ -1,49 +0,0 @@
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using Logging;
namespace CodexReleaseTests.MarketTests
{
public class ChainMonitor
{
private readonly ChainState chainMonitor;
private readonly TimeSpan interval;
private CancellationTokenSource cts = new CancellationTokenSource();
private Task worker = null!;
public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc, TimeSpan interval)
{
chainMonitor = new ChainState(log, contracts, new DoNothingChainEventHandler(), startUtc, true);
this.interval = interval;
}
public void Start()
{
cts = new CancellationTokenSource();
worker = Task.Run(Worker);
}
public void Stop()
{
cts.Cancel();
worker.Wait();
worker = null!;
cts = null!;
}
public PeriodMonitorResult GetPeriodReports()
{
return chainMonitor.PeriodMonitor.GetAndClearReports();
}
private void Worker()
{
while (!cts.IsCancellationRequested)
{
Thread.Sleep(interval);
chainMonitor.Update();
}
}
}
}

2
Tests/CodexReleaseTests/MarketTests/FailTest.cs
View File

@ -68,7 +68,7 @@ namespace CodexReleaseTests.MarketTests
private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
{
var cid = client.UploadFile(GenerateTestFile(5.MB()));
var cid = client.UploadFile(GenerateTestFile(3.MB()));
return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
{
Duration = HostAvailabilityMaxDuration / 2,

2
Tests/CodexReleaseTests/MarketTests/FinishTest.cs
View File

@ -12,7 +12,7 @@ namespace CodexReleaseTests.MarketTests
public FinishTest(int hosts, int slots, int tolerance)
{
this.hosts = hosts;
purchaseParams = new PurchaseParams(slots, tolerance, uploadFilesize: 10.MB());
purchaseParams = new PurchaseParams(slots, tolerance, uploadFilesize: 3.MB());
}
private readonly TestToken pricePerBytePerSecond = 10.TstWei();

51
Tests/CodexReleaseTests/MarketTests/JustTokensTest.cs Normal file
View File

@ -0,0 +1,51 @@
using BlockchainUtils;
using CodexContractsPlugin;
using CodexPlugin;
using DistTestCore;
using GethPlugin;
using NUnit.Framework;
using Utils;
namespace CodexReleaseTests.MarketTests
{
[TestFixture]
public class TestTokenTransferTest : DistTest
{
private readonly EthAccount user1 = EthAccountGenerator.GenerateNew();
private readonly EthAccount user2 = EthAccountGenerator.GenerateNew();
[Test]
public void CanTransferTokens()
{
var node = Ci.StartCodexNode();
var blockCache = new BlockCache();
var geth = Ci.StartGethNode(blockCache, s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth, node.Version);
geth.SendEth(user1.EthAddress, 1.Eth());
geth.SendEth(user2.EthAddress, 1.Eth());
contracts.MintTestTokens(user1.EthAddress, 10.Tst());
Balances(contracts, 10.Tst(), 0.Tst());
var geth1 = geth.WithDifferentAccount(user1);
var geth2 = geth.WithDifferentAccount(user2);
var contracts1 = contracts.WithDifferentGeth(geth1);
var contracts2 = contracts.WithDifferentGeth(geth2);
contracts1.TransferTestTokens(user2.EthAddress, 5.Tst());
Balances(contracts, 5.Tst(), 5.Tst());
contracts2.TransferTestTokens(user1.EthAddress, 2.Tst());
Balances(contracts, 7.Tst(), 3.Tst());
}
private void Balances(ICodexContracts contracts, TestToken one, TestToken two)
{
var balance1 = contracts.GetTestTokenBalance(user1.EthAddress);
var balance2 = contracts.GetTestTokenBalance(user2.EthAddress);
Assert.That(balance1, Is.EqualTo(one));
Assert.That(balance2, Is.EqualTo(two));
}
}
}

7
Tests/CodexReleaseTests/MarketTests/RepairTest.cs
View File

@ -29,13 +29,11 @@ namespace CodexReleaseTests.MarketTests
#endregion
[Ignore("Test is ready. Waiting for repair implementation. " +
"Slots are never freed because proofs are never marked as missing. Issue: https://github.com/codex-storage/nim-codex/issues/1153")]
[Test]
[Combinatorial]
public void RollingRepairSingleFailure(
[Rerun] int rerun,
[Values(10)] int numFailures)
[Values(5)] int numFailures)
{
var hosts = StartHosts().ToList();
var client = StartClients().Single();
@ -98,7 +96,7 @@ namespace CodexReleaseTests.MarketTests
private void WaitForNewSlotFilledEvent(IStoragePurchaseContract contract, ulong slotIndex)
{
Log(nameof(WaitForNewSlotFilledEvent));
var start = DateTime.UtcNow;
var start = DateTime.UtcNow - TimeSpan.FromSeconds(10.0);
var timeout = contract.Purchase.Expiry;
while (DateTime.UtcNow < start + timeout)
@ -122,6 +120,7 @@ namespace CodexReleaseTests.MarketTests
if (matches.Length == 1)
{
Log($"Found the correct new slotFilled event: {matches[0].ToString()}");
return;
}
Thread.Sleep(TimeSpan.FromSeconds(15));

4
Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs
View File

@ -27,7 +27,7 @@ namespace CodexReleaseTests.MarketTests
[Test]
[Combinatorial]
public void Sequential(
[Values(10)] int numGenerations)
[Values(5)] int numGenerations)
{
var hosts = StartHosts();
var clients = StartClients();
@ -93,7 +93,7 @@ namespace CodexReleaseTests.MarketTests
MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
NodeFailureTolerance = (uint)purchaseParams.Tolerance,
PricePerBytePerSecond = pricePerBytePerSecond,
ProofProbability = 10000,
ProofProbability = 100000,
CollateralPerByte = 1.TstWei()
});
}

2
Tests/CodexReleaseTests/MarketTests/StartTest.cs
View File

@ -11,7 +11,7 @@ namespace CodexReleaseTests.MarketTests
private readonly PurchaseParams purchaseParams = new PurchaseParams(
nodes: 3,
tolerance: 1,
uploadFilesize: 10.MB()
uploadFilesize: 3.MB()
);
private readonly TestToken pricePerBytePerSecond = 10.TstWei();

2
Tests/CodexReleaseTests/Utils/ChainMonitor.cs
View File

@ -41,7 +41,7 @@ namespace CodexReleaseTests.Utils
private void Worker(Action onFailure)
{
var state = new ChainState(log, contracts, new DoNothingChainEventHandler(), startUtc, doProofPeriodMonitoring: true);
var state = new ChainState(log, contracts, new DoNothingThrowingChainEventHandler(), startUtc, doProofPeriodMonitoring: true);
Thread.Sleep(updateInterval);
log.Log("Chain monitoring started");

31
Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs
View File

@ -176,8 +176,7 @@ namespace CodexReleaseTests.Utils
var result = new ChainMonitor(log, contracts, startUtc);
result.Start(() =>
{
log.Error("Failure in chain monitor. No chain updates after this point.");
//Assert.Fail("Failure in chain monitor.");
Assert.Fail("Failure in chain monitor.");
});
return result;
}
@ -262,10 +261,15 @@ namespace CodexReleaseTests.Utils
var fills = events.GetSlotFilledEvents();
return fills.Select(f =>
{
var host = possibleHosts.Single(h => h.EthAddress.Address == f.Host.Address);
// We can encounter a fill event that's from an old host.
// We must disregard those.
var host = possibleHosts.SingleOrDefault(h => h.EthAddress.Address == f.Host.Address);
if (host == null) return null;
return new SlotFill(f, host);
}).ToArray();
})
.Where(f => f != null)
.Cast<SlotFill>()
.ToArray();
}
protected void AssertClientHasPaidForContract(TestToken pricePerBytePerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
@ -363,7 +367,7 @@ namespace CodexReleaseTests.Utils
return Time.Retry(() =>
{
var events = GetContracts().GetEvents(GetTestRunTimeRange());
var submitEvent = events.GetStorageRequests().SingleOrDefault(e => e.RequestId.ToHex(false) == contract.PurchaseId);
var submitEvent = events.GetStorageRequestedEvents().SingleOrDefault(e => e.RequestId.ToHex() == contract.PurchaseId);
if (submitEvent == null)
{
// We're too early.
@ -402,12 +406,21 @@ namespace CodexReleaseTests.Utils
protected void AssertContractIsOnChain(IStoragePurchaseContract contract)
{
// Check the creation event.
AssertOnChainEvents(events =>
{
var onChainRequests = events.GetStorageRequests();
if (onChainRequests.Any(r => r.Id == contract.PurchaseId)) return;
var onChainRequests = events.GetStorageRequestedEvents();
if (onChainRequests.Any(r => r.RequestId.ToHex() == contract.PurchaseId)) return;
throw new Exception($"OnChain request {contract.PurchaseId} not found...");
}, nameof(AssertContractIsOnChain));
// Check that the getRequest call returns it.
var rid = contract.PurchaseId.HexToByteArray();
var r = GetContracts().GetRequest(rid);
if (r == null) throw new Exception($"Failed to get Request from {nameof(GetRequestFunction)}");
Assert.That(r.Ask.Duration, Is.EqualTo(contract.Purchase.Duration.TotalSeconds));
Assert.That(r.Ask.Slots, Is.EqualTo(contract.Purchase.MinRequiredNumberOfNodes));
Assert.That(((int)r.Ask.ProofProbability), Is.EqualTo(contract.Purchase.ProofProbability));
}
protected void AssertOnChainEvents(Action<ICodexContractsEvents> onEvents, string description)
@ -444,7 +457,7 @@ namespace CodexReleaseTests.Utils
float downtime = numBlocksInDowntimeSegment;
float window = 256.0f;
var chanceOfDowntime = downtime / window;
return 1.0f + chanceOfDowntime + chanceOfDowntime;
return 1.0f + (5.0f * chanceOfDowntime);
}
public class SlotFill
{

10
Tests/CodexReleaseTests/Utils/PurchaseParams.cs
View File

@ -33,7 +33,7 @@ namespace CodexReleaseTests.Utils
var numSlotBlocks = 1 + ((numBlocks - 1) / Nodes); // round-up div.
// Next power of two:
var numSlotBlocksPow2 = NextPowerOf2(numSlotBlocks);
var numSlotBlocksPow2 = IsOrNextPowerOf2(numSlotBlocks);
return new ByteSize(blockSize.SizeInBytes * numSlotBlocksPow2);
}
@ -51,8 +51,9 @@ namespace CodexReleaseTests.Utils
return new ByteSize(blockSize.SizeInBytes * totalBlocks);
}
private int NextPowerOf2(int n)
private int IsOrNextPowerOf2(int n)
{
if (IsPowerOfTwo(n)) return n;
n = n - 1;
var lg = Convert.ToInt32(Math.Round(Math.Log2(Convert.ToDouble(n))));
return 1 << (lg + 1);
@ -63,5 +64,10 @@ namespace CodexReleaseTests.Utils
var x = size.SizeInBytes;
return (x != 0) && ((x & (x - 1)) == 0);
}
private static bool IsPowerOfTwo(int x)
{
return (x != 0) && ((x & (x - 1)) == 0);
}
}
}

2
Tests/DistTestCore/DistTest.cs
View File

@ -274,7 +274,7 @@ namespace DistTestCore
private string GetCurrentTestName()
{
return $"[{NameUtils.GetTestMethodName()}]";
return $"[{NameUtils.GetRawFixtureName()}:{NameUtils.GetTestMethodName()}]";
}
public DistTestResult GetTestResult()

17
Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs
View File

@ -120,7 +120,7 @@ namespace ExperimentalTests.BasicTests
//Thread.Sleep(GethContainerRecipe.BlockInterval * blocks);
AssertBalance(contracts, client, Is.LessThan(clientInitialBalance), "Buyer was not charged for storage.");
Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished));
Assert.That(contracts.GetRequestState(request.RequestId), Is.EqualTo(RequestState.Finished));
}
private TrackedFile CreateFile(ByteSize fileSize)
@ -148,24 +148,25 @@ namespace ExperimentalTests.BasicTests
}, purchase.Expiry + TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5), "Checking SlotFilled events");
}
private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer)
private void AssertStorageRequest(StorageRequestedEventDTO request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer)
{
Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Started));
Assert.That(request.ClientAddress, Is.EqualTo(buyer.EthAddress));
Assert.That(contracts.GetRequestState(request.RequestId), Is.EqualTo(RequestState.Started));
var r = contracts.GetRequest(request.RequestId);
Assert.That(r.ClientAddress, Is.EqualTo(buyer.EthAddress));
Assert.That(request.Ask.Slots, Is.EqualTo(purchase.MinRequiredNumberOfNodes));
}
private Request GetOnChainStorageRequest(ICodexContracts contracts, IGethNode geth)
private StorageRequestedEventDTO GetOnChainStorageRequest(ICodexContracts contracts, IGethNode geth)
{
var events = contracts.GetEvents(GetTestRunTimeRange());
var requests = events.GetStorageRequests();
var requests = events.GetStorageRequestedEvents();
Assert.That(requests.Length, Is.EqualTo(1));
return requests.Single();
}
private void AssertContractSlot(ICodexContracts contracts, Request request, int contractSlotIndex)
private void AssertContractSlot(ICodexContracts contracts, StorageRequestedEventDTO request, int contractSlotIndex)
{
var slotHost = contracts.GetSlotHost(request, contractSlotIndex);
var slotHost = contracts.GetSlotHost(request.RequestId, contractSlotIndex);
Assert.That(slotHost?.Address, Is.Not.Null);
}
}

1
Tools/BiblioTech/CommandHandler.cs
View File

@ -6,7 +6,6 @@ using BiblioTech.Rewards;
using Logging;
using BiblioTech.CodexChecking;
using Nethereum.Model;
using static Org.BouncyCastle.Math.EC.ECCurve;
namespace BiblioTech
{

4
Tools/BiblioTech/Commands/MintCommand.cs
View File

@ -93,12 +93,12 @@ namespace BiblioTech.Commands
private bool ShouldSendEth(IGethNode gethNode, EthAddress addr)
{
var eth = gethNode.GetEthBalance(addr);
return eth.Eth < Program.Config.SendEth;
return ((decimal)eth.Eth) < Program.Config.SendEth;
}
private string FormatTransactionLink(string transaction)
{
var url = $"https://explorer.testnet.codex.storage/tx/{transaction}";
var url = Program.Config.TransactionLinkFormat.Replace("<ID>", transaction);
return $"- [View on block explorer](<{url}>){Environment.NewLine}Transaction ID - `{transaction}`";
}
}

5
Tools/BiblioTech/Configuration.cs
View File

@ -30,7 +30,7 @@ namespace BiblioTech
public int RewardApiPort { get; set; } = 31080;
[Uniform("send-eth", "se", "SENDETH", true, "Amount of Eth send by the mint command.")]
public int SendEth { get; set; } = 10;
public decimal SendEth { get; set; } = 10.0m;
[Uniform("mint-tt", "mt", "MINTTT", true, "Amount of TSTWEI minted by the mint command.")]
public BigInteger MintTT { get; set; } = 1073741824;
@ -44,6 +44,9 @@ namespace BiblioTech
[Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")]
public string CodexEndpointAuth { get; set; } = "";
[Uniform("transaction-link-format", "tlf", "TRANSACTIONLINKFORMAT", false, "Format of links to transactions on the blockchain. Use '<ID>' to inject the transaction ID into this string. (default 'https://explorer.testnet.codex.storage/tx/<ID>')")]
public string TransactionLinkFormat { get; set; } = "https://explorer.testnet.codex.storage/tx/<ID>";
#region Role Rewards
/// <summary>

3
Tools/BiblioTech/Program.cs
View File

@ -4,10 +4,7 @@ using BiblioTech.Commands;
using BiblioTech.Rewards;
using Discord;
using Discord.WebSocket;
using DiscordRewards;
using Logging;
using Nethereum.Model;
using Newtonsoft.Json;
namespace BiblioTech
{

11
Tools/BiblioTech/docker/Dockerfile
View File

@ -9,9 +9,9 @@ FROM ${IMAGE} AS builder
ARG APP_HOME
WORKDIR ${APP_HOME}
COPY ./Tools/BiblioTech ./Tools/BiblioTech
COPY ./Framework ./Framework
COPY ./ProjectPlugins ./ProjectPlugins
COPY Tools/BiblioTech Tools/BiblioTech
COPY Framework Framework
COPY ProjectPlugins ProjectPlugins
RUN dotnet restore Tools/BiblioTech
RUN dotnet publish Tools/BiblioTech -c Release -o out
@ -23,4 +23,7 @@ ENV APP_HOME=${APP_HOME}
WORKDIR ${APP_HOME}
COPY --from=builder ${APP_HOME}/out .
CMD dotnet ${APP_HOME}/BiblioTech.dll
COPY --chmod=0755 Tools/BiblioTech/docker/docker-entrypoint.sh /
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["bash", "-c", "dotnet ${APP_HOME}/BiblioTech.dll"]

30
Tools/BiblioTech/docker/docker-entrypoint.sh Normal file
View File

@ -0,0 +1,30 @@
#!/bin/bash
# Marketplace address from URL
if [[ -n "${MARKETPLACE_ADDRESS_FROM_URL}" ]]; then
WAIT=${MARKETPLACE_ADDRESS_FROM_URL_WAIT:-300}
SECONDS=0
SLEEP=1
# Run and retry if fail
while (( SECONDS < WAIT )); do
MARKETPLACE_ADDRESS=($(curl -s -f -m 5 "${MARKETPLACE_ADDRESS_FROM_URL}"))
# Check if exit code is 0 and returned value is not empty
if [[ $? -eq 0 && -n "${MARKETPLACE_ADDRESS}" ]]; then
export CODEXCONTRACTS_MARKETPLACEADDRESS="${MARKETPLACE_ADDRESS}"
break
else
# Sleep and check again
echo "Can't get Marketplace address from ${MARKETPLACE_ADDRESS_FROM_URL} - Retry in $SLEEP seconds / $((WAIT - SECONDS))"
sleep $SLEEP
fi
done
fi
# Show
echo -e "\nRun parameters:"
vars=$(env | grep "CODEX" | grep -v -e "[0-9]_SERVICE_" -e "[0-9]_NODEPORT_")
echo -e "${vars//CODEX/ - CODEX}"
echo -e " - $@\n"
# Run
exec "$@"

10
Tools/TestNetRewarder/EventsFormatter.cs
View File

@ -2,6 +2,7 @@
using CodexContractsPlugin.ChainMonitor;
using CodexContractsPlugin.Marketplace;
using DiscordRewards;
using Nethereum.Hex.HexConvertors.Extensions;
using System.Globalization;
using System.Numerics;
using Utils;
@ -237,14 +238,15 @@ namespace TestNetRewarder
private string FormatRequestId(IChainStateRequest request)
{
return FormatRequestId(request.Request.Id);
return FormatRequestId(request.RequestId);
}
private string FormatRequestId(string id)
private string FormatRequestId(byte[] id)
{
var str = id.ToHex();
return
$"({emojiMaps.StringToEmojis(id, 3)})" +
$"`{id}`";
$"({emojiMaps.StringToEmojis(str, 3)})" +
$"`{str}`";
}
private string BytesToHexString(byte[] bytes)

11
Tools/TestNetRewarder/docker/Dockerfile
View File

@ -9,9 +9,9 @@ FROM ${IMAGE} AS builder
ARG APP_HOME
WORKDIR ${APP_HOME}
COPY ./Tools/TestNetRewarder ./Tools/TestNetRewarder
COPY ./Framework ./Framework
COPY ./ProjectPlugins ./ProjectPlugins
COPY Tools/TestNetRewarder Tools/TestNetRewarder
COPY Framework Framework
COPY ProjectPlugins ProjectPlugins
RUN dotnet restore Tools/TestNetRewarder
RUN dotnet publish Tools/TestNetRewarder -c Release -o out
@ -23,4 +23,7 @@ ENV APP_HOME=${APP_HOME}
WORKDIR ${APP_HOME}
COPY --from=builder ${APP_HOME}/out .
CMD dotnet ${APP_HOME}/TestNetRewarder.dll
COPY --chmod=0755 Tools/TestNetRewarder/docker/docker-entrypoint.sh /
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["bash", "-c", "dotnet ${APP_HOME}/TestNetRewarder.dll"]

30
Tools/TestNetRewarder/docker/docker-entrypoint.sh Normal file
View File

@ -0,0 +1,30 @@
#!/bin/bash
# Marketplace address from URL
if [[ -n "${MARKETPLACE_ADDRESS_FROM_URL}" ]]; then
WAIT=${MARKETPLACE_ADDRESS_FROM_URL_WAIT:-300}
SECONDS=0
SLEEP=1
# Run and retry if fail
while (( SECONDS < WAIT )); do
MARKETPLACE_ADDRESS=($(curl -s -f -m 5 "${MARKETPLACE_ADDRESS_FROM_URL}"))
# Check if exit code is 0 and returned value is not empty
if [[ $? -eq 0 && -n "${MARKETPLACE_ADDRESS}" ]]; then
export CODEXCONTRACTS_MARKETPLACEADDRESS="${MARKETPLACE_ADDRESS}"
break
else
# Sleep and check again
echo "Can't get Marketplace address from ${MARKETPLACE_ADDRESS_FROM_URL} - Retry in $SLEEP seconds / $((WAIT - SECONDS))"
sleep $SLEEP
fi
done
fi
# Show
echo -e "\nRun parameters:"
vars=$(env | grep "CODEX" | grep -v -e "[0-9]_SERVICE_" -e "[0-9]_NODEPORT_")
echo -e "${vars//CODEX/ - CODEX}"
echo -e " - $@\n"
# Run
exec "$@"

3
Tools/TraceContract/ChainRequestTracker.cs
View File

@ -1,6 +1,7 @@
using System.Numerics;
using BlockchainUtils;
using CodexContractsPlugin.ChainMonitor;
using Nethereum.Hex.HexConvertors.Extensions;
using Utils;
namespace TraceContract
@ -96,7 +97,7 @@ namespace TraceContract
private bool IsMyRequest(RequestEvent requestEvent)
{
return requestId == requestEvent.Request.Request.Id.ToLowerInvariant();
return requestId == requestEvent.Request.RequestId.ToHex().ToLowerInvariant();
}
}

59
Tools/TraceContract/ChainTracer.cs
View File

@ -3,6 +3,7 @@ using CodexContractsPlugin.ChainMonitor;
using CodexContractsPlugin.Marketplace;
using Logging;
using Nethereum.Hex.HexConvertors.Extensions;
using Nethereum.Model;
using Utils;
namespace TraceContract
@ -28,10 +29,12 @@ namespace TraceContract
var request = GetRequest();
if (request == null) throw new Exception("Failed to find the purchase in the last week of transactions.");
log.Log($"Request started at {request.Block.Utc}");
var contractEnd = RunToContractEnd(request);
var creationEvent = FindRequestCreationEvent();
var requestTimeline = new TimeRange(request.Block.Utc.AddMinutes(-1.0), contractEnd.AddMinutes(1.0));
log.Log($"Request started at {creationEvent.Block.Utc}");
var contractEnd = RunToContractEnd(creationEvent);
var requestTimeline = new TimeRange(creationEvent.Block.Utc.AddMinutes(-1.0), contractEnd.AddMinutes(1.0));
log.Log($"Request timeline: {requestTimeline.From} -> {requestTimeline.To}");
// For this timeline, we log all the calls to reserve-slot.
@ -52,7 +55,7 @@ namespace TraceContract
return requestTimeline;
}
private DateTime RunToContractEnd(Request request)
private DateTime RunToContractEnd(StorageRequestedEventDTO request)
{
var utc = request.Block.Utc.AddMinutes(-1.0);
var tracker = new ChainRequestTracker(output, input.PurchaseId);
@ -78,35 +81,35 @@ namespace TraceContract
return tracker.FinishUtc;
}
private Request? GetRequest()
{
var request = FindRequest(LastHour());
if (request == null) request = FindRequest(LastDay());
if (request == null) request = FindRequest(LastWeek());
return request;
}
private Request? FindRequest(TimeRange timeRange)
{
var events = contracts.GetEvents(timeRange);
var requests = events.GetStorageRequests();
foreach (var r in requests)
{
if (IsThisRequest(r.RequestId))
{
return r;
}
}
return null;
}
private bool IsThisRequest(byte[] requestId)
{
return requestId.ToHex().ToLowerInvariant() == input.PurchaseId.ToLowerInvariant();
}
private Request? GetRequest()
{
return contracts.GetRequest(input.RequestId);
}
public StorageRequestedEventDTO FindRequestCreationEvent()
{
var range = new TimeRange(DateTime.UtcNow - TimeSpan.FromHours(3.0), DateTime.UtcNow);
var limit = DateTime.UtcNow - TimeSpan.FromDays(30);
while (range.From > limit)
{
var events = contracts.GetEvents(range);
foreach (var r in events.GetStorageRequestedEvents())
{
if (r.RequestId.ToHex() == input.RequestId.ToHex()) return r;
}
range = new TimeRange(range.From - TimeSpan.FromHours(3.0), range.From);
}
throw new Exception("Unable to find storage request creation event on-chain after (limit) " + Time.FormatTimestamp(limit));
}
private static TimeRange LastHour()
{
return new TimeRange(DateTime.UtcNow.AddHours(-1.0), DateTime.UtcNow);

14
Tools/TraceContract/Input.cs
View File

@ -1,4 +1,6 @@
namespace TraceContract
using Nethereum.Hex.HexConvertors.Extensions;
namespace TraceContract
{
public class Input
{
@ -17,5 +19,15 @@
//"066df09a3a2e2587cfd577a0e96186c915b113d02b331b06e56f808494cff2b4";
}
}
public byte[] RequestId
{
get
{
var r = PurchaseId.HexToByteArray();
if (r == null || r.Length != 32) throw new ArgumentException(nameof(PurchaseId));
return r;
}
}
}
}

4
Tools/TraceContract/Output.cs
View File

@ -3,6 +3,7 @@ using BlockchainUtils;
using CodexContractsPlugin.ChainMonitor;
using CodexContractsPlugin.Marketplace;
using Logging;
using Newtonsoft.Json;
using Utils;
namespace TraceContract
@ -49,7 +50,8 @@ namespace TraceContract
public void LogRequestCreated(RequestEvent requestEvent)
{
Add(requestEvent.Block, $"Storage request created: '{requestEvent.Request.Request.Id}'");
var msg = $"Storage request created: '{requestEvent.Request.RequestId}' = {Environment.NewLine}${JsonConvert.SerializeObject(requestEvent.Request.Request, Formatting.Indented)}{Environment.NewLine}";
Add(requestEvent.Block, msg);
}
public void LogRequestCancelled(RequestEvent requestEvent)