logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-01-02 13:33:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'wip/period-reporting'

Browse Source
This commit is contained in:
Ben 2025-09-01 10:35:50 +02:00
commit 7a192df0d0
No known key found for this signature in database
GPG Key ID: 0F16E812E736C24B
27 changed files with 910 additions and 446 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs
View File

@ -1,4 +1,5 @@
using Utils;
using System.Globalization;
using Utils;
namespace KubernetesWorkflow.Recipe
{
@ -88,6 +89,11 @@ namespace KubernetesWorkflow.Recipe
envVars.Add(factory.CreateEnvVar(name, value));
}
protected void AddEnvVar(string name, int value)
{
envVars.Add(factory.CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture)));
}
protected void AddEnvVar(string name, Port value)
{
envVars.Add(factory.CreateEnvVar(name, value.Number));

5
Framework/Utils/TimeRange.cs
View File

@ -20,5 +20,10 @@
public DateTime From { get; }
public DateTime To { get; }
public TimeSpan Duration { get; }
public override string ToString()
{
return $"{Time.FormatTimestamp(From)} -> {Time.FormatTimestamp(To)} ({Time.FormatDuration(Duration)})";
}
}
}

13
ProjectPlugins/CodexClient/TransferSpeeds.cs
View File

@ -6,6 +6,7 @@ namespace CodexClient
{
BytesPerSecond? GetUploadSpeed();
BytesPerSecond? GetDownloadSpeed();
ITransferSpeeds Combine(ITransferSpeeds? other);
}
public class TransferSpeeds : ITransferSpeeds
@ -35,6 +36,18 @@ namespace CodexClient
return downloads.Average();
}
public ITransferSpeeds Combine(ITransferSpeeds? other)
{
if (other == null) return this;
var o = (TransferSpeeds)other;
var result = new TransferSpeeds();
result.uploads.AddRange(uploads);
result.uploads.AddRange(o.uploads);
result.downloads.AddRange(downloads);
result.downloads.AddRange(o.downloads);
return result;
}
private static BytesPerSecond Convert(ByteSize size, TimeSpan duration)
{
double bytes = size.SizeInBytes;

5
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs
View File

@ -1,5 +1,6 @@
using BlockchainUtils;
using CodexContractsPlugin.Marketplace;
using GethPlugin;
using Logging;
using Nethereum.Contracts;
using Nethereum.Hex.HexConvertors.Extensions;
@ -42,14 +43,14 @@ namespace CodexContractsPlugin.ChainMonitor
private readonly IChainStateChangeHandler handler;
private readonly bool doProofPeriodMonitoring;
public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring)
public ChainState(ILog log, IGethNode geth, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring, IPeriodMonitorEventHandler periodEventHandler)
{
this.log = new LogPrefixer(log, "(ChainState) ");
this.contracts = contracts;
handler = changeHandler;
this.doProofPeriodMonitoring = doProofPeriodMonitoring;
TotalSpan = new TimeRange(startUtc, startUtc);
PeriodMonitor = new PeriodMonitor(log, contracts);
PeriodMonitor = new PeriodMonitor(log, contracts, geth, periodEventHandler);
}
public TimeRange TotalSpan { get; private set; }

239
ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs
View File

@ -1,35 +1,49 @@
using Logging;
using CodexContractsPlugin.Marketplace;
using GethPlugin;
using Logging;
using Nethereum.Contracts;
using Nethereum.Hex.HexConvertors.Extensions;
using Nethereum.Model;
using Utils;
using Nethereum.RPC.Eth.DTOs;
using Newtonsoft.Json;
using System.Reflection;
namespace CodexContractsPlugin.ChainMonitor
{
public interface IPeriodMonitorEventHandler
{
void OnPeriodReport(PeriodReport report);
}
public class PeriodMonitor
{
private readonly ILog log;
private readonly ICodexContracts contracts;
private readonly IGethNode geth;
private readonly IPeriodMonitorEventHandler eventHandler;
private readonly List<PeriodReport> reports = new List<PeriodReport>();
private ulong? currentPeriod = null;
private CurrentPeriod? currentPeriod = null;
public PeriodMonitor(ILog log, ICodexContracts contracts)
public PeriodMonitor(ILog log, ICodexContracts contracts, IGethNode geth, IPeriodMonitorEventHandler eventHandler)
{
this.log = log;
this.contracts = contracts;
this.geth = geth;
this.eventHandler = eventHandler;
}
public void Update(DateTime eventUtc, IChainStateRequest[] requests)
{
var period = contracts.GetPeriodNumber(eventUtc);
if (!currentPeriod.HasValue)
var periodNumber = contracts.GetPeriodNumber(eventUtc);
if (currentPeriod == null)
{
currentPeriod = period;
currentPeriod = CreateCurrentPeriod(periodNumber, requests);
return;
}
if (period == currentPeriod.Value) return;
if (periodNumber == currentPeriod.PeriodNumber) return;
CreateReportForPeriod(currentPeriod.Value, requests);
currentPeriod = period;
CreateReportForPeriod(currentPeriod, requests);
currentPeriod = CreateCurrentPeriod(periodNumber, requests);
}
public PeriodMonitorResult GetAndClearReports()
@ -39,26 +53,127 @@ namespace CodexContractsPlugin.ChainMonitor
return new PeriodMonitorResult(result);
}
private void CreateReportForPeriod(ulong periodNumber, IChainStateRequest[] requests)
private CurrentPeriod CreateCurrentPeriod(ulong periodNumber, IChainStateRequest[] requests)
{
var result = new CurrentPeriod(periodNumber);
ForEachActiveSlot(requests, (request, slotIndex) =>
{
if (contracts.IsProofRequired(request.RequestId, slotIndex) ||
contracts.WillProofBeRequired(request.RequestId, slotIndex))
{
var idx = Convert.ToInt32(slotIndex);
var host = request.Hosts.GetHost(idx);
var slotId = contracts.GetSlotId(request.RequestId, slotIndex);
if (host != null)
{
result.RequiredProofs.Add(new PeriodRequiredProof(host, request, idx, slotId));
}
}
});
return result;
}
private void CreateReportForPeriod(CurrentPeriod currentPeriod, IChainStateRequest[] requests)
{
// Fetch function calls during period. Format report.
var timeRange = contracts.GetPeriodTimeRange(currentPeriod.PeriodNumber);
var blockRange = geth.ConvertTimeRangeToBlockRange(timeRange);
var callReports = new List<FunctionCallReport>();
geth.IterateTransactions(blockRange, (t, blkI, blkUtc) =>
{
var reporter = new CallReporter(callReports, t, blkUtc, blkI);
reporter.Run();
});
var report = new PeriodReport(
new ProofPeriod(currentPeriod.PeriodNumber, timeRange, blockRange),
currentPeriod.RequiredProofs.ToArray(),
callReports.ToArray());
report.Log(log);
reports.Add(report);
eventHandler.OnPeriodReport(report);
}
private void ForEachActiveSlot(IChainStateRequest[] requests, Action<IChainStateRequest, ulong> action)
{
ulong total = 0;
var periodProofs = new List<PeriodProof>();
foreach (var request in requests)
{
for (ulong slotIndex = 0; slotIndex < request.Request.Ask.Slots; slotIndex++)
{
var state = contracts.GetProofState(request.RequestId, slotIndex, periodNumber);
total++;
var idx = Convert.ToInt32(slotIndex);
var host = request.Hosts.GetHost(idx);
var proof = new PeriodProof(host, request, idx, state);
periodProofs.Add(proof);
action(request, slotIndex);
}
}
var report = new PeriodReport(periodNumber, total, periodProofs.ToArray());
report.Log(log);
reports.Add(report);
}
}
public class DoNothingPeriodMonitorEventHandler : IPeriodMonitorEventHandler
{
public void OnPeriodReport(PeriodReport report)
{
}
}
public class CallReporter
{
private readonly List<FunctionCallReport> reports;
private readonly Transaction t;
private readonly DateTime blockUtc;
private readonly ulong blockNumber;
public CallReporter(List<FunctionCallReport> reports, Transaction t, DateTime blockUtc, ulong blockNumber)
{
this.reports = reports;
this.t = t;
this.blockUtc = blockUtc;
this.blockNumber = blockNumber;
}
public void Run()
{
CreateFunctionCallReport<CanMarkProofAsMissingFunction>();
CreateFunctionCallReport<CanReserveSlotFunction>();
CreateFunctionCallReport<ConfigurationFunction>();
CreateFunctionCallReport<CurrentCollateralFunction>();
CreateFunctionCallReport<FillSlotFunction>();
CreateFunctionCallReport<FreeSlot1Function>();
CreateFunctionCallReport<FreeSlotFunction>();
CreateFunctionCallReport<GetActiveSlotFunction>();
CreateFunctionCallReport<GetChallengeFunction>();
CreateFunctionCallReport<GetHostFunction>();
CreateFunctionCallReport<GetPointerFunction>();
CreateFunctionCallReport<GetRequestFunction>();
CreateFunctionCallReport<IsProofRequiredFunction>();
CreateFunctionCallReport<MarkProofAsMissingFunction>();
CreateFunctionCallReport<MissingProofsFunction>();
CreateFunctionCallReport<MyRequestsFunction>();
CreateFunctionCallReport<MySlotsFunction>();
CreateFunctionCallReport<RequestEndFunction>();
CreateFunctionCallReport<RequestExpiryFunction>();
CreateFunctionCallReport<RequestStateFunction>();
CreateFunctionCallReport<RequestStorageFunction>();
CreateFunctionCallReport<ReserveSlotFunction>();
CreateFunctionCallReport<SlotProbabilityFunction>();
CreateFunctionCallReport<SlotStateFunction>();
CreateFunctionCallReport<SubmitProofFunction>();
CreateFunctionCallReport<TokenFunction>();
CreateFunctionCallReport<WillProofBeRequiredFunction>();
CreateFunctionCallReport<WithdrawFundsFunction>();
CreateFunctionCallReport<WithdrawFunds1Function>();
}
private void CreateFunctionCallReport<TFunc>() where TFunc : FunctionMessage, new()
{
if (t.IsTransactionForFunctionMessage<TFunc>())
{
var func = t.DecodeTransactionToFunctionMessage<TFunc>();
reports.Add(new FunctionCallReport(blockUtc, blockNumber, typeof(TFunc).Name, JsonConvert.SerializeObject(func)));
}
}
}
@ -67,91 +182,19 @@ namespace CodexContractsPlugin.ChainMonitor
public PeriodMonitorResult(PeriodReport[] reports)
{
Reports = reports;
CalcStats();
}
public PeriodReport[] Reports { get; }
public bool IsEmpty { get; private set; }
public ulong PeriodLow { get; private set; }
public ulong PeriodHigh { get; private set; }
public float AverageNumSlots { get; private set; }
public float AverageNumProofsRequired { get; private set; }
private void CalcStats()
{
IsEmpty = Reports.All(r => r.PeriodProofs.Length == 0);
if (Reports.Length == 0) return;
PeriodLow = Reports.Min(r => r.PeriodNumber);
PeriodHigh = Reports.Max(r => r.PeriodNumber);
AverageNumSlots = Reports.Average(r => Convert.ToSingle(r.TotalNumSlots));
AverageNumProofsRequired = Reports.Average(r => Convert.ToSingle(r.PeriodProofs.Count(p => p.State != ProofState.NotRequired)));
}
}
public class PeriodReport
public class CurrentPeriod
{
public PeriodReport(ulong periodNumber, ulong totalNumSlots, PeriodProof[] periodProofs)
public CurrentPeriod(ulong periodNumber)
{
PeriodNumber = periodNumber;
TotalNumSlots = totalNumSlots;
PeriodProofs = periodProofs;
}
public ulong PeriodNumber { get; }
public ulong TotalNumSlots { get; }
public PeriodProof[] PeriodProofs { get; }
public PeriodProof[] GetMissedProofs()
{
return PeriodProofs.Where(p => p.State == ProofState.MissedAndMarked || p.State == ProofState.MissedNotMarked).ToArray();
}
public void Log(ILog log)
{
log.Log($"Period report: {PeriodNumber}");
log.Log($" - Slots: {TotalNumSlots}");
foreach (var p in PeriodProofs)
{
log.Log($" - {p.Describe()}");
}
}
private void Log(ILog log, PeriodProof[] proofs)
{
if (proofs.Length == 0) return;
foreach (var p in proofs)
{
}
}
}
public class PeriodProof
{
public PeriodProof(EthAddress? host, IChainStateRequest request, int slotIndex, ProofState state)
{
Host = host;
Request = request;
SlotIndex = slotIndex;
State = state;
}
public EthAddress? Host { get; }
public IChainStateRequest Request { get; }
public int SlotIndex { get; }
public ProofState State { get; }
public string FormatHost()
{
if (Host == null) return "Unknown host";
return Host.Address;
}
public string Describe()
{
return $"{FormatHost()} - {Request.RequestId.ToHex()} slotIndex:{SlotIndex} => {State}";
}
public List<PeriodRequiredProof> RequiredProofs { get; } = new List<PeriodRequiredProof>();
}
}

55
ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs Normal file
View File

@ -0,0 +1,55 @@
using Logging;
using Utils;
namespace CodexContractsPlugin.ChainMonitor
{
public class PeriodReport
{
public PeriodReport(ProofPeriod period, PeriodRequiredProof[] required, FunctionCallReport[] functionCalls)
{
Period = period;
Required = required;
FunctionCalls = functionCalls;
}
public ProofPeriod Period { get; }
public PeriodRequiredProof[] Required { get; }
public FunctionCallReport[] FunctionCalls { get; }
public void Log(ILog log)
{
log.Log($"Period report: {Period}");
log.Log($" - Proofs required: {Required.Length}");
foreach (var r in Required)
{
log.Log($" - {r.Describe()}");
}
log.Log($" - Calls: {FunctionCalls.Length}");
foreach (var f in FunctionCalls)
{
log.Log($" - {f.Describe()}");
}
}
}
public class FunctionCallReport
{
public FunctionCallReport(DateTime utc, ulong blockNumber, string name, string payload)
{
Utc = utc;
BlockNumber = blockNumber;
Name = name;
Payload = payload;
}
public DateTime Utc { get; }
public ulong BlockNumber { get; }
public string Name { get; }
public string Payload { get; }
public string Describe()
{
return $"[{Time.FormatTimestamp(Utc)}][{BlockNumber}] {Name} = \"{Payload}\"";
}
}
}

26
ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs Normal file
View File

@ -0,0 +1,26 @@
using Nethereum.Hex.HexConvertors.Extensions;
using Utils;
namespace CodexContractsPlugin.ChainMonitor
{
public class PeriodRequiredProof
{
public PeriodRequiredProof(EthAddress host, IChainStateRequest request, int slotIndex, byte[] slotId)
{
Host = host;
Request = request;
SlotIndex = slotIndex;
SlotId = slotId;
}
public EthAddress Host { get; }
public IChainStateRequest Request { get; }
public int SlotIndex { get; }
public byte[] SlotId { get; }
public string Describe()
{
return $"{Request.RequestId.ToHex()} slotId:{SlotId.ToHex()} slotIndex:{SlotIndex} by {Host}";
}
}
}

23
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs Normal file
View File

@ -0,0 +1,23 @@
using Utils;
namespace CodexContractsPlugin.ChainMonitor
{
public class ProofPeriod
{
public ProofPeriod(ulong periodNumber, TimeRange timeRange, BlockInterval blockRange)
{
PeriodNumber = periodNumber;
TimeRange = timeRange;
BlockRange = blockRange;
}
public ulong PeriodNumber { get; }
public TimeRange TimeRange { get; }
public BlockInterval BlockRange { get; }
public override string ToString()
{
return $"{{{PeriodNumber} - {TimeRange} {BlockRange}}}";
}
}
}

102
ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs
View File

@ -29,21 +29,15 @@ namespace CodexContractsPlugin
RequestState GetRequestState(byte[] requestId);
Request GetRequest(byte[] requestId);
ulong GetPeriodNumber(DateTime utc);
TimeRange GetPeriodTimeRange(ulong periodNumber);
void WaitUntilNextPeriod();
ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong period);
bool IsProofRequired(byte[] requestId, decimal slotIndex);
bool WillProofBeRequired(byte[] requestId, decimal slotIndex);
byte[] GetSlotId(byte[] requestId, decimal slotIndex);
ICodexContracts WithDifferentGeth(IGethNode node);
}
public enum ProofState
{
NotRequired,
NotMissed,
MissedNotMarked,
MissedAndMarked,
}
[JsonConverter(typeof(StringEnumConverter))]
public enum RequestState
{
@ -155,6 +149,16 @@ namespace CodexContractsPlugin
return Convert.ToUInt64(result);
}
public TimeRange GetPeriodTimeRange(ulong periodNumber)
{
var periodSeconds = (ulong)Deployment.Config.Proofs.Period;
var startUtco = Convert.ToInt64(periodSeconds * periodNumber);
var endUtco = Convert.ToInt64(periodSeconds * (periodNumber + 1));
var start = DateTimeOffset.FromUnixTimeSeconds(startUtco).UtcDateTime;
var end = DateTimeOffset.FromUnixTimeSeconds(endUtco).UtcDateTime;
return new TimeRange(start, end);
}
public void WaitUntilNextPeriod()
{
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
@ -163,14 +167,16 @@ namespace CodexContractsPlugin
Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
}
public ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong period)
public bool IsProofRequired(byte[] requestId, decimal slotIndex)
{
var slotId = GetSlotId(requestId, slotIndex);
return IsProofRequired(slotId);
}
var required = IsProofRequired(slotId);
if (!required) return ProofState.NotRequired;
return IsProofMissing(slotId, period);
public bool WillProofBeRequired(byte[] requestId, decimal slotIndex)
{
var slotId = GetSlotId(requestId, slotIndex);
return WillProofBeRequired(slotId);
}
public ICodexContracts WithDifferentGeth(IGethNode node)
@ -199,70 +205,14 @@ namespace CodexContractsPlugin
return result.ReturnValue1;
}
private ProofState IsProofMissing(byte[] slotId, ulong period)
private bool WillProofBeRequired(byte[] slotId)
{
// In case of a missed proof, one of two things can be true:
// 1 - The proof was missed but no validator marked it as missing:
// We can see this by calling "canMarkProofAsMissing" and it returns true/doesn't throw.
// 2 - The proof was missed and it was marked as missing by a validator:
// We can see this by a successful call to "MarkProofAsMissing" on-chain.
if (CallCanMarkProofAsMissing(slotId, period))
var func = new WillProofBeRequiredFunction
{
return ProofState.MissedNotMarked;
}
if (WasMarkProofAsMissingCalled(slotId, period))
{
return ProofState.MissedAndMarked;
}
return ProofState.NotMissed;
}
private bool CallCanMarkProofAsMissing(byte[] slotId, ulong period)
{
try
{
var func = new CanMarkProofAsMissingFunction
{
SlotId = slotId,
Period = period
};
gethNode.Call<CanMarkProofAsMissingFunction>(Deployment.MarketplaceAddress, func);
return true;
}
catch (AggregateException exc)
{
if (exc.InnerExceptions.Count == 1)
{
if (exc.InnerExceptions[0].GetType() == typeof(SmartContractCustomErrorRevertException))
{
return false;
}
}
throw;
}
}
private bool WasMarkProofAsMissingCalled(byte[] slotId, ulong period)
{
var now = DateTime.UtcNow;
var currentPeriod = new TimeRange(now - Deployment.Config.PeriodDuration, now);
var interval = gethNode.ConvertTimeRangeToBlockRange(currentPeriod);
var slot = slotId.ToHex().ToLowerInvariant();
var found = false;
gethNode.IterateFunctionCalls<MarkProofAsMissingFunction>(interval, (b, fn) =>
{
if (fn.Period == period && fn.SlotId.ToHex().ToLowerInvariant() == slot)
{
found = true;
}
});
return found;
Id = slotId
};
var result = gethNode.Call<WillProofBeRequiredFunction, WillProofBeRequiredOutputDTO>(Deployment.MarketplaceAddress, func);
return result.ReturnValue1;
}
private ContractInteractions StartInteraction()

21
ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs
View File

@ -9,6 +9,11 @@ namespace CodexContractsPlugin
{
public const string DeployedAddressesFilename = "/hardhat/ignition/deployments/chain-789988/deployed_addresses.json";
public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
public const int PeriodSeconds = 60;
public const int TimeoutSeconds = 30;
public const int DowntimeSeconds = 128;
private readonly DebugInfoVersion versionInfo;
public override string AppName => "codex-contracts";
@ -28,6 +33,22 @@ namespace CodexContractsPlugin
SetSchedulingAffinity(notIn: "false");
AddEnvVar("DISTTEST_NETWORK_URL", address.ToString());
// Default values:
AddEnvVar("DISTTEST_REPAIRREWARD", 10);
AddEnvVar("DISTTEST_MAXSLASHES", 2);
AddEnvVar("DISTTEST_SLASHPERCENTAGE", 20);
AddEnvVar("DISTTEST_VALIDATORREWARD", 20);
AddEnvVar("DISTTEST_DOWNTIMEPRODUCT", 67);
AddEnvVar("DISTTEST_MAXRESERVATIONS", 3);
AddEnvVar("DISTTEST_MAXDURATION", Convert.ToInt32(TimeSpan.FromDays(30).TotalSeconds));
// Customized values, required to operate in a network with
// block frequency of 1.
AddEnvVar("DISTTEST_PERIOD", PeriodSeconds);
AddEnvVar("DISTTEST_TIMEOUT", TimeoutSeconds);
AddEnvVar("DISTTEST_DOWNTIME", DowntimeSeconds);
AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");
AddEnvVar("HARDHAT_IGNITION_CONFIRM_DEPLOYMENT", "false");
AddEnvVar("KEEP_ALIVE", "1");

21
ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs
View File

@ -92,12 +92,31 @@ namespace CodexContractsPlugin
Log("Synced. Codex SmartContracts deployed. Getting configuration...");
var config = GetMarketplaceConfiguration(marketplaceAddress, gethNode);
Log("Got config: " + JsonConvert.SerializeObject(config));
ConfigShouldEqual(config.Proofs.Period, CodexContractsContainerRecipe.PeriodSeconds, "Period");
ConfigShouldEqual(config.Proofs.Timeout, CodexContractsContainerRecipe.TimeoutSeconds, "Timeout");
ConfigShouldEqual(config.Proofs.Downtime, CodexContractsContainerRecipe.DowntimeSeconds, "Downtime");
return new CodexContractsDeployment(config, marketplaceAddress, abi, tokenAddress);
}
private void ConfigShouldEqual(ulong value, int expected, string name)
{
if (Convert.ToInt32(value) != expected)
{
// Merge todo: https://github.com/codex-storage/nim-codex/pull/1303
// Once this is merged, the contract config values are settable via env-vars.
// This plugin is already updated to set the config vars to values compatible with a
// 1-second block frequency. AND it will read back the config and assert it is deployed correctly.
// This is waiting for that merge.
// Replace log with assert WHEN MERGED:
// throw new Exception($"Config value '{name}' should be deployed as '{expected}' but was '{value}'");
Log($"MERGE TODO. Config value '{name}' should be deployed as '{expected}' but was '{value}'");
}
}
private MarketplaceConfig GetMarketplaceConfiguration(string marketplaceAddress, IGethNode gethNode)
{
var func = new ConfigurationFunctionBase();

10
ProjectPlugins/CodexPlugin/CodexStartupConfig.cs
View File

@ -85,8 +85,11 @@ namespace CodexPlugin
{
"JSONRPC-WS-CLIENT",
"JSONRPC-HTTP-CLIENT",
"codex",
"repostore"
};
var alwaysIgnoreTopics = new []
{
"JSONRPC-CLIENT"
};
level = $"{level};" +
@ -94,7 +97,8 @@ namespace CodexPlugin
$"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" +
$"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)};" +
$"{CustomTopics.JsonSerialize.ToString().ToLowerInvariant()}:{string.Join(",", jsonSerializeTopics)};" +
$"{CustomTopics.MarketplaceInfra.ToString().ToLowerInvariant()}:{string.Join(",", marketplaceInfraTopics)}";
$"{CustomTopics.MarketplaceInfra.ToString().ToLowerInvariant()}:{string.Join(",", marketplaceInfraTopics)};" +
$"{CodexLogLevel.Error.ToString()}:{string.Join(",", alwaysIgnoreTopics)}";
if (CustomTopics.BlockExchange != null)
{

31
ProjectPlugins/GethPlugin/GethNode.cs
View File

@ -3,6 +3,7 @@ using Core;
using KubernetesWorkflow.Types;
using Logging;
using Nethereum.ABI.FunctionEncoding.Attributes;
using Nethereum.BlockchainProcessing.BlockStorage.Entities.Mapping;
using Nethereum.Contracts;
using Nethereum.RPC.Eth.DTOs;
using NethereumWorkflow;
@ -33,6 +34,7 @@ namespace GethPlugin
List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
BlockTimeEntry GetBlockForNumber(ulong number);
void IterateTransactions(BlockInterval blockRange, Action<Transaction, ulong, DateTime> action);
void IterateFunctionCalls<TFunc>(BlockInterval blockInterval, Action<BlockTimeEntry, TFunc> onCall) where TFunc : FunctionMessage, new();
IGethNode WithDifferentAccount(EthAccount account);
}
@ -223,28 +225,37 @@ namespace GethPlugin
return StartInteraction().GetBlockWithTransactions(number);
}
public void IterateFunctionCalls<TFunc>(BlockInterval blockRange, Action<BlockTimeEntry, TFunc> onCall) where TFunc : FunctionMessage, new()
public void IterateTransactions(BlockInterval blockRange, Action<Transaction, ulong, DateTime> action)
{
var i = StartInteraction();
for (var blkI = blockRange.From; blkI <= blockRange.To; blkI++)
{
var blk = i.GetBlockWithTransactions(blkI);
var blkUtc = DateTimeOffset.FromUnixTimeSeconds(blk.Timestamp.ToLong()).UtcDateTime;
foreach (var t in blk.Transactions)
{
if (t.IsTransactionForFunctionMessage<TFunc>())
{
var func = t.DecodeTransactionToFunctionMessage<TFunc>();
if (func != null)
{
var b = GetBlockForNumber(blkI);
onCall(b, func);
}
}
action(t, blkI, blkUtc);
}
}
}
public void IterateFunctionCalls<TFunc>(BlockInterval blockRange, Action<BlockTimeEntry, TFunc> onCall) where TFunc : FunctionMessage, new()
{
IterateTransactions(blockRange, (t, blkI, blkUtc) =>
{
if (t.IsTransactionForFunctionMessage<TFunc>())
{
var func = t.DecodeTransactionToFunctionMessage<TFunc>();
if (func != null)
{
var b = GetBlockForNumber(blkI);
onCall(b, func);
}
}
});
}
protected abstract NethereumInteraction StartInteraction();
}
}

286
Tests/CodexReleaseTests/DataTests/SwarmTest.cs
View File

@ -7,175 +7,191 @@ using Utils;
namespace CodexReleaseTests.DataTests
{
[TestFixture(2, 10)]
[TestFixture(5, 20)]
[TestFixture(10, 20)]
public class SwarmTests : AutoBootstrapDistTest
namespace SwarmTests
{
private readonly int numberOfNodes;
private readonly int filesizeMb;
public SwarmTests(int numberOfNodes, int filesizeMb)
[TestFixture(2, 10)]
[TestFixture(5, 20)]
[TestFixture(10, 20)]
public class SwarmTests : AutoBootstrapDistTest
{
this.numberOfNodes = numberOfNodes;
this.filesizeMb = filesizeMb;
}
private readonly int numberOfNodes;
private readonly int filesizeMb;
private ICodexNodeGroup nodes = null!;
[Test]
public void Swarm()
{
var filesize = filesizeMb.MB();
var nodes = StartCodex(numberOfNodes);
var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray();
var tasks = ParallelDownloadEachFile(nodes, files);
Task.WaitAll(tasks);
AssertAllFilesDownloadedCorrectly(files);
}
[Test]
public void StreamlessSwarm()
{
var filesize = filesizeMb.MB();
var nodes = StartCodex(numberOfNodes);
var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray();
var tasks = ParallelStreamlessDownloadEachFile(nodes, files);
Task.WaitAll(tasks);
AssertAllFilesStreamlesslyDownloadedCorrectly(nodes, files);
}
private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, ByteSize fileSize)
{
var file = GenerateTestFile(fileSize);
var cid = node.UploadFile(file);
return new SwarmTestNetworkFile(node, fileSize, file, cid);
}
private Task[] ParallelDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
{
var tasks = new List<Task>();
foreach (var node in nodes)
public SwarmTests(int numberOfNodes, int filesizeMb)
{
tasks.Add(StartDownload(node, files));
this.numberOfNodes = numberOfNodes;
this.filesizeMb = filesizeMb;
}
return tasks.ToArray();
}
private Task[] ParallelStreamlessDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
{
var tasks = new List<Task>();
foreach (var node in nodes)
[TearDown]
public void TearDown()
{
tasks.Add(StartStreamlessDownload(node, files));
}
return tasks.ToArray();
}
private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files)
{
return Task.Run(() =>
{
var remaining = files.ToList();
while (remaining.Count > 0)
ITransferSpeeds speeds = new TransferSpeeds();
foreach (var n in nodes)
{
var file = remaining.PickOneRandom();
try
{
var dl = node.DownloadContent(file.Cid, TimeSpan.FromMinutes(30));
lock (file.Lock)
{
file.Downloaded.Add(dl);
}
}
catch (Exception ex)
{
file.Error = ex;
}
speeds = speeds.Combine(n.TransferSpeeds);
}
});
}
Log($"Average upload speed: {speeds.GetUploadSpeed()}");
Log($"Average download speed: {speeds.GetDownloadSpeed()}");
}
private Task StartStreamlessDownload(ICodexNode node, SwarmTestNetworkFile[] files)
{
return Task.Run(() =>
[Test]
public void Stream()
{
var remaining = files.ToList();
var filesize = filesizeMb.MB();
nodes = StartCodex(numberOfNodes);
var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray();
while (remaining.Count > 0)
var tasks = ParallelDownloadEachFile(files);
Task.WaitAll(tasks);
AssertAllFilesDownloadedCorrectly(files);
}
[Test]
public void Streamless()
{
var filesize = filesizeMb.MB();
nodes = StartCodex(numberOfNodes);
var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray();
var tasks = ParallelStreamlessDownloadEachFile(files);
Task.WaitAll(tasks);
AssertAllFilesStreamlesslyDownloadedCorrectly(files);
}
private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, ByteSize fileSize)
{
var file = GenerateTestFile(fileSize);
var cid = node.UploadFile(file);
return new SwarmTestNetworkFile(node, fileSize, file, cid);
}
private Task[] ParallelDownloadEachFile(SwarmTestNetworkFile[] files)
{
var tasks = new List<Task>();
foreach (var node in nodes)
{
var file = remaining.PickOneRandom();
if (file.Uploader.GetName() != node.GetName())
tasks.Add(StartDownload(node, files));
}
return tasks.ToArray();
}
private Task[] ParallelStreamlessDownloadEachFile(SwarmTestNetworkFile[] files)
{
var tasks = new List<Task>();
foreach (var node in nodes)
{
tasks.Add(StartStreamlessDownload(node, files));
}
return tasks.ToArray();
}
private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files)
{
return Task.Run(() =>
{
var remaining = files.ToList();
while (remaining.Count > 0)
{
var file = remaining.PickOneRandom();
try
{
var startSpace = node.Space();
node.DownloadStreamlessWait(file.Cid, file.OriginalSize);
var dl = node.DownloadContent(file.Cid, TimeSpan.FromMinutes(30));
lock (file.Lock)
{
file.Downloaded.Add(dl);
}
}
catch (Exception ex)
{
file.Error = ex;
}
}
}
});
}
});
}
private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files)
{
foreach (var file in files)
private Task StartStreamlessDownload(ICodexNode node, SwarmTestNetworkFile[] files)
{
if (file.Error != null) throw file.Error;
lock (file.Lock)
return Task.Run(() =>
{
foreach (var dl in file.Downloaded)
var remaining = files.ToList();
while (remaining.Count > 0)
{
file.Original.AssertIsEqual(dl);
var file = remaining.PickOneRandom();
if (file.Uploader.GetName() != node.GetName())
{
try
{
var startSpace = node.Space();
node.DownloadStreamlessWait(file.Cid, file.OriginalSize);
}
catch (Exception ex)
{
file.Error = ex;
}
}
}
});
}
private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files)
{
foreach (var file in files)
{
if (file.Error != null) throw file.Error;
lock (file.Lock)
{
foreach (var dl in file.Downloaded)
{
file.Original.AssertIsEqual(dl);
}
}
}
}
}
private void AssertAllFilesStreamlesslyDownloadedCorrectly(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
{
var totalFilesSpace = 0.Bytes();
foreach (var file in files)
private void AssertAllFilesStreamlesslyDownloadedCorrectly(SwarmTestNetworkFile[] files)
{
if (file.Error != null) throw file.Error;
totalFilesSpace = new ByteSize(totalFilesSpace.SizeInBytes + file.Original.GetFilesize().SizeInBytes);
}
foreach (var node in nodes)
{
var currentSpace = node.Space();
Assert.That(currentSpace.QuotaUsedBytes, Is.GreaterThanOrEqualTo(totalFilesSpace.SizeInBytes));
}
}
var totalFilesSpace = 0.Bytes();
foreach (var file in files)
{
if (file.Error != null) throw file.Error;
totalFilesSpace = new ByteSize(totalFilesSpace.SizeInBytes + file.Original.GetFilesize().SizeInBytes);
}
private class SwarmTestNetworkFile
{
public SwarmTestNetworkFile(ICodexNode uploader, ByteSize originalSize, TrackedFile original, ContentId cid)
{
Uploader = uploader;
OriginalSize = originalSize;
Original = original;
Cid = cid;
foreach (var node in nodes)
{
var currentSpace = node.Space();
Assert.That(currentSpace.QuotaUsedBytes, Is.GreaterThanOrEqualTo(totalFilesSpace.SizeInBytes));
}
}
public ICodexNode Uploader { get; }
public ByteSize OriginalSize { get; }
public TrackedFile Original { get; }
public ContentId Cid { get; }
public object Lock { get; } = new object();
public List<TrackedFile?> Downloaded { get; } = new List<TrackedFile?>();
public Exception? Error { get; set; } = null;
private class SwarmTestNetworkFile
{
public SwarmTestNetworkFile(ICodexNode uploader, ByteSize originalSize, TrackedFile original, ContentId cid)
{
Uploader = uploader;
OriginalSize = originalSize;
Original = original;
Cid = cid;
}
public ICodexNode Uploader { get; }
public ByteSize OriginalSize { get; }
public TrackedFile Original { get; }
public ContentId Cid { get; }
public object Lock { get; } = new object();
public List<TrackedFile?> Downloaded { get; } = new List<TrackedFile?>();
public Exception? Error { get; set; } = null;
}
}
}
}

99
Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs Normal file
View File

@ -0,0 +1,99 @@
using CodexClient;
using CodexContractsPlugin.ChainMonitor;
using CodexReleaseTests.Utils;
using Nethereum.Hex.HexConvertors.Extensions;
using NUnit.Framework;
using Utils;
namespace CodexReleaseTests.MarketTests
{
[TestFixture]
public class IsProofRequiredTest : MarketplaceAutoBootstrapDistTest
{
#region Setup
private readonly PurchaseParams purchaseParams = new PurchaseParams(
nodes: 4,
tolerance: 2,
uploadFilesize: 32.MB()
);
public IsProofRequiredTest()
{
Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts));
}
protected override int NumberOfHosts => 6;
protected override int NumberOfClients => 1;
protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot.
protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0);
#endregion
[Test]
public void IsProofRequired()
{
var mins = TimeSpan.FromMinutes(10.0);
StartHosts();
StartValidator();
var client = StartClients().Single();
var purchase = CreateStorageRequest(client, mins);
purchase.WaitForStorageContractStarted();
var requestId = purchase.PurchaseId.HexToByteArray();
var numSlots = purchaseParams.Nodes;
//var map = new Dictionary<ulong, List<int>>();
Log($"Checking IsProofRequired every second for {Time.FormatDuration(mins)}.");
var endUtc = DateTime.UtcNow + mins;
while (DateTime.UtcNow < endUtc)
{
Thread.Sleep(TimeSpan.FromSeconds(1));
var requiredSlotIndices = new List<int>();
var willRequireSlotIndices = new List<int>();
for (var i = 0; i < numSlots; i++)
{
if (GetContracts().IsProofRequired(requestId, i)) requiredSlotIndices.Add(i);
if (GetContracts().WillProofBeRequired(requestId, i)) willRequireSlotIndices.Add(i);
}
var periodNumber = GetContracts().GetPeriodNumber(DateTime.UtcNow);
var blockNumber = GetGeth().GetSyncedBlockNumber();
Log($"[{blockNumber?.ToString().PadLeft(4, '0')}]" +
$"{periodNumber.ToString().PadLeft(12, '0')} => Required now: " +
$"[{string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}] " +
$"Will be required: " +
$"[{string.Join(",", willRequireSlotIndices.Select(i => i.ToString()))}]");
//var num = currentPeriod.PeriodNumber;
//if (!map.ContainsKey(num))
//{
// map.Add(num, requiredSlotIndices);
// Log($"Period {num} = required proof for slot indices {string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}");
//}
//else
//{
// var a = map[num];
// CollectionAssert.AreEquivalent(a, requiredSlotIndices);
//}
}
}
private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, TimeSpan minutes)
{
var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
var config = GetContracts().Deployment.Config;
return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
{
Duration = minutes * 1.1,
Expiry = TimeSpan.FromMinutes(8.0),
MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
NodeFailureTolerance = (uint)purchaseParams.Tolerance,
PricePerBytePerSecond = 10.TstWei(),
ProofProbability = 1, // One proof every period. Free slot as quickly as possible.
CollateralPerByte = 1.TstWei()
});
}
}
}

148
Tests/CodexReleaseTests/MarketTests/StabilityTest.cs Normal file
View File

@ -0,0 +1,148 @@
using CodexClient;
using CodexContractsPlugin.ChainMonitor;
using CodexContractsPlugin.Marketplace;
using CodexReleaseTests.Utils;
using Nethereum.Hex.HexConvertors.Extensions;
using Newtonsoft.Json;
using NUnit.Framework;
using Utils;
namespace CodexReleaseTests.MarketTests
{
[TestFixture]
public class StabilityTest : MarketplaceAutoBootstrapDistTest
{
#region Setup
private readonly PurchaseParams purchaseParams = new PurchaseParams(
nodes: 4,
tolerance: 2,
uploadFilesize: 32.MB()
);
public StabilityTest()
{
Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts));
}
protected override int NumberOfHosts => 6;
protected override int NumberOfClients => 1;
protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot.
protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0);
#endregion
private int numPeriods = 0;
private bool proofWasMissed = false;
[Test]
[Combinatorial]
public void Stability(
[Values(10, 120)] int minutes)
{
var mins = TimeSpan.FromMinutes(minutes);
var periodDuration = GetContracts().Deployment.Config.PeriodDuration;
Assert.That(HostAvailabilityMaxDuration, Is.GreaterThan(mins * 1.1));
numPeriods = 0;
proofWasMissed = false;
StartHosts();
StartValidator();
var client = StartClients().Single();
var purchase = CreateStorageRequest(client, mins);
purchase.WaitForStorageContractStarted();
Log($"Contract should remain stable for {Time.FormatDuration(mins)}.");
var endUtc = DateTime.UtcNow + mins;
while (DateTime.UtcNow < endUtc)
{
Thread.Sleep(TimeSpan.FromSeconds(10));
if (proofWasMissed)
{
// We wait because we want to log calls to MarkProofAsMissing.
Thread.Sleep(periodDuration * 1.1);
Assert.Fail("Proof was missed.");
}
}
var minNumPeriod = (mins / periodDuration) - 1.0;
Log($"{numPeriods} periods elapsed. Expected at least {minNumPeriod} periods.");
Assert.That(numPeriods, Is.GreaterThanOrEqualTo(minNumPeriod));
var status = client.GetPurchaseStatus(purchase.PurchaseId);
if (status == null) throw new Exception("Purchase status not found");
Assert.That(status.IsStarted || status.IsFinished);
}
protected override void OnPeriod(PeriodReport report)
{
numPeriods++;
// For each required proof, there should be a submit call.
var calls = GetSubmitProofCalls(report);
foreach (var required in report.Required)
{
var matchingCall = GetMatchingSubmitProofCall(calls, required);
if (matchingCall == null)
{
Log($"A proof was missed for {required.Describe()}. Failing test after a delay so chain events have time to log...");
proofWasMissed = true;
}
}
// There can't be any calls to mark a proof as missed.
foreach (var call in report.FunctionCalls)
{
var missedCall = nameof(MarkProofAsMissingFunction);
Assert.That(call.Name, Is.Not.EqualTo(missedCall));
}
}
private SubmitProofFunction? GetMatchingSubmitProofCall(SubmitProofFunction[] calls, PeriodRequiredProof required)
{
foreach (var call in calls)
{
if (
call.Id.SequenceEqual(required.SlotId) &&
call.FromAddress.ToLowerInvariant() == required.Host.Address.ToLowerInvariant()
)
{
return call;
}
}
return null;
}
private SubmitProofFunction[] GetSubmitProofCalls(PeriodReport report)
{
var submitCall = nameof(SubmitProofFunction);
var calls = report.FunctionCalls.Where(f => f.Name == submitCall).ToArray();
var callObjs = calls.Select(call => JsonConvert.DeserializeObject<SubmitProofFunction>(call.Payload)).ToArray();
Log($"SubmitProof calls: {callObjs.Length}");
foreach (var c in callObjs)
{
Log($" - slotId:{c.Id.ToHex()} host:{c.FromAddress}");
}
return callObjs!;
}
private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, TimeSpan minutes)
{
var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
var config = GetContracts().Deployment.Config;
return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
{
Duration = minutes * 1.1,
Expiry = TimeSpan.FromMinutes(8.0),
MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
NodeFailureTolerance = (uint)purchaseParams.Tolerance,
PricePerBytePerSecond = 10.TstWei(),
ProofProbability = 1, // One proof every period. Free slot as quickly as possible.
CollateralPerByte = 1.TstWei()
});
}
}
}

22
Tests/CodexReleaseTests/Utils/ChainMonitor.cs
View File

@ -1,5 +1,6 @@
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using GethPlugin;
using Logging;
using Utils;
@ -8,21 +9,25 @@ namespace CodexReleaseTests.Utils
public class ChainMonitor
{
private readonly ILog log;
private readonly IGethNode gethNode;
private readonly ICodexContracts contracts;
private readonly IPeriodMonitorEventHandler periodMonitorEventHandler;
private readonly DateTime startUtc;
private readonly TimeSpan updateInterval;
private CancellationTokenSource cts = new CancellationTokenSource();
private Task worker = Task.CompletedTask;
public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc)
: this(log, contracts, startUtc, TimeSpan.FromSeconds(3.0))
public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, IPeriodMonitorEventHandler periodMonitorEventHandler, DateTime startUtc)
: this(log, gethNode, contracts, periodMonitorEventHandler, startUtc, TimeSpan.FromSeconds(3.0))
{
}
public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc, TimeSpan updateInterval)
public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, IPeriodMonitorEventHandler periodMonitorEventHandler, DateTime startUtc, TimeSpan updateInterval)
{
this.log = log;
this.gethNode = gethNode;
this.contracts = contracts;
this.periodMonitorEventHandler = periodMonitorEventHandler;
this.startUtc = startUtc;
this.updateInterval = updateInterval;
}
@ -42,7 +47,7 @@ namespace CodexReleaseTests.Utils
private void Worker(Action onFailure)
{
var state = new ChainState(log, contracts, new DoNothingThrowingChainEventHandler(), startUtc, doProofPeriodMonitoring: true);
var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, true, periodMonitorEventHandler);
Thread.Sleep(updateInterval);
log.Log($"Chain monitoring started. Update interval: {Time.FormatDuration(updateInterval)}");
@ -66,15 +71,6 @@ namespace CodexReleaseTests.Utils
private void UpdateChainState(ChainState state)
{
state.Update();
var reports = state.PeriodMonitor.GetAndClearReports();
if (reports.IsEmpty) return;
var slots = reports.Reports.Sum(r => Convert.ToInt32(r.TotalNumSlots));
var required = reports.Reports.Sum(r => Convert.ToInt32(r.PeriodProofs.Count(p => p.State != ProofState.NotRequired)));
var missed = reports.Reports.Sum(r => r.GetMissedProofs().Length);
log.Log($"Proof report: Slots={slots} Required={required} Missed={missed}");
}
}
}

120
Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs
View File

@ -1,5 +1,6 @@
using CodexClient;
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using CodexContractsPlugin.Marketplace;
using CodexPlugin;
using CodexTests;
@ -11,7 +12,7 @@ using Utils;
namespace CodexReleaseTests.Utils
{
public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest
public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest, IPeriodMonitorEventHandler
{
private MarketplaceHandle handle = null!;
protected const int StartingBalanceTST = 1000;
@ -22,7 +23,7 @@ namespace CodexReleaseTests.Utils
{
var geth = StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version);
var monitor = SetupChainMonitor(GetTestLog(), contracts, GetTestRunTimeRange().From);
var monitor = SetupChainMonitor(GetTestLog(), geth, contracts, GetTestRunTimeRange().From);
handle = new MarketplaceHandle(geth, contracts, monitor);
}
@ -42,6 +43,12 @@ namespace CodexReleaseTests.Utils
return handle.Contracts;
}
protected ChainMonitor GetChainMonitor()
{
if (handle.ChainMonitor == null) throw new Exception($"Make sure {nameof(MonitorChainState)} is set to true.");
return handle.ChainMonitor;
}
protected TimeSpan GetPeriodDuration()
{
var config = GetContracts().Deployment.Config;
@ -54,6 +61,9 @@ namespace CodexReleaseTests.Utils
protected abstract TimeSpan HostAvailabilityMaxDuration { get; }
protected virtual bool MonitorChainState { get; } = true;
protected TimeSpan HostBlockTTL { get; } = TimeSpan.FromMinutes(1.0);
protected virtual void OnPeriod(PeriodReport report)
{
}
public ICodexNodeGroup StartHosts()
{
@ -169,56 +179,6 @@ namespace CodexReleaseTests.Utils
});
}
private ChainMonitor? SetupChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc)
{
if (!MonitorChainState) return null;
var result = new ChainMonitor(log, contracts, startUtc);
result.Start(() =>
{
Assert.Fail("Failure in chain monitor.");
});
return result;
}
private Retry GetBalanceAssertRetry()
{
return new Retry("AssertBalance",
maxTimeout: TimeSpan.FromMinutes(10.0),
sleepAfterFail: TimeSpan.FromSeconds(10.0),
onFail: f => { },
failFast: false);
}
private Retry GetAvailabilitySpaceAssertRetry()
{
return new Retry("AssertAvailabilitySpace",
maxTimeout: HostBlockTTL * 3,
sleepAfterFail: TimeSpan.FromSeconds(10.0),
onFail: f => { },
failFast: false);
}
private TestToken GetTstBalance(ICodexNode node)
{
return GetContracts().GetTestTokenBalance(node);
}
private TestToken GetTstBalance(EthAddress address)
{
return GetContracts().GetTestTokenBalance(address);
}
private Ether GetEthBalance(ICodexNode node)
{
return GetGeth().GetEthBalance(node);
}
private Ether GetEthBalance(EthAddress address)
{
return GetGeth().GetEthBalance(address);
}
public ICodexNodeGroup StartClients()
{
return StartClients(s => { });
@ -247,6 +207,11 @@ namespace CodexReleaseTests.Utils
);
}
public void OnPeriodReport(PeriodReport report)
{
OnPeriod(report);
}
public SlotFill[] GetOnChainSlotFills(IEnumerable<ICodexNode> possibleHosts, string purchaseId)
{
var fills = GetOnChainSlotFills(possibleHosts);
@ -345,6 +310,56 @@ namespace CodexReleaseTests.Utils
}
}
private ChainMonitor? SetupChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc)
{
if (!MonitorChainState) return null;
var result = new ChainMonitor(log, gethNode, contracts, this, startUtc);
result.Start(() =>
{
Assert.Fail("Failure in chain monitor.");
});
return result;
}
private Retry GetBalanceAssertRetry()
{
return new Retry("AssertBalance",
maxTimeout: TimeSpan.FromMinutes(10.0),
sleepAfterFail: TimeSpan.FromSeconds(10.0),
onFail: f => { },
failFast: false);
}
private Retry GetAvailabilitySpaceAssertRetry()
{
return new Retry("AssertAvailabilitySpace",
maxTimeout: HostBlockTTL * 3,
sleepAfterFail: TimeSpan.FromSeconds(10.0),
onFail: f => { },
failFast: false);
}
private TestToken GetTstBalance(ICodexNode node)
{
return GetContracts().GetTestTokenBalance(node);
}
private TestToken GetTstBalance(EthAddress address)
{
return GetContracts().GetTestTokenBalance(address);
}
private Ether GetEthBalance(ICodexNode node)
{
return GetGeth().GetEthBalance(node);
}
private Ether GetEthBalance(EthAddress address)
{
return GetGeth().GetEthBalance(address);
}
private TestToken GetContractFinalCost(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
{
var fills = GetOnChainSlotFills(hosts);
@ -459,6 +474,7 @@ namespace CodexReleaseTests.Utils
var chanceOfDowntime = downtime / window;
return 1.0f + (5.0f * chanceOfDowntime);
}
public class SlotFill
{
public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)

3
Tests/DistTestCore/NameUtils.cs
View File

@ -91,7 +91,8 @@ namespace DistTestCore
return Path.Join(
config.LogRoot,
$"{start.Year}-{Pad(start.Month)}",
Pad(start.Day));
Pad(start.Day),
$"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}");
}
private static string Pad(int n)

7
Tools/MarketInsights/AverageHistory.cs
View File

@ -1,5 +1,6 @@
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using GethPlugin;
using TestNetRewarder;
using Utils;
@ -13,12 +14,12 @@ namespace MarketInsights
private readonly int maxContributions;
private readonly ChainState chainState;
public AverageHistory(AppState appState, ICodexContracts contracts, int maxContributions)
public AverageHistory(AppState appState, IGethNode geth, ICodexContracts contracts, int maxContributions)
{
this.appState = appState;
this.maxContributions = maxContributions;
chainState = new ChainState(appState.Log, contracts, mux, appState.Config.HistoryStartUtc,
doProofPeriodMonitoring: false);
chainState = new ChainState(appState.Log, geth, contracts, mux, appState.Config.HistoryStartUtc,
doProofPeriodMonitoring: false, new DoNothingPeriodMonitorEventHandler());
}
public MarketTimeSegment[] Segments { get; private set; } = Array.Empty<MarketTimeSegment>();

2
Tools/MarketInsights/Program.cs
View File

@ -22,7 +22,7 @@ namespace MarketInsights
var connector = GethConnector.GethConnector.Initialize(appState.Log);
if (connector == null) throw new Exception("Invalid Geth information");
var updater = new Updater(appState, connector.CodexContracts, cts.Token);
var updater = new Updater(appState, connector.GethNode, connector.CodexContracts, cts.Token);
var builder = WebApplication.CreateBuilder(args);

5
Tools/MarketInsights/Updater.cs
View File

@ -1,4 +1,5 @@
using CodexContractsPlugin;
using GethPlugin;
using TestNetRewarder;
namespace MarketInsights
@ -11,13 +12,13 @@ namespace MarketInsights
private readonly Tracker[] trackers;
private readonly AverageHistory averageHistory;
public Updater(AppState appState, ICodexContracts contracts, CancellationToken ct)
public Updater(AppState appState, IGethNode geth, ICodexContracts contracts, CancellationToken ct)
{
this.appState = appState;
this.ct = ct;
trackers = CreateTrackers();
averageHistory = new AverageHistory(appState, contracts, trackers.Max(t => t.NumberOfSegments));
averageHistory = new AverageHistory(appState, geth, contracts, trackers.Max(t => t.NumberOfSegments));
}
private Tracker[] CreateTrackers()

77
Tools/TestNetRewarder/EventsFormatter.cs
View File

@ -123,17 +123,18 @@ namespace TestNetRewarder
public void ProcessPeriodReports(PeriodMonitorResult reports)
{
if (reports.IsEmpty) return;
//if (reports.IsEmpty) return;
var lines = new List<string> {
$"Periods: [{reports.PeriodLow} ... {reports.PeriodHigh}]",
$"Average number of slots: {reports.AverageNumSlots.ToString("F2")}",
$"Average number of proofs required: {reports.AverageNumProofsRequired.ToString("F2")}"
};
//var lines = new List<string> {
// $"Periods: [{reports.PeriodLow} ... {reports.PeriodHigh}]",
// $"Average number of slots: {reports.AverageNumSlots.ToString("F2")}",
// $"Average number of proofs required: {reports.AverageNumProofsRequired.ToString("F2")}"
//};
AddMissedProofDetails(lines, reports.Reports);
// AddMissedProofDetails(lines, reports.Reports);
// todo!
AddBlock(0, $"{emojiMaps.ProofReport} **Proof system report**", lines.ToArray());
//AddBlock(0, $"{emojiMaps.ProofReport} **Proof system report**", lines.ToArray());
}
private string GetSlotFilledIcon(bool isRepair)
@ -148,40 +149,40 @@ namespace TestNetRewarder
return $"Slot Filled";
}
private void AddMissedProofDetails(List<string> lines, PeriodReport[] reports)
{
var reportsWithMissedProofs = reports.Where(r => r.GetMissedProofs().Any()).ToArray();
if (reportsWithMissedProofs.Length < 1)
{
lines.Add($"No proofs were missed {emojiMaps.NoProofsMissed}");
return;
}
//private void AddMissedProofDetails(List<string> lines, PeriodReport[] reports)
//{
// var reportsWithMissedProofs = reports.Where(r => r.GetMissedProofs().Any()).ToArray();
// if (reportsWithMissedProofs.Length < 1)
// {
// lines.Add($"No proofs were missed {emojiMaps.NoProofsMissed}");
// return;
// }
var totalMissed = reportsWithMissedProofs.Sum(r => r.GetMissedProofs().Length);
if (totalMissed > 10)
{
lines.Add($"[{totalMissed}] proofs were missed {emojiMaps.ManyProofsMissed}");
return;
}
// var totalMissed = reportsWithMissedProofs.Sum(r => r.GetMissedProofs().Length);
// if (totalMissed > 10)
// {
// lines.Add($"[{totalMissed}] proofs were missed {emojiMaps.ManyProofsMissed}");
// return;
// }
foreach (var report in reportsWithMissedProofs)
{
DescribeMissedProof(lines, report);
}
}
// foreach (var report in reportsWithMissedProofs)
// {
// DescribeMissedProof(lines, report);
// }
//}
private void DescribeMissedProof(List<string> lines, PeriodReport report)
{
foreach (var missedProof in report.GetMissedProofs())
{
DescribeMissedProof(lines, missedProof);
}
}
//private void DescribeMissedProof(List<string> lines, PeriodReport report)
//{
// foreach (var missedProof in report.GetMissedProofs())
// {
// DescribeMissedProof(lines, missedProof);
// }
//}
private void DescribeMissedProof(List<string> lines, PeriodProof missedProof)
{
lines.Add($"[{missedProof.FormatHost()}] missed proof for {FormatRequestId(missedProof.Request)} (slotIndex: {missedProof.SlotIndex})");
}
//private void DescribeMissedProof(List<string> lines, PeriodRequiredProof missedProof)
//{
// lines.Add($"[{missedProof.FormatHost()}] missed proof for {FormatRequestId(missedProof.Request)} (slotIndex: {missedProof.SlotIndex})");
//}
private void AddRequestBlock(RequestEvent requestEvent, string icon, string eventName, params string[] content)
{

7
Tools/TestNetRewarder/Processor.cs
View File

@ -1,5 +1,6 @@
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using GethPlugin;
using Logging;
using Utils;
@ -15,7 +16,7 @@ namespace TestNetRewarder
private readonly ILog log;
private DateTime lastPeriodUpdateUtc;
public Processor(Configuration config, BotClient client, ICodexContracts contracts, ILog log)
public Processor(Configuration config, BotClient client, IGethNode geth, ICodexContracts contracts, ILog log)
{
this.config = config;
this.client = client;
@ -27,8 +28,8 @@ namespace TestNetRewarder
builder = new RequestBuilder();
eventsFormatter = new EventsFormatter(config, contracts.Deployment.Config);
chainState = new ChainState(log, contracts, eventsFormatter, config.HistoryStartUtc,
doProofPeriodMonitoring: config.ShowProofPeriodReports > 0);
chainState = new ChainState(log, geth, contracts, eventsFormatter, config.HistoryStartUtc,
doProofPeriodMonitoring: config.ShowProofPeriodReports > 0, new DoNothingPeriodMonitorEventHandler());
}
public async Task Initialize()

2
Tools/TestNetRewarder/Program.cs
View File

@ -31,7 +31,7 @@ namespace TestNetRewarder
if (connector == null) throw new Exception("Invalid Geth information");
BotClient = new BotClient(Config, Log);
processor = new Processor(Config, BotClient, connector.CodexContracts, Log);
processor = new Processor(Config, BotClient, connector.GethNode, connector.CodexContracts, Log);
EnsurePath(Config.DataPath);
EnsurePath(Config.LogPath);

7
Tools/TraceContract/ChainTracer.cs
View File

@ -1,6 +1,7 @@
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using CodexContractsPlugin.Marketplace;
using GethPlugin;
using Logging;
using Nethereum.Hex.HexConvertors.Extensions;
using Nethereum.Model;
@ -11,13 +12,15 @@ namespace TraceContract
public class ChainTracer
{
private readonly ILog log;
private readonly IGethNode geth;
private readonly ICodexContracts contracts;
private readonly Input input;
private readonly Output output;
public ChainTracer(ILog log, ICodexContracts contracts, Input input, Output output)
public ChainTracer(ILog log, IGethNode geth, ICodexContracts contracts, Input input, Output output)
{
this.log = log;
this.geth = geth;
this.contracts = contracts;
this.input = input;
this.output = output;
@ -60,7 +63,7 @@ namespace TraceContract
var utc = request.Block.Utc.AddMinutes(-1.0);
var tracker = new ChainRequestTracker(output, input.PurchaseId);
var ignoreLog = new NullLog();
var chainState = new ChainState(ignoreLog, contracts, tracker, utc, false);
var chainState = new ChainState(ignoreLog, geth, contracts, tracker, utc, false, new DoNothingPeriodMonitorEventHandler());
var atNow = false;
while (!tracker.IsFinished && !atNow)

12
Tools/TraceContract/Program.cs
View File

@ -47,9 +47,10 @@ namespace TraceContract
var entryPoint = new EntryPoint(log, new KubernetesWorkflow.Configuration(null, TimeSpan.FromMinutes(1.0), TimeSpan.FromSeconds(10.0), "_Unused!_"), Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData));
entryPoint.Announce();
var ci = entryPoint.CreateInterface();
var contracts = ConnectCodexContracts(ci);
var geth = ConnectGethNode();
var contracts = ConnectCodexContracts(ci, geth);
var chainTracer = new ChainTracer(log, contracts, input, output);
var chainTracer = new ChainTracer(log, geth, contracts, input, output);
var requestTimeRange = chainTracer.TraceChainTimeline();
Log("Downloading storage nodes logs for the request timerange...");
@ -61,12 +62,15 @@ namespace TraceContract
Log("Done");
}
private ICodexContracts ConnectCodexContracts(CoreInterface ci)
private IGethNode ConnectGethNode()
{
var account = EthAccountGenerator.GenerateNew();
var blockCache = new BlockCache();
var geth = new CustomGethNode(log, blockCache, config.RpcEndpoint, config.GethPort, account.PrivateKey);
return new CustomGethNode(log, blockCache, config.RpcEndpoint, config.GethPort, account.PrivateKey);
}
private ICodexContracts ConnectCodexContracts(CoreInterface ci, IGethNode geth)
{
var deployment = new CodexContractsDeployment(
config: new MarketplaceConfig(),
marketplaceAddress: config.MarketplaceAddress,