From 4e093f12d7bcd3ba4043b1e5c901be59c883ebe3 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 15 Aug 2025 12:02:33 +0200 Subject: [PATCH 01/11] wip --- .../ChainMonitor/ChainState.cs | 5 +- .../ChainMonitor/PeriodMonitor.cs | 181 ++++++++---------- .../ChainMonitor/PeriodReport.cs | 52 +++++ .../ChainMonitor/PeriodRequiredProof.cs | 24 +++ .../ChainMonitor/ProofPeriod.cs | 23 +++ .../CodexContractsAccess.cs | 95 ++------- ProjectPlugins/GethPlugin/GethNode.cs | 30 ++- Tests/CodexReleaseTests/Utils/ChainMonitor.cs | 20 +- .../Utils/MarketplaceAutoBootstrapDistTest.cs | 6 +- Tests/DistTestCore/NameUtils.cs | 3 +- Tools/MarketInsights/AverageHistory.cs | 5 +- Tools/MarketInsights/Program.cs | 2 +- Tools/MarketInsights/Updater.cs | 5 +- Tools/TestNetRewarder/EventsFormatter.cs | 77 ++++---- Tools/TestNetRewarder/Processor.cs | 5 +- Tools/TestNetRewarder/Program.cs | 2 +- Tools/TraceContract/ChainTracer.cs | 7 +- Tools/TraceContract/Program.cs | 12 +- 18 files changed, 294 insertions(+), 260 deletions(-) create mode 100644 ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs create mode 100644 ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs create mode 100644 ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 04d8e0e2..16c9e100 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -1,5 +1,6 @@ using BlockchainUtils; using CodexContractsPlugin.Marketplace; +using GethPlugin; using Logging; using Nethereum.Contracts; using Nethereum.Hex.HexConvertors.Extensions; @@ -42,14 +43,14 @@ namespace CodexContractsPlugin.ChainMonitor private readonly IChainStateChangeHandler handler; private readonly bool doProofPeriodMonitoring; - public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring) + public ChainState(ILog log, IGethNode geth, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring) { this.log = new LogPrefixer(log, "(ChainState) "); this.contracts = contracts; handler = changeHandler; this.doProofPeriodMonitoring = doProofPeriodMonitoring; TotalSpan = new TimeRange(startUtc, startUtc); - PeriodMonitor = new PeriodMonitor(log, contracts); + PeriodMonitor = new PeriodMonitor(log, contracts, geth); } public TimeRange TotalSpan { get; private set; } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs index d1e469a8..61ba5ecd 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs @@ -1,7 +1,12 @@ -using Logging; +using CodexContractsPlugin.Marketplace; +using GethPlugin; +using Logging; +using Nethereum.Contracts; using Nethereum.Hex.HexConvertors.Extensions; using Nethereum.Model; -using Utils; +using Nethereum.RPC.Eth.DTOs; +using Newtonsoft.Json; +using System.Reflection; namespace CodexContractsPlugin.ChainMonitor { @@ -9,27 +14,29 @@ namespace CodexContractsPlugin.ChainMonitor { private readonly ILog log; private readonly ICodexContracts contracts; + private readonly IGethNode geth; private readonly List reports = new List(); - private ulong? currentPeriod = null; + private CurrentPeriod? currentPeriod = null; - public PeriodMonitor(ILog log, ICodexContracts contracts) + public PeriodMonitor(ILog log, ICodexContracts contracts, IGethNode geth) { this.log = log; this.contracts = contracts; + this.geth = geth; } public void Update(DateTime eventUtc, IChainStateRequest[] requests) { - var period = contracts.GetPeriodNumber(eventUtc); - if (!currentPeriod.HasValue) + var periodNumber = contracts.GetPeriodNumber(eventUtc); + if (currentPeriod == null) { - currentPeriod = period; + currentPeriod = CreateCurrentPeriod(periodNumber, requests); return; } - if (period == currentPeriod.Value) return; + if (periodNumber == currentPeriod.PeriodNumber) return; - CreateReportForPeriod(currentPeriod.Value, requests); - currentPeriod = period; + CreateReportForPeriod(currentPeriod, requests); + currentPeriod = CreateCurrentPeriod(periodNumber, requests); } public PeriodMonitorResult GetAndClearReports() @@ -39,26 +46,76 @@ namespace CodexContractsPlugin.ChainMonitor return new PeriodMonitorResult(result); } - private void CreateReportForPeriod(ulong periodNumber, IChainStateRequest[] requests) + private CurrentPeriod CreateCurrentPeriod(ulong periodNumber, IChainStateRequest[] requests) + { + var result = new CurrentPeriod(periodNumber); + ForEachActiveSlot(requests, (request, slotIndex) => + { + if (contracts.IsProofRequired(request.RequestId, slotIndex)) + { + var idx = Convert.ToInt32(slotIndex); + var host = request.Hosts.GetHost(idx); + if (host != null) + { + result.RequiredProofs.Add(new PeriodRequiredProof(host, request, idx)); + } + } + }); + + return result; + } + + private void CreateReportForPeriod(CurrentPeriod currentPeriod, IChainStateRequest[] requests) + { + // Fetch function calls during period. Format report. + var timeRange = contracts.GetPeriodTimeRange(currentPeriod.PeriodNumber); + var blockRange = geth.ConvertTimeRangeToBlockRange(timeRange); + + // MarkProofAsMissingFunction + // SubmitProofFunction + // FreeSlot1Function + // FreeSlotFunction + + var callReports = new List(); + geth.IterateTransactions(blockRange, (t, blkI, blkUtc) => + { + CreateFunctionCallReport(callReports, t, blkUtc); + CreateFunctionCallReport(callReports, t, blkUtc); + CreateFunctionCallReport(callReports, t, blkUtc); + CreateFunctionCallReport(callReports, t, blkUtc); + }); + + var report = new PeriodReport( + new ProofPeriod( + currentPeriod.PeriodNumber, + timeRange.From, + timeRange.To), + currentPeriod.RequiredProofs.ToArray(), + callReports.ToArray()); + + report.Log(log); + reports.Add(report); + } + + private void CreateFunctionCallReport(List reports, Transaction t, DateTime blockUtc) where TFunc : FunctionMessage, new() + { + if (t.IsTransactionForFunctionMessage()) + { + var func = t.DecodeTransactionToFunctionMessage(); + + reports.Add(new FunctionCallReport(blockUtc, typeof(TFunc).Name, JsonConvert.SerializeObject(func))); + } + } + + private void ForEachActiveSlot(IChainStateRequest[] requests, Action action) { - ulong total = 0; - var periodProofs = new List(); foreach (var request in requests) { for (ulong slotIndex = 0; slotIndex < request.Request.Ask.Slots; slotIndex++) { - var state = contracts.GetProofState(request.RequestId, slotIndex, periodNumber); - - total++; - var idx = Convert.ToInt32(slotIndex); - var host = request.Hosts.GetHost(idx); - var proof = new PeriodProof(host, request, idx, state); - periodProofs.Add(proof); + action(request, slotIndex); } } - var report = new PeriodReport(periodNumber, total, periodProofs.ToArray()); - report.Log(log); - reports.Add(report); } } @@ -67,91 +124,19 @@ namespace CodexContractsPlugin.ChainMonitor public PeriodMonitorResult(PeriodReport[] reports) { Reports = reports; - - CalcStats(); } public PeriodReport[] Reports { get; } - - public bool IsEmpty { get; private set; } - public ulong PeriodLow { get; private set; } - public ulong PeriodHigh { get; private set; } - public float AverageNumSlots { get; private set; } - public float AverageNumProofsRequired { get; private set; } - - private void CalcStats() - { - IsEmpty = Reports.All(r => r.PeriodProofs.Length == 0); - if (Reports.Length == 0) return; - - PeriodLow = Reports.Min(r => r.PeriodNumber); - PeriodHigh = Reports.Max(r => r.PeriodNumber); - AverageNumSlots = Reports.Average(r => Convert.ToSingle(r.TotalNumSlots)); - AverageNumProofsRequired = Reports.Average(r => Convert.ToSingle(r.PeriodProofs.Count(p => p.State != ProofState.NotRequired))); - } } - public class PeriodReport + public class CurrentPeriod { - public PeriodReport(ulong periodNumber, ulong totalNumSlots, PeriodProof[] periodProofs) + public CurrentPeriod(ulong periodNumber) { PeriodNumber = periodNumber; - TotalNumSlots = totalNumSlots; - PeriodProofs = periodProofs; } public ulong PeriodNumber { get; } - public ulong TotalNumSlots { get; } - public PeriodProof[] PeriodProofs { get; } - - public PeriodProof[] GetMissedProofs() - { - return PeriodProofs.Where(p => p.State == ProofState.MissedAndMarked || p.State == ProofState.MissedNotMarked).ToArray(); - } - - public void Log(ILog log) - { - log.Log($"Period report: {PeriodNumber}"); - log.Log($" - Slots: {TotalNumSlots}"); - foreach (var p in PeriodProofs) - { - log.Log($" - {p.Describe()}"); - } - } - - private void Log(ILog log, PeriodProof[] proofs) - { - if (proofs.Length == 0) return; - foreach (var p in proofs) - { - } - } - } - - public class PeriodProof - { - public PeriodProof(EthAddress? host, IChainStateRequest request, int slotIndex, ProofState state) - { - Host = host; - Request = request; - SlotIndex = slotIndex; - State = state; - } - - public EthAddress? Host { get; } - public IChainStateRequest Request { get; } - public int SlotIndex { get; } - public ProofState State { get; } - - public string FormatHost() - { - if (Host == null) return "Unknown host"; - return Host.Address; - } - - public string Describe() - { - return $"{FormatHost()} - {Request.RequestId.ToHex()} slotIndex:{SlotIndex} => {State}"; - } + public List RequiredProofs { get; } = new List(); } } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs new file mode 100644 index 00000000..0e2f6dc3 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs @@ -0,0 +1,52 @@ +using Logging; +using Utils; + +namespace CodexContractsPlugin.ChainMonitor +{ + public class PeriodReport + { + public PeriodReport(ProofPeriod period, PeriodRequiredProof[] required, FunctionCallReport[] functionCalls) + { + Period = period; + Required = required; + FunctionCalls = functionCalls; + } + + public ProofPeriod Period { get; } + public PeriodRequiredProof[] Required { get; } + public FunctionCallReport[] FunctionCalls { get; } + + public void Log(ILog log) + { + log.Log($"Period report: {Period}"); + foreach (var r in Required) + { + log.Log($" Required: {r.Describe()}"); + } + log.Log(" - Calls:"); + foreach (var f in FunctionCalls) + { + log.Log($" - {f.Describe()}"); + } + } + } + + public class FunctionCallReport + { + public FunctionCallReport(DateTime utc, string name, string payload) + { + Utc = utc; + Name = name; + Payload = payload; + } + + public DateTime Utc { get; } + public string Name { get; } + public string Payload { get; } + + public string Describe() + { + return $"[{Time.FormatTimestamp(Utc)}] {Name} = \"{Payload}\""; + } + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs new file mode 100644 index 00000000..f8df4e23 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs @@ -0,0 +1,24 @@ +using Nethereum.Hex.HexConvertors.Extensions; +using Utils; + +namespace CodexContractsPlugin.ChainMonitor +{ + public class PeriodRequiredProof + { + public PeriodRequiredProof(EthAddress host, IChainStateRequest request, int slotIndex) + { + Host = host; + Request = request; + SlotIndex = slotIndex; + } + + public EthAddress Host { get; } + public IChainStateRequest Request { get; } + public int SlotIndex { get; } + + public string Describe() + { + return $"{Request.RequestId.ToHex()} slotIndex:{SlotIndex} by {Host}"; + } + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs new file mode 100644 index 00000000..ca428170 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs @@ -0,0 +1,23 @@ +using Utils; + +namespace CodexContractsPlugin.ChainMonitor +{ + public class ProofPeriod + { + public ProofPeriod(ulong periodNumber, DateTime startUtc, DateTime endUtc) + { + PeriodNumber = periodNumber; + StartUtc = startUtc; + EndUtc = endUtc; + } + + public ulong PeriodNumber { get; } + public DateTime StartUtc { get; } + public DateTime EndUtc { get; } + + public override string ToString() + { + return $"{PeriodNumber} - {Time.FormatTimestamp(StartUtc)} -> {Time.FormatTimestamp(EndUtc)}"; + } + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs index 17a8c36c..132451e7 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs @@ -29,21 +29,14 @@ namespace CodexContractsPlugin RequestState GetRequestState(byte[] requestId); Request GetRequest(byte[] requestId); ulong GetPeriodNumber(DateTime utc); + TimeRange GetPeriodTimeRange(ulong periodNumber); void WaitUntilNextPeriod(); - ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong period); + bool IsProofRequired(byte[] requestId, decimal slotIndex); byte[] GetSlotId(byte[] requestId, decimal slotIndex); ICodexContracts WithDifferentGeth(IGethNode node); } - public enum ProofState - { - NotRequired, - NotMissed, - MissedNotMarked, - MissedAndMarked, - } - [JsonConverter(typeof(StringEnumConverter))] public enum RequestState { @@ -155,6 +148,16 @@ namespace CodexContractsPlugin return Convert.ToUInt64(result); } + public TimeRange GetPeriodTimeRange(ulong periodNumber) + { + var periodSeconds = (ulong)Deployment.Config.Proofs.Period; + var startUtco = Convert.ToInt64(periodSeconds * periodNumber); + var endUtco = Convert.ToInt64(periodSeconds * (periodNumber + 1)); + var start = DateTimeOffset.FromUnixTimeSeconds(startUtco).UtcDateTime; + var end = DateTimeOffset.FromUnixTimeSeconds(endUtco).UtcDateTime; + return new TimeRange(start, end); + } + public void WaitUntilNextPeriod() { var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); @@ -163,14 +166,10 @@ namespace CodexContractsPlugin Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1)); } - public ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong period) + public bool IsProofRequired(byte[] requestId, decimal slotIndex) { var slotId = GetSlotId(requestId, slotIndex); - - var required = IsProofRequired(slotId); - if (!required) return ProofState.NotRequired; - - return IsProofMissing(slotId, period); + return IsProofRequired(slotId); } public ICodexContracts WithDifferentGeth(IGethNode node) @@ -199,72 +198,6 @@ namespace CodexContractsPlugin return result.ReturnValue1; } - private ProofState IsProofMissing(byte[] slotId, ulong period) - { - // In case of a missed proof, one of two things can be true: - // 1 - The proof was missed but no validator marked it as missing: - // We can see this by calling "canMarkProofAsMissing" and it returns true/doesn't throw. - // 2 - The proof was missed and it was marked as missing by a validator: - // We can see this by a successful call to "MarkProofAsMissing" on-chain. - - if (CallCanMarkProofAsMissing(slotId, period)) - { - return ProofState.MissedNotMarked; - } - if (WasMarkProofAsMissingCalled(slotId, period)) - { - return ProofState.MissedAndMarked; - } - - return ProofState.NotMissed; - } - - private bool CallCanMarkProofAsMissing(byte[] slotId, ulong period) - { - try - { - var func = new CanMarkProofAsMissingFunction - { - SlotId = slotId, - Period = period - }; - - gethNode.Call(Deployment.MarketplaceAddress, func); - - return true; - } - catch (AggregateException exc) - { - if (exc.InnerExceptions.Count == 1) - { - if (exc.InnerExceptions[0].GetType() == typeof(SmartContractCustomErrorRevertException)) - { - return false; - } - } - throw; - } - } - - private bool WasMarkProofAsMissingCalled(byte[] slotId, ulong period) - { - var now = DateTime.UtcNow; - var currentPeriod = new TimeRange(now - Deployment.Config.PeriodDuration, now); - var interval = gethNode.ConvertTimeRangeToBlockRange(currentPeriod); - var slot = slotId.ToHex().ToLowerInvariant(); - - var found = false; - gethNode.IterateFunctionCalls(interval, (b, fn) => - { - if (fn.Period == period && fn.SlotId.ToHex().ToLowerInvariant() == slot) - { - found = true; - } - }); - - return found; - } - private ContractInteractions StartInteraction() { return new ContractInteractions(log, gethNode); diff --git a/ProjectPlugins/GethPlugin/GethNode.cs b/ProjectPlugins/GethPlugin/GethNode.cs index 15fa0d4e..04653efe 100644 --- a/ProjectPlugins/GethPlugin/GethNode.cs +++ b/ProjectPlugins/GethPlugin/GethNode.cs @@ -33,6 +33,7 @@ namespace GethPlugin List> GetEvents(string address, TimeRange timeRange) where TEvent : IEventDTO, new(); BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange); BlockTimeEntry GetBlockForNumber(ulong number); + void IterateTransactions(BlockInterval blockRange, Action action); void IterateFunctionCalls(BlockInterval blockInterval, Action onCall) where TFunc : FunctionMessage, new(); IGethNode WithDifferentAccount(EthAccount account); } @@ -223,28 +224,37 @@ namespace GethPlugin return StartInteraction().GetBlockWithTransactions(number); } - public void IterateFunctionCalls(BlockInterval blockRange, Action onCall) where TFunc : FunctionMessage, new() + public void IterateTransactions(BlockInterval blockRange, Action action) { var i = StartInteraction(); for (var blkI = blockRange.From; blkI <= blockRange.To; blkI++) { var blk = i.GetBlockWithTransactions(blkI); + var blkUtc = DateTimeOffset.FromUnixTimeSeconds(blk.Timestamp.ToLong()).UtcDateTime; foreach (var t in blk.Transactions) { - if (t.IsTransactionForFunctionMessage()) - { - var func = t.DecodeTransactionToFunctionMessage(); - if (func != null) - { - var b = GetBlockForNumber(blkI); - onCall(b, func); - } - } + action(t, blkI, blkUtc); } } } + public void IterateFunctionCalls(BlockInterval blockRange, Action onCall) where TFunc : FunctionMessage, new() + { + IterateTransactions(blockRange, (t, blkI, blkUtc) => + { + if (t.IsTransactionForFunctionMessage()) + { + var func = t.DecodeTransactionToFunctionMessage(); + if (func != null) + { + var b = GetBlockForNumber(blkI); + onCall(b, func); + } + } + }); + } + protected abstract NethereumInteraction StartInteraction(); } } diff --git a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs index 8723246b..f34081e0 100644 --- a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs +++ b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs @@ -1,5 +1,6 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; +using GethPlugin; using Logging; using Utils; @@ -8,20 +9,22 @@ namespace CodexReleaseTests.Utils public class ChainMonitor { private readonly ILog log; + private readonly IGethNode gethNode; private readonly ICodexContracts contracts; private readonly DateTime startUtc; private readonly TimeSpan updateInterval; private CancellationTokenSource cts = new CancellationTokenSource(); private Task worker = Task.CompletedTask; - public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc) - : this(log, contracts, startUtc, TimeSpan.FromSeconds(3.0)) + public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc) + : this(log, gethNode, contracts, startUtc, TimeSpan.FromSeconds(3.0)) { } - public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc, TimeSpan updateInterval) + public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc, TimeSpan updateInterval) { this.log = log; + this.gethNode = gethNode; this.contracts = contracts; this.startUtc = startUtc; this.updateInterval = updateInterval; @@ -42,7 +45,7 @@ namespace CodexReleaseTests.Utils private void Worker(Action onFailure) { - var state = new ChainState(log, contracts, new DoNothingThrowingChainEventHandler(), startUtc, doProofPeriodMonitoring: true); + var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, doProofPeriodMonitoring: true); Thread.Sleep(updateInterval); log.Log($"Chain monitoring started. Update interval: {Time.FormatDuration(updateInterval)}"); @@ -66,15 +69,6 @@ namespace CodexReleaseTests.Utils private void UpdateChainState(ChainState state) { state.Update(); - - var reports = state.PeriodMonitor.GetAndClearReports(); - if (reports.IsEmpty) return; - - var slots = reports.Reports.Sum(r => Convert.ToInt32(r.TotalNumSlots)); - var required = reports.Reports.Sum(r => Convert.ToInt32(r.PeriodProofs.Count(p => p.State != ProofState.NotRequired))); - var missed = reports.Reports.Sum(r => r.GetMissedProofs().Length); - - log.Log($"Proof report: Slots={slots} Required={required} Missed={missed}"); } } } diff --git a/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs index 42bc03fa..f3dbdaa3 100644 --- a/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs @@ -22,7 +22,7 @@ namespace CodexReleaseTests.Utils { var geth = StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version); - var monitor = SetupChainMonitor(GetTestLog(), contracts, GetTestRunTimeRange().From); + var monitor = SetupChainMonitor(GetTestLog(), geth, contracts, GetTestRunTimeRange().From); handle = new MarketplaceHandle(geth, contracts, monitor); } @@ -169,11 +169,11 @@ namespace CodexReleaseTests.Utils }); } - private ChainMonitor? SetupChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc) + private ChainMonitor? SetupChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc) { if (!MonitorChainState) return null; - var result = new ChainMonitor(log, contracts, startUtc); + var result = new ChainMonitor(log, gethNode, contracts, startUtc); result.Start(() => { Assert.Fail("Failure in chain monitor."); diff --git a/Tests/DistTestCore/NameUtils.cs b/Tests/DistTestCore/NameUtils.cs index 10a4c498..76ea210e 100644 --- a/Tests/DistTestCore/NameUtils.cs +++ b/Tests/DistTestCore/NameUtils.cs @@ -91,7 +91,8 @@ namespace DistTestCore return Path.Join( config.LogRoot, $"{start.Year}-{Pad(start.Month)}", - Pad(start.Day)); + Pad(start.Day), + $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}"); } private static string Pad(int n) diff --git a/Tools/MarketInsights/AverageHistory.cs b/Tools/MarketInsights/AverageHistory.cs index 4baebd3b..412846d0 100644 --- a/Tools/MarketInsights/AverageHistory.cs +++ b/Tools/MarketInsights/AverageHistory.cs @@ -1,5 +1,6 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; +using GethPlugin; using TestNetRewarder; using Utils; @@ -13,11 +14,11 @@ namespace MarketInsights private readonly int maxContributions; private readonly ChainState chainState; - public AverageHistory(AppState appState, ICodexContracts contracts, int maxContributions) + public AverageHistory(AppState appState, IGethNode geth, ICodexContracts contracts, int maxContributions) { this.appState = appState; this.maxContributions = maxContributions; - chainState = new ChainState(appState.Log, contracts, mux, appState.Config.HistoryStartUtc, + chainState = new ChainState(appState.Log, geth, contracts, mux, appState.Config.HistoryStartUtc, doProofPeriodMonitoring: false); } diff --git a/Tools/MarketInsights/Program.cs b/Tools/MarketInsights/Program.cs index a616386f..6caf4433 100644 --- a/Tools/MarketInsights/Program.cs +++ b/Tools/MarketInsights/Program.cs @@ -22,7 +22,7 @@ namespace MarketInsights var connector = GethConnector.GethConnector.Initialize(appState.Log); if (connector == null) throw new Exception("Invalid Geth information"); - var updater = new Updater(appState, connector.CodexContracts, cts.Token); + var updater = new Updater(appState, connector.GethNode, connector.CodexContracts, cts.Token); var builder = WebApplication.CreateBuilder(args); diff --git a/Tools/MarketInsights/Updater.cs b/Tools/MarketInsights/Updater.cs index 87743f66..436a5ac2 100644 --- a/Tools/MarketInsights/Updater.cs +++ b/Tools/MarketInsights/Updater.cs @@ -1,4 +1,5 @@ using CodexContractsPlugin; +using GethPlugin; using TestNetRewarder; namespace MarketInsights @@ -11,13 +12,13 @@ namespace MarketInsights private readonly Tracker[] trackers; private readonly AverageHistory averageHistory; - public Updater(AppState appState, ICodexContracts contracts, CancellationToken ct) + public Updater(AppState appState, IGethNode geth, ICodexContracts contracts, CancellationToken ct) { this.appState = appState; this.ct = ct; trackers = CreateTrackers(); - averageHistory = new AverageHistory(appState, contracts, trackers.Max(t => t.NumberOfSegments)); + averageHistory = new AverageHistory(appState, geth, contracts, trackers.Max(t => t.NumberOfSegments)); } private Tracker[] CreateTrackers() diff --git a/Tools/TestNetRewarder/EventsFormatter.cs b/Tools/TestNetRewarder/EventsFormatter.cs index 3b1650cb..a9c3a0a1 100644 --- a/Tools/TestNetRewarder/EventsFormatter.cs +++ b/Tools/TestNetRewarder/EventsFormatter.cs @@ -123,17 +123,18 @@ namespace TestNetRewarder public void ProcessPeriodReports(PeriodMonitorResult reports) { - if (reports.IsEmpty) return; + //if (reports.IsEmpty) return; - var lines = new List { - $"Periods: [{reports.PeriodLow} ... {reports.PeriodHigh}]", - $"Average number of slots: {reports.AverageNumSlots.ToString("F2")}", - $"Average number of proofs required: {reports.AverageNumProofsRequired.ToString("F2")}" - }; + //var lines = new List { + // $"Periods: [{reports.PeriodLow} ... {reports.PeriodHigh}]", + // $"Average number of slots: {reports.AverageNumSlots.ToString("F2")}", + // $"Average number of proofs required: {reports.AverageNumProofsRequired.ToString("F2")}" + //}; - AddMissedProofDetails(lines, reports.Reports); + // AddMissedProofDetails(lines, reports.Reports); + // todo! - AddBlock(0, $"{emojiMaps.ProofReport} **Proof system report**", lines.ToArray()); + //AddBlock(0, $"{emojiMaps.ProofReport} **Proof system report**", lines.ToArray()); } private string GetSlotFilledIcon(bool isRepair) @@ -148,40 +149,40 @@ namespace TestNetRewarder return $"Slot Filled"; } - private void AddMissedProofDetails(List lines, PeriodReport[] reports) - { - var reportsWithMissedProofs = reports.Where(r => r.GetMissedProofs().Any()).ToArray(); - if (reportsWithMissedProofs.Length < 1) - { - lines.Add($"No proofs were missed {emojiMaps.NoProofsMissed}"); - return; - } + //private void AddMissedProofDetails(List lines, PeriodReport[] reports) + //{ + // var reportsWithMissedProofs = reports.Where(r => r.GetMissedProofs().Any()).ToArray(); + // if (reportsWithMissedProofs.Length < 1) + // { + // lines.Add($"No proofs were missed {emojiMaps.NoProofsMissed}"); + // return; + // } - var totalMissed = reportsWithMissedProofs.Sum(r => r.GetMissedProofs().Length); - if (totalMissed > 10) - { - lines.Add($"[{totalMissed}] proofs were missed {emojiMaps.ManyProofsMissed}"); - return; - } + // var totalMissed = reportsWithMissedProofs.Sum(r => r.GetMissedProofs().Length); + // if (totalMissed > 10) + // { + // lines.Add($"[{totalMissed}] proofs were missed {emojiMaps.ManyProofsMissed}"); + // return; + // } - foreach (var report in reportsWithMissedProofs) - { - DescribeMissedProof(lines, report); - } - } + // foreach (var report in reportsWithMissedProofs) + // { + // DescribeMissedProof(lines, report); + // } + //} - private void DescribeMissedProof(List lines, PeriodReport report) - { - foreach (var missedProof in report.GetMissedProofs()) - { - DescribeMissedProof(lines, missedProof); - } - } + //private void DescribeMissedProof(List lines, PeriodReport report) + //{ + // foreach (var missedProof in report.GetMissedProofs()) + // { + // DescribeMissedProof(lines, missedProof); + // } + //} - private void DescribeMissedProof(List lines, PeriodProof missedProof) - { - lines.Add($"[{missedProof.FormatHost()}] missed proof for {FormatRequestId(missedProof.Request)} (slotIndex: {missedProof.SlotIndex})"); - } + //private void DescribeMissedProof(List lines, PeriodRequiredProof missedProof) + //{ + // lines.Add($"[{missedProof.FormatHost()}] missed proof for {FormatRequestId(missedProof.Request)} (slotIndex: {missedProof.SlotIndex})"); + //} private void AddRequestBlock(RequestEvent requestEvent, string icon, string eventName, params string[] content) { diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 036f8795..0a459bc6 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -1,5 +1,6 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; +using GethPlugin; using Logging; using Utils; @@ -15,7 +16,7 @@ namespace TestNetRewarder private readonly ILog log; private DateTime lastPeriodUpdateUtc; - public Processor(Configuration config, BotClient client, ICodexContracts contracts, ILog log) + public Processor(Configuration config, BotClient client, IGethNode geth, ICodexContracts contracts, ILog log) { this.config = config; this.client = client; @@ -27,7 +28,7 @@ namespace TestNetRewarder builder = new RequestBuilder(); eventsFormatter = new EventsFormatter(config, contracts.Deployment.Config); - chainState = new ChainState(log, contracts, eventsFormatter, config.HistoryStartUtc, + chainState = new ChainState(log, geth, contracts, eventsFormatter, config.HistoryStartUtc, doProofPeriodMonitoring: config.ShowProofPeriodReports > 0); } diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index e3ed24e7..0cac9bd7 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -31,7 +31,7 @@ namespace TestNetRewarder if (connector == null) throw new Exception("Invalid Geth information"); BotClient = new BotClient(Config, Log); - processor = new Processor(Config, BotClient, connector.CodexContracts, Log); + processor = new Processor(Config, BotClient, connector.GethNode, connector.CodexContracts, Log); EnsurePath(Config.DataPath); EnsurePath(Config.LogPath); diff --git a/Tools/TraceContract/ChainTracer.cs b/Tools/TraceContract/ChainTracer.cs index a4227ab4..40f3195d 100644 --- a/Tools/TraceContract/ChainTracer.cs +++ b/Tools/TraceContract/ChainTracer.cs @@ -1,6 +1,7 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; using CodexContractsPlugin.Marketplace; +using GethPlugin; using Logging; using Nethereum.Hex.HexConvertors.Extensions; using Nethereum.Model; @@ -11,13 +12,15 @@ namespace TraceContract public class ChainTracer { private readonly ILog log; + private readonly IGethNode geth; private readonly ICodexContracts contracts; private readonly Input input; private readonly Output output; - public ChainTracer(ILog log, ICodexContracts contracts, Input input, Output output) + public ChainTracer(ILog log, IGethNode geth, ICodexContracts contracts, Input input, Output output) { this.log = log; + this.geth = geth; this.contracts = contracts; this.input = input; this.output = output; @@ -60,7 +63,7 @@ namespace TraceContract var utc = request.Block.Utc.AddMinutes(-1.0); var tracker = new ChainRequestTracker(output, input.PurchaseId); var ignoreLog = new NullLog(); - var chainState = new ChainState(ignoreLog, contracts, tracker, utc, false); + var chainState = new ChainState(ignoreLog, geth, contracts, tracker, utc, false); var atNow = false; while (!tracker.IsFinished && !atNow) diff --git a/Tools/TraceContract/Program.cs b/Tools/TraceContract/Program.cs index 50ade56d..5e24af4f 100644 --- a/Tools/TraceContract/Program.cs +++ b/Tools/TraceContract/Program.cs @@ -47,9 +47,10 @@ namespace TraceContract var entryPoint = new EntryPoint(log, new KubernetesWorkflow.Configuration(null, TimeSpan.FromMinutes(1.0), TimeSpan.FromSeconds(10.0), "_Unused!_"), Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData)); entryPoint.Announce(); var ci = entryPoint.CreateInterface(); - var contracts = ConnectCodexContracts(ci); + var geth = ConnectGethNode(); + var contracts = ConnectCodexContracts(ci, geth); - var chainTracer = new ChainTracer(log, contracts, input, output); + var chainTracer = new ChainTracer(log, geth, contracts, input, output); var requestTimeRange = chainTracer.TraceChainTimeline(); Log("Downloading storage nodes logs for the request timerange..."); @@ -61,12 +62,15 @@ namespace TraceContract Log("Done"); } - private ICodexContracts ConnectCodexContracts(CoreInterface ci) + private IGethNode ConnectGethNode() { var account = EthAccountGenerator.GenerateNew(); var blockCache = new BlockCache(); - var geth = new CustomGethNode(log, blockCache, config.RpcEndpoint, config.GethPort, account.PrivateKey); + return new CustomGethNode(log, blockCache, config.RpcEndpoint, config.GethPort, account.PrivateKey); + } + private ICodexContracts ConnectCodexContracts(CoreInterface ci, IGethNode geth) + { var deployment = new CodexContractsDeployment( config: new MarketplaceConfig(), marketplaceAddress: config.MarketplaceAddress, From 35816141a3685b1d9f26f1d5fc220fb958025c5d Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 19 Aug 2025 10:47:46 +0200 Subject: [PATCH 02/11] Trying to prove disconnects slow down clients --- .../ChainMonitor/PeriodMonitor.cs | 5 -- ProjectPlugins/GethPlugin/GethNode.cs | 1 + .../DataTests/TheseusTest.cs | 87 +++++++++++++++++++ 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs index 61ba5ecd..4991db7c 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs @@ -71,11 +71,6 @@ namespace CodexContractsPlugin.ChainMonitor var timeRange = contracts.GetPeriodTimeRange(currentPeriod.PeriodNumber); var blockRange = geth.ConvertTimeRangeToBlockRange(timeRange); - // MarkProofAsMissingFunction - // SubmitProofFunction - // FreeSlot1Function - // FreeSlotFunction - var callReports = new List(); geth.IterateTransactions(blockRange, (t, blkI, blkUtc) => { diff --git a/ProjectPlugins/GethPlugin/GethNode.cs b/ProjectPlugins/GethPlugin/GethNode.cs index 04653efe..b8c1e125 100644 --- a/ProjectPlugins/GethPlugin/GethNode.cs +++ b/ProjectPlugins/GethPlugin/GethNode.cs @@ -3,6 +3,7 @@ using Core; using KubernetesWorkflow.Types; using Logging; using Nethereum.ABI.FunctionEncoding.Attributes; +using Nethereum.BlockchainProcessing.BlockStorage.Entities.Mapping; using Nethereum.Contracts; using Nethereum.RPC.Eth.DTOs; using NethereumWorkflow; diff --git a/Tests/CodexReleaseTests/DataTests/TheseusTest.cs b/Tests/CodexReleaseTests/DataTests/TheseusTest.cs index 6f07a34b..36818b65 100644 --- a/Tests/CodexReleaseTests/DataTests/TheseusTest.cs +++ b/Tests/CodexReleaseTests/DataTests/TheseusTest.cs @@ -46,6 +46,93 @@ namespace CodexReleaseTests.DataTests } } + [Test] + [Combinatorial] + public void BlackHoleTest( + [Values(5)] int numBlackHoles, + [Values(50, 200)] int sourceNodes) + { + var blackHoles = StartCodex(numBlackHoles, n => n.WithName("BlackHole")); + + var remaining = sourceNodes; + var listLock = new object(); + var nodes = new List(); + var startTask = Task.Run(() => + { + while (remaining > 0) + { + if (nodes.Count < 3) + { + var count = Math.Min(5, remaining); + remaining -= count; + var n = StartCodex(count); + lock (listLock) + { + nodes.AddRange(n); + } + } + Thread.Sleep(100); + } + }); + + while (remaining > 0 || nodes.Count > 0) + { + var node = TakeNode(nodes, listLock); + + var file = GenerateTestFile(200.MB()); + var cid = node.UploadFile(file); + + SimultaneousDownload(blackHoles, file, cid); + + node.Stop(waitTillStopped: false); + + AllOk(blackHoles); + } + + startTask.Wait(); + } + + private void AllOk(ICodexNodeGroup blackHoles) + { + foreach (var n in blackHoles) + { + Assert.That(n.HasCrashed(), Is.False); + var info = n.GetDebugInfo(); + Assert.That(string.IsNullOrEmpty(info.Spr), Is.False); + } + } + + private void SimultaneousDownload(ICodexNodeGroup blackHoles, TrackedFile file, ContentId cid) + { + var tasks = blackHoles.Select(n => + Task.Run(() => n.DownloadContent(cid)) + ).ToArray(); + + Task.WaitAll(tasks); + + var received = tasks.Select(t => t.Result).ToArray(); + + foreach (var r in received) + { + file.AssertIsEqual(r); + } + } + + private ICodexNode TakeNode(List nodes, object listLock) + { + while (nodes.Count == 0) + { + Thread.Sleep(1000); + } + + lock (listLock) + { + var n = nodes[0]; + nodes.RemoveAt(0); + return n; + } + } + private void AllNodesHaveFile() { Log($"{nameof(AllNodesHaveFile)} {nodes.Names()}"); From 015eaef638831e67e9a3a2388bbfca181592a4e1 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 21 Aug 2025 09:54:46 +0200 Subject: [PATCH 03/11] Prints all chain calls every period --- Framework/Utils/TimeRange.cs | 5 ++ .../ChainMonitor/PeriodMonitor.cs | 81 ++++++++++++++----- .../ChainMonitor/PeriodReport.cs | 8 +- .../ChainMonitor/ProofPeriod.cs | 12 +-- .../CodexPlugin/CodexStartupConfig.cs | 10 ++- 5 files changed, 86 insertions(+), 30 deletions(-) diff --git a/Framework/Utils/TimeRange.cs b/Framework/Utils/TimeRange.cs index 35578d17..3812565b 100644 --- a/Framework/Utils/TimeRange.cs +++ b/Framework/Utils/TimeRange.cs @@ -20,5 +20,10 @@ public DateTime From { get; } public DateTime To { get; } public TimeSpan Duration { get; } + + public override string ToString() + { + return $"{Time.FormatTimestamp(From)} -> {Time.FormatTimestamp(To)} ({Time.FormatDuration(Duration)})"; + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs index 4991db7c..35bb462c 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs @@ -74,17 +74,13 @@ namespace CodexContractsPlugin.ChainMonitor var callReports = new List(); geth.IterateTransactions(blockRange, (t, blkI, blkUtc) => { - CreateFunctionCallReport(callReports, t, blkUtc); - CreateFunctionCallReport(callReports, t, blkUtc); - CreateFunctionCallReport(callReports, t, blkUtc); - CreateFunctionCallReport(callReports, t, blkUtc); + var reporter = new CallReporter(callReports, t, blkUtc, blkI); + reporter.Run(); + }); var report = new PeriodReport( - new ProofPeriod( - currentPeriod.PeriodNumber, - timeRange.From, - timeRange.To), + new ProofPeriod(currentPeriod.PeriodNumber, timeRange, blockRange), currentPeriod.RequiredProofs.ToArray(), callReports.ToArray()); @@ -92,16 +88,6 @@ namespace CodexContractsPlugin.ChainMonitor reports.Add(report); } - private void CreateFunctionCallReport(List reports, Transaction t, DateTime blockUtc) where TFunc : FunctionMessage, new() - { - if (t.IsTransactionForFunctionMessage()) - { - var func = t.DecodeTransactionToFunctionMessage(); - - reports.Add(new FunctionCallReport(blockUtc, typeof(TFunc).Name, JsonConvert.SerializeObject(func))); - } - } - private void ForEachActiveSlot(IChainStateRequest[] requests, Action action) { foreach (var request in requests) @@ -114,6 +100,65 @@ namespace CodexContractsPlugin.ChainMonitor } } + public class CallReporter + { + private readonly List reports; + private readonly Transaction t; + private readonly DateTime blockUtc; + private readonly ulong blockNumber; + + public CallReporter(List reports, Transaction t, DateTime blockUtc, ulong blockNumber) + { + this.reports = reports; + this.t = t; + this.blockUtc = blockUtc; + this.blockNumber = blockNumber; + } + + public void Run() + { + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + } + + private void CreateFunctionCallReport() where TFunc : FunctionMessage, new() + { + if (t.IsTransactionForFunctionMessage()) + { + var func = t.DecodeTransactionToFunctionMessage(); + + reports.Add(new FunctionCallReport(blockUtc, blockNumber, typeof(TFunc).Name, JsonConvert.SerializeObject(func))); + } + } + } + public class PeriodMonitorResult { public PeriodMonitorResult(PeriodReport[] reports) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs index 0e2f6dc3..177fe2e9 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs @@ -23,7 +23,7 @@ namespace CodexContractsPlugin.ChainMonitor { log.Log($" Required: {r.Describe()}"); } - log.Log(" - Calls:"); + log.Log($" - Calls: {FunctionCalls.Length}"); foreach (var f in FunctionCalls) { log.Log($" - {f.Describe()}"); @@ -33,20 +33,22 @@ namespace CodexContractsPlugin.ChainMonitor public class FunctionCallReport { - public FunctionCallReport(DateTime utc, string name, string payload) + public FunctionCallReport(DateTime utc, ulong blockNumber, string name, string payload) { Utc = utc; + BlockNumber = blockNumber; Name = name; Payload = payload; } public DateTime Utc { get; } + public ulong BlockNumber { get; } public string Name { get; } public string Payload { get; } public string Describe() { - return $"[{Time.FormatTimestamp(Utc)}] {Name} = \"{Payload}\""; + return $"[{Time.FormatTimestamp(Utc)}][{BlockNumber}] {Name} = \"{Payload}\""; } } } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs index ca428170..25ad5cc8 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs @@ -4,20 +4,20 @@ namespace CodexContractsPlugin.ChainMonitor { public class ProofPeriod { - public ProofPeriod(ulong periodNumber, DateTime startUtc, DateTime endUtc) + public ProofPeriod(ulong periodNumber, TimeRange timeRange, BlockInterval blockRange) { PeriodNumber = periodNumber; - StartUtc = startUtc; - EndUtc = endUtc; + TimeRange = timeRange; + BlockRange = blockRange; } public ulong PeriodNumber { get; } - public DateTime StartUtc { get; } - public DateTime EndUtc { get; } + public TimeRange TimeRange { get; } + public BlockInterval BlockRange { get; } public override string ToString() { - return $"{PeriodNumber} - {Time.FormatTimestamp(StartUtc)} -> {Time.FormatTimestamp(EndUtc)}"; + return $"{{{PeriodNumber} - {TimeRange} {BlockRange}}}"; } } } diff --git a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs index 30948506..e5c8db73 100644 --- a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs +++ b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs @@ -85,8 +85,11 @@ namespace CodexPlugin { "JSONRPC-WS-CLIENT", "JSONRPC-HTTP-CLIENT", - "codex", - "repostore" + }; + + var alwaysIgnoreTopics = new [] + { + "JSONRPC-CLIENT" }; level = $"{level};" + @@ -94,7 +97,8 @@ namespace CodexPlugin $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" + $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)};" + $"{CustomTopics.JsonSerialize.ToString().ToLowerInvariant()}:{string.Join(",", jsonSerializeTopics)};" + - $"{CustomTopics.MarketplaceInfra.ToString().ToLowerInvariant()}:{string.Join(",", marketplaceInfraTopics)}"; + $"{CustomTopics.MarketplaceInfra.ToString().ToLowerInvariant()}:{string.Join(",", marketplaceInfraTopics)};" + + $"{CodexLogLevel.Error.ToString()}:{string.Join(",", alwaysIgnoreTopics)}"; if (CustomTopics.BlockExchange != null) { From bdf114d1aee63ea5d1f83fcd9dcdee1f99215080 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 21 Aug 2025 10:06:02 +0200 Subject: [PATCH 04/11] we don't need this test --- .../DataTests/TheseusTest.cs | 87 ------------------- 1 file changed, 87 deletions(-) diff --git a/Tests/CodexReleaseTests/DataTests/TheseusTest.cs b/Tests/CodexReleaseTests/DataTests/TheseusTest.cs index 36818b65..6f07a34b 100644 --- a/Tests/CodexReleaseTests/DataTests/TheseusTest.cs +++ b/Tests/CodexReleaseTests/DataTests/TheseusTest.cs @@ -46,93 +46,6 @@ namespace CodexReleaseTests.DataTests } } - [Test] - [Combinatorial] - public void BlackHoleTest( - [Values(5)] int numBlackHoles, - [Values(50, 200)] int sourceNodes) - { - var blackHoles = StartCodex(numBlackHoles, n => n.WithName("BlackHole")); - - var remaining = sourceNodes; - var listLock = new object(); - var nodes = new List(); - var startTask = Task.Run(() => - { - while (remaining > 0) - { - if (nodes.Count < 3) - { - var count = Math.Min(5, remaining); - remaining -= count; - var n = StartCodex(count); - lock (listLock) - { - nodes.AddRange(n); - } - } - Thread.Sleep(100); - } - }); - - while (remaining > 0 || nodes.Count > 0) - { - var node = TakeNode(nodes, listLock); - - var file = GenerateTestFile(200.MB()); - var cid = node.UploadFile(file); - - SimultaneousDownload(blackHoles, file, cid); - - node.Stop(waitTillStopped: false); - - AllOk(blackHoles); - } - - startTask.Wait(); - } - - private void AllOk(ICodexNodeGroup blackHoles) - { - foreach (var n in blackHoles) - { - Assert.That(n.HasCrashed(), Is.False); - var info = n.GetDebugInfo(); - Assert.That(string.IsNullOrEmpty(info.Spr), Is.False); - } - } - - private void SimultaneousDownload(ICodexNodeGroup blackHoles, TrackedFile file, ContentId cid) - { - var tasks = blackHoles.Select(n => - Task.Run(() => n.DownloadContent(cid)) - ).ToArray(); - - Task.WaitAll(tasks); - - var received = tasks.Select(t => t.Result).ToArray(); - - foreach (var r in received) - { - file.AssertIsEqual(r); - } - } - - private ICodexNode TakeNode(List nodes, object listLock) - { - while (nodes.Count == 0) - { - Thread.Sleep(1000); - } - - lock (listLock) - { - var n = nodes[0]; - nodes.RemoveAt(0); - return n; - } - } - private void AllNodesHaveFile() { Log($"{nameof(AllNodesHaveFile)} {nodes.Names()}"); From 9aa377131d074a8554eefc1d4341f21861629d2b Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 21 Aug 2025 10:51:05 +0200 Subject: [PATCH 05/11] Adds stability test --- .../ChainMonitor/ChainState.cs | 4 +- .../ChainMonitor/PeriodMonitor.cs | 21 +++- .../ChainMonitor/PeriodRequiredProof.cs | 4 +- .../MarketTests/StabilityTest.cs | 100 ++++++++++++++++++ Tests/CodexReleaseTests/Utils/ChainMonitor.cs | 4 +- .../Utils/MarketplaceAutoBootstrapDistTest.cs | 6 ++ Tools/MarketInsights/AverageHistory.cs | 2 +- Tools/TestNetRewarder/Processor.cs | 2 +- Tools/TraceContract/ChainTracer.cs | 2 +- 9 files changed, 136 insertions(+), 9 deletions(-) create mode 100644 Tests/CodexReleaseTests/MarketTests/StabilityTest.cs diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 16c9e100..e32a593d 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -43,14 +43,14 @@ namespace CodexContractsPlugin.ChainMonitor private readonly IChainStateChangeHandler handler; private readonly bool doProofPeriodMonitoring; - public ChainState(ILog log, IGethNode geth, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring) + public ChainState(ILog log, IGethNode geth, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring, IPeriodMonitorEventHandler periodEventHandler) { this.log = new LogPrefixer(log, "(ChainState) "); this.contracts = contracts; handler = changeHandler; this.doProofPeriodMonitoring = doProofPeriodMonitoring; TotalSpan = new TimeRange(startUtc, startUtc); - PeriodMonitor = new PeriodMonitor(log, contracts, geth); + PeriodMonitor = new PeriodMonitor(log, contracts, geth, periodEventHandler); } public TimeRange TotalSpan { get; private set; } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs index 35bb462c..afac8993 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs @@ -10,19 +10,26 @@ using System.Reflection; namespace CodexContractsPlugin.ChainMonitor { + public interface IPeriodMonitorEventHandler + { + void OnPeriodReport(PeriodReport report); + } + public class PeriodMonitor { private readonly ILog log; private readonly ICodexContracts contracts; private readonly IGethNode geth; + private readonly IPeriodMonitorEventHandler eventHandler; private readonly List reports = new List(); private CurrentPeriod? currentPeriod = null; - public PeriodMonitor(ILog log, ICodexContracts contracts, IGethNode geth) + public PeriodMonitor(ILog log, ICodexContracts contracts, IGethNode geth, IPeriodMonitorEventHandler eventHandler) { this.log = log; this.contracts = contracts; this.geth = geth; + this.eventHandler = eventHandler; } public void Update(DateTime eventUtc, IChainStateRequest[] requests) @@ -55,9 +62,10 @@ namespace CodexContractsPlugin.ChainMonitor { var idx = Convert.ToInt32(slotIndex); var host = request.Hosts.GetHost(idx); + var slotId = contracts.GetSlotId(request.RequestId, slotIndex); if (host != null) { - result.RequiredProofs.Add(new PeriodRequiredProof(host, request, idx)); + result.RequiredProofs.Add(new PeriodRequiredProof(host, request, idx, slotId)); } } }); @@ -86,6 +94,8 @@ namespace CodexContractsPlugin.ChainMonitor report.Log(log); reports.Add(report); + + eventHandler.OnPeriodReport(report); } private void ForEachActiveSlot(IChainStateRequest[] requests, Action action) @@ -100,6 +110,13 @@ namespace CodexContractsPlugin.ChainMonitor } } + public class DoNothingPeriodMonitorEventHandler : IPeriodMonitorEventHandler + { + public void OnPeriodReport(PeriodReport report) + { + } + } + public class CallReporter { private readonly List reports; diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs index f8df4e23..7332ed1c 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs @@ -5,16 +5,18 @@ namespace CodexContractsPlugin.ChainMonitor { public class PeriodRequiredProof { - public PeriodRequiredProof(EthAddress host, IChainStateRequest request, int slotIndex) + public PeriodRequiredProof(EthAddress host, IChainStateRequest request, int slotIndex, byte[] slotId) { Host = host; Request = request; SlotIndex = slotIndex; + SlotId = slotId; } public EthAddress Host { get; } public IChainStateRequest Request { get; } public int SlotIndex { get; } + public byte[] SlotId { get; } public string Describe() { diff --git a/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs b/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs new file mode 100644 index 00000000..852842a2 --- /dev/null +++ b/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs @@ -0,0 +1,100 @@ +using CodexClient; +using CodexContractsPlugin.ChainMonitor; +using CodexContractsPlugin.Marketplace; +using CodexReleaseTests.Utils; +using Nethereum.Hex.HexConvertors.Extensions; +using Newtonsoft.Json; +using NUnit.Framework; +using Utils; + +namespace CodexReleaseTests.MarketTests +{ + [TestFixture] + public class StabilityTest : MarketplaceAutoBootstrapDistTest, IPeriodMonitorEventHandler + { + #region Setup + + private readonly PurchaseParams purchaseParams = new PurchaseParams( + nodes: 4, + tolerance: 2, + uploadFilesize: 32.MB() + ); + + public StabilityTest() + { + Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts)); + } + + protected override int NumberOfHosts => 6; + protected override int NumberOfClients => 1; + protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot. + protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0); + + #endregion + + [Test] + [Combinatorial] + public void Stability( + [Values(10, 120)] int minutes) + { + Assert.That(HostAvailabilityMaxDuration, Is.GreaterThan(TimeSpan.FromMinutes(minutes * 1.1))); + + GetChainMonitor().PeriodMonitorEventHandler = this; + + StartHosts(); + StartValidator(); + var client = StartClients().Single(); + var purchase = CreateStorageRequest(client, minutes); + + Log($"Contract should remain stable for {minutes} minutes."); + Thread.Sleep(TimeSpan.FromSeconds(minutes)); + + Assert.That(client.GetPurchaseStatus(purchase.PurchaseId)?.State, Is.EqualTo(StoragePurchaseState.Started)); + } + + public void OnPeriodReport(PeriodReport report) + { + // For each required proof, there should be a submit call. + foreach (var required in report.Required) + { + var matchingCall = GetMatchingSubmitProofCall(report, required); + + Assert.That(matchingCall.FromAddress.ToLowerInvariant(), Is.EqualTo(required.Host.Address.ToLowerInvariant())); + Assert.That(matchingCall.Id.ToHex(), Is.EqualTo(required.SlotId.ToHex())); + } + + // There can't be any calls to mark a proof as missed. + foreach (var call in report.FunctionCalls) + { + var missedCall = nameof(MarkProofAsMissingFunction); + Assert.That(call.Name, Is.Not.EqualTo(missedCall)); + } + } + + private SubmitProofFunction GetMatchingSubmitProofCall(PeriodReport report, PeriodRequiredProof required) + { + var submitCall = nameof(SubmitProofFunction); + var call = report.FunctionCalls.SingleOrDefault(f => f.Name == submitCall); + if (call == null) throw new Exception("Call to submitProof not found for " + required.Describe()); + var callObj = JsonConvert.DeserializeObject(call.Payload); + if (callObj == null) throw new Exception("Unable to deserialize call object"); + return callObj; + } + + private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, int minutes) + { + var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); + var config = GetContracts().Deployment.Config; + return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) + { + Duration = TimeSpan.FromMinutes(minutes) * 1.1, + Expiry = TimeSpan.FromMinutes(10.0), + MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, + NodeFailureTolerance = (uint)purchaseParams.Tolerance, + PricePerBytePerSecond = 10.TstWei(), + ProofProbability = 1, // One proof every period. Free slot as quickly as possible. + CollateralPerByte = 1.TstWei() + }); + } + } +} diff --git a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs index f34081e0..f8cae455 100644 --- a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs +++ b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs @@ -43,9 +43,11 @@ namespace CodexReleaseTests.Utils if (worker.Exception != null) throw worker.Exception; } + public IPeriodMonitorEventHandler PeriodMonitorEventHandler { get; set; } = new DoNothingPeriodMonitorEventHandler(); + private void Worker(Action onFailure) { - var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, doProofPeriodMonitoring: true); + var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, true, PeriodMonitorEventHandler); Thread.Sleep(updateInterval); log.Log($"Chain monitoring started. Update interval: {Time.FormatDuration(updateInterval)}"); diff --git a/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs index f3dbdaa3..026a405f 100644 --- a/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs @@ -42,6 +42,12 @@ namespace CodexReleaseTests.Utils return handle.Contracts; } + protected ChainMonitor GetChainMonitor() + { + if (handle.ChainMonitor == null) throw new Exception($"Make sure {nameof(MonitorChainState)} is set to true."); + return handle.ChainMonitor; + } + protected TimeSpan GetPeriodDuration() { var config = GetContracts().Deployment.Config; diff --git a/Tools/MarketInsights/AverageHistory.cs b/Tools/MarketInsights/AverageHistory.cs index 412846d0..9832ca46 100644 --- a/Tools/MarketInsights/AverageHistory.cs +++ b/Tools/MarketInsights/AverageHistory.cs @@ -19,7 +19,7 @@ namespace MarketInsights this.appState = appState; this.maxContributions = maxContributions; chainState = new ChainState(appState.Log, geth, contracts, mux, appState.Config.HistoryStartUtc, - doProofPeriodMonitoring: false); + doProofPeriodMonitoring: false, new DoNothingPeriodMonitorEventHandler()); } public MarketTimeSegment[] Segments { get; private set; } = Array.Empty(); diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 0a459bc6..58bef9ab 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -29,7 +29,7 @@ namespace TestNetRewarder eventsFormatter = new EventsFormatter(config, contracts.Deployment.Config); chainState = new ChainState(log, geth, contracts, eventsFormatter, config.HistoryStartUtc, - doProofPeriodMonitoring: config.ShowProofPeriodReports > 0); + doProofPeriodMonitoring: config.ShowProofPeriodReports > 0, new DoNothingPeriodMonitorEventHandler()); } public async Task Initialize() diff --git a/Tools/TraceContract/ChainTracer.cs b/Tools/TraceContract/ChainTracer.cs index 40f3195d..21a8a69d 100644 --- a/Tools/TraceContract/ChainTracer.cs +++ b/Tools/TraceContract/ChainTracer.cs @@ -63,7 +63,7 @@ namespace TraceContract var utc = request.Block.Utc.AddMinutes(-1.0); var tracker = new ChainRequestTracker(output, input.PurchaseId); var ignoreLog = new NullLog(); - var chainState = new ChainState(ignoreLog, geth, contracts, tracker, utc, false); + var chainState = new ChainState(ignoreLog, geth, contracts, tracker, utc, false, new DoNothingPeriodMonitorEventHandler()); var atNow = false; while (!tracker.IsFinished && !atNow) From 2a85a3649f34c14fd3825ff34785bf643c100a5e Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 21 Aug 2025 11:43:59 +0200 Subject: [PATCH 06/11] fixes new test --- Tests/CodexReleaseTests/MarketTests/StabilityTest.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs b/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs index 852842a2..9b7b08b8 100644 --- a/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs @@ -45,9 +45,10 @@ namespace CodexReleaseTests.MarketTests StartValidator(); var client = StartClients().Single(); var purchase = CreateStorageRequest(client, minutes); + purchase.WaitForStorageContractStarted(); Log($"Contract should remain stable for {minutes} minutes."); - Thread.Sleep(TimeSpan.FromSeconds(minutes)); + Thread.Sleep(TimeSpan.FromMinutes(minutes)); Assert.That(client.GetPurchaseStatus(purchase.PurchaseId)?.State, Is.EqualTo(StoragePurchaseState.Started)); } @@ -88,7 +89,7 @@ namespace CodexReleaseTests.MarketTests return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) { Duration = TimeSpan.FromMinutes(minutes) * 1.1, - Expiry = TimeSpan.FromMinutes(10.0), + Expiry = TimeSpan.FromMinutes(8.0), MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, NodeFailureTolerance = (uint)purchaseParams.Tolerance, PricePerBytePerSecond = 10.TstWei(), From 275937a814018d51ed5bfe6b5a284df8e7bef10e Mon Sep 17 00:00:00 2001 From: thatben Date: Fri, 22 Aug 2025 18:55:01 +0200 Subject: [PATCH 07/11] Cleanup stability test. Add test to check when proof is required --- .../ChainMonitor/PeriodReport.cs | 3 +- .../ChainMonitor/PeriodRequiredProof.cs | 2 +- .../MarketTests/IsProofRequiredTest.cs | 95 +++++++++++++++ .../MarketTests/StabilityTest.cs | 89 ++++++++++---- Tests/CodexReleaseTests/Utils/ChainMonitor.cs | 12 +- .../Utils/MarketplaceAutoBootstrapDistTest.cs | 112 ++++++++++-------- 6 files changed, 233 insertions(+), 80 deletions(-) create mode 100644 Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs index 177fe2e9..fd080820 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs @@ -19,9 +19,10 @@ namespace CodexContractsPlugin.ChainMonitor public void Log(ILog log) { log.Log($"Period report: {Period}"); + log.Log($" - Proofs required: {Required.Length}"); foreach (var r in Required) { - log.Log($" Required: {r.Describe()}"); + log.Log($" - {r.Describe()}"); } log.Log($" - Calls: {FunctionCalls.Length}"); foreach (var f in FunctionCalls) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs index 7332ed1c..5b560081 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs @@ -20,7 +20,7 @@ namespace CodexContractsPlugin.ChainMonitor public string Describe() { - return $"{Request.RequestId.ToHex()} slotIndex:{SlotIndex} by {Host}"; + return $"{Request.RequestId.ToHex()} slotId:{SlotId.ToHex()} slotIndex:{SlotIndex} by {Host}"; } } } diff --git a/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs b/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs new file mode 100644 index 00000000..698b4afa --- /dev/null +++ b/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs @@ -0,0 +1,95 @@ +using CodexClient; +using CodexContractsPlugin.ChainMonitor; +using CodexReleaseTests.Utils; +using Nethereum.Hex.HexConvertors.Extensions; +using NUnit.Framework; +using Utils; + +namespace CodexReleaseTests.MarketTests +{ + [TestFixture] + public class IsProofRequiredTest : MarketplaceAutoBootstrapDistTest + { + #region Setup + + private readonly PurchaseParams purchaseParams = new PurchaseParams( + nodes: 4, + tolerance: 2, + uploadFilesize: 32.MB() + ); + + public IsProofRequiredTest() + { + Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts)); + } + + protected override int NumberOfHosts => 6; + protected override int NumberOfClients => 1; + protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot. + protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0); + + #endregion + + [Test] + public void IsProofRequired() + { + var mins = TimeSpan.FromMinutes(10.0); + + StartHosts(); + StartValidator(); + var client = StartClients().Single(); + var purchase = CreateStorageRequest(client, mins); + purchase.WaitForStorageContractStarted(); + + var requestId = purchase.PurchaseId.HexToByteArray(); + var numSlots = purchaseParams.Nodes; + //var map = new Dictionary>(); + + Log($"Checking IsProofRequired every second for {Time.FormatDuration(mins)}."); + var endUtc = DateTime.UtcNow + mins; + while (DateTime.UtcNow < endUtc) + { + Thread.Sleep(TimeSpan.FromSeconds(1)); + var requiredSlotIndices = new List(); + for (var i = 0; i < numSlots; i++) + { + if (GetContracts().IsProofRequired(requestId, i)) requiredSlotIndices.Add(i); + } + + var periodNumber = GetContracts().GetPeriodNumber(DateTime.UtcNow); + var blockNumber = GetGeth().GetSyncedBlockNumber(); + Log($"[{blockNumber?.ToString().PadLeft(4, '0')}]" + + $"{periodNumber.ToString().PadLeft(12, '0')} => " + + $"{string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}"); + + //var num = currentPeriod.PeriodNumber; + //if (!map.ContainsKey(num)) + //{ + // map.Add(num, requiredSlotIndices); + // Log($"Period {num} = required proof for slot indices {string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}"); + //} + //else + //{ + // var a = map[num]; + // CollectionAssert.AreEquivalent(a, requiredSlotIndices); + //} + } + } + + private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, TimeSpan minutes) + { + var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); + var config = GetContracts().Deployment.Config; + return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) + { + Duration = minutes * 1.1, + Expiry = TimeSpan.FromMinutes(8.0), + MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, + NodeFailureTolerance = (uint)purchaseParams.Tolerance, + PricePerBytePerSecond = 10.TstWei(), + ProofProbability = 1, // One proof every period. Free slot as quickly as possible. + CollateralPerByte = 1.TstWei() + }); + } + } +} diff --git a/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs b/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs index 9b7b08b8..02f7a84d 100644 --- a/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs @@ -10,7 +10,7 @@ using Utils; namespace CodexReleaseTests.MarketTests { [TestFixture] - public class StabilityTest : MarketplaceAutoBootstrapDistTest, IPeriodMonitorEventHandler + public class StabilityTest : MarketplaceAutoBootstrapDistTest { #region Setup @@ -32,36 +32,63 @@ namespace CodexReleaseTests.MarketTests #endregion + private int numPeriods = 0; + private bool proofWasMissed = false; + [Test] [Combinatorial] public void Stability( [Values(10, 120)] int minutes) { - Assert.That(HostAvailabilityMaxDuration, Is.GreaterThan(TimeSpan.FromMinutes(minutes * 1.1))); + var mins = TimeSpan.FromMinutes(minutes); + var periodDuration = GetContracts().Deployment.Config.PeriodDuration; + Assert.That(HostAvailabilityMaxDuration, Is.GreaterThan(mins * 1.1)); - GetChainMonitor().PeriodMonitorEventHandler = this; + numPeriods = 0; + proofWasMissed = false; StartHosts(); StartValidator(); var client = StartClients().Single(); - var purchase = CreateStorageRequest(client, minutes); + var purchase = CreateStorageRequest(client, mins); purchase.WaitForStorageContractStarted(); - Log($"Contract should remain stable for {minutes} minutes."); - Thread.Sleep(TimeSpan.FromMinutes(minutes)); + Log($"Contract should remain stable for {Time.FormatDuration(mins)}."); + var endUtc = DateTime.UtcNow + mins; + while (DateTime.UtcNow < endUtc) + { + Thread.Sleep(TimeSpan.FromSeconds(10)); + if (proofWasMissed) + { + // We wait because we want to log calls to MarkProofAsMissing. + Thread.Sleep(periodDuration * 1.1); + Assert.Fail("Proof was missed."); + } + } - Assert.That(client.GetPurchaseStatus(purchase.PurchaseId)?.State, Is.EqualTo(StoragePurchaseState.Started)); + var minNumPeriod = (mins / periodDuration) - 1.0; + Log($"{numPeriods} periods elapsed. Expected at least {minNumPeriod} periods."); + Assert.That(numPeriods, Is.GreaterThanOrEqualTo(minNumPeriod)); + + var status = client.GetPurchaseStatus(purchase.PurchaseId); + if (status == null) throw new Exception("Purchase status not found"); + Assert.That(status.IsStarted || status.IsFinished); } - public void OnPeriodReport(PeriodReport report) + protected override void OnPeriod(PeriodReport report) { + numPeriods++; + // For each required proof, there should be a submit call. + var calls = GetSubmitProofCalls(report); foreach (var required in report.Required) { - var matchingCall = GetMatchingSubmitProofCall(report, required); - - Assert.That(matchingCall.FromAddress.ToLowerInvariant(), Is.EqualTo(required.Host.Address.ToLowerInvariant())); - Assert.That(matchingCall.Id.ToHex(), Is.EqualTo(required.SlotId.ToHex())); + var matchingCall = GetMatchingSubmitProofCall(calls, required); + if (matchingCall == null) + { + Log($"A proof was missed for {required.Describe()}. Failing test after a delay so chain events have time to log..."); + proofWasMissed = true; + } } // There can't be any calls to mark a proof as missed. @@ -72,23 +99,43 @@ namespace CodexReleaseTests.MarketTests } } - private SubmitProofFunction GetMatchingSubmitProofCall(PeriodReport report, PeriodRequiredProof required) + private SubmitProofFunction? GetMatchingSubmitProofCall(SubmitProofFunction[] calls, PeriodRequiredProof required) { - var submitCall = nameof(SubmitProofFunction); - var call = report.FunctionCalls.SingleOrDefault(f => f.Name == submitCall); - if (call == null) throw new Exception("Call to submitProof not found for " + required.Describe()); - var callObj = JsonConvert.DeserializeObject(call.Payload); - if (callObj == null) throw new Exception("Unable to deserialize call object"); - return callObj; + foreach (var call in calls) + { + if ( + call.Id.SequenceEqual(required.SlotId) && + call.FromAddress.ToLowerInvariant() == required.Host.Address.ToLowerInvariant() + ) + { + return call; + } + } + + return null; } - private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, int minutes) + private SubmitProofFunction[] GetSubmitProofCalls(PeriodReport report) + { + var submitCall = nameof(SubmitProofFunction); + var calls = report.FunctionCalls.Where(f => f.Name == submitCall).ToArray(); + var callObjs = calls.Select(call => JsonConvert.DeserializeObject(call.Payload)).ToArray(); + Log($"SubmitProof calls: {callObjs.Length}"); + foreach (var c in callObjs) + { + Log($" - slotId:{c.Id.ToHex()} host:{c.FromAddress}"); + } + + return callObjs!; + } + + private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, TimeSpan minutes) { var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); var config = GetContracts().Deployment.Config; return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) { - Duration = TimeSpan.FromMinutes(minutes) * 1.1, + Duration = minutes * 1.1, Expiry = TimeSpan.FromMinutes(8.0), MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, NodeFailureTolerance = (uint)purchaseParams.Tolerance, diff --git a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs index f8cae455..07f621fb 100644 --- a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs +++ b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs @@ -11,21 +11,23 @@ namespace CodexReleaseTests.Utils private readonly ILog log; private readonly IGethNode gethNode; private readonly ICodexContracts contracts; + private readonly IPeriodMonitorEventHandler periodMonitorEventHandler; private readonly DateTime startUtc; private readonly TimeSpan updateInterval; private CancellationTokenSource cts = new CancellationTokenSource(); private Task worker = Task.CompletedTask; - public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc) - : this(log, gethNode, contracts, startUtc, TimeSpan.FromSeconds(3.0)) + public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, IPeriodMonitorEventHandler periodMonitorEventHandler, DateTime startUtc) + : this(log, gethNode, contracts, periodMonitorEventHandler, startUtc, TimeSpan.FromSeconds(3.0)) { } - public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc, TimeSpan updateInterval) + public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, IPeriodMonitorEventHandler periodMonitorEventHandler, DateTime startUtc, TimeSpan updateInterval) { this.log = log; this.gethNode = gethNode; this.contracts = contracts; + this.periodMonitorEventHandler = periodMonitorEventHandler; this.startUtc = startUtc; this.updateInterval = updateInterval; } @@ -43,11 +45,9 @@ namespace CodexReleaseTests.Utils if (worker.Exception != null) throw worker.Exception; } - public IPeriodMonitorEventHandler PeriodMonitorEventHandler { get; set; } = new DoNothingPeriodMonitorEventHandler(); - private void Worker(Action onFailure) { - var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, true, PeriodMonitorEventHandler); + var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, true, periodMonitorEventHandler); Thread.Sleep(updateInterval); log.Log($"Chain monitoring started. Update interval: {Time.FormatDuration(updateInterval)}"); diff --git a/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs index 026a405f..43fcddae 100644 --- a/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs @@ -1,5 +1,6 @@ using CodexClient; using CodexContractsPlugin; +using CodexContractsPlugin.ChainMonitor; using CodexContractsPlugin.Marketplace; using CodexPlugin; using CodexTests; @@ -11,7 +12,7 @@ using Utils; namespace CodexReleaseTests.Utils { - public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest + public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest, IPeriodMonitorEventHandler { private MarketplaceHandle handle = null!; protected const int StartingBalanceTST = 1000; @@ -60,6 +61,9 @@ namespace CodexReleaseTests.Utils protected abstract TimeSpan HostAvailabilityMaxDuration { get; } protected virtual bool MonitorChainState { get; } = true; protected TimeSpan HostBlockTTL { get; } = TimeSpan.FromMinutes(1.0); + protected virtual void OnPeriod(PeriodReport report) + { + } public ICodexNodeGroup StartHosts() { @@ -175,56 +179,6 @@ namespace CodexReleaseTests.Utils }); } - private ChainMonitor? SetupChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc) - { - if (!MonitorChainState) return null; - - var result = new ChainMonitor(log, gethNode, contracts, startUtc); - result.Start(() => - { - Assert.Fail("Failure in chain monitor."); - }); - return result; - } - - private Retry GetBalanceAssertRetry() - { - return new Retry("AssertBalance", - maxTimeout: TimeSpan.FromMinutes(10.0), - sleepAfterFail: TimeSpan.FromSeconds(10.0), - onFail: f => { }, - failFast: false); - } - - private Retry GetAvailabilitySpaceAssertRetry() - { - return new Retry("AssertAvailabilitySpace", - maxTimeout: HostBlockTTL * 3, - sleepAfterFail: TimeSpan.FromSeconds(10.0), - onFail: f => { }, - failFast: false); - } - - private TestToken GetTstBalance(ICodexNode node) - { - return GetContracts().GetTestTokenBalance(node); - } - - private TestToken GetTstBalance(EthAddress address) - { - return GetContracts().GetTestTokenBalance(address); - } - - private Ether GetEthBalance(ICodexNode node) - { - return GetGeth().GetEthBalance(node); - } - - private Ether GetEthBalance(EthAddress address) - { - return GetGeth().GetEthBalance(address); - } - public ICodexNodeGroup StartClients() { return StartClients(s => { }); @@ -253,6 +207,11 @@ namespace CodexReleaseTests.Utils ); } + public void OnPeriodReport(PeriodReport report) + { + OnPeriod(report); + } + public SlotFill[] GetOnChainSlotFills(IEnumerable possibleHosts, string purchaseId) { var fills = GetOnChainSlotFills(possibleHosts); @@ -351,6 +310,56 @@ namespace CodexReleaseTests.Utils } } + private ChainMonitor? SetupChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc) + { + if (!MonitorChainState) return null; + + var result = new ChainMonitor(log, gethNode, contracts, this, startUtc); + result.Start(() => + { + Assert.Fail("Failure in chain monitor."); + }); + return result; + } + + private Retry GetBalanceAssertRetry() + { + return new Retry("AssertBalance", + maxTimeout: TimeSpan.FromMinutes(10.0), + sleepAfterFail: TimeSpan.FromSeconds(10.0), + onFail: f => { }, + failFast: false); + } + + private Retry GetAvailabilitySpaceAssertRetry() + { + return new Retry("AssertAvailabilitySpace", + maxTimeout: HostBlockTTL * 3, + sleepAfterFail: TimeSpan.FromSeconds(10.0), + onFail: f => { }, + failFast: false); + } + + private TestToken GetTstBalance(ICodexNode node) + { + return GetContracts().GetTestTokenBalance(node); + } + + private TestToken GetTstBalance(EthAddress address) + { + return GetContracts().GetTestTokenBalance(address); + } + + private Ether GetEthBalance(ICodexNode node) + { + return GetGeth().GetEthBalance(node); + } + + private Ether GetEthBalance(EthAddress address) + { + return GetGeth().GetEthBalance(address); + } + private TestToken GetContractFinalCost(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts) { var fills = GetOnChainSlotFills(hosts); @@ -465,6 +474,7 @@ namespace CodexReleaseTests.Utils var chanceOfDowntime = downtime / window; return 1.0f + (5.0f * chanceOfDowntime); } + public class SlotFill { public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host) From e03f5982d30854308eab7049ad60c2f28021f3f4 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 25 Aug 2025 11:11:56 +0200 Subject: [PATCH 08/11] Requires new contracts image with configurable marketplace config --- .../Recipe/ContainerRecipeFactory.cs | 8 ++++++- .../CodexContractsContainerRecipe.cs | 21 +++++++++++++++++++ .../CodexContractsStarter.cs | 13 +++++++++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs index 0e91dbc9..2e22dd61 100644 --- a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs @@ -1,4 +1,5 @@ -using Utils; +using System.Globalization; +using Utils; namespace KubernetesWorkflow.Recipe { @@ -88,6 +89,11 @@ namespace KubernetesWorkflow.Recipe envVars.Add(factory.CreateEnvVar(name, value)); } + protected void AddEnvVar(string name, int value) + { + envVars.Add(factory.CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture))); + } + protected void AddEnvVar(string name, Port value) { envVars.Add(factory.CreateEnvVar(name, value.Number)); diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs index 09c5b829..4b5dc172 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs @@ -9,6 +9,11 @@ namespace CodexContractsPlugin { public const string DeployedAddressesFilename = "/hardhat/ignition/deployments/chain-789988/deployed_addresses.json"; public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json"; + + public const int PeriodSeconds = 60; + public const int TimeoutSeconds = 30; + public const int DowntimeSeconds = 128; + private readonly DebugInfoVersion versionInfo; public override string AppName => "codex-contracts"; @@ -28,6 +33,22 @@ namespace CodexContractsPlugin SetSchedulingAffinity(notIn: "false"); AddEnvVar("DISTTEST_NETWORK_URL", address.ToString()); + + // Default values: + AddEnvVar("DISTTEST_REPAIRREWARD", 10); + AddEnvVar("DISTTEST_MAXSLASHES", 2); + AddEnvVar("DISTTEST_SLASHPERCENTAGE", 20); + AddEnvVar("DISTTEST_VALIDATORREWARD", 20); + AddEnvVar("DISTTEST_DOWNTIMEPRODUCT", 67); + AddEnvVar("DISTTEST_MAXRESERVATIONS", 3); + AddEnvVar("DISTTEST_MAXDURATION", Convert.ToInt32(TimeSpan.FromDays(30).TotalSeconds)); + + // Customized values, required to operate in a network with + // block frequency of 1. + AddEnvVar("DISTTEST_PERIOD", PeriodSeconds); + AddEnvVar("DISTTEST_TIMEOUT", TimeoutSeconds); + AddEnvVar("DISTTEST_DOWNTIME", DowntimeSeconds); + AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork"); AddEnvVar("HARDHAT_IGNITION_CONFIRM_DEPLOYMENT", "false"); AddEnvVar("KEEP_ALIVE", "1"); diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index 63032158..207a6681 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -92,12 +92,23 @@ namespace CodexContractsPlugin Log("Synced. Codex SmartContracts deployed. Getting configuration..."); var config = GetMarketplaceConfiguration(marketplaceAddress, gethNode); - Log("Got config: " + JsonConvert.SerializeObject(config)); + ConfigShouldEqual(config.Proofs.Period, CodexContractsContainerRecipe.PeriodSeconds, "Period"); + ConfigShouldEqual(config.Proofs.Timeout, CodexContractsContainerRecipe.TimeoutSeconds, "Timeout"); + ConfigShouldEqual(config.Proofs.Downtime, CodexContractsContainerRecipe.DowntimeSeconds, "Downtime"); + return new CodexContractsDeployment(config, marketplaceAddress, abi, tokenAddress); } + private void ConfigShouldEqual(ulong value, int expected, string name) + { + if (Convert.ToInt32(value) != expected) + { + throw new Exception($"Config value '{name}' should be deployed as '{expected}' but was '{value}'"); + } + } + private MarketplaceConfig GetMarketplaceConfiguration(string marketplaceAddress, IGethNode gethNode) { var func = new ConfigurationFunctionBase(); From 74dbdfba17a615303c9f9c3ca535f9dd709255dd Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 25 Aug 2025 11:15:00 +0200 Subject: [PATCH 09/11] Adds will-be-required to log --- .../CodexContractsAccess.cs | 17 +++++++++++++++++ .../MarketTests/IsProofRequiredTest.cs | 8 ++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs index 132451e7..0c952b1d 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs @@ -32,6 +32,7 @@ namespace CodexContractsPlugin TimeRange GetPeriodTimeRange(ulong periodNumber); void WaitUntilNextPeriod(); bool IsProofRequired(byte[] requestId, decimal slotIndex); + bool WillProofBeRequired(byte[] requestId, decimal slotIndex); byte[] GetSlotId(byte[] requestId, decimal slotIndex); ICodexContracts WithDifferentGeth(IGethNode node); @@ -172,6 +173,12 @@ namespace CodexContractsPlugin return IsProofRequired(slotId); } + public bool WillProofBeRequired(byte[] requestId, decimal slotIndex) + { + var slotId = GetSlotId(requestId, slotIndex); + return WillProofBeRequired(slotId); + } + public ICodexContracts WithDifferentGeth(IGethNode node) { return new CodexContractsAccess(log, node, Deployment); @@ -198,6 +205,16 @@ namespace CodexContractsPlugin return result.ReturnValue1; } + private bool WillProofBeRequired(byte[] slotId) + { + var func = new WillProofBeRequiredFunction + { + Id = slotId + }; + var result = gethNode.Call(Deployment.MarketplaceAddress, func); + return result.ReturnValue1; + } + private ContractInteractions StartInteraction() { return new ContractInteractions(log, gethNode); diff --git a/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs b/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs index 698b4afa..95f55fbd 100644 --- a/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs @@ -51,16 +51,20 @@ namespace CodexReleaseTests.MarketTests { Thread.Sleep(TimeSpan.FromSeconds(1)); var requiredSlotIndices = new List(); + var willRequireSlotIndices = new List(); for (var i = 0; i < numSlots; i++) { if (GetContracts().IsProofRequired(requestId, i)) requiredSlotIndices.Add(i); + if (GetContracts().WillProofBeRequired(requestId, i)) willRequireSlotIndices.Add(i); } var periodNumber = GetContracts().GetPeriodNumber(DateTime.UtcNow); var blockNumber = GetGeth().GetSyncedBlockNumber(); Log($"[{blockNumber?.ToString().PadLeft(4, '0')}]" + - $"{periodNumber.ToString().PadLeft(12, '0')} => " + - $"{string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}"); + $"{periodNumber.ToString().PadLeft(12, '0')} => Required now: " + + $"[{string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}] " + + $"Will be required: " + + $"[{string.Join(",", willRequireSlotIndices.Select(i => i.ToString()))}]"); //var num = currentPeriod.PeriodNumber; //if (!map.ContainsKey(num)) From 365e78f050b0b416f575abd9fa8bdedc12d8919b Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 25 Aug 2025 13:56:33 +0200 Subject: [PATCH 10/11] logging also when proof will be required --- .../CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs index afac8993..825db0d4 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs @@ -58,7 +58,8 @@ namespace CodexContractsPlugin.ChainMonitor var result = new CurrentPeriod(periodNumber); ForEachActiveSlot(requests, (request, slotIndex) => { - if (contracts.IsProofRequired(request.RequestId, slotIndex)) + if (contracts.IsProofRequired(request.RequestId, slotIndex) || + contracts.WillProofBeRequired(request.RequestId, slotIndex)) { var idx = Convert.ToInt32(slotIndex); var host = request.Hosts.GetHost(idx); From c16397ae47a37af2d3d78f9f3972b37f70c2b300 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 1 Sep 2025 10:35:40 +0200 Subject: [PATCH 11/11] Disables assert on contract config values check --- ProjectPlugins/CodexClient/TransferSpeeds.cs | 13 + .../CodexContractsStarter.cs | 10 +- .../CodexReleaseTests/DataTests/SwarmTest.cs | 286 +++++++++--------- 3 files changed, 173 insertions(+), 136 deletions(-) diff --git a/ProjectPlugins/CodexClient/TransferSpeeds.cs b/ProjectPlugins/CodexClient/TransferSpeeds.cs index 9e013efb..bb6afeed 100644 --- a/ProjectPlugins/CodexClient/TransferSpeeds.cs +++ b/ProjectPlugins/CodexClient/TransferSpeeds.cs @@ -6,6 +6,7 @@ namespace CodexClient { BytesPerSecond? GetUploadSpeed(); BytesPerSecond? GetDownloadSpeed(); + ITransferSpeeds Combine(ITransferSpeeds? other); } public class TransferSpeeds : ITransferSpeeds @@ -35,6 +36,18 @@ namespace CodexClient return downloads.Average(); } + public ITransferSpeeds Combine(ITransferSpeeds? other) + { + if (other == null) return this; + var o = (TransferSpeeds)other; + var result = new TransferSpeeds(); + result.uploads.AddRange(uploads); + result.uploads.AddRange(o.uploads); + result.downloads.AddRange(downloads); + result.downloads.AddRange(o.downloads); + return result; + } + private static BytesPerSecond Convert(ByteSize size, TimeSpan duration) { double bytes = size.SizeInBytes; diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index 207a6681..aeac9c21 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -105,7 +105,15 @@ namespace CodexContractsPlugin { if (Convert.ToInt32(value) != expected) { - throw new Exception($"Config value '{name}' should be deployed as '{expected}' but was '{value}'"); + // Merge todo: https://github.com/codex-storage/nim-codex/pull/1303 + // Once this is merged, the contract config values are settable via env-vars. + // This plugin is already updated to set the config vars to values compatible with a + // 1-second block frequency. AND it will read back the config and assert it is deployed correctly. + // This is waiting for that merge. + + // Replace log with assert WHEN MERGED: + // throw new Exception($"Config value '{name}' should be deployed as '{expected}' but was '{value}'"); + Log($"MERGE TODO. Config value '{name}' should be deployed as '{expected}' but was '{value}'"); } } diff --git a/Tests/CodexReleaseTests/DataTests/SwarmTest.cs b/Tests/CodexReleaseTests/DataTests/SwarmTest.cs index f4099ec0..2379fb08 100644 --- a/Tests/CodexReleaseTests/DataTests/SwarmTest.cs +++ b/Tests/CodexReleaseTests/DataTests/SwarmTest.cs @@ -7,175 +7,191 @@ using Utils; namespace CodexReleaseTests.DataTests { - [TestFixture(2, 10)] - [TestFixture(5, 20)] - [TestFixture(10, 20)] - public class SwarmTests : AutoBootstrapDistTest + namespace SwarmTests { - private readonly int numberOfNodes; - private readonly int filesizeMb; - - public SwarmTests(int numberOfNodes, int filesizeMb) + [TestFixture(2, 10)] + [TestFixture(5, 20)] + [TestFixture(10, 20)] + public class SwarmTests : AutoBootstrapDistTest { - this.numberOfNodes = numberOfNodes; - this.filesizeMb = filesizeMb; - } + private readonly int numberOfNodes; + private readonly int filesizeMb; + private ICodexNodeGroup nodes = null!; - [Test] - public void Swarm() - { - var filesize = filesizeMb.MB(); - var nodes = StartCodex(numberOfNodes); - var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); - - var tasks = ParallelDownloadEachFile(nodes, files); - Task.WaitAll(tasks); - - AssertAllFilesDownloadedCorrectly(files); - } - - [Test] - public void StreamlessSwarm() - { - var filesize = filesizeMb.MB(); - var nodes = StartCodex(numberOfNodes); - var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); - - var tasks = ParallelStreamlessDownloadEachFile(nodes, files); - Task.WaitAll(tasks); - - AssertAllFilesStreamlesslyDownloadedCorrectly(nodes, files); - } - - private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, ByteSize fileSize) - { - var file = GenerateTestFile(fileSize); - var cid = node.UploadFile(file); - return new SwarmTestNetworkFile(node, fileSize, file, cid); - } - - private Task[] ParallelDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files) - { - var tasks = new List(); - - foreach (var node in nodes) + public SwarmTests(int numberOfNodes, int filesizeMb) { - tasks.Add(StartDownload(node, files)); + this.numberOfNodes = numberOfNodes; + this.filesizeMb = filesizeMb; } - return tasks.ToArray(); - } - - private Task[] ParallelStreamlessDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files) - { - var tasks = new List(); - - foreach (var node in nodes) + [TearDown] + public void TearDown() { - tasks.Add(StartStreamlessDownload(node, files)); - } - - return tasks.ToArray(); - } - - private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files) - { - return Task.Run(() => - { - var remaining = files.ToList(); - - while (remaining.Count > 0) + ITransferSpeeds speeds = new TransferSpeeds(); + foreach (var n in nodes) { - var file = remaining.PickOneRandom(); - try - { - var dl = node.DownloadContent(file.Cid, TimeSpan.FromMinutes(30)); - lock (file.Lock) - { - file.Downloaded.Add(dl); - } - } - catch (Exception ex) - { - file.Error = ex; - } + speeds = speeds.Combine(n.TransferSpeeds); } - }); - } + Log($"Average upload speed: {speeds.GetUploadSpeed()}"); + Log($"Average download speed: {speeds.GetDownloadSpeed()}"); + } - private Task StartStreamlessDownload(ICodexNode node, SwarmTestNetworkFile[] files) - { - return Task.Run(() => + [Test] + public void Stream() { - var remaining = files.ToList(); + var filesize = filesizeMb.MB(); + nodes = StartCodex(numberOfNodes); + var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); - while (remaining.Count > 0) + var tasks = ParallelDownloadEachFile(files); + Task.WaitAll(tasks); + + AssertAllFilesDownloadedCorrectly(files); + } + + [Test] + public void Streamless() + { + var filesize = filesizeMb.MB(); + nodes = StartCodex(numberOfNodes); + var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); + + var tasks = ParallelStreamlessDownloadEachFile(files); + Task.WaitAll(tasks); + + AssertAllFilesStreamlesslyDownloadedCorrectly(files); + } + + private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, ByteSize fileSize) + { + var file = GenerateTestFile(fileSize); + var cid = node.UploadFile(file); + return new SwarmTestNetworkFile(node, fileSize, file, cid); + } + + private Task[] ParallelDownloadEachFile(SwarmTestNetworkFile[] files) + { + var tasks = new List(); + + foreach (var node in nodes) { - var file = remaining.PickOneRandom(); - if (file.Uploader.GetName() != node.GetName()) + tasks.Add(StartDownload(node, files)); + } + + return tasks.ToArray(); + } + + private Task[] ParallelStreamlessDownloadEachFile(SwarmTestNetworkFile[] files) + { + var tasks = new List(); + + foreach (var node in nodes) + { + tasks.Add(StartStreamlessDownload(node, files)); + } + + return tasks.ToArray(); + } + + private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files) + { + return Task.Run(() => + { + var remaining = files.ToList(); + + while (remaining.Count > 0) { + var file = remaining.PickOneRandom(); try { - var startSpace = node.Space(); - node.DownloadStreamlessWait(file.Cid, file.OriginalSize); + var dl = node.DownloadContent(file.Cid, TimeSpan.FromMinutes(30)); + lock (file.Lock) + { + file.Downloaded.Add(dl); + } } catch (Exception ex) { file.Error = ex; } } - } - }); - } + }); + } - private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files) - { - foreach (var file in files) + private Task StartStreamlessDownload(ICodexNode node, SwarmTestNetworkFile[] files) { - if (file.Error != null) throw file.Error; - lock (file.Lock) + return Task.Run(() => { - foreach (var dl in file.Downloaded) + var remaining = files.ToList(); + + while (remaining.Count > 0) { - file.Original.AssertIsEqual(dl); + var file = remaining.PickOneRandom(); + if (file.Uploader.GetName() != node.GetName()) + { + try + { + var startSpace = node.Space(); + node.DownloadStreamlessWait(file.Cid, file.OriginalSize); + } + catch (Exception ex) + { + file.Error = ex; + } + } + } + }); + } + + private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files) + { + foreach (var file in files) + { + if (file.Error != null) throw file.Error; + lock (file.Lock) + { + foreach (var dl in file.Downloaded) + { + file.Original.AssertIsEqual(dl); + } } } } - } - private void AssertAllFilesStreamlesslyDownloadedCorrectly(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files) - { - var totalFilesSpace = 0.Bytes(); - foreach (var file in files) + private void AssertAllFilesStreamlesslyDownloadedCorrectly(SwarmTestNetworkFile[] files) { - if (file.Error != null) throw file.Error; - totalFilesSpace = new ByteSize(totalFilesSpace.SizeInBytes + file.Original.GetFilesize().SizeInBytes); - } - - foreach (var node in nodes) - { - var currentSpace = node.Space(); - Assert.That(currentSpace.QuotaUsedBytes, Is.GreaterThanOrEqualTo(totalFilesSpace.SizeInBytes)); - } - } + var totalFilesSpace = 0.Bytes(); + foreach (var file in files) + { + if (file.Error != null) throw file.Error; + totalFilesSpace = new ByteSize(totalFilesSpace.SizeInBytes + file.Original.GetFilesize().SizeInBytes); + } - private class SwarmTestNetworkFile - { - public SwarmTestNetworkFile(ICodexNode uploader, ByteSize originalSize, TrackedFile original, ContentId cid) - { - Uploader = uploader; - OriginalSize = originalSize; - Original = original; - Cid = cid; + foreach (var node in nodes) + { + var currentSpace = node.Space(); + Assert.That(currentSpace.QuotaUsedBytes, Is.GreaterThanOrEqualTo(totalFilesSpace.SizeInBytes)); + } } - public ICodexNode Uploader { get; } - public ByteSize OriginalSize { get; } - public TrackedFile Original { get; } - public ContentId Cid { get; } - public object Lock { get; } = new object(); - public List Downloaded { get; } = new List(); - public Exception? Error { get; set; } = null; + private class SwarmTestNetworkFile + { + public SwarmTestNetworkFile(ICodexNode uploader, ByteSize originalSize, TrackedFile original, ContentId cid) + { + Uploader = uploader; + OriginalSize = originalSize; + Original = original; + Cid = cid; + } + + public ICodexNode Uploader { get; } + public ByteSize OriginalSize { get; } + public TrackedFile Original { get; } + public ContentId Cid { get; } + public object Lock { get; } = new object(); + public List Downloaded { get; } = new List(); + public Exception? Error { get; set; } = null; + } } } }