logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-01-07 16:03:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

better representation of proof state

Browse Source
This commit is contained in:
Ben 2025-08-13 15:23:31 +02:00
parent 82fde74c36
commit 2e1039a70f
No known key found for this signature in database
GPG Key ID: 0F16E812E736C24B
6 changed files with 133 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ProjectPlugins/CodexClient/MarketplaceAccess.cs
View File

@ -46,6 +46,9 @@ namespace CodexClient
Log($"Storage requested successfully. PurchaseId: '{response}'.");
var logName = $"<Purchase-{response.Substring(0, 3)}>";
log.AddStringReplace(response, logName);
log.AddStringReplace(response.ToLowerInvariant(), logName);
return new StoragePurchaseContract(log, codexAccess, response, purchase, hooks);
}

30
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs
View File

@ -1,6 +1,7 @@
using BlockchainUtils;
using CodexContractsPlugin.Marketplace;
using Logging;
using Nethereum.Contracts;
using Nethereum.Hex.HexConvertors.Extensions;
using System.Numerics;
using Utils;
@ -92,17 +93,17 @@ namespace CodexContractsPlugin.ChainMonitor
{
var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray();
ApplyEvents(b, blockEvents, eventUtc);
UpdatePeriodMonitor(b, eventUtc);
UpdatePeriodMonitor(eventUtc);
eventUtc += spanPerBlock;
}
}
private void UpdatePeriodMonitor(ulong blockNumber, DateTime eventUtc)
private void UpdatePeriodMonitor(DateTime eventUtc)
{
if (!doProofPeriodMonitoring) return;
var activeRequests = requests.Where(r => r.State == RequestState.Started).ToArray();
PeriodMonitor.Update(blockNumber, eventUtc, activeRequests);
PeriodMonitor.Update(eventUtc, activeRequests);
}
private void ApplyEvents(ulong blockNumber, IHasBlock[] blockEvents, DateTime eventsUtc)
@ -189,10 +190,31 @@ namespace CodexContractsPlugin.ChainMonitor
private void ApplyEvent(ProofSubmittedEventDTO @event)
{
var id = Base58.Encode(@event.Id);
log.Log($"[{@event.Block.BlockNumber}] Proof submitted (id:{id})");
var proofOrigin = SearchForProofOrigin(id);
log.Log($"[{@event.Block.BlockNumber}] Proof submitted (id:{id} {proofOrigin})");
handler.OnProofSubmitted(@event.Block, id);
}
private string SearchForProofOrigin(string slotId)
{
foreach (var r in requests)
{
for (decimal slotIndex = 0; slotIndex < r.Request.Ask.Slots; slotIndex++)
{
var thisSlotId = contracts.GetSlotId(r.RequestId, slotIndex);
var id = Base58.Encode(thisSlotId);
if (id.ToLowerInvariant() == slotId.ToLowerInvariant())
{
return $"({r.RequestId.ToHex()} slotIndex:{slotIndex})";
}
}
}
return "(Could not identify proof requestId + slot)";
}
private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
{
foreach (var r in requests)

77
ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs
View File

@ -1,5 +1,6 @@
using Logging;
using Nethereum.Hex.HexConvertors.Extensions;
using Nethereum.Model;
using Utils;
namespace CodexContractsPlugin.ChainMonitor
@ -17,7 +18,7 @@ namespace CodexContractsPlugin.ChainMonitor
this.contracts = contracts;
}
public void Update(ulong blockNumber, DateTime eventUtc, IChainStateRequest[] requests)
public void Update(DateTime eventUtc, IChainStateRequest[] requests)
{
var period = contracts.GetPeriodNumber(eventUtc);
if (!currentPeriod.HasValue)
@ -27,7 +28,7 @@ namespace CodexContractsPlugin.ChainMonitor
}
if (period == currentPeriod.Value) return;
CreateReportForPeriod(blockNumber - 1, currentPeriod.Value, requests);
CreateReportForPeriod(currentPeriod.Value, requests);
currentPeriod = period;
}
@ -38,34 +39,25 @@ namespace CodexContractsPlugin.ChainMonitor
return new PeriodMonitorResult(result);
}
private void CreateReportForPeriod(ulong lastBlockInPeriod, ulong periodNumber, IChainStateRequest[] requests)
private void CreateReportForPeriod(ulong periodNumber, IChainStateRequest[] requests)
{
ulong total = 0;
var required = new List<PeriodProof>();
var missed = new List<PeriodProof>();
var periodProofs = new List<PeriodProof>();
foreach (var request in requests)
{
for (ulong slotIndex = 0; slotIndex < request.Request.Ask.Slots; slotIndex++)
{
var state = contracts.GetProofState(request.RequestId, slotIndex, lastBlockInPeriod, periodNumber);
var state = contracts.GetProofState(request.RequestId, slotIndex, periodNumber);
total++;
if (state.Required)
{
var idx = Convert.ToInt32(slotIndex);
var host = request.Hosts.GetHost(idx);
var proof = new PeriodProof(host, request, idx);
required.Add(proof);
if (state.Missing)
{
missed.Add(proof);
}
}
var idx = Convert.ToInt32(slotIndex);
var host = request.Hosts.GetHost(idx);
var proof = new PeriodProof(host, request, idx, state);
periodProofs.Add(proof);
}
}
var report = new PeriodReport(periodNumber, total, required.ToArray(), missed.ToArray());
log.Log($"Period report: {report}");
var report = new PeriodReport(periodNumber, total, periodProofs.ToArray());
report.Log(log);
reports.Add(report);
}
}
@ -89,62 +81,77 @@ namespace CodexContractsPlugin.ChainMonitor
private void CalcStats()
{
IsEmpty = Reports.All(r => r.ProofsRequired.Length == 0);
IsEmpty = Reports.All(r => r.PeriodProofs.Length == 0);
if (Reports.Length == 0) return;
PeriodLow = Reports.Min(r => r.PeriodNumber);
PeriodHigh = Reports.Max(r => r.PeriodNumber);
AverageNumSlots = Reports.Average(r => Convert.ToSingle(r.TotalNumSlots));
AverageNumProofsRequired = Reports.Average(r => Convert.ToSingle(r.ProofsRequired.Length));
AverageNumProofsRequired = Reports.Average(r => Convert.ToSingle(r.PeriodProofs.Count(p => p.State != ProofState.NotRequired)));
}
}
public class PeriodReport
{
public PeriodReport(ulong periodNumber, ulong totalNumSlots, PeriodProof[] proofsRequired, PeriodProof[] missedProofs)
public PeriodReport(ulong periodNumber, ulong totalNumSlots, PeriodProof[] periodProofs)
{
PeriodNumber = periodNumber;
TotalNumSlots = totalNumSlots;
ProofsRequired = proofsRequired;
MissedProofs = missedProofs;
PeriodProofs = periodProofs;
}
public ulong PeriodNumber { get; }
public ulong TotalNumSlots { get; }
public PeriodProof[] ProofsRequired { get; }
public PeriodProof[] MissedProofs { get; }
public PeriodProof[] PeriodProofs { get; }
public override string ToString()
public PeriodProof[] GetMissedProofs()
{
var required = Describe(ProofsRequired);
var missed = Describe(MissedProofs);
return $"Period:{PeriodNumber}=[Slots:{TotalNumSlots},ProofsRequired:{required},ProofsMissed:{missed}]";
return PeriodProofs.Where(p => p.State == ProofState.MissedAndMarked || p.State == ProofState.MissedNotMarked).ToArray();
}
private string Describe(PeriodProof[] proofs)
public void Log(ILog log)
{
if (proofs.Length == 0) return "None";
return string.Join("+", proofs.Select(p => $"{p.FormatHost()} - {p.Request.RequestId.ToHex()} slot {p.SlotIndex}"));
log.Log($"Period report: {PeriodNumber}");
log.Log($" - Slots: {TotalNumSlots}");
foreach (var p in PeriodProofs)
{
log.Log($" - {p.Describe()}");
}
}
private void Log(ILog log, PeriodProof[] proofs)
{
if (proofs.Length == 0) return;
foreach (var p in proofs)
{
}
}
}
public class PeriodProof
{
public PeriodProof(EthAddress? host, IChainStateRequest request, int slotIndex)
public PeriodProof(EthAddress? host, IChainStateRequest request, int slotIndex, ProofState state)
{
Host = host;
Request = request;
SlotIndex = slotIndex;
State = state;
}
public EthAddress? Host { get; }
public IChainStateRequest Request { get; }
public int SlotIndex { get; }
public ProofState State { get; }
public string FormatHost()
{
if (Host == null) return "Unknown host";
return Host.Address;
}
public string Describe()
{
return $"{FormatHost()} - {Request.RequestId.ToHex()} slotIndex:{SlotIndex} => {State}";
}
}
}

79
ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs
View File

@ -4,6 +4,7 @@ using GethPlugin;
using Logging;
using Nethereum.ABI;
using Nethereum.Contracts;
using Nethereum.Hex.HexConvertors.Extensions;
using Nethereum.Util;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
@ -29,21 +30,18 @@ namespace CodexContractsPlugin
Request GetRequest(byte[] requestId);
ulong GetPeriodNumber(DateTime utc);
void WaitUntilNextPeriod();
ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong blockNumber, ulong period);
ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong period);
byte[] GetSlotId(byte[] requestId, decimal slotIndex);
ICodexContracts WithDifferentGeth(IGethNode node);
}
public class ProofState
public enum ProofState
{
public ProofState(bool required, bool missing)
{
Required = required;
Missing = missing;
}
public bool Required { get; }
public bool Missing { get; }
NotRequired,
NotMissed,
MissedNotMarked,
MissedAndMarked,
}
[JsonConverter(typeof(StringEnumConverter))]
@ -165,15 +163,14 @@ namespace CodexContractsPlugin
Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
}
public ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong blockNumber, ulong period)
public ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong period)
{
var slotId = GetSlotId(requestId, slotIndex);
var required = IsProofRequired(slotId, blockNumber);
if (!required) return new ProofState(false, false);
var required = IsProofRequired(slotId);
if (!required) return ProofState.NotRequired;
var missing = IsProofMissing(slotId, period, blockNumber);
return new ProofState(required, missing);
return IsProofMissing(slotId, period);
}
public ICodexContracts WithDifferentGeth(IGethNode node)
@ -181,7 +178,7 @@ namespace CodexContractsPlugin
return new CodexContractsAccess(log, node, Deployment);
}
private byte[] GetSlotId(byte[] requestId, decimal slotIndex)
public byte[] GetSlotId(byte[] requestId, decimal slotIndex)
{
var encoder = new ABIEncode();
var encoded = encoder.GetABIEncoded(
@ -192,17 +189,37 @@ namespace CodexContractsPlugin
return Sha3Keccack.Current.CalculateHash(encoded);
}
private bool IsProofRequired(byte[] slotId, ulong blockNumber)
private bool IsProofRequired(byte[] slotId)
{
var func = new IsProofRequiredFunction
{
Id = slotId
};
var result = gethNode.Call<IsProofRequiredFunction, IsProofRequiredOutputDTO>(Deployment.MarketplaceAddress, func, blockNumber);
var result = gethNode.Call<IsProofRequiredFunction, IsProofRequiredOutputDTO>(Deployment.MarketplaceAddress, func);
return result.ReturnValue1;
}
private bool IsProofMissing(byte[] slotId, ulong period, ulong blockNumber)
private ProofState IsProofMissing(byte[] slotId, ulong period)
{
// In case of a missed proof, one of two things can be true:
// 1 - The proof was missed but no validator marked it as missing:
// We can see this by calling "canMarkProofAsMissing" and it returns true/doesn't throw.
// 2 - The proof was missed and it was marked as missing by a validator:
// We can see this by a successful call to "MarkProofAsMissing" on-chain.
if (CallCanMarkProofAsMissing(slotId, period))
{
return ProofState.MissedNotMarked;
}
if (WasMarkProofAsMissingCalled(slotId, period))
{
return ProofState.MissedAndMarked;
}
return ProofState.NotMissed;
}
private bool CallCanMarkProofAsMissing(byte[] slotId, ulong period)
{
try
{
@ -212,9 +229,9 @@ namespace CodexContractsPlugin
Period = period
};
var result = gethNode.Call<CanMarkProofAsMissingFunction, bool>(Deployment.MarketplaceAddress, func, blockNumber);
gethNode.Call<CanMarkProofAsMissingFunction>(Deployment.MarketplaceAddress, func);
var a = 0;
return true;
}
catch (AggregateException exc)
{
@ -227,7 +244,25 @@ namespace CodexContractsPlugin
}
throw;
}
return true;
}
private bool WasMarkProofAsMissingCalled(byte[] slotId, ulong period)
{
var now = DateTime.UtcNow;
var currentPeriod = new TimeRange(now - Deployment.Config.PeriodDuration, now);
var interval = gethNode.ConvertTimeRangeToBlockRange(currentPeriod);
var slot = slotId.ToHex().ToLowerInvariant();
var found = false;
gethNode.IterateFunctionCalls<MarkProofAsMissingFunction>(interval, (b, fn) =>
{
if (fn.Period == period && fn.SlotId.ToHex().ToLowerInvariant() == slot)
{
found = true;
}
});
return found;
}
private ContractInteractions StartInteraction()

4
Tests/CodexReleaseTests/Utils/ChainMonitor.cs
View File

@ -71,8 +71,8 @@ namespace CodexReleaseTests.Utils
if (reports.IsEmpty) return;
var slots = reports.Reports.Sum(r => Convert.ToInt32(r.TotalNumSlots));
var required = reports.Reports.Sum(r => Convert.ToInt32(r.ProofsRequired.Length));
var missed = reports.Reports.Sum(r => r.MissedProofs.Length);
var required = reports.Reports.Sum(r => Convert.ToInt32(r.PeriodProofs.Count(p => p.State != ProofState.NotRequired)));
var missed = reports.Reports.Sum(r => r.GetMissedProofs().Length);
log.Log($"Proof report: Slots={slots} Required={required} Missed={missed}");
}

6
Tools/TestNetRewarder/EventsFormatter.cs
View File

@ -150,14 +150,14 @@ namespace TestNetRewarder
private void AddMissedProofDetails(List<string> lines, PeriodReport[] reports)
{
var reportsWithMissedProofs = reports.Where(r => r.MissedProofs.Length > 0).ToArray();
var reportsWithMissedProofs = reports.Where(r => r.GetMissedProofs().Any()).ToArray();
if (reportsWithMissedProofs.Length < 1)
{
lines.Add($"No proofs were missed {emojiMaps.NoProofsMissed}");
return;
}
var totalMissed = reportsWithMissedProofs.Sum(r => r.MissedProofs.Length);
var totalMissed = reportsWithMissedProofs.Sum(r => r.GetMissedProofs().Length);
if (totalMissed > 10)
{
lines.Add($"[{totalMissed}] proofs were missed {emojiMaps.ManyProofsMissed}");
@ -172,7 +172,7 @@ namespace TestNetRewarder
private void DescribeMissedProof(List<string> lines, PeriodReport report)
{
foreach (var missedProof in report.MissedProofs)
foreach (var missedProof in report.GetMissedProofs())
{
DescribeMissedProof(lines, missedProof);
}