diff --git a/ProjectPlugins/CodexClient/MarketplaceAccess.cs b/ProjectPlugins/CodexClient/MarketplaceAccess.cs index f1c5fddb..7d61e041 100644 --- a/ProjectPlugins/CodexClient/MarketplaceAccess.cs +++ b/ProjectPlugins/CodexClient/MarketplaceAccess.cs @@ -46,6 +46,9 @@ namespace CodexClient Log($"Storage requested successfully. PurchaseId: '{response}'."); + var logName = $""; + log.AddStringReplace(response, logName); + log.AddStringReplace(response.ToLowerInvariant(), logName); return new StoragePurchaseContract(log, codexAccess, response, purchase, hooks); } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 1cca3f4e..04d8e0e2 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -1,6 +1,7 @@ using BlockchainUtils; using CodexContractsPlugin.Marketplace; using Logging; +using Nethereum.Contracts; using Nethereum.Hex.HexConvertors.Extensions; using System.Numerics; using Utils; @@ -92,17 +93,17 @@ namespace CodexContractsPlugin.ChainMonitor { var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray(); ApplyEvents(b, blockEvents, eventUtc); - UpdatePeriodMonitor(b, eventUtc); + UpdatePeriodMonitor(eventUtc); eventUtc += spanPerBlock; } } - private void UpdatePeriodMonitor(ulong blockNumber, DateTime eventUtc) + private void UpdatePeriodMonitor(DateTime eventUtc) { if (!doProofPeriodMonitoring) return; var activeRequests = requests.Where(r => r.State == RequestState.Started).ToArray(); - PeriodMonitor.Update(blockNumber, eventUtc, activeRequests); + PeriodMonitor.Update(eventUtc, activeRequests); } private void ApplyEvents(ulong blockNumber, IHasBlock[] blockEvents, DateTime eventsUtc) @@ -189,10 +190,31 @@ namespace CodexContractsPlugin.ChainMonitor private void ApplyEvent(ProofSubmittedEventDTO @event) { var id = Base58.Encode(@event.Id); - log.Log($"[{@event.Block.BlockNumber}] Proof submitted (id:{id})"); + + var proofOrigin = SearchForProofOrigin(id); + + log.Log($"[{@event.Block.BlockNumber}] Proof submitted (id:{id} {proofOrigin})"); handler.OnProofSubmitted(@event.Block, id); } + private string SearchForProofOrigin(string slotId) + { + foreach (var r in requests) + { + for (decimal slotIndex = 0; slotIndex < r.Request.Ask.Slots; slotIndex++) + { + var thisSlotId = contracts.GetSlotId(r.RequestId, slotIndex); + var id = Base58.Encode(thisSlotId); + + if (id.ToLowerInvariant() == slotId.ToLowerInvariant()) + { + return $"({r.RequestId.ToHex()} slotIndex:{slotIndex})"; + } + } + } + return "(Could not identify proof requestId + slot)"; + } + private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc) { foreach (var r in requests) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs index fe75e89a..d1e469a8 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs @@ -1,5 +1,6 @@ using Logging; using Nethereum.Hex.HexConvertors.Extensions; +using Nethereum.Model; using Utils; namespace CodexContractsPlugin.ChainMonitor @@ -17,7 +18,7 @@ namespace CodexContractsPlugin.ChainMonitor this.contracts = contracts; } - public void Update(ulong blockNumber, DateTime eventUtc, IChainStateRequest[] requests) + public void Update(DateTime eventUtc, IChainStateRequest[] requests) { var period = contracts.GetPeriodNumber(eventUtc); if (!currentPeriod.HasValue) @@ -27,7 +28,7 @@ namespace CodexContractsPlugin.ChainMonitor } if (period == currentPeriod.Value) return; - CreateReportForPeriod(blockNumber - 1, currentPeriod.Value, requests); + CreateReportForPeriod(currentPeriod.Value, requests); currentPeriod = period; } @@ -38,34 +39,25 @@ namespace CodexContractsPlugin.ChainMonitor return new PeriodMonitorResult(result); } - private void CreateReportForPeriod(ulong lastBlockInPeriod, ulong periodNumber, IChainStateRequest[] requests) + private void CreateReportForPeriod(ulong periodNumber, IChainStateRequest[] requests) { ulong total = 0; - var required = new List(); - var missed = new List(); + var periodProofs = new List(); foreach (var request in requests) { for (ulong slotIndex = 0; slotIndex < request.Request.Ask.Slots; slotIndex++) { - var state = contracts.GetProofState(request.RequestId, slotIndex, lastBlockInPeriod, periodNumber); + var state = contracts.GetProofState(request.RequestId, slotIndex, periodNumber); total++; - if (state.Required) - { - var idx = Convert.ToInt32(slotIndex); - var host = request.Hosts.GetHost(idx); - var proof = new PeriodProof(host, request, idx); - - required.Add(proof); - if (state.Missing) - { - missed.Add(proof); - } - } + var idx = Convert.ToInt32(slotIndex); + var host = request.Hosts.GetHost(idx); + var proof = new PeriodProof(host, request, idx, state); + periodProofs.Add(proof); } } - var report = new PeriodReport(periodNumber, total, required.ToArray(), missed.ToArray()); - log.Log($"Period report: {report}"); + var report = new PeriodReport(periodNumber, total, periodProofs.ToArray()); + report.Log(log); reports.Add(report); } } @@ -89,62 +81,77 @@ namespace CodexContractsPlugin.ChainMonitor private void CalcStats() { - IsEmpty = Reports.All(r => r.ProofsRequired.Length == 0); + IsEmpty = Reports.All(r => r.PeriodProofs.Length == 0); if (Reports.Length == 0) return; PeriodLow = Reports.Min(r => r.PeriodNumber); PeriodHigh = Reports.Max(r => r.PeriodNumber); AverageNumSlots = Reports.Average(r => Convert.ToSingle(r.TotalNumSlots)); - AverageNumProofsRequired = Reports.Average(r => Convert.ToSingle(r.ProofsRequired.Length)); + AverageNumProofsRequired = Reports.Average(r => Convert.ToSingle(r.PeriodProofs.Count(p => p.State != ProofState.NotRequired))); } } public class PeriodReport { - public PeriodReport(ulong periodNumber, ulong totalNumSlots, PeriodProof[] proofsRequired, PeriodProof[] missedProofs) + public PeriodReport(ulong periodNumber, ulong totalNumSlots, PeriodProof[] periodProofs) { PeriodNumber = periodNumber; TotalNumSlots = totalNumSlots; - ProofsRequired = proofsRequired; - MissedProofs = missedProofs; + PeriodProofs = periodProofs; } public ulong PeriodNumber { get; } public ulong TotalNumSlots { get; } - public PeriodProof[] ProofsRequired { get; } - public PeriodProof[] MissedProofs { get; } + public PeriodProof[] PeriodProofs { get; } - public override string ToString() + public PeriodProof[] GetMissedProofs() { - var required = Describe(ProofsRequired); - var missed = Describe(MissedProofs); - return $"Period:{PeriodNumber}=[Slots:{TotalNumSlots},ProofsRequired:{required},ProofsMissed:{missed}]"; + return PeriodProofs.Where(p => p.State == ProofState.MissedAndMarked || p.State == ProofState.MissedNotMarked).ToArray(); } - private string Describe(PeriodProof[] proofs) + public void Log(ILog log) { - if (proofs.Length == 0) return "None"; - return string.Join("+", proofs.Select(p => $"{p.FormatHost()} - {p.Request.RequestId.ToHex()} slot {p.SlotIndex}")); + log.Log($"Period report: {PeriodNumber}"); + log.Log($" - Slots: {TotalNumSlots}"); + foreach (var p in PeriodProofs) + { + log.Log($" - {p.Describe()}"); + } + } + + private void Log(ILog log, PeriodProof[] proofs) + { + if (proofs.Length == 0) return; + foreach (var p in proofs) + { + } } } public class PeriodProof { - public PeriodProof(EthAddress? host, IChainStateRequest request, int slotIndex) + public PeriodProof(EthAddress? host, IChainStateRequest request, int slotIndex, ProofState state) { Host = host; Request = request; SlotIndex = slotIndex; + State = state; } public EthAddress? Host { get; } public IChainStateRequest Request { get; } public int SlotIndex { get; } + public ProofState State { get; } public string FormatHost() { if (Host == null) return "Unknown host"; return Host.Address; } + + public string Describe() + { + return $"{FormatHost()} - {Request.RequestId.ToHex()} slotIndex:{SlotIndex} => {State}"; + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs index 8c089e0f..17a8c36c 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs @@ -4,6 +4,7 @@ using GethPlugin; using Logging; using Nethereum.ABI; using Nethereum.Contracts; +using Nethereum.Hex.HexConvertors.Extensions; using Nethereum.Util; using Newtonsoft.Json; using Newtonsoft.Json.Converters; @@ -29,21 +30,18 @@ namespace CodexContractsPlugin Request GetRequest(byte[] requestId); ulong GetPeriodNumber(DateTime utc); void WaitUntilNextPeriod(); - ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong blockNumber, ulong period); + ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong period); + byte[] GetSlotId(byte[] requestId, decimal slotIndex); ICodexContracts WithDifferentGeth(IGethNode node); } - public class ProofState + public enum ProofState { - public ProofState(bool required, bool missing) - { - Required = required; - Missing = missing; - } - - public bool Required { get; } - public bool Missing { get; } + NotRequired, + NotMissed, + MissedNotMarked, + MissedAndMarked, } [JsonConverter(typeof(StringEnumConverter))] @@ -165,15 +163,14 @@ namespace CodexContractsPlugin Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1)); } - public ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong blockNumber, ulong period) + public ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong period) { var slotId = GetSlotId(requestId, slotIndex); - var required = IsProofRequired(slotId, blockNumber); - if (!required) return new ProofState(false, false); + var required = IsProofRequired(slotId); + if (!required) return ProofState.NotRequired; - var missing = IsProofMissing(slotId, period, blockNumber); - return new ProofState(required, missing); + return IsProofMissing(slotId, period); } public ICodexContracts WithDifferentGeth(IGethNode node) @@ -181,7 +178,7 @@ namespace CodexContractsPlugin return new CodexContractsAccess(log, node, Deployment); } - private byte[] GetSlotId(byte[] requestId, decimal slotIndex) + public byte[] GetSlotId(byte[] requestId, decimal slotIndex) { var encoder = new ABIEncode(); var encoded = encoder.GetABIEncoded( @@ -192,17 +189,37 @@ namespace CodexContractsPlugin return Sha3Keccack.Current.CalculateHash(encoded); } - private bool IsProofRequired(byte[] slotId, ulong blockNumber) + private bool IsProofRequired(byte[] slotId) { var func = new IsProofRequiredFunction { Id = slotId }; - var result = gethNode.Call(Deployment.MarketplaceAddress, func, blockNumber); + var result = gethNode.Call(Deployment.MarketplaceAddress, func); return result.ReturnValue1; } - private bool IsProofMissing(byte[] slotId, ulong period, ulong blockNumber) + private ProofState IsProofMissing(byte[] slotId, ulong period) + { + // In case of a missed proof, one of two things can be true: + // 1 - The proof was missed but no validator marked it as missing: + // We can see this by calling "canMarkProofAsMissing" and it returns true/doesn't throw. + // 2 - The proof was missed and it was marked as missing by a validator: + // We can see this by a successful call to "MarkProofAsMissing" on-chain. + + if (CallCanMarkProofAsMissing(slotId, period)) + { + return ProofState.MissedNotMarked; + } + if (WasMarkProofAsMissingCalled(slotId, period)) + { + return ProofState.MissedAndMarked; + } + + return ProofState.NotMissed; + } + + private bool CallCanMarkProofAsMissing(byte[] slotId, ulong period) { try { @@ -212,9 +229,9 @@ namespace CodexContractsPlugin Period = period }; - var result = gethNode.Call(Deployment.MarketplaceAddress, func, blockNumber); + gethNode.Call(Deployment.MarketplaceAddress, func); - var a = 0; + return true; } catch (AggregateException exc) { @@ -227,7 +244,25 @@ namespace CodexContractsPlugin } throw; } - return true; + } + + private bool WasMarkProofAsMissingCalled(byte[] slotId, ulong period) + { + var now = DateTime.UtcNow; + var currentPeriod = new TimeRange(now - Deployment.Config.PeriodDuration, now); + var interval = gethNode.ConvertTimeRangeToBlockRange(currentPeriod); + var slot = slotId.ToHex().ToLowerInvariant(); + + var found = false; + gethNode.IterateFunctionCalls(interval, (b, fn) => + { + if (fn.Period == period && fn.SlotId.ToHex().ToLowerInvariant() == slot) + { + found = true; + } + }); + + return found; } private ContractInteractions StartInteraction() diff --git a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs index 7b229c9f..8723246b 100644 --- a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs +++ b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs @@ -71,8 +71,8 @@ namespace CodexReleaseTests.Utils if (reports.IsEmpty) return; var slots = reports.Reports.Sum(r => Convert.ToInt32(r.TotalNumSlots)); - var required = reports.Reports.Sum(r => Convert.ToInt32(r.ProofsRequired.Length)); - var missed = reports.Reports.Sum(r => r.MissedProofs.Length); + var required = reports.Reports.Sum(r => Convert.ToInt32(r.PeriodProofs.Count(p => p.State != ProofState.NotRequired))); + var missed = reports.Reports.Sum(r => r.GetMissedProofs().Length); log.Log($"Proof report: Slots={slots} Required={required} Missed={missed}"); } diff --git a/Tools/TestNetRewarder/EventsFormatter.cs b/Tools/TestNetRewarder/EventsFormatter.cs index a7b6c8aa..3b1650cb 100644 --- a/Tools/TestNetRewarder/EventsFormatter.cs +++ b/Tools/TestNetRewarder/EventsFormatter.cs @@ -150,14 +150,14 @@ namespace TestNetRewarder private void AddMissedProofDetails(List lines, PeriodReport[] reports) { - var reportsWithMissedProofs = reports.Where(r => r.MissedProofs.Length > 0).ToArray(); + var reportsWithMissedProofs = reports.Where(r => r.GetMissedProofs().Any()).ToArray(); if (reportsWithMissedProofs.Length < 1) { lines.Add($"No proofs were missed {emojiMaps.NoProofsMissed}"); return; } - var totalMissed = reportsWithMissedProofs.Sum(r => r.MissedProofs.Length); + var totalMissed = reportsWithMissedProofs.Sum(r => r.GetMissedProofs().Length); if (totalMissed > 10) { lines.Add($"[{totalMissed}] proofs were missed {emojiMaps.ManyProofsMissed}"); @@ -172,7 +172,7 @@ namespace TestNetRewarder private void DescribeMissedProof(List lines, PeriodReport report) { - foreach (var missedProof in report.MissedProofs) + foreach (var missedProof in report.GetMissedProofs()) { DescribeMissedProof(lines, missedProof); }