diff --git a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs index 0e91dbc9..2e22dd61 100644 --- a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs @@ -1,4 +1,5 @@ -using Utils; +using System.Globalization; +using Utils; namespace KubernetesWorkflow.Recipe { @@ -88,6 +89,11 @@ namespace KubernetesWorkflow.Recipe envVars.Add(factory.CreateEnvVar(name, value)); } + protected void AddEnvVar(string name, int value) + { + envVars.Add(factory.CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture))); + } + protected void AddEnvVar(string name, Port value) { envVars.Add(factory.CreateEnvVar(name, value.Number)); diff --git a/Framework/Utils/TimeRange.cs b/Framework/Utils/TimeRange.cs index 35578d17..3812565b 100644 --- a/Framework/Utils/TimeRange.cs +++ b/Framework/Utils/TimeRange.cs @@ -20,5 +20,10 @@ public DateTime From { get; } public DateTime To { get; } public TimeSpan Duration { get; } + + public override string ToString() + { + return $"{Time.FormatTimestamp(From)} -> {Time.FormatTimestamp(To)} ({Time.FormatDuration(Duration)})"; + } } } diff --git a/ProjectPlugins/CodexClient/TransferSpeeds.cs b/ProjectPlugins/CodexClient/TransferSpeeds.cs index 9e013efb..bb6afeed 100644 --- a/ProjectPlugins/CodexClient/TransferSpeeds.cs +++ b/ProjectPlugins/CodexClient/TransferSpeeds.cs @@ -6,6 +6,7 @@ namespace CodexClient { BytesPerSecond? GetUploadSpeed(); BytesPerSecond? GetDownloadSpeed(); + ITransferSpeeds Combine(ITransferSpeeds? other); } public class TransferSpeeds : ITransferSpeeds @@ -35,6 +36,18 @@ namespace CodexClient return downloads.Average(); } + public ITransferSpeeds Combine(ITransferSpeeds? other) + { + if (other == null) return this; + var o = (TransferSpeeds)other; + var result = new TransferSpeeds(); + result.uploads.AddRange(uploads); + result.uploads.AddRange(o.uploads); + result.downloads.AddRange(downloads); + result.downloads.AddRange(o.downloads); + return result; + } + private static BytesPerSecond Convert(ByteSize size, TimeSpan duration) { double bytes = size.SizeInBytes; diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 04d8e0e2..e32a593d 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -1,5 +1,6 @@ using BlockchainUtils; using CodexContractsPlugin.Marketplace; +using GethPlugin; using Logging; using Nethereum.Contracts; using Nethereum.Hex.HexConvertors.Extensions; @@ -42,14 +43,14 @@ namespace CodexContractsPlugin.ChainMonitor private readonly IChainStateChangeHandler handler; private readonly bool doProofPeriodMonitoring; - public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring) + public ChainState(ILog log, IGethNode geth, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring, IPeriodMonitorEventHandler periodEventHandler) { this.log = new LogPrefixer(log, "(ChainState) "); this.contracts = contracts; handler = changeHandler; this.doProofPeriodMonitoring = doProofPeriodMonitoring; TotalSpan = new TimeRange(startUtc, startUtc); - PeriodMonitor = new PeriodMonitor(log, contracts); + PeriodMonitor = new PeriodMonitor(log, contracts, geth, periodEventHandler); } public TimeRange TotalSpan { get; private set; } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs index d1e469a8..825db0d4 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs @@ -1,35 +1,49 @@ -using Logging; +using CodexContractsPlugin.Marketplace; +using GethPlugin; +using Logging; +using Nethereum.Contracts; using Nethereum.Hex.HexConvertors.Extensions; using Nethereum.Model; -using Utils; +using Nethereum.RPC.Eth.DTOs; +using Newtonsoft.Json; +using System.Reflection; namespace CodexContractsPlugin.ChainMonitor { + public interface IPeriodMonitorEventHandler + { + void OnPeriodReport(PeriodReport report); + } + public class PeriodMonitor { private readonly ILog log; private readonly ICodexContracts contracts; + private readonly IGethNode geth; + private readonly IPeriodMonitorEventHandler eventHandler; private readonly List reports = new List(); - private ulong? currentPeriod = null; + private CurrentPeriod? currentPeriod = null; - public PeriodMonitor(ILog log, ICodexContracts contracts) + public PeriodMonitor(ILog log, ICodexContracts contracts, IGethNode geth, IPeriodMonitorEventHandler eventHandler) { this.log = log; this.contracts = contracts; + this.geth = geth; + this.eventHandler = eventHandler; } public void Update(DateTime eventUtc, IChainStateRequest[] requests) { - var period = contracts.GetPeriodNumber(eventUtc); - if (!currentPeriod.HasValue) + var periodNumber = contracts.GetPeriodNumber(eventUtc); + if (currentPeriod == null) { - currentPeriod = period; + currentPeriod = CreateCurrentPeriod(periodNumber, requests); return; } - if (period == currentPeriod.Value) return; + if (periodNumber == currentPeriod.PeriodNumber) return; - CreateReportForPeriod(currentPeriod.Value, requests); - currentPeriod = period; + CreateReportForPeriod(currentPeriod, requests); + currentPeriod = CreateCurrentPeriod(periodNumber, requests); } public PeriodMonitorResult GetAndClearReports() @@ -39,26 +53,127 @@ namespace CodexContractsPlugin.ChainMonitor return new PeriodMonitorResult(result); } - private void CreateReportForPeriod(ulong periodNumber, IChainStateRequest[] requests) + private CurrentPeriod CreateCurrentPeriod(ulong periodNumber, IChainStateRequest[] requests) + { + var result = new CurrentPeriod(periodNumber); + ForEachActiveSlot(requests, (request, slotIndex) => + { + if (contracts.IsProofRequired(request.RequestId, slotIndex) || + contracts.WillProofBeRequired(request.RequestId, slotIndex)) + { + var idx = Convert.ToInt32(slotIndex); + var host = request.Hosts.GetHost(idx); + var slotId = contracts.GetSlotId(request.RequestId, slotIndex); + if (host != null) + { + result.RequiredProofs.Add(new PeriodRequiredProof(host, request, idx, slotId)); + } + } + }); + + return result; + } + + private void CreateReportForPeriod(CurrentPeriod currentPeriod, IChainStateRequest[] requests) + { + // Fetch function calls during period. Format report. + var timeRange = contracts.GetPeriodTimeRange(currentPeriod.PeriodNumber); + var blockRange = geth.ConvertTimeRangeToBlockRange(timeRange); + + var callReports = new List(); + geth.IterateTransactions(blockRange, (t, blkI, blkUtc) => + { + var reporter = new CallReporter(callReports, t, blkUtc, blkI); + reporter.Run(); + + }); + + var report = new PeriodReport( + new ProofPeriod(currentPeriod.PeriodNumber, timeRange, blockRange), + currentPeriod.RequiredProofs.ToArray(), + callReports.ToArray()); + + report.Log(log); + reports.Add(report); + + eventHandler.OnPeriodReport(report); + } + + private void ForEachActiveSlot(IChainStateRequest[] requests, Action action) { - ulong total = 0; - var periodProofs = new List(); foreach (var request in requests) { for (ulong slotIndex = 0; slotIndex < request.Request.Ask.Slots; slotIndex++) { - var state = contracts.GetProofState(request.RequestId, slotIndex, periodNumber); - - total++; - var idx = Convert.ToInt32(slotIndex); - var host = request.Hosts.GetHost(idx); - var proof = new PeriodProof(host, request, idx, state); - periodProofs.Add(proof); + action(request, slotIndex); } } - var report = new PeriodReport(periodNumber, total, periodProofs.ToArray()); - report.Log(log); - reports.Add(report); + } + } + + public class DoNothingPeriodMonitorEventHandler : IPeriodMonitorEventHandler + { + public void OnPeriodReport(PeriodReport report) + { + } + } + + public class CallReporter + { + private readonly List reports; + private readonly Transaction t; + private readonly DateTime blockUtc; + private readonly ulong blockNumber; + + public CallReporter(List reports, Transaction t, DateTime blockUtc, ulong blockNumber) + { + this.reports = reports; + this.t = t; + this.blockUtc = blockUtc; + this.blockNumber = blockNumber; + } + + public void Run() + { + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + CreateFunctionCallReport(); + } + + private void CreateFunctionCallReport() where TFunc : FunctionMessage, new() + { + if (t.IsTransactionForFunctionMessage()) + { + var func = t.DecodeTransactionToFunctionMessage(); + + reports.Add(new FunctionCallReport(blockUtc, blockNumber, typeof(TFunc).Name, JsonConvert.SerializeObject(func))); + } } } @@ -67,91 +182,19 @@ namespace CodexContractsPlugin.ChainMonitor public PeriodMonitorResult(PeriodReport[] reports) { Reports = reports; - - CalcStats(); } public PeriodReport[] Reports { get; } - - public bool IsEmpty { get; private set; } - public ulong PeriodLow { get; private set; } - public ulong PeriodHigh { get; private set; } - public float AverageNumSlots { get; private set; } - public float AverageNumProofsRequired { get; private set; } - - private void CalcStats() - { - IsEmpty = Reports.All(r => r.PeriodProofs.Length == 0); - if (Reports.Length == 0) return; - - PeriodLow = Reports.Min(r => r.PeriodNumber); - PeriodHigh = Reports.Max(r => r.PeriodNumber); - AverageNumSlots = Reports.Average(r => Convert.ToSingle(r.TotalNumSlots)); - AverageNumProofsRequired = Reports.Average(r => Convert.ToSingle(r.PeriodProofs.Count(p => p.State != ProofState.NotRequired))); - } } - public class PeriodReport + public class CurrentPeriod { - public PeriodReport(ulong periodNumber, ulong totalNumSlots, PeriodProof[] periodProofs) + public CurrentPeriod(ulong periodNumber) { PeriodNumber = periodNumber; - TotalNumSlots = totalNumSlots; - PeriodProofs = periodProofs; } public ulong PeriodNumber { get; } - public ulong TotalNumSlots { get; } - public PeriodProof[] PeriodProofs { get; } - - public PeriodProof[] GetMissedProofs() - { - return PeriodProofs.Where(p => p.State == ProofState.MissedAndMarked || p.State == ProofState.MissedNotMarked).ToArray(); - } - - public void Log(ILog log) - { - log.Log($"Period report: {PeriodNumber}"); - log.Log($" - Slots: {TotalNumSlots}"); - foreach (var p in PeriodProofs) - { - log.Log($" - {p.Describe()}"); - } - } - - private void Log(ILog log, PeriodProof[] proofs) - { - if (proofs.Length == 0) return; - foreach (var p in proofs) - { - } - } - } - - public class PeriodProof - { - public PeriodProof(EthAddress? host, IChainStateRequest request, int slotIndex, ProofState state) - { - Host = host; - Request = request; - SlotIndex = slotIndex; - State = state; - } - - public EthAddress? Host { get; } - public IChainStateRequest Request { get; } - public int SlotIndex { get; } - public ProofState State { get; } - - public string FormatHost() - { - if (Host == null) return "Unknown host"; - return Host.Address; - } - - public string Describe() - { - return $"{FormatHost()} - {Request.RequestId.ToHex()} slotIndex:{SlotIndex} => {State}"; - } + public List RequiredProofs { get; } = new List(); } } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs new file mode 100644 index 00000000..fd080820 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs @@ -0,0 +1,55 @@ +using Logging; +using Utils; + +namespace CodexContractsPlugin.ChainMonitor +{ + public class PeriodReport + { + public PeriodReport(ProofPeriod period, PeriodRequiredProof[] required, FunctionCallReport[] functionCalls) + { + Period = period; + Required = required; + FunctionCalls = functionCalls; + } + + public ProofPeriod Period { get; } + public PeriodRequiredProof[] Required { get; } + public FunctionCallReport[] FunctionCalls { get; } + + public void Log(ILog log) + { + log.Log($"Period report: {Period}"); + log.Log($" - Proofs required: {Required.Length}"); + foreach (var r in Required) + { + log.Log($" - {r.Describe()}"); + } + log.Log($" - Calls: {FunctionCalls.Length}"); + foreach (var f in FunctionCalls) + { + log.Log($" - {f.Describe()}"); + } + } + } + + public class FunctionCallReport + { + public FunctionCallReport(DateTime utc, ulong blockNumber, string name, string payload) + { + Utc = utc; + BlockNumber = blockNumber; + Name = name; + Payload = payload; + } + + public DateTime Utc { get; } + public ulong BlockNumber { get; } + public string Name { get; } + public string Payload { get; } + + public string Describe() + { + return $"[{Time.FormatTimestamp(Utc)}][{BlockNumber}] {Name} = \"{Payload}\""; + } + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs new file mode 100644 index 00000000..5b560081 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs @@ -0,0 +1,26 @@ +using Nethereum.Hex.HexConvertors.Extensions; +using Utils; + +namespace CodexContractsPlugin.ChainMonitor +{ + public class PeriodRequiredProof + { + public PeriodRequiredProof(EthAddress host, IChainStateRequest request, int slotIndex, byte[] slotId) + { + Host = host; + Request = request; + SlotIndex = slotIndex; + SlotId = slotId; + } + + public EthAddress Host { get; } + public IChainStateRequest Request { get; } + public int SlotIndex { get; } + public byte[] SlotId { get; } + + public string Describe() + { + return $"{Request.RequestId.ToHex()} slotId:{SlotId.ToHex()} slotIndex:{SlotIndex} by {Host}"; + } + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs new file mode 100644 index 00000000..25ad5cc8 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ProofPeriod.cs @@ -0,0 +1,23 @@ +using Utils; + +namespace CodexContractsPlugin.ChainMonitor +{ + public class ProofPeriod + { + public ProofPeriod(ulong periodNumber, TimeRange timeRange, BlockInterval blockRange) + { + PeriodNumber = periodNumber; + TimeRange = timeRange; + BlockRange = blockRange; + } + + public ulong PeriodNumber { get; } + public TimeRange TimeRange { get; } + public BlockInterval BlockRange { get; } + + public override string ToString() + { + return $"{{{PeriodNumber} - {TimeRange} {BlockRange}}}"; + } + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs index 17a8c36c..0c952b1d 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs @@ -29,21 +29,15 @@ namespace CodexContractsPlugin RequestState GetRequestState(byte[] requestId); Request GetRequest(byte[] requestId); ulong GetPeriodNumber(DateTime utc); + TimeRange GetPeriodTimeRange(ulong periodNumber); void WaitUntilNextPeriod(); - ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong period); + bool IsProofRequired(byte[] requestId, decimal slotIndex); + bool WillProofBeRequired(byte[] requestId, decimal slotIndex); byte[] GetSlotId(byte[] requestId, decimal slotIndex); ICodexContracts WithDifferentGeth(IGethNode node); } - public enum ProofState - { - NotRequired, - NotMissed, - MissedNotMarked, - MissedAndMarked, - } - [JsonConverter(typeof(StringEnumConverter))] public enum RequestState { @@ -155,6 +149,16 @@ namespace CodexContractsPlugin return Convert.ToUInt64(result); } + public TimeRange GetPeriodTimeRange(ulong periodNumber) + { + var periodSeconds = (ulong)Deployment.Config.Proofs.Period; + var startUtco = Convert.ToInt64(periodSeconds * periodNumber); + var endUtco = Convert.ToInt64(periodSeconds * (periodNumber + 1)); + var start = DateTimeOffset.FromUnixTimeSeconds(startUtco).UtcDateTime; + var end = DateTimeOffset.FromUnixTimeSeconds(endUtco).UtcDateTime; + return new TimeRange(start, end); + } + public void WaitUntilNextPeriod() { var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); @@ -163,14 +167,16 @@ namespace CodexContractsPlugin Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1)); } - public ProofState GetProofState(byte[] requestId, decimal slotIndex, ulong period) + public bool IsProofRequired(byte[] requestId, decimal slotIndex) { var slotId = GetSlotId(requestId, slotIndex); + return IsProofRequired(slotId); + } - var required = IsProofRequired(slotId); - if (!required) return ProofState.NotRequired; - - return IsProofMissing(slotId, period); + public bool WillProofBeRequired(byte[] requestId, decimal slotIndex) + { + var slotId = GetSlotId(requestId, slotIndex); + return WillProofBeRequired(slotId); } public ICodexContracts WithDifferentGeth(IGethNode node) @@ -199,70 +205,14 @@ namespace CodexContractsPlugin return result.ReturnValue1; } - private ProofState IsProofMissing(byte[] slotId, ulong period) + private bool WillProofBeRequired(byte[] slotId) { - // In case of a missed proof, one of two things can be true: - // 1 - The proof was missed but no validator marked it as missing: - // We can see this by calling "canMarkProofAsMissing" and it returns true/doesn't throw. - // 2 - The proof was missed and it was marked as missing by a validator: - // We can see this by a successful call to "MarkProofAsMissing" on-chain. - - if (CallCanMarkProofAsMissing(slotId, period)) + var func = new WillProofBeRequiredFunction { - return ProofState.MissedNotMarked; - } - if (WasMarkProofAsMissingCalled(slotId, period)) - { - return ProofState.MissedAndMarked; - } - - return ProofState.NotMissed; - } - - private bool CallCanMarkProofAsMissing(byte[] slotId, ulong period) - { - try - { - var func = new CanMarkProofAsMissingFunction - { - SlotId = slotId, - Period = period - }; - - gethNode.Call(Deployment.MarketplaceAddress, func); - - return true; - } - catch (AggregateException exc) - { - if (exc.InnerExceptions.Count == 1) - { - if (exc.InnerExceptions[0].GetType() == typeof(SmartContractCustomErrorRevertException)) - { - return false; - } - } - throw; - } - } - - private bool WasMarkProofAsMissingCalled(byte[] slotId, ulong period) - { - var now = DateTime.UtcNow; - var currentPeriod = new TimeRange(now - Deployment.Config.PeriodDuration, now); - var interval = gethNode.ConvertTimeRangeToBlockRange(currentPeriod); - var slot = slotId.ToHex().ToLowerInvariant(); - - var found = false; - gethNode.IterateFunctionCalls(interval, (b, fn) => - { - if (fn.Period == period && fn.SlotId.ToHex().ToLowerInvariant() == slot) - { - found = true; - } - }); - - return found; + Id = slotId + }; + var result = gethNode.Call(Deployment.MarketplaceAddress, func); + return result.ReturnValue1; } private ContractInteractions StartInteraction() diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs index 09c5b829..4b5dc172 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs @@ -9,6 +9,11 @@ namespace CodexContractsPlugin { public const string DeployedAddressesFilename = "/hardhat/ignition/deployments/chain-789988/deployed_addresses.json"; public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json"; + + public const int PeriodSeconds = 60; + public const int TimeoutSeconds = 30; + public const int DowntimeSeconds = 128; + private readonly DebugInfoVersion versionInfo; public override string AppName => "codex-contracts"; @@ -28,6 +33,22 @@ namespace CodexContractsPlugin SetSchedulingAffinity(notIn: "false"); AddEnvVar("DISTTEST_NETWORK_URL", address.ToString()); + + // Default values: + AddEnvVar("DISTTEST_REPAIRREWARD", 10); + AddEnvVar("DISTTEST_MAXSLASHES", 2); + AddEnvVar("DISTTEST_SLASHPERCENTAGE", 20); + AddEnvVar("DISTTEST_VALIDATORREWARD", 20); + AddEnvVar("DISTTEST_DOWNTIMEPRODUCT", 67); + AddEnvVar("DISTTEST_MAXRESERVATIONS", 3); + AddEnvVar("DISTTEST_MAXDURATION", Convert.ToInt32(TimeSpan.FromDays(30).TotalSeconds)); + + // Customized values, required to operate in a network with + // block frequency of 1. + AddEnvVar("DISTTEST_PERIOD", PeriodSeconds); + AddEnvVar("DISTTEST_TIMEOUT", TimeoutSeconds); + AddEnvVar("DISTTEST_DOWNTIME", DowntimeSeconds); + AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork"); AddEnvVar("HARDHAT_IGNITION_CONFIRM_DEPLOYMENT", "false"); AddEnvVar("KEEP_ALIVE", "1"); diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index 63032158..aeac9c21 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -92,12 +92,31 @@ namespace CodexContractsPlugin Log("Synced. Codex SmartContracts deployed. Getting configuration..."); var config = GetMarketplaceConfiguration(marketplaceAddress, gethNode); - Log("Got config: " + JsonConvert.SerializeObject(config)); + ConfigShouldEqual(config.Proofs.Period, CodexContractsContainerRecipe.PeriodSeconds, "Period"); + ConfigShouldEqual(config.Proofs.Timeout, CodexContractsContainerRecipe.TimeoutSeconds, "Timeout"); + ConfigShouldEqual(config.Proofs.Downtime, CodexContractsContainerRecipe.DowntimeSeconds, "Downtime"); + return new CodexContractsDeployment(config, marketplaceAddress, abi, tokenAddress); } + private void ConfigShouldEqual(ulong value, int expected, string name) + { + if (Convert.ToInt32(value) != expected) + { + // Merge todo: https://github.com/codex-storage/nim-codex/pull/1303 + // Once this is merged, the contract config values are settable via env-vars. + // This plugin is already updated to set the config vars to values compatible with a + // 1-second block frequency. AND it will read back the config and assert it is deployed correctly. + // This is waiting for that merge. + + // Replace log with assert WHEN MERGED: + // throw new Exception($"Config value '{name}' should be deployed as '{expected}' but was '{value}'"); + Log($"MERGE TODO. Config value '{name}' should be deployed as '{expected}' but was '{value}'"); + } + } + private MarketplaceConfig GetMarketplaceConfiguration(string marketplaceAddress, IGethNode gethNode) { var func = new ConfigurationFunctionBase(); diff --git a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs index 30948506..e5c8db73 100644 --- a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs +++ b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs @@ -85,8 +85,11 @@ namespace CodexPlugin { "JSONRPC-WS-CLIENT", "JSONRPC-HTTP-CLIENT", - "codex", - "repostore" + }; + + var alwaysIgnoreTopics = new [] + { + "JSONRPC-CLIENT" }; level = $"{level};" + @@ -94,7 +97,8 @@ namespace CodexPlugin $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" + $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)};" + $"{CustomTopics.JsonSerialize.ToString().ToLowerInvariant()}:{string.Join(",", jsonSerializeTopics)};" + - $"{CustomTopics.MarketplaceInfra.ToString().ToLowerInvariant()}:{string.Join(",", marketplaceInfraTopics)}"; + $"{CustomTopics.MarketplaceInfra.ToString().ToLowerInvariant()}:{string.Join(",", marketplaceInfraTopics)};" + + $"{CodexLogLevel.Error.ToString()}:{string.Join(",", alwaysIgnoreTopics)}"; if (CustomTopics.BlockExchange != null) { diff --git a/ProjectPlugins/GethPlugin/GethNode.cs b/ProjectPlugins/GethPlugin/GethNode.cs index 15fa0d4e..b8c1e125 100644 --- a/ProjectPlugins/GethPlugin/GethNode.cs +++ b/ProjectPlugins/GethPlugin/GethNode.cs @@ -3,6 +3,7 @@ using Core; using KubernetesWorkflow.Types; using Logging; using Nethereum.ABI.FunctionEncoding.Attributes; +using Nethereum.BlockchainProcessing.BlockStorage.Entities.Mapping; using Nethereum.Contracts; using Nethereum.RPC.Eth.DTOs; using NethereumWorkflow; @@ -33,6 +34,7 @@ namespace GethPlugin List> GetEvents(string address, TimeRange timeRange) where TEvent : IEventDTO, new(); BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange); BlockTimeEntry GetBlockForNumber(ulong number); + void IterateTransactions(BlockInterval blockRange, Action action); void IterateFunctionCalls(BlockInterval blockInterval, Action onCall) where TFunc : FunctionMessage, new(); IGethNode WithDifferentAccount(EthAccount account); } @@ -223,28 +225,37 @@ namespace GethPlugin return StartInteraction().GetBlockWithTransactions(number); } - public void IterateFunctionCalls(BlockInterval blockRange, Action onCall) where TFunc : FunctionMessage, new() + public void IterateTransactions(BlockInterval blockRange, Action action) { var i = StartInteraction(); for (var blkI = blockRange.From; blkI <= blockRange.To; blkI++) { var blk = i.GetBlockWithTransactions(blkI); + var blkUtc = DateTimeOffset.FromUnixTimeSeconds(blk.Timestamp.ToLong()).UtcDateTime; foreach (var t in blk.Transactions) { - if (t.IsTransactionForFunctionMessage()) - { - var func = t.DecodeTransactionToFunctionMessage(); - if (func != null) - { - var b = GetBlockForNumber(blkI); - onCall(b, func); - } - } + action(t, blkI, blkUtc); } } } + public void IterateFunctionCalls(BlockInterval blockRange, Action onCall) where TFunc : FunctionMessage, new() + { + IterateTransactions(blockRange, (t, blkI, blkUtc) => + { + if (t.IsTransactionForFunctionMessage()) + { + var func = t.DecodeTransactionToFunctionMessage(); + if (func != null) + { + var b = GetBlockForNumber(blkI); + onCall(b, func); + } + } + }); + } + protected abstract NethereumInteraction StartInteraction(); } } diff --git a/Tests/CodexReleaseTests/DataTests/SwarmTest.cs b/Tests/CodexReleaseTests/DataTests/SwarmTest.cs index f4099ec0..2379fb08 100644 --- a/Tests/CodexReleaseTests/DataTests/SwarmTest.cs +++ b/Tests/CodexReleaseTests/DataTests/SwarmTest.cs @@ -7,175 +7,191 @@ using Utils; namespace CodexReleaseTests.DataTests { - [TestFixture(2, 10)] - [TestFixture(5, 20)] - [TestFixture(10, 20)] - public class SwarmTests : AutoBootstrapDistTest + namespace SwarmTests { - private readonly int numberOfNodes; - private readonly int filesizeMb; - - public SwarmTests(int numberOfNodes, int filesizeMb) + [TestFixture(2, 10)] + [TestFixture(5, 20)] + [TestFixture(10, 20)] + public class SwarmTests : AutoBootstrapDistTest { - this.numberOfNodes = numberOfNodes; - this.filesizeMb = filesizeMb; - } + private readonly int numberOfNodes; + private readonly int filesizeMb; + private ICodexNodeGroup nodes = null!; - [Test] - public void Swarm() - { - var filesize = filesizeMb.MB(); - var nodes = StartCodex(numberOfNodes); - var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); - - var tasks = ParallelDownloadEachFile(nodes, files); - Task.WaitAll(tasks); - - AssertAllFilesDownloadedCorrectly(files); - } - - [Test] - public void StreamlessSwarm() - { - var filesize = filesizeMb.MB(); - var nodes = StartCodex(numberOfNodes); - var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); - - var tasks = ParallelStreamlessDownloadEachFile(nodes, files); - Task.WaitAll(tasks); - - AssertAllFilesStreamlesslyDownloadedCorrectly(nodes, files); - } - - private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, ByteSize fileSize) - { - var file = GenerateTestFile(fileSize); - var cid = node.UploadFile(file); - return new SwarmTestNetworkFile(node, fileSize, file, cid); - } - - private Task[] ParallelDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files) - { - var tasks = new List(); - - foreach (var node in nodes) + public SwarmTests(int numberOfNodes, int filesizeMb) { - tasks.Add(StartDownload(node, files)); + this.numberOfNodes = numberOfNodes; + this.filesizeMb = filesizeMb; } - return tasks.ToArray(); - } - - private Task[] ParallelStreamlessDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files) - { - var tasks = new List(); - - foreach (var node in nodes) + [TearDown] + public void TearDown() { - tasks.Add(StartStreamlessDownload(node, files)); - } - - return tasks.ToArray(); - } - - private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files) - { - return Task.Run(() => - { - var remaining = files.ToList(); - - while (remaining.Count > 0) + ITransferSpeeds speeds = new TransferSpeeds(); + foreach (var n in nodes) { - var file = remaining.PickOneRandom(); - try - { - var dl = node.DownloadContent(file.Cid, TimeSpan.FromMinutes(30)); - lock (file.Lock) - { - file.Downloaded.Add(dl); - } - } - catch (Exception ex) - { - file.Error = ex; - } + speeds = speeds.Combine(n.TransferSpeeds); } - }); - } + Log($"Average upload speed: {speeds.GetUploadSpeed()}"); + Log($"Average download speed: {speeds.GetDownloadSpeed()}"); + } - private Task StartStreamlessDownload(ICodexNode node, SwarmTestNetworkFile[] files) - { - return Task.Run(() => + [Test] + public void Stream() { - var remaining = files.ToList(); + var filesize = filesizeMb.MB(); + nodes = StartCodex(numberOfNodes); + var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); - while (remaining.Count > 0) + var tasks = ParallelDownloadEachFile(files); + Task.WaitAll(tasks); + + AssertAllFilesDownloadedCorrectly(files); + } + + [Test] + public void Streamless() + { + var filesize = filesizeMb.MB(); + nodes = StartCodex(numberOfNodes); + var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); + + var tasks = ParallelStreamlessDownloadEachFile(files); + Task.WaitAll(tasks); + + AssertAllFilesStreamlesslyDownloadedCorrectly(files); + } + + private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, ByteSize fileSize) + { + var file = GenerateTestFile(fileSize); + var cid = node.UploadFile(file); + return new SwarmTestNetworkFile(node, fileSize, file, cid); + } + + private Task[] ParallelDownloadEachFile(SwarmTestNetworkFile[] files) + { + var tasks = new List(); + + foreach (var node in nodes) { - var file = remaining.PickOneRandom(); - if (file.Uploader.GetName() != node.GetName()) + tasks.Add(StartDownload(node, files)); + } + + return tasks.ToArray(); + } + + private Task[] ParallelStreamlessDownloadEachFile(SwarmTestNetworkFile[] files) + { + var tasks = new List(); + + foreach (var node in nodes) + { + tasks.Add(StartStreamlessDownload(node, files)); + } + + return tasks.ToArray(); + } + + private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files) + { + return Task.Run(() => + { + var remaining = files.ToList(); + + while (remaining.Count > 0) { + var file = remaining.PickOneRandom(); try { - var startSpace = node.Space(); - node.DownloadStreamlessWait(file.Cid, file.OriginalSize); + var dl = node.DownloadContent(file.Cid, TimeSpan.FromMinutes(30)); + lock (file.Lock) + { + file.Downloaded.Add(dl); + } } catch (Exception ex) { file.Error = ex; } } - } - }); - } + }); + } - private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files) - { - foreach (var file in files) + private Task StartStreamlessDownload(ICodexNode node, SwarmTestNetworkFile[] files) { - if (file.Error != null) throw file.Error; - lock (file.Lock) + return Task.Run(() => { - foreach (var dl in file.Downloaded) + var remaining = files.ToList(); + + while (remaining.Count > 0) { - file.Original.AssertIsEqual(dl); + var file = remaining.PickOneRandom(); + if (file.Uploader.GetName() != node.GetName()) + { + try + { + var startSpace = node.Space(); + node.DownloadStreamlessWait(file.Cid, file.OriginalSize); + } + catch (Exception ex) + { + file.Error = ex; + } + } + } + }); + } + + private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files) + { + foreach (var file in files) + { + if (file.Error != null) throw file.Error; + lock (file.Lock) + { + foreach (var dl in file.Downloaded) + { + file.Original.AssertIsEqual(dl); + } } } } - } - private void AssertAllFilesStreamlesslyDownloadedCorrectly(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files) - { - var totalFilesSpace = 0.Bytes(); - foreach (var file in files) + private void AssertAllFilesStreamlesslyDownloadedCorrectly(SwarmTestNetworkFile[] files) { - if (file.Error != null) throw file.Error; - totalFilesSpace = new ByteSize(totalFilesSpace.SizeInBytes + file.Original.GetFilesize().SizeInBytes); - } - - foreach (var node in nodes) - { - var currentSpace = node.Space(); - Assert.That(currentSpace.QuotaUsedBytes, Is.GreaterThanOrEqualTo(totalFilesSpace.SizeInBytes)); - } - } + var totalFilesSpace = 0.Bytes(); + foreach (var file in files) + { + if (file.Error != null) throw file.Error; + totalFilesSpace = new ByteSize(totalFilesSpace.SizeInBytes + file.Original.GetFilesize().SizeInBytes); + } - private class SwarmTestNetworkFile - { - public SwarmTestNetworkFile(ICodexNode uploader, ByteSize originalSize, TrackedFile original, ContentId cid) - { - Uploader = uploader; - OriginalSize = originalSize; - Original = original; - Cid = cid; + foreach (var node in nodes) + { + var currentSpace = node.Space(); + Assert.That(currentSpace.QuotaUsedBytes, Is.GreaterThanOrEqualTo(totalFilesSpace.SizeInBytes)); + } } - public ICodexNode Uploader { get; } - public ByteSize OriginalSize { get; } - public TrackedFile Original { get; } - public ContentId Cid { get; } - public object Lock { get; } = new object(); - public List Downloaded { get; } = new List(); - public Exception? Error { get; set; } = null; + private class SwarmTestNetworkFile + { + public SwarmTestNetworkFile(ICodexNode uploader, ByteSize originalSize, TrackedFile original, ContentId cid) + { + Uploader = uploader; + OriginalSize = originalSize; + Original = original; + Cid = cid; + } + + public ICodexNode Uploader { get; } + public ByteSize OriginalSize { get; } + public TrackedFile Original { get; } + public ContentId Cid { get; } + public object Lock { get; } = new object(); + public List Downloaded { get; } = new List(); + public Exception? Error { get; set; } = null; + } } } } diff --git a/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs b/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs new file mode 100644 index 00000000..95f55fbd --- /dev/null +++ b/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs @@ -0,0 +1,99 @@ +using CodexClient; +using CodexContractsPlugin.ChainMonitor; +using CodexReleaseTests.Utils; +using Nethereum.Hex.HexConvertors.Extensions; +using NUnit.Framework; +using Utils; + +namespace CodexReleaseTests.MarketTests +{ + [TestFixture] + public class IsProofRequiredTest : MarketplaceAutoBootstrapDistTest + { + #region Setup + + private readonly PurchaseParams purchaseParams = new PurchaseParams( + nodes: 4, + tolerance: 2, + uploadFilesize: 32.MB() + ); + + public IsProofRequiredTest() + { + Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts)); + } + + protected override int NumberOfHosts => 6; + protected override int NumberOfClients => 1; + protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot. + protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0); + + #endregion + + [Test] + public void IsProofRequired() + { + var mins = TimeSpan.FromMinutes(10.0); + + StartHosts(); + StartValidator(); + var client = StartClients().Single(); + var purchase = CreateStorageRequest(client, mins); + purchase.WaitForStorageContractStarted(); + + var requestId = purchase.PurchaseId.HexToByteArray(); + var numSlots = purchaseParams.Nodes; + //var map = new Dictionary>(); + + Log($"Checking IsProofRequired every second for {Time.FormatDuration(mins)}."); + var endUtc = DateTime.UtcNow + mins; + while (DateTime.UtcNow < endUtc) + { + Thread.Sleep(TimeSpan.FromSeconds(1)); + var requiredSlotIndices = new List(); + var willRequireSlotIndices = new List(); + for (var i = 0; i < numSlots; i++) + { + if (GetContracts().IsProofRequired(requestId, i)) requiredSlotIndices.Add(i); + if (GetContracts().WillProofBeRequired(requestId, i)) willRequireSlotIndices.Add(i); + } + + var periodNumber = GetContracts().GetPeriodNumber(DateTime.UtcNow); + var blockNumber = GetGeth().GetSyncedBlockNumber(); + Log($"[{blockNumber?.ToString().PadLeft(4, '0')}]" + + $"{periodNumber.ToString().PadLeft(12, '0')} => Required now: " + + $"[{string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}] " + + $"Will be required: " + + $"[{string.Join(",", willRequireSlotIndices.Select(i => i.ToString()))}]"); + + //var num = currentPeriod.PeriodNumber; + //if (!map.ContainsKey(num)) + //{ + // map.Add(num, requiredSlotIndices); + // Log($"Period {num} = required proof for slot indices {string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}"); + //} + //else + //{ + // var a = map[num]; + // CollectionAssert.AreEquivalent(a, requiredSlotIndices); + //} + } + } + + private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, TimeSpan minutes) + { + var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); + var config = GetContracts().Deployment.Config; + return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) + { + Duration = minutes * 1.1, + Expiry = TimeSpan.FromMinutes(8.0), + MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, + NodeFailureTolerance = (uint)purchaseParams.Tolerance, + PricePerBytePerSecond = 10.TstWei(), + ProofProbability = 1, // One proof every period. Free slot as quickly as possible. + CollateralPerByte = 1.TstWei() + }); + } + } +} diff --git a/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs b/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs new file mode 100644 index 00000000..02f7a84d --- /dev/null +++ b/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs @@ -0,0 +1,148 @@ +using CodexClient; +using CodexContractsPlugin.ChainMonitor; +using CodexContractsPlugin.Marketplace; +using CodexReleaseTests.Utils; +using Nethereum.Hex.HexConvertors.Extensions; +using Newtonsoft.Json; +using NUnit.Framework; +using Utils; + +namespace CodexReleaseTests.MarketTests +{ + [TestFixture] + public class StabilityTest : MarketplaceAutoBootstrapDistTest + { + #region Setup + + private readonly PurchaseParams purchaseParams = new PurchaseParams( + nodes: 4, + tolerance: 2, + uploadFilesize: 32.MB() + ); + + public StabilityTest() + { + Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts)); + } + + protected override int NumberOfHosts => 6; + protected override int NumberOfClients => 1; + protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot. + protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0); + + #endregion + + private int numPeriods = 0; + private bool proofWasMissed = false; + + [Test] + [Combinatorial] + public void Stability( + [Values(10, 120)] int minutes) + { + var mins = TimeSpan.FromMinutes(minutes); + var periodDuration = GetContracts().Deployment.Config.PeriodDuration; + Assert.That(HostAvailabilityMaxDuration, Is.GreaterThan(mins * 1.1)); + + numPeriods = 0; + proofWasMissed = false; + + StartHosts(); + StartValidator(); + var client = StartClients().Single(); + var purchase = CreateStorageRequest(client, mins); + purchase.WaitForStorageContractStarted(); + + Log($"Contract should remain stable for {Time.FormatDuration(mins)}."); + var endUtc = DateTime.UtcNow + mins; + while (DateTime.UtcNow < endUtc) + { + Thread.Sleep(TimeSpan.FromSeconds(10)); + if (proofWasMissed) + { + // We wait because we want to log calls to MarkProofAsMissing. + Thread.Sleep(periodDuration * 1.1); + Assert.Fail("Proof was missed."); + } + } + + var minNumPeriod = (mins / periodDuration) - 1.0; + Log($"{numPeriods} periods elapsed. Expected at least {minNumPeriod} periods."); + Assert.That(numPeriods, Is.GreaterThanOrEqualTo(minNumPeriod)); + + var status = client.GetPurchaseStatus(purchase.PurchaseId); + if (status == null) throw new Exception("Purchase status not found"); + Assert.That(status.IsStarted || status.IsFinished); + } + + protected override void OnPeriod(PeriodReport report) + { + numPeriods++; + + // For each required proof, there should be a submit call. + var calls = GetSubmitProofCalls(report); + foreach (var required in report.Required) + { + var matchingCall = GetMatchingSubmitProofCall(calls, required); + if (matchingCall == null) + { + Log($"A proof was missed for {required.Describe()}. Failing test after a delay so chain events have time to log..."); + proofWasMissed = true; + } + } + + // There can't be any calls to mark a proof as missed. + foreach (var call in report.FunctionCalls) + { + var missedCall = nameof(MarkProofAsMissingFunction); + Assert.That(call.Name, Is.Not.EqualTo(missedCall)); + } + } + + private SubmitProofFunction? GetMatchingSubmitProofCall(SubmitProofFunction[] calls, PeriodRequiredProof required) + { + foreach (var call in calls) + { + if ( + call.Id.SequenceEqual(required.SlotId) && + call.FromAddress.ToLowerInvariant() == required.Host.Address.ToLowerInvariant() + ) + { + return call; + } + } + + return null; + } + + private SubmitProofFunction[] GetSubmitProofCalls(PeriodReport report) + { + var submitCall = nameof(SubmitProofFunction); + var calls = report.FunctionCalls.Where(f => f.Name == submitCall).ToArray(); + var callObjs = calls.Select(call => JsonConvert.DeserializeObject(call.Payload)).ToArray(); + Log($"SubmitProof calls: {callObjs.Length}"); + foreach (var c in callObjs) + { + Log($" - slotId:{c.Id.ToHex()} host:{c.FromAddress}"); + } + + return callObjs!; + } + + private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, TimeSpan minutes) + { + var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); + var config = GetContracts().Deployment.Config; + return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) + { + Duration = minutes * 1.1, + Expiry = TimeSpan.FromMinutes(8.0), + MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, + NodeFailureTolerance = (uint)purchaseParams.Tolerance, + PricePerBytePerSecond = 10.TstWei(), + ProofProbability = 1, // One proof every period. Free slot as quickly as possible. + CollateralPerByte = 1.TstWei() + }); + } + } +} diff --git a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs index 8723246b..07f621fb 100644 --- a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs +++ b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs @@ -1,5 +1,6 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; +using GethPlugin; using Logging; using Utils; @@ -8,21 +9,25 @@ namespace CodexReleaseTests.Utils public class ChainMonitor { private readonly ILog log; + private readonly IGethNode gethNode; private readonly ICodexContracts contracts; + private readonly IPeriodMonitorEventHandler periodMonitorEventHandler; private readonly DateTime startUtc; private readonly TimeSpan updateInterval; private CancellationTokenSource cts = new CancellationTokenSource(); private Task worker = Task.CompletedTask; - public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc) - : this(log, contracts, startUtc, TimeSpan.FromSeconds(3.0)) + public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, IPeriodMonitorEventHandler periodMonitorEventHandler, DateTime startUtc) + : this(log, gethNode, contracts, periodMonitorEventHandler, startUtc, TimeSpan.FromSeconds(3.0)) { } - public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc, TimeSpan updateInterval) + public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, IPeriodMonitorEventHandler periodMonitorEventHandler, DateTime startUtc, TimeSpan updateInterval) { this.log = log; + this.gethNode = gethNode; this.contracts = contracts; + this.periodMonitorEventHandler = periodMonitorEventHandler; this.startUtc = startUtc; this.updateInterval = updateInterval; } @@ -42,7 +47,7 @@ namespace CodexReleaseTests.Utils private void Worker(Action onFailure) { - var state = new ChainState(log, contracts, new DoNothingThrowingChainEventHandler(), startUtc, doProofPeriodMonitoring: true); + var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, true, periodMonitorEventHandler); Thread.Sleep(updateInterval); log.Log($"Chain monitoring started. Update interval: {Time.FormatDuration(updateInterval)}"); @@ -66,15 +71,6 @@ namespace CodexReleaseTests.Utils private void UpdateChainState(ChainState state) { state.Update(); - - var reports = state.PeriodMonitor.GetAndClearReports(); - if (reports.IsEmpty) return; - - var slots = reports.Reports.Sum(r => Convert.ToInt32(r.TotalNumSlots)); - var required = reports.Reports.Sum(r => Convert.ToInt32(r.PeriodProofs.Count(p => p.State != ProofState.NotRequired))); - var missed = reports.Reports.Sum(r => r.GetMissedProofs().Length); - - log.Log($"Proof report: Slots={slots} Required={required} Missed={missed}"); } } } diff --git a/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs index 42bc03fa..43fcddae 100644 --- a/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs @@ -1,5 +1,6 @@ using CodexClient; using CodexContractsPlugin; +using CodexContractsPlugin.ChainMonitor; using CodexContractsPlugin.Marketplace; using CodexPlugin; using CodexTests; @@ -11,7 +12,7 @@ using Utils; namespace CodexReleaseTests.Utils { - public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest + public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest, IPeriodMonitorEventHandler { private MarketplaceHandle handle = null!; protected const int StartingBalanceTST = 1000; @@ -22,7 +23,7 @@ namespace CodexReleaseTests.Utils { var geth = StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version); - var monitor = SetupChainMonitor(GetTestLog(), contracts, GetTestRunTimeRange().From); + var monitor = SetupChainMonitor(GetTestLog(), geth, contracts, GetTestRunTimeRange().From); handle = new MarketplaceHandle(geth, contracts, monitor); } @@ -42,6 +43,12 @@ namespace CodexReleaseTests.Utils return handle.Contracts; } + protected ChainMonitor GetChainMonitor() + { + if (handle.ChainMonitor == null) throw new Exception($"Make sure {nameof(MonitorChainState)} is set to true."); + return handle.ChainMonitor; + } + protected TimeSpan GetPeriodDuration() { var config = GetContracts().Deployment.Config; @@ -54,6 +61,9 @@ namespace CodexReleaseTests.Utils protected abstract TimeSpan HostAvailabilityMaxDuration { get; } protected virtual bool MonitorChainState { get; } = true; protected TimeSpan HostBlockTTL { get; } = TimeSpan.FromMinutes(1.0); + protected virtual void OnPeriod(PeriodReport report) + { + } public ICodexNodeGroup StartHosts() { @@ -169,56 +179,6 @@ namespace CodexReleaseTests.Utils }); } - private ChainMonitor? SetupChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc) - { - if (!MonitorChainState) return null; - - var result = new ChainMonitor(log, contracts, startUtc); - result.Start(() => - { - Assert.Fail("Failure in chain monitor."); - }); - return result; - } - - private Retry GetBalanceAssertRetry() - { - return new Retry("AssertBalance", - maxTimeout: TimeSpan.FromMinutes(10.0), - sleepAfterFail: TimeSpan.FromSeconds(10.0), - onFail: f => { }, - failFast: false); - } - - private Retry GetAvailabilitySpaceAssertRetry() - { - return new Retry("AssertAvailabilitySpace", - maxTimeout: HostBlockTTL * 3, - sleepAfterFail: TimeSpan.FromSeconds(10.0), - onFail: f => { }, - failFast: false); - } - - private TestToken GetTstBalance(ICodexNode node) - { - return GetContracts().GetTestTokenBalance(node); - } - - private TestToken GetTstBalance(EthAddress address) - { - return GetContracts().GetTestTokenBalance(address); - } - - private Ether GetEthBalance(ICodexNode node) - { - return GetGeth().GetEthBalance(node); - } - - private Ether GetEthBalance(EthAddress address) - { - return GetGeth().GetEthBalance(address); - } - public ICodexNodeGroup StartClients() { return StartClients(s => { }); @@ -247,6 +207,11 @@ namespace CodexReleaseTests.Utils ); } + public void OnPeriodReport(PeriodReport report) + { + OnPeriod(report); + } + public SlotFill[] GetOnChainSlotFills(IEnumerable possibleHosts, string purchaseId) { var fills = GetOnChainSlotFills(possibleHosts); @@ -345,6 +310,56 @@ namespace CodexReleaseTests.Utils } } + private ChainMonitor? SetupChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc) + { + if (!MonitorChainState) return null; + + var result = new ChainMonitor(log, gethNode, contracts, this, startUtc); + result.Start(() => + { + Assert.Fail("Failure in chain monitor."); + }); + return result; + } + + private Retry GetBalanceAssertRetry() + { + return new Retry("AssertBalance", + maxTimeout: TimeSpan.FromMinutes(10.0), + sleepAfterFail: TimeSpan.FromSeconds(10.0), + onFail: f => { }, + failFast: false); + } + + private Retry GetAvailabilitySpaceAssertRetry() + { + return new Retry("AssertAvailabilitySpace", + maxTimeout: HostBlockTTL * 3, + sleepAfterFail: TimeSpan.FromSeconds(10.0), + onFail: f => { }, + failFast: false); + } + + private TestToken GetTstBalance(ICodexNode node) + { + return GetContracts().GetTestTokenBalance(node); + } + + private TestToken GetTstBalance(EthAddress address) + { + return GetContracts().GetTestTokenBalance(address); + } + + private Ether GetEthBalance(ICodexNode node) + { + return GetGeth().GetEthBalance(node); + } + + private Ether GetEthBalance(EthAddress address) + { + return GetGeth().GetEthBalance(address); + } + private TestToken GetContractFinalCost(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts) { var fills = GetOnChainSlotFills(hosts); @@ -459,6 +474,7 @@ namespace CodexReleaseTests.Utils var chanceOfDowntime = downtime / window; return 1.0f + (5.0f * chanceOfDowntime); } + public class SlotFill { public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host) diff --git a/Tests/DistTestCore/NameUtils.cs b/Tests/DistTestCore/NameUtils.cs index 10a4c498..76ea210e 100644 --- a/Tests/DistTestCore/NameUtils.cs +++ b/Tests/DistTestCore/NameUtils.cs @@ -91,7 +91,8 @@ namespace DistTestCore return Path.Join( config.LogRoot, $"{start.Year}-{Pad(start.Month)}", - Pad(start.Day)); + Pad(start.Day), + $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}"); } private static string Pad(int n) diff --git a/Tools/MarketInsights/AverageHistory.cs b/Tools/MarketInsights/AverageHistory.cs index 4baebd3b..9832ca46 100644 --- a/Tools/MarketInsights/AverageHistory.cs +++ b/Tools/MarketInsights/AverageHistory.cs @@ -1,5 +1,6 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; +using GethPlugin; using TestNetRewarder; using Utils; @@ -13,12 +14,12 @@ namespace MarketInsights private readonly int maxContributions; private readonly ChainState chainState; - public AverageHistory(AppState appState, ICodexContracts contracts, int maxContributions) + public AverageHistory(AppState appState, IGethNode geth, ICodexContracts contracts, int maxContributions) { this.appState = appState; this.maxContributions = maxContributions; - chainState = new ChainState(appState.Log, contracts, mux, appState.Config.HistoryStartUtc, - doProofPeriodMonitoring: false); + chainState = new ChainState(appState.Log, geth, contracts, mux, appState.Config.HistoryStartUtc, + doProofPeriodMonitoring: false, new DoNothingPeriodMonitorEventHandler()); } public MarketTimeSegment[] Segments { get; private set; } = Array.Empty(); diff --git a/Tools/MarketInsights/Program.cs b/Tools/MarketInsights/Program.cs index a616386f..6caf4433 100644 --- a/Tools/MarketInsights/Program.cs +++ b/Tools/MarketInsights/Program.cs @@ -22,7 +22,7 @@ namespace MarketInsights var connector = GethConnector.GethConnector.Initialize(appState.Log); if (connector == null) throw new Exception("Invalid Geth information"); - var updater = new Updater(appState, connector.CodexContracts, cts.Token); + var updater = new Updater(appState, connector.GethNode, connector.CodexContracts, cts.Token); var builder = WebApplication.CreateBuilder(args); diff --git a/Tools/MarketInsights/Updater.cs b/Tools/MarketInsights/Updater.cs index 87743f66..436a5ac2 100644 --- a/Tools/MarketInsights/Updater.cs +++ b/Tools/MarketInsights/Updater.cs @@ -1,4 +1,5 @@ using CodexContractsPlugin; +using GethPlugin; using TestNetRewarder; namespace MarketInsights @@ -11,13 +12,13 @@ namespace MarketInsights private readonly Tracker[] trackers; private readonly AverageHistory averageHistory; - public Updater(AppState appState, ICodexContracts contracts, CancellationToken ct) + public Updater(AppState appState, IGethNode geth, ICodexContracts contracts, CancellationToken ct) { this.appState = appState; this.ct = ct; trackers = CreateTrackers(); - averageHistory = new AverageHistory(appState, contracts, trackers.Max(t => t.NumberOfSegments)); + averageHistory = new AverageHistory(appState, geth, contracts, trackers.Max(t => t.NumberOfSegments)); } private Tracker[] CreateTrackers() diff --git a/Tools/TestNetRewarder/EventsFormatter.cs b/Tools/TestNetRewarder/EventsFormatter.cs index 3b1650cb..a9c3a0a1 100644 --- a/Tools/TestNetRewarder/EventsFormatter.cs +++ b/Tools/TestNetRewarder/EventsFormatter.cs @@ -123,17 +123,18 @@ namespace TestNetRewarder public void ProcessPeriodReports(PeriodMonitorResult reports) { - if (reports.IsEmpty) return; + //if (reports.IsEmpty) return; - var lines = new List { - $"Periods: [{reports.PeriodLow} ... {reports.PeriodHigh}]", - $"Average number of slots: {reports.AverageNumSlots.ToString("F2")}", - $"Average number of proofs required: {reports.AverageNumProofsRequired.ToString("F2")}" - }; + //var lines = new List { + // $"Periods: [{reports.PeriodLow} ... {reports.PeriodHigh}]", + // $"Average number of slots: {reports.AverageNumSlots.ToString("F2")}", + // $"Average number of proofs required: {reports.AverageNumProofsRequired.ToString("F2")}" + //}; - AddMissedProofDetails(lines, reports.Reports); + // AddMissedProofDetails(lines, reports.Reports); + // todo! - AddBlock(0, $"{emojiMaps.ProofReport} **Proof system report**", lines.ToArray()); + //AddBlock(0, $"{emojiMaps.ProofReport} **Proof system report**", lines.ToArray()); } private string GetSlotFilledIcon(bool isRepair) @@ -148,40 +149,40 @@ namespace TestNetRewarder return $"Slot Filled"; } - private void AddMissedProofDetails(List lines, PeriodReport[] reports) - { - var reportsWithMissedProofs = reports.Where(r => r.GetMissedProofs().Any()).ToArray(); - if (reportsWithMissedProofs.Length < 1) - { - lines.Add($"No proofs were missed {emojiMaps.NoProofsMissed}"); - return; - } + //private void AddMissedProofDetails(List lines, PeriodReport[] reports) + //{ + // var reportsWithMissedProofs = reports.Where(r => r.GetMissedProofs().Any()).ToArray(); + // if (reportsWithMissedProofs.Length < 1) + // { + // lines.Add($"No proofs were missed {emojiMaps.NoProofsMissed}"); + // return; + // } - var totalMissed = reportsWithMissedProofs.Sum(r => r.GetMissedProofs().Length); - if (totalMissed > 10) - { - lines.Add($"[{totalMissed}] proofs were missed {emojiMaps.ManyProofsMissed}"); - return; - } + // var totalMissed = reportsWithMissedProofs.Sum(r => r.GetMissedProofs().Length); + // if (totalMissed > 10) + // { + // lines.Add($"[{totalMissed}] proofs were missed {emojiMaps.ManyProofsMissed}"); + // return; + // } - foreach (var report in reportsWithMissedProofs) - { - DescribeMissedProof(lines, report); - } - } + // foreach (var report in reportsWithMissedProofs) + // { + // DescribeMissedProof(lines, report); + // } + //} - private void DescribeMissedProof(List lines, PeriodReport report) - { - foreach (var missedProof in report.GetMissedProofs()) - { - DescribeMissedProof(lines, missedProof); - } - } + //private void DescribeMissedProof(List lines, PeriodReport report) + //{ + // foreach (var missedProof in report.GetMissedProofs()) + // { + // DescribeMissedProof(lines, missedProof); + // } + //} - private void DescribeMissedProof(List lines, PeriodProof missedProof) - { - lines.Add($"[{missedProof.FormatHost()}] missed proof for {FormatRequestId(missedProof.Request)} (slotIndex: {missedProof.SlotIndex})"); - } + //private void DescribeMissedProof(List lines, PeriodRequiredProof missedProof) + //{ + // lines.Add($"[{missedProof.FormatHost()}] missed proof for {FormatRequestId(missedProof.Request)} (slotIndex: {missedProof.SlotIndex})"); + //} private void AddRequestBlock(RequestEvent requestEvent, string icon, string eventName, params string[] content) { diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 036f8795..58bef9ab 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -1,5 +1,6 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; +using GethPlugin; using Logging; using Utils; @@ -15,7 +16,7 @@ namespace TestNetRewarder private readonly ILog log; private DateTime lastPeriodUpdateUtc; - public Processor(Configuration config, BotClient client, ICodexContracts contracts, ILog log) + public Processor(Configuration config, BotClient client, IGethNode geth, ICodexContracts contracts, ILog log) { this.config = config; this.client = client; @@ -27,8 +28,8 @@ namespace TestNetRewarder builder = new RequestBuilder(); eventsFormatter = new EventsFormatter(config, contracts.Deployment.Config); - chainState = new ChainState(log, contracts, eventsFormatter, config.HistoryStartUtc, - doProofPeriodMonitoring: config.ShowProofPeriodReports > 0); + chainState = new ChainState(log, geth, contracts, eventsFormatter, config.HistoryStartUtc, + doProofPeriodMonitoring: config.ShowProofPeriodReports > 0, new DoNothingPeriodMonitorEventHandler()); } public async Task Initialize() diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index e3ed24e7..0cac9bd7 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -31,7 +31,7 @@ namespace TestNetRewarder if (connector == null) throw new Exception("Invalid Geth information"); BotClient = new BotClient(Config, Log); - processor = new Processor(Config, BotClient, connector.CodexContracts, Log); + processor = new Processor(Config, BotClient, connector.GethNode, connector.CodexContracts, Log); EnsurePath(Config.DataPath); EnsurePath(Config.LogPath); diff --git a/Tools/TraceContract/ChainTracer.cs b/Tools/TraceContract/ChainTracer.cs index a4227ab4..21a8a69d 100644 --- a/Tools/TraceContract/ChainTracer.cs +++ b/Tools/TraceContract/ChainTracer.cs @@ -1,6 +1,7 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; using CodexContractsPlugin.Marketplace; +using GethPlugin; using Logging; using Nethereum.Hex.HexConvertors.Extensions; using Nethereum.Model; @@ -11,13 +12,15 @@ namespace TraceContract public class ChainTracer { private readonly ILog log; + private readonly IGethNode geth; private readonly ICodexContracts contracts; private readonly Input input; private readonly Output output; - public ChainTracer(ILog log, ICodexContracts contracts, Input input, Output output) + public ChainTracer(ILog log, IGethNode geth, ICodexContracts contracts, Input input, Output output) { this.log = log; + this.geth = geth; this.contracts = contracts; this.input = input; this.output = output; @@ -60,7 +63,7 @@ namespace TraceContract var utc = request.Block.Utc.AddMinutes(-1.0); var tracker = new ChainRequestTracker(output, input.PurchaseId); var ignoreLog = new NullLog(); - var chainState = new ChainState(ignoreLog, contracts, tracker, utc, false); + var chainState = new ChainState(ignoreLog, geth, contracts, tracker, utc, false, new DoNothingPeriodMonitorEventHandler()); var atNow = false; while (!tracker.IsFinished && !atNow) diff --git a/Tools/TraceContract/Program.cs b/Tools/TraceContract/Program.cs index 50ade56d..5e24af4f 100644 --- a/Tools/TraceContract/Program.cs +++ b/Tools/TraceContract/Program.cs @@ -47,9 +47,10 @@ namespace TraceContract var entryPoint = new EntryPoint(log, new KubernetesWorkflow.Configuration(null, TimeSpan.FromMinutes(1.0), TimeSpan.FromSeconds(10.0), "_Unused!_"), Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData)); entryPoint.Announce(); var ci = entryPoint.CreateInterface(); - var contracts = ConnectCodexContracts(ci); + var geth = ConnectGethNode(); + var contracts = ConnectCodexContracts(ci, geth); - var chainTracer = new ChainTracer(log, contracts, input, output); + var chainTracer = new ChainTracer(log, geth, contracts, input, output); var requestTimeRange = chainTracer.TraceChainTimeline(); Log("Downloading storage nodes logs for the request timerange..."); @@ -61,12 +62,15 @@ namespace TraceContract Log("Done"); } - private ICodexContracts ConnectCodexContracts(CoreInterface ci) + private IGethNode ConnectGethNode() { var account = EthAccountGenerator.GenerateNew(); var blockCache = new BlockCache(); - var geth = new CustomGethNode(log, blockCache, config.RpcEndpoint, config.GethPort, account.PrivateKey); + return new CustomGethNode(log, blockCache, config.RpcEndpoint, config.GethPort, account.PrivateKey); + } + private ICodexContracts ConnectCodexContracts(CoreInterface ci, IGethNode geth) + { var deployment = new CodexContractsDeployment( config: new MarketplaceConfig(), marketplaceAddress: config.MarketplaceAddress,